Merge remote-tracking branch 'origin/3.0' into fix/refactorTqBackend

This commit is contained in:
Yihao Deng 2024-07-24 02:05:57 +00:00
commit ae999184ef
112 changed files with 4253 additions and 3338 deletions

View File

@ -49,7 +49,7 @@ typedef struct SBlockOrderInfo {
#define colDataSetNull_f_s(c_, r_) \ #define colDataSetNull_f_s(c_, r_) \
do { \ do { \
colDataSetNull_f((c_)->nullbitmap, r_); \ colDataSetNull_f((c_)->nullbitmap, r_); \
memset(((char*)(c_)->pData) + (c_)->info.bytes * (r_), 0, (c_)->info.bytes); \ (void)memset(((char*)(c_)->pData) + (c_)->info.bytes * (r_), 0, (c_)->info.bytes); \
} while (0) } while (0)
#define colDataClearNull_f(bm_, r_) \ #define colDataClearNull_f(bm_, r_) \

View File

@ -2828,19 +2828,24 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc
return tlen; return tlen;
} }
static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) { static FORCE_INLINE int32_t tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) {
buf = taosDecodeFixedI64(buf, &pReq->consumerId); buf = taosDecodeFixedI64(buf, &pReq->consumerId);
buf = taosDecodeStringTo(buf, pReq->cgroup); buf = taosDecodeStringTo(buf, pReq->cgroup);
buf = taosDecodeStringTo(buf, pReq->clientId); buf = taosDecodeStringTo(buf, pReq->clientId);
int32_t topicNum; int32_t topicNum = 0;
buf = taosDecodeFixedI32(buf, &topicNum); buf = taosDecodeFixedI32(buf, &topicNum);
pReq->topicNames = taosArrayInit(topicNum, sizeof(void*)); pReq->topicNames = taosArrayInit(topicNum, sizeof(void*));
if (pReq->topicNames == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < topicNum; i++) { for (int32_t i = 0; i < topicNum; i++) {
char* name; char* name = NULL;
buf = taosDecodeString(buf, &name); buf = taosDecodeString(buf, &name);
taosArrayPush(pReq->topicNames, &name); if (taosArrayPush(pReq->topicNames, &name) == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
buf = taosDecodeFixedI8(buf, &pReq->withTbName); buf = taosDecodeFixedI8(buf, &pReq->withTbName);
@ -2849,7 +2854,7 @@ static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq
buf = taosDecodeFixedI8(buf, &pReq->resetOffsetCfg); buf = taosDecodeFixedI8(buf, &pReq->resetOffsetCfg);
buf = taosDecodeFixedI8(buf, &pReq->enableReplay); buf = taosDecodeFixedI8(buf, &pReq->enableReplay);
buf = taosDecodeFixedI8(buf, &pReq->enableBatchMeta); buf = taosDecodeFixedI8(buf, &pReq->enableBatchMeta);
return buf; return 0;
} }
typedef struct { typedef struct {

View File

@ -34,9 +34,10 @@ typedef struct {
* @brief Start one Qnode in Dnode. * @brief Start one Qnode in Dnode.
* *
* @param pOption Option of the qnode. * @param pOption Option of the qnode.
* @return SQnode* The qnode object. * @param pQnode The qnode object.
* @return int32_t The error code.
*/ */
SQnode *qndOpen(const SQnodeOpt *pOption); int32_t qndOpen(const SQnodeOpt *pOption, SQnode **pQnode);
/** /**
* @brief Stop Qnode in Dnode. * @brief Stop Qnode in Dnode.

View File

@ -36,7 +36,7 @@ typedef struct SFuncExecEnv {
} SFuncExecEnv; } SFuncExecEnv;
typedef bool (*FExecGetEnv)(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv); typedef bool (*FExecGetEnv)(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv);
typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo); typedef int32_t (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo);
typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx); typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx);
typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock); typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock);
typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

View File

@ -260,7 +260,7 @@ bool fmIsProcessByRowFunc(int32_t funcId);
bool fmisSelectGroupConstValueFunc(int32_t funcId); bool fmisSelectGroupConstValueFunc(int32_t funcId);
void getLastCacheDataType(SDataType* pType, int32_t pkBytes); void getLastCacheDataType(SDataType* pType, int32_t pkBytes);
SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList); int32_t createFunction(const char* pName, SNodeList* pParameterList, SFunctionNode** pFunc);
int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMidFunc, SFunctionNode** pMergeFunc); int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMidFunc, SFunctionNode** pMergeFunc);
@ -273,7 +273,7 @@ typedef enum EFuncDataRequired {
} EFuncDataRequired; } EFuncDataRequired;
EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow); EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow);
EFuncDataRequired fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo); int32_t fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo, int32_t *reqStatus);
int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet); int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet); int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet);

View File

@ -77,7 +77,7 @@ void freeUdfInterBuf(SUdfInterBuf *buf);
// high level APIs // high level APIs
bool udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv); bool udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv);
bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo); int32_t udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo);
int32_t udfAggProcess(struct SqlFunctionCtx *pCtx); int32_t udfAggProcess(struct SqlFunctionCtx *pCtx);
int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock); int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock);

View File

@ -97,6 +97,7 @@ extern "C" {
#include <nmmintrin.h> #include <nmmintrin.h>
#endif #endif
#include "osThread.h" #include "osThread.h"
#include "osAtomic.h" #include "osAtomic.h"

View File

@ -54,7 +54,7 @@ extern SDiskSpace tsDataSpace;
extern SDiskSpace tsLogSpace; extern SDiskSpace tsLogSpace;
extern SDiskSpace tsTempSpace; extern SDiskSpace tsTempSpace;
void osDefaultInit(); int32_t osDefaultInit();
void osUpdate(); void osUpdate();
void osCleanup(); void osCleanup();
@ -66,7 +66,7 @@ bool osLogSpaceSufficient();
bool osDataSpaceSufficient(); bool osDataSpaceSufficient();
bool osTempSpaceSufficient(); bool osTempSpaceSufficient();
void osSetTimezone(const char *timezone); int32_t osSetTimezone(const char *timezone);
void osSetSystemLocale(const char *inLocale, const char *inCharSet); void osSetSystemLocale(const char *inLocale, const char *inCharSet);
void osSetProcPath(int32_t argc, char **argv); void osSetProcPath(int32_t argc, char **argv);

View File

@ -65,28 +65,28 @@ int32_t taosGetTimestampSec();
//@return timestamp in millisecond //@return timestamp in millisecond
static FORCE_INLINE int64_t taosGetTimestampMs() { static FORCE_INLINE int64_t taosGetTimestampMs() {
struct timeval systemTime; struct timeval systemTime;
taosGetTimeOfDay(&systemTime); (void)taosGetTimeOfDay(&systemTime);
return (int64_t)systemTime.tv_sec * 1000LL + (int64_t)systemTime.tv_usec / 1000; return (int64_t)systemTime.tv_sec * 1000LL + (int64_t)systemTime.tv_usec / 1000;
} }
//@return timestamp in microsecond //@return timestamp in microsecond
static FORCE_INLINE int64_t taosGetTimestampUs() { static FORCE_INLINE int64_t taosGetTimestampUs() {
struct timeval systemTime; struct timeval systemTime;
taosGetTimeOfDay(&systemTime); (void)taosGetTimeOfDay(&systemTime);
return (int64_t)systemTime.tv_sec * 1000000LL + (int64_t)systemTime.tv_usec; return (int64_t)systemTime.tv_sec * 1000000LL + (int64_t)systemTime.tv_usec;
} }
//@return timestamp in nanosecond //@return timestamp in nanosecond
static FORCE_INLINE int64_t taosGetTimestampNs() { static FORCE_INLINE int64_t taosGetTimestampNs() {
struct timespec systemTime = {0}; struct timespec systemTime = {0};
taosClockGetTime(CLOCK_REALTIME, &systemTime); (void)taosClockGetTime(CLOCK_REALTIME, &systemTime);
return (int64_t)systemTime.tv_sec * 1000000000LL + (int64_t)systemTime.tv_nsec; return (int64_t)systemTime.tv_sec * 1000000000LL + (int64_t)systemTime.tv_nsec;
} }
//@return timestamp of monotonic clock in millisecond //@return timestamp of monotonic clock in millisecond
static FORCE_INLINE int64_t taosGetMonoTimestampMs() { static FORCE_INLINE int64_t taosGetMonoTimestampMs() {
struct timespec systemTime = {0}; struct timespec systemTime = {0};
taosClockGetTime(CLOCK_MONOTONIC, &systemTime); (void)taosClockGetTime(CLOCK_MONOTONIC, &systemTime);
return (int64_t)systemTime.tv_sec * 1000LL + (int64_t)systemTime.tv_nsec / 1000000; return (int64_t)systemTime.tv_sec * 1000LL + (int64_t)systemTime.tv_nsec / 1000000;
} }

View File

@ -55,7 +55,7 @@ enum TdTimezone {
}; };
void taosGetSystemTimezone(char *outTimezone, enum TdTimezone *tsTimezone); void taosGetSystemTimezone(char *outTimezone, enum TdTimezone *tsTimezone);
void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight, enum TdTimezone *tsTimezone); int32_t taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight, enum TdTimezone *tsTimezone);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -860,6 +860,11 @@ int32_t taosGetErrSize();
#define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR TAOS_DEF_ERROR_CODE(0, 0x2807) #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR TAOS_DEF_ERROR_CODE(0, 0x2807)
#define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2808) #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2808)
#define TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2809) #define TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2809)
#define TSDB_CODE_FUNC_TIME_UNIT_INVALID TAOS_DEF_ERROR_CODE(0, 0x280A)
#define TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL TAOS_DEF_ERROR_CODE(0, 0x280B)
#define TSDB_CODE_FUNC_INVALID_VALUE_RANGE TAOS_DEF_ERROR_CODE(0, 0x280C)
#define TSDB_CODE_FUNC_SETUP_ERROR TAOS_DEF_ERROR_CODE(0, 0x280D)
//udf //udf
#define TSDB_CODE_UDF_STOPPING TAOS_DEF_ERROR_CODE(0, 0x2901) #define TSDB_CODE_UDF_STOPPING TAOS_DEF_ERROR_CODE(0, 0x2901)

View File

@ -556,7 +556,8 @@ typedef struct {
char name[TSDB_LOG_VAR_LEN]; char name[TSDB_LOG_VAR_LEN];
} SLogVar; } SLogVar;
#define TMQ_SEPARATOR ':' #define TMQ_SEPARATOR ":"
#define TMQ_SEPARATOR_CHAR ':'
enum { enum {
SND_WORKER_TYPE__SHARED = 1, SND_WORKER_TYPE__SHARED = 1,

View File

@ -897,7 +897,11 @@ void taos_init_imp(void) {
tscError("failed to init task queue"); tscError("failed to init task queue");
return; return;
} }
fmFuncMgtInit(); if (fmFuncMgtInit() != TSDB_CODE_SUCCESS) {
tscInitRes = -1;
tscError("failed to init function manager");
return;
}
nodesInitAllocatorSet(); nodesInitAllocatorSet();
clientConnRefPool = taosOpenRef(200, destroyTscObj); clientConnRefPool = taosOpenRef(200, destroyTscObj);

View File

@ -801,9 +801,8 @@ int32_t monitorInit() {
} }
if (taosMulModeMkDir(tmpSlowLogPath, 0777, true) != 0) { if (taosMulModeMkDir(tmpSlowLogPath, 0777, true) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
tscError("failed to create dir:%s since %s", tmpSlowLogPath, terrstr()); tscError("failed to create dir:%s since %s", tmpSlowLogPath, terrstr());
return -1; return terrno;
} }
if (tsem2_init(&monitorSem, 0, 0) != 0) { if (tsem2_init(&monitorSem, 0, 0) != 0) {

View File

@ -456,11 +456,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse
pOffset.consumerId = tmq->consumerId; pOffset.consumerId = tmq->consumerId;
pOffset.offset.val = *offset; pOffset.offset.val = *offset;
int32_t groupLen = strlen(tmq->groupId); (void)snprintf(pOffset.offset.subKey, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", tmq->groupId, TMQ_SEPARATOR, pTopicName);
memcpy(pOffset.offset.subKey, tmq->groupId, groupLen);
pOffset.offset.subKey[groupLen] = TMQ_SEPARATOR;
strcpy(pOffset.offset.subKey + groupLen + 1, pTopicName);
int32_t len = 0; int32_t len = 0;
int32_t code = 0; int32_t code = 0;
tEncodeSize(tEncodeMqVgOffset, &pOffset, len, code); tEncodeSize(tEncodeMqVgOffset, &pOffset, len, code);
@ -1667,11 +1663,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
} }
void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) { void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) {
int32_t groupLen = strlen(tmq->groupId); (void)snprintf(pReq->subKey, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", tmq->groupId, TMQ_SEPARATOR, pTopic->topicName);
memcpy(pReq->subKey, tmq->groupId, groupLen);
pReq->subKey[groupLen] = TMQ_SEPARATOR;
strcpy(pReq->subKey + groupLen + 1, pTopic->topicName);
pReq->withTbName = tmq->withTbName; pReq->withTbName = tmq->withTbName;
pReq->consumerId = tmq->consumerId; pReq->consumerId = tmq->consumerId;
pReq->timeout = timeout; pReq->timeout = timeout;
@ -2902,11 +2894,7 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep
SMqVgOffset pOffset = {0}; SMqVgOffset pOffset = {0};
pOffset.consumerId = tmq->consumerId; pOffset.consumerId = tmq->consumerId;
(void)snprintf(pOffset.offset.subKey, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", tmq->groupId, TMQ_SEPARATOR, tname);
int32_t groupLen = strlen(tmq->groupId);
memcpy(pOffset.offset.subKey, tmq->groupId, groupLen);
pOffset.offset.subKey[groupLen] = TMQ_SEPARATOR;
strcpy(pOffset.offset.subKey + groupLen + 1, tname);
int32_t len = 0; int32_t len = 0;
tEncodeSize(tEncodeMqVgOffset, &pOffset, len, code); tEncodeSize(tEncodeMqVgOffset, &pOffset, len, code);

View File

@ -1306,7 +1306,6 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
taosSetAllDebugFlag(pCfg, cfgGetItem(pCfg, "debugFlag")->i32); taosSetAllDebugFlag(pCfg, cfgGetItem(pCfg, "debugFlag")->i32);
if (taosMulModeMkDir(tsLogDir, 0777, true) != 0) { if (taosMulModeMkDir(tsLogDir, 0777, true) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
printf("failed to create dir:%s since %s", tsLogDir, terrstr()); printf("failed to create dir:%s since %s", tsLogDir, terrstr());
cfgCleanup(pCfg); cfgCleanup(pCfg);
return -1; return -1;

View File

@ -9197,23 +9197,23 @@ int32_t tDecodeSTqOffsetVal(SDecoder *pDecoder, STqOffsetVal *pOffsetVal) {
int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) { int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) {
if (pVal->type == TMQ_OFFSET__RESET_NONE) { if (pVal->type == TMQ_OFFSET__RESET_NONE) {
snprintf(buf, maxLen, "none"); (void)snprintf(buf, maxLen, "none");
} else if (pVal->type == TMQ_OFFSET__RESET_EARLIEST) { } else if (pVal->type == TMQ_OFFSET__RESET_EARLIEST) {
snprintf(buf, maxLen, "earliest"); (void)snprintf(buf, maxLen, "earliest");
} else if (pVal->type == TMQ_OFFSET__RESET_LATEST) { } else if (pVal->type == TMQ_OFFSET__RESET_LATEST) {
snprintf(buf, maxLen, "latest"); (void)snprintf(buf, maxLen, "latest");
} else if (pVal->type == TMQ_OFFSET__LOG) { } else if (pVal->type == TMQ_OFFSET__LOG) {
snprintf(buf, maxLen, "wal:%" PRId64, pVal->version); (void)snprintf(buf, maxLen, "wal:%" PRId64, pVal->version);
} else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA || pVal->type == TMQ_OFFSET__SNAPSHOT_META) { } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA || pVal->type == TMQ_OFFSET__SNAPSHOT_META) {
if (IS_VAR_DATA_TYPE(pVal->primaryKey.type)) { if (IS_VAR_DATA_TYPE(pVal->primaryKey.type)) {
char *tmp = taosMemoryCalloc(1, pVal->primaryKey.nData + 1); char *tmp = taosMemoryCalloc(1, pVal->primaryKey.nData + 1);
if (tmp == NULL) return TSDB_CODE_OUT_OF_MEMORY; if (tmp == NULL) return TSDB_CODE_OUT_OF_MEMORY;
memcpy(tmp, pVal->primaryKey.pData, pVal->primaryKey.nData); (void)memcpy(tmp, pVal->primaryKey.pData, pVal->primaryKey.nData);
snprintf(buf, maxLen, "tsdb:%" PRId64 "|%" PRId64 ",pk type:%d,val:%s", pVal->uid, pVal->ts, (void)snprintf(buf, maxLen, "tsdb:%" PRId64 "|%" PRId64 ",pk type:%d,val:%s", pVal->uid, pVal->ts,
pVal->primaryKey.type, tmp); pVal->primaryKey.type, tmp);
taosMemoryFree(tmp); taosMemoryFree(tmp);
} else { } else {
snprintf(buf, maxLen, "tsdb:%" PRId64 "|%" PRId64 ",pk type:%d,val:%" PRId64, pVal->uid, pVal->ts, (void)snprintf(buf, maxLen, "tsdb:%" PRId64 "|%" PRId64 ",pk type:%d,val:%" PRId64, pVal->uid, pVal->ts,
pVal->primaryKey.type, pVal->primaryKey.val); pVal->primaryKey.type, pVal->primaryKey.val);
} }
} else { } else {

View File

@ -34,11 +34,8 @@ static void qmClose(SQnodeMgmt *pMgmt) {
} }
static int32_t qndOpenWrapper(SQnodeOpt *pOption, SQnode **pQnode) { static int32_t qndOpenWrapper(SQnodeOpt *pOption, SQnode **pQnode) {
*pQnode = qndOpen(pOption); int32_t code = qndOpen(pOption, pQnode);
if (*pQnode == NULL) { return code;
return terrno;
}
return 0;
} }
static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
int32_t code = 0; int32_t code = 0;
@ -62,7 +59,7 @@ static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
if (code != 0) { if (code != 0) {
dError("failed to open qnode since %s", tstrerror(code)); dError("failed to open qnode since %s", tstrerror(code));
qmClose(pMgmt); qmClose(pMgmt);
return -1; return code;
} }
tmsgReportStartup("qnode-impl", "initialized"); tmsgReportStartup("qnode-impl", "initialized");

View File

@ -48,8 +48,8 @@ static int32_t qmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) {
} }
int32_t qmPutNodeMsgToQueryQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t qmPutNodeMsgToQueryQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg) {
qndPreprocessQueryMsg(pMgmt->pQnode, pMsg); int32_t code = qndPreprocessQueryMsg(pMgmt->pQnode, pMsg);
if (code) return code;
return qmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg); return qmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg);
} }

View File

@ -262,7 +262,7 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
pWrapper = &pDnode->wrappers[ntype]; pWrapper = &pDnode->wrappers[ntype];
if (taosMkDir(pWrapper->path) != 0) { if (taosMkDir(pWrapper->path) != 0) {
dmReleaseWrapper(pWrapper); dmReleaseWrapper(pWrapper);
code = TAOS_SYSTEM_ERROR(errno); code = terrno;
dError("failed to create dir:%s since %s", pWrapper->path, tstrerror(code)); dError("failed to create dir:%s since %s", pWrapper->path, tstrerror(code));
return code; return code;
} }
@ -333,7 +333,7 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
pWrapper = &pDnode->wrappers[ntype]; pWrapper = &pDnode->wrappers[ntype];
if (taosMkDir(pWrapper->path) != 0) { if (taosMkDir(pWrapper->path) != 0) {
taosThreadMutexUnlock(&pDnode->mutex); taosThreadMutexUnlock(&pDnode->mutex);
code = TAOS_SYSTEM_ERROR(errno); code = terrno;
dError("failed to create dir:%s since %s", pWrapper->path, tstrerror(code)); dError("failed to create dir:%s since %s", pWrapper->path, tstrerror(code));
return code; return code;
} }

View File

@ -21,7 +21,7 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) {
SDnode *pDnode = pWrapper->pDnode; SDnode *pDnode = pWrapper->pDnode;
if (taosMkDir(pWrapper->path) != 0) { if (taosMkDir(pWrapper->path) != 0) {
code = TAOS_SYSTEM_ERROR(errno); code = terrno;
dError("node:%s, failed to create dir:%s since %s", pWrapper->name, pWrapper->path, tstrerror(code)); dError("node:%s, failed to create dir:%s since %s", pWrapper->name, pWrapper->path, tstrerror(code));
return code; return code;
} }

View File

@ -417,7 +417,7 @@ int32_t dmUpdateEncryptKey(char *key, bool toLogFile) {
} }
if (taosMkDir(folder) != 0) { if (taosMkDir(folder) != 0) {
code = TAOS_SYSTEM_ERROR(errno); code = terrno;
encryptError("failed to create dir:%s since %s", folder, tstrerror(code)); encryptError("failed to create dir:%s since %s", folder, tstrerror(code));
goto _OVER; goto _OVER;
} }
@ -578,4 +578,4 @@ _OVER:
#else #else
return 0; return 0;
#endif #endif
} }

View File

@ -30,10 +30,10 @@ enum {
int32_t mndInitConsumer(SMnode *pMnode); int32_t mndInitConsumer(SMnode *pMnode);
void mndCleanupConsumer(SMnode *pMnode); void mndCleanupConsumer(SMnode *pMnode);
void mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SRpcHandleInfo* info); int32_t mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SRpcHandleInfo* info);
SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int64_t consumerId); int32_t mndAcquireConsumer(SMnode *pMnode, int64_t consumerId, SMqConsumerObj** pConsumer);
void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer); void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer);
SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer); SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer);
SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw); SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw);
@ -43,6 +43,22 @@ int32_t mndSetConsumerDropLogs(STrans *pTrans, SMqConsumerObj *pConsumer);
const char *mndConsumerStatusName(int status); const char *mndConsumerStatusName(int status);
#define MND_TMQ_NULL_CHECK(c) \
do { \
if (c == NULL) { \
code = TSDB_CODE_OUT_OF_MEMORY; \
goto END; \
} \
} while (0)
#define MND_TMQ_RETURN_CHECK(c) \
do { \
code = c; \
if (code != 0) { \
goto END; \
} \
} while (0)
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -622,7 +622,8 @@ typedef struct {
int32_t resetOffsetCfg; int32_t resetOffsetCfg;
} SMqConsumerObj; } SMqConsumerObj;
SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, char *topic, SCMSubscribeReq *subscribe); int32_t tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType,
char *topic, SCMSubscribeReq *subscribe, SMqConsumerObj** ppConsumer);
void tClearSMqConsumerObj(SMqConsumerObj* pConsumer); void tClearSMqConsumerObj(SMqConsumerObj* pConsumer);
void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer); void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer);
int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer); int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer);
@ -665,8 +666,8 @@ typedef struct {
char* qmsg; // SubPlanToString char* qmsg; // SubPlanToString
} SMqSubscribeObj; } SMqSubscribeObj;
SMqSubscribeObj* tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]); int32_t tNewSubscribeObj(const char *key, SMqSubscribeObj **ppSub);
SMqSubscribeObj* tCloneSubscribeObj(const SMqSubscribeObj* pSub); int32_t tCloneSubscribeObj(const SMqSubscribeObj* pSub, SMqSubscribeObj **ppSub);
void tDeleteSubscribeObj(SMqSubscribeObj* pSub); void tDeleteSubscribeObj(SMqSubscribeObj* pSub);
int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub); int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub);
void* tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub, int8_t sver); void* tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub, int8_t sver);

View File

@ -26,12 +26,9 @@ int32_t mndInitSubscribe(SMnode *pMnode);
void mndCleanupSubscribe(SMnode *pMnode); void mndCleanupSubscribe(SMnode *pMnode);
int32_t mndGetGroupNumByTopic(SMnode *pMnode, const char *topicName); int32_t mndGetGroupNumByTopic(SMnode *pMnode, const char *topicName);
SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, const char *CGroup, const char *topicName); int32_t mndAcquireSubscribeByKey(SMnode *pMnode, const char *key, SMqSubscribeObj** pSub);
SMqSubscribeObj *mndAcquireSubscribeByKey(SMnode *pMnode, const char *key);
void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub); void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub);
void mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName);
int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topic); int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topic);
int32_t mndSetDropSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub); int32_t mndSetDropSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub);

View File

@ -25,13 +25,12 @@ extern "C" {
int32_t mndInitTopic(SMnode *pMnode); int32_t mndInitTopic(SMnode *pMnode);
void mndCleanupTopic(SMnode *pMnode); void mndCleanupTopic(SMnode *pMnode);
SMqTopicObj *mndAcquireTopic(SMnode *pMnode, const char *topicName); int32_t mndAcquireTopic(SMnode *pMnode, const char *topicName, SMqTopicObj **pTopic);
void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic); void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic);
int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
bool mndTopicExistsForDb(SMnode *pMnode, SDbObj *pDb); bool mndTopicExistsForDb(SMnode *pMnode, SDbObj *pDb);
const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]); void mndTopicGetShowName(const char* fullTopic, char* topic);
int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic);
int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics); int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics);
#ifdef __cplusplus #ifdef __cplusplus

View File

@ -40,7 +40,6 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg);
static int32_t mndProcessAskEpReq(SRpcMsg *pMsg); static int32_t mndProcessAskEpReq(SRpcMsg *pMsg);
static int32_t mndProcessMqHbReq(SRpcMsg *pMsg); static int32_t mndProcessMqHbReq(SRpcMsg *pMsg);
static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg); static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg);
static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg);
int32_t mndInitConsumer(SMnode *pMnode) { int32_t mndInitConsumer(SMnode *pMnode) {
SSdbTable table = { SSdbTable table = {
@ -57,7 +56,6 @@ int32_t mndInitConsumer(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_TMQ_HB, mndProcessMqHbReq); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_HB, mndProcessMqHbReq);
mndSetMsgHandle(pMnode, TDMT_MND_TMQ_ASK_EP, mndProcessAskEpReq); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_ASK_EP, mndProcessAskEpReq);
// mndSetMsgHandle(pMnode, TDMT_MND_TMQ_TIMER, mndProcessMqTimerMsg); // mndSetMsgHandle(pMnode, TDMT_MND_TMQ_TIMER, mndProcessMqTimerMsg);
mndSetMsgHandle(pMnode, TDMT_MND_TMQ_CONSUMER_RECOVER, mndProcessConsumerRecoverMsg);
mndSetMsgHandle(pMnode, TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, mndProcessConsumerClearMsg); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, mndProcessConsumerClearMsg);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONSUMERS, mndRetrieveConsumer); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONSUMERS, mndRetrieveConsumer);
@ -68,13 +66,10 @@ int32_t mndInitConsumer(SMnode *pMnode) {
void mndCleanupConsumer(SMnode *pMnode) {} void mndCleanupConsumer(SMnode *pMnode) {}
void mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SRpcHandleInfo *info) { int32_t mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SRpcHandleInfo *info) {
void *msg = rpcMallocCont(sizeof(int64_t)); int32_t code = 0;
if (msg == NULL) { void *msg = rpcMallocCont(sizeof(int64_t));
mError("consumer:0x%" PRIx64 " failed to clear consumer due to out of memory. alloc size:%d", consumerId, MND_TMQ_NULL_CHECK(msg);
(int32_t)sizeof(int64_t));
return;
}
*(int64_t*)msg = consumerId; *(int64_t*)msg = consumerId;
SRpcMsg rpcMsg = { SRpcMsg rpcMsg = {
@ -85,108 +80,51 @@ void mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SR
}; };
mInfo("mndSendConsumerMsg type:%d consumer:0x%" PRIx64, msgType, consumerId); mInfo("mndSendConsumerMsg type:%d consumer:0x%" PRIx64, msgType, consumerId);
int32_t code = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); MND_TMQ_RETURN_CHECK(tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg));
if (code != 0){ return code;
mError("consumer:%"PRId64" send consumer msg:%d error:%d", consumerId, msgType, code);
} END:
taosMemoryFree(msg);
return code;
} }
static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const char *pUser, static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const char *pUser,
bool enableReplay) { bool enableReplay) {
SMqTopicObj *pTopic = NULL; SMqTopicObj *pTopic = NULL;
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
int32_t numOfTopics = taosArrayGetSize(pTopicList); int32_t numOfTopics = taosArrayGetSize(pTopicList);
for (int32_t i = 0; i < numOfTopics; i++) { for (int32_t i = 0; i < numOfTopics; i++) {
char *pOneTopic = taosArrayGetP(pTopicList, i); char *pOneTopic = taosArrayGetP(pTopicList, i);
pTopic = mndAcquireTopic(pMnode, pOneTopic); MND_TMQ_RETURN_CHECK(mndAcquireTopic(pMnode, pOneTopic, &pTopic));
if (pTopic == NULL) { // terrno has been set by callee function MND_TMQ_RETURN_CHECK(mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic));
code = TSDB_CODE_MND_RETURN_VALUE_NULL; MND_TMQ_RETURN_CHECK(grantCheckExpire(TSDB_GRANT_SUBSCRIPTION));
if (terrno != 0) code = terrno;
goto FAILED;
}
TAOS_CHECK_GOTO(mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic), &lino, FAILED);
TAOS_CHECK_GOTO(grantCheckExpire(TSDB_GRANT_SUBSCRIPTION), &lino, FAILED);
if (enableReplay) { if (enableReplay) {
if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) { if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) {
code = TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT; code = TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT;
goto FAILED; goto END;
} else if (pTopic->ntbUid == 0 && pTopic->ctbStbUid == 0) { } else if (pTopic->ntbUid == 0 && pTopic->ctbStbUid == 0) {
SDbObj *pDb = mndAcquireDb(pMnode, pTopic->db); SDbObj *pDb = mndAcquireDb(pMnode, pTopic->db);
if (pDb == NULL) { if (pDb == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno; goto END;
goto FAILED;
} }
if (pDb->cfg.numOfVgroups != 1) { if (pDb->cfg.numOfVgroups != 1) {
mndReleaseDb(pMnode, pDb); mndReleaseDb(pMnode, pDb);
code = TSDB_CODE_TMQ_REPLAY_NEED_ONE_VGROUP; code = TSDB_CODE_TMQ_REPLAY_NEED_ONE_VGROUP;
goto FAILED; goto END;
} }
mndReleaseDb(pMnode, pDb); mndReleaseDb(pMnode, pDb);
} }
} }
mndReleaseTopic(pMnode, pTopic); mndReleaseTopic(pMnode, pTopic);
} }
return 0; return 0;
FAILED:
mndReleaseTopic(pMnode, pTopic);
TAOS_RETURN(code);
}
static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
int32_t code = 0;
int32_t lino = 0;
SMnode *pMnode = pMsg->info.node;
SMqConsumerRecoverMsg *pRecoverMsg = pMsg->pCont;
SMqConsumerObj *pConsumerNew = NULL;
STrans *pTrans = NULL;
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pRecoverMsg->consumerId);
if (pConsumer == NULL) {
mError("cannot find consumer %" PRId64 " when processing consumer recover msg", pRecoverMsg->consumerId);
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END;
}
mInfo("receive consumer recover msg, consumer:0x%" PRIx64 " status:%d(%s)", pRecoverMsg->consumerId,
pConsumer->status, mndConsumerStatusName(pConsumer->status));
if (pConsumer->status != MQ_CONSUMER_STATUS_LOST) {
code = TSDB_CODE_MND_CONSUMER_NOT_READY;
goto END;
}
pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup, CONSUMER_UPDATE_REC, NULL, NULL);
if (pConsumerNew == NULL){
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END;
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "recover-csm");
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END;
}
TAOS_CHECK_GOTO(validateTopics(pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false), &lino, END);
TAOS_CHECK_GOTO(mndSetConsumerCommitLogs(pTrans, pConsumerNew), &lino, END);
code = mndTransPrepare(pMnode, pTrans);
END: END:
mndReleaseConsumer(pMnode, pConsumer); mndReleaseTopic(pMnode, pTopic);
tDeleteSMqConsumerObj(pConsumerNew); return code;
mndTransDrop(pTrans);
TAOS_RETURN(code);
} }
static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) { static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) {
@ -196,55 +134,39 @@ static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) {
SMqConsumerClearMsg *pClearMsg = pMsg->pCont; SMqConsumerClearMsg *pClearMsg = pMsg->pCont;
SMqConsumerObj *pConsumerNew = NULL; SMqConsumerObj *pConsumerNew = NULL;
STrans *pTrans = NULL; STrans *pTrans = NULL;
SMqConsumerObj *pConsumer = NULL;
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pClearMsg->consumerId); MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, pClearMsg->consumerId, &pConsumer));
if (pConsumer == NULL) {
mError("consumer:0x%" PRIx64 " failed to be found to clear it", pClearMsg->consumerId);
TAOS_RETURN(code);
}
mInfo("consumer:0x%" PRIx64 " needs to be cleared, status %s", pClearMsg->consumerId, mInfo("consumer:0x%" PRIx64 " needs to be cleared, status %s", pClearMsg->consumerId,
mndConsumerStatusName(pConsumer->status)); mndConsumerStatusName(pConsumer->status));
pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup, -1, NULL, NULL); MND_TMQ_RETURN_CHECK(tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup, -1, NULL, NULL, &pConsumerNew));
if (pConsumerNew == NULL){
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END;
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "clear-csm"); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "clear-csm");
if (pTrans == NULL) { MND_TMQ_NULL_CHECK(pTrans);
code = TSDB_CODE_MND_RETURN_VALUE_NULL; MND_TMQ_RETURN_CHECK(mndSetConsumerDropLogs(pTrans, pConsumerNew));
if (terrno != 0) code = terrno;
goto END;
}
// this is the drop action, not the update action
TAOS_CHECK_GOTO(mndSetConsumerDropLogs(pTrans, pConsumerNew), &lino, END);
code = mndTransPrepare(pMnode, pTrans); code = mndTransPrepare(pMnode, pTrans);
END: END:
mndReleaseConsumer(pMnode, pConsumer); mndReleaseConsumer(pMnode, pConsumer);
tDeleteSMqConsumerObj(pConsumerNew); tDeleteSMqConsumerObj(pConsumerNew);
mndTransDrop(pTrans); mndTransDrop(pTrans);
TAOS_RETURN(code); return code;
} }
static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbRsp *rsp, char *user) { static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbRsp *rsp, char *user) {
int32_t code = 0;
rsp->topicPrivileges = taosArrayInit(taosArrayGetSize(pConsumer->currentTopics), sizeof(STopicPrivilege)); rsp->topicPrivileges = taosArrayInit(taosArrayGetSize(pConsumer->currentTopics), sizeof(STopicPrivilege));
if (rsp->topicPrivileges == NULL) { MND_TMQ_NULL_CHECK(rsp->topicPrivileges);
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < taosArrayGetSize(pConsumer->currentTopics); i++) { for (int32_t i = 0; i < taosArrayGetSize(pConsumer->currentTopics); i++) {
char *topic = taosArrayGetP(pConsumer->currentTopics, i); char *topic = taosArrayGetP(pConsumer->currentTopics, i);
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); SMqTopicObj *pTopic = NULL;
if (pTopic == NULL) { // terrno has been set by callee function code = mndAcquireTopic(pMnode, topic, &pTopic);
if (code != TDB_CODE_SUCCESS) {
continue; continue;
} }
STopicPrivilege *data = taosArrayReserve(rsp->topicPrivileges, 1); STopicPrivilege *data = taosArrayReserve(rsp->topicPrivileges, 1);
strcpy(data->topic, topic); MND_TMQ_NULL_CHECK(data);
(void)strcpy(data->topic, topic);
if (mndCheckTopicPrivilege(pMnode, user, MND_OPER_SUBSCRIBE, pTopic) != 0 || if (mndCheckTopicPrivilege(pMnode, user, MND_OPER_SUBSCRIBE, pTopic) != 0 ||
grantCheckExpire(TSDB_GRANT_SUBSCRIPTION) < 0) { grantCheckExpire(TSDB_GRANT_SUBSCRIPTION) < 0) {
data->noPrivilege = 1; data->noPrivilege = 1;
@ -253,22 +175,30 @@ static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbRs
} }
mndReleaseTopic(pMnode, pTopic); mndReleaseTopic(pMnode, pTopic);
} }
return 0; END:
return code;
} }
static void storeOffsetRows(SMnode *pMnode, SMqHbReq *req, SMqConsumerObj *pConsumer){ static void storeOffsetRows(SMnode *pMnode, SMqHbReq *req, SMqConsumerObj *pConsumer){
for (int i = 0; i < taosArrayGetSize(req->topics); i++) { for (int i = 0; i < taosArrayGetSize(req->topics); i++) {
TopicOffsetRows *data = taosArrayGet(req->topics, i); TopicOffsetRows *data = taosArrayGet(req->topics, i);
if (data == NULL){
continue;
}
mInfo("heartbeat report offset rows.%s:%s", pConsumer->cgroup, data->topicName); mInfo("heartbeat report offset rows.%s:%s", pConsumer->cgroup, data->topicName);
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, data->topicName); SMqSubscribeObj *pSub = NULL;
if (pSub == NULL) { char key[TSDB_SUBSCRIBE_KEY_LEN] = {0};
(void)snprintf(key, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", pConsumer->cgroup, TMQ_SEPARATOR, data->topicName);
int32_t code = mndAcquireSubscribeByKey(pMnode, key, &pSub);
if (code != 0) {
mError("failed to acquire subscribe by key:%s, code:%d", key, code);
continue; continue;
} }
taosWLockLatch(&pSub->lock); taosWLockLatch(&pSub->lock);
SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &pConsumer->consumerId, sizeof(int64_t)); SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &pConsumer->consumerId, sizeof(int64_t));
if (pConsumerEp) { if (pConsumerEp) {
taosArrayDestroy(pConsumerEp->offsetRows); (void)taosArrayDestroy(pConsumerEp->offsetRows);
pConsumerEp->offsetRows = data->offsetRows; pConsumerEp->offsetRows = data->offsetRows;
data->offsetRows = NULL; data->offsetRows = NULL;
} }
@ -303,36 +233,24 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
SMqHbReq req = {0}; SMqHbReq req = {0};
SMqHbRsp rsp = {0}; SMqHbRsp rsp = {0};
SMqConsumerObj *pConsumer = NULL; SMqConsumerObj *pConsumer = NULL;
MND_TMQ_RETURN_CHECK(tDeserializeSMqHbReq(pMsg->pCont, pMsg->contLen, &req));
TAOS_CHECK_GOTO(tDeserializeSMqHbReq(pMsg->pCont, pMsg->contLen, &req), NULL, end);
int64_t consumerId = req.consumerId; int64_t consumerId = req.consumerId;
pConsumer = mndAcquireConsumer(pMnode, consumerId); MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, consumerId, &pConsumer));
if (pConsumer == NULL) { MND_TMQ_RETURN_CHECK(checkPrivilege(pMnode, pConsumer, &rsp, pMsg->info.conn.user));
mError("consumer:0x%" PRIx64 " not exist", consumerId);
code = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
goto end;
}
TAOS_CHECK_GOTO(checkPrivilege(pMnode, pConsumer, &rsp, pMsg->info.conn.user), NULL, end);
atomic_store_32(&pConsumer->hbStatus, 0); atomic_store_32(&pConsumer->hbStatus, 0);
int32_t status = atomic_load_32(&pConsumer->status); int32_t status = atomic_load_32(&pConsumer->status);
if (status == MQ_CONSUMER_STATUS_LOST) { if (status == MQ_CONSUMER_STATUS_LOST) {
mInfo("try to recover consumer:0x%" PRIx64, consumerId); mInfo("try to recover consumer:0x%" PRIx64, consumerId);
mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_CONSUMER_RECOVER, &pMsg->info); MND_TMQ_RETURN_CHECK(mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_CONSUMER_RECOVER, &pMsg->info));
} }
storeOffsetRows(pMnode, &req, pConsumer); storeOffsetRows(pMnode, &req, pConsumer);
code = buildMqHbRsp(pMsg, &rsp); code = buildMqHbRsp(pMsg, &rsp);
end: END:
tDestroySMqHbRsp(&rsp); tDestroySMqHbRsp(&rsp);
mndReleaseConsumer(pMnode, pConsumer); mndReleaseConsumer(pMnode, pConsumer);
tDestroySMqHbReq(&req); tDestroySMqHbReq(&req);
TAOS_RETURN(code); return code;
} }
static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t epoch, SMqAskEpRsp *rsp){ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t epoch, SMqAskEpRsp *rsp){
@ -349,19 +267,22 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e
// handle all topics subscribed by this consumer // handle all topics subscribed by this consumer
for (int32_t i = 0; i < numOfTopics; i++) { for (int32_t i = 0; i < numOfTopics; i++) {
char *topic = taosArrayGetP(pConsumer->currentTopics, i); char *topic = taosArrayGetP(pConsumer->currentTopics, i);
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic); SMqSubscribeObj *pSub = NULL;
// txn guarantees pSub is created char key[TSDB_SUBSCRIBE_KEY_LEN] = {0};
if (pSub == NULL) { (void)snprintf(key, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", pConsumer->cgroup, TMQ_SEPARATOR, topic);
int32_t code = mndAcquireSubscribeByKey(pMnode, key, &pSub);
if (code != 0) {
continue; continue;
} }
taosRLockLatch(&pSub->lock); taosRLockLatch(&pSub->lock);
SMqSubTopicEp topicEp = {0}; SMqSubTopicEp topicEp = {0};
strcpy(topicEp.topic, topic); (void)strcpy(topicEp.topic, topic);
// 2.1 fetch topic schema // 2.1 fetch topic schema
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); SMqTopicObj *pTopic = NULL;
if (pTopic == NULL) { code = mndAcquireTopic(pMnode, topic, &pTopic);
if (code != TDB_CODE_SUCCESS) {
taosRUnLockLatch(&pSub->lock); taosRUnLockLatch(&pSub->lock);
mndReleaseSubscribe(pMnode, pSub); mndReleaseSubscribe(pMnode, pSub);
continue; continue;
@ -371,16 +292,27 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e
topicEp.schema.nCols = pTopic->schema.nCols; topicEp.schema.nCols = pTopic->schema.nCols;
if (topicEp.schema.nCols) { if (topicEp.schema.nCols) {
topicEp.schema.pSchema = taosMemoryCalloc(topicEp.schema.nCols, sizeof(SSchema)); topicEp.schema.pSchema = taosMemoryCalloc(topicEp.schema.nCols, sizeof(SSchema));
memcpy(topicEp.schema.pSchema, pTopic->schema.pSchema, topicEp.schema.nCols * sizeof(SSchema)); if (topicEp.schema.pSchema == NULL) {
taosRUnLockLatch(&pTopic->lock);
taosRUnLockLatch(&pSub->lock);
mndReleaseSubscribe(pMnode, pSub);
mndReleaseTopic(pMnode, pTopic);
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(topicEp.schema.pSchema, pTopic->schema.pSchema, topicEp.schema.nCols * sizeof(SSchema));
} }
taosRUnLockLatch(&pTopic->lock); taosRUnLockLatch(&pTopic->lock);
mndReleaseTopic(pMnode, pTopic); mndReleaseTopic(pMnode, pTopic);
// 2.2 iterate all vg assigned to the consumer of that topic // 2.2 iterate all vg assigned to the consumer of that topic
SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &pConsumer->consumerId, sizeof(int64_t)); SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &pConsumer->consumerId, sizeof(int64_t));
int32_t vgNum = taosArrayGetSize(pConsumerEp->vgs); if (pConsumerEp == NULL) {
taosRUnLockLatch(&pConsumer->lock);
// this customer assigned vgroups taosRUnLockLatch(&pSub->lock);
mndReleaseSubscribe(pMnode, pSub);
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t vgNum = taosArrayGetSize(pConsumerEp->vgs);
topicEp.vgs = taosArrayInit(vgNum, sizeof(SMqSubVgEp)); topicEp.vgs = taosArrayInit(vgNum, sizeof(SMqSubVgEp));
if (topicEp.vgs == NULL) { if (topicEp.vgs == NULL) {
taosRUnLockLatch(&pConsumer->lock); taosRUnLockLatch(&pConsumer->lock);
@ -391,9 +323,9 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e
for (int32_t j = 0; j < vgNum; j++) { for (int32_t j = 0; j < vgNum; j++) {
SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j); SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j);
// char offsetKey[TSDB_PARTITION_KEY_LEN]; if (pVgEp == NULL) {
// mndMakePartitionKey(offsetKey, pConsumer->cgroup, topic, pVgEp->vgId); continue;
}
if (epoch == -1) { if (epoch == -1) {
SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVgEp->vgId); SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVgEp->vgId);
if (pVgroup) { if (pVgroup) {
@ -401,17 +333,10 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e
mndReleaseVgroup(pMnode, pVgroup); mndReleaseVgroup(pMnode, pVgroup);
} }
} }
// 2.2.1 build vg ep SMqSubVgEp vgEp = {.epSet = pVgEp->epSet, .vgId = pVgEp->vgId, .offset = -1};
SMqSubVgEp vgEp = { (void)taosArrayPush(topicEp.vgs, &vgEp);
.epSet = pVgEp->epSet,
.vgId = pVgEp->vgId,
.offset = -1,
};
taosArrayPush(topicEp.vgs, &vgEp);
} }
taosArrayPush(rsp->topics, &topicEp); (void)taosArrayPush(rsp->topics, &topicEp);
taosRUnLockLatch(&pSub->lock); taosRUnLockLatch(&pSub->lock);
mndReleaseSubscribe(pMnode, pSub); mndReleaseSubscribe(pMnode, pSub);
} }
@ -437,12 +362,15 @@ static int32_t buildAskEpRsp(SRpcMsg *pMsg, SMqAskEpRsp *rsp, int32_t serverEpoc
pHead->walever = 0; pHead->walever = 0;
void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
tEncodeSMqAskEpRsp(&abuf, rsp); if (tEncodeSMqAskEpRsp(&abuf, rsp) < 0) {
rpcFreeCont(buf);
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
// send rsp // send rsp
pMsg->info.rsp = buf; pMsg->info.rsp = buf;
pMsg->info.rspLen = tlen; pMsg->info.rspLen = tlen;
TAOS_RETURN(code); return code;
} }
static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
@ -450,33 +378,22 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
SMqAskEpReq req = {0}; SMqAskEpReq req = {0};
SMqAskEpRsp rsp = {0}; SMqAskEpRsp rsp = {0};
int32_t code = 0; int32_t code = 0;
SMqConsumerObj *pConsumer = NULL;
TAOS_CHECK_RETURN(tDeserializeSMqAskEpReq(pMsg->pCont, pMsg->contLen, &req)); MND_TMQ_RETURN_CHECK(tDeserializeSMqAskEpReq(pMsg->pCont, pMsg->contLen, &req));
int64_t consumerId = req.consumerId; int64_t consumerId = req.consumerId;
MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, consumerId, &pConsumer));
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId);
if (pConsumer == NULL) {
mError("consumer:0x%" PRIx64 " group:%s not exists in sdb", consumerId, req.cgroup);
return TSDB_CODE_MND_CONSUMER_NOT_EXIST;
}
if (strncmp(req.cgroup, pConsumer->cgroup, tListLen(pConsumer->cgroup)) != 0) { if (strncmp(req.cgroup, pConsumer->cgroup, tListLen(pConsumer->cgroup)) != 0) {
mError("consumer:0x%" PRIx64 " group:%s not consistent with data in sdb, saved cgroup:%s", consumerId, req.cgroup, mError("consumer:0x%" PRIx64 " group:%s not consistent with data in sdb, saved cgroup:%s", consumerId, req.cgroup,
pConsumer->cgroup); pConsumer->cgroup);
code = TSDB_CODE_MND_CONSUMER_NOT_EXIST; code = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
goto END; goto END;
} }
atomic_store_32(&pConsumer->hbStatus, 0); atomic_store_32(&pConsumer->hbStatus, 0);
// 1. check consumer status
int32_t status = atomic_load_32(&pConsumer->status); int32_t status = atomic_load_32(&pConsumer->status);
if (status == MQ_CONSUMER_STATUS_LOST) { if (status == MQ_CONSUMER_STATUS_LOST) {
mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_CONSUMER_RECOVER, &pMsg->info); MND_TMQ_RETURN_CHECK(mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_CONSUMER_RECOVER, &pMsg->info));
} }
if (status != MQ_CONSUMER_STATUS_READY) { if (status != MQ_CONSUMER_STATUS_READY) {
mInfo("consumer:0x%" PRIx64 " not ready, status: %s", consumerId, mndConsumerStatusName(status)); mInfo("consumer:0x%" PRIx64 " not ready, status: %s", consumerId, mndConsumerStatusName(status));
code = TSDB_CODE_MND_CONSUMER_NOT_READY; code = TSDB_CODE_MND_CONSUMER_NOT_READY;
@ -490,10 +407,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
if (epoch != serverEpoch) { if (epoch != serverEpoch) {
mInfo("process ask ep, consumer:0x%" PRIx64 "(epoch %d) update with server epoch %d", mInfo("process ask ep, consumer:0x%" PRIx64 "(epoch %d) update with server epoch %d",
consumerId, epoch, serverEpoch); consumerId, epoch, serverEpoch);
code = addEpSetInfo(pMnode, pConsumer, epoch, &rsp); MND_TMQ_RETURN_CHECK(addEpSetInfo(pMnode, pConsumer, epoch, &rsp));
if(code != 0){
goto END;
}
} }
code = buildAskEpRsp(pMsg, &rsp, serverEpoch, consumerId); code = buildAskEpRsp(pMsg, &rsp, serverEpoch, consumerId);
@ -507,27 +421,29 @@ END:
int32_t mndSetConsumerDropLogs(STrans *pTrans, SMqConsumerObj *pConsumer) { int32_t mndSetConsumerDropLogs(STrans *pTrans, SMqConsumerObj *pConsumer) {
int32_t code = 0; int32_t code = 0;
SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer); SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer);
if (pCommitRaw == NULL) { MND_TMQ_NULL_CHECK(pCommitRaw);
code = TSDB_CODE_MND_RETURN_VALUE_NULL; code = mndTransAppendCommitlog(pTrans, pCommitRaw);
if (terrno != 0) code = terrno; if (code != 0) {
TAOS_RETURN(code); sdbFreeRaw(pCommitRaw);
goto END;
} }
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw)); MND_TMQ_RETURN_CHECK(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED)); END:
TAOS_RETURN(code); return code;
} }
int32_t mndSetConsumerCommitLogs(STrans *pTrans, SMqConsumerObj *pConsumer) { int32_t mndSetConsumerCommitLogs(STrans *pTrans, SMqConsumerObj *pConsumer) {
int32_t code = 0; int32_t code = 0;
SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer); SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer);
if (pCommitRaw == NULL) { MND_TMQ_NULL_CHECK(pCommitRaw);
code = TSDB_CODE_MND_RETURN_VALUE_NULL; code = mndTransAppendCommitlog(pTrans, pCommitRaw);
if (terrno != 0) code = terrno; if (code != 0) {
TAOS_RETURN(code); sdbFreeRaw(pCommitRaw);
goto END;
} }
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw)); MND_TMQ_RETURN_CHECK(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)); END:
TAOS_RETURN(code); return code;
} }
static void freeItem(void *param) { static void freeItem(void *param) {
@ -538,29 +454,35 @@ static void freeItem(void *param) {
} }
static int32_t getTopicAddDelete(SMqConsumerObj *pExistedConsumer, SMqConsumerObj *pConsumerNew){ static int32_t getTopicAddDelete(SMqConsumerObj *pExistedConsumer, SMqConsumerObj *pConsumerNew){
int32_t code = 0;
pConsumerNew->rebNewTopics = taosArrayInit(0, sizeof(void *)); pConsumerNew->rebNewTopics = taosArrayInit(0, sizeof(void *));
MND_TMQ_NULL_CHECK(pConsumerNew->rebNewTopics);
pConsumerNew->rebRemovedTopics = taosArrayInit(0, sizeof(void *)); pConsumerNew->rebRemovedTopics = taosArrayInit(0, sizeof(void *));
if(pConsumerNew->rebNewTopics == NULL || pConsumerNew->rebRemovedTopics == NULL){ MND_TMQ_NULL_CHECK(pConsumerNew->rebRemovedTopics);
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t newTopicNum = taosArrayGetSize(pConsumerNew->assignedTopics); int32_t newTopicNum = taosArrayGetSize(pConsumerNew->assignedTopics);
int32_t oldTopicNum = taosArrayGetSize(pExistedConsumer->currentTopics); int32_t oldTopicNum = taosArrayGetSize(pExistedConsumer->currentTopics);
int32_t i = 0, j = 0; int32_t i = 0, j = 0;
while (i < oldTopicNum || j < newTopicNum) { while (i < oldTopicNum || j < newTopicNum) {
if (i >= oldTopicNum) { if (i >= oldTopicNum) {
char *newTopicCopy = taosStrdup(taosArrayGetP(pConsumerNew->assignedTopics, j)); void* tmp = taosArrayGetP(pConsumerNew->assignedTopics, j);
taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy); MND_TMQ_NULL_CHECK(tmp);
char *newTopicCopy = taosStrdup(tmp);
MND_TMQ_NULL_CHECK(taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy));
j++; j++;
continue; continue;
} else if (j >= newTopicNum) { } else if (j >= newTopicNum) {
char *oldTopicCopy = taosStrdup(taosArrayGetP(pExistedConsumer->currentTopics, i)); void* tmp = taosArrayGetP(pExistedConsumer->currentTopics, i);
taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy); MND_TMQ_NULL_CHECK(tmp);
char *oldTopicCopy = taosStrdup(tmp);
MND_TMQ_NULL_CHECK(taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy));
i++; i++;
continue; continue;
} else { } else {
char *oldTopic = taosArrayGetP(pExistedConsumer->currentTopics, i); char *oldTopic = taosArrayGetP(pExistedConsumer->currentTopics, i);
MND_TMQ_NULL_CHECK(oldTopic);
char *newTopic = taosArrayGetP(pConsumerNew->assignedTopics, j); char *newTopic = taosArrayGetP(pConsumerNew->assignedTopics, j);
MND_TMQ_NULL_CHECK(newTopic);
int comp = strcmp(oldTopic, newTopic); int comp = strcmp(oldTopic, newTopic);
if (comp == 0) { if (comp == 0) {
i++; i++;
@ -568,22 +490,22 @@ static int32_t getTopicAddDelete(SMqConsumerObj *pExistedConsumer, SMqConsumerOb
continue; continue;
} else if (comp < 0) { } else if (comp < 0) {
char *oldTopicCopy = taosStrdup(oldTopic); char *oldTopicCopy = taosStrdup(oldTopic);
taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy); MND_TMQ_NULL_CHECK(taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy));
i++; i++;
continue; continue;
} else { } else {
char *newTopicCopy = taosStrdup(newTopic); char *newTopicCopy = taosStrdup(newTopic);
taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy); MND_TMQ_NULL_CHECK(taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy));
j++; j++;
continue; continue;
} }
} }
} }
return 0; END:
return code;
} }
static int32_t checkAndSortTopic(SMnode *pMnode, SArray *pTopicList){ static int32_t checkAndSortTopic(SMnode *pMnode, SArray *pTopicList){
int32_t code = 0;
taosArraySort(pTopicList, taosArrayCompareString); taosArraySort(pTopicList, taosArrayCompareString);
taosArrayRemoveDuplicate(pTopicList, taosArrayCompareString, freeItem); taosArrayRemoveDuplicate(pTopicList, taosArrayCompareString, freeItem);
@ -594,24 +516,21 @@ static int32_t checkAndSortTopic(SMnode *pMnode, SArray *pTopicList){
return TSDB_CODE_TMQ_GROUP_OUT_OF_RANGE; return TSDB_CODE_TMQ_GROUP_OUT_OF_RANGE;
} }
} }
TAOS_RETURN(code); return 0;
} }
static SMqConsumerObj* buildSubConsumer(SMnode *pMnode, SCMSubscribeReq *subscribe){ static int32_t buildSubConsumer(SMnode *pMnode, SCMSubscribeReq *subscribe, SMqConsumerObj** ppConsumer){
int64_t consumerId = subscribe->consumerId; int64_t consumerId = subscribe->consumerId;
char *cgroup = subscribe->cgroup; char *cgroup = subscribe->cgroup;
SMqConsumerObj *pConsumerNew = NULL; SMqConsumerObj *pConsumerNew = NULL;
SMqConsumerObj *pExistedConsumer = NULL; SMqConsumerObj *pExistedConsumer = NULL;
pExistedConsumer = mndAcquireConsumer(pMnode, consumerId); int32_t code = mndAcquireConsumer(pMnode, consumerId, &pExistedConsumer);
if (pExistedConsumer == NULL) { if (code != 0) {
mInfo("receive subscribe request from new consumer:0x%" PRIx64 mInfo("receive subscribe request from new consumer:0x%" PRIx64
",cgroup:%s, numOfTopics:%d", consumerId, ",cgroup:%s, numOfTopics:%d", consumerId,
subscribe->cgroup, (int32_t)taosArrayGetSize(subscribe->topicNames)); subscribe->cgroup, (int32_t)taosArrayGetSize(subscribe->topicNames));
pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup, CONSUMER_INSERT_SUB, NULL, subscribe); MND_TMQ_RETURN_CHECK(tNewSMqConsumerObj(consumerId, cgroup, CONSUMER_INSERT_SUB, NULL, subscribe, &pConsumerNew));
if (pConsumerNew == NULL) {
goto _over;
}
} else { } else {
int32_t status = atomic_load_32(&pExistedConsumer->status); int32_t status = atomic_load_32(&pExistedConsumer->status);
@ -621,82 +540,53 @@ static SMqConsumerObj* buildSubConsumer(SMnode *pMnode, SCMSubscribeReq *subscri
(int32_t)taosArrayGetSize(subscribe->topicNames)); (int32_t)taosArrayGetSize(subscribe->topicNames));
if (status != MQ_CONSUMER_STATUS_READY) { if (status != MQ_CONSUMER_STATUS_READY) {
terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; code = TSDB_CODE_MND_CONSUMER_NOT_READY;
goto _over; goto END;
}
pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup, CONSUMER_UPDATE_SUB, NULL, subscribe);
if (pConsumerNew == NULL) {
goto _over;
}
int32_t code = getTopicAddDelete(pExistedConsumer, pConsumerNew);
if (code != 0){
terrno = code;
goto _over;
} }
MND_TMQ_RETURN_CHECK(tNewSMqConsumerObj(consumerId, cgroup, CONSUMER_UPDATE_SUB, NULL, subscribe, &pConsumerNew));
MND_TMQ_RETURN_CHECK(getTopicAddDelete(pExistedConsumer, pConsumerNew));
} }
mndReleaseConsumer(pMnode, pExistedConsumer); mndReleaseConsumer(pMnode, pExistedConsumer);
return pConsumerNew; if (ppConsumer){
*ppConsumer = pConsumerNew;
}
return code;
_over: END:
mndReleaseConsumer(pMnode, pExistedConsumer); mndReleaseConsumer(pMnode, pExistedConsumer);
tDeleteSMqConsumerObj(pConsumerNew); tDeleteSMqConsumerObj(pConsumerNew);
return NULL; return code;
} }
int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
char *msgStr = pMsg->pCont; char *msgStr = pMsg->pCont;
int32_t code = 0; int32_t code = 0;
SCMSubscribeReq subscribe = {0};
tDeserializeSCMSubscribeReq(msgStr, &subscribe);
SMqConsumerObj *pConsumerNew = NULL; SMqConsumerObj *pConsumerNew = NULL;
STrans *pTrans = NULL; STrans *pTrans = NULL;
SCMSubscribeReq subscribe = {0};
MND_TMQ_RETURN_CHECK(tDeserializeSCMSubscribeReq(msgStr, &subscribe));
if(taosArrayGetSize(subscribe.topicNames) == 0){ if(taosArrayGetSize(subscribe.topicNames) == 0){
SMqConsumerObj *pConsumerTmp = mndAcquireConsumer(pMnode, subscribe.consumerId); SMqConsumerObj *pConsumerTmp = NULL;
if(pConsumerTmp == NULL){ MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, subscribe.consumerId, &pConsumerTmp));
goto _over;
}
mndReleaseConsumer(pMnode, pConsumerTmp); mndReleaseConsumer(pMnode, pConsumerTmp);
} }
MND_TMQ_RETURN_CHECK(checkAndSortTopic(pMnode, subscribe.topicNames));
code = checkAndSortTopic(pMnode, subscribe.topicNames);
if(code != TSDB_CODE_SUCCESS){
goto _over;
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe"); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe");
if (pTrans == NULL) { MND_TMQ_NULL_CHECK(pTrans);
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _over;
}
code = validateTopics(subscribe.topicNames, pMnode, pMsg->info.conn.user, subscribe.enableReplay); MND_TMQ_RETURN_CHECK(validateTopics(subscribe.topicNames, pMnode, pMsg->info.conn.user, subscribe.enableReplay));
if (code != TSDB_CODE_SUCCESS) { MND_TMQ_RETURN_CHECK(buildSubConsumer(pMnode, &subscribe, &pConsumerNew));
goto _over; MND_TMQ_RETURN_CHECK(mndSetConsumerCommitLogs(pTrans, pConsumerNew));
} MND_TMQ_RETURN_CHECK(mndTransPrepare(pMnode, pTrans));
pConsumerNew = buildSubConsumer(pMnode, &subscribe);
if(pConsumerNew == NULL){
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _over;
}
TAOS_CHECK_GOTO(mndSetConsumerCommitLogs(pTrans, pConsumerNew), NULL, _over);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _over);
code = TSDB_CODE_ACTION_IN_PROGRESS; code = TSDB_CODE_ACTION_IN_PROGRESS;
_over: END:
mndTransDrop(pTrans); mndTransDrop(pTrans);
tDeleteSMqConsumerObj(pConsumerNew); tDeleteSMqConsumerObj(pConsumerNew);
taosArrayDestroyP(subscribe.topicNames, (FDelete)taosMemoryFree); taosArrayDestroyP(subscribe.topicNames, (FDelete)taosMemoryFree);
TAOS_RETURN(code); return code;
} }
SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) {
@ -713,7 +603,9 @@ SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) {
if (buf == NULL) goto CM_ENCODE_OVER; if (buf == NULL) goto CM_ENCODE_OVER;
void *abuf = buf; void *abuf = buf;
tEncodeSMqConsumerObj(&abuf, pConsumer); if(tEncodeSMqConsumerObj(&abuf, pConsumer) < 0){
goto CM_ENCODE_OVER;
}
int32_t dataPos = 0; int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, tlen, CM_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, tlen, CM_ENCODE_OVER);
@ -838,8 +730,7 @@ static bool existInCurrentTopicList(const SMqConsumerObj *pConsumer, const char
int32_t size = taosArrayGetSize(pConsumer->currentTopics); int32_t size = taosArrayGetSize(pConsumer->currentTopics);
for (int32_t i = 0; i < size; i++) { for (int32_t i = 0; i < size; i++) {
char *topic = taosArrayGetP(pConsumer->currentTopics, i); char *topic = taosArrayGetP(pConsumer->currentTopics, i);
if (topic && strcmp(topic, pTopic) == 0) {
if (strcmp(topic, pTopic) == 0) {
existing = true; existing = true;
break; break;
} }
@ -865,32 +756,45 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
} else if (pNewConsumer->updateType == CONSUMER_UPDATE_REC) { } else if (pNewConsumer->updateType == CONSUMER_UPDATE_REC) {
int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics); int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics);
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
char *topic = taosStrdup(taosArrayGetP(pOldConsumer->assignedTopics, i)); void * tmp = taosArrayGetP(pOldConsumer->assignedTopics, i);
taosArrayPush(pOldConsumer->rebNewTopics, &topic); if (tmp == NULL){
return TSDB_CODE_TMQ_INVALID_MSG;
}
char *topic = taosStrdup(tmp);
if (taosArrayPush(pOldConsumer->rebNewTopics, &topic) == NULL) {
taosMemoryFree(topic);
return TSDB_CODE_TMQ_INVALID_MSG;
}
} }
pOldConsumer->status = MQ_CONSUMER_STATUS_REBALANCE; pOldConsumer->status = MQ_CONSUMER_STATUS_REBALANCE;
mInfo("consumer:0x%" PRIx64 " recover update", pOldConsumer->consumerId); mInfo("consumer:0x%" PRIx64 " recover update", pOldConsumer->consumerId);
} else if (pNewConsumer->updateType == CONSUMER_UPDATE_REB) { } else if (pNewConsumer->updateType == CONSUMER_UPDATE_REB) {
atomic_add_fetch_32(&pOldConsumer->epoch, 1); (void)atomic_add_fetch_32(&pOldConsumer->epoch, 1);
pOldConsumer->rebalanceTime = taosGetTimestampMs(); pOldConsumer->rebalanceTime = taosGetTimestampMs();
mInfo("[rebalance] consumer:0x%" PRIx64 " rebalance update, only rebalance time", pOldConsumer->consumerId); mInfo("[rebalance] consumer:0x%" PRIx64 " rebalance update, only rebalance time", pOldConsumer->consumerId);
} else if (pNewConsumer->updateType == CONSUMER_ADD_REB) { } else if (pNewConsumer->updateType == CONSUMER_ADD_REB) {
char *pNewTopic = taosStrdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0)); void *tmp = taosArrayGetP(pNewConsumer->rebNewTopics, 0);
if (tmp == NULL){
return TSDB_CODE_TMQ_INVALID_MSG;
}
char *pNewTopic = taosStrdup(tmp);
removeFromTopicList(pOldConsumer->rebNewTopics, pNewTopic, pOldConsumer->consumerId, "new"); removeFromTopicList(pOldConsumer->rebNewTopics, pNewTopic, pOldConsumer->consumerId, "new");
bool existing = existInCurrentTopicList(pOldConsumer, pNewTopic); bool existing = existInCurrentTopicList(pOldConsumer, pNewTopic);
if (existing) { if (existing) {
mError("[rebalance] consumer:0x%" PRIx64 " add new topic:%s should not in currentTopics", pOldConsumer->consumerId, pNewTopic); mError("[rebalance] consumer:0x%" PRIx64 " add new topic:%s should not in currentTopics", pOldConsumer->consumerId, pNewTopic);
taosMemoryFree(pNewTopic); taosMemoryFree(pNewTopic);
} else { } else {
taosArrayPush(pOldConsumer->currentTopics, &pNewTopic); if (taosArrayPush(pOldConsumer->currentTopics, &pNewTopic) == NULL) {
taosMemoryFree(pNewTopic);
return TSDB_CODE_TMQ_INVALID_MSG;
}
taosArraySort(pOldConsumer->currentTopics, taosArrayCompareString); taosArraySort(pOldConsumer->currentTopics, taosArrayCompareString);
} }
int32_t status = pOldConsumer->status; int32_t status = pOldConsumer->status;
updateConsumerStatus(pOldConsumer); updateConsumerStatus(pOldConsumer);
pOldConsumer->rebalanceTime = taosGetTimestampMs(); pOldConsumer->rebalanceTime = taosGetTimestampMs();
atomic_add_fetch_32(&pOldConsumer->epoch, 1); (void)atomic_add_fetch_32(&pOldConsumer->epoch, 1);
mInfo("[rebalance] consumer:0x%" PRIx64 " rebalance update add, state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64 mInfo("[rebalance] consumer:0x%" PRIx64 " rebalance update add, state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64
", current topics:%d, newTopics:%d, removeTopics:%d", ", current topics:%d, newTopics:%d, removeTopics:%d",
@ -901,13 +805,16 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
} else if (pNewConsumer->updateType == CONSUMER_REMOVE_REB) { } else if (pNewConsumer->updateType == CONSUMER_REMOVE_REB) {
char *topic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0); char *topic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0);
if (topic == NULL){
return TSDB_CODE_TMQ_INVALID_MSG;
}
removeFromTopicList(pOldConsumer->rebRemovedTopics, topic, pOldConsumer->consumerId, "remove"); removeFromTopicList(pOldConsumer->rebRemovedTopics, topic, pOldConsumer->consumerId, "remove");
removeFromTopicList(pOldConsumer->currentTopics, topic, pOldConsumer->consumerId, "current"); removeFromTopicList(pOldConsumer->currentTopics, topic, pOldConsumer->consumerId, "current");
int32_t status = pOldConsumer->status; int32_t status = pOldConsumer->status;
updateConsumerStatus(pOldConsumer); updateConsumerStatus(pOldConsumer);
pOldConsumer->rebalanceTime = taosGetTimestampMs(); pOldConsumer->rebalanceTime = taosGetTimestampMs();
atomic_add_fetch_32(&pOldConsumer->epoch, 1); (void)atomic_add_fetch_32(&pOldConsumer->epoch, 1);
mInfo("[rebalance]consumer:0x%" PRIx64 " rebalance update remove, state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64 mInfo("[rebalance]consumer:0x%" PRIx64 " rebalance update remove, state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64
", current topics:%d, newTopics:%d, removeTopics:%d", ", current topics:%d, newTopics:%d, removeTopics:%d",
@ -921,13 +828,13 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
return 0; return 0;
} }
SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int64_t consumerId) { int32_t mndAcquireConsumer(SMnode *pMnode, int64_t consumerId, SMqConsumerObj** pConsumer) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SMqConsumerObj *pConsumer = sdbAcquire(pSdb, SDB_CONSUMER, &consumerId); *pConsumer = sdbAcquire(pSdb, SDB_CONSUMER, &consumerId);
if (pConsumer == NULL) { if (*pConsumer == NULL) {
terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; return TSDB_CODE_MND_CONSUMER_NOT_EXIST;
} }
return pConsumer; return 0;
} }
void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer) { void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer) {
@ -940,6 +847,7 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0; int32_t numOfRows = 0;
SMqConsumerObj *pConsumer = NULL; SMqConsumerObj *pConsumer = NULL;
int32_t code = 0;
while (numOfRows < rowsCapacity) { while (numOfRows < rowsCapacity) {
pShow->pIter = sdbFetch(pSdb, SDB_CONSUMER, pShow->pIter, (void **)&pConsumer); pShow->pIter = sdbFetch(pSdb, SDB_CONSUMER, pShow->pIter, (void **)&pConsumer);
@ -964,34 +872,37 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *
} }
if (numOfRows + topicSz > rowsCapacity) { if (numOfRows + topicSz > rowsCapacity) {
blockDataEnsureCapacity(pBlock, numOfRows + topicSz); MND_TMQ_RETURN_CHECK(blockDataEnsureCapacity(pBlock, numOfRows + topicSz));
} }
for (int32_t i = 0; i < topicSz; i++) { for (int32_t i = 0; i < topicSz; i++) {
SColumnInfoData *pColInfo; SColumnInfoData *pColInfo = NULL;
int32_t cols = 0; int32_t cols = 0;
// consumer id // consumer id
char consumerIdHex[32] = {0}; char consumerIdHex[32] = {0};
sprintf(varDataVal(consumerIdHex), "0x%" PRIx64, pConsumer->consumerId); (void)sprintf(varDataVal(consumerIdHex), "0x%" PRIx64, pConsumer->consumerId);
varDataSetLen(consumerIdHex, strlen(varDataVal(consumerIdHex))); varDataSetLen(consumerIdHex, strlen(varDataVal(consumerIdHex)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)consumerIdHex, false); MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)consumerIdHex, false));
// consumer group // consumer group
char cgroup[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; char cgroup[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(cgroup, pConsumer->cgroup); STR_TO_VARSTR(cgroup, pConsumer->cgroup);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)cgroup, false); MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)cgroup, false));
// client id // client id
char clientId[256 + VARSTR_HEADER_SIZE] = {0}; char clientId[256 + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(clientId, pConsumer->clientId); STR_TO_VARSTR(clientId, pConsumer->clientId);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)clientId, false); MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)clientId, false));
// status // status
char status[20 + VARSTR_HEADER_SIZE] = {0}; char status[20 + VARSTR_HEADER_SIZE] = {0};
@ -999,46 +910,48 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *
STR_TO_VARSTR(status, pStatusName); STR_TO_VARSTR(status, pStatusName);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)status, false); MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)status, false));
// one subscribed topic // one subscribed topic
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
MND_TMQ_NULL_CHECK(pColInfo);
if (hasTopic) { if (hasTopic) {
char topic[TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char topic[TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
const char *topicName = mndTopicGetShowName(taosArrayGetP(pConsumer->assignedTopics, i)); mndTopicGetShowName(taosArrayGetP(pConsumer->assignedTopics, i), topic + VARSTR_HEADER_SIZE);
STR_TO_VARSTR(topic, topicName); *(VarDataLenT *)(topic) = strlen(topic + VARSTR_HEADER_SIZE);
colDataSetVal(pColInfo, numOfRows, (const char *)topic, false); MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)topic, false));
} else { } else {
colDataSetVal(pColInfo, numOfRows, NULL, true); MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, NULL, true));
} }
// end point
/*pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);*/
/*colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->ep, true);*/
// up time // up time
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->createTime, false); MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->createTime, false));
// subscribe time // subscribe time
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->subscribeTime, false); MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->subscribeTime, false));
// rebalance time // rebalance time
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0); MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0));
char buf[TSDB_OFFSET_LEN] = {0}; char buf[TSDB_OFFSET_LEN] = {0};
STqOffsetVal pVal = {.type = pConsumer->resetOffsetCfg}; STqOffsetVal pVal = {.type = pConsumer->resetOffsetCfg};
tFormatOffset(buf, TSDB_OFFSET_LEN, &pVal); MND_TMQ_RETURN_CHECK(tFormatOffset(buf, TSDB_OFFSET_LEN, &pVal));
char parasStr[64 + TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE] = {0}; char parasStr[64 + TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE] = {0};
sprintf(varDataVal(parasStr), "tbname:%d,commit:%d,interval:%dms,reset:%s", pConsumer->withTbName, (void)sprintf(varDataVal(parasStr), "tbname:%d,commit:%d,interval:%dms,reset:%s", pConsumer->withTbName,
pConsumer->autoCommit, pConsumer->autoCommitInterval, buf); pConsumer->autoCommit, pConsumer->autoCommitInterval, buf);
varDataSetLen(parasStr, strlen(varDataVal(parasStr))); varDataSetLen(parasStr, strlen(varDataVal(parasStr)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)parasStr, false); MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)parasStr, false));
numOfRows++; numOfRows++;
} }
@ -1051,6 +964,9 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *
pShow->numOfRows += numOfRows; pShow->numOfRows += numOfRows;
return numOfRows; return numOfRows;
END:
return code;
} }
static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) { static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) {

View File

@ -12,173 +12,179 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "mndDef.h" #include "mndDef.h"
#include "mndConsumer.h" #include "mndConsumer.h"
#include "taoserror.h"
static void *freeStreamTasks(SArray *pTaskLevel); static void *freeStreamTasks(SArray *pTaskLevel);
int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
if (tStartEncode(pEncoder) < 0) return -1; TAOS_CHECK_RETURN(tStartEncode(pEncoder));
if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->name));
if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->createTime));
if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->updateTime));
if (tEncodeI32(pEncoder, pObj->version) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pObj->version));
if (tEncodeI32(pEncoder, pObj->totalLevel) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pObj->totalLevel));
if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->smaId));
if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->uid));
if (tEncodeI8(pEncoder, pObj->status) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->status));
if (tEncodeI8(pEncoder, pObj->conf.igExpired) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->conf.igExpired));
if (tEncodeI8(pEncoder, pObj->conf.trigger) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->conf.trigger));
if (tEncodeI8(pEncoder, pObj->conf.fillHistory) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->conf.fillHistory));
if (tEncodeI64(pEncoder, pObj->conf.triggerParam) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->conf.triggerParam));
if (tEncodeI64(pEncoder, pObj->conf.watermark) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->conf.watermark));
if (tEncodeI64(pEncoder, pObj->sourceDbUid) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->sourceDbUid));
if (tEncodeI64(pEncoder, pObj->targetDbUid) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->targetDbUid));
if (tEncodeCStr(pEncoder, pObj->sourceDb) < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->sourceDb));
if (tEncodeCStr(pEncoder, pObj->targetDb) < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->targetDb));
if (tEncodeCStr(pEncoder, pObj->targetSTbName) < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->targetSTbName));
if (tEncodeI64(pEncoder, pObj->targetStbUid) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->targetStbUid));
if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pObj->fixedSinkVgId));
if (pObj->sql != NULL) { if (pObj->sql != NULL) {
if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->sql));
} else { } else {
if (tEncodeCStr(pEncoder, "") < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, ""));
} }
if (pObj->ast != NULL) { if (pObj->ast != NULL) {
if (tEncodeCStr(pEncoder, pObj->ast) < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->ast));
} else { } else {
if (tEncodeCStr(pEncoder, "") < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, ""));
} }
if (pObj->physicalPlan != NULL) { if (pObj->physicalPlan != NULL) {
if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->physicalPlan));
} else { } else {
if (tEncodeCStr(pEncoder, "") < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, ""));
} }
int32_t sz = taosArrayGetSize(pObj->tasks); int32_t sz = taosArrayGetSize(pObj->tasks);
if (tEncodeI32(pEncoder, sz) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32(pEncoder, sz));
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
SArray *pArray = taosArrayGetP(pObj->tasks, i); SArray *pArray = taosArrayGetP(pObj->tasks, i);
int32_t innerSz = taosArrayGetSize(pArray); int32_t innerSz = taosArrayGetSize(pArray);
if (tEncodeI32(pEncoder, innerSz) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32(pEncoder, innerSz));
for (int32_t j = 0; j < innerSz; j++) { for (int32_t j = 0; j < innerSz; j++) {
SStreamTask *pTask = taosArrayGetP(pArray, j); SStreamTask *pTask = taosArrayGetP(pArray, j);
if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER){ if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER){
pTask->ver = SSTREAM_TASK_VER; pTask->ver = SSTREAM_TASK_VER;
} }
if (tEncodeStreamTask(pEncoder, pTask) < 0) return -1; TAOS_CHECK_RETURN(tEncodeStreamTask(pEncoder, pTask));
} }
} }
if (tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema) < 0) return -1; TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema));
// 3.0.20 ver =2 // 3.0.20 ver =2
if (tEncodeI64(pEncoder, pObj->checkpointFreq) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->checkpointFreq));
if (tEncodeI8(pEncoder, pObj->igCheckUpdate) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->igCheckUpdate));
// 3.0.50 ver = 3 // 3.0.50 ver = 3
if (tEncodeI64(pEncoder, pObj->checkpointId) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->checkpointId));
if (tEncodeI8(pEncoder, pObj->subTableWithoutMd5) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->subTableWithoutMd5));
if (tEncodeCStrWithLen(pEncoder, pObj->reserve, sizeof(pObj->reserve) - 1) < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStrWithLen(pEncoder, pObj->reserve, sizeof(pObj->reserve) - 1));
tEndEncode(pEncoder); tEndEncode(pEncoder);
return pEncoder->pos; return pEncoder->pos;
} }
int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) { int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) {
if (tStartDecode(pDecoder) < 0) return -1; int32_t code = 0;
if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1; TAOS_CHECK_RETURN(tStartDecode(pDecoder));
TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->name));
if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->createTime));
if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->updateTime));
if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pObj->version));
if (tDecodeI32(pDecoder, &pObj->totalLevel) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pObj->totalLevel));
if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->smaId));
if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->uid));
if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->status));
if (tDecodeI8(pDecoder, &pObj->conf.igExpired) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->conf.igExpired));
if (tDecodeI8(pDecoder, &pObj->conf.trigger) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->conf.trigger));
if (tDecodeI8(pDecoder, &pObj->conf.fillHistory) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->conf.fillHistory));
if (tDecodeI64(pDecoder, &pObj->conf.triggerParam) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->conf.triggerParam));
if (tDecodeI64(pDecoder, &pObj->conf.watermark) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->conf.watermark));
if (tDecodeI64(pDecoder, &pObj->sourceDbUid) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->sourceDbUid));
if (tDecodeI64(pDecoder, &pObj->targetDbUid) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->targetDbUid));
if (tDecodeCStrTo(pDecoder, pObj->sourceDb) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->sourceDb));
if (tDecodeCStrTo(pDecoder, pObj->targetDb) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->targetDb));
if (tDecodeCStrTo(pDecoder, pObj->targetSTbName) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->targetSTbName));
if (tDecodeI64(pDecoder, &pObj->targetStbUid) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->targetStbUid));
if (tDecodeI32(pDecoder, &pObj->fixedSinkVgId) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pObj->fixedSinkVgId));
if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStrAlloc(pDecoder, &pObj->sql));
if (tDecodeCStrAlloc(pDecoder, &pObj->ast) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStrAlloc(pDecoder, &pObj->ast));
if (tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan));
if (pObj->tasks != NULL) { if (pObj->tasks != NULL) {
pObj->tasks = freeStreamTasks(pObj->tasks); pObj->tasks = freeStreamTasks(pObj->tasks);
} }
int32_t sz; int32_t sz;
if (tDecodeI32(pDecoder, &sz) < 0) { TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &sz));
return -1;
}
if (sz != 0) { if (sz != 0) {
pObj->tasks = taosArrayInit(sz, sizeof(void *)); pObj->tasks = taosArrayInit(sz, sizeof(void *));
if (pObj->tasks == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
int32_t innerSz; int32_t innerSz;
if (tDecodeI32(pDecoder, &innerSz) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &innerSz));
SArray *pArray = taosArrayInit(innerSz, sizeof(void *)); SArray *pArray = taosArrayInit(innerSz, sizeof(void *));
for (int32_t j = 0; j < innerSz; j++) { if (pArray != NULL) {
SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); for (int32_t j = 0; j < innerSz; j++) {
if (pTask == NULL) { SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask));
taosArrayDestroy(pArray); if (pTask == NULL) {
return -1; taosArrayDestroy(pArray);
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
if ((code = tDecodeStreamTask(pDecoder, pTask)) < 0) {
taosMemoryFree(pTask);
taosArrayDestroy(pArray);
TAOS_RETURN(code);
}
taosArrayPush(pArray, &pTask);
} }
if (tDecodeStreamTask(pDecoder, pTask) < 0) {
taosMemoryFree(pTask);
taosArrayDestroy(pArray);
return -1;
}
taosArrayPush(pArray, &pTask);
} }
taosArrayPush(pObj->tasks, &pArray); taosArrayPush(pObj->tasks, &pArray);
} }
} }
if (tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema) < 0) return -1; TAOS_CHECK_RETURN(tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema));
// 3.0.20 // 3.0.20
if (sver >= 2) { if (sver >= 2) {
if (tDecodeI64(pDecoder, &pObj->checkpointFreq) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->checkpointFreq));
if (!tDecodeIsEnd(pDecoder)) { if (!tDecodeIsEnd(pDecoder)) {
if (tDecodeI8(pDecoder, &pObj->igCheckUpdate) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->igCheckUpdate));
} }
} }
if (sver >= 3) { if (sver >= 3) {
if (tDecodeI64(pDecoder, &pObj->checkpointId) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->checkpointId));
} }
if (sver >= 5) { if (sver >= 5) {
if (tDecodeI8(pDecoder, &pObj->subTableWithoutMd5) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->subTableWithoutMd5));
} }
if (tDecodeCStrTo(pDecoder, pObj->reserve) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->reserve));
tEndDecode(pDecoder); tEndDecode(pDecoder);
return 0; TAOS_RETURN(code);
} }
void *freeStreamTasks(SArray *pTaskLevel) { void *freeStreamTasks(SArray *pTaskLevel) {
@ -220,7 +226,10 @@ void tFreeStreamObj(SStreamObj *pStream) {
SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) { SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) {
SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp)); SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp));
if (pVgEpNew == NULL) return NULL; if (pVgEpNew == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pVgEpNew->vgId = pVgEp->vgId; pVgEpNew->vgId = pVgEp->vgId;
// pVgEpNew->qmsg = taosStrdup(pVgEp->qmsg); // pVgEpNew->qmsg = taosStrdup(pVgEp->qmsg);
pVgEpNew->epSet = pVgEp->epSet; pVgEpNew->epSet = pVgEp->epSet;
@ -255,15 +264,17 @@ void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp, int8_t sver) {
static void *topicNameDup(void *p) { return taosStrdup((char *)p); } static void *topicNameDup(void *p) { return taosStrdup((char *)p); }
SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, char *topic, SCMSubscribeReq *subscribe) { int32_t tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType,
char *topic, SCMSubscribeReq *subscribe, SMqConsumerObj** ppConsumer) {
int32_t code = 0;
SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj)); SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj));
if (pConsumer == NULL) { if (pConsumer == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return NULL; goto END;
} }
pConsumer->consumerId = consumerId; pConsumer->consumerId = consumerId;
memcpy(pConsumer->cgroup, cgroup, TSDB_CGROUP_LEN); (void)memcpy(pConsumer->cgroup, cgroup, TSDB_CGROUP_LEN);
pConsumer->epoch = 0; pConsumer->epoch = 0;
pConsumer->status = MQ_CONSUMER_STATUS_REBALANCE; pConsumer->status = MQ_CONSUMER_STATUS_REBALANCE;
@ -276,20 +287,26 @@ SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t upda
if (updateType == CONSUMER_ADD_REB){ if (updateType == CONSUMER_ADD_REB){
pConsumer->rebNewTopics = taosArrayInit(0, sizeof(void *)); pConsumer->rebNewTopics = taosArrayInit(0, sizeof(void *));
if(pConsumer->rebNewTopics == NULL){ if(pConsumer->rebNewTopics == NULL){
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto END; goto END;
} }
char* topicTmp = taosStrdup(topic); char* topicTmp = taosStrdup(topic);
taosArrayPush(pConsumer->rebNewTopics, &topicTmp); if (taosArrayPush(pConsumer->rebNewTopics, &topicTmp) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
}else if (updateType == CONSUMER_REMOVE_REB) { }else if (updateType == CONSUMER_REMOVE_REB) {
pConsumer->rebRemovedTopics = taosArrayInit(0, sizeof(void *)); pConsumer->rebRemovedTopics = taosArrayInit(0, sizeof(void *));
if(pConsumer->rebRemovedTopics == NULL){ if(pConsumer->rebRemovedTopics == NULL){
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto END; goto END;
} }
char* topicTmp = taosStrdup(topic); char* topicTmp = taosStrdup(topic);
taosArrayPush(pConsumer->rebRemovedTopics, &topicTmp); if (taosArrayPush(pConsumer->rebRemovedTopics, &topicTmp) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
}else if (updateType == CONSUMER_INSERT_SUB){ }else if (updateType == CONSUMER_INSERT_SUB){
tstrncpy(pConsumer->clientId, subscribe->clientId, tListLen(pConsumer->clientId)); tstrncpy(pConsumer->clientId, subscribe->clientId, tListLen(pConsumer->clientId));
pConsumer->withTbName = subscribe->withTbName; pConsumer->withTbName = subscribe->withTbName;
@ -297,10 +314,9 @@ SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t upda
pConsumer->autoCommitInterval = subscribe->autoCommitInterval; pConsumer->autoCommitInterval = subscribe->autoCommitInterval;
pConsumer->resetOffsetCfg = subscribe->resetOffsetCfg; pConsumer->resetOffsetCfg = subscribe->resetOffsetCfg;
pConsumer->rebNewTopics = taosArrayDup(subscribe->topicNames, topicNameDup); pConsumer->rebNewTopics = taosArrayDup(subscribe->topicNames, topicNameDup);
if (pConsumer->rebNewTopics == NULL){ if (pConsumer->rebNewTopics == NULL){
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto END; goto END;
} }
pConsumer->assignedTopics = subscribe->topicNames; pConsumer->assignedTopics = subscribe->topicNames;
@ -310,11 +326,12 @@ SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t upda
subscribe->topicNames = NULL; subscribe->topicNames = NULL;
} }
return pConsumer; *ppConsumer = pConsumer;
return 0;
END: END:
tDeleteSMqConsumerObj(pConsumer); tDeleteSMqConsumerObj(pConsumer);
return NULL; return code;
} }
void tClearSMqConsumerObj(SMqConsumerObj *pConsumer) { void tClearSMqConsumerObj(SMqConsumerObj *pConsumer) {
@ -530,27 +547,36 @@ void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp, int8_t s
return (void *)buf; return (void *)buf;
} }
SMqSubscribeObj *tNewSubscribeObj(const char *key) { int32_t tNewSubscribeObj(const char *key, SMqSubscribeObj **ppSub) {
int32_t code = 0;
SMqSubscribeObj *pSubObj = taosMemoryCalloc(1, sizeof(SMqSubscribeObj)); SMqSubscribeObj *pSubObj = taosMemoryCalloc(1, sizeof(SMqSubscribeObj));
if (pSubObj == NULL) { MND_TMQ_NULL_CHECK(pSubObj);
return NULL;
}
memcpy(pSubObj->key, key, TSDB_SUBSCRIBE_KEY_LEN); (void)memcpy(pSubObj->key, key, TSDB_SUBSCRIBE_KEY_LEN);
taosInitRWLatch(&pSubObj->lock); taosInitRWLatch(&pSubObj->lock);
pSubObj->vgNum = 0; pSubObj->vgNum = 0;
pSubObj->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); pSubObj->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
MND_TMQ_NULL_CHECK(pSubObj->consumerHash);
// TODO set hash free fp
/*taosHashSetFreeFp(pSubObj->consumerHash, tDeleteSMqConsumerEp);*/
pSubObj->unassignedVgs = taosArrayInit(0, POINTER_BYTES); pSubObj->unassignedVgs = taosArrayInit(0, POINTER_BYTES);
return pSubObj; MND_TMQ_NULL_CHECK(pSubObj->unassignedVgs);
if (ppSub){
*ppSub = pSubObj;
}
return code;
END:
taosMemoryFree(pSubObj);
return code;
} }
SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { int32_t tCloneSubscribeObj(const SMqSubscribeObj *pSub, SMqSubscribeObj **ppSub) {
int32_t code = 0;
SMqSubscribeObj *pSubNew = taosMemoryMalloc(sizeof(SMqSubscribeObj)); SMqSubscribeObj *pSubNew = taosMemoryMalloc(sizeof(SMqSubscribeObj));
if (pSubNew == NULL) return NULL; if (pSubNew == NULL) {
memcpy(pSubNew->key, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); code = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
(void)memcpy(pSubNew->key, pSub->key, TSDB_SUBSCRIBE_KEY_LEN);
taosInitRWLatch(&pSubNew->lock); taosInitRWLatch(&pSubNew->lock);
pSubNew->dbUid = pSub->dbUid; pSubNew->dbUid = pSub->dbUid;
@ -560,8 +586,7 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) {
pSubNew->vgNum = pSub->vgNum; pSubNew->vgNum = pSub->vgNum;
pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
// TODO set hash free fp
/*taosHashSetFreeFp(pSubNew->consumerHash, tDeleteSMqConsumerEp);*/
void *pIter = NULL; void *pIter = NULL;
SMqConsumerEp *pConsumerEp = NULL; SMqConsumerEp *pConsumerEp = NULL;
while (1) { while (1) {
@ -576,9 +601,13 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) {
} }
pSubNew->unassignedVgs = taosArrayDup(pSub->unassignedVgs, (__array_item_dup_fn_t)tCloneSMqVgEp); pSubNew->unassignedVgs = taosArrayDup(pSub->unassignedVgs, (__array_item_dup_fn_t)tCloneSMqVgEp);
pSubNew->offsetRows = taosArrayDup(pSub->offsetRows, NULL); pSubNew->offsetRows = taosArrayDup(pSub->offsetRows, NULL);
memcpy(pSubNew->dbName, pSub->dbName, TSDB_DB_FNAME_LEN); (void)memcpy(pSubNew->dbName, pSub->dbName, TSDB_DB_FNAME_LEN);
pSubNew->qmsg = taosStrdup(pSub->qmsg); pSubNew->qmsg = taosStrdup(pSub->qmsg);
return pSubNew; if (ppSub) {
*ppSub = pSubNew;
}
END:
return code;
} }
void tDeleteSubscribeObj(SMqSubscribeObj *pSub) { void tDeleteSubscribeObj(SMqSubscribeObj *pSub) {

View File

@ -165,15 +165,24 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) {
} }
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL, "create-dnode"); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL, "create-dnode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mInfo("trans:%d, used to create dnode:%s on first deploy", pTrans->id, dnodeObj.ep); mInfo("trans:%d, used to create dnode:%s on first deploy", pTrans->id, dnodeObj.ep);
pRaw = mndDnodeActionEncode(&dnodeObj); pRaw = mndDnodeActionEncode(&dnodeObj);
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; if (pRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pRaw), NULL, _OVER);
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
pRaw = NULL; pRaw = NULL;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1); mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1);
@ -563,10 +572,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
SStatisReq statisReq = {0}; SStatisReq statisReq = {0};
int32_t code = -1; int32_t code = -1;
if (tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq) != 0) { TAOS_CHECK_RETURN(tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq));
terrno = TSDB_CODE_INVALID_MSG;
return code;
}
if (tsMonitorLogProtocol) { if (tsMonitorLogProtocol) {
mInfo("process statis req,\n %s", statisReq.pCont); mInfo("process statis req,\n %s", statisReq.pCont);
@ -586,23 +592,28 @@ static int32_t mndUpdateDnodeObj(SMnode *pMnode, SDnodeObj *pDnode) {
int32_t code = 0; int32_t code = 0;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "update-dnode-obj"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "update-dnode-obj");
if (pTrans == NULL) { if (pTrans == NULL) {
code = terrno; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _exit; goto _exit;
} }
pDnode->updateTime = taosGetTimestampMs(); pDnode->updateTime = taosGetTimestampMs();
SSdbRaw *pCommitRaw = mndDnodeActionEncode(pDnode); SSdbRaw *pCommitRaw = mndDnodeActionEncode(pDnode);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { if (pCommitRaw == NULL) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _exit;
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, tstrerror(code));
code = terrno; code = terrno;
goto _exit; goto _exit;
} }
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) { if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, tstrerror(code));
code = terrno;
goto _exit; goto _exit;
} }
@ -617,10 +628,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
SDnodeObj *pDnode = NULL; SDnodeObj *pDnode = NULL;
int32_t code = -1; int32_t code = -1;
if (tDeserializeSStatusReq(pReq->pCont, pReq->contLen, &statusReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSStatusReq(pReq->pCont, pReq->contLen, &statusReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
int64_t clusterid = mndGetClusterId(pMnode); int64_t clusterid = mndGetClusterId(pMnode);
if (statusReq.clusterId != 0 && statusReq.clusterId != clusterid) { if (statusReq.clusterId != 0 && statusReq.clusterId != clusterid) {
@ -634,6 +642,8 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
pDnode = mndAcquireDnodeByEp(pMnode, statusReq.dnodeEp); pDnode = mndAcquireDnodeByEp(pMnode, statusReq.dnodeEp);
if (pDnode == NULL) { if (pDnode == NULL) {
mInfo("dnode:%s, not created yet", statusReq.dnodeEp); mInfo("dnode:%s, not created yet", statusReq.dnodeEp);
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER; goto _OVER;
} }
} else { } else {
@ -871,16 +881,25 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC
snprintf(dnodeObj.ep, TSDB_EP_LEN - 1, "%s:%u", pCreate->fqdn, pCreate->port); snprintf(dnodeObj.ep, TSDB_EP_LEN - 1, "%s:%u", pCreate->fqdn, pCreate->port);
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq, "create-dnode"); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq, "create-dnode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mInfo("trans:%d, used to create dnode:%s", pTrans->id, dnodeObj.ep); mInfo("trans:%d, used to create dnode:%s", pTrans->id, dnodeObj.ep);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
pRaw = mndDnodeActionEncode(&dnodeObj); pRaw = mndDnodeActionEncode(&dnodeObj);
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; if (pRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pRaw), NULL, _OVER);
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
pRaw = NULL; pRaw = NULL;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1); mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1);
@ -901,7 +920,7 @@ static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) {
rsp.dnodeList = taosArrayInit(5, sizeof(SEpSet)); rsp.dnodeList = taosArrayInit(5, sizeof(SEpSet));
if (NULL == rsp.dnodeList) { if (NULL == rsp.dnodeList) {
mError("failed to alloc epSet while process dnode list req"); mError("failed to alloc epSet while process dnode list req");
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
@ -922,7 +941,7 @@ static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) {
int32_t rspLen = tSerializeSDnodeListRsp(NULL, 0, &rsp); int32_t rspLen = tSerializeSDnodeListRsp(NULL, 0, &rsp);
void *pRsp = rpcMallocCont(rspLen); void *pRsp = rpcMallocCont(rspLen);
if (pRsp == NULL) { if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
@ -935,12 +954,12 @@ static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0) { if (code != 0) {
mError("failed to get dnode list since %s", terrstr()); mError("failed to get dnode list since %s", tstrerror(code));
} }
tFreeSDnodeListRsp(&rsp); tFreeSDnodeListRsp(&rsp);
return code; TAOS_RETURN(code);
} }
static void getSlowLogScopeString(int32_t scope, char* result){ static void getSlowLogScopeString(int32_t scope, char* result){
@ -980,7 +999,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) {
rsp.variables = taosArrayInit(16, sizeof(SVariablesInfo)); rsp.variables = taosArrayInit(16, sizeof(SVariablesInfo));
if (NULL == rsp.variables) { if (NULL == rsp.variables) {
mError("failed to alloc SVariablesInfo array while process show variables req"); mError("failed to alloc SVariablesInfo array while process show variables req");
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
@ -1036,7 +1055,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) {
int32_t rspLen = tSerializeSShowVariablesRsp(NULL, 0, &rsp); int32_t rspLen = tSerializeSShowVariablesRsp(NULL, 0, &rsp);
void *pRsp = rpcMallocCont(rspLen); void *pRsp = rpcMallocCont(rspLen);
if (pRsp == NULL) { if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
@ -1049,11 +1068,11 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0) { if (code != 0) {
mError("failed to get show variables info since %s", terrstr()); mError("failed to get show variables info since %s", tstrerror(code));
} }
tFreeSShowVariablesRsp(&rsp); tFreeSShowVariablesRsp(&rsp);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
@ -1062,23 +1081,17 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
SDnodeObj *pDnode = NULL; SDnodeObj *pDnode = NULL;
SCreateDnodeReq createReq = {0}; SCreateDnodeReq createReq = {0};
if ((terrno = grantCheck(TSDB_GRANT_DNODE)) != 0 || (terrno = grantCheck(TSDB_GRANT_CPU_CORES)) != 0) { if ((code = grantCheck(TSDB_GRANT_DNODE)) != 0 || (code = grantCheck(TSDB_GRANT_CPU_CORES)) != 0) {
code = terrno;
goto _OVER; goto _OVER;
} }
if (tDeserializeSCreateDnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSCreateDnodeReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("dnode:%s:%d, start to create", createReq.fqdn, createReq.port); mInfo("dnode:%s:%d, start to create", createReq.fqdn, createReq.port);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE), NULL, _OVER);
goto _OVER;
}
if (createReq.fqdn[0] == 0 || createReq.port <= 0 || createReq.port > UINT16_MAX) { if (createReq.fqdn[0] == 0 || createReq.port <= 0 || createReq.port > UINT16_MAX) {
terrno = TSDB_CODE_MND_INVALID_DNODE_EP; code = TSDB_CODE_MND_INVALID_DNODE_EP;
goto _OVER; goto _OVER;
} }
@ -1086,7 +1099,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
snprintf(ep, TSDB_EP_LEN, "%s:%d", createReq.fqdn, createReq.port); snprintf(ep, TSDB_EP_LEN, "%s:%d", createReq.fqdn, createReq.port);
pDnode = mndAcquireDnodeByEp(pMnode, ep); pDnode = mndAcquireDnodeByEp(pMnode, ep);
if (pDnode != NULL) { if (pDnode != NULL) {
terrno = TSDB_CODE_MND_DNODE_ALREADY_EXIST; code = TSDB_CODE_MND_DNODE_ALREADY_EXIST;
goto _OVER; goto _OVER;
} }
@ -1103,12 +1116,12 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("dnode:%s:%d, failed to create since %s", createReq.fqdn, createReq.port, terrstr()); mError("dnode:%s:%d, failed to create since %s", createReq.fqdn, createReq.port, tstrerror(code));
} }
mndReleaseDnode(pMnode, pDnode); mndReleaseDnode(pMnode, pDnode);
tFreeSCreateDnodeReq(&createReq); tFreeSCreateDnodeReq(&createReq);
return code; TAOS_RETURN(code);
} }
extern int32_t mndProcessRestoreDnodeReqImpl(SRpcMsg *pReq); extern int32_t mndProcessRestoreDnodeReqImpl(SRpcMsg *pReq);
@ -1126,44 +1139,56 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM
STrans *pTrans = NULL; STrans *pTrans = NULL;
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "drop-dnode"); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "drop-dnode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans); mndTransSetSerial(pTrans);
mInfo("trans:%d, used to drop dnode:%d, force:%d", pTrans->id, pDnode->id, force); mInfo("trans:%d, used to drop dnode:%d, force:%d", pTrans->id, pDnode->id, force);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
pRaw = mndDnodeActionEncode(pDnode); pRaw = mndDnodeActionEncode(pDnode);
if (pRaw == NULL) goto _OVER; if (pRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendRedolog(pTrans, pRaw), NULL, _OVER);
(void)sdbSetRawStatus(pRaw, SDB_STATUS_DROPPING); (void)sdbSetRawStatus(pRaw, SDB_STATUS_DROPPING);
pRaw = NULL; pRaw = NULL;
pRaw = mndDnodeActionEncode(pDnode); pRaw = mndDnodeActionEncode(pDnode);
if (pRaw == NULL) goto _OVER; if (pRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pRaw), NULL, _OVER);
(void)sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); (void)sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED);
pRaw = NULL; pRaw = NULL;
if (pMObj != NULL) { if (pMObj != NULL) {
mInfo("trans:%d, mnode on dnode:%d will be dropped", pTrans->id, pDnode->id); mInfo("trans:%d, mnode on dnode:%d will be dropped", pTrans->id, pDnode->id);
if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pMObj, force) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetDropMnodeInfoToTrans(pMnode, pTrans, pMObj, force), NULL, _OVER);
} }
if (pQObj != NULL) { if (pQObj != NULL) {
mInfo("trans:%d, qnode on dnode:%d will be dropped", pTrans->id, pDnode->id); mInfo("trans:%d, qnode on dnode:%d will be dropped", pTrans->id, pDnode->id);
if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pQObj, force) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetDropQnodeInfoToTrans(pMnode, pTrans, pQObj, force), NULL, _OVER);
} }
if (pSObj != NULL) { if (pSObj != NULL) {
mInfo("trans:%d, snode on dnode:%d will be dropped", pTrans->id, pDnode->id); mInfo("trans:%d, snode on dnode:%d will be dropped", pTrans->id, pDnode->id);
if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pSObj, force) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetDropSnodeInfoToTrans(pMnode, pTrans, pSObj, force), NULL, _OVER);
} }
if (numOfVnodes > 0) { if (numOfVnodes > 0) {
mInfo("trans:%d, %d vnodes on dnode:%d will be dropped", pTrans->id, numOfVnodes, pDnode->id); mInfo("trans:%d, %d vnodes on dnode:%d will be dropped", pTrans->id, numOfVnodes, pDnode->id);
if (mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id, force, unsafe) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id, force, unsafe), NULL, _OVER);
} }
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, pDnode->fqdn, IP_WHITE_DROP, 1); mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, pDnode->fqdn, IP_WHITE_DROP, 1);
code = 0; code = 0;
@ -1171,7 +1196,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM
_OVER: _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
sdbFreeRaw(pRaw); sdbFreeRaw(pRaw);
return code; TAOS_RETURN(code);
} }
static bool mndIsEmptyDnode(SMnode *pMnode, int32_t dnodeId) { static bool mndIsEmptyDnode(SMnode *pMnode, int32_t dnodeId) {
@ -1209,16 +1234,11 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
SSnodeObj *pSObj = NULL; SSnodeObj *pSObj = NULL;
SDropDnodeReq dropReq = {0}; SDropDnodeReq dropReq = {0};
if (tDeserializeSDropDnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSDropDnodeReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("dnode:%d, start to drop, ep:%s:%d, force:%s, unsafe:%s", dropReq.dnodeId, dropReq.fqdn, dropReq.port, mInfo("dnode:%d, start to drop, ep:%s:%d, force:%s, unsafe:%s", dropReq.dnodeId, dropReq.fqdn, dropReq.port,
dropReq.force ? "true" : "false", dropReq.unsafe ? "true" : "false"); dropReq.force ? "true" : "false", dropReq.unsafe ? "true" : "false");
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE), NULL, _OVER);
goto _OVER;
}
bool force = dropReq.force; bool force = dropReq.force;
if (dropReq.unsafe) { if (dropReq.unsafe) {
@ -1232,7 +1252,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
snprintf(ep, sizeof(ep), dropReq.fqdn, dropReq.port); snprintf(ep, sizeof(ep), dropReq.fqdn, dropReq.port);
pDnode = mndAcquireDnodeByEp(pMnode, ep); pDnode = mndAcquireDnodeByEp(pMnode, ep);
if (pDnode == NULL) { if (pDnode == NULL) {
terrno = err; code = err;
goto _OVER; goto _OVER;
} }
} }
@ -1242,11 +1262,11 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
pMObj = mndAcquireMnode(pMnode, dropReq.dnodeId); pMObj = mndAcquireMnode(pMnode, dropReq.dnodeId);
if (pMObj != NULL) { if (pMObj != NULL) {
if (sdbGetSize(pMnode->pSdb, SDB_MNODE) <= 1) { if (sdbGetSize(pMnode->pSdb, SDB_MNODE) <= 1) {
terrno = TSDB_CODE_MND_TOO_FEW_MNODES; code = TSDB_CODE_MND_TOO_FEW_MNODES;
goto _OVER; goto _OVER;
} }
if (pMnode->selfDnodeId == dropReq.dnodeId) { if (pMnode->selfDnodeId == dropReq.dnodeId) {
terrno = TSDB_CODE_MND_CANT_DROP_LEADER; code = TSDB_CODE_MND_CANT_DROP_LEADER;
goto _OVER; goto _OVER;
} }
} }
@ -1255,16 +1275,16 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
bool isonline = mndIsDnodeOnline(pDnode, taosGetTimestampMs()); bool isonline = mndIsDnodeOnline(pDnode, taosGetTimestampMs());
if (isonline && force) { if (isonline && force) {
terrno = TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE; code = TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE;
mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, tstrerror(code),
numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL);
goto _OVER; goto _OVER;
} }
bool isEmpty = mndIsEmptyDnode(pMnode, pDnode->id); bool isEmpty = mndIsEmptyDnode(pMnode, pDnode->id);
if (!isonline && !force && !isEmpty) { if (!isonline && !force && !isEmpty) {
terrno = TSDB_CODE_DNODE_OFFLINE; code = TSDB_CODE_DNODE_OFFLINE;
mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, tstrerror(code),
numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL);
goto _OVER; goto _OVER;
} }
@ -1279,7 +1299,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("dnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); mError("dnode:%d, failed to drop since %s", dropReq.dnodeId, tstrerror(code));
} }
mndReleaseDnode(pMnode, pDnode); mndReleaseDnode(pMnode, pDnode);
@ -1287,11 +1307,11 @@ _OVER:
mndReleaseQnode(pMnode, pQObj); mndReleaseQnode(pMnode, pQObj);
mndReleaseSnode(pMnode, pSObj); mndReleaseSnode(pMnode, pSObj);
tFreeSDropDnodeReq(&dropReq); tFreeSDropDnodeReq(&dropReq);
return code; TAOS_RETURN(code);
} }
static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) { static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) {
terrno = 0; int32_t code = 0;
char *p = pMCfgReq->config; char *p = pMCfgReq->config;
while (*p) { while (*p) {
if (*p == ' ') { if (*p == ' ') {
@ -1314,12 +1334,12 @@ static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) {
strcpy(pDCfgReq->value, pMCfgReq->value); strcpy(pDCfgReq->value, pMCfgReq->value);
} }
return 0; TAOS_RETURN(code);
_err: _err:
mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config); mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config);
terrno = TSDB_CODE_INVALID_CFG; code = TSDB_CODE_INVALID_CFG;
return -1; TAOS_RETURN(code);
} }
static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq *pDcfgReq) { static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq *pDcfgReq) {
@ -1349,23 +1369,21 @@ static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq
} }
if (code == -1) { if (code == -1) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; code = TSDB_CODE_MND_DNODE_NOT_EXIST;
} }
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SMCfgDnodeReq cfgReq = {0}; SMCfgDnodeReq cfgReq = {0};
if (tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) { TAOS_CHECK_RETURN(tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq));
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
int8_t updateIpWhiteList = 0; int8_t updateIpWhiteList = 0;
mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) { if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE)) != 0) {
tFreeSMCfgDnodeReq(&cfgReq); tFreeSMCfgDnodeReq(&cfgReq);
return -1; TAOS_RETURN(code);
} }
SDCfgDnodeReq dcfgReq = {0}; SDCfgDnodeReq dcfgReq = {0};
@ -1381,26 +1399,26 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
if (flag > 1024 * 1024 || (flag > -1 && flag < 1024) || flag < -1) { if (flag > 1024 * 1024 || (flag > -1 && flag < 1024) || flag < -1) {
mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: -1 or [1024, 1024 * 1024]", mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: -1 or [1024, 1024 * 1024]",
cfgReq.dnodeId, flag); cfgReq.dnodeId, flag);
terrno = TSDB_CODE_INVALID_CFG; code = TSDB_CODE_INVALID_CFG;
tFreeSMCfgDnodeReq(&cfgReq); tFreeSMCfgDnodeReq(&cfgReq);
return -1; TAOS_RETURN(code);
} }
strcpy(dcfgReq.config, "s3blocksize"); strcpy(dcfgReq.config, "s3blocksize");
snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
#endif #endif
} else { } else {
if (mndMCfg2DCfg(&cfgReq, &dcfgReq)) goto _err_out; TAOS_CHECK_GOTO (mndMCfg2DCfg(&cfgReq, &dcfgReq), NULL, _err_out);
if (strlen(dcfgReq.config) > TSDB_DNODE_CONFIG_LEN) { if (strlen(dcfgReq.config) > TSDB_DNODE_CONFIG_LEN) {
mError("dnode:%d, failed to config since config is too long", cfgReq.dnodeId); mError("dnode:%d, failed to config since config is too long", cfgReq.dnodeId);
terrno = TSDB_CODE_INVALID_CFG; code = TSDB_CODE_INVALID_CFG;
goto _err_out; goto _err_out;
} }
if (strncasecmp(dcfgReq.config, "enableWhiteList", strlen("enableWhiteList")) == 0) { if (strncasecmp(dcfgReq.config, "enableWhiteList", strlen("enableWhiteList")) == 0) {
updateIpWhiteList = 1; updateIpWhiteList = 1;
} }
if (cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true) != 0) goto _err_out; TAOS_CHECK_GOTO(cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true), NULL, _err_out);
} }
{ // audit { // audit
@ -1412,15 +1430,15 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
tFreeSMCfgDnodeReq(&cfgReq); tFreeSMCfgDnodeReq(&cfgReq);
int32_t code = mndSendCfgDnodeReq(pMnode, cfgReq.dnodeId, &dcfgReq); code = mndSendCfgDnodeReq(pMnode, cfgReq.dnodeId, &dcfgReq);
// dont care suss or succ; // dont care suss or succ;
if (updateIpWhiteList) mndRefreshUserIpWhiteList(pMnode); if (updateIpWhiteList) mndRefreshUserIpWhiteList(pMnode);
return code; TAOS_RETURN(code);
_err_out: _err_out:
tFreeSMCfgDnodeReq(&cfgReq); tFreeSMCfgDnodeReq(&cfgReq);
return -1; TAOS_RETURN(code);
} }
static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp) { static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp) {
@ -1503,17 +1521,16 @@ _exit:
} }
static int32_t mndProcessCreateEncryptKeyReq(SRpcMsg *pReq) { static int32_t mndProcessCreateEncryptKeyReq(SRpcMsg *pReq) {
int32_t code = 0;
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SMCfgDnodeReq cfgReq = {0}; SMCfgDnodeReq cfgReq = {0};
if (tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) { TAOS_CHECK_RETURN(tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq));
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) { if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE)) != 0) {
tFreeSMCfgDnodeReq(&cfgReq); tFreeSMCfgDnodeReq(&cfgReq);
return -1; TAOS_RETURN(code);
} }
const STraceId *trace = &pReq->info.traceId; const STraceId *trace = &pReq->info.traceId;
SDCfgDnodeReq dcfgReq = {0}; SDCfgDnodeReq dcfgReq = {0};
@ -1523,13 +1540,13 @@ static int32_t mndProcessCreateEncryptKeyReq(SRpcMsg *pReq) {
tFreeSMCfgDnodeReq(&cfgReq); tFreeSMCfgDnodeReq(&cfgReq);
return mndProcessCreateEncryptKeyReqImpl(pReq, cfgReq.dnodeId, &dcfgReq); return mndProcessCreateEncryptKeyReqImpl(pReq, cfgReq.dnodeId, &dcfgReq);
} else { } else {
terrno = TSDB_CODE_PAR_INTERNAL_ERROR; code = TSDB_CODE_PAR_INTERNAL_ERROR;
tFreeSMCfgDnodeReq(&cfgReq); tFreeSMCfgDnodeReq(&cfgReq);
return -1; TAOS_RETURN(code);
} }
#else #else
return 0; TAOS_RETURN(code);
#endif #endif
} }
@ -1710,7 +1727,7 @@ static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter) {
// get int32_t value from 'SMCfgDnodeReq' // get int32_t value from 'SMCfgDnodeReq'
static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t optLen, int32_t *pOutValue) { static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t optLen, int32_t *pOutValue) {
terrno = 0; int32_t code = 0;
if (' ' != pMCfgReq->config[optLen] && 0 != pMCfgReq->config[optLen]) { if (' ' != pMCfgReq->config[optLen] && 0 != pMCfgReq->config[optLen]) {
goto _err; goto _err;
} }
@ -1725,12 +1742,12 @@ static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t optLen, int32
*pOutValue = atoi(pMCfgReq->value); *pOutValue = atoi(pMCfgReq->value);
} }
return 0; TAOS_RETURN(code);
_err: _err:
mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config); mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config);
terrno = TSDB_CODE_INVALID_CFG; code = TSDB_CODE_INVALID_CFG;
return -1; TAOS_RETURN(code);
} }
SArray *mndGetAllDnodeFqdns(SMnode *pMnode) { SArray *mndGetAllDnodeFqdns(SMnode *pMnode) {

View File

@ -29,12 +29,14 @@ void reportStartup(const char *name, const char *desc) {}
void sendRsp(SRpcMsg *pMsg) { rpcFreeCont(pMsg->pCont); } void sendRsp(SRpcMsg *pMsg) { rpcFreeCont(pMsg->pCont); }
int32_t sendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { int32_t sendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
terrno = TSDB_CODE_INVALID_PTR; int32_t code = 0;
return -1; code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
} }
int32_t sendSyncReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { int32_t sendSyncReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
terrno = TSDB_CODE_INVALID_PTR; int32_t code = 0;
return -1; code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
} }
char *i642str(int64_t val) { char *i642str(int64_t val) {

View File

@ -223,6 +223,7 @@ static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew) {
} }
static SFuncObj *mndAcquireFunc(SMnode *pMnode, char *funcName) { static SFuncObj *mndAcquireFunc(SMnode *pMnode, char *funcName) {
terrno = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SFuncObj *pFunc = sdbAcquire(pSdb, SDB_FUNC, funcName); SFuncObj *pFunc = sdbAcquire(pSdb, SDB_FUNC, funcName);
if (pFunc == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { if (pFunc == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
@ -240,7 +241,7 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre
int32_t code = -1; int32_t code = -1;
STrans *pTrans = NULL; STrans *pTrans = NULL;
if ((terrno = grantCheck(TSDB_GRANT_USER)) < 0) { if ((code = grantCheck(TSDB_GRANT_USER)) < 0) {
return code; return code;
} }
@ -260,7 +261,7 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre
func.codeSize = pCreate->codeLen; func.codeSize = pCreate->codeLen;
func.pCode = taosMemoryMalloc(func.codeSize); func.pCode = taosMemoryMalloc(func.codeSize);
if (func.pCode == NULL || func.pCode == NULL) { if (func.pCode == NULL || func.pCode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
@ -270,7 +271,11 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre
memcpy(func.pCode, pCreate->pCode, func.codeSize); memcpy(func.pCode, pCreate->pCode, func.codeSize);
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-func"); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-func");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mInfo("trans:%d, used to create func:%s", pTrans->id, pCreate->name); mInfo("trans:%d, used to create func:%s", pTrans->id, pCreate->name);
SFuncObj *oldFunc = mndAcquireFunc(pMnode, pCreate->name); SFuncObj *oldFunc = mndAcquireFunc(pMnode, pCreate->name);
@ -279,31 +284,61 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre
func.createdTime = oldFunc->createdTime; func.createdTime = oldFunc->createdTime;
SSdbRaw *pRedoRaw = mndFuncActionEncode(oldFunc); SSdbRaw *pRedoRaw = mndFuncActionEncode(oldFunc);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; if (pRedoRaw == NULL) {
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendRedolog(pTrans, pRedoRaw), NULL, _OVER);
TAOS_CHECK_GOTO(sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY), NULL, _OVER);
SSdbRaw *pUndoRaw = mndFuncActionEncode(oldFunc); SSdbRaw *pUndoRaw = mndFuncActionEncode(oldFunc);
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; if (pUndoRaw == NULL) {
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendUndolog(pTrans, pUndoRaw), NULL, _OVER);
TAOS_CHECK_GOTO(sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY), NULL, _OVER);
SSdbRaw *pCommitRaw = mndFuncActionEncode(&func); SSdbRaw *pCommitRaw = mndFuncActionEncode(&func);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; if (pCommitRaw == NULL) {
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pCommitRaw), NULL, _OVER);
TAOS_CHECK_GOTO(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY), NULL, _OVER);
} else { } else {
SSdbRaw *pRedoRaw = mndFuncActionEncode(&func); SSdbRaw *pRedoRaw = mndFuncActionEncode(&func);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; if (pRedoRaw == NULL) {
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendRedolog(pTrans, pRedoRaw), NULL, _OVER);
TAOS_CHECK_GOTO(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING), NULL, _OVER);
SSdbRaw *pUndoRaw = mndFuncActionEncode(&func); SSdbRaw *pUndoRaw = mndFuncActionEncode(&func);
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; if (pUndoRaw == NULL) {
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendUndolog(pTrans, pUndoRaw), NULL, _OVER);
TAOS_CHECK_GOTO(sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED), NULL, _OVER);
SSdbRaw *pCommitRaw = mndFuncActionEncode(&func); SSdbRaw *pCommitRaw = mndFuncActionEncode(&func);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; if (pCommitRaw == NULL) {
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pCommitRaw), NULL, _OVER);
TAOS_CHECK_GOTO(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY), NULL, _OVER);
} }
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
@ -315,32 +350,48 @@ _OVER:
taosMemoryFree(func.pCode); taosMemoryFree(func.pCode);
taosMemoryFree(func.pComment); taosMemoryFree(func.pComment);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t mndDropFunc(SMnode *pMnode, SRpcMsg *pReq, SFuncObj *pFunc) { static int32_t mndDropFunc(SMnode *pMnode, SRpcMsg *pReq, SFuncObj *pFunc) {
int32_t code = -1; int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "drop-func"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "drop-func");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mInfo("trans:%d, used to drop user:%s", pTrans->id, pFunc->name); mInfo("trans:%d, used to drop user:%s", pTrans->id, pFunc->name);
SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc); SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc);
if (pRedoRaw == NULL) goto _OVER; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendRedolog(pTrans, pRedoRaw), NULL, _OVER);
(void)sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); (void)sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING);
SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc); SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc);
if (pUndoRaw == NULL) goto _OVER; if (pUndoRaw == NULL) {
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendUndolog(pTrans, pUndoRaw), NULL, _OVER);
(void)sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY);
SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc); SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc);
if (pCommitRaw == NULL) goto _OVER; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pCommitRaw), NULL, _OVER);
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
@ -355,18 +406,14 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) {
SFuncObj *pFunc = NULL; SFuncObj *pFunc = NULL;
SCreateFuncReq createReq = {0}; SCreateFuncReq createReq = {0};
if (tDeserializeSCreateFuncReq(pReq->pCont, pReq->contLen, &createReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSCreateFuncReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
#ifdef WINDOWS #ifdef WINDOWS
terrno = TSDB_CODE_MND_INVALID_PLATFORM; code = TSDB_CODE_MND_INVALID_PLATFORM;
goto _OVER; goto _OVER;
#endif #endif
mInfo("func:%s, start to create, size:%d", createReq.name, createReq.codeLen); mInfo("func:%s, start to create, size:%d", createReq.name, createReq.codeLen);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC), NULL, _OVER);
goto _OVER;
}
pFunc = mndAcquireFunc(pMnode, createReq.name); pFunc = mndAcquireFunc(pMnode, createReq.name);
if (pFunc != NULL) { if (pFunc != NULL) {
@ -378,7 +425,7 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) {
mInfo("func:%s, replace function is set", createReq.name); mInfo("func:%s, replace function is set", createReq.name);
code = 0; code = 0;
} else { } else {
terrno = TSDB_CODE_MND_FUNC_ALREADY_EXIST; code = TSDB_CODE_MND_FUNC_ALREADY_EXIST;
goto _OVER; goto _OVER;
} }
} else if (terrno == TSDB_CODE_MND_FUNC_ALREADY_EXIST) { } else if (terrno == TSDB_CODE_MND_FUNC_ALREADY_EXIST) {
@ -386,22 +433,22 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) {
} }
if (createReq.name[0] == 0) { if (createReq.name[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_FUNC_NAME; code = TSDB_CODE_MND_INVALID_FUNC_NAME;
goto _OVER; goto _OVER;
} }
if (createReq.pCode == NULL) { if (createReq.pCode == NULL) {
terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; code = TSDB_CODE_MND_INVALID_FUNC_CODE;
goto _OVER; goto _OVER;
} }
if (createReq.codeLen <= 1) { if (createReq.codeLen <= 1) {
terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; code = TSDB_CODE_MND_INVALID_FUNC_CODE;
goto _OVER; goto _OVER;
} }
if (createReq.bufSize < 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) { if (createReq.bufSize < 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) {
terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; code = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE;
goto _OVER; goto _OVER;
} }
@ -410,12 +457,12 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("func:%s, failed to create since %s", createReq.name, terrstr()); mError("func:%s, failed to create since %s", createReq.name, tstrerror(code));
} }
mndReleaseFunc(pMnode, pFunc); mndReleaseFunc(pMnode, pFunc);
tFreeSCreateFuncReq(&createReq); tFreeSCreateFuncReq(&createReq);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) {
@ -424,18 +471,13 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) {
SFuncObj *pFunc = NULL; SFuncObj *pFunc = NULL;
SDropFuncReq dropReq = {0}; SDropFuncReq dropReq = {0};
if (tDeserializeSDropFuncReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSDropFuncReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("func:%s, start to drop", dropReq.name); mInfo("func:%s, start to drop", dropReq.name);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC), NULL, _OVER);
goto _OVER;
}
if (dropReq.name[0] == 0) { if (dropReq.name[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_FUNC_NAME; code = TSDB_CODE_MND_INVALID_FUNC_NAME;
goto _OVER; goto _OVER;
} }
@ -446,7 +488,7 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) {
code = 0; code = 0;
goto _OVER; goto _OVER;
} else { } else {
terrno = TSDB_CODE_MND_FUNC_NOT_EXIST; code = TSDB_CODE_MND_FUNC_NOT_EXIST;
goto _OVER; goto _OVER;
} }
} }
@ -456,11 +498,11 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("func:%s, failed to drop since %s", dropReq.name, terrstr()); mError("func:%s, failed to drop since %s", dropReq.name, tstrerror(code));
} }
mndReleaseFunc(pMnode, pFunc); mndReleaseFunc(pMnode, pFunc);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) { static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) {
@ -470,25 +512,25 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) {
SRetrieveFuncRsp retrieveRsp = {0}; SRetrieveFuncRsp retrieveRsp = {0};
if (tDeserializeSRetrieveFuncReq(pReq->pCont, pReq->contLen, &retrieveReq) != 0) { if (tDeserializeSRetrieveFuncReq(pReq->pCont, pReq->contLen, &retrieveReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
goto RETRIEVE_FUNC_OVER; goto RETRIEVE_FUNC_OVER;
} }
if (retrieveReq.numOfFuncs <= 0 || retrieveReq.numOfFuncs > TSDB_FUNC_MAX_RETRIEVE) { if (retrieveReq.numOfFuncs <= 0 || retrieveReq.numOfFuncs > TSDB_FUNC_MAX_RETRIEVE) {
terrno = TSDB_CODE_MND_INVALID_FUNC_RETRIEVE; code = TSDB_CODE_MND_INVALID_FUNC_RETRIEVE;
goto RETRIEVE_FUNC_OVER; goto RETRIEVE_FUNC_OVER;
} }
retrieveRsp.numOfFuncs = retrieveReq.numOfFuncs; retrieveRsp.numOfFuncs = retrieveReq.numOfFuncs;
retrieveRsp.pFuncInfos = taosArrayInit(retrieveReq.numOfFuncs, sizeof(SFuncInfo)); retrieveRsp.pFuncInfos = taosArrayInit(retrieveReq.numOfFuncs, sizeof(SFuncInfo));
if (retrieveRsp.pFuncInfos == NULL) { if (retrieveRsp.pFuncInfos == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto RETRIEVE_FUNC_OVER; goto RETRIEVE_FUNC_OVER;
} }
retrieveRsp.pFuncExtraInfos = taosArrayInit(retrieveReq.numOfFuncs, sizeof(SFuncExtraInfo)); retrieveRsp.pFuncExtraInfos = taosArrayInit(retrieveReq.numOfFuncs, sizeof(SFuncExtraInfo));
if (retrieveRsp.pFuncExtraInfos == NULL) { if (retrieveRsp.pFuncExtraInfos == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto RETRIEVE_FUNC_OVER; goto RETRIEVE_FUNC_OVER;
} }
@ -497,6 +539,7 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) {
SFuncObj *pFunc = mndAcquireFunc(pMnode, funcName); SFuncObj *pFunc = mndAcquireFunc(pMnode, funcName);
if (pFunc == NULL) { if (pFunc == NULL) {
if (terrno != 0) code = terrno;
goto RETRIEVE_FUNC_OVER; goto RETRIEVE_FUNC_OVER;
} }
@ -541,7 +584,7 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) {
int32_t contLen = tSerializeSRetrieveFuncRsp(NULL, 0, &retrieveRsp); int32_t contLen = tSerializeSRetrieveFuncRsp(NULL, 0, &retrieveRsp);
void *pRsp = rpcMallocCont(contLen); void *pRsp = rpcMallocCont(contLen);
if (pRsp == NULL) { if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto RETRIEVE_FUNC_OVER; goto RETRIEVE_FUNC_OVER;
} }
@ -556,7 +599,7 @@ RETRIEVE_FUNC_OVER:
tFreeSRetrieveFuncReq(&retrieveReq); tFreeSRetrieveFuncReq(&retrieveReq);
tFreeSRetrieveFuncRsp(&retrieveRsp); tFreeSRetrieveFuncRsp(&retrieveRsp);
return code; TAOS_RETURN(code);
} }
static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int32_t len) { static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int32_t len) {

View File

@ -457,7 +457,7 @@ static int32_t mndCreateDir(SMnode *pMnode, const char *path) {
} }
if (taosMkDir(pMnode->path) != 0) { if (taosMkDir(pMnode->path) != 0) {
code = TAOS_SYSTEM_ERROR(errno); code = terrno;
TAOS_RETURN(code); TAOS_RETURN(code);
} }

View File

@ -771,21 +771,22 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
if (pTopic->subType == TOPIC_SUB_TYPE__COLUMN) { if (pTopic->subType == TOPIC_SUB_TYPE__COLUMN) {
pPlan = qStringToQueryPlan(pTopic->physicalPlan); pPlan = qStringToQueryPlan(pTopic->physicalPlan);
if (pPlan == NULL) { if (pPlan == NULL) {
code = TSDB_CODE_QRY_INVALID_INPUT; return TSDB_CODE_QRY_INVALID_INPUT;
TAOS_RETURN(code);
} }
} else if (pTopic->subType == TOPIC_SUB_TYPE__TABLE && pTopic->ast != NULL) { } else if (pTopic->subType == TOPIC_SUB_TYPE__TABLE && pTopic->ast != NULL) {
SNode* pAst = NULL; SNode* pAst = NULL;
if ((code = nodesStringToNode(pTopic->ast, &pAst)) != 0) { code = nodesStringToNode(pTopic->ast, &pAst);
if (code != 0) {
mError("topic:%s, failed to create since %s", pTopic->name, terrstr()); mError("topic:%s, failed to create since %s", pTopic->name, terrstr());
TAOS_RETURN(code); return code;
} }
SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true}; SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true};
if ((code = qCreateQueryPlan(&cxt, &pPlan, NULL)) != 0) { code = qCreateQueryPlan(&cxt, &pPlan, NULL);
if (code != 0) {
mError("failed to create topic:%s since %s", pTopic->name, terrstr()); mError("failed to create topic:%s since %s", pTopic->name, terrstr());
nodesDestroyNode(pAst); nodesDestroyNode(pAst);
TAOS_RETURN(code); return code;
} }
nodesDestroyNode(pAst); nodesDestroyNode(pAst);
} }
@ -793,18 +794,19 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
if (pPlan) { if (pPlan) {
int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
if (levelNum != 1) { if (levelNum != 1) {
qDestroyQueryPlan(pPlan);
code = TSDB_CODE_MND_INVALID_TOPIC_QUERY; code = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
TAOS_RETURN(code); goto END;
} }
SNodeListNode* pNodeListNode = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); SNodeListNode* pNodeListNode = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
if (pNodeListNode == NULL){
code = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
int32_t opNum = LIST_LENGTH(pNodeListNode->pNodeList); int32_t opNum = LIST_LENGTH(pNodeListNode->pNodeList);
if (opNum != 1) { if (opNum != 1) {
qDestroyQueryPlan(pPlan);
code = TSDB_CODE_MND_INVALID_TOPIC_QUERY; code = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
TAOS_RETURN(code); goto END;
} }
pSubplan = (SSubplan*)nodesListGetNode(pNodeListNode->pNodeList, 0); pSubplan = (SSubplan*)nodesListGetNode(pNodeListNode->pNodeList, 0);
@ -825,12 +827,18 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
pSub->vgNum++; pSub->vgNum++;
SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
if (pVgEp == NULL){
code = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup); pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup);
pVgEp->vgId = pVgroup->vgId; pVgEp->vgId = pVgroup->vgId;
taosArrayPush(pSub->unassignedVgs, &pVgEp); if (taosArrayPush(pSub->unassignedVgs, &pVgEp) == NULL){
code = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pVgEp);
goto END;
}
mInfo("init subscription %s for topic:%s assign vgId:%d", pSub->key, pTopic->name, pVgEp->vgId); mInfo("init subscription %s for topic:%s assign vgId:%d", pSub->key, pTopic->name, pVgEp->vgId);
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
} }
@ -838,14 +846,14 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
int32_t msgLen; int32_t msgLen;
if (qSubPlanToString(pSubplan, &pSub->qmsg, &msgLen) < 0) { if (qSubPlanToString(pSubplan, &pSub->qmsg, &msgLen) < 0) {
qDestroyQueryPlan(pPlan);
code = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
TAOS_RETURN(code); goto END;
} }
} else { } else {
pSub->qmsg = taosStrdup(""); pSub->qmsg = taosStrdup("");
} }
END:
qDestroyQueryPlan(pPlan); qDestroyQueryPlan(pPlan);
TAOS_RETURN(code); return code;
} }

File diff suppressed because it is too large Load Diff

View File

@ -123,15 +123,16 @@ _OUT:
} }
static int32_t mndTransValidateImp(SMnode *pMnode, STrans *pTrans) { static int32_t mndTransValidateImp(SMnode *pMnode, STrans *pTrans) {
int32_t code = 0;
if (pTrans->stage == TRN_STAGE_PREPARE) { if (pTrans->stage == TRN_STAGE_PREPARE) {
if (mndTransCheckConflict(pMnode, pTrans) < 0) { if ((code = mndTransCheckConflict(pMnode, pTrans)) < 0) {
mError("trans:%d, failed to validate trans conflicts.", pTrans->id); mError("trans:%d, failed to validate trans conflicts.", pTrans->id);
return -1; TAOS_RETURN(code);
} }
return mndTransValidatePrepareStage(pMnode, pTrans); return mndTransValidatePrepareStage(pMnode, pTrans);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndTransValidate(SMnode *pMnode, SSdbRaw *pRaw) { static int32_t mndTransValidate(SMnode *pMnode, SSdbRaw *pRaw) {
@ -139,10 +140,18 @@ static int32_t mndTransValidate(SMnode *pMnode, SSdbRaw *pRaw) {
int32_t code = -1; int32_t code = -1;
SSdbRow *pRow = mndTransDecode(pRaw); SSdbRow *pRow = mndTransDecode(pRaw);
if (pRow == NULL) goto _OUT; if (pRow == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OUT;
}
pTrans = sdbGetRowObj(pRow); pTrans = sdbGetRowObj(pRow);
if (pTrans == NULL) goto _OUT; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OUT;
}
code = mndTransValidateImp(pMnode, pTrans); code = mndTransValidateImp(pMnode, pTrans);
@ -150,11 +159,10 @@ _OUT:
if (pTrans) mndTransDropData(pTrans); if (pTrans) mndTransDropData(pTrans);
if (pRow) taosMemoryFreeClear(pRow); if (pRow) taosMemoryFreeClear(pRow);
if (code) terrno = (terrno ? terrno : TSDB_CODE_MND_TRANS_CONFLICT); if (code) terrno = (terrno ? terrno : TSDB_CODE_MND_TRANS_CONFLICT);
return code; TAOS_RETURN(code);
} }
int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
terrno = TSDB_CODE_SUCCESS;
SSyncMgmt *pMgmt = &pMnode->syncMgmt; SSyncMgmt *pMgmt = &pMnode->syncMgmt;
SSdbRaw *pRaw = pMsg->pCont; SSdbRaw *pRaw = pMsg->pCont;
STrans *pTrans = NULL; STrans *pTrans = NULL;
@ -163,7 +171,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
if (transId <= 0) { if (transId <= 0) {
mError("trans:%d, invalid commit msg, cache transId:%d seq:%" PRId64, transId, pMgmt->transId, pMgmt->transSeq); mError("trans:%d, invalid commit msg, cache transId:%d seq:%" PRId64, transId, pMgmt->transId, pMgmt->transSeq);
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
goto _OUT; goto _OUT;
} }
@ -176,7 +184,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
if (code != 0) { if (code != 0) {
mError("trans:%d, failed to validate requested trans since %s", transId, terrstr()); mError("trans:%d, failed to validate requested trans since %s", transId, terrstr());
code = 0; code = 0;
pMeta->code = terrno; pMeta->code = code;
goto _OUT; goto _OUT;
} }
@ -184,13 +192,15 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
if (code != 0) { if (code != 0) {
mError("trans:%d, failed to write to sdb since %s", transId, terrstr()); mError("trans:%d, failed to write to sdb since %s", transId, terrstr());
code = 0; code = 0;
pMeta->code = terrno; pMeta->code = code;
goto _OUT; goto _OUT;
} }
pTrans = mndAcquireTrans(pMnode, transId); pTrans = mndAcquireTrans(pMnode, transId);
if (pTrans == NULL) { if (pTrans == NULL) {
mError("trans:%d, not found while execute in mnode since %s", transId, terrstr()); code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
mError("trans:%d, not found while execute in mnode since %s", transId, tstrerror(code));
goto _OUT; goto _OUT;
} }
@ -207,7 +217,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
_OUT: _OUT:
if (pTrans) mndReleaseTrans(pMnode, pTrans); if (pTrans) mndReleaseTrans(pMnode, pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t mndPostMgmtCode(SMnode *pMnode, int32_t code) { static int32_t mndPostMgmtCode(SMnode *pMnode, int32_t code) {
@ -258,7 +268,7 @@ _OUT:
mndPostMgmtCode(pMnode, code ? code : pMeta->code); mndPostMgmtCode(pMnode, code ? code : pMeta->code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL; pMsg->pCont = NULL;
return code; TAOS_RETURN(code);
} }
SyncIndex mndSyncAppliedIndex(const SSyncFSM *pFSM) { SyncIndex mndSyncAppliedIndex(const SSyncFSM *pFSM) {
@ -466,16 +476,18 @@ int32_t mndInitSync(SMnode *pMnode) {
pNode->clusterId); pNode->clusterId);
} }
int32_t code = 0;
tsem_init(&pMgmt->syncSem, 0, 0); tsem_init(&pMgmt->syncSem, 0, 0);
pMgmt->sync = syncOpen(&syncInfo, true); pMgmt->sync = syncOpen(&syncInfo, true);
if (pMgmt->sync <= 0) { if (pMgmt->sync <= 0) {
mError("failed to open sync since %s", terrstr()); if (terrno != 0) code = terrno;
return -1; mError("failed to open sync since %s", tstrerror(code));
TAOS_RETURN(code);
} }
pMnode->pSdb->sync = pMgmt->sync; pMnode->pSdb->sync = pMgmt->sync;
mInfo("mnode-sync is opened, id:%" PRId64, pMgmt->sync); mInfo("mnode-sync is opened, id:%" PRId64, pMgmt->sync);
return 0; TAOS_RETURN(code);
} }
void mndCleanupSync(SMnode *pMnode) { void mndCleanupSync(SMnode *pMnode) {
@ -518,10 +530,10 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) {
SSyncMgmt *pMgmt = &pMnode->syncMgmt; SSyncMgmt *pMgmt = &pMnode->syncMgmt;
SRpcMsg req = {.msgType = TDMT_MND_APPLY_MSG, .contLen = sdbGetRawTotalSize(pRaw)}; SRpcMsg req = {.msgType = TDMT_MND_APPLY_MSG, .contLen = sdbGetRawTotalSize(pRaw)};
if (req.contLen <= 0) return -1; if (req.contLen <= 0) return TSDB_CODE_OUT_OF_MEMORY;
req.pCont = rpcMallocCont(req.contLen); req.pCont = rpcMallocCont(req.contLen);
if (req.pCont == NULL) return -1; if (req.pCont == NULL) return TSDB_CODE_OUT_OF_MEMORY;
memcpy(req.pCont, pRaw, req.contLen); memcpy(req.pCont, pRaw, req.contLen);
taosThreadMutexLock(&pMgmt->lock); taosThreadMutexLock(&pMgmt->lock);
@ -531,8 +543,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) {
mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId); mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId);
taosThreadMutexUnlock(&pMgmt->lock); taosThreadMutexUnlock(&pMgmt->lock);
rpcFreeCont(req.pCont); rpcFreeCont(req.pCont);
terrno = TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED; TAOS_RETURN(TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED);
return terrno;
} }
mInfo("trans:%d, will be proposed", transId); mInfo("trans:%d, will be proposed", transId);

View File

@ -44,8 +44,6 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq);
static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter); static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter);
static int32_t mndSetDropTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic);
int32_t mndInitTopic(SMnode *pMnode) { int32_t mndInitTopic(SMnode *pMnode) {
SSdbTable table = { SSdbTable table = {
.sdbType = SDB_TOPIC, .sdbType = SDB_TOPIC,
@ -70,9 +68,16 @@ int32_t mndInitTopic(SMnode *pMnode) {
void mndCleanupTopic(SMnode *pMnode) {} void mndCleanupTopic(SMnode *pMnode) {}
const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]) { void mndTopicGetShowName(const char* fullTopic, char* topic) {
// if (fullTopic == NULL) {
return strchr(topic, '.') + 1; return;
}
char* tmp = strchr(fullTopic, '.');
if (tmp == NULL) {
tstrncpy(topic, fullTopic, TSDB_TOPIC_FNAME_LEN);
}else {
tstrncpy(topic, tmp+1, TSDB_TOPIC_FNAME_LEN);
}
} }
SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
@ -129,7 +134,9 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
goto TOPIC_ENCODE_OVER; goto TOPIC_ENCODE_OVER;
} }
void *aswBuf = swBuf; void *aswBuf = swBuf;
taosEncodeSSchemaWrapper(&aswBuf, &pTopic->schema); if(taosEncodeSSchemaWrapper(&aswBuf, &pTopic->schema) < 0){
goto TOPIC_ENCODE_OVER;
}
SDB_SET_BINARY(pRaw, dataPos, swBuf, schemaLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, swBuf, schemaLen, TOPIC_ENCODE_OVER);
} }
@ -259,7 +266,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
} }
int16_t colId; int16_t colId;
SDB_GET_INT16(pRaw, dataPos, &colId, TOPIC_DECODE_OVER); SDB_GET_INT16(pRaw, dataPos, &colId, TOPIC_DECODE_OVER);
taosArrayPush(pTopic->ntbColIds, &colId); (void)taosArrayPush(pTopic->ntbColIds, &colId);
} }
SDB_GET_INT64(pRaw, dataPos, &pTopic->ctbStbUid, TOPIC_DECODE_OVER); SDB_GET_INT64(pRaw, dataPos, &pTopic->ctbStbUid, TOPIC_DECODE_OVER);
@ -289,25 +296,25 @@ static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic) {
taosMemoryFreeClear(pTopic->ast); taosMemoryFreeClear(pTopic->ast);
taosMemoryFreeClear(pTopic->physicalPlan); taosMemoryFreeClear(pTopic->physicalPlan);
if (pTopic->schema.nCols) taosMemoryFreeClear(pTopic->schema.pSchema); if (pTopic->schema.nCols) taosMemoryFreeClear(pTopic->schema.pSchema);
taosArrayDestroy(pTopic->ntbColIds); (void)taosArrayDestroy(pTopic->ntbColIds);
return 0; return 0;
} }
static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopicObj *pNewTopic) { static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopicObj *pNewTopic) {
mTrace("topic:%s perform update action", pOldTopic->name); mTrace("topic:%s perform update action", pOldTopic->name);
atomic_exchange_64(&pOldTopic->updateTime, pNewTopic->updateTime); (void)atomic_exchange_64(&pOldTopic->updateTime, pNewTopic->updateTime);
atomic_exchange_32(&pOldTopic->version, pNewTopic->version); (void)atomic_exchange_32(&pOldTopic->version, pNewTopic->version);
return 0; return 0;
} }
SMqTopicObj *mndAcquireTopic(SMnode *pMnode, const char *topicName) { int32_t mndAcquireTopic(SMnode *pMnode, const char *topicName, SMqTopicObj **pTopic) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SMqTopicObj *pTopic = sdbAcquire(pSdb, SDB_TOPIC, topicName); *pTopic = sdbAcquire(pSdb, SDB_TOPIC, topicName);
if (pTopic == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { if (*pTopic == NULL) {
terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; return TSDB_CODE_MND_TOPIC_NOT_EXIST;
} }
return pTopic; return TDB_CODE_SUCCESS;
} }
void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) { void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) {
@ -316,25 +323,23 @@ void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) {
} }
static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) { static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) {
terrno = TSDB_CODE_MND_INVALID_TOPIC; if (pCreate->sql == NULL) return TSDB_CODE_MND_INVALID_TOPIC;
if (pCreate->sql == NULL) return -1;
if (pCreate->subType == TOPIC_SUB_TYPE__COLUMN) { if (pCreate->subType == TOPIC_SUB_TYPE__COLUMN) {
if (pCreate->ast == NULL || pCreate->ast[0] == 0) return -1; if (pCreate->ast == NULL || pCreate->ast[0] == 0) return TSDB_CODE_MND_INVALID_TOPIC;
} else if (pCreate->subType == TOPIC_SUB_TYPE__TABLE) { } else if (pCreate->subType == TOPIC_SUB_TYPE__TABLE) {
if (pCreate->subStbName[0] == 0) return -1; if (pCreate->subStbName[0] == 0) return TSDB_CODE_MND_INVALID_TOPIC;
} else if (pCreate->subType == TOPIC_SUB_TYPE__DB) { } else if (pCreate->subType == TOPIC_SUB_TYPE__DB) {
if (pCreate->subDbName[0] == 0) return -1; if (pCreate->subDbName[0] == 0) return TSDB_CODE_MND_INVALID_TOPIC;
} }
terrno = TSDB_CODE_SUCCESS;
return 0; return 0;
} }
static int32_t extractTopicTbInfo(SNode *pAst, SMqTopicObj *pTopic) { static int32_t extractTopicTbInfo(SNode *pAst, SMqTopicObj *pTopic) {
SNodeList *pNodeList = NULL; SNodeList *pNodeList = NULL;
nodesCollectColumns((SSelectStmt *)pAst, SQL_CLAUSE_FROM, NULL, COLLECT_COL_TYPE_ALL, &pNodeList); int32_t code = 0;
MND_TMQ_RETURN_CHECK(nodesCollectColumns((SSelectStmt *)pAst, SQL_CLAUSE_FROM, NULL, COLLECT_COL_TYPE_ALL, &pNodeList));
int64_t suid = ((SRealTableNode *)((SSelectStmt *)pAst)->pFromTable)->pMeta->suid; int64_t suid = ((SRealTableNode *)((SSelectStmt *)pAst)->pFromTable)->pMeta->suid;
int8_t tableType = ((SRealTableNode *)((SSelectStmt *)pAst)->pFromTable)->pMeta->tableType; int8_t tableType = ((SRealTableNode *)((SSelectStmt *)pAst)->pFromTable)->pMeta->tableType;
if (tableType == TSDB_CHILD_TABLE) { if (tableType == TSDB_CHILD_TABLE) {
@ -345,17 +350,19 @@ static int32_t extractTopicTbInfo(SNode *pAst, SMqTopicObj *pTopic) {
SColumnNode *pCol = (SColumnNode *)pNode; SColumnNode *pCol = (SColumnNode *)pNode;
if (pCol->tableType == TSDB_NORMAL_TABLE) { if (pCol->tableType == TSDB_NORMAL_TABLE) {
pTopic->ntbUid = pCol->tableId; pTopic->ntbUid = pCol->tableId;
taosArrayPush(pTopic->ntbColIds, &pCol->colId); MND_TMQ_NULL_CHECK(taosArrayPush(pTopic->ntbColIds, &pCol->colId));
} }
} }
} }
nodesDestroyList(pNodeList); nodesDestroyList(pNodeList);
return 0;
END:
return code;
} }
static int32_t sendCheckInfoToVnode(STrans *pTrans, SMnode *pMnode, SMqTopicObj *topicObj){ static int32_t sendCheckInfoToVnode(STrans *pTrans, SMnode *pMnode, SMqTopicObj *topicObj){
STqCheckInfo info; STqCheckInfo info = {0};
memcpy(info.topic, topicObj->name, TSDB_TOPIC_FNAME_LEN); (void)memcpy(info.topic, topicObj->name, TSDB_TOPIC_FNAME_LEN);
info.ntbUid = topicObj->ntbUid; info.ntbUid = topicObj->ntbUid;
info.colIdList = topicObj->ntbColIds; info.colIdList = topicObj->ntbColIds;
// broadcast forbid alter info // broadcast forbid alter info
@ -363,7 +370,7 @@ static int32_t sendCheckInfoToVnode(STrans *pTrans, SMnode *pMnode, SMqTopicObj
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL; SVgObj *pVgroup = NULL;
int32_t code = 0; int32_t code = 0;
void *buf = NULL; void *buf = NULL;
while (1) { while (1) {
// iterate vg // iterate vg
@ -378,16 +385,17 @@ static int32_t sendCheckInfoToVnode(STrans *pTrans, SMnode *pMnode, SMqTopicObj
int32_t len; int32_t len;
tEncodeSize(tEncodeSTqCheckInfo, &info, len, code); tEncodeSize(tEncodeSTqCheckInfo, &info, len, code);
if (code != 0) { if (code != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto END; goto END;
} }
buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len);
MND_TMQ_NULL_CHECK(buf);
void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
SEncoder encoder; SEncoder encoder = {0};
tEncoderInit(&encoder, abuf, len); tEncoderInit(&encoder, abuf, len);
code = tEncodeSTqCheckInfo(&encoder, &info); code = tEncodeSTqCheckInfo(&encoder, &info);
if (code < 0) { if (code < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; tEncoderClear(&encoder);
goto END; goto END;
} }
tEncoderClear(&encoder); tEncoderClear(&encoder);
@ -398,10 +406,7 @@ static int32_t sendCheckInfoToVnode(STrans *pTrans, SMnode *pMnode, SMqTopicObj
action.pCont = buf; action.pCont = buf;
action.contLen = sizeof(SMsgHead) + len; action.contLen = sizeof(SMsgHead) + len;
action.msgType = TDMT_VND_TMQ_ADD_CHECKINFO; action.msgType = TDMT_VND_TMQ_ADD_CHECKINFO;
code = mndTransAppendRedoAction(pTrans, &action); MND_TMQ_RETURN_CHECK(mndTransAppendRedoAction(pTrans, &action));
if (code != 0) {
goto END;
}
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
buf = NULL; buf = NULL;
} }
@ -417,33 +422,22 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
const char *userName) { const char *userName) {
mInfo("start to create topic:%s", pCreate->name); mInfo("start to create topic:%s", pCreate->name);
STrans *pTrans = NULL; STrans *pTrans = NULL;
int32_t code = -1; int32_t code = 0;
SNode *pAst = NULL; SNode *pAst = NULL;
SQueryPlan *pPlan = NULL; SQueryPlan *pPlan = NULL;
SMqTopicObj topicObj = {0}; SMqTopicObj topicObj = {0};
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "create-topic"); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "create-topic");
if (pTrans == NULL) { MND_TMQ_NULL_CHECK(pTrans);
mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
code = -1;
goto _OUT;
}
mndTransSetDbName(pTrans, pDb->name, NULL); mndTransSetDbName(pTrans, pDb->name, NULL);
code = mndTransCheckConflict(pMnode, pTrans); MND_TMQ_RETURN_CHECK(mndTransCheckConflict(pMnode, pTrans));
if (code != 0) {
goto _OUT;
}
mInfo("trans:%d to create topic:%s", pTrans->id, pCreate->name); mInfo("trans:%d to create topic:%s", pTrans->id, pCreate->name);
tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN); tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN);
tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN); tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN);
tstrncpy(topicObj.createUser, userName, TSDB_USER_LEN); tstrncpy(topicObj.createUser, userName, TSDB_USER_LEN);
code = mndCheckTopicPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_TOPIC, &topicObj); MND_TMQ_RETURN_CHECK(mndCheckTopicPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_TOPIC, &topicObj));
if (code != 0) {
goto _OUT;
}
topicObj.createTime = taosGetTimestampMs(); topicObj.createTime = taosGetTimestampMs();
topicObj.updateTime = topicObj.createTime; topicObj.updateTime = topicObj.createTime;
@ -458,56 +452,25 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
if (pCreate->subType == TOPIC_SUB_TYPE__COLUMN) { if (pCreate->subType == TOPIC_SUB_TYPE__COLUMN) {
topicObj.ast = taosStrdup(pCreate->ast); topicObj.ast = taosStrdup(pCreate->ast);
topicObj.astLen = strlen(pCreate->ast) + 1; topicObj.astLen = strlen(pCreate->ast) + 1;
qDebugL("topic:%s ast %s", topicObj.name, topicObj.ast); qDebugL("topic:%s ast %s", topicObj.name, topicObj.ast);
MND_TMQ_RETURN_CHECK(nodesStringToNode(pCreate->ast, &pAst));
code = nodesStringToNode(pCreate->ast, &pAst);
if (code != 0) {
mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
goto _OUT;
}
SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true}; SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true};
code = qCreateQueryPlan(&cxt, &pPlan, NULL); MND_TMQ_RETURN_CHECK(qCreateQueryPlan(&cxt, &pPlan, NULL));
if (code != 0) {
mError("failed to create topic:%s since %s", pCreate->name, terrstr());
goto _OUT;
}
topicObj.ntbColIds = taosArrayInit(0, sizeof(int16_t)); topicObj.ntbColIds = taosArrayInit(0, sizeof(int16_t));
if (topicObj.ntbColIds == NULL) { MND_TMQ_NULL_CHECK(topicObj.ntbColIds);
terrno = TSDB_CODE_OUT_OF_MEMORY; MND_TMQ_RETURN_CHECK(extractTopicTbInfo(pAst, &topicObj));
code = terrno;
goto _OUT;
}
extractTopicTbInfo(pAst, &topicObj);
if (topicObj.ntbUid == 0) { if (topicObj.ntbUid == 0) {
taosArrayDestroy(topicObj.ntbColIds); (void)taosArrayDestroy(topicObj.ntbColIds);
topicObj.ntbColIds = NULL; topicObj.ntbColIds = NULL;
} }
code = qExtractResultSchema(pAst, &topicObj.schema.nCols, &topicObj.schema.pSchema); MND_TMQ_RETURN_CHECK(qExtractResultSchema(pAst, &topicObj.schema.nCols, &topicObj.schema.pSchema));
if (code != 0) { MND_TMQ_RETURN_CHECK(nodesNodeToString((SNode *)pPlan, false, &topicObj.physicalPlan, NULL));
mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
goto _OUT;
}
code = nodesNodeToString((SNode *)pPlan, false, &topicObj.physicalPlan, NULL);
if (code != 0) {
mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
goto _OUT;
}
} else if (pCreate->subType == TOPIC_SUB_TYPE__TABLE) { } else if (pCreate->subType == TOPIC_SUB_TYPE__TABLE) {
SStbObj *pStb = mndAcquireStb(pMnode, pCreate->subStbName); SStbObj *pStb = mndAcquireStb(pMnode, pCreate->subStbName);
if (pStb == NULL) { MND_TMQ_NULL_CHECK(pStb);
terrno = TSDB_CODE_MND_STB_NOT_EXIST; tstrncpy(topicObj.stbName, pCreate->subStbName, TSDB_TABLE_FNAME_LEN);
code = terrno;
goto _OUT;
}
strcpy(topicObj.stbName, pCreate->subStbName);
topicObj.stbUid = pStb->uid; topicObj.stbUid = pStb->uid;
mndReleaseStb(pMnode, pStb); mndReleaseStb(pMnode, pStb);
if(pCreate->ast != NULL){ if(pCreate->ast != NULL){
@ -518,34 +481,25 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
} }
SSdbRaw *pCommitRaw = mndTopicActionEncode(&topicObj); SSdbRaw *pCommitRaw = mndTopicActionEncode(&topicObj);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { MND_TMQ_NULL_CHECK(pCommitRaw);
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); code = mndTransAppendCommitlog(pTrans, pCommitRaw);
code = -1; if(code != 0) {
goto _OUT; sdbFreeRaw(pCommitRaw);
goto END;
} }
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); MND_TMQ_RETURN_CHECK(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
if (topicObj.ntbUid != 0) { if (topicObj.ntbUid != 0) {
code = sendCheckInfoToVnode(pTrans, pMnode, &topicObj); MND_TMQ_RETURN_CHECK(sendCheckInfoToVnode(pTrans, pMnode, &topicObj));
if (code != 0){
goto _OUT;
}
} }
MND_TMQ_RETURN_CHECK(mndTransPrepare(pMnode, pTrans));
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
code = -1;
goto _OUT;
}
code = TSDB_CODE_ACTION_IN_PROGRESS; code = TSDB_CODE_ACTION_IN_PROGRESS;
_OUT: END:
taosMemoryFreeClear(topicObj.physicalPlan); taosMemoryFreeClear(topicObj.physicalPlan);
taosMemoryFreeClear(topicObj.sql); taosMemoryFreeClear(topicObj.sql);
taosMemoryFreeClear(topicObj.ast); taosMemoryFreeClear(topicObj.ast);
taosArrayDestroy(topicObj.ntbColIds); (void)taosArrayDestroy(topicObj.ntbColIds);
if (topicObj.schema.nCols) { if (topicObj.schema.nCols) {
taosMemoryFreeClear(topicObj.schema.pSchema); taosMemoryFreeClear(topicObj.schema.pSchema);
} }
@ -557,75 +511,64 @@ _OUT:
static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
int32_t code = -1; int32_t code = TDB_CODE_SUCCESS;
SMqTopicObj *pTopic = NULL; SMqTopicObj *pTopic = NULL;
SDbObj *pDb = NULL; SDbObj *pDb = NULL;
SCMCreateTopicReq createTopicReq = {0}; SCMCreateTopicReq createTopicReq = {0};
if (tDeserializeSCMCreateTopicReq(pReq->pCont, pReq->contLen, &createTopicReq) != 0) { if (tDeserializeSCMCreateTopicReq(pReq->pCont, pReq->contLen, &createTopicReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
goto _OVER; goto END;
} }
mInfo("topic:%s start to create, sql:%s", createTopicReq.name, createTopicReq.sql); mInfo("topic:%s start to create, sql:%s", createTopicReq.name, createTopicReq.sql);
if (mndCheckCreateTopicReq(&createTopicReq) != 0) { MND_TMQ_RETURN_CHECK(mndCheckCreateTopicReq(&createTopicReq));
mError("topic:%s failed to create since %s", createTopicReq.name, terrstr());
goto _OVER;
}
pTopic = mndAcquireTopic(pMnode, createTopicReq.name); code = mndAcquireTopic(pMnode, createTopicReq.name, &pTopic);
if (pTopic != NULL) { if (code == TDB_CODE_SUCCESS) {
if (createTopicReq.igExists) { if (createTopicReq.igExists) {
mInfo("topic:%s already exist, ignore exist is set", createTopicReq.name); mInfo("topic:%s already exist, ignore exist is set", createTopicReq.name);
code = 0; goto END;
goto _OVER;
} else { } else {
terrno = TSDB_CODE_MND_TOPIC_ALREADY_EXIST; code = TSDB_CODE_MND_TOPIC_ALREADY_EXIST;
goto _OVER; goto END;
} }
} else if (terrno != TSDB_CODE_MND_TOPIC_NOT_EXIST) { } else if (code != TSDB_CODE_MND_TOPIC_NOT_EXIST) {
goto _OVER; goto END;
} }
pDb = mndAcquireDb(pMnode, createTopicReq.subDbName); pDb = mndAcquireDb(pMnode, createTopicReq.subDbName);
if (pDb == NULL) { MND_TMQ_NULL_CHECK(pDb);
terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
goto _OVER;
}
if (pDb->cfg.walRetentionPeriod == 0) { if (pDb->cfg.walRetentionPeriod == 0) {
terrno = TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO; code = TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO;
mError("db:%s, not allowed to create topic when WAL_RETENTION_PERIOD is zero", pDb->name); mError("db:%s, not allowed to create topic when WAL_RETENTION_PERIOD is zero", pDb->name);
goto _OVER; goto END;
} }
if (sdbGetSize(pMnode->pSdb, SDB_TOPIC) >= tmqMaxTopicNum){ if (sdbGetSize(pMnode->pSdb, SDB_TOPIC) >= tmqMaxTopicNum){
terrno = TSDB_CODE_TMQ_TOPIC_OUT_OF_RANGE; code = TSDB_CODE_TMQ_TOPIC_OUT_OF_RANGE;
mError("topic num out of range"); mError("topic num out of range");
goto _OVER; goto END;
}
if ((terrno = grantCheck(TSDB_GRANT_SUBSCRIPTION)) < 0) {
goto _OVER;
} }
MND_TMQ_RETURN_CHECK(grantCheck(TSDB_GRANT_SUBSCRIPTION));
code = mndCreateTopic(pMnode, pReq, &createTopicReq, pDb, pReq->info.conn.user); code = mndCreateTopic(pMnode, pReq, &createTopicReq, pDb, pReq->info.conn.user);
if (code == 0) { if (code == 0) {
code = TSDB_CODE_ACTION_IN_PROGRESS; code = TSDB_CODE_ACTION_IN_PROGRESS;
} }
SName dbname = {0}; {
tNameFromString(&dbname, createTopicReq.subDbName, T_NAME_ACCT | T_NAME_DB); SName dbname = {0};
(void)tNameFromString(&dbname, createTopicReq.subDbName, T_NAME_ACCT | T_NAME_DB); // ignore error
SName topicName = {0};
(void)tNameFromString(&topicName, createTopicReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); // ignore error
auditRecord(pReq, pMnode->clusterId, "createTopic", dbname.dbname, topicName.dbname,
createTopicReq.sql, strlen(createTopicReq.sql));
}
SName topicName = {0}; END:
tNameFromString(&topicName, createTopicReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
//reuse this function for topic
auditRecord(pReq, pMnode->clusterId, "createTopic", dbname.dbname, topicName.dbname,
createTopicReq.sql, strlen(createTopicReq.sql));
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("failed to create topic:%s since %s", createTopicReq.name, terrstr()); mError("failed to create topic:%s since %s", createTopicReq.name, terrstr());
} }
@ -638,22 +581,20 @@ _OVER:
} }
static int32_t mndDropTopic(SMnode *pMnode, STrans *pTrans, SRpcMsg *pReq, SMqTopicObj *pTopic) { static int32_t mndDropTopic(SMnode *pMnode, STrans *pTrans, SRpcMsg *pReq, SMqTopicObj *pTopic) {
int32_t code = -1; int32_t code = 0;
if (mndUserRemoveTopic(pMnode, pTrans, pTopic->name) != 0) { SSdbRaw *pCommitRaw = NULL;
goto _OVER; MND_TMQ_RETURN_CHECK(mndUserRemoveTopic(pMnode, pTrans, pTopic->name));
pCommitRaw = mndTopicActionEncode(pTopic);
MND_TMQ_NULL_CHECK(pCommitRaw);
code = mndTransAppendCommitlog(pTrans, pCommitRaw);
if(code != 0) {
sdbFreeRaw(pCommitRaw);
goto END;
} }
MND_TMQ_RETURN_CHECK(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
MND_TMQ_RETURN_CHECK(mndTransPrepare(pMnode, pTrans));
SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic); END:
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER;
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pMnode, pTrans) != 0) {
goto _OVER;
}
code = 0;
_OVER:
return code; return code;
} }
@ -661,7 +602,7 @@ static bool checkTopic(SArray *topics, char *topicName){
int32_t sz = taosArrayGetSize(topics); int32_t sz = taosArrayGetSize(topics);
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
char *name = taosArrayGetP(topics, i); char *name = taosArrayGetP(topics, i);
if (strcmp(name, topicName) == 0) { if (name && strcmp(name, topicName) == 0) {
return true; return true;
} }
} }
@ -682,29 +623,26 @@ static int32_t mndDropConsumerByTopic(SMnode *pMnode, STrans *pTrans, char *topi
bool found = checkTopic(pConsumer->assignedTopics, topicName); bool found = checkTopic(pConsumer->assignedTopics, topicName);
if (found){ if (found){
if (pConsumer->status == MQ_CONSUMER_STATUS_LOST) { if (pConsumer->status == MQ_CONSUMER_STATUS_LOST) {
code = mndSetConsumerDropLogs(pTrans, pConsumer); MND_TMQ_RETURN_CHECK(mndSetConsumerDropLogs(pTrans, pConsumer));
if (code != 0) {
goto end;
}
sdbRelease(pSdb, pConsumer); sdbRelease(pSdb, pConsumer);
continue; continue;
} }
mError("topic:%s, failed to drop since subscribed by consumer:0x%" PRIx64 ", in consumer group %s", mError("topic:%s, failed to drop since subscribed by consumer:0x%" PRIx64 ", in consumer group %s",
topicName, pConsumer->consumerId, pConsumer->cgroup); topicName, pConsumer->consumerId, pConsumer->cgroup);
code = TSDB_CODE_MND_TOPIC_SUBSCRIBED; code = TSDB_CODE_MND_TOPIC_SUBSCRIBED;
goto end; goto END;
} }
if (checkTopic(pConsumer->rebNewTopics, topicName) || checkTopic(pConsumer->rebRemovedTopics, topicName)) { if (checkTopic(pConsumer->rebNewTopics, topicName) || checkTopic(pConsumer->rebRemovedTopics, topicName)) {
code = TSDB_CODE_MND_TOPIC_SUBSCRIBED; code = TSDB_CODE_MND_TOPIC_SUBSCRIBED;
mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s (reb new)", mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s (reb new)",
topicName, pConsumer->consumerId, pConsumer->cgroup); topicName, pConsumer->consumerId, pConsumer->cgroup);
goto end; goto END;
} }
sdbRelease(pSdb, pConsumer); sdbRelease(pSdb, pConsumer);
} }
end: END:
sdbRelease(pSdb, pConsumer); sdbRelease(pSdb, pConsumer);
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
return code; return code;
@ -726,13 +664,10 @@ static int32_t mndDropCheckInfoByTopic(SMnode *pMnode, STrans *pTrans, SMqTopicO
} }
buf = taosMemoryCalloc(1, sizeof(SMsgHead) + TSDB_TOPIC_FNAME_LEN); buf = taosMemoryCalloc(1, sizeof(SMsgHead) + TSDB_TOPIC_FNAME_LEN);
if (buf == NULL){ MND_TMQ_NULL_CHECK(buf);
code = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
((SMsgHead *)buf)->vgId = htonl(pVgroup->vgId); ((SMsgHead *)buf)->vgId = htonl(pVgroup->vgId);
memcpy(abuf, pTopic->name, TSDB_TOPIC_FNAME_LEN); (void)memcpy(abuf, pTopic->name, TSDB_TOPIC_FNAME_LEN);
STransAction action = {0}; STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
@ -742,12 +677,12 @@ static int32_t mndDropCheckInfoByTopic(SMnode *pMnode, STrans *pTrans, SMqTopicO
code = mndTransAppendRedoAction(pTrans, &action); code = mndTransAppendRedoAction(pTrans, &action);
if (code != 0) { if (code != 0) {
taosMemoryFree(buf); taosMemoryFree(buf);
goto end; goto END;
} }
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
} }
end: END:
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
return code; return code;
@ -761,70 +696,40 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
STrans *pTrans = NULL; STrans *pTrans = NULL;
if (tDeserializeSMDropTopicReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { if (tDeserializeSMDropTopicReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; return TSDB_CODE_INVALID_MSG;
return -1;
} }
pTopic = mndAcquireTopic(pMnode, dropReq.name); code = mndAcquireTopic(pMnode, dropReq.name, &pTopic);
if (pTopic == NULL) { if (code != 0) {
if (dropReq.igNotExists) { if (dropReq.igNotExists) {
mInfo("topic:%s, not exist, ignore not exist is set", dropReq.name); mInfo("topic:%s, not exist, ignore not exist is set", dropReq.name);
tFreeSMDropTopicReq(&dropReq); tFreeSMDropTopicReq(&dropReq);
return 0; return 0;
} else { } else {
terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST;
mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); mError("topic:%s, failed to drop since %s", dropReq.name, terrstr());
tFreeSMDropTopicReq(&dropReq); tFreeSMDropTopicReq(&dropReq);
return -1; return code;
} }
} }
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-topic"); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-topic");
if (pTrans == NULL) { MND_TMQ_NULL_CHECK(pTrans);
mError("topic:%s, failed to drop since %s", pTopic->name, terrstr());
code = -1;
goto end;
}
mndTransSetDbName(pTrans, pTopic->db, NULL); mndTransSetDbName(pTrans, pTopic->db, NULL);
code = mndTransCheckConflict(pMnode, pTrans); MND_TMQ_RETURN_CHECK(mndTransCheckConflict(pMnode, pTrans));
if (code != 0) {
goto end;
}
mInfo("trans:%d, used to drop topic:%s", pTrans->id, pTopic->name); mInfo("trans:%d, used to drop topic:%s", pTrans->id, pTopic->name);
MND_TMQ_RETURN_CHECK(mndCheckTopicPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_TOPIC, pTopic));
code = mndCheckTopicPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_TOPIC, pTopic); MND_TMQ_RETURN_CHECK(mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db));
if (code != 0) { MND_TMQ_RETURN_CHECK(mndDropConsumerByTopic(pMnode, pTrans, dropReq.name));
goto end; MND_TMQ_RETURN_CHECK(mndDropSubByTopic(pMnode, pTrans, dropReq.name));
}
code = mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db);
if (code != 0) {
goto end;
}
code = mndDropConsumerByTopic(pMnode, pTrans, dropReq.name);
if (code != 0) {
goto end;
}
code = mndDropSubByTopic(pMnode, pTrans, dropReq.name);
if (code < 0) {
mError("topic:%s, failed to drop since %s", pTopic->name, terrstr());
goto end;
}
if (pTopic->ntbUid != 0) { if (pTopic->ntbUid != 0) {
code = mndDropCheckInfoByTopic(pMnode, pTrans, pTopic); MND_TMQ_RETURN_CHECK(mndDropCheckInfoByTopic(pMnode, pTrans, pTopic));
if (code != 0) {
goto end;
}
} }
code = mndDropTopic(pMnode, pTrans, pReq, pTopic); code = mndDropTopic(pMnode, pTrans, pReq, pTopic);
end: END:
mndReleaseTopic(pMnode, pTopic); mndReleaseTopic(pMnode, pTopic);
mndTransDrop(pTrans); mndTransDrop(pTrans);
if (code != 0) { if (code != 0) {
@ -834,8 +739,7 @@ end:
} }
SName name = {0}; SName name = {0};
tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); (void)tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); // ignore error
auditRecord(pReq, pMnode->clusterId, "dropTopic", name.dbname, name.tname, dropReq.sql, dropReq.sqlLen); auditRecord(pReq, pMnode->clusterId, "dropTopic", name.dbname, name.tname, dropReq.sql, dropReq.sqlLen);
tFreeSMDropTopicReq(&dropReq); tFreeSMDropTopicReq(&dropReq);
@ -849,8 +753,7 @@ int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SDbObj *pDb = mndAcquireDb(pMnode, dbName); SDbObj *pDb = mndAcquireDb(pMnode, dbName);
if (pDb == NULL) { if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED; return TSDB_CODE_MND_DB_NOT_SELECTED;
return -1;
} }
int32_t numOfTopics = 0; int32_t numOfTopics = 0;
@ -875,18 +778,24 @@ int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) {
} }
static void schemaToJson(SSchema *schema, int32_t nCols, char *schemaJson){ static void schemaToJson(SSchema *schema, int32_t nCols, char *schemaJson){
char* string = NULL; char* string = NULL;
int32_t code = 0;
cJSON* columns = cJSON_CreateArray(); cJSON* columns = cJSON_CreateArray();
if (columns == NULL) { MND_TMQ_NULL_CHECK(columns);
return;
}
for (int i = 0; i < nCols; i++) { for (int i = 0; i < nCols; i++) {
cJSON* column = cJSON_CreateObject(); cJSON* column = cJSON_CreateObject();
MND_TMQ_NULL_CHECK(column);
SSchema* s = schema + i; SSchema* s = schema + i;
cJSON* cname = cJSON_CreateString(s->name); cJSON* cname = cJSON_CreateString(s->name);
cJSON_AddItemToObject(column, "name", cname); MND_TMQ_NULL_CHECK(cname);
if (!cJSON_AddItemToObject(column, "name", cname)) {
return;
}
cJSON* ctype = cJSON_CreateString(tDataTypes[s->type].name); cJSON* ctype = cJSON_CreateString(tDataTypes[s->type].name);
cJSON_AddItemToObject(column, "type", ctype); MND_TMQ_NULL_CHECK(ctype);
if (!cJSON_AddItemToObject(column, "type", ctype)) {
return;
}
int32_t length = 0; int32_t length = 0;
if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_VARBINARY || if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_VARBINARY ||
s->type == TSDB_DATA_TYPE_GEOMETRY) { s->type == TSDB_DATA_TYPE_GEOMETRY) {
@ -897,8 +806,13 @@ static void schemaToJson(SSchema *schema, int32_t nCols, char *schemaJson){
length = s->bytes; length = s->bytes;
} }
cJSON* cbytes = cJSON_CreateNumber(length); cJSON* cbytes = cJSON_CreateNumber(length);
cJSON_AddItemToObject(column, "length", cbytes); MND_TMQ_NULL_CHECK(cbytes);
cJSON_AddItemToArray(columns, column); if (!cJSON_AddItemToObject(column, "length", cbytes)){
return;
}
if (!cJSON_AddItemToArray(columns, column)){
return;
}
} }
string = cJSON_PrintUnformatted(columns); string = cJSON_PrintUnformatted(columns);
cJSON_Delete(columns); cJSON_Delete(columns);
@ -910,6 +824,9 @@ static void schemaToJson(SSchema *schema, int32_t nCols, char *schemaJson){
mError("mndRetrieveTopic build schema error json:%p, json len:%zu", string, len); mError("mndRetrieveTopic build schema error json:%p, json len:%zu", string, len);
} }
taosMemoryFree(string); taosMemoryFree(string);
END:
return;
} }
static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) {
@ -917,13 +834,16 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0; int32_t numOfRows = 0;
SMqTopicObj *pTopic = NULL; SMqTopicObj *pTopic = NULL;
int32_t code = 0;
char *sql = NULL;
char *schemaJson = NULL;
while (numOfRows < rowsCapacity) { while (numOfRows < rowsCapacity) {
pShow->pIter = sdbFetch(pSdb, SDB_TOPIC, pShow->pIter, (void **)&pTopic); pShow->pIter = sdbFetch(pSdb, SDB_TOPIC, pShow->pIter, (void **)&pTopic);
if (pShow->pIter == NULL) break; if (pShow->pIter == NULL) break;
SColumnInfoData *pColInfo; SColumnInfoData *pColInfo= NULL;
SName n; SName n = {0};
int32_t cols = 0; int32_t cols = 0;
char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE + 5] = {0}; char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE + 5] = {0};
@ -931,28 +851,34 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
STR_TO_VARSTR(topicName, pName); STR_TO_VARSTR(topicName, pName);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)topicName, false); MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)topicName, false));
char dbName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char dbName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
tNameFromString(&n, pTopic->db, T_NAME_ACCT | T_NAME_DB); MND_TMQ_RETURN_CHECK(tNameFromString(&n, pTopic->db, T_NAME_ACCT | T_NAME_DB));
tNameGetDbName(&n, varDataVal(dbName)); MND_TMQ_RETURN_CHECK(tNameGetDbName(&n, varDataVal(dbName)));
varDataSetLen(dbName, strlen(varDataVal(dbName))); varDataSetLen(dbName, strlen(varDataVal(dbName)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)dbName, false); MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)dbName, false));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pTopic->createTime, false); MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)&pTopic->createTime, false));
char *sql = taosMemoryMalloc(strlen(pTopic->sql) + VARSTR_HEADER_SIZE); sql = taosMemoryMalloc(strlen(pTopic->sql) + VARSTR_HEADER_SIZE);
MND_TMQ_NULL_CHECK(sql);
STR_TO_VARSTR(sql, pTopic->sql); STR_TO_VARSTR(sql, pTopic->sql);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)sql, false); MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)sql, false));
taosMemoryFree(sql); taosMemoryFreeClear(sql);
char *schemaJson = taosMemoryMalloc(TSDB_SHOW_SCHEMA_JSON_LEN + VARSTR_HEADER_SIZE); schemaJson = taosMemoryMalloc(TSDB_SHOW_SCHEMA_JSON_LEN + VARSTR_HEADER_SIZE);
MND_TMQ_NULL_CHECK(schemaJson);
if(pTopic->subType == TOPIC_SUB_TYPE__COLUMN){ if(pTopic->subType == TOPIC_SUB_TYPE__COLUMN){
schemaToJson(pTopic->schema.pSchema, pTopic->schema.nCols, schemaJson); schemaToJson(pTopic->schema.pSchema, pTopic->schema.nCols, schemaJson);
}else if(pTopic->subType == TOPIC_SUB_TYPE__TABLE){ }else if(pTopic->subType == TOPIC_SUB_TYPE__TABLE){
@ -969,8 +895,9 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
} }
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)schemaJson, false); MND_TMQ_NULL_CHECK(pColInfo);
taosMemoryFree(schemaJson); MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)schemaJson, false));
taosMemoryFreeClear(schemaJson);
char mete[4 + VARSTR_HEADER_SIZE] = {0}; char mete[4 + VARSTR_HEADER_SIZE] = {0};
if(pTopic->withMeta){ if(pTopic->withMeta){
@ -980,7 +907,8 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
} }
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)mete, false); MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)mete, false));
char type[8 + VARSTR_HEADER_SIZE] = {0}; char type[8 + VARSTR_HEADER_SIZE] = {0};
if(pTopic->subType == TOPIC_SUB_TYPE__COLUMN){ if(pTopic->subType == TOPIC_SUB_TYPE__COLUMN){
@ -992,7 +920,8 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
} }
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)type, false); MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)type, false));
numOfRows++; numOfRows++;
sdbRelease(pSdb, pTopic); sdbRelease(pSdb, pTopic);
@ -1000,24 +929,11 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
pShow->numOfRows += numOfRows; pShow->numOfRows += numOfRows;
return numOfRows; return numOfRows;
}
int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic) { END:
SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic); taosMemoryFreeClear(schemaJson);
if (pCommitRaw == NULL) return -1; taosMemoryFreeClear(sql);
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; return code;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
}
static int32_t mndSetDropTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic) {
SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
} }
static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter) { static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter) {
@ -1047,33 +963,3 @@ bool mndTopicExistsForDb(SMnode *pMnode, SDbObj *pDb) {
return false; return false;
} }
#if 0
int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
SMqTopicObj *pTopic = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic);
if (pIter == NULL) break;
if (pTopic->dbUid != pDb->uid) {
sdbRelease(pSdb, pTopic);
continue;
}
if (mndSetDropTopicCommitLogs(pMnode, pTrans, pTopic) < 0) {
sdbRelease(pSdb, pTopic);
sdbCancelFetch(pSdb, pIter);
code = -1;
break;
}
sdbRelease(pSdb, pTopic);
}
return code;
}
#endif

View File

@ -1357,7 +1357,7 @@ SHashObj *mndDupUseDbHash(SHashObj *pOld) {
} }
int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) {
memcpy(pNew, pUser, sizeof(SUserObj)); (void)memcpy(pNew, pUser, sizeof(SUserObj));
pNew->authVersion++; pNew->authVersion++;
pNew->updateTime = taosGetTimestampMs(); pNew->updateTime = taosGetTimestampMs();
@ -1377,7 +1377,7 @@ int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) {
taosRUnLockLatch(&pUser->lock); taosRUnLockLatch(&pUser->lock);
if (pNew->readDbs == NULL || pNew->writeDbs == NULL || pNew->topics == NULL) { if (pNew->readDbs == NULL || pNew->writeDbs == NULL || pNew->topics == NULL) {
return -1; return TSDB_CODE_INVALID_PARA;
} }
return 0; return 0;
} }
@ -1983,10 +1983,11 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode
if (ALTER_USER_ADD_SUBSCRIBE_TOPIC_PRIV(pAlterReq->alterType, pAlterReq->privileges)) { if (ALTER_USER_ADD_SUBSCRIBE_TOPIC_PRIV(pAlterReq->alterType, pAlterReq->privileges)) {
int32_t len = strlen(pAlterReq->objname) + 1; int32_t len = strlen(pAlterReq->objname) + 1;
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pAlterReq->objname); SMqTopicObj *pTopic = NULL;
if (pTopic == NULL) { int32_t code = mndAcquireTopic(pMnode, pAlterReq->objname, &pTopic);
if (code != 0) {
mndReleaseTopic(pMnode, pTopic); mndReleaseTopic(pMnode, pTopic);
return -1; return code;
} }
taosHashPut(pNewUser->topics, pTopic->name, len, pTopic->name, TSDB_TOPIC_FNAME_LEN); taosHashPut(pNewUser->topics, pTopic->name, len, pTopic->name, TSDB_TOPIC_FNAME_LEN);
mndReleaseTopic(pMnode, pTopic); mndReleaseTopic(pMnode, pTopic);
@ -1994,10 +1995,11 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode
if (ALTER_USER_DEL_SUBSCRIBE_TOPIC_PRIV(pAlterReq->alterType, pAlterReq->privileges)) { if (ALTER_USER_DEL_SUBSCRIBE_TOPIC_PRIV(pAlterReq->alterType, pAlterReq->privileges)) {
int32_t len = strlen(pAlterReq->objname) + 1; int32_t len = strlen(pAlterReq->objname) + 1;
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pAlterReq->objname); SMqTopicObj *pTopic = NULL;
if (pTopic == NULL) { int32_t code = mndAcquireTopic(pMnode, pAlterReq->objname, &pTopic);
if (code != 0) {
mndReleaseTopic(pMnode, pTopic); mndReleaseTopic(pMnode, pTopic);
return -1; return code;
} }
taosHashRemove(pNewUser->topics, pAlterReq->objname, len); taosHashRemove(pNewUser->topics, pAlterReq->objname, len);
mndReleaseTopic(pMnode, pTopic); mndReleaseTopic(pMnode, pTopic);

View File

@ -2725,15 +2725,6 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro
SDbObj dbObj = {0}; SDbObj dbObj = {0};
SArray *pArray = mndBuildDnodesArray(pMnode, 0); SArray *pArray = mndBuildDnodesArray(pMnode, 0);
// int32_t numOfTopics = 0;
// if (mndGetNumOfTopics(pMnode, pDb->name, &numOfTopics) != 0) {
// goto _OVER;
// }
// if (numOfTopics > 0) {
// terrno = TSDB_CODE_MND_TOPIC_MUST_BE_DELETED;
// goto _OVER;
// }
int32_t numOfStreams = 0; int32_t numOfStreams = 0;
if (mndGetNumOfStreams(pMnode, pDb->name, &numOfStreams) != 0) { if (mndGetNumOfStreams(pMnode, pDb->name, &numOfStreams) != 0) {
goto _OVER; goto _OVER;

View File

@ -19,21 +19,22 @@
#include "query.h" #include "query.h"
#include "qworker.h" #include "qworker.h"
SQnode *qndOpen(const SQnodeOpt *pOption) { int32_t qndOpen(const SQnodeOpt *pOption, SQnode **pQnode) {
SQnode *pQnode = taosMemoryCalloc(1, sizeof(SQnode)); *pQnode = taosMemoryCalloc(1, sizeof(SQnode));
if (NULL == pQnode) { if (NULL == *pQnode) {
qError("calloc SQnode failed"); qError("calloc SQnode failed");
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
pQnode->qndId = QNODE_HANDLE; (*pQnode)->qndId = QNODE_HANDLE;
if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, (void **)&pQnode->pQuery, &pOption->msgCb)) { int32_t code = qWorkerInit(NODE_TYPE_QNODE, (*pQnode)->qndId, (void **)&(*pQnode)->pQuery, &pOption->msgCb);
if (TSDB_CODE_SUCCESS != code) {
taosMemoryFreeClear(pQnode); taosMemoryFreeClear(pQnode);
return NULL; return code;
} }
pQnode->msgCb = pOption->msgCb; (*pQnode)->msgCb = pOption->msgCb;
return pQnode; return TSDB_CODE_SUCCESS;
} }
void qndClose(SQnode *pQnode) { void qndClose(SQnode *pQnode) {

View File

@ -22,7 +22,6 @@ set(
# meta # meta
"src/meta/metaOpen.c" "src/meta/metaOpen.c"
"src/meta/metaIdx.c"
"src/meta/metaTable.c" "src/meta/metaTable.c"
"src/meta/metaSma.c" "src/meta/metaSma.c"
"src/meta/metaQuery.c" "src/meta/metaQuery.c"

View File

@ -226,7 +226,7 @@ void tMapDataGetItemByIdx(SMapData *pMapData, int32_t idx, void *pItem, int32
int32_t tMapDataSearch(SMapData *pMapData, void *pSearchItem, int32_t (*tGetItemFn)(uint8_t *, void *), int32_t tMapDataSearch(SMapData *pMapData, void *pSearchItem, int32_t (*tGetItemFn)(uint8_t *, void *),
int32_t (*tItemCmprFn)(const void *, const void *), void *pItem); int32_t (*tItemCmprFn)(const void *, const void *), void *pItem);
int32_t tPutMapData(uint8_t *p, SMapData *pMapData); int32_t tPutMapData(uint8_t *p, SMapData *pMapData);
int32_t tGetMapData(uint8_t *p, SMapData *pMapData); int32_t tGetMapData(uint8_t *p, SMapData *pMapData, int32_t *decodedSize);
int32_t tMapDataToArray(SMapData *pMapData, int32_t itemSize, int32_t (*tGetItemFn)(uint8_t *, void *), int32_t tMapDataToArray(SMapData *pMapData, int32_t itemSize, int32_t (*tGetItemFn)(uint8_t *, void *),
SArray **ppArray); SArray **ppArray);
// other // other
@ -706,7 +706,6 @@ typedef TARRAY2(STsdbFSetPartition *) STsdbFSetPartList;
STsdbFSetPartList *tsdbFSetPartListCreate(); STsdbFSetPartList *tsdbFSetPartListCreate();
void tsdbFSetPartListDestroy(STsdbFSetPartList **ppList); void tsdbFSetPartListDestroy(STsdbFSetPartList **ppList);
int32_t tSerializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList);
int32_t tDeserializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList); int32_t tDeserializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList);
int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList *pList, TFileSetRangeArray **ppRanges); int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList *pList, TFileSetRangeArray **ppRanges);

View File

@ -318,7 +318,7 @@ void* tdUidStoreFree(STbUidStore* pStore);
// SMetaSnapReader ======================================== // SMetaSnapReader ========================================
int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader); int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader);
int32_t metaSnapReaderClose(SMetaSnapReader** ppReader); void metaSnapReaderClose(SMetaSnapReader** ppReader);
int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData); int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData);
// SMetaSnapWriter ======================================== // SMetaSnapWriter ========================================
int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter); int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter);

View File

@ -117,91 +117,81 @@ static void freeCacheEntryFp(void* param) {
} }
int32_t metaCacheOpen(SMeta* pMeta) { int32_t metaCacheOpen(SMeta* pMeta) {
int32_t code = 0; int32_t code = 0;
SMetaCache* pCache = NULL; int32_t lino;
pCache = (SMetaCache*)taosMemoryMalloc(sizeof(SMetaCache)); pMeta->pCache = (SMetaCache*)taosMemoryCalloc(1, sizeof(SMetaCache));
if (pCache == NULL) { if (pMeta->pCache == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
// open entry cache // open entry cache
pCache->sEntryCache.nEntry = 0; pMeta->pCache->sEntryCache.nEntry = 0;
pCache->sEntryCache.nBucket = META_CACHE_BASE_BUCKET; pMeta->pCache->sEntryCache.nBucket = META_CACHE_BASE_BUCKET;
pCache->sEntryCache.aBucket = pMeta->pCache->sEntryCache.aBucket =
(SMetaCacheEntry**)taosMemoryCalloc(pCache->sEntryCache.nBucket, sizeof(SMetaCacheEntry*)); (SMetaCacheEntry**)taosMemoryCalloc(pMeta->pCache->sEntryCache.nBucket, sizeof(SMetaCacheEntry*));
if (pCache->sEntryCache.aBucket == NULL) { if (pMeta->pCache->sEntryCache.aBucket == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
// open stats cache // open stats cache
pCache->sStbStatsCache.nEntry = 0; pMeta->pCache->sStbStatsCache.nEntry = 0;
pCache->sStbStatsCache.nBucket = META_CACHE_STATS_BUCKET; pMeta->pCache->sStbStatsCache.nBucket = META_CACHE_STATS_BUCKET;
pCache->sStbStatsCache.aBucket = pMeta->pCache->sStbStatsCache.aBucket =
(SMetaStbStatsEntry**)taosMemoryCalloc(pCache->sStbStatsCache.nBucket, sizeof(SMetaStbStatsEntry*)); (SMetaStbStatsEntry**)taosMemoryCalloc(pMeta->pCache->sStbStatsCache.nBucket, sizeof(SMetaStbStatsEntry*));
if (pCache->sStbStatsCache.aBucket == NULL) { if (pMeta->pCache->sStbStatsCache.aBucket == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err2;
} }
pCache->sTagFilterResCache.pUidResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5); pMeta->pCache->sTagFilterResCache.pUidResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5);
if (pCache->sTagFilterResCache.pUidResCache == NULL) { if (pMeta->pCache->sTagFilterResCache.pUidResCache == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = terrno, lino, _exit);
goto _err2;
} }
pCache->sTagFilterResCache.accTimes = 0; pMeta->pCache->sTagFilterResCache.accTimes = 0;
pCache->sTagFilterResCache.pTableEntry = pMeta->pCache->sTagFilterResCache.pTableEntry =
taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
if (pCache->sTagFilterResCache.pTableEntry == NULL) { if (pMeta->pCache->sTagFilterResCache.pTableEntry == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = terrno, lino, _exit);
goto _err2;
} }
taosHashSetFreeFp(pCache->sTagFilterResCache.pTableEntry, freeCacheEntryFp); taosHashSetFreeFp(pMeta->pCache->sTagFilterResCache.pTableEntry, freeCacheEntryFp);
taosThreadMutexInit(&pCache->sTagFilterResCache.lock, NULL); taosThreadMutexInit(&pMeta->pCache->sTagFilterResCache.lock, NULL);
pCache->STbGroupResCache.pResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5); pMeta->pCache->STbGroupResCache.pResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5);
if (pCache->STbGroupResCache.pResCache == NULL) { if (pMeta->pCache->STbGroupResCache.pResCache == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = terrno, lino, _exit);
goto _err2;
} }
pCache->STbGroupResCache.accTimes = 0; pMeta->pCache->STbGroupResCache.accTimes = 0;
pCache->STbGroupResCache.pTableEntry = pMeta->pCache->STbGroupResCache.pTableEntry =
taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
if (pCache->STbGroupResCache.pTableEntry == NULL) { if (pMeta->pCache->STbGroupResCache.pTableEntry == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = terrno, lino, _exit);
goto _err2;
} }
taosHashSetFreeFp(pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp); taosHashSetFreeFp(pMeta->pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp);
taosThreadMutexInit(&pCache->STbGroupResCache.lock, NULL); taosThreadMutexInit(&pMeta->pCache->STbGroupResCache.lock, NULL);
pCache->STbFilterCache.pStb = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); pMeta->pCache->STbFilterCache.pStb =
if (pCache->STbFilterCache.pStb == NULL) { taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
code = TSDB_CODE_OUT_OF_MEMORY; if (pMeta->pCache->STbFilterCache.pStb == NULL) {
goto _err2; TSDB_CHECK_CODE(code = terrno, lino, _exit);
} }
pCache->STbFilterCache.pStbName = pMeta->pCache->STbFilterCache.pStbName =
taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
if (pCache->STbFilterCache.pStbName == NULL) { if (pMeta->pCache->STbFilterCache.pStbName == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = terrno, lino, _exit);
goto _err2;
} }
pMeta->pCache = pCache; _exit:
return code; if (code) {
metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code));
_err2: metaCacheClose(pMeta);
entryCacheClose(pMeta); } else {
metaDebug("vgId:%d, %s success", TD_VID(pMeta->pVnode), __func__);
_err: }
taosMemoryFree(pCache);
metaError("vgId:%d, meta open cache failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
return code; return code;
} }
@ -289,8 +279,7 @@ int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo) {
} }
} else { // insert } else { // insert
if (pCache->sEntryCache.nEntry >= pCache->sEntryCache.nBucket) { if (pCache->sEntryCache.nEntry >= pCache->sEntryCache.nBucket) {
code = metaRehashCache(pCache, 1); TAOS_UNUSED(metaRehashCache(pCache, 1));
if (code) goto _exit;
iBucket = TABS(pInfo->uid) % pCache->sEntryCache.nBucket; iBucket = TABS(pInfo->uid) % pCache->sEntryCache.nBucket;
} }
@ -328,8 +317,7 @@ int32_t metaCacheDrop(SMeta* pMeta, int64_t uid) {
pCache->sEntryCache.nEntry--; pCache->sEntryCache.nEntry--;
if (pCache->sEntryCache.nEntry < pCache->sEntryCache.nBucket / 4 && if (pCache->sEntryCache.nEntry < pCache->sEntryCache.nBucket / 4 &&
pCache->sEntryCache.nBucket > META_CACHE_BASE_BUCKET) { pCache->sEntryCache.nBucket > META_CACHE_BASE_BUCKET) {
code = metaRehashCache(pCache, 0); TAOS_UNUSED(metaRehashCache(pCache, 0));
if (code) goto _exit;
} }
} else { } else {
code = TSDB_CODE_NOT_FOUND; code = TSDB_CODE_NOT_FOUND;
@ -351,7 +339,9 @@ int32_t metaCacheGet(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo) {
} }
if (pEntry) { if (pEntry) {
*pInfo = pEntry->info; if (pInfo) {
*pInfo = pEntry->info;
}
} else { } else {
code = TSDB_CODE_NOT_FOUND; code = TSDB_CODE_NOT_FOUND;
} }
@ -415,9 +405,7 @@ int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo) {
(*ppEntry)->info.ctbNum = pInfo->ctbNum; (*ppEntry)->info.ctbNum = pInfo->ctbNum;
} else { // insert } else { // insert
if (pCache->sStbStatsCache.nEntry >= pCache->sStbStatsCache.nBucket) { if (pCache->sStbStatsCache.nEntry >= pCache->sStbStatsCache.nBucket) {
code = metaRehashStatsCache(pCache, 1); TAOS_UNUSED(metaRehashStatsCache(pCache, 1));
if (code) goto _exit;
iBucket = TABS(pInfo->uid) % pCache->sStbStatsCache.nBucket; iBucket = TABS(pInfo->uid) % pCache->sStbStatsCache.nBucket;
} }
@ -454,8 +442,7 @@ int32_t metaStatsCacheDrop(SMeta* pMeta, int64_t uid) {
pCache->sStbStatsCache.nEntry--; pCache->sStbStatsCache.nEntry--;
if (pCache->sStbStatsCache.nEntry < pCache->sStbStatsCache.nBucket / 4 && if (pCache->sStbStatsCache.nEntry < pCache->sStbStatsCache.nBucket / 4 &&
pCache->sStbStatsCache.nBucket > META_CACHE_STATS_BUCKET) { pCache->sStbStatsCache.nBucket > META_CACHE_STATS_BUCKET) {
code = metaRehashStatsCache(pCache, 0); TAOS_UNUSED(metaRehashStatsCache(pCache, 0));
if (code) goto _exit;
} }
} else { } else {
code = TSDB_CODE_NOT_FOUND; code = TSDB_CODE_NOT_FOUND;
@ -477,7 +464,9 @@ int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo) {
} }
if (pEntry) { if (pEntry) {
*pInfo = pEntry->info; if (pInfo) {
*pInfo = pEntry->info;
}
} else { } else {
code = TSDB_CODE_NOT_FOUND; code = TSDB_CODE_NOT_FOUND;
} }
@ -502,7 +491,9 @@ static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInv
// check whether it is existed in LRU cache, and remove it from linked list if not. // check whether it is existed in LRU cache, and remove it from linked list if not.
LRUHandle* pRes = taosLRUCacheLookup(pCache, buf, len); LRUHandle* pRes = taosLRUCacheLookup(pCache, buf, len);
if (pRes == NULL) { // remove the item in the linked list if (pRes == NULL) { // remove the item in the linked list
taosArrayPush(pInvalidRes, &pNode); if (taosArrayPush(pInvalidRes, &pNode) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} else { } else {
taosLRUCacheRelease(pCache, pRes, false); taosLRUCacheRelease(pCache, pRes, false);
} }
@ -626,7 +617,7 @@ static int32_t addNewEntry(SHashObj* pTableEntry, const void* pKey, int32_t keyL
p->hitTimes = 0; p->hitTimes = 0;
tdListInit(&p->list, keyLen); tdListInit(&p->list, keyLen);
taosHashPut(pTableEntry, &suid, sizeof(uint64_t), &p, POINTER_BYTES); TAOS_CHECK_RETURN(taosHashPut(pTableEntry, &suid, sizeof(uint64_t), &p, POINTER_BYTES));
tdListAppend(&p->list, pKey); tdListAppend(&p->list, pKey);
return 0; return 0;
} }
@ -956,9 +947,7 @@ int32_t metaInitTbFilterCache(SMeta* pMeta) {
} }
if (tbNum && pTbArr) { if (tbNum && pTbArr) {
for (int32_t i = 0; i < tbNum; ++i) { for (int32_t i = 0; i < tbNum; ++i) {
if (metaPutTbToFilterCache(pMeta, pTbArr[i], 1) != 0) { TAOS_CHECK_RETURN(metaPutTbToFilterCache(pMeta, pTbArr[i], 1));
return terrno ? terrno : -1;
}
} }
} }
#else #else

View File

@ -21,7 +21,10 @@ static FORCE_INLINE void *metaMalloc(void *pPool, size_t size) {
static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); } static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); }
// begin a meta txn // begin a meta txn
int metaBegin(SMeta *pMeta, int8_t heap) { int32_t metaBegin(SMeta *pMeta, int8_t heap) {
int32_t code = 0;
int32_t lino;
void *(*xMalloc)(void *, size_t) = NULL; void *(*xMalloc)(void *, size_t) = NULL;
void (*xFree)(void *, void *) = NULL; void (*xFree)(void *, void *) = NULL;
void *xArg = NULL; void *xArg = NULL;
@ -36,12 +39,19 @@ int metaBegin(SMeta *pMeta, int8_t heap) {
xArg = pMeta->pVnode->inUse; xArg = pMeta->pVnode->inUse;
} }
if (tdbBegin(pMeta->pEnv, &pMeta->txn, xMalloc, xFree, xArg, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { code = tdbBegin(pMeta->pEnv, &pMeta->txn, xMalloc, xFree, xArg, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
return -1; TSDB_CHECK_CODE(code, lino, _exit);
code = tdbCommit(pMeta->pEnv, pMeta->txn);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__,
tstrerror(terrno));
} else {
metaDebug("vgId:%d %s success", TD_VID(pMeta->pVnode), __func__);
} }
tdbCommit(pMeta->pEnv, pMeta->txn);
return 0; return 0;
} }
@ -49,20 +59,36 @@ int metaBegin(SMeta *pMeta, int8_t heap) {
TXN *metaGetTxn(SMeta *pMeta) { return pMeta->txn; } TXN *metaGetTxn(SMeta *pMeta) { return pMeta->txn; }
int metaCommit(SMeta *pMeta, TXN *txn) { return tdbCommit(pMeta->pEnv, txn); } int metaCommit(SMeta *pMeta, TXN *txn) { return tdbCommit(pMeta->pEnv, txn); }
int metaFinishCommit(SMeta *pMeta, TXN *txn) { return tdbPostCommit(pMeta->pEnv, txn); } int metaFinishCommit(SMeta *pMeta, TXN *txn) { return tdbPostCommit(pMeta->pEnv, txn); }
int metaPrepareAsyncCommit(SMeta *pMeta) {
// return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn); int metaPrepareAsyncCommit(SMeta *pMeta) {
int code = 0; // return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn);
int code = 0;
int32_t lino;
metaWLock(pMeta); metaWLock(pMeta);
code = ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn); TAOS_UNUSED(ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn));
metaULock(pMeta); metaULock(pMeta);
code = tdbCommit(pMeta->pEnv, pMeta->txn); code = tdbCommit(pMeta->pEnv, pMeta->txn);
TSDB_CHECK_CODE(code, lino, _exit);
pMeta->changed = false; pMeta->changed = false;
_exit:
if (code) {
metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__,
tstrerror(terrno));
} else {
metaDebug("vgId:%d %s success", TD_VID(pMeta->pVnode), __func__);
}
return code; return code;
} }
// abort the meta txn // abort the meta txn
int metaAbort(SMeta *pMeta) { int metaAbort(SMeta *pMeta) {
if (!pMeta->txn) return 0; if (!pMeta->txn) {
return 0;
}
int code = tdbAbort(pMeta->pEnv, pMeta->txn); int code = tdbAbort(pMeta->pEnv, pMeta->txn);
if (code) { if (code) {
metaError("vgId:%d, failed to abort meta since %s", TD_VID(pMeta->pVnode), tstrerror(terrno)); metaError("vgId:%d, failed to abort meta since %s", TD_VID(pMeta->pVnode), tstrerror(terrno));

View File

@ -17,159 +17,166 @@
int meteEncodeColCmprEntry(SEncoder *pCoder, const SMetaEntry *pME) { int meteEncodeColCmprEntry(SEncoder *pCoder, const SMetaEntry *pME) {
const SColCmprWrapper *pw = &pME->colCmpr; const SColCmprWrapper *pw = &pME->colCmpr;
if (tEncodeI32v(pCoder, pw->nCols) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pw->nCols));
if (tEncodeI32v(pCoder, pw->version) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pw->version));
uDebug("encode cols:%d", pw->nCols); uDebug("encode cols:%d", pw->nCols);
for (int32_t i = 0; i < pw->nCols; i++) { for (int32_t i = 0; i < pw->nCols; i++) {
SColCmpr *p = &pw->pColCmpr[i]; SColCmpr *p = &pw->pColCmpr[i];
if (tEncodeI16v(pCoder, p->id) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI16v(pCoder, p->id));
if (tEncodeU32(pCoder, p->alg) < 0) return -1; TAOS_CHECK_RETURN(tEncodeU32(pCoder, p->alg));
} }
return 0; return 0;
} }
int meteDecodeColCmprEntry(SDecoder *pDecoder, SMetaEntry *pME) { int meteDecodeColCmprEntry(SDecoder *pDecoder, SMetaEntry *pME) {
SColCmprWrapper *pWrapper = &pME->colCmpr; SColCmprWrapper *pWrapper = &pME->colCmpr;
if (tDecodeI32v(pDecoder, &pWrapper->nCols) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pWrapper->nCols));
if (pWrapper->nCols == 0) { if (pWrapper->nCols == 0) {
return 0; return 0;
} }
if (tDecodeI32v(pDecoder, &pWrapper->version) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pWrapper->version));
uDebug("dencode cols:%d", pWrapper->nCols); uDebug("dencode cols:%d", pWrapper->nCols);
pWrapper->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pWrapper->nCols * sizeof(SColCmpr)); pWrapper->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pWrapper->nCols * sizeof(SColCmpr));
if (pWrapper->pColCmpr == NULL) return -1; if (pWrapper->pColCmpr == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int i = 0; i < pWrapper->nCols; i++) { for (int i = 0; i < pWrapper->nCols; i++) {
SColCmpr *p = &pWrapper->pColCmpr[i]; SColCmpr *p = &pWrapper->pColCmpr[i];
if (tDecodeI16v(pDecoder, &p->id) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI16v(pDecoder, &p->id));
if (tDecodeU32(pDecoder, &p->alg) < 0) return -1; TAOS_CHECK_RETURN(tDecodeU32(pDecoder, &p->alg));
} }
return 0; return 0;
} }
static FORCE_INLINE void metatInitDefaultSColCmprWrapper(SDecoder *pDecoder, SColCmprWrapper *pCmpr, static FORCE_INLINE int32_t metatInitDefaultSColCmprWrapper(SDecoder *pDecoder, SColCmprWrapper *pCmpr,
SSchemaWrapper *pSchema) { SSchemaWrapper *pSchema) {
pCmpr->nCols = pSchema->nCols; pCmpr->nCols = pSchema->nCols;
pCmpr->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pCmpr->nCols * sizeof(SColCmpr)); if ((pCmpr->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pCmpr->nCols * sizeof(SColCmpr))) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < pCmpr->nCols; i++) { for (int32_t i = 0; i < pCmpr->nCols; i++) {
SColCmpr *pColCmpr = &pCmpr->pColCmpr[i]; SColCmpr *pColCmpr = &pCmpr->pColCmpr[i];
SSchema *pColSchema = &pSchema->pSchema[i]; SSchema *pColSchema = &pSchema->pSchema[i];
pColCmpr->id = pColSchema->colId; pColCmpr->id = pColSchema->colId;
pColCmpr->alg = createDefaultColCmprByType(pColSchema->type); pColCmpr->alg = createDefaultColCmprByType(pColSchema->type);
} }
return 0;
} }
int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
if (tStartEncode(pCoder) < 0) return -1; TAOS_CHECK_RETURN(tStartEncode(pCoder));
TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->version));
TAOS_CHECK_RETURN(tEncodeI8(pCoder, pME->type));
TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->uid));
if (tEncodeI64(pCoder, pME->version) < 0) return -1; if (pME->name == NULL) {
if (tEncodeI8(pCoder, pME->type) < 0) return -1; return TSDB_CODE_INVALID_PARA;
if (tEncodeI64(pCoder, pME->uid) < 0) return -1; }
if (pME->name == NULL || tEncodeCStr(pCoder, pME->name) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->name));
if (pME->type == TSDB_SUPER_TABLE) { if (pME->type == TSDB_SUPER_TABLE) {
if (tEncodeI8(pCoder, pME->flags) < 0) return -1; // TODO: need refactor? TAOS_CHECK_RETURN(tEncodeI8(pCoder, pME->flags));
if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaRow) < 0) return -1; TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaRow));
if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1; TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag));
if (TABLE_IS_ROLLUP(pME->flags)) { if (TABLE_IS_ROLLUP(pME->flags)) {
if (tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1; TAOS_CHECK_RETURN(tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam));
} }
} else if (pME->type == TSDB_CHILD_TABLE) { } else if (pME->type == TSDB_CHILD_TABLE) {
if (tEncodeI64(pCoder, pME->ctbEntry.btime) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ctbEntry.btime));
if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32(pCoder, pME->ctbEntry.ttlDays));
if (tEncodeI32v(pCoder, pME->ctbEntry.commentLen) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ctbEntry.commentLen));
if (pME->ctbEntry.commentLen > 0) { if (pME->ctbEntry.commentLen > 0) {
if (tEncodeCStr(pCoder, pME->ctbEntry.comment) < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->ctbEntry.comment));
} }
if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ctbEntry.suid));
if (tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags) < 0) return -1; TAOS_CHECK_RETURN(tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags));
} else if (pME->type == TSDB_NORMAL_TABLE) { } else if (pME->type == TSDB_NORMAL_TABLE) {
if (tEncodeI64(pCoder, pME->ntbEntry.btime) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ntbEntry.btime));
if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32(pCoder, pME->ntbEntry.ttlDays));
if (tEncodeI32v(pCoder, pME->ntbEntry.commentLen) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ntbEntry.commentLen));
if (pME->ntbEntry.commentLen > 0) { if (pME->ntbEntry.commentLen > 0) {
if (tEncodeCStr(pCoder, pME->ntbEntry.comment) < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->ntbEntry.comment));
} }
if (tEncodeI32v(pCoder, pME->ntbEntry.ncid) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ntbEntry.ncid));
if (tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schemaRow) < 0) return -1; TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schemaRow));
} else if (pME->type == TSDB_TSMA_TABLE) { } else if (pME->type == TSDB_TSMA_TABLE) {
if (tEncodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1; TAOS_CHECK_RETURN(tEncodeTSma(pCoder, pME->smaEntry.tsma));
} else { } else {
metaError("meta/entry: invalide table type: %" PRId8 " encode failed.", pME->type); metaError("meta/entry: invalide table type: %" PRId8 " encode failed.", pME->type);
return TSDB_CODE_INVALID_PARA;
return -1;
} }
if (meteEncodeColCmprEntry(pCoder, pME) < 0) return -1; TAOS_CHECK_RETURN(meteEncodeColCmprEntry(pCoder, pME));
tEndEncode(pCoder); tEndEncode(pCoder);
return 0; return 0;
} }
int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
if (tStartDecode(pCoder) < 0) return -1; TAOS_CHECK_RETURN(tStartDecode(pCoder));
TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->version));
if (tDecodeI64(pCoder, &pME->version) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI8(pCoder, &pME->type));
if (tDecodeI8(pCoder, &pME->type) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->uid));
if (tDecodeI64(pCoder, &pME->uid) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->name));
if (tDecodeCStr(pCoder, &pME->name) < 0) return -1;
if (pME->type == TSDB_SUPER_TABLE) { if (pME->type == TSDB_SUPER_TABLE) {
if (tDecodeI8(pCoder, &pME->flags) < 0) return -1; // TODO: need refactor? TAOS_CHECK_RETURN(tDecodeI8(pCoder, &pME->flags));
if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaRow) < 0) return -1; TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaRow));
if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag) < 0) return -1; TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag));
if (TABLE_IS_ROLLUP(pME->flags)) { if (TABLE_IS_ROLLUP(pME->flags)) {
if (tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1; TAOS_CHECK_RETURN(tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam));
} }
} else if (pME->type == TSDB_CHILD_TABLE) { } else if (pME->type == TSDB_CHILD_TABLE) {
if (tDecodeI64(pCoder, &pME->ctbEntry.btime) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ctbEntry.btime));
if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(pCoder, &pME->ctbEntry.ttlDays));
if (tDecodeI32v(pCoder, &pME->ctbEntry.commentLen) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ctbEntry.commentLen));
if (pME->ctbEntry.commentLen > 0) { if (pME->ctbEntry.commentLen > 0) {
if (tDecodeCStr(pCoder, &pME->ctbEntry.comment) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->ctbEntry.comment));
} }
if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ctbEntry.suid));
if (tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags) < 0) return -1; // (TODO) TAOS_CHECK_RETURN(tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags));
} else if (pME->type == TSDB_NORMAL_TABLE) { } else if (pME->type == TSDB_NORMAL_TABLE) {
if (tDecodeI64(pCoder, &pME->ntbEntry.btime) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ntbEntry.btime));
if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(pCoder, &pME->ntbEntry.ttlDays));
if (tDecodeI32v(pCoder, &pME->ntbEntry.commentLen) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ntbEntry.commentLen));
if (pME->ntbEntry.commentLen > 0) { if (pME->ntbEntry.commentLen > 0) {
if (tDecodeCStr(pCoder, &pME->ntbEntry.comment) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->ntbEntry.comment));
} }
if (tDecodeI32v(pCoder, &pME->ntbEntry.ncid) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ntbEntry.ncid));
if (tDecodeSSchemaWrapperEx(pCoder, &pME->ntbEntry.schemaRow) < 0) return -1; TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->ntbEntry.schemaRow));
} else if (pME->type == TSDB_TSMA_TABLE) { } else if (pME->type == TSDB_TSMA_TABLE) {
pME->smaEntry.tsma = tDecoderMalloc(pCoder, sizeof(STSma)); pME->smaEntry.tsma = tDecoderMalloc(pCoder, sizeof(STSma));
if (!pME->smaEntry.tsma) { if (!pME->smaEntry.tsma) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
if (tDecodeTSma(pCoder, pME->smaEntry.tsma, true) < 0) return -1; TAOS_CHECK_RETURN(tDecodeTSma(pCoder, pME->smaEntry.tsma, true));
} else { } else {
metaError("meta/entry: invalide table type: %" PRId8 " decode failed.", pME->type); metaError("meta/entry: invalide table type: %" PRId8 " decode failed.", pME->type);
return -1; return TSDB_CODE_INVALID_PARA;
} }
if (pME->type == TSDB_SUPER_TABLE) { if (pME->type == TSDB_SUPER_TABLE) {
if (TABLE_IS_COL_COMPRESSED(pME->flags)) { if (TABLE_IS_COL_COMPRESSED(pME->flags)) {
if (meteDecodeColCmprEntry(pCoder, pME) < 0) return -1; TAOS_CHECK_RETURN(meteDecodeColCmprEntry(pCoder, pME));
if (pME->colCmpr.nCols == 0) { if (pME->colCmpr.nCols == 0) {
metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow); TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow));
} }
} else { } else {
metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow); TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow));
TABLE_SET_COL_COMPRESSED(pME->flags); TABLE_SET_COL_COMPRESSED(pME->flags);
} }
} else if (pME->type == TSDB_NORMAL_TABLE) { } else if (pME->type == TSDB_NORMAL_TABLE) {
if (!tDecodeIsEnd(pCoder)) { if (!tDecodeIsEnd(pCoder)) {
uDebug("set type: %d, tableName:%s", pME->type, pME->name); uDebug("set type: %d, tableName:%s", pME->type, pME->name);
if (meteDecodeColCmprEntry(pCoder, pME) < 0) return -1; TAOS_CHECK_RETURN(meteDecodeColCmprEntry(pCoder, pME));
if (pME->colCmpr.nCols == 0) { if (pME->colCmpr.nCols == 0) {
metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow); TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow));
} }
} else { } else {
uDebug("set default type: %d, tableName:%s", pME->type, pME->name); uDebug("set default type: %d, tableName:%s", pME->type, pME->name);
metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow); TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow));
} }
TABLE_SET_COL_COMPRESSED(pME->flags); TABLE_SET_COL_COMPRESSED(pME->flags);
} }

View File

@ -1,118 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifdef USE_INVERTED_INDEX
#include "index.h"
#endif
#include "meta.h"
struct SMetaIdx {
#ifdef USE_INVERTED_INDEX
SIndex *pIdx;
#endif
/* data */
#ifdef WINDOWS
size_t avoidCompilationErrors;
#endif
};
int metaOpenIdx(SMeta *pMeta) {
#if 0
char idxDir[128]; // TODO
char * err = NULL;
rocksdb_options_t *options = rocksdb_options_create();
// TODO
sprintf(idxDir, "%s/index", pMeta->path);
if (pMeta->pCache) {
rocksdb_options_set_row_cache(options, pMeta->pCache);
}
rocksdb_options_set_create_if_missing(options, 1);
pMeta->pIdx = rocksdb_open(options, idxDir, &err);
if (pMeta->pIdx == NULL) {
// TODO: handle error
rocksdb_options_destroy(options);
return -1;
}
rocksdb_options_destroy(options);
#endif
#ifdef USE_INVERTED_INDEX
// SIndexOpts opts;
// if (indexOpen(&opts, pMeta->path, &pMeta->pIdx->pIdx) != 0) {
// return -1;
//}
#endif
return 0;
}
#ifdef BUILD_NO_CALL
void metaCloseIdx(SMeta *pMeta) { /* TODO */
#if 0
if (pMeta->pIdx) {
rocksdb_close(pMeta->pIdx);
pMeta->pIdx = NULL;
}
#endif
#ifdef USE_INVERTED_INDEX
// SIndexOpts opts;
// if (indexClose(pMeta->pIdx->pIdx) != 0) {
// return -1;
//}
// return 0;
#endif
}
int metaSaveTableToIdx(SMeta *pMeta, const STbCfg *pTbCfg) {
#ifdef USE_INVERTED_INDEX
// if (pTbCfgs->type == META_CHILD_TABLE) {
// char buf[8] = {0};
// int16_t colId = (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId;
// sprintf(buf, "%d", colId); // colname
// char *pTagVal = (char *)tdGetKVRowValOfCol(pTbCfg->ctbCfg.pTag, (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId);
// tb_uid_t suid = pTbCfg->ctbCfg.suid; // super id
// tb_uid_t tuid = 0; // child table uid
// SIndexMultiTerm *terms = indexMultiTermCreate();
// SIndexTerm *term =
// indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BINARY, buf, strlen(buf), pTagVal, strlen(pTagVal), tuid);
// indexMultiTermAdd(terms, term);
// int ret = indexPut(pMeta->pIdx->pIdx, terms);
// indexMultiTermDestroy(terms);
// return ret;
//} else {
// return DB_DONOTINDEX;
//}
#endif
// TODO
return 0;
}
int metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid) {
#ifdef USE_INVERTED_INDEX
#endif
// TODO
return 0;
}
#endif

View File

@ -39,13 +39,13 @@ static int32_t metaDestroyLock(SMeta *pMeta) { return taosThreadRwlockDestroy(&p
static void metaCleanup(SMeta **ppMeta); static void metaCleanup(SMeta **ppMeta);
int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
SMeta *pMeta = NULL; SMeta *pMeta = NULL;
int ret; int32_t code = 0;
int offset; int32_t lino;
char path[TSDB_FILENAME_LEN] = {0}; int32_t offset;
char path[TSDB_FILENAME_LEN] = {0};
*ppMeta = NULL; char indexFullPath[128] = {0};
// create handle // create handle
vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, path, TSDB_FILENAME_LEN); vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, path, TSDB_FILENAME_LEN);
@ -53,8 +53,7 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
snprintf(path + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VNODE_META_DIR); snprintf(path + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VNODE_META_DIR);
if ((pMeta = taosMemoryCalloc(1, sizeof(*pMeta) + strlen(path) + 1)) == NULL) { if ((pMeta = taosMemoryCalloc(1, sizeof(*pMeta) + strlen(path) + 1)) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
return -1;
} }
metaInitLock(pMeta); metaInitLock(pMeta);
@ -69,163 +68,104 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
taosMkDir(pMeta->path); taosMkDir(pMeta->path);
// open env // open env
ret = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback, code = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback,
pVnode->config.tdbEncryptAlgorithm, pVnode->config.tdbEncryptKey); pVnode->config.tdbEncryptAlgorithm, pVnode->config.tdbEncryptKey);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pTbDb // open pTbDb
ret = tdbTbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb, 0); code = tdbTbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pSkmDb // open pSkmDb
ret = tdbTbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb, 0); code = tdbTbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pUidIdx // open pUidIdx
ret = tdbTbOpen("uid.idx", sizeof(tb_uid_t), sizeof(SUidIdxVal), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx, 0); code = tdbTbOpen("uid.idx", sizeof(tb_uid_t), sizeof(SUidIdxVal), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pNameIdx // open pNameIdx
ret = tdbTbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx, 0); code = tdbTbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pCtbIdx // open pCtbIdx
ret = tdbTbOpen("ctb.idx", sizeof(SCtbIdxKey), -1, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx, 0); code = tdbTbOpen("ctb.idx", sizeof(SCtbIdxKey), -1, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pSuidIdx // open pSuidIdx
ret = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx, 0); code = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta super table index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
char indexFullPath[128] = {0};
sprintf(indexFullPath, "%s/%s", pMeta->path, "invert"); sprintf(indexFullPath, "%s/%s", pMeta->path, "invert");
taosMkDir(indexFullPath); TAOS_UNUSED(taosMkDir(indexFullPath));
SIndexOpts opts = {.cacheSize = 8 * 1024 * 1024}; SIndexOpts opts = {.cacheSize = 8 * 1024 * 1024};
ret = indexOpen(&opts, indexFullPath, (SIndex **)&pMeta->pTagIvtIdx); code = indexOpen(&opts, indexFullPath, (SIndex **)&pMeta->pTagIvtIdx);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
ret = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx, 0); code = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pTtlMgr ("ttlv1.idx") // open pTtlMgr ("ttlv1.idx")
char logPrefix[128] = {0}; char logPrefix[128] = {0};
sprintf(logPrefix, "vgId:%d", TD_VID(pVnode)); sprintf(logPrefix, "vgId:%d", TD_VID(pVnode));
ret = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0, logPrefix, tsTtlFlushThreshold); code = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0, logPrefix, tsTtlFlushThreshold);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pSmaIdx // open pSmaIdx
ret = tdbTbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx, 0); code = tdbTbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// idx table create time // idx table create time
ret = tdbTbOpen("ctime.idx", sizeof(SBtimeIdxKey), 0, btimeIdxCmpr, pMeta->pEnv, &pMeta->pBtimeIdx, 0); code = tdbTbOpen("ctime.idx", sizeof(SBtimeIdxKey), 0, btimeIdxCmpr, pMeta->pEnv, &pMeta->pBtimeIdx, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta ctime index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// idx num of col, normal table only // idx num of col, normal table only
ret = tdbTbOpen("ncol.idx", sizeof(SNcolIdxKey), 0, ncolIdxCmpr, pMeta->pEnv, &pMeta->pNcolIdx, 0); code = tdbTbOpen("ncol.idx", sizeof(SNcolIdxKey), 0, ncolIdxCmpr, pMeta->pEnv, &pMeta->pNcolIdx, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta ncol index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
ret = tdbTbOpen("stream.task.db", sizeof(int64_t), -1, taskIdxKeyCmpr, pMeta->pEnv, &pMeta->pStreamDb, 0); code = tdbTbOpen("stream.task.db", sizeof(int64_t), -1, taskIdxKeyCmpr, pMeta->pEnv, &pMeta->pStreamDb, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta stream task index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open index code = metaCacheOpen(pMeta);
if (metaOpenIdx(pMeta) < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
int32_t code = metaCacheOpen(pMeta); code = metaInitTbFilterCache(pMeta);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) { if (code) {
terrno = code; metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, __LINE__, tstrerror(code));
metaError("vgId:%d, failed to open meta cache since %s", TD_VID(pVnode), tstrerror(terrno)); metaCleanup(&pMeta);
goto _err; *ppMeta = NULL;
} else {
metaDebug("vgId:%d %s success", TD_VID(pVnode), __func__);
*ppMeta = pMeta;
} }
return code;
if (metaInitTbFilterCache(pMeta) != 0) {
goto _err;
}
metaDebug("vgId:%d, meta is opened", TD_VID(pVnode));
*ppMeta = pMeta;
return 0;
_err:
metaCleanup(&pMeta);
return -1;
} }
int metaUpgrade(SVnode *pVnode, SMeta **ppMeta) { int32_t metaUpgrade(SVnode *pVnode, SMeta **ppMeta) {
int code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SMeta *pMeta = *ppMeta; int32_t lino;
SMeta *pMeta = *ppMeta;
if (ttlMgrNeedUpgrade(pMeta->pEnv)) { if (ttlMgrNeedUpgrade(pMeta->pEnv)) {
code = metaBegin(pMeta, META_BEGIN_HEAP_OS); code = metaBegin(pMeta, META_BEGIN_HEAP_OS);
if (code < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to upgrade meta, meta begin failed since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = ttlMgrUpgrade(pMeta->pTtlMgr, pMeta); code = ttlMgrUpgrade(pMeta->pTtlMgr, pMeta);
if (code < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to upgrade meta ttl since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = metaCommit(pMeta, pMeta->txn); code = metaCommit(pMeta, pMeta->txn);
if (code < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to upgrade meta ttl, meta commit failed since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
} }
return TSDB_CODE_SUCCESS; _exit:
if (code) {
_err: metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, __LINE__, tstrerror(code));
metaCleanup(ppMeta); metaCleanup(ppMeta);
}
return code; return code;
} }
@ -235,33 +175,42 @@ int metaClose(SMeta **ppMeta) {
} }
int metaAlterCache(SMeta *pMeta, int32_t nPage) { int metaAlterCache(SMeta *pMeta, int32_t nPage) {
int32_t code = 0;
metaWLock(pMeta); metaWLock(pMeta);
code = tdbAlter(pMeta->pEnv, nPage);
if (tdbAlter(pMeta->pEnv, nPage) < 0) {
metaULock(pMeta);
return -1;
}
metaULock(pMeta); metaULock(pMeta);
return 0;
if (code) {
metaError("vgId:%d %s failed since %s", TD_VID(pMeta->pVnode), __func__, tstrerror(code));
}
return code;
} }
int32_t metaRLock(SMeta *pMeta) { int32_t metaRLock(SMeta *pMeta) {
metaTrace("meta rlock %p", &pMeta->lock); metaTrace("meta rlock %p", &pMeta->lock);
int32_t ret = taosThreadRwlockRdlock(&pMeta->lock); int32_t code = taosThreadRwlockRdlock(&pMeta->lock);
return ret; if (code) {
return TAOS_SYSTEM_ERROR(code);
}
return 0;
} }
int32_t metaWLock(SMeta *pMeta) { int32_t metaWLock(SMeta *pMeta) {
metaTrace("meta wlock %p", &pMeta->lock); metaTrace("meta wlock %p", &pMeta->lock);
int32_t ret = taosThreadRwlockWrlock(&pMeta->lock); int32_t code = taosThreadRwlockWrlock(&pMeta->lock);
return ret; if (code) {
return TAOS_SYSTEM_ERROR(code);
}
return 0;
} }
int32_t metaULock(SMeta *pMeta) { int32_t metaULock(SMeta *pMeta) {
metaTrace("meta ulock %p", &pMeta->lock); metaTrace("meta ulock %p", &pMeta->lock);
int32_t ret = taosThreadRwlockUnlock(&pMeta->lock); int32_t code = taosThreadRwlockUnlock(&pMeta->lock);
return ret; if (code) {
return TAOS_SYSTEM_ERROR(code);
}
return 0;
} }
static void metaCleanup(SMeta **ppMeta) { static void metaCleanup(SMeta **ppMeta) {

View File

@ -1542,9 +1542,14 @@ int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables, int32_t
// slow path: search TDB // slow path: search TDB
int64_t ctbNum = 0; int64_t ctbNum = 0;
int32_t colNum = 0; int32_t colNum = 0;
vnodeGetCtbNum(pVnode, uid, &ctbNum); code = vnodeGetCtbNum(pVnode, uid, &ctbNum);
vnodeGetStbColumnNum(pVnode, uid, &colNum); if (TSDB_CODE_SUCCESS == code) {
code = vnodeGetStbColumnNum(pVnode, uid, &colNum);
}
metaULock(pVnodeObj->pMeta); metaULock(pVnodeObj->pMeta);
if (TSDB_CODE_SUCCESS != code) {
goto _exit;
}
if (numOfTables) *numOfTables = ctbNum; if (numOfTables) *numOfTables = ctbNum;
if (numOfCols) *numOfCols = colNum; if (numOfCols) *numOfCols = colNum;

View File

@ -25,14 +25,14 @@ struct SMetaSnapReader {
int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader) { int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
int32_t c = 0; int32_t c = 0;
SMetaSnapReader* pReader = NULL; SMetaSnapReader* pReader = NULL;
// alloc // alloc
pReader = (SMetaSnapReader*)taosMemoryCalloc(1, sizeof(*pReader)); pReader = (SMetaSnapReader*)taosMemoryCalloc(1, sizeof(*pReader));
if (pReader == NULL) { if (pReader == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
pReader->pMeta = pMeta; pReader->pMeta = pMeta;
pReader->sver = sver; pReader->sver = sver;
@ -40,36 +40,29 @@ int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapRe
// impl // impl
code = tdbTbcOpen(pMeta->pTbDb, &pReader->pTbc, NULL); code = tdbTbcOpen(pMeta->pTbDb, &pReader->pTbc, NULL);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
taosMemoryFree(pReader);
goto _err;
}
code = tdbTbcMoveTo(pReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c); code = tdbTbcMoveTo(pReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) { if (code) {
taosMemoryFree(pReader); metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code));
goto _err; metaSnapReaderClose(&pReader);
*ppReader = NULL;
} else {
metaInfo("vgId:%d, %s success", TD_VID(pMeta->pVnode), __func__);
*ppReader = pReader;
} }
metaInfo("vgId:%d, vnode snapshot meta reader opened", TD_VID(pMeta->pVnode));
*ppReader = pReader;
return code;
_err:
metaError("vgId:%d, vnode snapshot meta reader open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
*ppReader = NULL;
return code; return code;
} }
int32_t metaSnapReaderClose(SMetaSnapReader** ppReader) { void metaSnapReaderClose(SMetaSnapReader** ppReader) {
int32_t code = 0; if (ppReader && *ppReader) {
tdbTbcClose((*ppReader)->pTbc);
tdbTbcClose((*ppReader)->pTbc); taosMemoryFree(*ppReader);
taosMemoryFree(*ppReader); *ppReader = NULL;
*ppReader = NULL; }
return code;
} }
int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) {
@ -106,7 +99,7 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) {
*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + nData); *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + nData);
if (*ppData == NULL) { if (*ppData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _exit;
} }
SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData); SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData);
@ -122,10 +115,10 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) {
} }
_exit: _exit:
return code; if (code) {
metaError("vgId:%d, vnode snapshot meta read data failed since %s", TD_VID(pReader->pMeta->pVnode),
_err: tstrerror(code));
metaError("vgId:%d, vnode snapshot meta read data failed since %s", TD_VID(pReader->pMeta->pVnode), tstrerror(code)); }
return code; return code;
} }
@ -138,26 +131,30 @@ struct SMetaSnapWriter {
int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter) { int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
SMetaSnapWriter* pWriter; SMetaSnapWriter* pWriter;
// alloc // alloc
pWriter = (SMetaSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter)); pWriter = (SMetaSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter));
if (pWriter == NULL) { if (pWriter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
pWriter->pMeta = pMeta; pWriter->pMeta = pMeta;
pWriter->sver = sver; pWriter->sver = sver;
pWriter->ever = ever; pWriter->ever = ever;
metaBegin(pMeta, META_BEGIN_HEAP_NIL); code = metaBegin(pMeta, META_BEGIN_HEAP_NIL);
TSDB_CHECK_CODE(code, lino, _exit);
*ppWriter = pWriter; _exit:
return code; if (code) {
metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code));
_err: taosMemoryFree(pWriter);
metaError("vgId:%d, meta snapshot writer open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code)); *ppWriter = NULL;
*ppWriter = NULL; } else {
metaDebug("vgId:%d, %s success", TD_VID(pMeta->pVnode), __func__);
*ppWriter = pWriter;
}
return code; return code;
} }
@ -189,25 +186,24 @@ _err:
int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t code = 0; int32_t code = 0;
int32_t line = 0; int32_t lino = 0;
SMeta* pMeta = pWriter->pMeta; SMeta* pMeta = pWriter->pMeta;
SMetaEntry metaEntry = {0}; SMetaEntry metaEntry = {0};
SDecoder* pDecoder = &(SDecoder){0}; SDecoder* pDecoder = &(SDecoder){0};
tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
code = metaDecodeEntry(pDecoder, &metaEntry); code = metaDecodeEntry(pDecoder, &metaEntry);
VND_CHECK_CODE(code, line, _err); TSDB_CHECK_CODE(code, lino, _exit);
code = metaHandleEntry(pMeta, &metaEntry); code = metaHandleEntry(pMeta, &metaEntry);
VND_CHECK_CODE(code, line, _err); TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code));
}
tDecoderClear(pDecoder); tDecoderClear(pDecoder);
return code; return code;
_err:
tDecoderClear(pDecoder);
metaError("vgId:%d, vnode snapshot meta write failed since %s at line:%d", TD_VID(pMeta->pVnode), terrstr(), line);
return code;
} }
typedef struct STableInfoForChildTable { typedef struct STableInfoForChildTable {
@ -468,19 +464,17 @@ int32_t setForSnapShot(SSnapContext* ctx, int64_t uid) {
return c; return c;
} }
void taosXSetTablePrimaryKey(SSnapContext* ctx, int64_t uid){ void taosXSetTablePrimaryKey(SSnapContext* ctx, int64_t uid) {
bool ret = false; bool ret = false;
SSchemaWrapper *schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1); SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1);
if (schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY){ if (schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) {
ret = true; ret = true;
} }
tDeleteSchemaWrapper(schema); tDeleteSchemaWrapper(schema);
ctx->hasPrimaryKey = ret; ctx->hasPrimaryKey = ret;
} }
bool taosXGetTablePrimaryKey(SSnapContext* ctx){ bool taosXGetTablePrimaryKey(SSnapContext* ctx) { return ctx->hasPrimaryKey; }
return ctx->hasPrimaryKey;
}
int32_t getTableInfoFromSnapshot(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid) { int32_t getTableInfoFromSnapshot(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid) {
int32_t ret = 0; int32_t ret = 0;

View File

@ -757,10 +757,9 @@ int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *m
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// decode // decode
int64_t n = tGetMapData(pReader->aBuf[0], mDataBlk); int32_t n;
if (n < 0) { code = tGetMapData(pReader->aBuf[0], mDataBlk, &n);
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); if (code) goto _exit;
}
ASSERT(n == size); ASSERT(n == size);
_exit: _exit:

View File

@ -81,7 +81,7 @@ static int32_t tsdbFTypeToFRangeType(tsdb_ftype_t ftype) {
static int32_t tsdbTFileSetToFSetPartition(STFileSet* fset, STsdbFSetPartition** ppSP) { static int32_t tsdbTFileSetToFSetPartition(STFileSet* fset, STsdbFSetPartition** ppSP) {
STsdbFSetPartition* p = tsdbFSetPartitionCreate(); STsdbFSetPartition* p = tsdbFSetPartitionCreate();
if (p == NULL) { if (p == NULL) {
goto _err; return terrno;
} }
p->fid = fset->fid; p->fid = fset->fid;
@ -134,10 +134,6 @@ static int32_t tsdbTFileSetToFSetPartition(STFileSet* fset, STsdbFSetPartition**
} }
ppSP[0] = p; ppSP[0] = p;
return 0; return 0;
_err:
tsdbFSetPartitionClear(&p);
return -1;
} }
// fset partition list // fset partition list
@ -160,9 +156,11 @@ void tsdbFSetPartListDestroy(STsdbFSetPartList** ppList) {
} }
int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList* pList, TFileSetRangeArray** ppRanges) { int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList* pList, TFileSetRangeArray** ppRanges) {
int32_t code = 0;
TFileSetRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TFileSetRangeArray)); TFileSetRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TFileSetRangeArray));
if (pDiff == NULL) { if (pDiff == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
TARRAY2_INIT(pDiff); TARRAY2_INIT(pDiff);
@ -171,7 +169,7 @@ int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList* pList, TFileSetRangeArray
TARRAY2_FOREACH(pList, part) { TARRAY2_FOREACH(pList, part) {
STFileSetRange* r = taosMemoryCalloc(1, sizeof(STFileSetRange)); STFileSetRange* r = taosMemoryCalloc(1, sizeof(STFileSetRange));
if (r == NULL) { if (r == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
int64_t maxVerValid = -1; int64_t maxVerValid = -1;
@ -202,7 +200,7 @@ _err:
if (pDiff) { if (pDiff) {
tsdbTFileSetRangeArrayDestroy(&pDiff); tsdbTFileSetRangeArrayDestroy(&pDiff);
} }
return -1; return code;
} }
// serialization // serialization
@ -235,96 +233,103 @@ int32_t tTsdbFSetPartListDataLenCalc(STsdbFSetPartList* pList) {
return datLen; return datLen;
} }
int32_t tSerializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* pList) { static int32_t tSerializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* pList, int32_t* encodeSize) {
SEncoder encoder = {0}; SEncoder encoder = {0};
int8_t reserved8 = 0;
int16_t reserved16 = 0;
int64_t reserved64 = 0;
int8_t msgVer = TSDB_SNAP_MSG_VER;
int32_t len = TARRAY2_SIZE(pList);
int32_t code = 0;
tEncoderInit(&encoder, buf, bufLen); tEncoderInit(&encoder, buf, bufLen);
if ((code = tStartEncode(&encoder))) goto _exit;
int8_t reserved8 = 0; if ((code = tEncodeI8(&encoder, msgVer))) goto _exit;
int16_t reserved16 = 0; if ((code = tEncodeI32(&encoder, len))) goto _exit;
int64_t reserved64 = 0;
int8_t msgVer = TSDB_SNAP_MSG_VER;
int32_t len = TARRAY2_SIZE(pList);
if (tStartEncode(&encoder) < 0) goto _err;
if (tEncodeI8(&encoder, msgVer) < 0) goto _err;
if (tEncodeI32(&encoder, len) < 0) goto _err;
for (int32_t u = 0; u < len; u++) { for (int32_t u = 0; u < len; u++) {
STsdbFSetPartition* p = TARRAY2_GET(pList, u); STsdbFSetPartition* p = TARRAY2_GET(pList, u);
if (tEncodeI64(&encoder, p->fid) < 0) goto _err; if ((code = tEncodeI64(&encoder, p->fid))) goto _exit;
if (tEncodeI8(&encoder, p->stat) < 0) goto _err; if ((code = tEncodeI8(&encoder, p->stat))) goto _exit;
if (tEncodeI8(&encoder, reserved8) < 0) goto _err; if ((code = tEncodeI8(&encoder, reserved8))) goto _exit;
if (tEncodeI16(&encoder, reserved16) < 0) goto _err; if ((code = tEncodeI16(&encoder, reserved16))) goto _exit;
int32_t typMax = TSDB_FSET_RANGE_TYP_MAX; int32_t typMax = TSDB_FSET_RANGE_TYP_MAX;
if (tEncodeI32(&encoder, typMax) < 0) goto _err; if ((code = tEncodeI32(&encoder, typMax))) goto _exit;
for (int32_t i = 0; i < typMax; i++) { for (int32_t i = 0; i < typMax; i++) {
SVerRangeList* iList = &p->verRanges[i]; SVerRangeList* iList = &p->verRanges[i];
int32_t iLen = TARRAY2_SIZE(iList); int32_t iLen = TARRAY2_SIZE(iList);
if (tEncodeI32(&encoder, iLen) < 0) goto _err; if ((code = tEncodeI32(&encoder, iLen))) goto _exit;
for (int32_t j = 0; j < iLen; j++) { for (int32_t j = 0; j < iLen; j++) {
SVersionRange r = TARRAY2_GET(iList, j); SVersionRange r = TARRAY2_GET(iList, j);
if (tEncodeI64(&encoder, r.minVer) < 0) goto _err; if ((code = tEncodeI64(&encoder, r.minVer))) goto _exit;
if (tEncodeI64(&encoder, r.maxVer) < 0) goto _err; if ((code = tEncodeI64(&encoder, r.maxVer))) goto _exit;
if (tEncodeI64(&encoder, reserved64) < 0) goto _err; if ((code = tEncodeI64(&encoder, reserved64))) goto _exit;
} }
} }
} }
tEndEncode(&encoder); tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
_err: if (encodeSize) {
encodeSize[0] = encoder.pos;
}
_exit:
tEncoderClear(&encoder); tEncoderClear(&encoder);
return -1; return code;
} }
int32_t tDeserializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* pList) { int32_t tDeserializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* pList) {
SDecoder decoder = {0}; SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen); int8_t reserved8 = 0;
int16_t reserved16 = 0;
int8_t reserved8 = 0; int64_t reserved64 = 0;
int16_t reserved16 = 0; int32_t code = 0;
int64_t reserved64 = 0;
STsdbFSetPartition* p = NULL; STsdbFSetPartition* p = NULL;
tDecoderInit(&decoder, buf, bufLen);
int8_t msgVer = 0; int8_t msgVer = 0;
int32_t len = 0; int32_t len = 0;
if (tStartDecode(&decoder) < 0) goto _err; if ((code = tStartDecode(&decoder))) goto _err;
if (tDecodeI8(&decoder, &msgVer) < 0) goto _err; if ((code = tDecodeI8(&decoder, &msgVer))) goto _err;
if (msgVer != TSDB_SNAP_MSG_VER) goto _err; if (msgVer != TSDB_SNAP_MSG_VER) {
if (tDecodeI32(&decoder, &len) < 0) goto _err; code = TSDB_CODE_INVALID_MSG;
goto _err;
}
if ((code = tDecodeI32(&decoder, &len))) goto _err;
for (int32_t u = 0; u < len; u++) { for (int32_t u = 0; u < len; u++) {
p = tsdbFSetPartitionCreate(); p = tsdbFSetPartitionCreate();
if (p == NULL) goto _err; if (p == NULL) {
if (tDecodeI64(&decoder, &p->fid) < 0) goto _err; code = terrno;
if (tDecodeI8(&decoder, &p->stat) < 0) goto _err; goto _err;
if (tDecodeI8(&decoder, &reserved8) < 0) goto _err; }
if (tDecodeI16(&decoder, &reserved16) < 0) goto _err;
if ((code = tDecodeI64(&decoder, &p->fid))) goto _err;
if ((code = tDecodeI8(&decoder, &p->stat))) goto _err;
if ((code = tDecodeI8(&decoder, &reserved8))) goto _err;
if ((code = tDecodeI16(&decoder, &reserved16))) goto _err;
int32_t typMax = 0; int32_t typMax = 0;
if (tDecodeI32(&decoder, &typMax) < 0) goto _err; if ((code = tDecodeI32(&decoder, &typMax))) goto _err;
for (int32_t i = 0; i < typMax; i++) { for (int32_t i = 0; i < typMax; i++) {
SVerRangeList* iList = &p->verRanges[i]; SVerRangeList* iList = &p->verRanges[i];
int32_t iLen = 0; int32_t iLen = 0;
if (tDecodeI32(&decoder, &iLen) < 0) goto _err; if ((code = tDecodeI32(&decoder, &iLen))) goto _err;
for (int32_t j = 0; j < iLen; j++) { for (int32_t j = 0; j < iLen; j++) {
SVersionRange r = {0}; SVersionRange r = {0};
if (tDecodeI64(&decoder, &r.minVer) < 0) goto _err; if ((code = tDecodeI64(&decoder, &r.minVer))) goto _err;
if (tDecodeI64(&decoder, &r.maxVer) < 0) goto _err; if ((code = tDecodeI64(&decoder, &r.maxVer))) goto _err;
if (tDecodeI64(&decoder, &reserved64) < 0) goto _err; if ((code = tDecodeI64(&decoder, &reserved64))) goto _err;
if (TARRAY2_APPEND(iList, r)) goto _err; if ((code = TARRAY2_APPEND(iList, r))) goto _err;
} }
} }
if (TARRAY2_APPEND(pList, p)) goto _err; if ((code = TARRAY2_APPEND(pList, p))) goto _err;
p = NULL; p = NULL;
} }
@ -337,7 +342,7 @@ _err:
tsdbFSetPartitionClear(&p); tsdbFSetPartitionClear(&p);
} }
tDecoderClear(&decoder); tDecoderClear(&decoder);
return -1; return code;
} }
// fs state // fs state
@ -352,8 +357,9 @@ static STsdbFSetPartList* tsdbSnapGetFSetPartList(STFileSystem* fs) {
STFileSet* fset; STFileSet* fset;
TARRAY2_FOREACH(fs->fSetArr, fset) { TARRAY2_FOREACH(fs->fSetArr, fset) {
STsdbFSetPartition* pItem = NULL; STsdbFSetPartition* pItem = NULL;
if (tsdbTFileSetToFSetPartition(fset, &pItem) < 0) { code = tsdbTFileSetToFSetPartition(fset, &pItem);
code = -1; if (code) {
terrno = code;
break; break;
} }
ASSERT(pItem != NULL); ASSERT(pItem != NULL);
@ -404,7 +410,9 @@ static int32_t tsdbPartitionInfoInit(SVnode* pVnode, STsdbPartitionInfo* pInfo)
for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) { for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) {
STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, j); STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, j);
pInfo->pLists[j] = tsdbSnapGetFSetPartList(pTsdb->pFS); pInfo->pLists[j] = tsdbSnapGetFSetPartList(pTsdb->pFS);
if (pInfo->pLists[j] == NULL) return -1; if (pInfo->pLists[j] == NULL) {
return terrno;
}
} }
return 0; return 0;
} }
@ -432,9 +440,10 @@ static int32_t tsdbPartitionInfoSerialize(STsdbPartitionInfo* pInfo, uint8_t* bu
SSyncTLV* pSubHead = (void*)((char*)buf + offset); SSyncTLV* pSubHead = (void*)((char*)buf + offset);
int32_t valOffset = offset + sizeof(*pSubHead); int32_t valOffset = offset + sizeof(*pSubHead);
ASSERT(pSubHead->val == (char*)buf + valOffset); ASSERT(pSubHead->val == (char*)buf + valOffset);
if ((tlen = tSerializeTsdbFSetPartList(pSubHead->val, bufLen - valOffset, pInfo->pLists[j])) < 0) { int32_t code = tSerializeTsdbFSetPartList(pSubHead->val, bufLen - valOffset, pInfo->pLists[j], &tlen);
if (code) {
tsdbError("vgId:%d, failed to serialize fset partition list of tsdb %d since %s", pInfo->vgId, j, terrstr()); tsdbError("vgId:%d, failed to serialize fset partition list of tsdb %d since %s", pInfo->vgId, j, terrstr());
return -1; return code;
} }
pSubHead->typ = pInfo->subTyps[j]; pSubHead->typ = pInfo->subTyps[j];
pSubHead->len = tlen; pSubHead->len = tlen;
@ -460,17 +469,18 @@ static int32_t tTsdbRepOptsDataLenCalc(STsdbRepOpts* pInfo) {
} }
int32_t tSerializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) { int32_t tSerializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) {
int32_t code = 0;
SEncoder encoder = {0}; SEncoder encoder = {0};
int64_t reserved64 = 0;
int8_t msgVer = TSDB_SNAP_MSG_VER;
tEncoderInit(&encoder, buf, bufLen); tEncoderInit(&encoder, buf, bufLen);
int64_t reserved64 = 0; if ((code = tStartEncode(&encoder))) goto _err;
int8_t msgVer = TSDB_SNAP_MSG_VER; if ((code = tEncodeI8(&encoder, msgVer))) goto _err;
if (tStartEncode(&encoder) < 0) goto _err;
if (tEncodeI8(&encoder, msgVer) < 0) goto _err;
int16_t format = pOpts->format; int16_t format = pOpts->format;
if (tEncodeI16(&encoder, format) < 0) goto _err; if ((code = tEncodeI16(&encoder, format))) goto _err;
if (tEncodeI64(&encoder, reserved64) < 0) goto _err; if ((code = tEncodeI64(&encoder, reserved64))) goto _err;
tEndEncode(&encoder); tEndEncode(&encoder);
int32_t tlen = encoder.pos; int32_t tlen = encoder.pos;
@ -479,23 +489,24 @@ int32_t tSerializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) {
_err: _err:
tEncoderClear(&encoder); tEncoderClear(&encoder);
return -1; return code;
} }
int32_t tDeserializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) { int32_t tDeserializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) {
int32_t code;
SDecoder decoder = {0}; SDecoder decoder = {0};
int64_t reserved64 = 0;
int8_t msgVer = 0;
tDecoderInit(&decoder, buf, bufLen); tDecoderInit(&decoder, buf, bufLen);
int64_t reserved64 = 0; if ((code = tStartDecode(&decoder))) goto _err;
int8_t msgVer = 0; if ((code = tDecodeI8(&decoder, &msgVer))) goto _err;
if (tStartDecode(&decoder) < 0) goto _err;
if (tDecodeI8(&decoder, &msgVer) < 0) goto _err;
if (msgVer != TSDB_SNAP_MSG_VER) goto _err; if (msgVer != TSDB_SNAP_MSG_VER) goto _err;
int16_t format = 0; int16_t format = 0;
if (tDecodeI16(&decoder, &format) < 0) goto _err; if ((code = tDecodeI16(&decoder, &format))) goto _err;
pOpts->format = format; pOpts->format = format;
if (tDecodeI64(&decoder, &reserved64) < 0) goto _err; if ((code = tDecodeI64(&decoder, &reserved64))) goto _err;
tEndDecode(&decoder); tEndDecode(&decoder);
tDecoderClear(&decoder); tDecoderClear(&decoder);
@ -503,7 +514,7 @@ int32_t tDeserializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts)
_err: _err:
tDecoderClear(&decoder); tDecoderClear(&decoder);
return -1; return code;
} }
static int32_t tsdbRepOptsEstSize(STsdbRepOpts* pOpts) { static int32_t tsdbRepOptsEstSize(STsdbRepOpts* pOpts) {
@ -518,7 +529,7 @@ static int32_t tsdbRepOptsSerialize(STsdbRepOpts* pOpts, void* buf, int32_t bufL
int32_t offset = 0; int32_t offset = 0;
int32_t tlen = 0; int32_t tlen = 0;
if ((tlen = tSerializeTsdbRepOpts(pSubHead->val, bufLen, pOpts)) < 0) { if ((tlen = tSerializeTsdbRepOpts(pSubHead->val, bufLen, pOpts)) < 0) {
return -1; return tlen;
} }
pSubHead->typ = SNAP_DATA_RAW; pSubHead->typ = SNAP_DATA_RAW;
pSubHead->len = tlen; pSubHead->len = tlen;
@ -528,8 +539,10 @@ static int32_t tsdbRepOptsSerialize(STsdbRepOpts* pOpts, void* buf, int32_t bufL
// snap info // snap info
static int32_t tsdbSnapPrepDealWithSnapInfo(SVnode* pVnode, SSnapshot* pSnap, STsdbRepOpts* pInfo) { static int32_t tsdbSnapPrepDealWithSnapInfo(SVnode* pVnode, SSnapshot* pSnap, STsdbRepOpts* pInfo) {
if (!pSnap->data) return 0; if (!pSnap->data) {
int32_t code = -1; return 0;
}
int32_t code = 0;
SSyncTLV* pHead = (void*)pSnap->data; SSyncTLV* pHead = (void*)pSnap->data;
int32_t offset = 0; int32_t offset = 0;
@ -546,30 +559,30 @@ static int32_t tsdbSnapPrepDealWithSnapInfo(SVnode* pVnode, SSnapshot* pSnap, ST
case SNAP_DATA_RSMA2: { case SNAP_DATA_RSMA2: {
} break; } break;
case SNAP_DATA_RAW: { case SNAP_DATA_RAW: {
if (tDeserializeTsdbRepOpts(buf, bufLen, pInfo) < 0) { code = tDeserializeTsdbRepOpts(buf, bufLen, pInfo);
terrno = TSDB_CODE_INVALID_DATA_FMT; if (code < 0) {
tsdbError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr()); tsdbError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr());
goto _out; return code;
} }
} break; } break;
default: default:
code = TSDB_CODE_INVALID_MSG;
tsdbError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), pField->typ); tsdbError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), pField->typ);
goto _out; return code;
} }
} }
code = 0;
_out:
return code; return code;
} }
int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) {
ASSERT(pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY); ASSERT(pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY);
STsdbPartitionInfo partitionInfo = {0}; STsdbPartitionInfo partitionInfo = {0};
int code = -1; int code = 0;
STsdbPartitionInfo* pInfo = &partitionInfo; STsdbPartitionInfo* pInfo = &partitionInfo;
if (tsdbPartitionInfoInit(pVnode, pInfo) != 0) { code = tsdbPartitionInfoInit(pVnode, pInfo);
if (code) {
goto _out; goto _out;
} }
@ -577,7 +590,7 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) {
STsdbRepOpts opts = {.format = TSDB_SNAP_REP_FMT_RAW}; STsdbRepOpts opts = {.format = TSDB_SNAP_REP_FMT_RAW};
if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
STsdbRepOpts leaderOpts = {0}; STsdbRepOpts leaderOpts = {0};
if (tsdbSnapPrepDealWithSnapInfo(pVnode, pSnap, &leaderOpts) < 0) { if ((code = tsdbSnapPrepDealWithSnapInfo(pVnode, pSnap, &leaderOpts)) < 0) {
tsdbError("vgId:%d, failed to deal with snap info for reply since %s", TD_VID(pVnode), terrstr()); tsdbError("vgId:%d, failed to deal with snap info for reply since %s", TD_VID(pVnode), terrstr());
goto _out; goto _out;
} }
@ -589,7 +602,7 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) {
int32_t bufLen = headLen; int32_t bufLen = headLen;
bufLen += tsdbPartitionInfoEstSize(pInfo); bufLen += tsdbPartitionInfoEstSize(pInfo);
bufLen += tsdbRepOptsEstSize(&opts); bufLen += tsdbRepOptsEstSize(&opts);
if (syncSnapInfoDataRealloc(pSnap, bufLen) != 0) { if ((code = syncSnapInfoDataRealloc(pSnap, bufLen)) != 0) {
tsdbError("vgId:%d, failed to realloc memory for data of snap info. bytes:%d", TD_VID(pVnode), bufLen); tsdbError("vgId:%d, failed to realloc memory for data of snap info. bytes:%d", TD_VID(pVnode), bufLen);
goto _out; goto _out;
} }
@ -599,7 +612,8 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) {
int32_t offset = headLen; int32_t offset = headLen;
int32_t tlen = 0; int32_t tlen = 0;
if ((tlen = tsdbPartitionInfoSerialize(pInfo, buf + offset, bufLen - offset)) < 0) { if ((tlen = tsdbPartitionInfoSerialize(pInfo, (uint8_t*)(buf + offset), bufLen - offset)) < 0) {
code = tlen;
tsdbError("vgId:%d, failed to serialize tsdb partition info since %s", TD_VID(pVnode), terrstr()); tsdbError("vgId:%d, failed to serialize tsdb partition info since %s", TD_VID(pVnode), terrstr());
goto _out; goto _out;
} }
@ -607,6 +621,7 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) {
ASSERT(offset <= bufLen); ASSERT(offset <= bufLen);
if ((tlen = tsdbRepOptsSerialize(&opts, buf + offset, bufLen - offset)) < 0) { if ((tlen = tsdbRepOptsSerialize(&opts, buf + offset, bufLen - offset)) < 0) {
code = tlen;
tsdbError("vgId:%d, failed to serialize tsdb rep opts since %s", TD_VID(pVnode), terrstr()); tsdbError("vgId:%d, failed to serialize tsdb rep opts since %s", TD_VID(pVnode), terrstr());
goto _out; goto _out;
} }
@ -620,7 +635,7 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) {
tsdbInfo("vgId:%d, tsdb snap info prepared. type:%s, val length:%d", TD_VID(pVnode), TMSG_INFO(pHead->typ), tsdbInfo("vgId:%d, tsdb snap info prepared. type:%s, val length:%d", TD_VID(pVnode), TMSG_INFO(pHead->typ),
pHead->len); pHead->len);
code = 0;
_out: _out:
tsdbPartitionInfoClear(pInfo); tsdbPartitionInfoClear(pInfo);
return code; return code;

View File

@ -152,7 +152,7 @@ int32_t tPutMapData(uint8_t *p, SMapData *pMapData) {
} }
#endif #endif
int32_t tGetMapData(uint8_t *p, SMapData *pMapData) { int32_t tGetMapData(uint8_t *p, SMapData *pMapData, int32_t *decodedSize) {
int32_t n = 0; int32_t n = 0;
int32_t offset; int32_t offset;
@ -160,7 +160,9 @@ int32_t tGetMapData(uint8_t *p, SMapData *pMapData) {
n += tGetI32v(p + n, &pMapData->nItem); n += tGetI32v(p + n, &pMapData->nItem);
if (pMapData->nItem) { if (pMapData->nItem) {
if (tRealloc((uint8_t **)&pMapData->aOffset, sizeof(int32_t) * pMapData->nItem)) return -1; if (tRealloc((uint8_t **)&pMapData->aOffset, sizeof(int32_t) * pMapData->nItem)) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t lOffset = 0; int32_t lOffset = 0;
for (int32_t iItem = 0; iItem < pMapData->nItem; iItem++) { for (int32_t iItem = 0; iItem < pMapData->nItem; iItem++) {
@ -170,12 +172,18 @@ int32_t tGetMapData(uint8_t *p, SMapData *pMapData) {
} }
n += tGetI32v(p + n, &pMapData->nData); n += tGetI32v(p + n, &pMapData->nData);
if (tRealloc(&pMapData->pData, pMapData->nData)) return -1; if (tRealloc(&pMapData->pData, pMapData->nData)) {
return TSDB_CODE_OUT_OF_MEMORY;
}
memcpy(pMapData->pData, p + n, pMapData->nData); memcpy(pMapData->pData, p + n, pMapData->nData);
n += pMapData->nData; n += pMapData->nData;
} }
return n; if (decodedSize) {
*decodedSize = n;
}
return 0;
} }
#ifdef BUILD_NO_CALL #ifdef BUILD_NO_CALL
@ -680,20 +688,17 @@ int32_t tsdbRowCompareWithoutVersion(const void *p1, const void *p2) {
// STSDBRowIter ====================================================== // STSDBRowIter ======================================================
int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) {
int32_t code = 0;
pIter->pRow = pRow; pIter->pRow = pRow;
if (pRow->type == TSDBROW_ROW_FMT) { if (pRow->type == TSDBROW_ROW_FMT) {
code = tRowIterOpen(pRow->pTSRow, pTSchema, &pIter->pIter); int32_t code = tRowIterOpen(pRow->pTSRow, pTSchema, &pIter->pIter);
if (code) goto _exit; if (code) return code;
} else if (pRow->type == TSDBROW_COL_FMT) { } else if (pRow->type == TSDBROW_COL_FMT) {
pIter->iColData = 0; pIter->iColData = 0;
} else { } else {
ASSERT(0); ASSERT(0);
} }
_exit: return 0;
return code;
} }
void tsdbRowClose(STSDBRowIter *pIter) { void tsdbRowClose(STSDBRowIter *pIter) {
@ -760,7 +765,9 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema)
--iCol; --iCol;
continue; continue;
} else if (pTSchema->columns[jCol].colId > pTColumn->colId) { } else if (pTSchema->columns[jCol].colId > pTColumn->colId) {
taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type)); if (taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type)) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
continue; continue;
} }
@ -780,14 +787,15 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema)
} }
if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return code;
} }
} }
for (; iCol < pMerger->pTSchema->numOfCols; ++iCol) { for (; iCol < pMerger->pTSchema->numOfCols; ++iCol) {
pTColumn = &pMerger->pTSchema->columns[iCol]; pTColumn = &pMerger->pTSchema->columns[iCol];
taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type)); if (taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type)) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
pMerger->version = key.version; pMerger->version = key.version;
@ -962,8 +970,7 @@ _exit:
*/ */
// delete skyline ====================================================== // delete skyline ======================================================
static int32_t tsdbMergeSkyline(SArray *pSkyline1, SArray *pSkyline2, SArray *pSkyline) { static void tsdbMergeSkyline(SArray *pSkyline1, SArray *pSkyline2, SArray *pSkyline) {
int32_t code = 0;
int32_t i1 = 0; int32_t i1 = 0;
int32_t n1 = taosArrayGetSize(pSkyline1); int32_t n1 = taosArrayGetSize(pSkyline1);
int32_t i2 = 0; int32_t i2 = 0;
@ -1017,7 +1024,6 @@ static int32_t tsdbMergeSkyline(SArray *pSkyline1, SArray *pSkyline2, SArray *pS
} }
pSkyline->size = TARRAY_ELEM_IDX(pSkyline, pItem); pSkyline->size = TARRAY_ELEM_IDX(pSkyline, pItem);
return code;
} }
int32_t tsdbBuildDeleteSkylineImpl(SArray *aSkyline, int32_t sidx, int32_t eidx, SArray *pSkyline) { int32_t tsdbBuildDeleteSkylineImpl(SArray *aSkyline, int32_t sidx, int32_t eidx, SArray *pSkyline) {
@ -1029,8 +1035,13 @@ int32_t tsdbBuildDeleteSkylineImpl(SArray *aSkyline, int32_t sidx, int32_t eidx,
if (sidx == eidx) { if (sidx == eidx) {
TSDBKEY *pItem1 = taosArrayGet(aSkyline, sidx * 2); TSDBKEY *pItem1 = taosArrayGet(aSkyline, sidx * 2);
TSDBKEY *pItem2 = taosArrayGet(aSkyline, sidx * 2 + 1); TSDBKEY *pItem2 = taosArrayGet(aSkyline, sidx * 2 + 1);
taosArrayPush(pSkyline, &pItem1); if (taosArrayPush(pSkyline, &pItem1) == NULL) {
taosArrayPush(pSkyline, &pItem2); return TSDB_CODE_OUT_OF_MEMORY;
}
if (taosArrayPush(pSkyline, &pItem2) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} else { } else {
SArray *pSkyline1 = NULL; SArray *pSkyline1 = NULL;
SArray *pSkyline2 = NULL; SArray *pSkyline2 = NULL;
@ -1049,7 +1060,7 @@ int32_t tsdbBuildDeleteSkylineImpl(SArray *aSkyline, int32_t sidx, int32_t eidx,
code = tsdbBuildDeleteSkylineImpl(aSkyline, midx + 1, eidx, pSkyline2); code = tsdbBuildDeleteSkylineImpl(aSkyline, midx + 1, eidx, pSkyline2);
if (code) goto _clear; if (code) goto _clear;
code = tsdbMergeSkyline(pSkyline1, pSkyline2, pSkyline); tsdbMergeSkyline(pSkyline1, pSkyline2, pSkyline);
_clear: _clear:
taosArrayDestroy(pSkyline1); taosArrayDestroy(pSkyline1);
@ -1064,13 +1075,28 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr
int32_t code = 0; int32_t code = 0;
int32_t dataNum = eidx - sidx + 1; int32_t dataNum = eidx - sidx + 1;
SArray *aTmpSkyline = taosArrayInit(dataNum * 2, sizeof(TSDBKEY)); SArray *aTmpSkyline = taosArrayInit(dataNum * 2, sizeof(TSDBKEY));
SArray *pSkyline = taosArrayInit(dataNum * 2, POINTER_BYTES); if (aTmpSkyline == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SArray *pSkyline = taosArrayInit(dataNum * 2, POINTER_BYTES);
if (pSkyline == NULL) {
taosArrayDestroy(aTmpSkyline);
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArrayClear(aSkyline); taosArrayClear(aSkyline);
for (int32_t i = sidx; i <= eidx; ++i) { for (int32_t i = sidx; i <= eidx; ++i) {
pDelData = (SDelData *)taosArrayGet(aDelData, i); pDelData = (SDelData *)taosArrayGet(aDelData, i);
taosArrayPush(aTmpSkyline, &(TSDBKEY){.ts = pDelData->sKey, .version = pDelData->version}); if (taosArrayPush(aTmpSkyline, &(TSDBKEY){.ts = pDelData->sKey, .version = pDelData->version}) == NULL) {
taosArrayPush(aTmpSkyline, &(TSDBKEY){.ts = pDelData->eKey, .version = 0}); code = TSDB_CODE_OUT_OF_MEMORY;
goto _clear;
}
if (taosArrayPush(aTmpSkyline, &(TSDBKEY){.ts = pDelData->eKey, .version = 0}) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _clear;
}
} }
code = tsdbBuildDeleteSkylineImpl(aTmpSkyline, sidx, eidx, pSkyline); code = tsdbBuildDeleteSkylineImpl(aTmpSkyline, sidx, eidx, pSkyline);
@ -1079,7 +1105,10 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr
int32_t skylineNum = taosArrayGetSize(pSkyline); int32_t skylineNum = taosArrayGetSize(pSkyline);
for (int32_t i = 0; i < skylineNum; ++i) { for (int32_t i = 0; i < skylineNum; ++i) {
TSDBKEY *p = taosArrayGetP(pSkyline, i); TSDBKEY *p = taosArrayGetP(pSkyline, i);
taosArrayPush(aSkyline, p); if (taosArrayPush(aSkyline, p) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _clear;
}
} }
_clear: _clear:
@ -1519,8 +1548,8 @@ int32_t tBlockDataDecompress(SBufferReader *br, SBlockData *blockData, SBuffer *
SBlockCol blockCol; SBlockCol blockCol;
code = tGetBlockCol(&br2, &blockCol, hdr.fmtVer, hdr.cmprAlg); code = tGetBlockCol(&br2, &blockCol, hdr.fmtVer, hdr.cmprAlg);
if (blockCol.alg == 0) blockCol.alg = hdr.cmprAlg;
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
if (blockCol.alg == 0) blockCol.alg = hdr.cmprAlg;
code = tBlockDataDecompressColData(&hdr, &blockCol, br, blockData, assist); code = tBlockDataDecompressColData(&hdr, &blockCol, br, blockData, assist);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
@ -1812,7 +1841,9 @@ int32_t tsdbGetColCmprAlgFromSet(SHashObj *set, int16_t colId, uint32_t *alg) {
if (set == NULL) return -1; if (set == NULL) return -1;
uint32_t *ret = taosHashGet(set, &colId, sizeof(colId)); uint32_t *ret = taosHashGet(set, &colId, sizeof(colId));
if (ret == NULL) return -1; if (ret == NULL) {
return TSDB_CODE_NOT_FOUND;
}
*alg = *ret; *alg = *ret;
return 0; return 0;

View File

@ -47,7 +47,7 @@ int32_t fillTableColCmpr(SMetaReader *reader, SSchemaExt *pExt, int32_t numOfCol
return 0; return 0;
} }
int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
STableInfoReq infoReq = {0}; STableInfoReq infoReq = {0};
STableMetaRsp metaRsp = {0}; STableMetaRsp metaRsp = {0};
SMetaReader mer1 = {0}; SMetaReader mer1 = {0};
@ -62,15 +62,15 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
// decode req // decode req
if (tDeserializeSTableInfoReq(pMsg->pCont, pMsg->contLen, &infoReq) != 0) { if (tDeserializeSTableInfoReq(pMsg->pCont, pMsg->contLen, &infoReq) != 0) {
code = TSDB_CODE_INVALID_MSG; code = terrno;
goto _exit4; goto _exit4;
} }
metaRsp.dbId = pVnode->config.dbId; metaRsp.dbId = pVnode->config.dbId;
strcpy(metaRsp.tbName, infoReq.tbName); (void)strcpy(metaRsp.tbName, infoReq.tbName);
memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName)); (void)memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName));
sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName); (void)sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName);
code = vnodeValidateTableHash(pVnode, tableFName); code = vnodeValidateTableHash(pVnode, tableFName);
if (code) { if (code) {
goto _exit4; goto _exit4;
@ -89,7 +89,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
metaRsp.tuid = mer1.me.uid; metaRsp.tuid = mer1.me.uid;
if (mer1.me.type == TSDB_SUPER_TABLE) { if (mer1.me.type == TSDB_SUPER_TABLE) {
strcpy(metaRsp.stbName, mer1.me.name); (void)strcpy(metaRsp.stbName, mer1.me.name);
schema = mer1.me.stbEntry.schemaRow; schema = mer1.me.stbEntry.schemaRow;
schemaTag = mer1.me.stbEntry.schemaTag; schemaTag = mer1.me.stbEntry.schemaTag;
metaRsp.suid = mer1.me.uid; metaRsp.suid = mer1.me.uid;
@ -97,7 +97,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_NOLOCK); metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_NOLOCK);
if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit2; if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit2;
strcpy(metaRsp.stbName, mer2.me.name); (void)strcpy(metaRsp.stbName, mer2.me.name);
metaRsp.suid = mer2.me.uid; metaRsp.suid = mer2.me.uid;
schema = mer2.me.stbEntry.schemaRow; schema = mer2.me.stbEntry.schemaRow;
schemaTag = mer2.me.stbEntry.schemaTag; schemaTag = mer2.me.stbEntry.schemaTag;
@ -114,10 +114,13 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
metaRsp.tversion = schemaTag.version; metaRsp.tversion = schemaTag.version;
metaRsp.pSchemas = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * (metaRsp.numOfColumns + metaRsp.numOfTags)); metaRsp.pSchemas = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * (metaRsp.numOfColumns + metaRsp.numOfTags));
metaRsp.pSchemaExt = (SSchemaExt *)taosMemoryCalloc(metaRsp.numOfColumns, sizeof(SSchemaExt)); metaRsp.pSchemaExt = (SSchemaExt *)taosMemoryCalloc(metaRsp.numOfColumns, sizeof(SSchemaExt));
if (NULL == metaRsp.pSchemas || NULL == metaRsp.pSchemaExt) {
memcpy(metaRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols); code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
(void)memcpy(metaRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols);
if (schemaTag.nCols) { if (schemaTag.nCols) {
memcpy(metaRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols); (void)memcpy(metaRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols);
} }
if (metaRsp.pSchemaExt) { if (metaRsp.pSchemaExt) {
SMetaReader *pReader = mer1.me.type == TSDB_CHILD_TABLE ? &mer2 : &mer1; SMetaReader *pReader = mer1.me.type == TSDB_CHILD_TABLE ? &mer2 : &mer1;
@ -134,7 +137,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
// encode and send response // encode and send response
rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp);
if (rspLen < 0) { if (rspLen < 0) {
code = TSDB_CODE_INVALID_MSG; code = terrno;
goto _exit; goto _exit;
} }
@ -148,7 +151,12 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp);
rspLen = tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp);
if (rspLen < 0) {
code = terrno;
goto _exit;
}
_exit: _exit:
taosMemoryFree(metaRsp.pSchemas); taosMemoryFree(metaRsp.pSchemas);
@ -174,10 +182,10 @@ _exit4:
*pMsg = rpcMsg; *pMsg = rpcMsg;
} }
return TSDB_CODE_SUCCESS; return code;
} }
int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
STableCfgReq cfgReq = {0}; STableCfgReq cfgReq = {0};
STableCfgRsp cfgRsp = {0}; STableCfgRsp cfgRsp = {0};
SMetaReader mer1 = {0}; SMetaReader mer1 = {0};
@ -192,14 +200,14 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
// decode req // decode req
if (tDeserializeSTableCfgReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) { if (tDeserializeSTableCfgReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) {
code = TSDB_CODE_INVALID_MSG; code = terrno;
goto _exit; goto _exit;
} }
strcpy(cfgRsp.tbName, cfgReq.tbName); (void)strcpy(cfgRsp.tbName, cfgReq.tbName);
memcpy(cfgRsp.dbFName, cfgReq.dbFName, sizeof(cfgRsp.dbFName)); (void)memcpy(cfgRsp.dbFName, cfgReq.dbFName, sizeof(cfgRsp.dbFName));
sprintf(tableFName, "%s.%s", cfgReq.dbFName, cfgReq.tbName); (void)sprintf(tableFName, "%s.%s", cfgReq.dbFName, cfgReq.tbName);
code = vnodeValidateTableHash(pVnode, tableFName); code = vnodeValidateTableHash(pVnode, tableFName);
if (code) { if (code) {
goto _exit; goto _exit;
@ -222,24 +230,36 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_LOCK); metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_LOCK);
if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit;
strcpy(cfgRsp.stbName, mer2.me.name); (void)strcpy(cfgRsp.stbName, mer2.me.name);
schema = mer2.me.stbEntry.schemaRow; schema = mer2.me.stbEntry.schemaRow;
schemaTag = mer2.me.stbEntry.schemaTag; schemaTag = mer2.me.stbEntry.schemaTag;
cfgRsp.ttl = mer1.me.ctbEntry.ttlDays; cfgRsp.ttl = mer1.me.ctbEntry.ttlDays;
cfgRsp.commentLen = mer1.me.ctbEntry.commentLen; cfgRsp.commentLen = mer1.me.ctbEntry.commentLen;
if (mer1.me.ctbEntry.commentLen > 0) { if (mer1.me.ctbEntry.commentLen > 0) {
cfgRsp.pComment = taosStrdup(mer1.me.ctbEntry.comment); cfgRsp.pComment = taosStrdup(mer1.me.ctbEntry.comment);
if (NULL == cfgRsp.pComment) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
} }
STag *pTag = (STag *)mer1.me.ctbEntry.pTags; STag *pTag = (STag *)mer1.me.ctbEntry.pTags;
cfgRsp.tagsLen = pTag->len; cfgRsp.tagsLen = pTag->len;
cfgRsp.pTags = taosMemoryMalloc(cfgRsp.tagsLen); cfgRsp.pTags = taosMemoryMalloc(cfgRsp.tagsLen);
memcpy(cfgRsp.pTags, pTag, cfgRsp.tagsLen); if (NULL == cfgRsp.pTags) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
(void)memcpy(cfgRsp.pTags, pTag, cfgRsp.tagsLen);
} else if (mer1.me.type == TSDB_NORMAL_TABLE) { } else if (mer1.me.type == TSDB_NORMAL_TABLE) {
schema = mer1.me.ntbEntry.schemaRow; schema = mer1.me.ntbEntry.schemaRow;
cfgRsp.ttl = mer1.me.ntbEntry.ttlDays; cfgRsp.ttl = mer1.me.ntbEntry.ttlDays;
cfgRsp.commentLen = mer1.me.ntbEntry.commentLen; cfgRsp.commentLen = mer1.me.ntbEntry.commentLen;
if (mer1.me.ntbEntry.commentLen > 0) { if (mer1.me.ntbEntry.commentLen > 0) {
cfgRsp.pComment = taosStrdup(mer1.me.ntbEntry.comment); cfgRsp.pComment = taosStrdup(mer1.me.ntbEntry.comment);
if (NULL == cfgRsp.pComment) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
} }
} else { } else {
ASSERT(0); ASSERT(0);
@ -250,9 +270,13 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
cfgRsp.pSchemas = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * (cfgRsp.numOfColumns + cfgRsp.numOfTags)); cfgRsp.pSchemas = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * (cfgRsp.numOfColumns + cfgRsp.numOfTags));
cfgRsp.pSchemaExt = (SSchemaExt *)taosMemoryMalloc(cfgRsp.numOfColumns * sizeof(SSchemaExt)); cfgRsp.pSchemaExt = (SSchemaExt *)taosMemoryMalloc(cfgRsp.numOfColumns * sizeof(SSchemaExt));
memcpy(cfgRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols); if (NULL == cfgRsp.pSchemas || NULL == cfgRsp.pSchemaExt) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
(void)memcpy(cfgRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols);
if (schemaTag.nCols) { if (schemaTag.nCols) {
memcpy(cfgRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols); (void)memcpy(cfgRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols);
} }
// if (useCompress(cfgRsp.tableType)) { // if (useCompress(cfgRsp.tableType)) {
@ -271,7 +295,7 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
// encode and send response // encode and send response
rspLen = tSerializeSTableCfgRsp(NULL, 0, &cfgRsp); rspLen = tSerializeSTableCfgRsp(NULL, 0, &cfgRsp);
if (rspLen < 0) { if (rspLen < 0) {
code = TSDB_CODE_INVALID_MSG; code = terrno;
goto _exit; goto _exit;
} }
@ -285,7 +309,12 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
tSerializeSTableCfgRsp(pRsp, rspLen, &cfgRsp);
rspLen = tSerializeSTableCfgRsp(pRsp, rspLen, &cfgRsp);
if (rspLen < 0) {
code = terrno;
goto _exit;
}
_exit: _exit:
rpcMsg.info = pMsg->info; rpcMsg.info = pMsg->info;
@ -307,7 +336,7 @@ _exit:
tFreeSTableCfgRsp(&cfgRsp); tFreeSTableCfgRsp(&cfgRsp);
metaReaderClear(&mer2); metaReaderClear(&mer2);
metaReaderClear(&mer1); metaReaderClear(&mer1);
return TSDB_CODE_SUCCESS; return code;
} }
static FORCE_INLINE void vnodeFreeSBatchRspMsg(void *p) { static FORCE_INLINE void vnodeFreeSBatchRspMsg(void *p) {
@ -331,7 +360,7 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) {
void *pRsp = NULL; void *pRsp = NULL;
if (tDeserializeSBatchReq(pMsg->pCont, pMsg->contLen, &batchReq)) { if (tDeserializeSBatchReq(pMsg->pCont, pMsg->contLen, &batchReq)) {
code = TSDB_CODE_OUT_OF_MEMORY; code = terrno;
qError("tDeserializeSBatchReq failed"); qError("tDeserializeSBatchReq failed");
goto _exit; goto _exit;
} }
@ -352,6 +381,10 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) {
for (int32_t i = 0; i < msgNum; ++i) { for (int32_t i = 0; i < msgNum; ++i) {
req = taosArrayGet(batchReq.pMsgs, i); req = taosArrayGet(batchReq.pMsgs, i);
if (req == NULL) {
code = TSDB_CODE_OUT_OF_RANGE;
goto _exit;
}
reqMsg.msgType = req->msgType; reqMsg.msgType = req->msgType;
reqMsg.pCont = req->msg; reqMsg.pCont = req->msg;
@ -359,13 +392,16 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) {
switch (req->msgType) { switch (req->msgType) {
case TDMT_VND_TABLE_META: case TDMT_VND_TABLE_META:
vnodeGetTableMeta(pVnode, &reqMsg, false); // error code has been set into reqMsg, no need to handle it here.
(void)vnodeGetTableMeta(pVnode, &reqMsg, false);
break; break;
case TDMT_VND_TABLE_CFG: case TDMT_VND_TABLE_CFG:
vnodeGetTableCfg(pVnode, &reqMsg, false); // error code has been set into reqMsg, no need to handle it here.
(void)vnodeGetTableCfg(pVnode, &reqMsg, false);
break; break;
case TDMT_VND_GET_STREAM_PROGRESS: case TDMT_VND_GET_STREAM_PROGRESS:
vnodeGetStreamProgress(pVnode, &reqMsg, false); // error code has been set into reqMsg, no need to handle it here.
(void)vnodeGetStreamProgress(pVnode, &reqMsg, false);
break; break;
default: default:
qError("invalid req msgType %d", req->msgType); qError("invalid req msgType %d", req->msgType);
@ -381,24 +417,28 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) {
rsp.rspCode = reqMsg.code; rsp.rspCode = reqMsg.code;
rsp.msg = reqMsg.pCont; rsp.msg = reqMsg.pCont;
taosArrayPush(batchRsp.pRsps, &rsp); if (NULL == taosArrayPush(batchRsp.pRsps, &rsp)) {
qError("taosArrayPush failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
} }
rspSize = tSerializeSBatchRsp(NULL, 0, &batchRsp); rspSize = tSerializeSBatchRsp(NULL, 0, &batchRsp);
if (rspSize < 0) { if (rspSize < 0) {
qError("tSerializeSBatchRsp failed"); qError("tSerializeSBatchRsp failed");
code = TSDB_CODE_OUT_OF_MEMORY; code = terrno;
goto _exit; goto _exit;
} }
pRsp = rpcMallocCont(rspSize); pRsp = rpcMallocCont(rspSize);
if (pRsp == NULL) { if (pRsp == NULL) {
qError("rpcMallocCont %d failed", rspSize); qError("rpcMallocCont %d failed", rspSize);
code = TSDB_CODE_OUT_OF_MEMORY; code = terrno;
goto _exit; goto _exit;
} }
if (tSerializeSBatchRsp(pRsp, rspSize, &batchRsp) < 0) { if (tSerializeSBatchRsp(pRsp, rspSize, &batchRsp) < 0) {
qError("tSerializeSBatchRsp %d failed", rspSize); qError("tSerializeSBatchRsp %d failed", rspSize);
code = TSDB_CODE_OUT_OF_MEMORY; code = terrno;
goto _exit; goto _exit;
} }
@ -501,7 +541,12 @@ int32_t vnodeGetTableList(void *pVnode, int8_t type, SArray *pList) {
} }
int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) { int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) {
int32_t code = TSDB_CODE_SUCCESS;
SMCtbCursor *pCur = metaOpenCtbCursor(pVnode, uid, 1); SMCtbCursor *pCur = metaOpenCtbCursor(pVnode, uid, 1);
if (NULL == pCur) {
qError("vnode get all table list failed");
return TSDB_CODE_FAILED;
}
while (1) { while (1) {
tb_uid_t id = metaCtbCursorNext(pCur); tb_uid_t id = metaCtbCursorNext(pCur);
@ -510,11 +555,15 @@ int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) {
} }
STableKeyInfo info = {uid = id}; STableKeyInfo info = {uid = id};
taosArrayPush(list, &info); if (NULL == taosArrayPush(list, &info)) {
qError("taosArrayPush failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
} }
_exit:
metaCloseCtbCursor(pCur); metaCloseCtbCursor(pCur);
return TSDB_CODE_SUCCESS; return code;
} }
int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg) { int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg) {
@ -522,8 +571,13 @@ int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bo
} }
int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) { int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) {
int32_t code = TSDB_CODE_SUCCESS;
SVnode *pVnodeObj = pVnode; SVnode *pVnodeObj = pVnode;
SMCtbCursor *pCur = metaOpenCtbCursor(pVnodeObj, suid, 1); SMCtbCursor *pCur = metaOpenCtbCursor(pVnodeObj, suid, 1);
if (NULL == pCur) {
qError("vnode get all table list failed");
return TSDB_CODE_FAILED;
}
while (1) { while (1) {
tb_uid_t id = metaCtbCursorNext(pCur); tb_uid_t id = metaCtbCursorNext(pCur);
@ -531,14 +585,20 @@ int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) {
break; break;
} }
taosArrayPush(list, &id); if (NULL == taosArrayPush(list, &id)) {
qError("taosArrayPush failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
} }
_exit:
metaCloseCtbCursor(pCur); metaCloseCtbCursor(pCur);
return TSDB_CODE_SUCCESS; return code;
} }
int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list) { int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list) {
int32_t code = TSDB_CODE_SUCCESS;
SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid); SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid);
if (!pCur) { if (!pCur) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
@ -550,15 +610,21 @@ int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list) {
break; break;
} }
taosArrayPush(list, &id); if (NULL == taosArrayPush(list, &id)) {
qError("taosArrayPush failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
} }
_exit:
metaCloseStbCursor(pCur); metaCloseStbCursor(pCur);
return TSDB_CODE_SUCCESS; return code;
} }
int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg, void *arg1), int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg, void *arg1),
void *arg) { void *arg) {
int32_t code = TSDB_CODE_SUCCESS;
SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid); SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid);
if (!pCur) { if (!pCur) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
@ -574,11 +640,16 @@ int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bo
continue; continue;
} }
taosArrayPush(list, &id); if (NULL == taosArrayPush(list, &id)) {
qError("taosArrayPush failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
} }
_exit:
metaCloseStbCursor(pCur); metaCloseStbCursor(pCur);
return TSDB_CODE_SUCCESS; return code;
} }
int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) { int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) {
@ -700,16 +771,20 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) {
*num = 0; *num = 0;
int64_t arrSize = taosArrayGetSize(suidList); int64_t arrSize = taosArrayGetSize(suidList);
int32_t code = TSDB_CODE_SUCCESS;
for (int64_t i = 0; i < arrSize; ++i) { for (int64_t i = 0; i < arrSize; ++i) {
tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
int64_t ctbNum = 0; int64_t ctbNum = 0;
int32_t numOfCols = 0; int32_t numOfCols = 0;
metaGetStbStats(pVnode, suid, &ctbNum, &numOfCols); code = metaGetStbStats(pVnode, suid, &ctbNum, &numOfCols);
if (TSDB_CODE_SUCCESS != code) {
goto _exit;
}
*num += ctbNum * (numOfCols - 1); *num += ctbNum * (numOfCols - 1);
} }
_exit:
taosArrayDestroy(suidList); taosArrayDestroy(suidList);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -728,7 +803,11 @@ int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num) {
} }
int64_t ctbNum = 0; int64_t ctbNum = 0;
vnodeGetCtbNum(pVnode, id, &ctbNum); int32_t code = vnodeGetCtbNum(pVnode, id, &ctbNum);
if (TSDB_CODE_SUCCESS != code) {
metaCloseStbCursor(pCur);
return code;
}
*num += ctbNum; *num += ctbNum;
} }
@ -771,14 +850,17 @@ int32_t vnodeGetStreamProgress(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
rsp.vgId = req.vgId; rsp.vgId = req.vgId;
rsp.streamId = req.streamId; rsp.streamId = req.streamId;
rspLen = tSerializeStreamProgressRsp(0, 0, &rsp); rspLen = tSerializeStreamProgressRsp(0, 0, &rsp);
if (rspLen < 0) {
code = terrno;
goto _OVER;
}
if (direct) { if (direct) {
buf = rpcMallocCont(rspLen); buf = rpcMallocCont(rspLen);
} else { } else {
buf = taosMemoryCalloc(1, rspLen); buf = taosMemoryCalloc(1, rspLen);
} }
if (!buf) { if (!buf) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
code = -1;
goto _OVER; goto _OVER;
} }
} }
@ -787,7 +869,11 @@ int32_t vnodeGetStreamProgress(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
code = tqGetStreamExecInfo(pVnode, req.streamId, &rsp.progressDelay, &rsp.fillHisFinished); code = tqGetStreamExecInfo(pVnode, req.streamId, &rsp.progressDelay, &rsp.fillHisFinished);
} }
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
tSerializeStreamProgressRsp(buf, rspLen, &rsp); rspLen = tSerializeStreamProgressRsp(buf, rspLen, &rsp);
if (rspLen < 0) {
code = terrno;
goto _OVER;
}
rpcMsg.pCont = buf; rpcMsg.pCont = buf;
buf = NULL; buf = NULL;
rpcMsg.contLen = rspLen; rpcMsg.contLen = rspLen;

View File

@ -312,8 +312,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
goto _exit; goto _exit;
} else { } else {
pReader->metaDone = 1; pReader->metaDone = 1;
code = metaSnapReaderClose(&pReader->pMetaReader); metaSnapReaderClose(&pReader->pMetaReader);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} }

View File

@ -1684,14 +1684,17 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
if (fmIsAggFunc(pCtx->functionId) || fmIsIndefiniteRowsFunc(pCtx->functionId)) { if (fmIsAggFunc(pCtx->functionId) || fmIsIndefiniteRowsFunc(pCtx->functionId)) {
bool isUdaf = fmIsUserDefinedFunc(pCtx->functionId); bool isUdaf = fmIsUserDefinedFunc(pCtx->functionId);
if (!isUdaf) { if (!isUdaf) {
// TODO(xxx) : need handle return value of fmGetFuncExecFuncs.
fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet); fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet);
} else { } else {
char* udfName = pExpr->pExpr->_function.pFunctNode->functionName; char* udfName = pExpr->pExpr->_function.pFunctNode->functionName;
pCtx->udfName = taosStrdup(udfName); pCtx->udfName = taosStrdup(udfName);
// TODO(xxx) : need handle return value of fmGetUdafExecFuncs.
fmGetUdafExecFuncs(pCtx->functionId, &pCtx->fpSet); fmGetUdafExecFuncs(pCtx->functionId, &pCtx->fpSet);
} }
pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env); pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env);
} else { } else {
// TODO(xxx) : need handle return value of fmGetScalarFuncExecFuncs.
fmGetScalarFuncExecFuncs(pCtx->functionId, &pCtx->sfp); fmGetScalarFuncExecFuncs(pCtx->functionId, &pCtx->sfp);
if (pCtx->sfp.getEnv != NULL) { if (pCtx->sfp.getEnv != NULL) {
pCtx->sfp.getEnv(pExpr->pExpr->_function.pFunctNode, &env); pCtx->sfp.getEnv(pExpr->pExpr->_function.pFunctNode, &env);

View File

@ -521,8 +521,8 @@ int32_t setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t n
if (!pResInfo->initialized) { if (!pResInfo->initialized) {
if (pCtx[i].functionId != -1) { if (pCtx[i].functionId != -1) {
bool ini = pCtx[i].fpSet.init(&pCtx[i], pResInfo); int32_t code = pCtx[i].fpSet.init(&pCtx[i], pResInfo);
if (!ini && fmIsUserDefinedFunc(pCtx[i].functionId)) { if (code != TSDB_CODE_SUCCESS && fmIsUserDefinedFunc(pCtx[i].functionId)){
pResInfo->initialized = false; pResInfo->initialized = false;
return TSDB_CODE_UDF_FUNC_EXEC_FAILURE; return TSDB_CODE_UDF_FUNC_EXEC_FAILURE;
} }

View File

@ -44,8 +44,8 @@ static int32_t doGenerateSourceData(SOperatorInfo* pOperator);
static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator); static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator);
static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator); static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator);
static SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols); static SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols);
static void setFunctionResultOutput(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t stage, static int32_t setFunctionResultOutput(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SAggSupporter* pSup,
int32_t numOfExprs); int32_t stage, int32_t numOfExprs);
static void destroyProjectOperatorInfo(void* param) { static void destroyProjectOperatorInfo(void* param) {
if (NULL == param) { if (NULL == param) {
@ -142,7 +142,10 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys
} }
initBasicInfo(&pInfo->binfo, pResBlock); initBasicInfo(&pInfo->binfo, pResBlock);
setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfCols); code = setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfCols);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
code = filterInitFromNode((SNode*)pProjPhyNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); code = filterInitFromNode((SNode*)pProjPhyNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -447,7 +450,11 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
goto _error; goto _error;
} }
setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfExpr); code = setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfExpr);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
code = filterInitFromNode((SNode*)pPhyNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); code = filterInitFromNode((SNode*)pPhyNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
@ -589,7 +596,8 @@ SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) {
return (rows > 0) ? pInfo->pRes : NULL; return (rows > 0) ? pInfo->pRes : NULL;
} }
void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) { int32_t initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) {
int32_t code = TSDB_CODE_SUCCESS;
for (int32_t j = 0; j < size; ++j) { for (int32_t j = 0; j < size; ++j) {
struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]); struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]);
if (isRowEntryInitialized(pResInfo) || fmIsPseudoColumnFunc(pCtx[j].functionId) || pCtx[j].functionId == -1 || if (isRowEntryInitialized(pResInfo) || fmIsPseudoColumnFunc(pCtx[j].functionId) || pCtx[j].functionId == -1 ||
@ -597,8 +605,12 @@ void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) {
continue; continue;
} }
pCtx[j].fpSet.init(&pCtx[j], pCtx[j].resultInfo); code = pCtx[j].fpSet.init(&pCtx[j], pCtx[j].resultInfo);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
} }
return code;
} }
/* /*
@ -610,7 +622,7 @@ void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) {
* offset[0] offset[1] offset[2] * offset[0] offset[1] offset[2]
*/ */
// TODO refactor: some function move away // TODO refactor: some function move away
void setFunctionResultOutput(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t stage, int32_t setFunctionResultOutput(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t stage,
int32_t numOfExprs) { int32_t numOfExprs) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
@ -632,7 +644,7 @@ void setFunctionResultOutput(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SA
pCtx[i].scanFlag = stage; pCtx[i].scanFlag = stage;
} }
initCtxOutputBuffer(pCtx, numOfExprs); return initCtxOutputBuffer(pCtx, numOfExprs);
} }
SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols) { SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols) {
@ -841,7 +853,10 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
// do nothing // do nothing
} else if (fmIsIndefiniteRowsFunc(pfCtx->functionId)) { } else if (fmIsIndefiniteRowsFunc(pfCtx->functionId)) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pfCtx); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pfCtx);
pfCtx->fpSet.init(pfCtx, pResInfo); code = pfCtx->fpSet.init(pfCtx, pResInfo);
if (TSDB_CODE_SUCCESS != code) {
goto _exit;
}
pfCtx->pOutput = taosArrayGet(pResult->pDataBlock, outputSlotId); pfCtx->pOutput = taosArrayGet(pResult->pDataBlock, outputSlotId);
pfCtx->offset = createNewColModel ? 0 : pResult->info.rows; // set the start offset pfCtx->offset = createNewColModel ? 0 : pResult->info.rows; // set the start offset

View File

@ -202,8 +202,9 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo*
SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, i, pTableScanInfo->base.pdInfo.pExprSup->rowEntryInfoOffset); SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, i, pTableScanInfo->base.pdInfo.pExprSup->rowEntryInfoOffset);
int32_t reqStatus = fmFuncDynDataRequired(functionId, pEntry, pBlockInfo); int32_t reqStatus;
if (reqStatus != FUNC_DATA_REQUIRED_NOT_LOAD) { code = fmFuncDynDataRequired(functionId, pEntry, pBlockInfo, &reqStatus);
if (code != TSDB_CODE_SUCCESS || reqStatus != FUNC_DATA_REQUIRED_NOT_LOAD) {
notLoadBlock = false; notLoadBlock = false;
break; break;
} }

View File

@ -1086,12 +1086,13 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
return (rows == 0) ? NULL : pBlock; return (rows == 0) ? NULL : pBlock;
} }
static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) { static int32_t doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) {
SResultRow* pResult = getResultRowByPos(pResultBuf, p1, false); SResultRow* pResult = getResultRowByPos(pResultBuf, p1, false);
if (NULL == pResult) { if (NULL == pResult) {
return; return TSDB_CODE_SUCCESS;
} }
int32_t code = TSDB_CODE_SUCCESS;
SqlFunctionCtx* pCtx = pSup->pCtx; SqlFunctionCtx* pCtx = pSup->pCtx;
for (int32_t i = 0; i < numOfOutput; ++i) { for (int32_t i = 0; i < numOfOutput; ++i) {
pCtx[i].resultInfo = getResultEntryInfo(pResult, i, pSup->rowEntryInfoOffset); pCtx[i].resultInfo = getResultEntryInfo(pResult, i, pSup->rowEntryInfoOffset);
@ -1101,15 +1102,19 @@ static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf,
} }
pResInfo->initialized = false; pResInfo->initialized = false;
if (pCtx[i].functionId != -1) { if (pCtx[i].functionId != -1) {
pCtx[i].fpSet.init(&pCtx[i], pResInfo); code = pCtx[i].fpSet.init(&pCtx[i], pResInfo);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
} }
} }
SFilePage* bufPage = getBufPage(pResultBuf, p1->pageId); SFilePage* bufPage = getBufPage(pResultBuf, p1->pageId);
if (NULL == bufPage) { if (NULL == bufPage) {
return; return TSDB_CODE_SUCCESS;
} }
setBufPageDirty(bufPage, true); setBufPageDirty(bufPage, true);
releaseBufPage(pResultBuf, bufPage); releaseBufPage(pResultBuf, bufPage);
return TSDB_CODE_SUCCESS;
} }
static void destroyStateWindowOperatorInfo(void* param) { static void destroyStateWindowOperatorInfo(void* param) {

View File

@ -49,9 +49,9 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems)
int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos);
int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos);
const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos); int32_t loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos, char** value);
bool functionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t functionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult); int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult);
int32_t combineFunction(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t combineFunction(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
@ -74,7 +74,7 @@ int32_t sumInvertFunction(SqlFunctionCtx* pCtx);
int32_t sumCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t sumCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
bool minmaxFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t minmaxFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
bool getMinmaxFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getMinmaxFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
int32_t minFunction(SqlFunctionCtx* pCtx); int32_t minFunction(SqlFunctionCtx* pCtx);
int32_t maxFunction(SqlFunctionCtx* pCtx); int32_t maxFunction(SqlFunctionCtx* pCtx);
@ -83,7 +83,7 @@ int32_t minCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
int32_t maxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t maxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
bool getAvgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getAvgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t avgFunction(SqlFunctionCtx* pCtx); int32_t avgFunction(SqlFunctionCtx* pCtx);
int32_t avgFunctionMerge(SqlFunctionCtx* pCtx); int32_t avgFunctionMerge(SqlFunctionCtx* pCtx);
int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
@ -97,7 +97,7 @@ int32_t avgCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
int32_t getAvgInfoSize(); int32_t getAvgInfoSize();
bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool stddevFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t stddevFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t stddevFunction(SqlFunctionCtx* pCtx); int32_t stddevFunction(SqlFunctionCtx* pCtx);
int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx); int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx);
int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
@ -111,18 +111,18 @@ int32_t stddevCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
int32_t getStddevInfoSize(); int32_t getStddevInfoSize();
bool getLeastSQRFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getLeastSQRFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool leastSQRFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t leastSQRFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t leastSQRFunction(SqlFunctionCtx* pCtx); int32_t leastSQRFunction(SqlFunctionCtx* pCtx);
int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t leastSQRCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t leastSQRCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool percentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t percentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t percentileFunction(SqlFunctionCtx* pCtx); int32_t percentileFunction(SqlFunctionCtx* pCtx);
int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getApercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getApercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool apercentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t apercentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t apercentileFunction(SqlFunctionCtx* pCtx); int32_t apercentileFunction(SqlFunctionCtx* pCtx);
int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx); int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx);
int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
@ -131,16 +131,16 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx)
int32_t getApercentileMaxSize(); int32_t getApercentileMaxSize();
bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo); int32_t diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo);
int32_t diffFunction(SqlFunctionCtx* pCtx); int32_t diffFunction(SqlFunctionCtx* pCtx);
int32_t diffFunctionByRow(SArray* pCtx); int32_t diffFunctionByRow(SArray* pCtx);
bool getDerivativeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getDerivativeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool derivativeFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo); int32_t derivativeFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo);
int32_t derivativeFunction(SqlFunctionCtx* pCtx); int32_t derivativeFunction(SqlFunctionCtx* pCtx);
bool getIrateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getIrateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool irateFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo); int32_t irateFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo);
int32_t irateFunction(SqlFunctionCtx* pCtx); int32_t irateFunction(SqlFunctionCtx* pCtx);
int32_t irateFunctionMerge(SqlFunctionCtx* pCtx); int32_t irateFunctionMerge(SqlFunctionCtx* pCtx);
int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
@ -165,7 +165,7 @@ EFuncDataRequired lastDynDataReq(void* pRes, SDataBlockInfo* pBlockInfo);
int32_t lastRowFunction(SqlFunctionCtx* pCtx); int32_t lastRowFunction(SqlFunctionCtx* pCtx);
bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
bool topBotFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t topBotFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t topFunction(SqlFunctionCtx* pCtx); int32_t topFunction(SqlFunctionCtx* pCtx);
int32_t bottomFunction(SqlFunctionCtx* pCtx); int32_t bottomFunction(SqlFunctionCtx* pCtx);
int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
@ -174,7 +174,7 @@ int32_t bottomCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
int32_t getTopBotInfoSize(int64_t numOfItems); int32_t getTopBotInfoSize(int64_t numOfItems);
bool getSpreadFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getSpreadFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool spreadFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t spreadFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t spreadFunction(SqlFunctionCtx* pCtx); int32_t spreadFunction(SqlFunctionCtx* pCtx);
int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx); int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx);
int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
@ -183,7 +183,7 @@ int32_t getSpreadInfoSize();
int32_t spreadCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t spreadCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
bool getElapsedFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getElapsedFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool elapsedFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t elapsedFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t elapsedFunction(SqlFunctionCtx* pCtx); int32_t elapsedFunction(SqlFunctionCtx* pCtx);
int32_t elapsedFunctionMerge(SqlFunctionCtx* pCtx); int32_t elapsedFunctionMerge(SqlFunctionCtx* pCtx);
int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
@ -192,7 +192,7 @@ int32_t getElapsedInfoSize();
int32_t elapsedCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t elapsedCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool histogramFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t histogramFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t histogramFunction(SqlFunctionCtx* pCtx); int32_t histogramFunction(SqlFunctionCtx* pCtx);
int32_t histogramFunctionPartial(SqlFunctionCtx* pCtx); int32_t histogramFunctionPartial(SqlFunctionCtx* pCtx);
int32_t histogramFunctionMerge(SqlFunctionCtx* pCtx); int32_t histogramFunctionMerge(SqlFunctionCtx* pCtx);
@ -210,7 +210,7 @@ int32_t getHLLInfoSize();
int32_t hllCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t hllCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
bool getStateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getStateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool stateFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t stateFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t stateCountFunction(SqlFunctionCtx* pCtx); int32_t stateCountFunction(SqlFunctionCtx* pCtx);
int32_t stateDurationFunction(SqlFunctionCtx* pCtx); int32_t stateDurationFunction(SqlFunctionCtx* pCtx);
@ -218,35 +218,35 @@ bool getCsumFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
int32_t csumFunction(SqlFunctionCtx* pCtx); int32_t csumFunction(SqlFunctionCtx* pCtx);
bool getMavgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getMavgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool mavgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t mavgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t mavgFunction(SqlFunctionCtx* pCtx); int32_t mavgFunction(SqlFunctionCtx* pCtx);
bool getSampleFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getSampleFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool sampleFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t sampleFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t sampleFunction(SqlFunctionCtx* pCtx); int32_t sampleFunction(SqlFunctionCtx* pCtx);
int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getTailFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getTailFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool tailFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t tailFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t tailFunction(SqlFunctionCtx* pCtx); int32_t tailFunction(SqlFunctionCtx* pCtx);
bool getUniqueFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getUniqueFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool uniqueFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t uniqueFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t uniqueFunction(SqlFunctionCtx* pCtx); int32_t uniqueFunction(SqlFunctionCtx* pCtx);
bool getModeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getModeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t modeFunction(SqlFunctionCtx* pCtx); int32_t modeFunction(SqlFunctionCtx* pCtx);
int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getTwaFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getTwaFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool twaFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t twaFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t twaFunction(SqlFunctionCtx* pCtx); int32_t twaFunction(SqlFunctionCtx* pCtx);
int32_t twaFinalize(struct SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t twaFinalize(struct SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getSelectivityFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getSelectivityFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool blockDistSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t blockDistSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t blockDistFunction(SqlFunctionCtx* pCtx); int32_t blockDistFunction(SqlFunctionCtx* pCtx);
int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);

View File

@ -37,7 +37,7 @@ static FORCE_INLINE void initResultRowEntry(SResultRowEntryInfo *pResInfo, int32
pResInfo->complete = false; pResInfo->complete = false;
pResInfo->numOfRes = 0; pResInfo->numOfRes = 0;
memset(GET_ROWCELL_INTERBUF(pResInfo), 0, bufLen); (void)memset(GET_ROWCELL_INTERBUF(pResInfo), 0, bufLen);
} }
#ifdef __cplusplus #ifdef __cplusplus

View File

@ -55,14 +55,15 @@ typedef struct SHistogramInfo {
#endif #endif
} SHistogramInfo; } SHistogramInfo;
SHistogramInfo* tHistogramCreate(int32_t numOfBins); int32_t tHistogramCreate(int32_t numOfEntries, SHistogramInfo** pHisto);
SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins); SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins);
int32_t tHistogramAdd(SHistogramInfo** pHisto, double val); int32_t tHistogramAdd(SHistogramInfo** pHisto, double val);
int64_t tHistogramSum(SHistogramInfo* pHisto, double v); int64_t tHistogramSum(SHistogramInfo* pHisto, double v);
double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num); int32_t tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num, double** pVal);
SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries); int32_t tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries,
SHistogramInfo** pResHistogram);
void tHistogramDestroy(SHistogramInfo** pHisto); void tHistogramDestroy(SHistogramInfo** pHisto);
void tHistogramPrint(SHistogramInfo* pHisto); void tHistogramPrint(SHistogramInfo* pHisto);

View File

@ -67,7 +67,8 @@ typedef struct tMemBucket {
SHashObj *groupPagesMap; // disk page map for different groups; SHashObj *groupPagesMap; // disk page map for different groups;
} tMemBucket; } tMemBucket;
tMemBucket *tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval, bool hasWindowOrGroup); int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval, bool hasWindowOrGroup,
tMemBucket **pBucket);
void tMemBucketDestroy(tMemBucket *pBucket); void tMemBucketDestroy(tMemBucket *pBucket);

View File

@ -25,7 +25,7 @@
static int32_t buildFuncErrMsg(char* pErrBuf, int32_t len, int32_t errCode, const char* pFormat, ...) { static int32_t buildFuncErrMsg(char* pErrBuf, int32_t len, int32_t errCode, const char* pFormat, ...) {
va_list vArgList; va_list vArgList;
va_start(vArgList, pFormat); va_start(vArgList, pFormat);
vsnprintf(pErrBuf, len, pFormat, vArgList); (void)vsnprintf(pErrBuf, len, pFormat, vArgList);
va_end(vArgList); va_end(vArgList);
return errCode; return errCode;
} }
@ -42,27 +42,24 @@ static int32_t invaildFuncParaValueErrMsg(char* pErrBuf, int32_t len, const char
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_VALUE, "Invalid parameter value : %s", pFuncName); return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_VALUE, "Invalid parameter value : %s", pFuncName);
} }
#define TIME_UNIT_INVALID 1
#define TIME_UNIT_TOO_SMALL 2
static int32_t validateTimeUnitParam(uint8_t dbPrec, const SValueNode* pVal) { static int32_t validateTimeUnitParam(uint8_t dbPrec, const SValueNode* pVal) {
if (!IS_DURATION_VAL(pVal->flag)) { if (!IS_DURATION_VAL(pVal->flag)) {
return TIME_UNIT_INVALID; return TSDB_CODE_FUNC_TIME_UNIT_INVALID;
} }
if (TSDB_TIME_PRECISION_MILLI == dbPrec && if (TSDB_TIME_PRECISION_MILLI == dbPrec &&
(0 == strcasecmp(pVal->literal, "1u") || 0 == strcasecmp(pVal->literal, "1b"))) { (0 == strcasecmp(pVal->literal, "1u") || 0 == strcasecmp(pVal->literal, "1b"))) {
return TIME_UNIT_TOO_SMALL; return TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL;
} }
if (TSDB_TIME_PRECISION_MICRO == dbPrec && 0 == strcasecmp(pVal->literal, "1b")) { if (TSDB_TIME_PRECISION_MICRO == dbPrec && 0 == strcasecmp(pVal->literal, "1b")) {
return TIME_UNIT_TOO_SMALL; return TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL;
} }
if (pVal->literal[0] != '1' || if (pVal->literal[0] != '1' ||
(pVal->literal[1] != 'u' && pVal->literal[1] != 'a' && pVal->literal[1] != 's' && pVal->literal[1] != 'm' && (pVal->literal[1] != 'u' && pVal->literal[1] != 'a' && pVal->literal[1] != 's' && pVal->literal[1] != 'm' &&
pVal->literal[1] != 'h' && pVal->literal[1] != 'd' && pVal->literal[1] != 'w' && pVal->literal[1] != 'b')) { pVal->literal[1] != 'h' && pVal->literal[1] != 'd' && pVal->literal[1] != 'w' && pVal->literal[1] != 'b')) {
return TIME_UNIT_INVALID; return TSDB_CODE_FUNC_TIME_UNIT_INVALID;
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -138,13 +135,13 @@ static bool validateTimezoneFormat(const SValueNode* pVal) {
} }
if (i == 2) { if (i == 2) {
memcpy(buf, &tz[i - 1], 2); (void)memcpy(buf, &tz[i - 1], 2);
hour = taosStr2Int8(buf, NULL, 10); hour = taosStr2Int8(buf, NULL, 10);
if (!validateHourRange(hour)) { if (!validateHourRange(hour)) {
return false; return false;
} }
} else if (i == 4) { } else if (i == 4) {
memcpy(buf, &tz[i - 1], 2); (void)memcpy(buf, &tz[i - 1], 2);
minute = taosStr2Int8(buf, NULL, 10); minute = taosStr2Int8(buf, NULL, 10);
if (!validateMinuteRange(hour, minute, tz[0])) { if (!validateMinuteRange(hour, minute, tz[0])) {
return false; return false;
@ -167,13 +164,13 @@ static bool validateTimezoneFormat(const SValueNode* pVal) {
} }
if (i == 2) { if (i == 2) {
memcpy(buf, &tz[i - 1], 2); (void)memcpy(buf, &tz[i - 1], 2);
hour = taosStr2Int8(buf, NULL, 10); hour = taosStr2Int8(buf, NULL, 10);
if (!validateHourRange(hour)) { if (!validateHourRange(hour)) {
return false; return false;
} }
} else if (i == 5) { } else if (i == 5) {
memcpy(buf, &tz[i - 1], 2); (void)memcpy(buf, &tz[i - 1], 2);
minute = taosStr2Int8(buf, NULL, 10); minute = taosStr2Int8(buf, NULL, 10);
if (!validateMinuteRange(hour, minute, tz[0])) { if (!validateMinuteRange(hour, minute, tz[0])) {
return false; return false;
@ -215,7 +212,7 @@ static int32_t addTimezoneParam(SNodeList* pList) {
time_t t = taosTime(NULL); time_t t = taosTime(NULL);
struct tm tmInfo; struct tm tmInfo;
if (taosLocalTime(&t, &tmInfo, buf) != NULL) { if (taosLocalTime(&t, &tmInfo, buf) != NULL) {
strftime(buf, sizeof(buf), "%z", &tmInfo); (void)strftime(buf, sizeof(buf), "%z", &tmInfo);
} }
int32_t len = (int32_t)strlen(buf); int32_t len = (int32_t)strlen(buf);
@ -225,15 +222,27 @@ static int32_t addTimezoneParam(SNodeList* pList) {
} }
pVal->literal = strndup(buf, len); pVal->literal = strndup(buf, len);
if (pVal->literal == NULL) {
nodesDestroyNode((SNode*)pVal);
return TSDB_CODE_OUT_OF_MEMORY;
}
pVal->translate = true; pVal->translate = true;
pVal->node.resType.type = TSDB_DATA_TYPE_BINARY; pVal->node.resType.type = TSDB_DATA_TYPE_BINARY;
pVal->node.resType.bytes = len + VARSTR_HEADER_SIZE; pVal->node.resType.bytes = len + VARSTR_HEADER_SIZE;
pVal->node.resType.precision = TSDB_TIME_PRECISION_MILLI; pVal->node.resType.precision = TSDB_TIME_PRECISION_MILLI;
pVal->datum.p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE + 1); pVal->datum.p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE + 1);
if (pVal->datum.p == NULL) {
nodesDestroyNode((SNode*)pVal);
return TSDB_CODE_OUT_OF_MEMORY;
}
varDataSetLen(pVal->datum.p, len); varDataSetLen(pVal->datum.p, len);
strncpy(varDataVal(pVal->datum.p), pVal->literal, len); (void)strncpy(varDataVal(pVal->datum.p), pVal->literal, len);
nodesListAppend(pList, (SNode*)pVal); int32_t code = nodesListAppend(pList, (SNode*)pVal);
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode((SNode*)pVal);
return code;
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -252,7 +261,11 @@ static int32_t addDbPrecisonParam(SNodeList** pList, uint8_t precision) {
pVal->datum.i = (int64_t)precision; pVal->datum.i = (int64_t)precision;
pVal->typeData = (int64_t)precision; pVal->typeData = (int64_t)precision;
nodesListMakeAppend(pList, (SNode*)pVal); int32_t code = nodesListMakeAppend(pList, (SNode*)pVal);
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode((SNode*)pVal);
return code;
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -936,13 +949,13 @@ static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len
uint8_t dbPrec = pFunc->node.resType.precision; uint8_t dbPrec = pFunc->node.resType.precision;
int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1)); int32_t code = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1));
if (ret == TIME_UNIT_TOO_SMALL) { if (code == TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, return buildFuncErrMsg(pErrBuf, len, code,
"ELAPSED function time unit parameter should be greater than db precision"); "ELAPSED function time unit parameter should be greater than db precision");
} else if (ret == TIME_UNIT_INVALID) { } else if (code == TSDB_CODE_FUNC_TIME_UNIT_INVALID) {
return buildFuncErrMsg( return buildFuncErrMsg(
pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, pErrBuf, len, code,
"ELAPSED function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]"); "ELAPSED function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]");
} }
} }
@ -1062,7 +1075,7 @@ static int8_t validateHistogramBinType(char* binTypeStr) {
return binType; return binType;
} }
static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* errMsg, int32_t msgLen) { static int32_t validateHistogramBinDesc(char* binDescStr, int8_t binType, char* errMsg, int32_t msgLen) {
const char* msg1 = "HISTOGRAM function requires four parameters"; const char* msg1 = "HISTOGRAM function requires four parameters";
const char* msg3 = "HISTOGRAM function invalid format for binDesc parameter"; const char* msg3 = "HISTOGRAM function invalid format for binDesc parameter";
const char* msg4 = "HISTOGRAM function binDesc parameter \"count\" should be in range [1, 1000]"; const char* msg4 = "HISTOGRAM function binDesc parameter \"count\" should be in range [1, 1000]";
@ -1070,6 +1083,7 @@ static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* err
const char* msg6 = "HISTOGRAM function binDesc parameter \"width\" cannot be 0"; const char* msg6 = "HISTOGRAM function binDesc parameter \"width\" cannot be 0";
const char* msg7 = "HISTOGRAM function binDesc parameter \"start\" cannot be 0 with \"log_bin\" type"; const char* msg7 = "HISTOGRAM function binDesc parameter \"start\" cannot be 0 with \"log_bin\" type";
const char* msg8 = "HISTOGRAM function binDesc parameter \"factor\" cannot be negative or equal to 0/1"; const char* msg8 = "HISTOGRAM function binDesc parameter \"factor\" cannot be negative or equal to 0/1";
const char* msg9 = "HISTOGRAM function out of memory";
cJSON* binDesc = cJSON_Parse(binDescStr); cJSON* binDesc = cJSON_Parse(binDescStr);
int32_t numOfBins; int32_t numOfBins;
@ -1078,9 +1092,9 @@ static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* err
int32_t numOfParams = cJSON_GetArraySize(binDesc); int32_t numOfParams = cJSON_GetArraySize(binDesc);
int32_t startIndex; int32_t startIndex;
if (numOfParams != 4) { if (numOfParams != 4) {
snprintf(errMsg, msgLen, "%s", msg1); (void)snprintf(errMsg, msgLen, "%s", msg1);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
cJSON* start = cJSON_GetObjectItem(binDesc, "start"); cJSON* start = cJSON_GetObjectItem(binDesc, "start");
@ -1090,22 +1104,22 @@ static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* err
cJSON* infinity = cJSON_GetObjectItem(binDesc, "infinity"); cJSON* infinity = cJSON_GetObjectItem(binDesc, "infinity");
if (!cJSON_IsNumber(start) || !cJSON_IsNumber(count) || !cJSON_IsBool(infinity)) { if (!cJSON_IsNumber(start) || !cJSON_IsNumber(count) || !cJSON_IsBool(infinity)) {
snprintf(errMsg, msgLen, "%s", msg3); (void)snprintf(errMsg, msgLen, "%s", msg3);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
if (count->valueint <= 0 || count->valueint > 1000) { // limit count to 1000 if (count->valueint <= 0 || count->valueint > 1000) { // limit count to 1000
snprintf(errMsg, msgLen, "%s", msg4); (void)snprintf(errMsg, msgLen, "%s", msg4);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
if (isinf(start->valuedouble) || (width != NULL && isinf(width->valuedouble)) || if (isinf(start->valuedouble) || (width != NULL && isinf(width->valuedouble)) ||
(factor != NULL && isinf(factor->valuedouble)) || (count != NULL && isinf(count->valuedouble))) { (factor != NULL && isinf(factor->valuedouble)) || (count != NULL && isinf(count->valuedouble))) {
snprintf(errMsg, msgLen, "%s", msg5); (void)snprintf(errMsg, msgLen, "%s", msg5);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
int32_t counter = (int32_t)count->valueint; int32_t counter = (int32_t)count->valueint;
@ -1118,53 +1132,58 @@ static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* err
} }
intervals = taosMemoryCalloc(numOfBins, sizeof(double)); intervals = taosMemoryCalloc(numOfBins, sizeof(double));
if (intervals == NULL) {
(void)snprintf(errMsg, msgLen, "%s", msg9);
cJSON_Delete(binDesc);
return TSDB_CODE_FAILED;
}
if (cJSON_IsNumber(width) && factor == NULL && binType == LINEAR_BIN) { if (cJSON_IsNumber(width) && factor == NULL && binType == LINEAR_BIN) {
// linear bin process // linear bin process
if (width->valuedouble == 0) { if (width->valuedouble == 0) {
snprintf(errMsg, msgLen, "%s", msg6); (void)snprintf(errMsg, msgLen, "%s", msg6);
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
for (int i = 0; i < counter + 1; ++i) { for (int i = 0; i < counter + 1; ++i) {
intervals[startIndex] = start->valuedouble + i * width->valuedouble; intervals[startIndex] = start->valuedouble + i * width->valuedouble;
if (isinf(intervals[startIndex])) { if (isinf(intervals[startIndex])) {
snprintf(errMsg, msgLen, "%s", msg5); (void)snprintf(errMsg, msgLen, "%s", msg5);
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
startIndex++; startIndex++;
} }
} else if (cJSON_IsNumber(factor) && width == NULL && binType == LOG_BIN) { } else if (cJSON_IsNumber(factor) && width == NULL && binType == LOG_BIN) {
// log bin process // log bin process
if (start->valuedouble == 0) { if (start->valuedouble == 0) {
snprintf(errMsg, msgLen, "%s", msg7); (void)snprintf(errMsg, msgLen, "%s", msg7);
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
if (factor->valuedouble < 0 || factor->valuedouble == 0 || factor->valuedouble == 1) { if (factor->valuedouble < 0 || factor->valuedouble == 0 || factor->valuedouble == 1) {
snprintf(errMsg, msgLen, "%s", msg8); (void)snprintf(errMsg, msgLen, "%s", msg8);
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
for (int i = 0; i < counter + 1; ++i) { for (int i = 0; i < counter + 1; ++i) {
intervals[startIndex] = start->valuedouble * pow(factor->valuedouble, i * 1.0); intervals[startIndex] = start->valuedouble * pow(factor->valuedouble, i * 1.0);
if (isinf(intervals[startIndex])) { if (isinf(intervals[startIndex])) {
snprintf(errMsg, msgLen, "%s", msg5); (void)snprintf(errMsg, msgLen, "%s", msg5);
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
startIndex++; startIndex++;
} }
} else { } else {
snprintf(errMsg, msgLen, "%s", msg3); (void)snprintf(errMsg, msgLen, "%s", msg3);
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
if (infinity->valueint == true) { if (infinity->valueint == true) {
@ -1172,7 +1191,7 @@ static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* err
intervals[numOfBins - 1] = INFINITY; intervals[numOfBins - 1] = INFINITY;
// in case of desc bin orders, -inf/inf should be swapped // in case of desc bin orders, -inf/inf should be swapped
if (numOfBins < 4) { if (numOfBins < 4) {
return false; return TSDB_CODE_FAILED;
} }
if (intervals[1] > intervals[numOfBins - 2]) { if (intervals[1] > intervals[numOfBins - 2]) {
@ -1181,46 +1200,51 @@ static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* err
} }
} else if (cJSON_IsArray(binDesc)) { /* user input bins */ } else if (cJSON_IsArray(binDesc)) { /* user input bins */
if (binType != USER_INPUT_BIN) { if (binType != USER_INPUT_BIN) {
snprintf(errMsg, msgLen, "%s", msg3); (void)snprintf(errMsg, msgLen, "%s", msg3);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
numOfBins = cJSON_GetArraySize(binDesc); numOfBins = cJSON_GetArraySize(binDesc);
intervals = taosMemoryCalloc(numOfBins, sizeof(double)); intervals = taosMemoryCalloc(numOfBins, sizeof(double));
if (intervals == NULL) {
(void)snprintf(errMsg, msgLen, "%s", msg9);
cJSON_Delete(binDesc);
return TSDB_CODE_OUT_OF_MEMORY;
}
cJSON* bin = binDesc->child; cJSON* bin = binDesc->child;
if (bin == NULL) { if (bin == NULL) {
snprintf(errMsg, msgLen, "%s", msg3); (void)snprintf(errMsg, msgLen, "%s", msg3);
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
int i = 0; int i = 0;
while (bin) { while (bin) {
intervals[i] = bin->valuedouble; intervals[i] = bin->valuedouble;
if (!cJSON_IsNumber(bin)) { if (!cJSON_IsNumber(bin)) {
snprintf(errMsg, msgLen, "%s", msg3); (void)snprintf(errMsg, msgLen, "%s", msg3);
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
if (i != 0 && intervals[i] <= intervals[i - 1]) { if (i != 0 && intervals[i] <= intervals[i - 1]) {
snprintf(errMsg, msgLen, "%s", msg3); (void)snprintf(errMsg, msgLen, "%s", msg3);
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
bin = bin->next; bin = bin->next;
i++; i++;
} }
} else { } else {
snprintf(errMsg, msgLen, "%s", msg3); (void)snprintf(errMsg, msgLen, "%s", msg3);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
taosMemoryFree(intervals); taosMemoryFree(intervals);
return true; return TSDB_CODE_SUCCESS;
} }
static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
@ -1265,7 +1289,7 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l
if (i == 2) { if (i == 2) {
char errMsg[128] = {0}; char errMsg[128] = {0};
binDesc = varDataVal(pValue->datum.p); binDesc = varDataVal(pValue->datum.p);
if (!validateHistogramBinDesc(binDesc, binType, errMsg, (int32_t)sizeof(errMsg))) { if (TSDB_CODE_SUCCESS != validateHistogramBinDesc(binDesc, binType, errMsg, (int32_t)sizeof(errMsg))) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, errMsg); return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, errMsg);
} }
} }
@ -1323,7 +1347,7 @@ static int32_t translateHistogramImpl(SFunctionNode* pFunc, char* pErrBuf, int32
if (i == 2) { if (i == 2) {
char errMsg[128] = {0}; char errMsg[128] = {0};
binDesc = varDataVal(pValue->datum.p); binDesc = varDataVal(pValue->datum.p);
if (!validateHistogramBinDesc(binDesc, binType, errMsg, (int32_t)sizeof(errMsg))) { if (TSDB_CODE_SUCCESS != validateHistogramBinDesc(binDesc, binType, errMsg, (int32_t)sizeof(errMsg))) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, errMsg); return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, errMsg);
} }
} }
@ -1507,12 +1531,12 @@ static int32_t translateStateDuration(SFunctionNode* pFunc, char* pErrBuf, int32
if (numOfParams == 4) { if (numOfParams == 4) {
uint8_t dbPrec = pFunc->node.resType.precision; uint8_t dbPrec = pFunc->node.resType.precision;
int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 3)); int32_t code = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 3));
if (ret == TIME_UNIT_TOO_SMALL) { if (code == TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, return buildFuncErrMsg(pErrBuf, len, code,
"STATEDURATION function time unit parameter should be greater than db precision"); "STATEDURATION function time unit parameter should be greater than db precision");
} else if (ret == TIME_UNIT_INVALID) { } else if (code == TSDB_CODE_FUNC_TIME_UNIT_INVALID) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, return buildFuncErrMsg(pErrBuf, len, code,
"STATEDURATION function time unit parameter should be one of the following: [1b, 1u, 1a, " "STATEDURATION function time unit parameter should be one of the following: [1b, 1u, 1a, "
"1s, 1m, 1h, 1d, 1w]"); "1s, 1m, 1h, 1d, 1w]");
} }
@ -2268,13 +2292,13 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_
} }
uint8_t dbPrec = pFunc->node.resType.precision; uint8_t dbPrec = pFunc->node.resType.precision;
int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1)); int32_t code = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1));
if (ret == TIME_UNIT_TOO_SMALL) { if (code == TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, return buildFuncErrMsg(pErrBuf, len, code,
"TIMETRUNCATE function time unit parameter should be greater than db precision"); "TIMETRUNCATE function time unit parameter should be greater than db precision");
} else if (ret == TIME_UNIT_INVALID) { } else if (code == TSDB_CODE_FUNC_TIME_UNIT_INVALID) {
return buildFuncErrMsg( return buildFuncErrMsg(
pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, pErrBuf, len, code,
"TIMETRUNCATE function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]"); "TIMETRUNCATE function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]");
} }
@ -2291,7 +2315,7 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_
// add database precision as param // add database precision as param
int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
@ -2330,13 +2354,13 @@ static int32_t translateTimeDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t le
uint8_t dbPrec = pFunc->node.resType.precision; uint8_t dbPrec = pFunc->node.resType.precision;
if (3 == numOfParams) { if (3 == numOfParams) {
int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 2)); int32_t code = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 2));
if (ret == TIME_UNIT_TOO_SMALL) { if (code == TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, return buildFuncErrMsg(pErrBuf, len, code,
"TIMEDIFF function time unit parameter should be greater than db precision"); "TIMEDIFF function time unit parameter should be greater than db precision");
} else if (ret == TIME_UNIT_INVALID) { } else if (code == TSDB_CODE_FUNC_TIME_UNIT_INVALID) {
return buildFuncErrMsg( return buildFuncErrMsg(
pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, pErrBuf, len, code,
"TIMEDIFF function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]"); "TIMEDIFF function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]");
} }
} }

File diff suppressed because it is too large Load Diff

View File

@ -354,14 +354,17 @@ bool getAvgFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
return true; return true;
} }
bool avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { int32_t avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) {
if (!functionSetup(pCtx, pResultInfo)) { if (pResultInfo->initialized) {
return false; return TSDB_CODE_SUCCESS;
}
if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResultInfo)) {
return TSDB_CODE_FUNC_SETUP_ERROR;
} }
SAvgRes* pRes = GET_ROWCELL_INTERBUF(pResultInfo); SAvgRes* pRes = GET_ROWCELL_INTERBUF(pResultInfo);
memset(pRes, 0, sizeof(SAvgRes)); (void)memset(pRes, 0, sizeof(SAvgRes));
return true; return TSDB_CODE_SUCCESS;
} }
static int32_t calculateAvgBySMAInfo(SAvgRes* pRes, int32_t numOfRows, int32_t type, const SColumnDataAgg* pAgg) { static int32_t calculateAvgBySMAInfo(SAvgRes* pRes, int32_t numOfRows, int32_t type, const SColumnDataAgg* pAgg) {
@ -849,15 +852,23 @@ int32_t avgPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
int32_t resultBytes = getAvgInfoSize(); int32_t resultBytes = getAvgInfoSize();
char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
int32_t code = TSDB_CODE_SUCCESS;
memcpy(varDataVal(res), pInfo, resultBytes); if (NULL == res) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(varDataVal(res), pInfo, resultBytes);
varDataSetLen(res, resultBytes); varDataSetLen(res, resultBytes);
int32_t slotId = pCtx->pExpr->base.resSchema.slotId; int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
if(NULL == pCol) {
code = TSDB_CODE_OUT_OF_RANGE;
goto _exit;
}
colDataSetVal(pCol, pBlock->info.rows, res, false); code = colDataSetVal(pCol, pBlock->info.rows, res, false);
_exit:
taosMemoryFree(res); taosMemoryFree(res);
return pResInfo->numOfRes; return code;
} }

View File

@ -20,6 +20,7 @@
#include "tglobal.h" #include "tglobal.h"
#define __COMPARE_ACQUIRED_MAX(i, end, bm, _data, ctx, val, pos) \ #define __COMPARE_ACQUIRED_MAX(i, end, bm, _data, ctx, val, pos) \
int32_t code = TSDB_CODE_SUCCESS; \
for (; i < (end); ++i) { \ for (; i < (end); ++i) { \
if (colDataIsNull_f(bm, i)) { \ if (colDataIsNull_f(bm, i)) { \
continue; \ continue; \
@ -28,12 +29,16 @@
if ((val) < (_data)[i]) { \ if ((val) < (_data)[i]) { \
(val) = (_data)[i]; \ (val) = (_data)[i]; \
if ((ctx)->subsidiaries.num > 0) { \ if ((ctx)->subsidiaries.num > 0) { \
updateTupleData((ctx), i, (ctx)->pSrcBlock, pos); \ code = updateTupleData((ctx), i, (ctx)->pSrcBlock, pos); \
if (TSDB_CODE_SUCCESS != code) { \
return code; \
} \
} \ } \
} \ } \
} }
#define __COMPARE_ACQUIRED_MIN(i, end, bm, _data, ctx, val, pos) \ #define __COMPARE_ACQUIRED_MIN(i, end, bm, _data, ctx, val, pos) \
int32_t code = TSDB_CODE_SUCCESS; \
for (; i < (end); ++i) { \ for (; i < (end); ++i) { \
if (colDataIsNull_f(bm, i)) { \ if (colDataIsNull_f(bm, i)) { \
continue; \ continue; \
@ -42,7 +47,10 @@
if ((val) > (_data)[i]) { \ if ((val) > (_data)[i]) { \
(val) = (_data)[i]; \ (val) = (_data)[i]; \
if ((ctx)->subsidiaries.num > 0) { \ if ((ctx)->subsidiaries.num > 0) { \
updateTupleData((ctx), i, (ctx)->pSrcBlock, pos); \ code = updateTupleData((ctx), i, (ctx)->pSrcBlock, pos); \
if (TSDB_CODE_SUCCESS != code) { \
return code; \
} \
} \ } \
} \ } \
} }
@ -571,7 +579,7 @@ static int32_t findRowIndex(int32_t start, int32_t num, SColumnInfoData* pCol, c
return -1; return -1;
} }
static void doExtractVal(SColumnInfoData* pCol, int32_t i, int32_t end, SqlFunctionCtx* pCtx, SMinmaxResInfo* pBuf, static int32_t doExtractVal(SColumnInfoData* pCol, int32_t i, int32_t end, SqlFunctionCtx* pCtx, SMinmaxResInfo* pBuf,
bool isMinFunc) { bool isMinFunc) {
if (isMinFunc) { if (isMinFunc) {
switch (pCol->info.type) { switch (pCol->info.type) {
@ -700,6 +708,7 @@ static void doExtractVal(SColumnInfoData* pCol, int32_t i, int32_t end, SqlFunct
} }
} }
} }
return TSDB_CODE_SUCCESS;
} }
static int32_t saveRelatedTupleTag(SqlFunctionCtx* pCtx, SInputColumnInfoData* pInput, void* tval) { static int32_t saveRelatedTupleTag(SqlFunctionCtx* pCtx, SInputColumnInfoData* pInput, void* tval) {
@ -840,7 +849,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems)
break; break;
} }
default: default:
memcpy(&pBuf->v, p, pCol->info.bytes); (void)memcpy(&pBuf->v, p, pCol->info.bytes);
break; break;
} }
@ -858,7 +867,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems)
goto _over; goto _over;
} }
doExtractVal(pCol, i, end, pCtx, pBuf, isMinFunc); code = doExtractVal(pCol, i, end, pCtx, pBuf, isMinFunc);
} else { } else {
numOfElems = numOfRows; numOfElems = numOfRows;

View File

@ -35,14 +35,14 @@ static void doInitFunctionTable() {
gFunMgtService.pFuncNameHashTable = gFunMgtService.pFuncNameHashTable =
taosHashInit(funcMgtBuiltinsNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); taosHashInit(funcMgtBuiltinsNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
if (NULL == gFunMgtService.pFuncNameHashTable) { if (NULL == gFunMgtService.pFuncNameHashTable) {
initFunctionCode = TSDB_CODE_FAILED; initFunctionCode = terrno;
return; return;
} }
for (int32_t i = 0; i < funcMgtBuiltinsNum; ++i) { for (int32_t i = 0; i < funcMgtBuiltinsNum; ++i) {
if (TSDB_CODE_SUCCESS != taosHashPut(gFunMgtService.pFuncNameHashTable, funcMgtBuiltins[i].name, if (TSDB_CODE_SUCCESS != taosHashPut(gFunMgtService.pFuncNameHashTable, funcMgtBuiltins[i].name,
strlen(funcMgtBuiltins[i].name), &i, sizeof(int32_t))) { strlen(funcMgtBuiltins[i].name), &i, sizeof(int32_t))) {
initFunctionCode = TSDB_CODE_FAILED; initFunctionCode = terrno;
return; return;
} }
} }
@ -61,7 +61,7 @@ static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) {
} }
int32_t fmFuncMgtInit() { int32_t fmFuncMgtInit() {
taosThreadOnce(&functionHashTableInit, doInitFunctionTable); (void)taosThreadOnce(&functionHashTableInit, doInitFunctionTable);
return initFunctionCode; return initFunctionCode;
} }
@ -115,20 +115,24 @@ EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWin
return funcMgtBuiltins[pFunc->funcId].dataRequiredFunc(pFunc, pTimeWindow); return funcMgtBuiltins[pFunc->funcId].dataRequiredFunc(pFunc, pTimeWindow);
} }
EFuncDataRequired fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo) { int32_t fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo, int32_t *reqStatus) {
if (fmIsUserDefinedFunc(funcId) || funcId < 0 || funcId >= funcMgtBuiltinsNum) { if (fmIsUserDefinedFunc(funcId) || funcId < 0 || funcId >= funcMgtBuiltinsNum) {
*reqStatus = -1;
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
const char* name = funcMgtBuiltins[funcId].name; const char* name = funcMgtBuiltins[funcId].name;
if ((strcmp(name, "_group_key") == 0) || (strcmp(name, "_select_value") == 0)) { if ((strcmp(name, "_group_key") == 0) || (strcmp(name, "_select_value") == 0)) {
return FUNC_DATA_REQUIRED_NOT_LOAD; *reqStatus = FUNC_DATA_REQUIRED_NOT_LOAD;
return TSDB_CODE_SUCCESS;;
} }
if (funcMgtBuiltins[funcId].dynDataRequiredFunc == NULL) { if (funcMgtBuiltins[funcId].dynDataRequiredFunc == NULL) {
return FUNC_DATA_REQUIRED_DATA_LOAD; *reqStatus = FUNC_DATA_REQUIRED_DATA_LOAD;
return TSDB_CODE_SUCCESS;
} else { } else {
return funcMgtBuiltins[funcId].dynDataRequiredFunc(pRes, pBlockInfo); *reqStatus = funcMgtBuiltins[funcId].dynDataRequiredFunc(pRes, pBlockInfo);
return TSDB_CODE_SUCCESS;
} }
} }
@ -378,29 +382,30 @@ static int32_t getFuncInfo(SFunctionNode* pFunc) {
return fmGetFuncInfo(pFunc, msg, sizeof(msg)); return fmGetFuncInfo(pFunc, msg, sizeof(msg));
} }
SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList) { int32_t createFunction(const char* pName, SNodeList* pParameterList, SFunctionNode** pFunc) {
SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); *pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
if (NULL == pFunc) { if (NULL == *pFunc) {
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", pName); (void)snprintf((*pFunc)->functionName, sizeof((*pFunc)->functionName), "%s", pName);
pFunc->pParameterList = pParameterList; (*pFunc)->pParameterList = pParameterList;
if (TSDB_CODE_SUCCESS != getFuncInfo(pFunc)) { int32_t code = getFuncInfo((*pFunc));
pFunc->pParameterList = NULL; if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode((SNode*)pFunc); (*pFunc)->pParameterList = NULL;
return NULL; nodesDestroyNode((SNode*)*pFunc);
return code;
} }
return pFunc; return code;
} }
static SNode* createColumnByFunc(const SFunctionNode* pFunc) { static int32_t createColumnByFunc(const SFunctionNode* pFunc, SColumnNode** pCol) {
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); *pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) { if (NULL == *pCol) {
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
strcpy(pCol->colName, pFunc->node.aliasName); (void)strcpy((*pCol)->colName, pFunc->node.aliasName);
pCol->node.resType = pFunc->node.resType; (*pCol)->node.resType = pFunc->node.resType;
return (SNode*)pCol; return TSDB_CODE_SUCCESS;
} }
bool fmIsDistExecFunc(int32_t funcId) { bool fmIsDistExecFunc(int32_t funcId) {
@ -418,17 +423,17 @@ static int32_t createPartialFunction(const SFunctionNode* pSrcFunc, SFunctionNod
if (NULL == pParameterList) { if (NULL == pParameterList) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
*pPartialFunc = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pPartialFunc, pParameterList); int32_t code = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pPartialFunc, pParameterList,pPartialFunc );
if (NULL == *pPartialFunc) { if (TSDB_CODE_SUCCESS != code) {
nodesDestroyList(pParameterList); nodesDestroyList(pParameterList);
return TSDB_CODE_OUT_OF_MEMORY; return code;
} }
(*pPartialFunc)->hasOriginalFunc = true; (*pPartialFunc)->hasOriginalFunc = true;
(*pPartialFunc)->originalFuncId = pSrcFunc->hasOriginalFunc ? pSrcFunc->originalFuncId : pSrcFunc->funcId; (*pPartialFunc)->originalFuncId = pSrcFunc->hasOriginalFunc ? pSrcFunc->originalFuncId : pSrcFunc->funcId;
char name[TSDB_FUNC_NAME_LEN + TSDB_NAME_DELIMITER_LEN + TSDB_POINTER_PRINT_BYTES + 1] = {0}; char name[TSDB_FUNC_NAME_LEN + TSDB_NAME_DELIMITER_LEN + TSDB_POINTER_PRINT_BYTES + 1] = {0};
int32_t len = snprintf(name, sizeof(name) - 1, "%s.%p", (*pPartialFunc)->functionName, pSrcFunc); int32_t len = snprintf(name, sizeof(name) - 1, "%s.%p", (*pPartialFunc)->functionName, pSrcFunc);
taosCreateMD5Hash(name, len); (void)taosCreateMD5Hash(name, len);
strncpy((*pPartialFunc)->node.aliasName, name, TSDB_COL_NAME_LEN - 1); (void)strncpy((*pPartialFunc)->node.aliasName, name, TSDB_COL_NAME_LEN - 1);
(*pPartialFunc)->hasPk = pSrcFunc->hasPk; (*pPartialFunc)->hasPk = pSrcFunc->hasPk;
(*pPartialFunc)->pkBytes = pSrcFunc->pkBytes; (*pPartialFunc)->pkBytes = pSrcFunc->pkBytes;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -436,7 +441,11 @@ static int32_t createPartialFunction(const SFunctionNode* pSrcFunc, SFunctionNod
static int32_t createMergeFuncPara(const SFunctionNode* pSrcFunc, const SFunctionNode* pPartialFunc, static int32_t createMergeFuncPara(const SFunctionNode* pSrcFunc, const SFunctionNode* pPartialFunc,
SNodeList** pParameterList) { SNodeList** pParameterList) {
SNode* pRes = createColumnByFunc(pPartialFunc); SNode *pRes = NULL;
int32_t code = createColumnByFunc(pPartialFunc, (SColumnNode**)&pRes);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
if (NULL != funcMgtBuiltins[pSrcFunc->funcId].createMergeParaFuc) { if (NULL != funcMgtBuiltins[pSrcFunc->funcId].createMergeParaFuc) {
return funcMgtBuiltins[pSrcFunc->funcId].createMergeParaFuc(pSrcFunc->pParameterList, pRes, pParameterList); return funcMgtBuiltins[pSrcFunc->funcId].createMergeParaFuc(pSrcFunc->pParameterList, pRes, pParameterList);
} else { } else {
@ -452,16 +461,13 @@ static int32_t createMidFunction(const SFunctionNode* pSrcFunc, const SFunctionN
int32_t code = createMergeFuncPara(pSrcFunc, pPartialFunc, &pParameterList); int32_t code = createMergeFuncPara(pSrcFunc, pPartialFunc, &pParameterList);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
if(funcMgtBuiltins[pSrcFunc->funcId].pMiddleFunc != NULL){ if(funcMgtBuiltins[pSrcFunc->funcId].pMiddleFunc != NULL){
pFunc = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMiddleFunc, pParameterList); code = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMiddleFunc, pParameterList, &pFunc);
}else{ }else{
pFunc = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMergeFunc, pParameterList); code = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMergeFunc, pParameterList, &pFunc);
}
if (NULL == pFunc) {
code = TSDB_CODE_OUT_OF_MEMORY;
} }
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
strcpy(pFunc->node.aliasName, pPartialFunc->node.aliasName); (void)strcpy(pFunc->node.aliasName, pPartialFunc->node.aliasName);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
@ -481,10 +487,7 @@ static int32_t createMergeFunction(const SFunctionNode* pSrcFunc, const SFunctio
int32_t code = createMergeFuncPara(pSrcFunc, pPartialFunc, &pParameterList); int32_t code = createMergeFuncPara(pSrcFunc, pPartialFunc, &pParameterList);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
pFunc = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMergeFunc, pParameterList); code = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMergeFunc, pParameterList, &pFunc);
if (NULL == pFunc) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
pFunc->hasOriginalFunc = true; pFunc->hasOriginalFunc = true;
@ -493,7 +496,7 @@ static int32_t createMergeFunction(const SFunctionNode* pSrcFunc, const SFunctio
if (fmIsSameInOutType(pSrcFunc->funcId)) { if (fmIsSameInOutType(pSrcFunc->funcId)) {
pFunc->node.resType = pSrcFunc->node.resType; pFunc->node.resType = pSrcFunc->node.resType;
} }
strcpy(pFunc->node.aliasName, pSrcFunc->node.aliasName); (void)strcpy(pFunc->node.aliasName, pSrcFunc->node.aliasName);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
@ -541,13 +544,13 @@ static int32_t fmCreateStateFunc(const SFunctionNode* pFunc, SFunctionNode** pSt
if (funcMgtBuiltins[pFunc->funcId].pStateFunc) { if (funcMgtBuiltins[pFunc->funcId].pStateFunc) {
SNodeList* pParams = nodesCloneList(pFunc->pParameterList); SNodeList* pParams = nodesCloneList(pFunc->pParameterList);
if (!pParams) return TSDB_CODE_OUT_OF_MEMORY; if (!pParams) return TSDB_CODE_OUT_OF_MEMORY;
*pStateFunc = createFunction(funcMgtBuiltins[pFunc->funcId].pStateFunc, pParams); int32_t code = createFunction(funcMgtBuiltins[pFunc->funcId].pStateFunc, pParams, pStateFunc);
if (!*pStateFunc) { if (TSDB_CODE_SUCCESS != code) {
nodesDestroyList(pParams); nodesDestroyList(pParams);
return TSDB_CODE_FUNC_FUNTION_ERROR; return TSDB_CODE_FUNC_FUNTION_ERROR;
} }
strcpy((*pStateFunc)->node.aliasName, pFunc->node.aliasName); (void)strcpy((*pStateFunc)->node.aliasName, pFunc->node.aliasName);
strcpy((*pStateFunc)->node.userAlias, pFunc->node.userAlias); (void)strcpy((*pStateFunc)->node.userAlias, pFunc->node.userAlias);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -587,13 +590,13 @@ static int32_t fmCreateStateMergeFunc(SFunctionNode* pFunc, SFunctionNode** pSta
if (funcMgtBuiltins[pFunc->funcId].pMergeFunc) { if (funcMgtBuiltins[pFunc->funcId].pMergeFunc) {
SNodeList* pParams = nodesCloneList(pFunc->pParameterList); SNodeList* pParams = nodesCloneList(pFunc->pParameterList);
if (!pParams) return TSDB_CODE_OUT_OF_MEMORY; if (!pParams) return TSDB_CODE_OUT_OF_MEMORY;
*pStateMergeFunc = createFunction(funcMgtBuiltins[pFunc->funcId].pMergeFunc, pParams); int32_t code = createFunction(funcMgtBuiltins[pFunc->funcId].pMergeFunc, pParams, pStateMergeFunc);
if (!*pStateMergeFunc) { if (TSDB_CODE_SUCCESS != code) {
nodesDestroyList(pParams); nodesDestroyList(pParams);
return TSDB_CODE_FUNC_FUNTION_ERROR; return code;
} }
strcpy((*pStateMergeFunc)->node.aliasName, pFunc->node.aliasName); (void)strcpy((*pStateMergeFunc)->node.aliasName, pFunc->node.aliasName);
strcpy((*pStateMergeFunc)->node.userAlias, pFunc->node.userAlias); (void)strcpy((*pStateMergeFunc)->node.userAlias, pFunc->node.userAlias);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -646,6 +649,9 @@ bool fmIsMyStateFunc(int32_t funcId, int32_t stateFuncId) {
} }
if (strcmp(pFunc->pStateFunc, pStateFunc->name) == 0) return true; if (strcmp(pFunc->pStateFunc, pStateFunc->name) == 0) return true;
int32_t stateMergeFuncId = fmGetFuncId(pFunc->pStateFunc); int32_t stateMergeFuncId = fmGetFuncId(pFunc->pStateFunc);
if (stateMergeFuncId == -1) {
return false;
}
const SBuiltinFuncDefinition* pStateMergeFunc = &funcMgtBuiltins[stateMergeFuncId]; const SBuiltinFuncDefinition* pStateMergeFunc = &funcMgtBuiltins[stateMergeFuncId];
return strcmp(pStateFunc->name, pStateMergeFunc->pMergeFunc) == 0; return strcmp(pStateFunc->name, pStateMergeFunc->pMergeFunc) == 0;
} }

View File

@ -32,9 +32,12 @@
*/ */
static int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val); static int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val);
SHistogramInfo* tHistogramCreate(int32_t numOfEntries) { int32_t tHistogramCreate(int32_t numOfEntries, SHistogramInfo** pHisto) {
/* need one redundant slot */ /* need one redundant slot */
SHistogramInfo* pHisto = taosMemoryMalloc(sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfEntries + 1)); *pHisto = taosMemoryMalloc(sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfEntries + 1));
if (NULL == *pHisto) {
return TSDB_CODE_OUT_OF_MEMORY;
}
#if !defined(USE_ARRAYLIST) #if !defined(USE_ARRAYLIST)
pHisto->pList = SSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); pHisto->pList = SSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
@ -46,11 +49,12 @@ SHistogramInfo* tHistogramCreate(int32_t numOfEntries) {
pss->pTree = pHisto->pLoserTree; pss->pTree = pHisto->pLoserTree;
#endif #endif
return tHistogramCreateFrom(pHisto, numOfEntries); *pHisto = tHistogramCreateFrom(*pHisto, numOfEntries);
return TSDB_CODE_SUCCESS;
} }
SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins) { SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins) {
memset(pBuf, 0, sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfBins + 1)); (void)memset(pBuf, 0, sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfBins + 1));
SHistogramInfo* pHisto = (SHistogramInfo*)pBuf; SHistogramInfo* pHisto = (SHistogramInfo*)pBuf;
pHisto->elems = (SHistBin*)((char*)pBuf + sizeof(SHistogramInfo)); pHisto->elems = (SHistBin*)((char*)pBuf + sizeof(SHistogramInfo));
@ -67,15 +71,19 @@ SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins) {
} }
int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
int32_t code = TSDB_CODE_SUCCESS;
if (*pHisto == NULL) { if (*pHisto == NULL) {
*pHisto = tHistogramCreate(MAX_HISTOGRAM_BIN); code = tHistogramCreate(MAX_HISTOGRAM_BIN, pHisto);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
} }
#if defined(USE_ARRAYLIST) #if defined(USE_ARRAYLIST)
int32_t idx = histoBinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val); int32_t idx = histoBinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val);
if (ASSERTS(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL, "tHistogramAdd Error, idx:%d, maxEntries:%d, elems:%p", if (ASSERTS(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL, "tHistogramAdd Error, idx:%d, maxEntries:%d, elems:%p",
idx, (*pHisto)->maxEntries, (*pHisto)->elems)) { idx, (*pHisto)->maxEntries, (*pHisto)->elems)) {
return -1; return TSDB_CODE_FAILED;
} }
if ((*pHisto)->elems[idx].val == val && idx >= 0) { if ((*pHisto)->elems[idx].val == val && idx >= 0) {
@ -89,23 +97,23 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
if (idx > 0) { if (idx > 0) {
if (ASSERTS((*pHisto)->elems[idx - 1].val <= val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf", if (ASSERTS((*pHisto)->elems[idx - 1].val <= val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf",
idx - 1, (*pHisto)->elems[idx - 1].val, val)) { idx - 1, (*pHisto)->elems[idx - 1].val, val)) {
return -1; return TSDB_CODE_FAILED;
} }
} else { } else {
if (ASSERTS((*pHisto)->elems[idx].val > val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf", if (ASSERTS((*pHisto)->elems[idx].val > val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf",
idx, (*pHisto)->elems[idx].val, val)) { idx, (*pHisto)->elems[idx].val, val)) {
return -1; return TSDB_CODE_FAILED;
} }
} }
} else if ((*pHisto)->numOfElems > 0) { } else if ((*pHisto)->numOfElems > 0) {
if (ASSERTS((*pHisto)->elems[(*pHisto)->numOfEntries].val <= val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf", if (ASSERTS((*pHisto)->elems[(*pHisto)->numOfEntries].val <= val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf",
(*pHisto)->numOfEntries, (*pHisto)->elems[idx].val, val)) { (*pHisto)->numOfEntries, (*pHisto)->elems[idx].val, val)) {
return -1; return TSDB_CODE_FAILED;
} }
} }
int32_t code = histogramCreateBin(*pHisto, idx, val); code = histogramCreateBin(*pHisto, idx, val);
if (code != 0) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
} }
@ -286,7 +294,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
} }
(*pHisto)->numOfElems += 1; (*pHisto)->numOfElems += 1;
return 0; return code;
} }
int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val) { int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val) {
@ -335,7 +343,7 @@ static void histogramMergeImpl(SHistBin* pHistBin, int32_t* size) {
s1->val = newVal; s1->val = newVal;
s1->num = s1->num + s2->num; s1->num = s1->num + s2->num;
memmove(&pHistBin[index + 1], &pHistBin[index + 2], (oldSize - index - 2) * sizeof(SHistBin)); (void)memmove(&pHistBin[index + 1], &pHistBin[index + 2], (oldSize - index - 2) * sizeof(SHistBin));
(*size) -= 1; (*size) -= 1;
#endif #endif
} }
@ -345,12 +353,12 @@ int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val) {
#if defined(USE_ARRAYLIST) #if defined(USE_ARRAYLIST)
int32_t remain = pHisto->numOfEntries - index; int32_t remain = pHisto->numOfEntries - index;
if (remain > 0) { if (remain > 0) {
memmove(&pHisto->elems[index + 1], &pHisto->elems[index], sizeof(SHistBin) * remain); (void)memmove(&pHisto->elems[index + 1], &pHisto->elems[index], sizeof(SHistBin) * remain);
} }
if (ASSERTS(index >= 0 && index <= pHisto->maxEntries, "histogramCreateBin Error, index:%d, maxEntries:%d", if (ASSERTS(index >= 0 && index <= pHisto->maxEntries, "histogramCreateBin Error, index:%d, maxEntries:%d",
index, pHisto->maxEntries)) { index, pHisto->maxEntries)) {
return -1; return TSDB_CODE_FAILED;
} }
pHisto->elems[index].num = 1; pHisto->elems[index].num = 1;
@ -367,10 +375,10 @@ int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val) {
#endif #endif
if (ASSERTS(pHisto->numOfEntries <= pHisto->maxEntries, "histogramCreateBin Error, numOfEntries:%d, maxEntries:%d", if (ASSERTS(pHisto->numOfEntries <= pHisto->maxEntries, "histogramCreateBin Error, numOfEntries:%d, maxEntries:%d",
pHisto->numOfEntries, pHisto->maxEntries)) { pHisto->numOfEntries, pHisto->maxEntries)) {
return -1; return TSDB_CODE_FAILED;
} }
return 0; return TSDB_CODE_SUCCESS;
} }
void tHistogramDestroy(SHistogramInfo** pHisto) { void tHistogramDestroy(SHistogramInfo** pHisto) {
@ -383,17 +391,17 @@ void tHistogramDestroy(SHistogramInfo** pHisto) {
} }
void tHistogramPrint(SHistogramInfo* pHisto) { void tHistogramPrint(SHistogramInfo* pHisto) {
printf("total entries: %d, elements: %" PRId64 "\n", pHisto->numOfEntries, pHisto->numOfElems); (void)printf("total entries: %d, elements: %" PRId64 "\n", pHisto->numOfEntries, pHisto->numOfElems);
#if defined(USE_ARRAYLIST) #if defined(USE_ARRAYLIST)
for (int32_t i = 0; i < pHisto->numOfEntries; ++i) { for (int32_t i = 0; i < pHisto->numOfEntries; ++i) {
printf("%d: (%f, %" PRId64 ")\n", i + 1, pHisto->elems[i].val, pHisto->elems[i].num); (void)printf("%d: (%f, %" PRId64 ")\n", i + 1, pHisto->elems[i].val, pHisto->elems[i].num);
} }
#else #else
tSkipListNode* pNode = pHisto->pList->pHead.pForward[0]; tSkipListNode* pNode = pHisto->pList->pHead.pForward[0];
for (int32_t i = 0; i < pHisto->numOfEntries; ++i) { for (int32_t i = 0; i < pHisto->numOfEntries; ++i) {
SHistBin* pEntry = (SHistBin*)pNode->pData; SHistBin* pEntry = (SHistBin*)pNode->pData;
printf("%d: (%f, %" PRId64 ")\n", i + 1, pEntry->val, pEntry->num); (void)printf("%d: (%f, %" PRId64 ")\n", i + 1, pEntry->val, pEntry->num);
pNode = pNode->pForward[0]; pNode = pNode->pForward[0];
} }
#endif #endif
@ -443,21 +451,24 @@ int64_t tHistogramSum(SHistogramInfo* pHisto, double v) {
#endif #endif
} }
double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) { int32_t tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num, double** pVal) {
#if defined(USE_ARRAYLIST) #if defined(USE_ARRAYLIST)
double* pVal = taosMemoryMalloc(num * sizeof(double)); *pVal = taosMemoryMalloc(num * sizeof(double));
if (NULL == *pVal) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
double numOfElem = (ratio[i] / 100) * pHisto->numOfElems; double numOfElem = (ratio[i] / 100) * pHisto->numOfElems;
if (numOfElem == 0) { if (numOfElem == 0) {
pVal[i] = pHisto->min; (*pVal)[i] = pHisto->min;
continue; continue;
} else if (numOfElem <= pHisto->elems[0].num) { } else if (numOfElem <= pHisto->elems[0].num) {
pVal[i] = pHisto->elems[0].val; (*pVal)[i] = pHisto->elems[0].val;
continue; continue;
} else if (numOfElem == pHisto->numOfElems) { } else if (numOfElem == pHisto->numOfElems) {
pVal[i] = pHisto->max; (*pVal)[i] = pHisto->max;
continue; continue;
} }
@ -479,37 +490,39 @@ double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) {
double delta = numOfElem - total; double delta = numOfElem - total;
if (fabs(delta) < FLT_EPSILON) { if (fabs(delta) < FLT_EPSILON) {
pVal[i] = pHisto->elems[j].val; (*pVal)[i] = pHisto->elems[j].val;
} }
double start = (double)pHisto->elems[j].num; double start = (double)pHisto->elems[j].num;
double range = pHisto->elems[j + 1].num - start; double range = pHisto->elems[j + 1].num - start;
if (range == 0) { if (range == 0) {
pVal[i] = (pHisto->elems[j + 1].val - pHisto->elems[j].val) * delta / start + pHisto->elems[j].val; (*pVal)[i] = (pHisto->elems[j + 1].val - pHisto->elems[j].val) * delta / start + pHisto->elems[j].val;
} else { } else {
double factor = (-2 * start + sqrt(4 * start * start - 4 * range * (-2 * delta))) / (2 * range); double factor = (-2 * start + sqrt(4 * start * start - 4 * range * (-2 * delta))) / (2 * range);
pVal[i] = pHisto->elems[j].val + (pHisto->elems[j + 1].val - pHisto->elems[j].val) * factor; (*pVal)[i] = pHisto->elems[j].val + (pHisto->elems[j + 1].val - pHisto->elems[j].val) * factor;
} }
} }
#else #else
double* pVal = taosMemoryMalloc(num * sizeof(double)); double* pVal = taosMemoryMalloc(num * sizeof(double));
if (NULL == *pVal) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
double numOfElem = ratio[i] * pHisto->numOfElems; double numOfElem = ratio[i] * pHisto->numOfElems;
tSkipListNode* pFirst = pHisto->pList->pHead.pForward[0]; tSkipListNode* pFirst = pHisto->pList->pHead.pForward[0];
SHistBin* pEntry = (SHistBin*)pFirst->pData; SHistBin* pEntry = (SHistBin*)pFirst->pData;
if (numOfElem == 0) { if (numOfElem == 0) {
pVal[i] = pHisto->min; (*pVal)[i] = pHisto->min;
printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]); printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]);
continue; continue;
} else if (numOfElem <= pEntry->num) { } else if (numOfElem <= pEntry->num) {
pVal[i] = pEntry->val; (*pVal)[i] = pEntry->val;
printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]); printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]);
continue; continue;
} else if (numOfElem == pHisto->numOfElems) { } else if (numOfElem == pHisto->numOfElems) {
pVal[i] = pHisto->max; (*pVal)[i] = pHisto->max;
printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]); printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]);
continue; continue;
} }
@ -540,34 +553,40 @@ double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) {
if (fabs(delta) < FLT_EPSILON) { if (fabs(delta) < FLT_EPSILON) {
// printf("i/numofSlot: %f, v:%f, %f\n", // printf("i/numofSlot: %f, v:%f, %f\n",
// (double)i/numOfSlots, numOfElem, pHisto->elems[j].val); // (double)i/numOfSlots, numOfElem, pHisto->elems[j].val);
pVal[i] = pPrev->val; (*pVal)[i] = pPrev->val;
} }
double start = pPrev->num; double start = pPrev->num;
double range = pEntry->num - start; double range = pEntry->num - start;
if (range == 0) { if (range == 0) {
pVal[i] = (pEntry->val - pPrev->val) * delta / start + pPrev->val; (*pVal)[i] = (pEntry->val - pPrev->val) * delta / start + pPrev->val;
} else { } else {
double factor = (-2 * start + sqrt(4 * start * start - 4 * range * (-2 * delta))) / (2 * range); double factor = (-2 * start + sqrt(4 * start * start - 4 * range * (-2 * delta))) / (2 * range);
pVal[i] = pPrev->val + (pEntry->val - pPrev->val) * factor; (*pVal)[i] = pPrev->val + (pEntry->val - pPrev->val) * factor;
} }
// printf("i/numofSlot: %f, v:%f, %f\n", (double)i/numOfSlots, // printf("i/numofSlot: %f, v:%f, %f\n", (double)i/numOfSlots,
// numOfElem, val); // numOfElem, val);
} }
#endif #endif
return pVal; return TSDB_CODE_SUCCESS;
} }
SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries) { int32_t tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries, SHistogramInfo** pResHistogram) {
SHistogramInfo* pResHistogram = tHistogramCreate(numOfEntries); int32_t code = tHistogramCreate(numOfEntries, pResHistogram);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
// error in histogram info // error in histogram info
if (pHisto1->numOfEntries > MAX_HISTOGRAM_BIN || pHisto2->numOfEntries > MAX_HISTOGRAM_BIN) { if (pHisto1->numOfEntries > MAX_HISTOGRAM_BIN || pHisto2->numOfEntries > MAX_HISTOGRAM_BIN) {
return pResHistogram; return code;
} }
SHistBin* pHistoBins = taosMemoryCalloc(1, sizeof(SHistBin) * (pHisto1->numOfEntries + pHisto2->numOfEntries)); SHistBin* pHistoBins = taosMemoryCalloc(1, sizeof(SHistBin) * (pHisto1->numOfEntries + pHisto2->numOfEntries));
if (NULL == pHistoBins) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t i = 0, j = 0, k = 0; int32_t i = 0, j = 0, k = 0;
while (i < pHisto1->numOfEntries && j < pHisto2->numOfEntries) { while (i < pHisto1->numOfEntries && j < pHisto2->numOfEntries) {
@ -583,28 +602,28 @@ SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2
if (i < pHisto1->numOfEntries) { if (i < pHisto1->numOfEntries) {
int32_t remain = pHisto1->numOfEntries - i; int32_t remain = pHisto1->numOfEntries - i;
memcpy(&pHistoBins[k], &pHisto1->elems[i], sizeof(SHistBin) * remain); (void)memcpy(&pHistoBins[k], &pHisto1->elems[i], sizeof(SHistBin) * remain);
k += remain; k += remain;
} }
if (j < pHisto2->numOfEntries) { if (j < pHisto2->numOfEntries) {
int32_t remain = pHisto2->numOfEntries - j; int32_t remain = pHisto2->numOfEntries - j;
memcpy(&pHistoBins[k], &pHisto2->elems[j], sizeof(SHistBin) * remain); (void)memcpy(&pHistoBins[k], &pHisto2->elems[j], sizeof(SHistBin) * remain);
k += remain; k += remain;
} }
/* update other information */ /* update other information */
pResHistogram->numOfElems = pHisto1->numOfElems + pHisto2->numOfElems; (*pResHistogram)->numOfElems = pHisto1->numOfElems + pHisto2->numOfElems;
pResHistogram->min = (pHisto1->min < pHisto2->min) ? pHisto1->min : pHisto2->min; (*pResHistogram)->min = (pHisto1->min < pHisto2->min) ? pHisto1->min : pHisto2->min;
pResHistogram->max = (pHisto1->max > pHisto2->max) ? pHisto1->max : pHisto2->max; (*pResHistogram)->max = (pHisto1->max > pHisto2->max) ? pHisto1->max : pHisto2->max;
while (k > numOfEntries) { while (k > numOfEntries) {
histogramMergeImpl(pHistoBins, &k); histogramMergeImpl(pHistoBins, &k);
} }
pResHistogram->numOfEntries = k; (*pResHistogram)->numOfEntries = k;
memcpy(pResHistogram->elems, pHistoBins, sizeof(SHistBin) * k); (void)memcpy((*pResHistogram)->elems, pHistoBins, sizeof(SHistBin) * k);
taosMemoryFree(pHistoBins); taosMemoryFree(pHistoBins);
return pResHistogram; return TSDB_CODE_SUCCESS;
} }

View File

@ -28,9 +28,12 @@
int32_t getGroupId(int32_t numOfSlots, int32_t slotIndex, int32_t times) { return (times * numOfSlots) + slotIndex; } int32_t getGroupId(int32_t numOfSlots, int32_t slotIndex, int32_t times) { return (times * numOfSlots) + slotIndex; }
static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) { static int32_t loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx, SFilePage ** buffer) {
SFilePage *buffer = *buffer =
(SFilePage *)taosMemoryCalloc(1, pMemBucket->bytes * pMemBucket->pSlots[slotIdx].info.size + sizeof(SFilePage)); (SFilePage *)taosMemoryCalloc(1, pMemBucket->bytes * pMemBucket->pSlots[slotIdx].info.size + sizeof(SFilePage));
if (NULL == *buffer) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t groupId = getGroupId(pMemBucket->numOfSlots, slotIdx, pMemBucket->times); int32_t groupId = getGroupId(pMemBucket->numOfSlots, slotIdx, pMemBucket->times);
@ -39,26 +42,30 @@ static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx)
if (p != NULL) { if (p != NULL) {
pIdList = *(SArray **)p; pIdList = *(SArray **)p;
} else { } else {
taosMemoryFree(buffer); taosMemoryFree(*buffer);
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
int32_t offset = 0; int32_t offset = 0;
for (int32_t i = 0; i < taosArrayGetSize(pIdList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pIdList); ++i) {
int32_t *pageId = taosArrayGet(pIdList, i); int32_t *pageId = taosArrayGet(pIdList, i);
if (pageId == NULL) {
taosMemoryFree(*buffer);
return TSDB_CODE_OUT_OF_RANGE;
}
SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId); SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId);
if (pg == NULL) { if (pg == NULL) {
taosMemoryFree(buffer); taosMemoryFree(*buffer);
return NULL; return terrno;
} }
memcpy(buffer->data + offset, pg->data, (size_t)(pg->num * pMemBucket->bytes)); (void)memcpy((*buffer)->data + offset, pg->data, (size_t)(pg->num * pMemBucket->bytes));
offset += (int32_t)(pg->num * pMemBucket->bytes); offset += (int32_t)(pg->num * pMemBucket->bytes);
} }
taosSort(buffer->data, pMemBucket->pSlots[slotIdx].info.size, pMemBucket->bytes, pMemBucket->comparFn); taosSort((*buffer)->data, pMemBucket->pSlots[slotIdx].info.size, pMemBucket->bytes, pMemBucket->comparFn);
return buffer; return TSDB_CODE_SUCCESS;
} }
static void resetBoundingBox(MinMaxEntry *range, int32_t type) { static void resetBoundingBox(MinMaxEntry *range, int32_t type) {
@ -116,6 +123,9 @@ int32_t findOnlyResult(tMemBucket *pMemBucket, double *result) {
ASSERT(list->size == 1); ASSERT(list->size == 1);
int32_t *pageId = taosArrayGet(list, 0); int32_t *pageId = taosArrayGet(list, 0);
if (NULL == pageId) {
return TSDB_CODE_OUT_OF_RANGE;
}
SFilePage *pPage = getBufPage(pMemBucket->pBuffer, *pageId); SFilePage *pPage = getBufPage(pMemBucket->pBuffer, *pageId);
if (pPage == NULL) { if (pPage == NULL) {
return terrno; return terrno;
@ -238,67 +248,71 @@ static void resetSlotInfo(tMemBucket *pBucket) {
} }
} }
tMemBucket *tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval, bool hasWindowOrGroup) { int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval, bool hasWindowOrGroup,
tMemBucket *pBucket = (tMemBucket *)taosMemoryCalloc(1, sizeof(tMemBucket)); tMemBucket **pBucket) {
if (pBucket == NULL) { *pBucket = (tMemBucket *)taosMemoryCalloc(1, sizeof(tMemBucket));
return NULL; if (*pBucket == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
} }
if (hasWindowOrGroup) { if (hasWindowOrGroup) {
// With window or group by, we need to shrink page size and reduce page num to save memory. // With window or group by, we need to shrink page size and reduce page num to save memory.
pBucket->numOfSlots = DEFAULT_NUM_OF_SLOT / 8 ; // 128 bucket (*pBucket)->numOfSlots = DEFAULT_NUM_OF_SLOT / 8 ; // 128 bucket
pBucket->bufPageSize = 4096; // 4k per page (*pBucket)->bufPageSize = 4096; // 4k per page
} else { } else {
pBucket->numOfSlots = DEFAULT_NUM_OF_SLOT; (*pBucket)->numOfSlots = DEFAULT_NUM_OF_SLOT;
pBucket->bufPageSize = 16384 * 4; // 16k per page (*pBucket)->bufPageSize = 16384 * 4; // 16k per page
} }
pBucket->type = dataType; (*pBucket)->type = dataType;
pBucket->bytes = nElemSize; (*pBucket)->bytes = nElemSize;
pBucket->total = 0; (*pBucket)->total = 0;
pBucket->times = 1; (*pBucket)->times = 1;
pBucket->maxCapacity = 200000; (*pBucket)->maxCapacity = 200000;
pBucket->groupPagesMap = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); (*pBucket)->groupPagesMap = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
if (setBoundingBox(&pBucket->range, pBucket->type, minval, maxval) != 0) { if ((*pBucket)->groupPagesMap == NULL) {
tMemBucketDestroy(*pBucket);
return terrno;
}
if (setBoundingBox(&(*pBucket)->range, (*pBucket)->type, minval, maxval) != 0) {
// qError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval); // qError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval);
taosMemoryFree(pBucket); tMemBucketDestroy(*pBucket);
return NULL; return TSDB_CODE_FUNC_INVALID_VALUE_RANGE;
} }
pBucket->elemPerPage = (pBucket->bufPageSize - sizeof(SFilePage)) / pBucket->bytes; (*pBucket)->elemPerPage = ((*pBucket)->bufPageSize - sizeof(SFilePage)) / (*pBucket)->bytes;
pBucket->comparFn = getKeyComparFunc(pBucket->type, TSDB_ORDER_ASC); (*pBucket)->comparFn = getKeyComparFunc((*pBucket)->type, TSDB_ORDER_ASC);
pBucket->hashFunc = getHashFunc(pBucket->type); (*pBucket)->hashFunc = getHashFunc((*pBucket)->type);
if (pBucket->hashFunc == NULL) { if ((*pBucket)->hashFunc == NULL) {
// qError("MemBucket:%p, not support data type %d, failed", pBucket, pBucket->type); // qError("MemBucket:%p, not support data type %d, failed", pBucket, pBucket->type);
taosMemoryFree(pBucket); tMemBucketDestroy(*pBucket);
return NULL; return TSDB_CODE_FUNC_FUNTION_PARA_TYPE;
} }
pBucket->pSlots = (tMemBucketSlot *)taosMemoryCalloc(pBucket->numOfSlots, sizeof(tMemBucketSlot)); (*pBucket)->pSlots = (tMemBucketSlot *)taosMemoryCalloc((*pBucket)->numOfSlots, sizeof(tMemBucketSlot));
if (pBucket->pSlots == NULL) { if ((*pBucket)->pSlots == NULL) {
taosMemoryFree(pBucket); tMemBucketDestroy(*pBucket);
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
resetSlotInfo(pBucket); resetSlotInfo((*pBucket));
if (!osTempSpaceAvailable()) { if (!osTempSpaceAvailable()) {
terrno = TSDB_CODE_NO_DISKSPACE;
// qError("MemBucket create disk based Buf failed since %s", terrstr(terrno)); // qError("MemBucket create disk based Buf failed since %s", terrstr(terrno));
tMemBucketDestroy(pBucket); tMemBucketDestroy(*pBucket);
return NULL; return TSDB_CODE_NO_DISKSPACE;
} }
int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 1024, "1", tsTempDir); int32_t ret = createDiskbasedBuf(&(*pBucket)->pBuffer, (*pBucket)->bufPageSize, (*pBucket)->bufPageSize * 1024, "1", tsTempDir);
if (ret != 0) { if (ret != 0) {
tMemBucketDestroy(pBucket); tMemBucketDestroy(*pBucket);
return NULL; return ret;
} }
// qDebug("MemBucket:%p, elem size:%d", pBucket, pBucket->bytes); // qDebug("MemBucket:%p, elem size:%d", pBucket, pBucket->bytes);
return pBucket; return TSDB_CODE_SUCCESS;
} }
void tMemBucketDestroy(tMemBucket *pBucket) { void tMemBucketDestroy(tMemBucket *pBucket) {
@ -394,7 +408,14 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) {
void *p = taosHashGet(pBucket->groupPagesMap, &groupId, sizeof(groupId)); void *p = taosHashGet(pBucket->groupPagesMap, &groupId, sizeof(groupId));
if (p == NULL) { if (p == NULL) {
pPageIdList = taosArrayInit(4, sizeof(int32_t)); pPageIdList = taosArrayInit(4, sizeof(int32_t));
taosHashPut(pBucket->groupPagesMap, &groupId, sizeof(groupId), &pPageIdList, POINTER_BYTES); if (NULL == pPageIdList) {
return terrno;
}
int32_t code = taosHashPut(pBucket->groupPagesMap, &groupId, sizeof(groupId), &pPageIdList, POINTER_BYTES);
if (TSDB_CODE_SUCCESS != code) {
taosArrayDestroy(pPageIdList);
return code;
}
} else { } else {
pPageIdList = *(SArray **)p; pPageIdList = *(SArray **)p;
} }
@ -404,10 +425,13 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) {
return terrno; return terrno;
} }
pSlot->info.pageId = pageId; pSlot->info.pageId = pageId;
taosArrayPush(pPageIdList, &pageId); if (taosArrayPush(pPageIdList, &pageId) == NULL) {
taosArrayDestroy(pPageIdList);
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
memcpy(pSlot->info.data->data + pSlot->info.data->num * pBucket->bytes, d, pBucket->bytes); (void)memcpy(pSlot->info.data->data + pSlot->info.data->num * pBucket->bytes, d, pBucket->bytes);
pSlot->info.data->num += 1; pSlot->info.data->num += 1;
pSlot->info.size += 1; pSlot->info.size += 1;
@ -493,9 +517,10 @@ int32_t getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction
if (pSlot->info.size <= pMemBucket->maxCapacity) { if (pSlot->info.size <= pMemBucket->maxCapacity) {
// data in buffer and file are merged together to be processed. // data in buffer and file are merged together to be processed.
SFilePage *buffer = loadDataFromFilePage(pMemBucket, i); SFilePage *buffer = NULL;
if (buffer == NULL) { int32_t code = loadDataFromFilePage(pMemBucket, i, &buffer);
return terrno; if (TSDB_CODE_SUCCESS != code) {
return code;
} }
int32_t currentIdx = count - num; int32_t currentIdx = count - num;
@ -541,6 +566,9 @@ int32_t getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction
for (int32_t f = 0; f < list->size; ++f) { for (int32_t f = 0; f < list->size; ++f) {
int32_t *pageId = taosArrayGet(list, f); int32_t *pageId = taosArrayGet(list, f);
if (NULL == pageId) {
return TSDB_CODE_OUT_OF_RANGE;
}
SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId); SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId);
if (pg == NULL) { if (pg == NULL) {
return terrno; return terrno;

View File

@ -1011,7 +1011,7 @@ void releaseUdfFuncHandle(char *udfName, UdfcFuncHandle handle);
int32_t cleanUpUdfs(); int32_t cleanUpUdfs();
bool udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv); bool udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv);
bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo); int32_t udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo);
int32_t udfAggProcess(struct SqlFunctionCtx *pCtx); int32_t udfAggProcess(struct SqlFunctionCtx *pCtx);
int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock); int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock);
@ -1196,15 +1196,18 @@ bool udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv) {
return true; return true;
} }
bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo) { int32_t udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo) {
if (functionSetup(pCtx, pResultCellInfo) != true) { if (pResultCellInfo->initialized) {
return false; return TSDB_CODE_SUCCESS;
}
if (functionSetup(pCtx, pResultCellInfo) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_FUNC_SETUP_ERROR;
} }
UdfcFuncHandle handle; UdfcFuncHandle handle;
int32_t udfCode = 0; int32_t udfCode = 0;
if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) { if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) {
fnError("udfAggInit error. step doSetupUdf. udf code: %d", udfCode); fnError("udfAggInit error. step doSetupUdf. udf code: %d", udfCode);
return false; return TSDB_CODE_FUNC_SETUP_ERROR;
} }
SUdfcUvSession *session = (SUdfcUvSession *)handle; SUdfcUvSession *session = (SUdfcUvSession *)handle;
SUdfAggRes *udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(pResultCellInfo); SUdfAggRes *udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(pResultCellInfo);
@ -1218,7 +1221,7 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResult
if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) { if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) {
fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode); fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode);
releaseUdfFuncHandle(pCtx->udfName, handle); releaseUdfFuncHandle(pCtx->udfName, handle);
return false; return TSDB_CODE_FUNC_SETUP_ERROR;
} }
if (buf.bufLen <= session->bufSize) { if (buf.bufLen <= session->bufSize) {
memcpy(udfRes->interResBuf, buf.buf, buf.bufLen); memcpy(udfRes->interResBuf, buf.buf, buf.bufLen);
@ -1227,11 +1230,11 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResult
} else { } else {
fnError("udfc inter buf size %d is greater than function bufSize %d", buf.bufLen, session->bufSize); fnError("udfc inter buf size %d is greater than function bufSize %d", buf.bufLen, session->bufSize);
releaseUdfFuncHandle(pCtx->udfName, handle); releaseUdfFuncHandle(pCtx->udfName, handle);
return false; return TSDB_CODE_FUNC_SETUP_ERROR;
} }
releaseUdfFuncHandle(pCtx->udfName, handle); releaseUdfFuncHandle(pCtx->udfName, handle);
freeUdfInterBuf(&buf); freeUdfInterBuf(&buf);
return true; return TSDB_CODE_SUCCESS;
} }
int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {

View File

@ -10420,8 +10420,9 @@ static int32_t createLastTsSelectStmt(char* pDb, const char* pTable, const char*
return code; return code;
} }
SNode* pFunc = (SNode*)createFunction("last", pParameterList); SNode* pFunc = NULL;
if (NULL == pFunc) { code = createFunction("last", pParameterList, (SFunctionNode**)&pFunc);
if (code) {
nodesDestroyList(pParameterList); nodesDestroyList(pParameterList);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
@ -10438,8 +10439,9 @@ static int32_t createLastTsSelectStmt(char* pDb, const char* pTable, const char*
return code; return code;
} }
SFunctionNode* pFunc1 = createFunction("_vgid", NULL); SFunctionNode* pFunc1 = NULL;
if (NULL == pFunc1) { code = createFunction("_vgid", NULL, &pFunc1);
if (code) {
nodesDestroyList(pProjectionList); nodesDestroyList(pProjectionList);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
@ -10451,8 +10453,9 @@ static int32_t createLastTsSelectStmt(char* pDb, const char* pTable, const char*
return code; return code;
} }
SFunctionNode* pFunc2 = createFunction("_vgver", NULL); SFunctionNode* pFunc2 = NULL;
if (NULL == pFunc2) { code = createFunction("_vgver", NULL, &pFunc2);
if (code) {
nodesDestroyList(pProjectionList); nodesDestroyList(pProjectionList);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }

View File

@ -35,7 +35,8 @@ namespace ParserTest {
class ParserEnv : public testing::Environment { class ParserEnv : public testing::Environment {
public: public:
virtual void SetUp() { virtual void SetUp() {
fmFuncMgtInit(); // TODO(smj) : How to handle return value of fmFuncMgtInit
(void)fmFuncMgtInit();
initMetaDataEnv(); initMetaDataEnv();
generateMetaData(); generateMetaData();
initLog(TD_TMP_DIR_PATH "td"); initLog(TD_TMP_DIR_PATH "td");

View File

@ -5336,9 +5336,13 @@ static bool stbJoinOptShouldBeOptimized(SLogicNode* pNode) {
} }
int32_t stbJoinOptAddFuncToScanNode(char* funcName, SScanLogicNode* pScan) { int32_t stbJoinOptAddFuncToScanNode(char* funcName, SScanLogicNode* pScan) {
SFunctionNode* pUidFunc = createFunction(funcName, NULL); SFunctionNode* pUidFunc = NULL;
int32_t code = createFunction(funcName, NULL, &pUidFunc);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
snprintf(pUidFunc->node.aliasName, sizeof(pUidFunc->node.aliasName), "%s.%p", pUidFunc->functionName, pUidFunc); snprintf(pUidFunc->node.aliasName, sizeof(pUidFunc->node.aliasName), "%s.%p", pUidFunc->functionName, pUidFunc);
int32_t code = nodesListStrictAppend(pScan->pScanPseudoCols, (SNode*)pUidFunc); code = nodesListStrictAppend(pScan->pScanPseudoCols, (SNode*)pUidFunc);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = createColumnByRewriteExpr((SNode*)pUidFunc, &pScan->node.pTargets); code = createColumnByRewriteExpr((SNode*)pUidFunc, &pScan->node.pTargets);
} }

View File

@ -27,7 +27,8 @@
class PlannerEnv : public testing::Environment { class PlannerEnv : public testing::Environment {
public: public:
virtual void SetUp() { virtual void SetUp() {
fmFuncMgtInit(); // TODO(smj) : How to handle return value of fmFuncMgtInit
(void)fmFuncMgtInit();
initMetaDataEnv(); initMetaDataEnv();
generateMetaData(); generateMetaData();
initLog(TD_TMP_DIR_PATH "td"); initLog(TD_TMP_DIR_PATH "td");

View File

@ -997,9 +997,15 @@ int32_t filterDetachCnfGroups(SArray *group, SArray *left, SArray *right) {
for (int32_t l = 0; l < leftSize; ++l) { for (int32_t l = 0; l < leftSize; ++l) {
SFilterGroup *gp1 = taosArrayGet(left, l); SFilterGroup *gp1 = taosArrayGet(left, l);
if (NULL == gp1) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE);
}
for (int32_t r = 0; r < rightSize; ++r) { for (int32_t r = 0; r < rightSize; ++r) {
SFilterGroup *gp2 = taosArrayGet(right, r); SFilterGroup *gp2 = taosArrayGet(right, r);
if (NULL == gp2) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE);
}
FLT_ERR_RET(filterDetachCnfGroup(gp1, gp2, group)); FLT_ERR_RET(filterDetachCnfGroup(gp1, gp2, group));
} }
@ -1212,7 +1218,7 @@ int32_t filterAddUnitImpl(SFilterInfo *info, uint8_t optr, SFilterFieldId *left,
FLT_PACKAGE_UNIT_HASH_KEY(&v, optr, optr2, left->idx, (right ? right->idx : -1), (right2 ? right2->idx : -1)); FLT_PACKAGE_UNIT_HASH_KEY(&v, optr, optr2, left->idx, (right ? right->idx : -1), (right2 ? right2->idx : -1));
if (taosHashPut(info->pctx.unitHash, v, sizeof(v), uidx, sizeof(*uidx))) { if (taosHashPut(info->pctx.unitHash, v, sizeof(v), uidx, sizeof(*uidx))) {
fltError("taosHashPut to set failed"); fltError("taosHashPut to set failed");
FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); FLT_ERR_RET(terrno);
} }
} }
@ -2157,7 +2163,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData)); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData));
if (len < 0) { if (len < 0) {
qError("filterInitValFieldData taosUcs4ToMbs error 1"); qError("filterInitValFieldData taosUcs4ToMbs error 1");
return TSDB_CODE_APP_ERROR; return TSDB_CODE_SCALAR_CONVERT_ERROR;
} }
varDataSetLen(newValData, len); varDataSetLen(newValData, len);
(void)varDataCopy(fi->data, newValData); (void)varDataCopy(fi->data, newValData);
@ -2307,6 +2313,9 @@ int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx *gRes, uint32_t colI
for (uint32_t i = 0; i < size; ++i) { for (uint32_t i = 0; i < size; ++i) {
SFilterUnit *u = taosArrayGetP(colArray, i); SFilterUnit *u = taosArrayGetP(colArray, i);
if (NULL == u) {
FLT_ERR_JRET(TSDB_CODE_OUT_OF_RANGE);
}
uint8_t optr = FILTER_UNIT_OPTR(u); uint8_t optr = FILTER_UNIT_OPTR(u);
FLT_ERR_RET(filterAddRangeOptr(ctx, optr, LOGIC_COND_TYPE_AND, empty, NULL)); FLT_ERR_RET(filterAddRangeOptr(ctx, optr, LOGIC_COND_TYPE_AND, empty, NULL));
@ -2637,6 +2646,9 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx **gRes1, SFilter
for (int32_t i = 0; i < ctxSize; ++i) { for (int32_t i = 0; i < ctxSize; ++i) {
pctx = taosArrayGet(colCtxs, i); pctx = taosArrayGet(colCtxs, i);
if (NULL == pctx) {
FLT_ERR_JRET(TSDB_CODE_OUT_OF_RANGE);
}
colInfo = &(*gRes1)->colInfo[pctx->colIdx]; colInfo = &(*gRes1)->colInfo[pctx->colIdx];
filterFreeColInfo(colInfo); filterFreeColInfo(colInfo);
@ -2763,6 +2775,9 @@ int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray *group) {
for (size_t i = 0; i < groupSize; ++i) { for (size_t i = 0; i < groupSize; ++i) {
SFilterGroup *pg = taosArrayGet(group, i); SFilterGroup *pg = taosArrayGet(group, i);
if (NULL == pg) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE);
}
pg->unitFlags = taosMemoryCalloc(pg->unitNum, sizeof(*pg->unitFlags)); pg->unitFlags = taosMemoryCalloc(pg->unitNum, sizeof(*pg->unitFlags));
if (pg->unitFlags == NULL) { if (pg->unitFlags == NULL) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
@ -2819,7 +2834,9 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum
for (int32_t n = 0; n < usize; ++n) { for (int32_t n = 0; n < usize; ++n) {
SFilterUnit *u = (SFilterUnit *)taosArrayGetP((SArray *)colInfo->info, n); SFilterUnit *u = (SFilterUnit *)taosArrayGetP((SArray *)colInfo->info, n);
if (NULL == u) {
FLT_ERR_JRET(TSDB_CODE_OUT_OF_RANGE);
}
FLT_ERR_JRET(filterAddUnitFromUnit(info, &oinfo, u, &uidx)); FLT_ERR_JRET(filterAddUnitFromUnit(info, &oinfo, u, &uidx));
FLT_ERR_JRET(filterAddUnitToGroup(&ng, uidx)); FLT_ERR_JRET(filterAddUnitToGroup(&ng, uidx));
} }
@ -3425,7 +3442,7 @@ int32_t filterExecuteImplMisc(void *pinfo, int32_t numOfRows, SColumnInfoData *p
if (len < 0) { if (len < 0) {
qError("castConvert1 taosUcs4ToMbs error"); qError("castConvert1 taosUcs4ToMbs error");
taosMemoryFreeClear(newColData); taosMemoryFreeClear(newColData);
FLT_ERR_RET(TSDB_CODE_APP_ERROR); FLT_ERR_RET(TSDB_CODE_SCALAR_CONVERT_ERROR);
} else { } else {
varDataSetLen(newColData, len); varDataSetLen(newColData, len);
p[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, p[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData,
@ -3500,7 +3517,7 @@ int32_t filterExecuteImpl(void *pinfo, int32_t numOfRows, SColumnInfoData *pRes,
if (len < 0) { if (len < 0) {
qError("castConvert1 taosUcs4ToMbs error"); qError("castConvert1 taosUcs4ToMbs error");
taosMemoryFreeClear(newColData); taosMemoryFreeClear(newColData);
FLT_ERR_RET(TSDB_CODE_APP_ERROR); FLT_ERR_RET(TSDB_CODE_SCALAR_CONVERT_ERROR);
} else { } else {
varDataSetLen(newColData, len); varDataSetLen(newColData, len);
p[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData); p[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData);
@ -3814,6 +3831,9 @@ int32_t fltSclMergeSort(SArray *pts1, SArray *pts2, SArray *result) {
while (i < len1 && j < len2) { while (i < len1 && j < len2) {
SFltSclPoint *pt1 = taosArrayGet(pts1, i); SFltSclPoint *pt1 = taosArrayGet(pts1, i);
SFltSclPoint *pt2 = taosArrayGet(pts2, j); SFltSclPoint *pt2 = taosArrayGet(pts2, j);
if (NULL == pt1 || NULL == pt2) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE);
}
bool less = fltSclLessPoint(pt1, pt2); bool less = fltSclLessPoint(pt1, pt2);
if (less) { if (less) {
if (NULL == taosArrayPush(result, pt1)) { if (NULL == taosArrayPush(result, pt1)) {
@ -3830,6 +3850,9 @@ int32_t fltSclMergeSort(SArray *pts1, SArray *pts2, SArray *result) {
if (i < len1) { if (i < len1) {
for (; i < len1; ++i) { for (; i < len1; ++i) {
SFltSclPoint *pt1 = taosArrayGet(pts1, i); SFltSclPoint *pt1 = taosArrayGet(pts1, i);
if (NULL == pt1) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE);
}
if (NULL == taosArrayPush(result, pt1)) { if (NULL == taosArrayPush(result, pt1)) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
@ -3838,6 +3861,9 @@ int32_t fltSclMergeSort(SArray *pts1, SArray *pts2, SArray *result) {
if (j < len2) { if (j < len2) {
for (; j < len2; ++j) { for (; j < len2; ++j) {
SFltSclPoint *pt2 = taosArrayGet(pts2, j); SFltSclPoint *pt2 = taosArrayGet(pts2, j);
if (NULL == pt2) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE);
}
if (NULL == taosArrayPush(result, pt2)) { if (NULL == taosArrayPush(result, pt2)) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
@ -3859,6 +3885,9 @@ int32_t fltSclMerge(SArray *pts1, SArray *pts2, bool isUnion, SArray *merged) {
int32_t count = 0; int32_t count = 0;
for (int32_t i = 0; i < taosArrayGetSize(all); ++i) { for (int32_t i = 0; i < taosArrayGetSize(all); ++i) {
SFltSclPoint *pt = taosArrayGet(all, i); SFltSclPoint *pt = taosArrayGet(all, i);
if (NULL == pt) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE);
}
if (pt->start) { if (pt->start) {
++count; ++count;
if (count == countRequired) { if (count == countRequired) {
@ -3893,6 +3922,9 @@ typedef struct {
int32_t fltSclGetOrCreateColumnRange(SColumnNode *colNode, SArray *colRangeList, SFltSclColumnRange **colRange) { int32_t fltSclGetOrCreateColumnRange(SColumnNode *colNode, SArray *colRangeList, SFltSclColumnRange **colRange) {
for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) {
*colRange = taosArrayGet(colRangeList, i); *colRange = taosArrayGet(colRangeList, i);
if (NULL == colRange) {
return TSDB_CODE_OUT_OF_RANGE;
}
if (nodesEqualNode((SNode *)(*colRange)->colNode, (SNode *)colNode)) { if (nodesEqualNode((SNode *)(*colRange)->colNode, (SNode *)colNode)) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -4044,6 +4076,9 @@ int32_t filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32
SArray *colRanges = info->sclCtx.fltSclRange; SArray *colRanges = info->sclCtx.fltSclRange;
for (int32_t i = 0; i < taosArrayGetSize(colRanges); ++i) { for (int32_t i = 0; i < taosArrayGetSize(colRanges); ++i) {
SFltSclColumnRange *colRange = taosArrayGet(colRanges, i); SFltSclColumnRange *colRange = taosArrayGet(colRanges, i);
if (NULL == colRange) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE);
}
bool foundCol = false; bool foundCol = false;
int32_t j = 0; int32_t j = 0;
for (; j < numOfCols; ++j) { for (; j < numOfCols; ++j) {
@ -4332,11 +4367,17 @@ int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) {
SArray *colRanges = info->sclCtx.fltSclRange; SArray *colRanges = info->sclCtx.fltSclRange;
if (taosArrayGetSize(colRanges) == 1) { if (taosArrayGetSize(colRanges) == 1) {
SFltSclColumnRange *colRange = taosArrayGet(colRanges, 0); SFltSclColumnRange *colRange = taosArrayGet(colRanges, 0);
if (NULL == colRange) {
FLT_ERR_JRET(TSDB_CODE_OUT_OF_RANGE);
}
SArray *points = colRange->points; SArray *points = colRange->points;
if (taosArrayGetSize(points) == 2) { if (taosArrayGetSize(points) == 2) {
*win = TSWINDOW_DESC_INITIALIZER; *win = TSWINDOW_DESC_INITIALIZER;
SFltSclPoint *startPt = taosArrayGet(points, 0); SFltSclPoint *startPt = taosArrayGet(points, 0);
SFltSclPoint *endPt = taosArrayGet(points, 1); SFltSclPoint *endPt = taosArrayGet(points, 1);
if (NULL == startPt || NULL == endPt) {
FLT_ERR_JRET(TSDB_CODE_OUT_OF_RANGE);
}
SFltSclDatum start; SFltSclDatum start;
SFltSclDatum end; SFltSclDatum end;
FLT_ERR_JRET(fltSclGetTimeStampDatum(startPt, &start)); FLT_ERR_JRET(fltSclGetTimeStampDatum(startPt, &start));
@ -4398,7 +4439,7 @@ int32_t filterConverNcharColumns(SFilterInfo *info, int32_t rows, bool *gotNchar
bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), (TdUcs4 *)varDataVal(dst), bufSize, &len); bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), (TdUcs4 *)varDataVal(dst), bufSize, &len);
if (!ret) { if (!ret) {
qError("filterConverNcharColumns taosMbsToUcs4 error"); qError("filterConverNcharColumns taosMbsToUcs4 error");
return TSDB_CODE_FAILED; return TSDB_CODE_SCALAR_CONVERT_ERROR;
} }
varDataLen(dst) = len; varDataLen(dst) = len;
} }
@ -4432,7 +4473,7 @@ int32_t fltAddValueNodeToConverList(SFltTreeStat *stat, SValueNode *pNode) {
if (NULL == stat->nodeList) { if (NULL == stat->nodeList) {
stat->nodeList = taosArrayInit(10, POINTER_BYTES); stat->nodeList = taosArrayInit(10, POINTER_BYTES);
if (NULL == stat->nodeList) { if (NULL == stat->nodeList) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); FLT_ERR_RET(terrno);
} }
} }
@ -4816,6 +4857,9 @@ int32_t fltSclProcessCNF(SArray *sclOpListCNF, SArray *colRangeList) {
size_t sz = taosArrayGetSize(sclOpListCNF); size_t sz = taosArrayGetSize(sclOpListCNF);
for (int32_t i = 0; i < sz; ++i) { for (int32_t i = 0; i < sz; ++i) {
SFltSclOperator *sclOper = taosArrayGet(sclOpListCNF, i); SFltSclOperator *sclOper = taosArrayGet(sclOpListCNF, i);
if (NULL == sclOper) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE);
}
SFltSclColumnRange *colRange = NULL; SFltSclColumnRange *colRange = NULL;
FLT_ERR_RET(fltSclGetOrCreateColumnRange(sclOper->colNode, colRangeList, &colRange)); FLT_ERR_RET(fltSclGetOrCreateColumnRange(sclOper->colNode, colRangeList, &colRange));
SArray *points = taosArrayInit(4, sizeof(SFltSclPoint)); SArray *points = taosArrayInit(4, sizeof(SFltSclPoint));
@ -4920,22 +4964,27 @@ static int32_t fltSclCollectOperators(SNode *pNode, SArray *sclOpList) {
int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) { int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) {
SArray *sclOpList = taosArrayInit(16, sizeof(SFltSclOperator)); SArray *sclOpList = taosArrayInit(16, sizeof(SFltSclOperator));
int32_t code = TSDB_CODE_SUCCESS;
if (NULL == sclOpList) { if (NULL == sclOpList) {
FLT_ERR_RET(terrno); FLT_ERR_RET(terrno);
} }
FLT_ERR_RET(fltSclCollectOperators(*pNode, sclOpList)); FLT_ERR_JRET(fltSclCollectOperators(*pNode, sclOpList));
SArray *colRangeList = taosArrayInit(16, sizeof(SFltSclColumnRange)); SArray *colRangeList = taosArrayInit(16, sizeof(SFltSclColumnRange));
if (NULL == colRangeList) { if (NULL == colRangeList) {
FLT_ERR_RET(terrno); FLT_ERR_RET(terrno);
} }
FLT_ERR_RET(fltSclProcessCNF(sclOpList, colRangeList)); FLT_ERR_JRET(fltSclProcessCNF(sclOpList, colRangeList));
pInfo->sclCtx.fltSclRange = colRangeList; pInfo->sclCtx.fltSclRange = colRangeList;
for (int32_t i = 0; i < taosArrayGetSize(sclOpList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(sclOpList); ++i) {
SFltSclOperator *sclOp = taosArrayGet(sclOpList, i); SFltSclOperator *sclOp = taosArrayGet(sclOpList, i);
if (NULL == sclOp) {
FLT_ERR_JRET(TSDB_CODE_OUT_OF_RANGE);
}
nodesDestroyNode((SNode *)sclOp->colNode); nodesDestroyNode((SNode *)sclOp->colNode);
nodesDestroyNode((SNode *)sclOp->valNode); nodesDestroyNode((SNode *)sclOp->valNode);
} }
_return:
taosArrayDestroy(sclOpList); taosArrayDestroy(sclOpList);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -4946,6 +4995,9 @@ int32_t fltGetDataFromColId(void *param, int32_t id, void **data) {
for (int32_t j = 0; j < numOfCols; ++j) { for (int32_t j = 0; j < numOfCols; ++j) {
SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, j); SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, j);
if (NULL == pColInfo) {
FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE);
}
if (id == pColInfo->info.colId) { if (id == pColInfo->info.colId) {
*data = pColInfo; *data = pColInfo;
break; break;
@ -4965,6 +5017,9 @@ int32_t fltGetDataFromSlotId(void *param, int32_t id, void **data) {
} }
SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, id); SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, id);
if (NULL == pColInfo) {
return TSDB_CODE_OUT_OF_RANGE;
}
*data = pColInfo; *data = pColInfo;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;

View File

@ -90,6 +90,9 @@ int32_t sclConvertValueToSclParam(SValueNode *pValueNode, SScalarParam *out, int
int32_t sclExtendResRows(SScalarParam *pDst, SScalarParam *pSrc, SArray *pBlockList) { int32_t sclExtendResRows(SScalarParam *pDst, SScalarParam *pSrc, SArray *pBlockList) {
SSDataBlock *pb = taosArrayGetP(pBlockList, 0); SSDataBlock *pb = taosArrayGetP(pBlockList, 0);
if (NULL == pb) {
SCL_ERR_RET(TSDB_CODE_OUT_OF_RANGE);
}
SScalarParam *pLeft = taosMemoryCalloc(1, sizeof(SScalarParam)); SScalarParam *pLeft = taosMemoryCalloc(1, sizeof(SScalarParam));
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (NULL == pLeft) { if (NULL == pLeft) {
@ -116,7 +119,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) {
SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false);
if (NULL == pObj) { if (NULL == pObj) {
sclError("taosHashInit failed, size:%d", 256); sclError("taosHashInit failed, size:%d", 256);
SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); SCL_ERR_RET(terrno);
} }
taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(type)); taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(type));
@ -176,7 +179,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) {
if (taosHashPut(pObj, buf, (size_t)len, NULL, 0)) { if (taosHashPut(pObj, buf, (size_t)len, NULL, 0)) {
sclError("taosHashPut to set failed"); sclError("taosHashPut to set failed");
SCL_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); SCL_ERR_JRET(terrno);
} }
colInfoDataCleanup(out.columnData, out.numOfRows); colInfoDataCleanup(out.columnData, out.numOfRows);
@ -336,6 +339,9 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t
switch (nodeType(node)) { switch (nodeType(node)) {
case QUERY_NODE_LEFT_VALUE: { case QUERY_NODE_LEFT_VALUE: {
SSDataBlock *pb = taosArrayGetP(ctx->pBlockList, 0); SSDataBlock *pb = taosArrayGetP(ctx->pBlockList, 0);
if (NULL == pb) {
SCL_ERR_RET(TSDB_CODE_OUT_OF_RANGE);
}
param->numOfRows = pb->info.rows; param->numOfRows = pb->info.rows;
break; break;
} }
@ -347,10 +353,7 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t
SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
param->numOfRows = 1; param->numOfRows = 1;
int32_t code = sclCreateColumnInfoData(&valueNode->node.resType, 1, param); SCL_ERR_RET(sclCreateColumnInfoData(&valueNode->node.resType, 1, param));
if (code != TSDB_CODE_SUCCESS) {
SCL_RET(TSDB_CODE_OUT_OF_MEMORY);
}
if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type || valueNode->isNull) { if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type || valueNode->isNull) {
colDataSetNULL(param->columnData, 0); colDataSetNULL(param->columnData, 0);
} else { } else {
@ -377,7 +380,7 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t
taosHashCleanup(param->pHashFilter); taosHashCleanup(param->pHashFilter);
param->pHashFilter = NULL; param->pHashFilter = NULL;
sclError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); sclError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param));
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
param->colAlloced = false; param->colAlloced = false;
break; break;
@ -393,6 +396,9 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t
int32_t index = -1; int32_t index = -1;
for (int32_t i = 0; i < taosArrayGetSize(ctx->pBlockList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(ctx->pBlockList); ++i) {
SSDataBlock *pb = taosArrayGetP(ctx->pBlockList, i); SSDataBlock *pb = taosArrayGetP(ctx->pBlockList, i);
if (NULL == pb) {
SCL_ERR_RET(TSDB_CODE_OUT_OF_RANGE);
}
if (pb->info.id.blockId == ref->dataBlockId) { if (pb->info.id.blockId == ref->dataBlockId) {
index = i; index = i;
break; break;
@ -461,6 +467,9 @@ int32_t sclInitParamList(SScalarParam **pParams, SNodeList *pParamList, SScalarC
if (NULL == pParamList) { if (NULL == pParamList) {
if (ctx->pBlockList) { if (ctx->pBlockList) {
SSDataBlock *pBlock = taosArrayGetP(ctx->pBlockList, 0); SSDataBlock *pBlock = taosArrayGetP(ctx->pBlockList, 0);
if (NULL == pBlock) {
SCL_ERR_RET(TSDB_CODE_OUT_OF_RANGE);
}
*rowNum = pBlock->info.rows; *rowNum = pBlock->info.rows;
} else { } else {
*rowNum = 1; *rowNum = 1;
@ -919,6 +928,9 @@ int32_t sclExecCaseWhen(SCaseWhenNode *node, SScalarCtx *ctx, SScalarParam *outp
if (ctx->pBlockList) { if (ctx->pBlockList) {
SSDataBlock *pb = taosArrayGetP(ctx->pBlockList, 0); SSDataBlock *pb = taosArrayGetP(ctx->pBlockList, 0);
if (NULL == pb) {
SCL_ERR_RET(TSDB_CODE_OUT_OF_RANGE);
}
rowNum = pb->info.rows; rowNum = pb->info.rows;
output->numOfRows = pb->info.rows; output->numOfRows = pb->info.rows;
} }
@ -1532,6 +1544,10 @@ EDealRes sclWalkTarget(SNode *pNode, SScalarCtx *ctx) {
int32_t index = -1; int32_t index = -1;
for (int32_t i = 0; i < taosArrayGetSize(ctx->pBlockList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(ctx->pBlockList); ++i) {
SSDataBlock *pb = taosArrayGetP(ctx->pBlockList, i); SSDataBlock *pb = taosArrayGetP(ctx->pBlockList, i);
if (NULL == pb) {
ctx->code = TSDB_CODE_OUT_OF_RANGE;
return DEAL_RES_ERROR;
}
if (pb->info.id.blockId == target->dataBlockId) { if (pb->info.id.blockId == target->dataBlockId) {
index = i; index = i;
break; break;
@ -1636,7 +1652,7 @@ int32_t sclCalcConstants(SNode *pNode, bool dual, SNode **pRes) {
ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
if (NULL == ctx.pRes) { if (NULL == ctx.pRes) {
sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM);
SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); SCL_ERR_RET(terrno);
} }
nodesRewriteExprPostOrder(&pNode, sclConstantsRewriter, (void *)&ctx); nodesRewriteExprPostOrder(&pNode, sclConstantsRewriter, (void *)&ctx);
@ -1770,7 +1786,7 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) {
ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
if (NULL == ctx.pRes) { if (NULL == ctx.pRes) {
sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM);
SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); SCL_ERR_RET(terrno);
} }
nodesWalkExprPostOrder(pNode, sclCalcWalker, (void *)&ctx); nodesWalkExprPostOrder(pNode, sclCalcWalker, (void *)&ctx);
@ -1784,6 +1800,9 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) {
} }
SSDataBlock *pb = taosArrayGetP(pBlockList, 0); SSDataBlock *pb = taosArrayGetP(pBlockList, 0);
if (NULL == pb) {
SCL_ERR_JRET(TSDB_CODE_OUT_OF_RANGE);
}
if (1 == res->numOfRows && pb->info.rows > 0) { if (1 == res->numOfRows && pb->info.rows > 0) {
SCL_ERR_JRET(sclExtendResRows(pDst, res, pBlockList)); SCL_ERR_JRET(sclExtendResRows(pDst, res, pBlockList));
} else { } else {

View File

@ -399,7 +399,7 @@ static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar,
bool ret = taosMbsToUcs4(varDataVal(input), len, newBuf, (varDataLen(input) + 1) * TSDB_NCHAR_SIZE, &len); bool ret = taosMbsToUcs4(varDataVal(input), len, newBuf, (varDataLen(input) + 1) * TSDB_NCHAR_SIZE, &len);
if (!ret) { if (!ret) {
taosMemoryFree(newBuf); taosMemoryFree(newBuf);
return TSDB_CODE_FAILED; return TSDB_CODE_SCALAR_CONVERT_ERROR;
} }
memcpy(varDataVal(output) + *dataLen, newBuf, varDataLen(input) * TSDB_NCHAR_SIZE); memcpy(varDataVal(output) + *dataLen, newBuf, varDataLen(input) * TSDB_NCHAR_SIZE);
*dataLen += varDataLen(input) * TSDB_NCHAR_SIZE; *dataLen += varDataLen(input) * TSDB_NCHAR_SIZE;
@ -818,7 +818,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_FAILED; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
} }
@ -837,7 +837,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_FAILED; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
} }
convBuf[len] = 0; convBuf[len] = 0;
@ -855,7 +855,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_FAILED; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
} }
@ -874,7 +874,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_FAILED; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
} }
convBuf[len] = 0; convBuf[len] = 0;
@ -892,7 +892,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_FAILED; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
} }
convBuf[len] = 0; convBuf[len] = 0;
@ -910,7 +910,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_FAILED; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
} }
convBuf[len] = 0; convBuf[len] = 0;
@ -928,7 +928,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_FAILED; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
} }
convBuf[len] = 0; convBuf[len] = 0;
@ -946,7 +946,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_FAILED; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
} }
@ -965,7 +965,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_FAILED; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
} }
convBuf[len] = 0; convBuf[len] = 0;
@ -983,7 +983,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_FAILED; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
} }
convBuf[len] = 0; convBuf[len] = 0;
@ -1001,7 +1001,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_FAILED; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
} }
convBuf[len] = 0; convBuf[len] = 0;
@ -1041,7 +1041,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_FAILED; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
} }
len = TMIN(len, outputLen - VARSTR_HEADER_SIZE); len = TMIN(len, outputLen - VARSTR_HEADER_SIZE);
@ -1075,7 +1075,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
len = sprintf(tmp, "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false"); len = sprintf(tmp, "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false");
bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len);
if (!ret) { if (!ret) {
code = TSDB_CODE_FAILED; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
} }
@ -1085,7 +1085,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
bool ret = taosMbsToUcs4(input + VARSTR_HEADER_SIZE, len, (TdUcs4 *)varDataVal(output), bool ret = taosMbsToUcs4(input + VARSTR_HEADER_SIZE, len, (TdUcs4 *)varDataVal(output),
outputLen - VARSTR_HEADER_SIZE, &len); outputLen - VARSTR_HEADER_SIZE, &len);
if (!ret) { if (!ret) {
code = TSDB_CODE_FAILED; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
} }
varDataSetLen(output, len); varDataSetLen(output, len);
@ -1099,7 +1099,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
len = outputCharLen > len ? len : outputCharLen; len = outputCharLen > len ? len : outputCharLen;
bool ret = taosMbsToUcs4(buf, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); bool ret = taosMbsToUcs4(buf, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len);
if (!ret) { if (!ret) {
code = TSDB_CODE_FAILED; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
} }
varDataSetLen(output, len); varDataSetLen(output, len);

View File

@ -105,7 +105,7 @@ int32_t convertNcharToDouble(const void *inData, void *outData) {
int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp); int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp);
if (len < 0) { if (len < 0) {
sclError("castConvert taosUcs4ToMbs error 1"); sclError("castConvert taosUcs4ToMbs error 1");
SCL_ERR_JRET(TSDB_CODE_FAILED); SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR);
} }
tmp[len] = 0; tmp[len] = 0;
@ -596,7 +596,7 @@ int32_t ncharTobinary(void *buf, void **out) { // todo need to remove , if tobi
sclError("charset:%s to %s. val:%s convert ncharTobinary failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, sclError("charset:%s to %s. val:%s convert ncharTobinary failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
(char *)varDataVal(buf)); (char *)varDataVal(buf));
taosMemoryFree(*out); taosMemoryFree(*out);
SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); SCL_ERR_RET(TSDB_CODE_SCALAR_CONVERT_ERROR);
} }
varDataSetLen(*out, len); varDataSetLen(*out, len);
SCL_RET(TSDB_CODE_SUCCESS); SCL_RET(TSDB_CODE_SUCCESS);

View File

@ -549,7 +549,7 @@ int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
if (tEncodeCStrWithLen(pEncoder, pTask->reserve, sizeof(pTask->reserve) - 1) < 0) return -1; if (tEncodeCStrWithLen(pEncoder, pTask->reserve, sizeof(pTask->reserve) - 1) < 0) return -1;
tEndEncode(pEncoder); tEndEncode(pEncoder);
return pEncoder->pos; return 0;
} }
int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {

View File

@ -43,17 +43,16 @@ void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet) {
bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId) { bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId) {
uint32_t ipv4 = 0xFFFFFFFF; uint32_t ipv4 = 0xFFFFFFFF;
sDebug("vgId:%d, start to resolve sync addr fqdn in %d seconds, " sDebug(
"dnode:%d cluster:%" PRId64 " fqdn:%s port:%u ", "vgId:%d, start to resolve sync addr fqdn in %d seconds, "
vgId, tsResolveFQDNRetryTime, "dnode:%d cluster:%" PRId64 " fqdn:%s port:%u ",
pInfo->nodeId, pInfo->clusterId, pInfo->nodeFqdn, pInfo->nodePort); vgId, tsResolveFQDNRetryTime, pInfo->nodeId, pInfo->clusterId, pInfo->nodeFqdn, pInfo->nodePort);
for(int i = 0; i < tsResolveFQDNRetryTime; i++){ for (int i = 0; i < tsResolveFQDNRetryTime; i++) {
ipv4 = taosGetIpv4FromFqdn(pInfo->nodeFqdn); ipv4 = taosGetIpv4FromFqdn(pInfo->nodeFqdn);
if (ipv4 == 0xFFFFFFFF || ipv4 == 1) { if (ipv4 == 0xFFFFFFFF || ipv4 == 1) {
sError("failed to resolve ipv4 addr, fqdn:%s, wait one second", pInfo->nodeFqdn); sError("failed to resolve ipv4 addr, fqdn:%s, wait one second", pInfo->nodeFqdn);
taosSsleep(1); taosSsleep(1);
} } else {
else{
break; break;
} }
} }
@ -501,8 +500,7 @@ void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteRepl
int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size) { int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size) {
void* data = taosMemoryRealloc(pSnap->data, size); void* data = taosMemoryRealloc(pSnap->data, size);
if (data == NULL) { if (data == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
pSnap->data = data; pSnap->data = data;
return 0; return 0;

View File

@ -10,7 +10,6 @@ target_sources(tdb
"src/db/tdbTable.c" "src/db/tdbTable.c"
"src/db/tdbTxn.c" "src/db/tdbTxn.c"
"src/db/tdbPage.c" "src/db/tdbPage.c"
"src/db/tdbOs.c"
) )
target_include_directories( target_include_directories(

View File

@ -76,14 +76,14 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg
if (keyLen == 0) { if (keyLen == 0) {
tdbError("tdb/btree-open: key len cannot be zero."); tdbError("tdb/btree-open: key len cannot be zero.");
return -1; return TSDB_CODE_INVALID_PARA;
} }
*ppBt = NULL; *ppBt = NULL;
pBt = (SBTree *)tdbOsCalloc(1, sizeof(*pBt)); pBt = (SBTree *)tdbOsCalloc(1, sizeof(*pBt));
if (pBt == NULL) { if (pBt == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
// pBt->keyLen // pBt->keyLen

View File

@ -15,8 +15,8 @@
#include "tdbInt.h" #include "tdbInt.h"
int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, int8_t rollback, int32_t encryptAlgorithm, int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, int8_t rollback,
char *encryptKey) { int32_t encryptAlgorithm, char *encryptKey) {
TDB *pDb; TDB *pDb;
int dsize; int dsize;
int zsize; int zsize;
@ -31,7 +31,7 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i
pPtr = (uint8_t *)tdbOsCalloc(1, zsize); pPtr = (uint8_t *)tdbOsCalloc(1, zsize);
if (pPtr == NULL) { if (pPtr == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
pDb = (TDB *)pPtr; pDb = (TDB *)pPtr;
@ -51,38 +51,38 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i
pDb->jfd = -1; pDb->jfd = -1;
pDb->encryptAlgorithm = encryptAlgorithm; pDb->encryptAlgorithm = encryptAlgorithm;
if(encryptKey != NULL){ if (encryptKey != NULL) {
strncpy(pDb->encryptKey, encryptKey, ENCRYPT_KEY_LEN); strncpy(pDb->encryptKey, encryptKey, ENCRYPT_KEY_LEN);
} }
ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache)); ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache));
if (ret < 0) { if (ret < 0) {
return -1; return ret;
} }
pDb->nPgrHash = 8; pDb->nPgrHash = 8;
tsize = sizeof(SPager *) * pDb->nPgrHash; tsize = sizeof(SPager *) * pDb->nPgrHash;
pDb->pgrHash = tdbOsMalloc(tsize); pDb->pgrHash = tdbOsMalloc(tsize);
if (pDb->pgrHash == NULL) { if (pDb->pgrHash == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
memset(pDb->pgrHash, 0, tsize); memset(pDb->pgrHash, 0, tsize);
ret = taosMulModeMkDir(dbname, 0755, false); ret = taosMulModeMkDir(dbname, 0755, false);
if (ret < 0) { if (ret < 0) {
return -1; return terrno;
} }
#ifdef USE_MAINDB #ifdef USE_MAINDB
// open main db // open main db
ret = tdbTbOpen(TDB_MAINDB_NAME, -1, sizeof(SBtInfo), NULL, pDb, &pDb->pMainDb, rollback); ret = tdbTbOpen(TDB_MAINDB_NAME, -1, sizeof(SBtInfo), NULL, pDb, &pDb->pMainDb, rollback);
if (ret < 0) { if (ret < 0) {
return -1; return ret;
} }
ret = tdbTbOpen(TDB_FREEDB_NAME, sizeof(SPgno), 0, NULL, pDb, &pDb->pFreeDb, rollback); ret = tdbTbOpen(TDB_FREEDB_NAME, sizeof(SPgno), 0, NULL, pDb, &pDb->pFreeDb, rollback);
if (ret < 0) { if (ret < 0) {
return -1; return ret;
} }
#endif #endif
@ -125,12 +125,13 @@ int32_t tdbBegin(TDB *pDb, TXN **ppTxn, void *(*xMalloc)(void *, size_t), void (
TXN *pTxn = tdbOsCalloc(1, sizeof(*pTxn)); TXN *pTxn = tdbOsCalloc(1, sizeof(*pTxn));
if (!pTxn) { if (!pTxn) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
if (tdbTxnOpen(pTxn, txnId, xMalloc, xFree, xArg, flags) < 0) { ret = tdbTxnOpen(pTxn, txnId, xMalloc, xFree, xArg, flags);
if (ret < 0) {
tdbOsFree(pTxn); tdbOsFree(pTxn);
return -1; return ret;
} }
for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
@ -139,7 +140,7 @@ int32_t tdbBegin(TDB *pDb, TXN **ppTxn, void *(*xMalloc)(void *, size_t), void (
tdbError("failed to begin pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, tdbError("failed to begin pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
pTxn->txnId); pTxn->txnId);
tdbTxnClose(pTxn); tdbTxnClose(pTxn);
return -1; return ret;
} }
} }
@ -157,7 +158,7 @@ int32_t tdbCommit(TDB *pDb, TXN *pTxn) {
if (ret < 0) { if (ret < 0) {
tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
pTxn->txnId); pTxn->txnId);
return -1; return ret;
} }
} }
@ -173,7 +174,7 @@ int32_t tdbPostCommit(TDB *pDb, TXN *pTxn) {
if (ret < 0) { if (ret < 0) {
tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
pTxn->txnId); pTxn->txnId);
return -1; return ret;
} }
} }
@ -191,7 +192,7 @@ int32_t tdbPrepareAsyncCommit(TDB *pDb, TXN *pTxn) {
if (ret < 0) { if (ret < 0) {
tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
pTxn->txnId); pTxn->txnId);
return -1; return ret;
} }
} }
@ -207,7 +208,7 @@ int32_t tdbAbort(TDB *pDb, TXN *pTxn) {
if (ret < 0) { if (ret < 0) {
tdbError("failed to abort pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, tdbError("failed to abort pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
pTxn->txnId); pTxn->txnId);
return -1; return ret;
} }
} }

View File

@ -1,98 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbOs.h"
#ifndef TDB_FOR_TDENGINE
// tdbOsRead
i64 tdbOsRead(tdb_fd_t fd, void *pData, i64 nBytes) {
i64 nRead = 0;
i64 iRead = 0;
u8 *pBuf = (u8 *)pData;
while (nBytes > 0) {
iRead = read(fd, pBuf, nBytes);
if (iRead < 0) {
if (errno == EINTR) {
continue;
} else {
return -1;
}
} else if (iRead == 0) {
break;
}
nRead += iRead;
pBuf += iRead;
nBytes -= iRead;
}
return nRead;
}
// tdbOsPRead
i64 tdbOsPRead(tdb_fd_t fd, void *pData, i64 nBytes, i64 offset) {
i64 nRead = 0;
i64 iRead = 0;
i64 iOffset = offset;
u8 *pBuf = (u8 *)pData;
while (nBytes > 0) {
iRead = pread(fd, pBuf, nBytes, iOffset);
if (iRead < 0) {
if (errno == EINTR) {
continue;
} else {
return -1;
}
} else if (iRead == 0) {
break;
}
nRead += iRead;
pBuf += iRead;
iOffset += iRead;
nBytes -= iRead;
}
return nRead;
}
// tdbOsWrite
i64 tdbOsWrite(tdb_fd_t fd, const void *pData, i64 nBytes) {
i64 nWrite = 0;
i64 iWrite = 0;
u8 *pBuf = (u8 *)pData;
while (nBytes > 0) {
iWrite = write(fd, pBuf, nBytes);
if (iWrite < 0) {
if (errno == EINTR) {
continue;
}
return -1;
}
nWrite += iWrite;
pBuf += iWrite;
nBytes -= iWrite;
}
return nWrite;
}
#endif

View File

@ -50,30 +50,36 @@ static void tdbPCacheLock(SPCache *pCache) { tdbMutexLock(&(pCache->mutex)); }
static void tdbPCacheUnlock(SPCache *pCache) { tdbMutexUnlock(&(pCache->mutex)); } static void tdbPCacheUnlock(SPCache *pCache) { tdbMutexUnlock(&(pCache->mutex)); }
int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) { int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) {
int32_t code = 0;
int32_t lino;
SPCache *pCache; SPCache *pCache;
void *pPtr; void *pPtr;
SPage *pPgHdr; SPage *pPgHdr;
pCache = (SPCache *)tdbOsCalloc(1, sizeof(*pCache) + sizeof(SPage *) * cacheSize); pCache = (SPCache *)tdbOsCalloc(1, sizeof(*pCache) + sizeof(SPage *) * cacheSize);
if (pCache == NULL) { if (pCache == NULL) {
return -1; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
} }
pCache->szPage = pageSize; pCache->szPage = pageSize;
pCache->nPages = cacheSize; pCache->nPages = cacheSize;
pCache->aPage = (SPage **)tdbOsCalloc(cacheSize, sizeof(SPage *)); pCache->aPage = (SPage **)tdbOsCalloc(cacheSize, sizeof(SPage *));
if (pCache->aPage == NULL) { if (pCache->aPage == NULL) {
tdbOsFree(pCache); TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
return -1;
} }
if (tdbPCacheOpenImpl(pCache) < 0) { code = tdbPCacheOpenImpl(pCache);
tdbOsFree(pCache); TSDB_CHECK_CODE(code, lino, _exit);
return -1;
}
*ppCache = pCache; _exit:
return 0; if (code) {
tdbError("%s failed at %s:%d since %s", __func__, __FILE__, __LINE__, tstrerror(code));
tdbPCacheClose(pCache);
*ppCache = NULL;
} else {
*ppCache = pCache;
}
return code;
} }
int tdbPCacheClose(SPCache *pCache) { int tdbPCacheClose(SPCache *pCache) {
@ -99,14 +105,14 @@ static int tdbPCacheAlterImpl(SPCache *pCache, int32_t nPage) {
} else if (pCache->nPages < nPage) { } else if (pCache->nPages < nPage) {
SPage **aPage = tdbOsCalloc(nPage, sizeof(SPage *)); SPage **aPage = tdbOsCalloc(nPage, sizeof(SPage *));
if (aPage == NULL) { if (aPage == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
for (int32_t iPage = pCache->nPages; iPage < nPage; iPage++) { for (int32_t iPage = pCache->nPages; iPage < nPage; iPage++) {
if (tdbPageCreate(pCache->szPage, &aPage[iPage], tdbDefaultMalloc, NULL) < 0) { int32_t code = tdbPageCreate(pCache->szPage, &aPage[iPage], tdbDefaultMalloc, NULL);
// TODO: handle error if (code) {
tdbOsFree(aPage); tdbOsFree(aPage);
return -1; return code;
} }
// pPage->pgid = 0; // pPage->pgid = 0;
@ -156,15 +162,11 @@ static int tdbPCacheAlterImpl(SPCache *pCache, int32_t nPage) {
} }
int tdbPCacheAlter(SPCache *pCache, int32_t nPage) { int tdbPCacheAlter(SPCache *pCache, int32_t nPage) {
int ret = 0; int code;
tdbPCacheLock(pCache); tdbPCacheLock(pCache);
code = tdbPCacheAlterImpl(pCache, nPage);
ret = tdbPCacheAlterImpl(pCache, nPage);
tdbPCacheUnlock(pCache); tdbPCacheUnlock(pCache);
return code;
return ret;
} }
SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) { SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) {
@ -180,9 +182,6 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) {
tdbPCacheUnlock(pCache); tdbPCacheUnlock(pCache);
// printf("thread %" PRId64 " fetch page %d pgno %d pPage %p nRef %d\n", taosGetSelfPthreadId(), pPage->id,
// TDB_PAGE_PGNO(pPage), pPage, nRef);
if (pPage) { if (pPage) {
tdbTrace("pcache/fetch page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef); tdbTrace("pcache/fetch page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef);
} else { } else {
@ -285,6 +284,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
if (!pTxn) { if (!pTxn) {
tdbError("tdb/pcache: null ptr pTxn, fetch impl failed."); tdbError("tdb/pcache: null ptr pTxn, fetch impl failed.");
terrno = TSDB_CODE_INVALID_PARA;
return NULL; return NULL;
} }
@ -327,7 +327,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg); ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg);
if (ret < 0 || pPage == NULL) { if (ret < 0 || pPage == NULL) {
tdbError("tdb/pcache: ret: %" PRId32 " pPage: %p, page create failed.", ret, pPage); tdbError("tdb/pcache: ret: %" PRId32 " pPage: %p, page create failed.", ret, pPage);
// TODO: recycle other backup pages terrno = ret;
return NULL; return NULL;
} }
@ -475,10 +475,8 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
pCache->nFree = 0; pCache->nFree = 0;
pCache->pFree = NULL; pCache->pFree = NULL;
for (int i = 0; i < pCache->nPages; i++) { for (int i = 0; i < pCache->nPages; i++) {
if (tdbPageCreate(pCache->szPage, &pPage, tdbDefaultMalloc, NULL) < 0) { ret = tdbPageCreate(pCache->szPage, &pPage, tdbDefaultMalloc, NULL);
// TODO: handle error if (ret) return ret;
return -1;
}
// pPage->pgid = 0; // pPage->pgid = 0;
pPage->isAnchor = 0; pPage->isAnchor = 0;
@ -504,8 +502,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
pCache->nHash = pCache->nPages < 8 ? 8 : pCache->nPages; pCache->nHash = pCache->nPages < 8 ? 8 : pCache->nPages;
pCache->pgHash = (SPage **)tdbOsCalloc(pCache->nHash, sizeof(SPage *)); pCache->pgHash = (SPage **)tdbOsCalloc(pCache->nHash, sizeof(SPage *));
if (pCache->pgHash == NULL) { if (pCache->pgHash == NULL) {
// TODO return TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
// Open LRU list // Open LRU list

View File

@ -45,12 +45,12 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t)
if (!xMalloc) { if (!xMalloc) {
tdbError("tdb/page-create: null xMalloc."); tdbError("tdb/page-create: null xMalloc.");
return -1; return TSDB_CODE_INVALID_PARA;
} }
if (!TDB_IS_PGSIZE_VLD(pageSize)) { if (!TDB_IS_PGSIZE_VLD(pageSize)) {
tdbError("tdb/page-create: invalid pageSize: %d.", pageSize); tdbError("tdb/page-create: invalid pageSize: %d.", pageSize);
return -1; return TSDB_CODE_INVALID_PARA;
} }
*ppPage = NULL; *ppPage = NULL;
@ -58,7 +58,7 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t)
ptr = (u8 *)(xMalloc(arg, size)); ptr = (u8 *)(xMalloc(arg, size));
if (ptr == NULL) { if (ptr == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
memset(ptr, 0, size); memset(ptr, 0, size);
@ -86,12 +86,12 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg)
if (pPage->isDirty) { if (pPage->isDirty) {
tdbError("tdb/page-destroy: dirty page: %" PRIu8 ".", pPage->isDirty); tdbError("tdb/page-destroy: dirty page: %" PRIu8 ".", pPage->isDirty);
return -1; return TSDB_CODE_INVALID_PARA;
} }
if (!xFree) { if (!xFree) {
tdbError("tdb/page-destroy: null xFree."); tdbError("tdb/page-destroy: null xFree.");
return -1; return TSDB_CODE_INVALID_PARA;
} }
for (int iOvfl = 0; iOvfl < pPage->nOverflow; iOvfl++) { for (int iOvfl = 0; iOvfl < pPage->nOverflow; iOvfl++) {
@ -129,7 +129,8 @@ void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell
tdbTrace("page/init: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize); tdbTrace("page/init: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize);
pPage->pPageHdr = pPage->pData + szAmHdr; pPage->pPageHdr = pPage->pData + szAmHdr;
if (TDB_PAGE_NCELLS(pPage) == 0) { if (TDB_PAGE_NCELLS(pPage) == 0) {
return tdbPageZero(pPage, szAmHdr, xCellSize); tdbPageZero(pPage, szAmHdr, xCellSize);
return;
} }
pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage);
pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage); pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage);
@ -159,7 +160,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl
if (szCell > TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)) { if (szCell > TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)) {
tdbError("tdb/page-insert-cell: invalid page, szCell: %d, max free: %lu", szCell, tdbError("tdb/page-insert-cell: invalid page, szCell: %d, max free: %lu", szCell,
TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)); TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData));
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
nFree = TDB_PAGE_NFREE(pPage); nFree = TDB_PAGE_NFREE(pPage);
@ -207,7 +208,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl
if (pPage->pFreeStart != pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)) { if (pPage->pFreeStart != pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)) {
tdbError("tdb/page-insert-cell: invalid page, pFreeStart: %p, pCellIdx: %p, nCells: %d", pPage->pFreeStart, tdbError("tdb/page-insert-cell: invalid page, pFreeStart: %p, pCellIdx: %p, nCells: %d", pPage->pFreeStart,
pPage->pCellIdx, nCells); pPage->pCellIdx, nCells);
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
} }
@ -234,7 +235,7 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) {
if (idx < 0 || idx >= nCells + pPage->nOverflow) { if (idx < 0 || idx >= nCells + pPage->nOverflow) {
tdbError("tdb/page-drop-cell: idx: %d out of range, nCells: %d, nOvfl: %d.", idx, nCells, pPage->nOverflow); tdbError("tdb/page-drop-cell: idx: %d out of range, nCells: %d, nOvfl: %d.", idx, nCells, pPage->nOverflow);
return -1; return TSDB_CODE_INVALID_PARA;
} }
iOvfl = 0; iOvfl = 0;
@ -265,14 +266,14 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) {
pPage->aiOvfl[iOvfl]--; pPage->aiOvfl[iOvfl]--;
if (pPage->aiOvfl[iOvfl] <= 0) { if (pPage->aiOvfl[iOvfl] <= 0) {
tdbError("tdb/page-drop-cell: invalid ai idx: %d", pPage->aiOvfl[iOvfl]); tdbError("tdb/page-drop-cell: invalid ai idx: %d", pPage->aiOvfl[iOvfl]);
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
} }
return 0; return 0;
} }
void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { int32_t tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) {
int delta, nFree; int delta, nFree;
pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr); pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr);
@ -280,7 +281,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) {
if (pToPage->pFreeEnd < pToPage->pFreeStart) { if (pToPage->pFreeEnd < pToPage->pFreeStart) {
tdbError("tdb/page-copy: invalid to page, pFreeStart: %p, pFreeEnd: %p", pToPage->pFreeStart, pToPage->pFreeEnd); tdbError("tdb/page-copy: invalid to page, pFreeStart: %p, pFreeEnd: %p", pToPage->pFreeStart, pToPage->pFreeEnd);
return; return TSDB_CODE_INVALID_DATA_FMT;
} }
memcpy(pToPage->pPageHdr, pFromPage->pPageHdr, pFromPage->pFreeStart - pFromPage->pPageHdr); memcpy(pToPage->pPageHdr, pFromPage->pPageHdr, pFromPage->pFreeStart - pFromPage->pPageHdr);
@ -289,7 +290,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) {
if (TDB_PAGE_CCELLS(pToPage) != pToPage->pFreeEnd - pToPage->pData) { if (TDB_PAGE_CCELLS(pToPage) != pToPage->pFreeEnd - pToPage->pData) {
tdbError("tdb/page-copy: invalid to page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pToPage), tdbError("tdb/page-copy: invalid to page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pToPage),
pToPage->pFreeEnd - pToPage->pData); pToPage->pFreeEnd - pToPage->pData);
return; return TSDB_CODE_INVALID_DATA_FMT;
} }
delta = (pToPage->pPageHdr - pToPage->pData) - (pFromPage->pPageHdr - pFromPage->pData); delta = (pToPage->pPageHdr - pToPage->pData) - (pFromPage->pPageHdr - pFromPage->pData);
@ -304,6 +305,10 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) {
if (deepCopyOvfl) { if (deepCopyOvfl) {
int szCell = (*pFromPage->xCellSize)(pFromPage, pFromPage->apOvfl[iOvfl], 0, NULL, NULL); int szCell = (*pFromPage->xCellSize)(pFromPage, pFromPage->apOvfl[iOvfl], 0, NULL, NULL);
pNewCell = (SCell *)tdbOsMalloc(szCell); pNewCell = (SCell *)tdbOsMalloc(szCell);
if (pNewCell == NULL) {
tdbError("tdb/page-copy: out of memory, size: %d", szCell);
return TSDB_CODE_OUT_OF_MEMORY;
}
memcpy(pNewCell, pFromPage->apOvfl[iOvfl], szCell); memcpy(pNewCell, pFromPage->apOvfl[iOvfl], szCell);
tdbTrace("tdbPage/copy/new ovfl cell: %p/%p/%p", pNewCell, pToPage, pFromPage); tdbTrace("tdbPage/copy/new ovfl cell: %p/%p/%p", pNewCell, pToPage, pFromPage);
} }
@ -312,6 +317,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) {
pToPage->aiOvfl[iOvfl] = pFromPage->aiOvfl[iOvfl]; pToPage->aiOvfl[iOvfl] = pFromPage->aiOvfl[iOvfl];
} }
pToPage->nOverflow = pFromPage->nOverflow; pToPage->nOverflow = pFromPage->nOverflow;
return 0;
} }
int tdbPageCapacity(int pageSize, int amHdrSize) { int tdbPageCapacity(int pageSize, int amHdrSize) {
@ -343,12 +349,12 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) {
if (nFree < szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { if (nFree < szCell + TDB_PAGE_OFFSET_SIZE(pPage)) {
tdbError("tdb/page-allocate: invalid cell size, nFree: %d, szCell: %d, szOffset: %d", nFree, szCell, tdbError("tdb/page-allocate: invalid cell size, nFree: %d, szCell: %d, szOffset: %d", nFree, szCell,
TDB_PAGE_OFFSET_SIZE(pPage)); TDB_PAGE_OFFSET_SIZE(pPage));
return -1; return TSDB_CODE_INVALID_PARA;
} }
if (TDB_PAGE_CCELLS(pPage) != pPage->pFreeEnd - pPage->pData) { if (TDB_PAGE_CCELLS(pPage) != pPage->pFreeEnd - pPage->pData) {
tdbError("tdb/page-allocate: invalid page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pPage), tdbError("tdb/page-allocate: invalid page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pPage),
pPage->pFreeEnd - pPage->pData); pPage->pFreeEnd - pPage->pData);
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
// 1. Try to allocate from the free space block area // 1. Try to allocate from the free space block area
@ -363,7 +369,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) {
cellFree = TDB_PAGE_FCELL(pPage); cellFree = TDB_PAGE_FCELL(pPage);
if (cellFree != 0 && cellFree < pPage->pFreeEnd - pPage->pData) { if (cellFree != 0 && cellFree < pPage->pFreeEnd - pPage->pData) {
tdbError("tdb/page-allocate: cellFree: %d, pFreeEnd: %p, pData: %p.", cellFree, pPage->pFreeEnd, pPage->pData); tdbError("tdb/page-allocate: cellFree: %d, pFreeEnd: %p, pData: %p.", cellFree, pPage->pFreeEnd, pPage->pData);
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
if (cellFree && pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) { if (cellFree && pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) {
SCell *pPrevFreeCell = NULL; SCell *pPrevFreeCell = NULL;
@ -408,19 +414,19 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) {
} }
// 3. Try to dfragment and allocate again // 3. Try to dfragment and allocate again
tdbPageDefragment(pPage); TAOS_CHECK_RETURN(tdbPageDefragment(pPage));
if (pPage->pFreeEnd - pPage->pFreeStart != nFree) { if (pPage->pFreeEnd - pPage->pFreeStart != nFree) {
tdbError("tdb/page-allocate: nFree: %d, pFreeStart: %p, pFreeEnd: %p.", nFree, pPage->pFreeStart, pPage->pFreeEnd); tdbError("tdb/page-allocate: nFree: %d, pFreeStart: %p, pFreeEnd: %p.", nFree, pPage->pFreeStart, pPage->pFreeEnd);
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
if (TDB_PAGE_NFREE(pPage) != nFree) { if (TDB_PAGE_NFREE(pPage) != nFree) {
tdbError("tdb/page-allocate: nFree: %d, page free: %d.", nFree, TDB_PAGE_NFREE(pPage)); tdbError("tdb/page-allocate: nFree: %d, page free: %d.", nFree, TDB_PAGE_NFREE(pPage));
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
if (pPage->pFreeEnd - pPage->pData != TDB_PAGE_CCELLS(pPage)) { if (pPage->pFreeEnd - pPage->pData != TDB_PAGE_CCELLS(pPage)) {
tdbError("tdb/page-allocate: ccells: %d, pFreeStart: %p, pData: %p.", TDB_PAGE_CCELLS(pPage), pPage->pFreeStart, tdbError("tdb/page-allocate: ccells: %d, pFreeStart: %p, pData: %p.", TDB_PAGE_CCELLS(pPage), pPage->pFreeStart,
pPage->pData); pPage->pData);
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
pPage->pFreeEnd -= szCell; pPage->pFreeEnd -= szCell;
@ -430,7 +436,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) {
_alloc_finish: _alloc_finish:
if (NULL == pCell) { if (NULL == pCell) {
tdbError("tdb/page-allocate: null ptr pCell."); tdbError("tdb/page-allocate: null ptr pCell.");
return -1; return TSDB_CODE_OUT_OF_BUFFER;
} }
pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage);
@ -447,15 +453,15 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) {
if (pCell < pPage->pFreeEnd) { if (pCell < pPage->pFreeEnd) {
tdbError("tdb/page-free: invalid cell, cell: %p, free end: %p", pCell, pPage->pFreeEnd); tdbError("tdb/page-free: invalid cell, cell: %p, free end: %p", pCell, pPage->pFreeEnd);
return -1; return TSDB_CODE_INVALID_PARA;
} }
if (pCell + szCell > (u8 *)(pPage->pPageFtr)) { if (pCell + szCell > (u8 *)(pPage->pPageFtr)) {
tdbError("tdb/page-free: cell crosses page footer, cell: %p, size: %d footer: %p", pCell, szCell, pPage->pFreeEnd); tdbError("tdb/page-free: cell crosses page footer, cell: %p, size: %d footer: %p", pCell, szCell, pPage->pFreeEnd);
return -1; return TSDB_CODE_INVALID_PARA;
} }
if (pCell != TDB_PAGE_CELL_AT(pPage, idx)) { if (pCell != TDB_PAGE_CELL_AT(pPage, idx)) {
tdbError("tdb/page-free: cell pos incorrect, cell: %p, pos: %p", pCell, TDB_PAGE_CELL_AT(pPage, idx)); tdbError("tdb/page-free: cell pos incorrect, cell: %p, pos: %p", pCell, TDB_PAGE_CELL_AT(pPage, idx));
return -1; return TSDB_CODE_INVALID_PARA;
} }
nFree = TDB_PAGE_NFREE(pPage); nFree = TDB_PAGE_NFREE(pPage);
@ -470,7 +476,7 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) {
TDB_PAGE_FCELL_SET(pPage, pCell - pPage->pData); TDB_PAGE_FCELL_SET(pPage, pCell - pPage->pData);
} else { } else {
tdbError("tdb/page-free: invalid cell size: %d", szCell); tdbError("tdb/page-free: invalid cell size: %d", szCell);
return -1; return TSDB_CODE_INVALID_PARA;
} }
} }
@ -502,7 +508,9 @@ static int tdbPageDefragment(SPage *pPage) {
int32_t nCell = TDB_PAGE_NCELLS(pPage); int32_t nCell = TDB_PAGE_NCELLS(pPage);
SCellIdx *aCellIdx = (SCellIdx *)tdbOsMalloc(sizeof(SCellIdx) * nCell); SCellIdx *aCellIdx = (SCellIdx *)tdbOsMalloc(sizeof(SCellIdx) * nCell);
if (aCellIdx == NULL) return -1; if (aCellIdx == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t iCell = 0; iCell < nCell; iCell++) { for (int32_t iCell = 0; iCell < nCell; iCell++) {
aCellIdx[iCell].iCell = iCell; aCellIdx[iCell].iCell = iCell;
aCellIdx[iCell].offset = TDB_PAGE_CELL_OFFSET_AT(pPage, iCell); aCellIdx[iCell].offset = TDB_PAGE_CELL_OFFSET_AT(pPage, iCell);

View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tdbInt.h"
#include "crypt.h" #include "crypt.h"
#include "tdbInt.h"
#include "tglobal.h" #include "tglobal.h"
/* /*
#pragma pack(push, 1) #pragma pack(push, 1)
@ -41,9 +41,10 @@ struct hashset_st {
static const unsigned int prime = 39; static const unsigned int prime = 39;
static const unsigned int prime2 = 5009; static const unsigned int prime2 = 5009;
hashset_t hashset_create(void) { static hashset_t hashset_create(void) {
hashset_t set = tdbOsCalloc(1, sizeof(struct hashset_st)); hashset_t set = tdbOsCalloc(1, sizeof(struct hashset_st));
if (!set) { if (!set) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
@ -52,6 +53,7 @@ hashset_t hashset_create(void) {
set->items = tdbOsCalloc(set->capacity, sizeof(size_t)); set->items = tdbOsCalloc(set->capacity, sizeof(size_t));
if (!set->items) { if (!set->items) {
tdbOsFree(set); tdbOsFree(set);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
set->mask = set->capacity - 1; set->mask = set->capacity - 1;
@ -69,7 +71,7 @@ void hashset_destroy(hashset_t set) {
} }
} }
int hashset_add_member(hashset_t set, void *item) { static int hashset_add_member(hashset_t set, void *item) {
size_t value = (size_t)item; size_t value = (size_t)item;
size_t h; size_t h;
@ -88,7 +90,7 @@ int hashset_add_member(hashset_t set, void *item) {
return 1; return 1;
} }
int hashset_add(hashset_t set, void *item) { static int hashset_add(hashset_t set, void *item) {
int ret = hashset_add_member(set, item); int ret = hashset_add_member(set, item);
size_t old_capacity = set->capacity; size_t old_capacity = set->capacity;
@ -113,7 +115,7 @@ int hashset_add(hashset_t set, void *item) {
return ret; return ret;
} }
int hashset_remove(hashset_t set, void *item) { static int hashset_remove(hashset_t set, void *item) {
size_t value = (size_t)item; size_t value = (size_t)item;
for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) { for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) {
@ -127,7 +129,7 @@ int hashset_remove(hashset_t set, void *item) {
return 0; return 0;
} }
int hashset_contains(hashset_t set, void *item) { static int hashset_contains(hashset_t set, void *item) {
size_t value = (size_t)item; size_t value = (size_t)item;
for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) { for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) {
@ -177,7 +179,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
+ fsize + 8 + 1; /* jFileName */ + fsize + 8 + 1; /* jFileName */
pPtr = (uint8_t *)tdbOsCalloc(1, zsize); pPtr = (uint8_t *)tdbOsCalloc(1, zsize);
if (pPtr == NULL) { if (pPtr == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
pPager = (SPager *)pPtr; pPager = (SPager *)pPtr;
@ -198,12 +200,12 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
pPager->fd = tdbOsOpen(pPager->dbFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); pPager->fd = tdbOsOpen(pPager->dbFileName, TDB_O_CREAT | TDB_O_RDWR, 0755);
if (TDB_FD_INVALID(pPager->fd)) { if (TDB_FD_INVALID(pPager->fd)) {
// if (pPager->fd < 0) { // if (pPager->fd < 0) {
return -1; return TAOS_SYSTEM_ERROR(errno);
} }
ret = tdbGnrtFileID(pPager->fd, pPager->fid, false); ret = tdbGnrtFileID(pPager->fd, pPager->fid, false);
if (ret < 0) { if (ret < 0) {
return -1; return TAOS_SYSTEM_ERROR(errno);
} }
// pPager->jfd = -1; // pPager->jfd = -1;
@ -221,11 +223,6 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
int tdbPagerClose(SPager *pPager) { int tdbPagerClose(SPager *pPager) {
if (pPager) { if (pPager) {
/*
if (pPager->inTran) {
tdbOsClose(pPager->jfd);
}
*/
tdbOsClose(pPager->fd); tdbOsClose(pPager->fd);
tdbOsFree(pPager); tdbOsFree(pPager);
} }
@ -254,8 +251,8 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
!hashset_contains(pPager->pActiveTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))))) { !hashset_contains(pPager->pActiveTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))))) {
ret = tdbPagerWritePageToJournal(pPager, pPage); ret = tdbPagerWritePageToJournal(pPager, pPage);
if (ret < 0) { if (ret < 0) {
tdbError("failed to write page to journal since %s", tstrerror(terrno)); tdbError("failed to write page to journal since %s", tstrerror(ret));
return -1; return ret;
} }
if (pPager->pActiveTxn->jPageSet) { if (pPager->pActiveTxn->jPageSet) {
@ -278,11 +275,13 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) {
pTxn->jfd = tdbOsOpen(jTxnFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); pTxn->jfd = tdbOsOpen(jTxnFileName, TDB_O_CREAT | TDB_O_RDWR, 0755);
if (TDB_FD_INVALID(pTxn->jfd)) { if (TDB_FD_INVALID(pTxn->jfd)) {
tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
pTxn->jPageSet = hashset_create(); pTxn->jPageSet = hashset_create();
if (pTxn->jPageSet == NULL) {
return terrno;
}
pPager->pActiveTxn = pTxn; pPager->pActiveTxn = pTxn;
@ -319,8 +318,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
ret = tdbOsFSync(pTxn->jfd); ret = tdbOsFSync(pTxn->jfd);
if (ret < 0) { if (ret < 0) {
tdbError("failed to fsync: %s. jFileName:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); tdbError("failed to fsync: %s. jFileName:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
// loop to write the dirty pages to file // loop to write the dirty pages to file
@ -331,13 +329,13 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
if (pPage->nOverflow != 0) { if (pPage->nOverflow != 0) {
tdbError("tdb/pager-commit: %p, pPage: %p, ovfl: %d, commit page failed.", pPager, pPage, pPage->nOverflow); tdbError("tdb/pager-commit: %p, pPage: %p, ovfl: %d, commit page failed.", pPager, pPage, pPage->nOverflow);
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
ret = tdbPagerPWritePageToDB(pPager, pPage); ret = tdbPagerPWritePageToDB(pPager, pPage);
if (ret < 0) { if (ret < 0) {
tdbError("failed to write page to db since %s", tstrerror(terrno)); tdbError("failed to write page to db since %s", tstrerror(terrno));
return -1; return ret;
} }
} }
@ -368,8 +366,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
// sync the db file // sync the db file
if (tdbOsFSync(pPager->fd) < 0) { if (tdbOsFSync(pPager->fd) < 0) {
tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName); tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
return 0; return 0;
@ -382,14 +379,12 @@ int tdbPagerPostCommit(SPager *pPager, TXN *pTxn) {
// remove the journal file // remove the journal file
if (tdbOsClose(pTxn->jfd) < 0) { if (tdbOsClose(pTxn->jfd) < 0) {
tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) { if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) {
tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName); tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
// pPager->inTran = 0; // pPager->inTran = 0;
@ -408,8 +403,7 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) {
ret = tdbOsFSync(pTxn->jfd); ret = tdbOsFSync(pTxn->jfd);
if (ret < 0) { if (ret < 0) {
tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
// loop to write the dirty pages to file // loop to write the dirty pages to file
@ -426,7 +420,7 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) {
ret = tdbPagerPWritePageToDB(pPager, pPage); ret = tdbPagerPWritePageToDB(pPager, pPage);
if (ret < 0) { if (ret < 0) {
tdbError("failed to write page to db since %s", tstrerror(terrno)); tdbError("failed to write page to db since %s", tstrerror(terrno));
return -1; return ret;
} }
} }
@ -445,36 +439,28 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) {
tdbPCacheRelease(pPager->pCache, pPage, pTxn); tdbPCacheRelease(pPager->pCache, pPage, pTxn);
} }
/*
tdbTrace("reset dirty tree: %p", &pPager->rbt);
tRBTreeCreate(&pPager->rbt, pageCmpFn);
// sync the db file
if (tdbOsFSync(pPager->fd) < 0) {
tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
*/
return 0; return 0;
} }
static char* tdbEncryptPage(SPager *pPager, char* pPageData, int32_t pageSize, const char* function, static char *tdbEncryptPage(SPager *pPager, char *pPageData, int32_t pageSize, const char *function, int64_t offset) {
int64_t offset){
int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm; int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm;
char* encryptKey = pPager->pEnv->encryptKey; char *encryptKey = pPager->pEnv->encryptKey;
char* buf = pPageData; char *buf = pPageData;
if(encryptAlgorithm == DND_CA_SM4){ if (encryptAlgorithm == DND_CA_SM4) {
//tdbInfo("CBC_Encrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__); // tdbInfo("CBC_Encrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__);
//ASSERT(strlen(encryptKey) > 0); // ASSERT(strlen(encryptKey) > 0);
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Encrypt", offset, pPage->pData[0]); // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Encrypt", offset, pPage->pData[0]);
buf = taosMemoryMalloc(pageSize); buf = taosMemoryMalloc(pageSize);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
unsigned char packetData[128]; unsigned char packetData[128];
int32_t count = 0; int32_t count = 0;
while (count < pageSize) { while (count < pageSize) {
@ -488,19 +474,19 @@ static char* tdbEncryptPage(SPager *pPager, char* pPageData, int32_t pageSize, c
int32_t newLen = CBC_Encrypt(&opts); int32_t newLen = CBC_Encrypt(&opts);
memcpy(buf + count, packetData, newLen); memcpy(buf + count, packetData, newLen);
count += newLen; count += newLen;
} }
//tdbInfo("CBC tdb offset:%" PRId64 ", Encrypt count:%d %s", offset, count, function); // tdbInfo("CBC tdb offset:%" PRId64 ", Encrypt count:%d %s", offset, count, function);
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Encrypt", offset, (uint8_t)buf[0]); // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Encrypt", offset, (uint8_t)buf[0]);
} }
return buf; return buf;
} }
void tdbFreeEncryptBuf(SPager *pPager, char* buf){ void tdbFreeEncryptBuf(SPager *pPager, char *buf) {
int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm; int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm;
if(encryptAlgorithm == DND_CA_SM4) taosMemoryFreeClear(buf); if (encryptAlgorithm == DND_CA_SM4) taosMemoryFreeClear(buf);
} }
// recovery dirty pages // recovery dirty pages
int tdbPagerAbort(SPager *pPager, TXN *pTxn) { int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
@ -518,26 +504,24 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
ret = tdbOsFSync(pTxn->jfd); ret = tdbOsFSync(pTxn->jfd);
if (ret < 0) { if (ret < 0) {
tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
tdb_fd_t jfd = pTxn->jfd; tdb_fd_t jfd = pTxn->jfd;
ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize); ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize);
if (ret < 0) { if (ret < 0) {
return -1; return ret;
} }
if (tdbOsLSeek(jfd, 0L, SEEK_SET) < 0) { if (tdbOsLSeek(jfd, 0L, SEEK_SET) < 0) {
tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(errno), pPager->dbFileName); tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(errno), pPager->dbFileName);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
u8 *pageBuf = tdbOsCalloc(1, pPager->pageSize); u8 *pageBuf = tdbOsCalloc(1, pPager->pageSize);
if (pageBuf == NULL) { if (pageBuf == NULL) {
return -1; return terrno = TSDB_CODE_OUT_OF_MEMORY;
} }
tdbDebug("pager/abort: %p, %d/%d, txnId:%" PRId64, pPager, pPager->dbOrigSize, pPager->dbFileSize, pTxn->txnId); tdbDebug("pager/abort: %p, %d/%d, txnId:%" PRId64, pPager, pPager->dbOrigSize, pPager->dbFileSize, pTxn->txnId);
@ -549,7 +533,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
int ret = tdbOsRead(jfd, &pgno, sizeof(pgno)); int ret = tdbOsRead(jfd, &pgno, sizeof(pgno));
if (ret < 0) { if (ret < 0) {
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return terrno = TAOS_SYSTEM_ERROR(errno);
} }
tdbTrace("pager/abort: restore pgno:%d,", pgno); tdbTrace("pager/abort: restore pgno:%d,", pgno);
@ -559,27 +543,28 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
ret = tdbOsRead(jfd, pageBuf, pPager->pageSize); ret = tdbOsRead(jfd, pageBuf, pPager->pageSize);
if (ret < 0) { if (ret < 0) {
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return terrno = TAOS_SYSTEM_ERROR(errno);
} }
i64 offset = pPager->pageSize * (pgno - 1); i64 offset = pPager->pageSize * (pgno - 1);
if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
tdbError("failed to lseek fd due to %s. file:%s, offset:%" PRId64, strerror(errno), pPager->dbFileName, offset); tdbError("failed to lseek fd due to %s. file:%s, offset:%" PRId64, strerror(errno), pPager->dbFileName, offset);
terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return terrno = TAOS_SYSTEM_ERROR(errno);
} }
char* buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset); char *buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset);
if (buf == NULL) {
return terrno;
}
ret = tdbOsWrite(pPager->fd, buf, pPager->pageSize); ret = tdbOsWrite(pPager->fd, buf, pPager->pageSize);
if (ret < 0) { if (ret < 0) {
tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName,
pPager->pageSize); pPager->pageSize);
tdbFreeEncryptBuf(pPager, buf); tdbFreeEncryptBuf(pPager, buf);
terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return terrno = TAOS_SYSTEM_ERROR(errno);
} }
tdbFreeEncryptBuf(pPager, buf); tdbFreeEncryptBuf(pPager, buf);
@ -587,9 +572,8 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
if (tdbOsFSync(pPager->fd) < 0) { if (tdbOsFSync(pPager->fd) < 0) {
tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName); tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName);
terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return terrno = TAOS_SYSTEM_ERROR(errno);
} }
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
@ -617,8 +601,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
// 4, remove the journal file // 4, remove the journal file
if (tdbOsClose(pTxn->jfd) < 0) { if (tdbOsClose(pTxn->jfd) < 0) {
tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
char jTxnFileName[TDB_FILENAME_LEN]; char jTxnFileName[TDB_FILENAME_LEN];
@ -626,8 +609,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) { if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) {
tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName); tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
// pPager->inTran = 0; // pPager->inTran = 0;
@ -658,7 +640,7 @@ int tdbPagerFlushPage(SPager *pPager, TXN *pTxn) {
ret = tdbPagerPWritePageToDB(pPager, pPage); ret = tdbPagerPWritePageToDB(pPager, pPage);
if (ret < 0) { if (ret < 0) {
tdbError("failed to write page to db since %s", tstrerror(terrno)); tdbError("failed to write page to db since %s", tstrerror(terrno));
return -1; return ret;
} }
tdbTrace("tdb/flush:%p, pgno:%d, %d/%d/%d", pPager, pgno, pPager->dbOrigSize, pPager->dbFileSize, maxPgno); tdbTrace("tdb/flush:%p, pgno:%d, %d/%d/%d", pPager, pgno, pPager->dbOrigSize, pPager->dbFileSize, maxPgno);
@ -717,13 +699,13 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa
ret = tdbPagerAllocPage(pPager, &pgno, pTxn); ret = tdbPagerAllocPage(pPager, &pgno, pTxn);
if (ret < 0) { if (ret < 0) {
tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno); tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno);
return -1; return ret;
} }
} }
if (pgno == 0) { if (pgno == 0) {
tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno); tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno);
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
// fetch a page container // fetch a page container
@ -739,7 +721,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa
ret = tdbPagerInitPage(pPager, pPage, initPage, arg, loadPage); ret = tdbPagerInitPage(pPager, pPage, initPage, arg, loadPage);
if (ret < 0) { if (ret < 0) {
tdbError("tdb/pager: %p, pPage: %p, init page failed.", pPager, pPage); tdbError("tdb/pager: %p, pPage: %p, init page failed.", pPager, pPage);
return -1; return ret;
} }
} }
@ -748,11 +730,11 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa
if (!TDB_PAGE_INITIALIZED(pPage)) { if (!TDB_PAGE_INITIALIZED(pPage)) {
tdbError("tdb/pager: %p, pPage: %p, fetch page uninited.", pPager, pPage); tdbError("tdb/pager: %p, pPage: %p, fetch page uninited.", pPager, pPage);
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
if (pPage->pPager != pPager) { if (pPage->pPager != pPager) {
tdbError("tdb/pager: %p/%p, fetch page failed.", pPager, pPage->pPager); tdbError("tdb/pager: %p/%p, fetch page failed.", pPager, pPage->pPager);
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
*ppgno = pgno; *ppgno = pgno;
@ -771,12 +753,17 @@ int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) {
SPgno pgno = TDB_PAGE_PGNO(pPage); SPgno pgno = TDB_PAGE_PGNO(pPage);
if (pPager->frps) { if (pPager->frps) {
taosArrayPush(pPager->frps, &pgno); if (taosArrayPush(pPager->frps, &pgno) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pPage->pPager = NULL; pPage->pPager = NULL;
return code; return code;
} }
pPager->frps = taosArrayInit(8, sizeof(SPgno)); pPager->frps = taosArrayInit(8, sizeof(SPgno));
if (pPager->frps == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
// memset(pPage->pData, 0, pPage->pageSize); // memset(pPage->pData, 0, pPage->pageSize);
tdbTrace("tdb/insert-free-page: tbc recycle page: %d.", pgno); tdbTrace("tdb/insert-free-page: tbc recycle page: %d.", pgno);
// printf("tdb/insert-free-page: tbc recycle page: %d.\n", pgno); // printf("tdb/insert-free-page: tbc recycle page: %d.\n", pgno);
@ -785,7 +772,7 @@ int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) {
tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code); tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code);
taosArrayDestroy(pPager->frps); taosArrayDestroy(pPager->frps);
pPager->frps = NULL; pPager->frps = NULL;
return -1; return code;
} }
while (TARRAY_SIZE(pPager->frps) > 0) { while (TARRAY_SIZE(pPager->frps) > 0) {
@ -796,7 +783,7 @@ int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) {
tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code); tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code);
taosArrayDestroy(pPager->frps); taosArrayDestroy(pPager->frps);
pPager->frps = NULL; pPager->frps = NULL;
return -1; return code;
} }
} }
@ -822,7 +809,7 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) {
code = tdbTbcOpen(pPager->pEnv->pFreeDb, &pCur, pTxn); code = tdbTbcOpen(pPager->pEnv->pFreeDb, &pCur, pTxn);
if (code < 0) { if (code < 0) {
return 0; return code;
} }
code = tdbTbcMoveToFirst(pCur); code = tdbTbcMoveToFirst(pCur);
@ -924,20 +911,19 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage
} }
int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm; int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm;
char* encryptKey = pPager->pEnv->encryptKey; char *encryptKey = pPager->pEnv->encryptKey;
if(encryptAlgorithm == DND_CA_SM4){ if (encryptAlgorithm == DND_CA_SM4) {
//tdbInfo("CBC_Decrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__); // tdbInfo("CBC_Decrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__);
//ASSERT(strlen(encryptKey) > 0); // ASSERT(strlen(encryptKey) > 0);
//uint8_t flags = pPage->pData[0]; // uint8_t flags = pPage->pData[0];
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Decrypt", ((i64)pPage->pageSize) * (pgno - 1), flags); // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Decrypt", ((i64)pPage->pageSize) * (pgno - 1), flags);
unsigned char packetData[128]; unsigned char packetData[128];
int32_t count = 0; int32_t count = 0;
while(count < pPage->pageSize) while (count < pPage->pageSize) {
{
SCryptOpts opts = {0}; SCryptOpts opts = {0};
opts.len = 128; opts.len = 128;
opts.source = pPage->pData + count; opts.source = pPage->pData + count;
@ -945,20 +931,23 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage
opts.unitLen = 128; opts.unitLen = 128;
strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN); strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN);
int newLen = CBC_Decrypt(&opts); int newLen = CBC_Decrypt(&opts);
memcpy(pPage->pData + count, packetData, newLen); memcpy(pPage->pData + count, packetData, newLen);
count += newLen; count += newLen;
} }
//tdbInfo("CBC tdb offset:%" PRId64 ", Decrypt count:%d %s", ((i64)pPage->pageSize) * (pgno - 1), count, __FUNCTION__); // tdbInfo("CBC tdb offset:%" PRId64 ", Decrypt count:%d %s", ((i64)pPage->pageSize) * (pgno - 1), count,
// __FUNCTION__);
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Decrypt %s", ((i64)pPage->pageSize) * (pgno - 1), pPage->pData[0], __FUNCTION__); // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Decrypt %s", ((i64)pPage->pageSize) * (pgno - 1),
// pPage->pData[0], __FUNCTION__);
} }
} else { } else {
init = 0; init = 0;
} }
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d initPage %s", ((i64)pPage->pageSize) * (pgno - 1), pPage->pData[0], __FUNCTION__); // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d initPage %s", ((i64)pPage->pageSize) * (pgno - 1), pPage->pData[0],
// __FUNCTION__);
ret = (*initPage)(pPage, arg, init); ret = (*initPage)(pPage, arg, init);
if (ret < 0) { if (ret < 0) {
@ -1001,16 +990,14 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) {
if (ret < 0) { if (ret < 0) {
tdbError("failed to write pgno due to %s. file:%s, pgno:%u, txnId:%" PRId64, strerror(errno), pPager->jFileName, tdbError("failed to write pgno due to %s. file:%s, pgno:%u, txnId:%" PRId64, strerror(errno), pPager->jFileName,
pgno, pPager->pActiveTxn->txnId); pgno, pPager->pActiveTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
ret = tdbOsWrite(pPager->pActiveTxn->jfd, pPage->pData, pPage->pageSize); ret = tdbOsWrite(pPager->pActiveTxn->jfd, pPage->pData, pPage->pageSize);
if (ret < 0) { if (ret < 0) {
tdbError("failed to write page data due to %s. file:%s, pageSize:%d, txnId:%" PRId64, strerror(errno), tdbError("failed to write page data due to %s. file:%s, pageSize:%d, txnId:%" PRId64, strerror(errno),
pPager->jFileName, pPage->pageSize, pPager->pActiveTxn->txnId); pPager->jFileName, pPage->pageSize, pPager->pActiveTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
return 0; return 0;
@ -1044,15 +1031,14 @@ static int tdbPagerPWritePageToDB(SPager *pPager, SPage *pPage) {
offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1); offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1);
char* buf = tdbEncryptPage(pPager, pPage->pData, pPage->pageSize, __FUNCTION__, offset); char *buf = tdbEncryptPage(pPager, pPage->pData, pPage->pageSize, __FUNCTION__, offset);
ret = tdbOsPWrite(pPager->fd, buf, pPage->pageSize, offset); ret = tdbOsPWrite(pPager->fd, buf, pPage->pageSize, offset);
if (ret < 0) { if (ret < 0) {
tdbFreeEncryptBuf(pPager, buf); tdbFreeEncryptBuf(pPager, buf);
tdbError("failed to pwrite page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->dbFileName, tdbError("failed to pwrite page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->dbFileName,
pPage->pageSize); pPage->pageSize);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
tdbFreeEncryptBuf(pPager, buf); tdbFreeEncryptBuf(pPager, buf);
@ -1072,18 +1058,17 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) {
ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize); ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize);
if (ret < 0) { if (ret < 0) {
return -1; return TAOS_SYSTEM_ERROR(errno);
} }
if (tdbOsLSeek(jfd, 0L, SEEK_SET) < 0) { if (tdbOsLSeek(jfd, 0L, SEEK_SET) < 0) {
tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(errno), pPager->dbFileName); tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(errno), pPager->dbFileName);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
pageBuf = tdbOsCalloc(1, pPager->pageSize); pageBuf = tdbOsCalloc(1, pPager->pageSize);
if (pageBuf == NULL) { if (pageBuf == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
tdbDebug("pager/restore: %p, %d/%d, txnId:%s", pPager, pPager->dbOrigSize, pPager->dbFileSize, jFileName); tdbDebug("pager/restore: %p, %d/%d, txnId:%s", pPager, pPager->dbOrigSize, pPager->dbFileSize, jFileName);
@ -1095,7 +1080,7 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) {
int ret = tdbOsRead(jfd, &pgno, sizeof(pgno)); int ret = tdbOsRead(jfd, &pgno, sizeof(pgno));
if (ret < 0) { if (ret < 0) {
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return TAOS_SYSTEM_ERROR(errno);
} }
tdbTrace("pager/restore: restore pgno:%d,", pgno); tdbTrace("pager/restore: restore pgno:%d,", pgno);
@ -1103,27 +1088,28 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) {
ret = tdbOsRead(jfd, pageBuf, pPager->pageSize); ret = tdbOsRead(jfd, pageBuf, pPager->pageSize);
if (ret < 0) { if (ret < 0) {
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return TAOS_SYSTEM_ERROR(errno);
} }
i64 offset = pPager->pageSize * (pgno - 1); i64 offset = pPager->pageSize * (pgno - 1);
if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
tdbError("failed to lseek fd due to %s. file:%s, offset:%" PRId64, strerror(errno), pPager->dbFileName, offset); tdbError("failed to lseek fd due to %s. file:%s, offset:%" PRId64, strerror(errno), pPager->dbFileName, offset);
terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return terrno = TAOS_SYSTEM_ERROR(errno);
} }
char* buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset); char *buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset);
if (buf == NULL) {
return terrno;
}
ret = tdbOsWrite(pPager->fd, buf, pPager->pageSize); ret = tdbOsWrite(pPager->fd, buf, pPager->pageSize);
if (ret < 0) { if (ret < 0) {
tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName,
pPager->pageSize); pPager->pageSize);
tdbFreeEncryptBuf(pPager, buf); tdbFreeEncryptBuf(pPager, buf);
terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return terrno = TAOS_SYSTEM_ERROR(errno);
} }
tdbFreeEncryptBuf(pPager, buf); tdbFreeEncryptBuf(pPager, buf);
@ -1131,9 +1117,8 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) {
if (tdbOsFSync(pPager->fd) < 0) { if (tdbOsFSync(pPager->fd) < 0) {
tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName); tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName);
terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return terrno = TAOS_SYSTEM_ERROR(errno);
} }
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
@ -1160,21 +1145,27 @@ static int32_t txnIdCompareDesc(const void *pLeft, const void *pRight) {
} }
int tdbPagerRestoreJournals(SPager *pPager) { int tdbPagerRestoreJournals(SPager *pPager) {
int32_t code = 0;
tdbDirEntryPtr pDirEntry; tdbDirEntryPtr pDirEntry;
tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName); tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName);
if (pDir == NULL) { if (pDir == NULL) {
tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno)); tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno));
return -1; return TAOS_SYSTEM_ERROR(errno);
} }
SArray *pTxnList = taosArrayInit(16, sizeof(int64_t)); SArray *pTxnList = taosArrayInit(16, sizeof(int64_t));
if (pTxnList == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
while ((pDirEntry = tdbReadDir(pDir)) != NULL) { while ((pDirEntry = tdbReadDir(pDir)) != NULL) {
char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry)); char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry));
if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) { if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) {
int64_t txnId = -1; int64_t txnId = -1;
sscanf(name, TDB_MAINDB_NAME "-journal.%" PRId64, &txnId); sscanf(name, TDB_MAINDB_NAME "-journal.%" PRId64, &txnId);
taosArrayPush(pTxnList, &txnId); if (taosArrayPush(pTxnList, &txnId) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
} }
taosArraySort(pTxnList, txnIdCompareDesc); taosArraySort(pTxnList, txnIdCompareDesc);
@ -1185,12 +1176,12 @@ int tdbPagerRestoreJournals(SPager *pPager) {
memcpy(jname, pPager->pEnv->dbName, dirLen); memcpy(jname, pPager->pEnv->dbName, dirLen);
jname[dirLen] = '/'; jname[dirLen] = '/';
sprintf(jname + dirLen + 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId); sprintf(jname + dirLen + 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId);
if (tdbPagerRestore(pPager, jname) < 0) { code = tdbPagerRestore(pPager, jname);
if (code) {
taosArrayDestroy(pTxnList); taosArrayDestroy(pTxnList);
tdbCloseDir(&pDir); tdbCloseDir(&pDir);
tdbError("failed to restore file due to %s. jFileName:%s", strerror(code), jname);
tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), jname); return code;
return -1;
} }
} }
@ -1205,7 +1196,7 @@ int tdbPagerRollback(SPager *pPager) {
tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName); tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName);
if (pDir == NULL) { if (pDir == NULL) {
tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno)); tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno));
return -1; return terrno = TAOS_SYSTEM_ERROR(errno);
} }
while ((pDirEntry = tdbReadDir(pDir)) != NULL) { while ((pDirEntry = tdbReadDir(pDir)) != NULL) {
@ -1221,8 +1212,7 @@ int tdbPagerRollback(SPager *pPager) {
tdbCloseDir(&pDir); tdbCloseDir(&pDir);
tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), name); tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), name);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
} }
} }

View File

@ -41,7 +41,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
pTb = (TTB *)tdbOsCalloc(1, sizeof(*pTb)); pTb = (TTB *)tdbOsCalloc(1, sizeof(*pTb));
if (pTb == NULL) { if (pTb == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
// pTb->pEnv // pTb->pEnv
@ -54,7 +54,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
pPager = tdbEnvGetPager(pEnv, fFullName); pPager = tdbEnvGetPager(pEnv, fFullName);
if (!pPager) { if (!pPager) {
tdbOsFree(pTb); tdbOsFree(pTb);
return -1; return terrno;
} }
ret = tdbTbGet(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pData, &nData); ret = tdbTbGet(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pData, &nData);
@ -74,7 +74,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager); ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager);
if (ret < 0) { if (ret < 0) {
tdbOsFree(pTb); tdbOsFree(pTb);
return -1; return ret;
} }
tdbEnvAddPager(pEnv, pPager); tdbEnvAddPager(pEnv, pPager);
@ -109,7 +109,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
ret = tdbPagerRestoreJournals(pPager); ret = tdbPagerRestoreJournals(pPager);
if (ret < 0) { if (ret < 0) {
tdbOsFree(pTb); tdbOsFree(pTb);
return -1; return ret;
} }
} else { } else {
tdbPagerRollback(pPager); tdbPagerRollback(pPager);
@ -119,7 +119,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, pEnv, &(pTb->pBt)); ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, pEnv, &(pTb->pBt));
if (ret < 0) { if (ret < 0) {
tdbOsFree(pTb); tdbOsFree(pTb);
return -1; return ret;
} }
*ppTb = pTb; *ppTb = pTb;

View File

@ -20,7 +20,7 @@ int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void
// not support read-committed version at the moment // not support read-committed version at the moment
if (flags != 0 && flags != (TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)) { if (flags != 0 && flags != (TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)) {
tdbError("tdb/txn: invalid txn flags: %" PRId32, flags); tdbError("tdb/txn: invalid txn flags: %" PRId32, flags);
return -1; return TSDB_CODE_INVALID_PARA;
} }
pTxn->flags = flags; pTxn->flags = flags;
@ -39,7 +39,7 @@ int tdbTxnCloseImpl(TXN *pTxn) {
} }
if (pTxn->jfd) { if (pTxn->jfd) {
tdbOsClose(pTxn->jfd); TAOS_UNUSED(tdbOsClose(pTxn->jfd));
ASSERT(pTxn->jfd == NULL); ASSERT(pTxn->jfd == NULL);
} }

View File

@ -38,9 +38,8 @@ void tdbFree(void *p) {
int tdbGnrtFileID(tdb_fd_t fd, uint8_t *fileid, bool unique) { int tdbGnrtFileID(tdb_fd_t fd, uint8_t *fileid, bool unique) {
int64_t stDev = 0, stIno = 0; int64_t stDev = 0, stIno = 0;
if (taosDevInoFile(fd, &stDev, &stIno) < 0) { int32_t code = taosDevInoFile(fd, &stDev, &stIno);
return -1; return code;
}
memset(fileid, 0, TDB_FILE_ID_LEN); memset(fileid, 0, TDB_FILE_ID_LEN);
@ -59,7 +58,7 @@ int tdbGetFileSize(tdb_fd_t fd, int szPage, SPgno *size) {
ret = tdbOsFileSize(fd, &szBytes); ret = tdbOsFileSize(fd, &szBytes);
if (ret < 0) { if (ret < 0) {
return -1; return TAOS_SYSTEM_ERROR(errno);
} }
*size = szBytes / szPage; *size = szBytes / szPage;

View File

@ -17,6 +17,7 @@
#define _TD_TDB_INTERNAL_H_ #define _TD_TDB_INTERNAL_H_
#include "tdb.h" #include "tdb.h"
#include "tutil.h"
#include "tdef.h" #include "tdef.h"
#include "tlog.h" #include "tlog.h"
@ -338,15 +339,15 @@ static inline int tdbTryLockPage(tdb_spinlock_t *pLock) {
((*(pPage)->xCellSize)(pPage, pCell, 0, NULL, NULL) + (pPage)->pPageMethods->szOffset) ((*(pPage)->xCellSize)(pPage, pCell, 0, NULL, NULL) + (pPage)->pPageMethods->szOffset)
#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) #define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset)
int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg);
int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg);
void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)); void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt));
void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)); void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt));
int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl); int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl);
int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt); int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt);
int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell, TXN *pTxn, SBTree *pBt); int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell, TXN *pTxn, SBTree *pBt);
void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int copyOvflCells); int32_t tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl);
int tdbPageCapacity(int pageSize, int amHdrSize); int tdbPageCapacity(int pageSize, int amHdrSize);
static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { static inline SCell *tdbPageGetCell(SPage *pPage, int idx) {
SCell *pCell; SCell *pCell;

View File

@ -103,7 +103,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
tstrncpy(pWal->path, path, sizeof(pWal->path)); tstrncpy(pWal->path, path, sizeof(pWal->path));
if (taosMkDir(pWal->path) != 0) { if (taosMkDir(pWal->path) != 0) {
wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, tstrerror(terrno));
goto _err; goto _err;
} }

View File

@ -44,7 +44,7 @@ int wordexp(char *words, wordexp_t *pwordexp, int flags) {
pwordexp->we_wordc = 1; pwordexp->we_wordc = 1;
pwordexp->we_wordv[0] = pwordexp->wordPos; pwordexp->we_wordv[0] = pwordexp->wordPos;
memset(pwordexp->wordPos, 0, 1025); (void)memset(pwordexp->wordPos, 0, 1025);
if (_fullpath(pwordexp->wordPos, words, 1024) == NULL) { if (_fullpath(pwordexp->wordPos, words, 1024) == NULL) {
pwordexp->we_wordv[0] = words; pwordexp->we_wordv[0] = words;
printf("failed to parse relative path:%s to abs path\n", words); printf("failed to parse relative path:%s to abs path\n", words);
@ -100,7 +100,7 @@ void taosRemoveDir(const char *dirname) {
if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue; if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue;
char filename[1024] = {0}; char filename[1024] = {0};
snprintf(filename, sizeof(filename), "%s%s%s", dirname, TD_DIRSEP, taosGetDirEntryName(de)); (void)snprintf(filename, sizeof(filename), "%s%s%s", dirname, TD_DIRSEP, taosGetDirEntryName(de));
if (taosDirEntryIsDir(de)) { if (taosDirEntryIsDir(de)) {
taosRemoveDir(filename); taosRemoveDir(filename);
} else { } else {
@ -109,8 +109,8 @@ void taosRemoveDir(const char *dirname) {
} }
} }
taosCloseDir(&pDir); (void)taosCloseDir(&pDir);
rmdir(dirname); (void)rmdir(dirname);
// printf("dir:%s is removed\n", dirname); // printf("dir:%s is removed\n", dirname);
return; return;
@ -127,8 +127,13 @@ int32_t taosMkDir(const char *dirname) {
#else #else
int32_t code = mkdir(dirname, 0755); int32_t code = mkdir(dirname, 0755);
#endif #endif
if (code < 0 && errno == EEXIST) { if (-1 == code) {
return 0; if (errno == EEXIST) {
return 0;
} else {
terrno = TAOS_SYSTEM_ERROR(errno);
code = terrno;
}
} }
return code; return code;
@ -143,7 +148,7 @@ int32_t taosMulMkDir(const char *dirname) {
taosRealPath(dirname, temp, sizeof(temp)); taosRealPath(dirname, temp, sizeof(temp));
if (temp[1] == ':') pos += 3; if (temp[1] == ':') pos += 3;
#else #else
strcpy(temp, dirname); (void)strcpy(temp, dirname);
#endif #endif
if (taosDirExist(temp)) return code; if (taosDirExist(temp)) return code;
@ -165,7 +170,7 @@ int32_t taosMulMkDir(const char *dirname) {
code = mkdir(temp, 0755); code = mkdir(temp, 0755);
#endif #endif
if (code < 0 && errno != EEXIST) { if (code < 0 && errno != EEXIST) {
// terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return code; return code;
} }
*pos = TD_DIRSEP[0]; *pos = TD_DIRSEP[0];
@ -181,7 +186,7 @@ int32_t taosMulMkDir(const char *dirname) {
code = mkdir(temp, 0755); code = mkdir(temp, 0755);
#endif #endif
if (code < 0 && errno != EEXIST) { if (code < 0 && errno != EEXIST) {
// terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return code; return code;
} }
} }
@ -194,7 +199,10 @@ int32_t taosMulMkDir(const char *dirname) {
} }
int32_t taosMulModeMkDir(const char *dirname, int mode, bool checkAccess) { int32_t taosMulModeMkDir(const char *dirname, int mode, bool checkAccess) {
if (dirname == NULL || strlen(dirname) >= TDDIRMAXLEN) return -1; if (dirname == NULL || strlen(dirname) >= TDDIRMAXLEN) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
char temp[TDDIRMAXLEN]; char temp[TDDIRMAXLEN];
char *pos = temp; char *pos = temp;
int32_t code = 0; int32_t code = 0;
@ -202,14 +210,18 @@ int32_t taosMulModeMkDir(const char *dirname, int mode, bool checkAccess) {
taosRealPath(dirname, temp, sizeof(temp)); taosRealPath(dirname, temp, sizeof(temp));
if (temp[1] == ':') pos += 3; if (temp[1] == ':') pos += 3;
#else #else
strcpy(temp, dirname); (void)strcpy(temp, dirname);
#endif #endif
if (taosDirExist(temp)) { if (taosDirExist(temp)) {
if (checkAccess && taosCheckAccessFile(temp, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) { if (checkAccess && taosCheckAccessFile(temp, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) {
return 0; return 0;
} }
return chmod(temp, mode); code = chmod(temp, mode);
if (-1 == code) {
terrno = TAOS_SYSTEM_ERROR(errno);
return terrno;
}
} }
if (strncmp(temp, TD_DIRSEP, 1) == 0) { if (strncmp(temp, TD_DIRSEP, 1) == 0) {
@ -229,8 +241,8 @@ int32_t taosMulModeMkDir(const char *dirname, int mode, bool checkAccess) {
code = mkdir(temp, mode); code = mkdir(temp, mode);
#endif #endif
if (code < 0 && errno != EEXIST) { if (code < 0 && errno != EEXIST) {
// terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return code; return terrno;
} }
*pos = TD_DIRSEP[0]; *pos = TD_DIRSEP[0];
} }
@ -245,8 +257,8 @@ int32_t taosMulModeMkDir(const char *dirname, int mode, bool checkAccess) {
code = mkdir(temp, mode); code = mkdir(temp, mode);
#endif #endif
if (code < 0 && errno != EEXIST) { if (code < 0 && errno != EEXIST) {
// terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return code; return terrno;
} }
} }
@ -254,10 +266,14 @@ int32_t taosMulModeMkDir(const char *dirname, int mode, bool checkAccess) {
if (checkAccess && taosCheckAccessFile(temp, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) { if (checkAccess && taosCheckAccessFile(temp, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) {
return 0; return 0;
} }
return chmod(temp, mode);
} }
return chmod(temp, mode); code = chmod(temp, mode);
if (-1 == code) {
terrno = TAOS_SYSTEM_ERROR(errno);
return terrno;
}
return code;
} }
void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { void taosRemoveOldFiles(const char *dirname, int32_t keepDays) {
@ -271,7 +287,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) {
if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue; if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue;
char filename[1024]; char filename[1024];
snprintf(filename, sizeof(filename), "%s/%s", dirname, taosGetDirEntryName(de)); (void)snprintf(filename, sizeof(filename), "%s/%s", dirname, taosGetDirEntryName(de));
if (taosDirEntryIsDir(de)) { if (taosDirEntryIsDir(de)) {
continue; continue;
} else { } else {
@ -299,34 +315,25 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) {
} }
} }
taosCloseDir(&pDir); (void)taosCloseDir(&pDir);
rmdir(dirname); (void)rmdir(dirname);
} }
int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) {
wordexp_t full_path; wordexp_t full_path;
switch (wordexp(dirname, &full_path, 0)) { int32_t code = wordexp(dirname, &full_path, 0);
switch (code) {
case 0: case 0:
break; break;
case WRDE_NOSPACE: case WRDE_NOSPACE:
wordfree(&full_path); wordfree(&full_path);
// printf("failed to expand path:%s since Out of memory\n", dirname); // FALL THROUGH
return -1;
case WRDE_BADCHAR:
// printf("failed to expand path:%s since illegal occurrence of newline or one of |, &, ;, <, >, (, ), {, }\n",
// dirname);
return -1;
case WRDE_SYNTAX:
// printf("failed to expand path:%s since Shell syntax error, such as unbalanced parentheses or unmatched
// quotes\n", dirname);
return -1;
default: default:
// printf("failed to expand path:%s since %s\n", dirname, strerror(errno)); return code;
return -1;
} }
if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) { if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) {
strncpy(outname, full_path.we_wordv[0], maxlen); (void)strncpy(outname, full_path.we_wordv[0], maxlen);
} }
wordfree(&full_path); wordfree(&full_path);
@ -343,21 +350,23 @@ int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen) {
#endif #endif
if (strlen(tmp) < maxlen) { if (strlen(tmp) < maxlen) {
if (realPath == NULL) { if (realPath == NULL) {
strncpy(dirname, tmp, maxlen); (void)strncpy(dirname, tmp, maxlen);
} else { } else {
strncpy(realPath, tmp, maxlen); (void)strncpy(realPath, tmp, maxlen);
} }
return 0; return 0;
} }
} }
return -1; terrno = TAOS_SYSTEM_ERROR(errno);
return terrno;
} }
bool taosIsDir(const char *dirname) { bool taosIsDir(const char *dirname) {
TdDirPtr pDir = taosOpenDir(dirname); TdDirPtr pDir = taosOpenDir(dirname);
if (pDir != NULL) { if (pDir != NULL) {
taosCloseDir(&pDir); (void)taosCloseDir(&pDir);
return true; return true;
} }
return false; return false;
@ -410,6 +419,7 @@ char *taosDirEntryBaseName(char *name) {
TdDirPtr taosOpenDir(const char *dirname) { TdDirPtr taosOpenDir(const char *dirname) {
if (dirname == NULL) { if (dirname == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL; return NULL;
} }
@ -436,12 +446,17 @@ TdDirPtr taosOpenDir(const char *dirname) {
dirPtr->pDir = pDir; dirPtr->pDir = pDir;
return dirPtr; return dirPtr;
#else #else
return (TdDirPtr)opendir(dirname); TdDirPtr ptr = (TdDirPtr)opendir(dirname);
if (NULL == ptr) {
terrno = TAOS_SYSTEM_ERROR(errno);
}
return ptr;
#endif #endif
} }
TdDirEntryPtr taosReadDir(TdDirPtr pDir) { TdDirEntryPtr taosReadDir(TdDirPtr pDir) {
if (pDir == NULL) { if (pDir == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL; return NULL;
} }
#ifdef WINDOWS #ifdef WINDOWS
@ -456,7 +471,13 @@ TdDirEntryPtr taosReadDir(TdDirPtr pDir) {
return NULL; return NULL;
} }
#else #else
return (TdDirEntryPtr)readdir((DIR *)pDir); errno = 0;
terrno = 0;
TdDirEntryPtr p = (TdDirEntryPtr)readdir((DIR *)pDir);
if (NULL == p && errno) {
terrno = TAOS_SYSTEM_ERROR(errno);
}
return p;
#endif #endif
} }
@ -484,7 +505,8 @@ char *taosGetDirEntryName(TdDirEntryPtr pDirEntry) {
int32_t taosCloseDir(TdDirPtr *ppDir) { int32_t taosCloseDir(TdDirPtr *ppDir) {
if (ppDir == NULL || *ppDir == NULL) { if (ppDir == NULL || *ppDir == NULL) {
return -1; terrno = TSDB_CODE_INVALID_PARA;
return terrno;
} }
#ifdef WINDOWS #ifdef WINDOWS
FindClose((*ppDir)->hFind); FindClose((*ppDir)->hFind);
@ -497,8 +519,12 @@ int32_t taosCloseDir(TdDirPtr *ppDir) {
*ppDir = NULL; *ppDir = NULL;
return 0; return 0;
#else #else
closedir((DIR *)*ppDir); int32_t code = closedir((DIR *)*ppDir);
*ppDir = NULL; *ppDir = NULL;
if (-1 == code) {
terrno = TAOS_SYSTEM_ERROR(errno);
return terrno;
}
return 0; return 0;
#endif #endif
} }

View File

@ -45,11 +45,16 @@ char tsAVX2Supported = 0;
char tsFMASupported = 0; char tsFMASupported = 0;
char tsAVX512Supported = 0; char tsAVX512Supported = 0;
void osDefaultInit() { int32_t osDefaultInit() {
int32_t code = TSDB_CODE_SUCCESS;
taosSeedRand(taosSafeRand()); taosSeedRand(taosSafeRand());
taosGetSystemLocale(tsLocale, tsCharset); taosGetSystemLocale(tsLocale, tsCharset);
taosGetSystemTimezone(tsTimezoneStr, &tsTimezone); taosGetSystemTimezone(tsTimezoneStr, &tsTimezone);
taosSetSystemTimezone(tsTimezoneStr, tsTimezoneStr, &tsDaylight, &tsTimezone); code = taosSetSystemTimezone(tsTimezoneStr, tsTimezoneStr, &tsDaylight, &tsTimezone);
if (code) {
return code;
}
taosGetSystemInfo(); taosGetSystemInfo();
// deadlock in query // deadlock in query
@ -65,33 +70,35 @@ void osDefaultInit() {
tmpDir = getenv("temp"); tmpDir = getenv("temp");
} }
if (tmpDir != NULL) { if (tmpDir != NULL) {
strcpy(tsTempDir, tmpDir); (void)strcpy(tsTempDir, tmpDir);
} }
strcpy(tsOsName, "Windows"); (void)strcpy(tsOsName, "Windows");
#elif defined(_TD_DARWIN_64) #elif defined(_TD_DARWIN_64)
strcpy(tsOsName, "Darwin"); (void)strcpy(tsOsName, "Darwin");
#else #else
strcpy(tsOsName, "Linux"); (void)strcpy(tsOsName, "Linux");
#endif #endif
if (configDir[0] == 0) { if (configDir[0] == 0) {
strcpy(configDir, TD_CFG_DIR_PATH); (void)strcpy(configDir, TD_CFG_DIR_PATH);
} }
strcpy(tsDataDir, TD_DATA_DIR_PATH); (void)strcpy(tsDataDir, TD_DATA_DIR_PATH);
strcpy(tsLogDir, TD_LOG_DIR_PATH); (void)strcpy(tsLogDir, TD_LOG_DIR_PATH);
if(strlen(tsTempDir) == 0){ if(strlen(tsTempDir) == 0){
strcpy(tsTempDir, TD_TMP_DIR_PATH); (void)strcpy(tsTempDir, TD_TMP_DIR_PATH);
} }
return code;
} }
void osUpdate() { void osUpdate() {
if (tsLogDir[0] != 0) { if (tsLogDir[0] != 0) {
taosGetDiskSize(tsLogDir, &tsLogSpace.size); (void)taosGetDiskSize(tsLogDir, &tsLogSpace.size);
} }
if (tsDataDir[0] != 0) { if (tsDataDir[0] != 0) {
taosGetDiskSize(tsDataDir, &tsDataSpace.size); (void)taosGetDiskSize(tsDataDir, &tsDataSpace.size);
} }
if (tsTempDir[0] != 0) { if (tsTempDir[0] != 0) {
taosGetDiskSize(tsTempDir, &tsTempSpace.size); (void)taosGetDiskSize(tsTempDir, &tsTempSpace.size);
} }
} }
@ -109,11 +116,11 @@ bool osDataSpaceSufficient() { return tsDataSpace.size.avail > tsDataSpace.reser
bool osTempSpaceSufficient() { return tsTempSpace.size.avail > tsTempSpace.reserved; } bool osTempSpaceSufficient() { return tsTempSpace.size.avail > tsTempSpace.reserved; }
void osSetTimezone(const char *tz) { taosSetSystemTimezone(tz, tsTimezoneStr, &tsDaylight, &tsTimezone); } int32_t osSetTimezone(const char *tz) { return taosSetSystemTimezone(tz, tsTimezoneStr, &tsDaylight, &tsTimezone); }
void osSetSystemLocale(const char *inLocale, const char *inCharSet) { void osSetSystemLocale(const char *inLocale, const char *inCharSet) {
memcpy(tsLocale, inLocale, strlen(inLocale) + 1); (void)memcpy(tsLocale, inLocale, strlen(inLocale) + 1);
memcpy(tsCharset, inCharSet, strlen(inCharSet) + 1); (void)memcpy(tsCharset, inCharSet, strlen(inCharSet) + 1);
} }
void osSetProcPath(int32_t argc, char **argv) { tsProcPath = argv[0]; } void osSetProcPath(int32_t argc, char **argv) { tsProcPath = argv[0]; }

View File

@ -67,7 +67,7 @@ typedef struct TdFile {
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) { void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) {
#ifdef WINDOWS #ifdef WINDOWS
char tmpPath[PATH_MAX]; char tmpPath[PATH_MAX];
int32_t len = (int32_t)strlen(inputTmpDir); int32_t len = (int32_t)strlen(inputTmpDir);
memcpy(tmpPath, inputTmpDir, len); memcpy(tmpPath, inputTmpDir, len);
@ -90,24 +90,24 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, cha
char tmpPath[PATH_MAX]; char tmpPath[PATH_MAX];
int32_t len = strlen(inputTmpDir); int32_t len = strlen(inputTmpDir);
memcpy(tmpPath, inputTmpDir, len); (void)memcpy(tmpPath, inputTmpDir, len);
static uint64_t seqId = 0; static uint64_t seqId = 0;
if (tmpPath[len - 1] != '/') { if (tmpPath[len - 1] != '/') {
tmpPath[len++] = '/'; tmpPath[len++] = '/';
} }
strcpy(tmpPath + len, TD_TMP_FILE_PREFIX); (void)strcpy(tmpPath + len, TD_TMP_FILE_PREFIX);
if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) { if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
strcat(tmpPath, fileNamePrefix); (void)strcat(tmpPath, fileNamePrefix);
strcat(tmpPath, "-%d-%s"); (void)strcat(tmpPath, "-%d-%s");
} }
char rand[32] = {0}; char rand[32] = {0};
sprintf(rand, "%" PRIu64, atomic_add_fetch_64(&seqId, 1)); (void)sprintf(rand, "%" PRIu64, atomic_add_fetch_64(&seqId, 1));
snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); (void)snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand);
#endif #endif
} }
@ -123,41 +123,60 @@ int64_t taosCopyFile(const char *from, const char *to) {
char buffer[4096]; char buffer[4096];
int64_t size = 0; int64_t size = 0;
int64_t bytes; int64_t bytes;
int32_t code = TSDB_CODE_SUCCESS;
// fidfrom = open(from, O_RDONLY); // fidfrom = open(from, O_RDONLY);
TdFilePtr pFileFrom = taosOpenFile(from, TD_FILE_READ); TdFilePtr pFileFrom = taosOpenFile(from, TD_FILE_READ);
if (pFileFrom == NULL) goto _err; if (pFileFrom == NULL) {
code = terrno;
goto _err;
}
// fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755); // fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755);
TdFilePtr pFileTo = taosOpenFile(to, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_EXCL); TdFilePtr pFileTo = taosOpenFile(to, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_EXCL);
if (pFileTo == NULL) goto _err; if (pFileTo == NULL) {
code = terrno;
goto _err;
}
while (true) { while (true) {
bytes = taosReadFile(pFileFrom, buffer, sizeof(buffer)); bytes = taosReadFile(pFileFrom, buffer, sizeof(buffer));
if (bytes < 0) goto _err; if (bytes < 0) {
code = terrno;
goto _err;
}
if (bytes == 0) break; if (bytes == 0) break;
size += bytes; size += bytes;
if (taosWriteFile(pFileTo, (void *)buffer, bytes) < bytes) goto _err; if (taosWriteFile(pFileTo, (void *)buffer, bytes) < bytes) {
code = terrno;
goto _err;
}
if (bytes < sizeof(buffer)) break; if (bytes < sizeof(buffer)) break;
} }
int code = taosFsyncFile(pFileTo); code = taosFsyncFile(pFileTo);
taosCloseFile(&pFileFrom); (void)taosCloseFile(&pFileFrom);
taosCloseFile(&pFileTo); (void)taosCloseFile(&pFileTo);
if (code != 0) { if (code != 0) {
terrno = code;
return -1; return -1;
} }
return size; return size;
_err: _err:
if (pFileFrom != NULL) taosCloseFile(&pFileFrom);
if (pFileTo != NULL) taosCloseFile(&pFileTo); if (pFileFrom != NULL) (void)taosCloseFile(&pFileFrom);
if (pFileTo != NULL) (void)taosCloseFile(&pFileTo);
/* coverity[+retval] */ /* coverity[+retval] */
taosRemoveFile(to); (void)taosRemoveFile(to);
terrno = code;
return -1; return -1;
#endif #endif
} }
@ -165,24 +184,31 @@ _err:
TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions) { TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions) {
TdFilePtr fp = taosOpenFile(path, tdFileOptions); TdFilePtr fp = taosOpenFile(path, tdFileOptions);
if (!fp) { if (!fp) {
if (errno == ENOENT) { if (terrno == TAOS_SYSTEM_ERROR(ENOENT)) {
// Try to create directory recursively // Try to create directory recursively
char *s = taosStrdup(path); char s[PATH_MAX];
tstrncpy(s, path, sizeof(s));
if (taosMulMkDir(taosDirName(s)) != 0) { if (taosMulMkDir(taosDirName(s)) != 0) {
taosMemoryFree(s);
return NULL; return NULL;
} }
taosMemoryFree(s);
fp = taosOpenFile(path, tdFileOptions); fp = taosOpenFile(path, tdFileOptions);
if (!fp) { if (!fp) {
return NULL; return NULL;
} }
} }
} }
return fp; return fp;
} }
int32_t taosRemoveFile(const char *path) { return remove(path); } int32_t taosRemoveFile(const char *path) {
int32_t code = remove(path);
if (-1 == code) {
terrno = TAOS_SYSTEM_ERROR(errno);
return terrno;
}
return code;
}
int32_t taosRenameFile(const char *oldName, const char *newName) { int32_t taosRenameFile(const char *oldName, const char *newName) {
#ifdef WINDOWS #ifdef WINDOWS
@ -214,8 +240,9 @@ int32_t taosRenameFile(const char *oldName, const char *newName) {
return finished ? 0 : -1; return finished ? 0 : -1;
#else #else
int32_t code = rename(oldName, newName); int32_t code = rename(oldName, newName);
if (code < 0) { if (-1 == code) {
printf("failed to rename file %s to %s, reason:%s\n", oldName, newName, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno);
return terrno;
} }
return code; return code;
@ -230,8 +257,9 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *a
struct stat fileStat; struct stat fileStat;
int32_t code = stat(path, &fileStat); int32_t code = stat(path, &fileStat);
#endif #endif
if (code < 0) { if (-1 == code) {
return code; terrno = TAOS_SYSTEM_ERROR(errno);
return terrno;
} }
if (size != NULL) { if (size != NULL) {
@ -269,13 +297,15 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) {
#else #else
if (pFile == NULL || pFile->fd < 0) { if (pFile == NULL || pFile->fd < 0) {
return -1; terrno = TSDB_CODE_INVALID_PARA;
return terrno;
} }
struct stat fileStat; struct stat fileStat;
int32_t code = fstat(pFile->fd, &fileStat); int32_t code = fstat(pFile->fd, &fileStat);
if (code < 0) { if (-1 == code) {
printf("taosFStatFile run fstat fail."); terrno = TAOS_SYSTEM_ERROR(errno);
return code; return terrno;
} }
if (stDev != NULL) { if (stDev != NULL) {
@ -301,11 +331,15 @@ FILE *taosOpenFileForStream(const char *path, int32_t tdFileOptions) {
} else { } else {
mode = (tdFileOptions & TD_FILE_TEXT) ? "rt+" : "rb+"; mode = (tdFileOptions & TD_FILE_TEXT) ? "rt+" : "rb+";
} }
ASSERT(!(tdFileOptions & TD_FILE_EXCL));
if (tdFileOptions & TD_FILE_EXCL) { if (tdFileOptions & TD_FILE_EXCL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL; return NULL;
} }
return fopen(path, mode); FILE* f = fopen(path, mode);
if (NULL == f) {
terrno = TAOS_SYSTEM_ERROR(errno);
}
return f;
} }
#ifdef WINDOWS #ifdef WINDOWS
@ -653,6 +687,9 @@ int taosOpenFileNotStream(const char *path, int32_t tdFileOptions) {
access |= (tdFileOptions & TD_FILE_CLOEXEC) ? O_CLOEXEC : 0; access |= (tdFileOptions & TD_FILE_CLOEXEC) ? O_CLOEXEC : 0;
int fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO); int fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO);
if (-1 == fd) {
terrno = TAOS_SYSTEM_ERROR(errno);
}
return fd; return fd;
} }
@ -1060,14 +1097,14 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) {
#ifdef WINDOWS #ifdef WINDOWS
if (hFile != NULL) CloseHandle(hFile); if (hFile != NULL) CloseHandle(hFile);
#else #else
if (fd >= 0) close(fd); if (fd >= 0) (void)close(fd);
#endif #endif
if (fp != NULL) fclose(fp); if (fp != NULL) (void)fclose(fp);
return NULL; return NULL;
} }
#if FILE_WITH_LOCK #if FILE_WITH_LOCK
taosThreadRwlockInit(&(pFile->rwlock), NULL); (void)taosThreadRwlockInit(&(pFile->rwlock), NULL);
#endif #endif
pFile->fp = fp; pFile->fp = fp;
pFile->refId = 0; pFile->refId = 0;
@ -1082,9 +1119,15 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) {
// Remove it instantly, so when the program exits normally/abnormally, the file // Remove it instantly, so when the program exits normally/abnormally, the file
// will be automatically remove by OS. // will be automatically remove by OS.
if (tdFileOptions & TD_FILE_AUTO_DEL) { if (tdFileOptions & TD_FILE_AUTO_DEL) {
unlink(path); if (-1 == unlink(path)) {
terrno = TAOS_SYSTEM_ERROR(errno);
(void)close(fd);
taosMemoryFree(pFile);
return NULL;
}
} }
#endif #endif
return pFile; return pFile;
} }
@ -1094,11 +1137,11 @@ int32_t taosCloseFile(TdFilePtr *ppFile) {
return 0; return 0;
} }
#if FILE_WITH_LOCK #if FILE_WITH_LOCK
taosThreadRwlockWrlock(&((*ppFile)->rwlock)); (void)taosThreadRwlockWrlock(&((*ppFile)->rwlock));
#endif #endif
if ((*ppFile)->fp != NULL) { if ((*ppFile)->fp != NULL) {
fflush((*ppFile)->fp); (void)fflush((*ppFile)->fp);
fclose((*ppFile)->fp); (void)fclose((*ppFile)->fp);
(*ppFile)->fp = NULL; (*ppFile)->fp = NULL;
} }
#ifdef WINDOWS #ifdef WINDOWS
@ -1113,16 +1156,20 @@ int32_t taosCloseFile(TdFilePtr *ppFile) {
// warning: never fsync silently in base lib // warning: never fsync silently in base lib
/*fsync((*ppFile)->fd);*/ /*fsync((*ppFile)->fd);*/
code = close((*ppFile)->fd); code = close((*ppFile)->fd);
if (-1 == code) {
terrno = TAOS_SYSTEM_ERROR(errno);
}
(*ppFile)->fd = -1; (*ppFile)->fd = -1;
#endif #endif
} }
(*ppFile)->refId = 0; (*ppFile)->refId = 0;
#if FILE_WITH_LOCK #if FILE_WITH_LOCK
taosThreadRwlockUnlock(&((*ppFile)->rwlock)); (void)taosThreadRwlockUnlock(&((*ppFile)->rwlock));
taosThreadRwlockDestroy(&((*ppFile)->rwlock)); (void)taosThreadRwlockDestroy(&((*ppFile)->rwlock));
#endif #endif
taosMemoryFree(*ppFile); taosMemoryFree(*ppFile);
*ppFile = NULL; *ppFile = NULL;
return code; return code;
} }
@ -1130,6 +1177,8 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset)
if (pFile == NULL) { if (pFile == NULL) {
return 0; return 0;
} }
int32_t code = 0;
#ifdef WINDOWS #ifdef WINDOWS
#if FILE_WITH_LOCK #if FILE_WITH_LOCK
@ -1155,20 +1204,27 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset)
} }
#else #else
#if FILE_WITH_LOCK #if FILE_WITH_LOCK
taosThreadRwlockRdlock(&(pFile->rwlock)); (void)taosThreadRwlockRdlock(&(pFile->rwlock));
#endif #endif
ASSERT(pFile->fd >= 0); // Please check if you have closed the file. ASSERT(pFile->fd >= 0); // Please check if you have closed the file.
if (pFile->fd < 0) { if (pFile->fd < 0) {
#if FILE_WITH_LOCK #if FILE_WITH_LOCK
taosThreadRwlockUnlock(&(pFile->rwlock)); (void)taosThreadRwlockUnlock(&(pFile->rwlock));
#endif #endif
terrno = TSDB_CODE_INVALID_PARA;
return -1; return -1;
} }
int64_t ret = pread(pFile->fd, buf, count, offset); int64_t ret = pread(pFile->fd, buf, count, offset);
if (-1 == ret) {
code = TAOS_SYSTEM_ERROR(errno);
}
#endif #endif
#if FILE_WITH_LOCK #if FILE_WITH_LOCK
taosThreadRwlockUnlock(&(pFile->rwlock)); (void)taosThreadRwlockUnlock(&(pFile->rwlock));
#endif #endif
terrno = code;
return ret; return ret;
} }
@ -1177,20 +1233,36 @@ int32_t taosFsyncFile(TdFilePtr pFile) {
return 0; return 0;
} }
int32_t code = 0;
// this implementation is WRONG // this implementation is WRONG
// fflush is not a replacement of fsync // fflush is not a replacement of fsync
if (pFile->fp != NULL) return fflush(pFile->fp); if (pFile->fp != NULL) {
code = fflush(pFile->fp);
if (0 != code) {
terrno = TAOS_SYSTEM_ERROR(errno);
return terrno;
}
return code;
}
#ifdef WINDOWS #ifdef WINDOWS
if (pFile->hFile != NULL) { if (pFile->hFile != NULL) {
if (pFile->tdFileOptions & TD_FILE_WRITE_THROUGH) { if (pFile->tdFileOptions & TD_FILE_WRITE_THROUGH) {
return 0; return 0;
} }
return !FlushFileBuffers(pFile->hFile); return !FlushFileBuffers(pFile->hFile);
}
#else #else
if (pFile->fd >= 0) { if (pFile->fd >= 0) {
return fsync(pFile->fd); code = fsync(pFile->fd);
#endif if (-1 == code) {
terrno = TAOS_SYSTEM_ERROR(errno);
return terrno;
}
} }
#endif
return 0; return 0;
} }
@ -1239,7 +1311,7 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) {
#ifdef WINDOWS #ifdef WINDOWS
size_t bufferSize = 512; size_t bufferSize = 512;
*ptrBuf = taosMemoryMalloc(bufferSize); *ptrBuf = taosMemoryMalloc(bufferSize);
if (*ptrBuf == NULL) goto END; if (*ptrBuf == NULL) goto END;
size_t bytesRead = 0; size_t bytesRead = 0;
size_t totalBytesRead = 0; size_t totalBytesRead = 0;
@ -1274,7 +1346,7 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) {
ret = getline(ptrBuf, &len, pFile->fp); ret = getline(ptrBuf, &len, pFile->fp);
#endif #endif
END: END:
#if FILE_WITH_LOCK #if FILE_WITH_LOCK
taosThreadRwlockUnlock(&(pFile->rwlock)); taosThreadRwlockUnlock(&(pFile->rwlock));
#endif #endif
@ -1413,34 +1485,30 @@ int32_t taosLinkFile(char *src, char *dst) {
return 0; return 0;
} }
FILE* taosOpenCFile(const char* filename, const char* mode) { FILE *taosOpenCFile(const char *filename, const char *mode) { return fopen(filename, mode); }
return fopen(filename, mode);
}
int taosSeekCFile(FILE* file, int64_t offset, int whence) { int taosSeekCFile(FILE *file, int64_t offset, int whence) {
#ifdef WINDOWS #ifdef WINDOWS
return _fseeki64(file, offset, whence); return _fseeki64(file, offset, whence);
#else #else
return fseeko(file, offset, whence); return fseeko(file, offset, whence);
#endif #endif
} }
size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream ) { size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream) {
return fread(buffer, size, count, stream); return fread(buffer, size, count, stream);
} }
size_t taosWriteToCFile(const void* ptr, size_t size, size_t nitems, FILE* stream) { size_t taosWriteToCFile(const void *ptr, size_t size, size_t nitems, FILE *stream) {
return fwrite(ptr, size, nitems, stream); return fwrite(ptr, size, nitems, stream);
} }
int taosCloseCFile(FILE *f) { int taosCloseCFile(FILE *f) { return fclose(f); }
return fclose(f);
}
int taosSetAutoDelFile(char* path) { int taosSetAutoDelFile(char *path) {
#ifdef WINDOWS #ifdef WINDOWS
return SetFileAttributes(path, FILE_ATTRIBUTE_TEMPORARY); return SetFileAttributes(path, FILE_ATTRIBUTE_TEMPORARY);
#else #else
return unlink(path); return unlink(path);
#endif #endif
} }

View File

@ -186,7 +186,7 @@ static void print_line(Dwarf_Debug dbg, Dwarf_Line line, Dwarf_Addr pc) {
dwarf_linesrc(line, &linesrc, NULL); dwarf_linesrc(line, &linesrc, NULL);
dwarf_lineno(line, &lineno, NULL); dwarf_lineno(line, &lineno, NULL);
} }
printf("BackTrace %08" PRId64 " %s:%" DW_PR_DUu "\n", taosGetSelfPthreadId(), linesrc, lineno); (void)printf("BackTrace %08" PRId64 " %s:%" DW_PR_DUu "\n", taosGetSelfPthreadId(), linesrc, lineno);
if (line) dwarf_dealloc(dbg, linesrc, DW_DLA_STRING); if (line) dwarf_dealloc(dbg, linesrc, DW_DLA_STRING);
} }
void taosPrintBackTrace() { void taosPrintBackTrace() {
@ -266,7 +266,11 @@ void *taosMemoryMalloc(int64_t size) {
return (char *)tmp + sizeof(TdMemoryInfo); return (char *)tmp + sizeof(TdMemoryInfo);
#else #else
return malloc(size); void* p = malloc(size);
if (NULL == p) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
}
return p;
#endif #endif
} }
@ -283,7 +287,11 @@ void *taosMemoryCalloc(int64_t num, int64_t size) {
return (char *)tmp + sizeof(TdMemoryInfo); return (char *)tmp + sizeof(TdMemoryInfo);
#else #else
return calloc(num, size); void* p = calloc(num, size);
if (NULL == p) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
}
return p;
#endif #endif
} }
@ -309,7 +317,11 @@ void *taosMemoryRealloc(void *ptr, int64_t size) {
return (char *)tmp + sizeof(TdMemoryInfo); return (char *)tmp + sizeof(TdMemoryInfo);
#else #else
return realloc(ptr, size); void* p = realloc(ptr, size);
if (size > 0 && NULL == p) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
}
return p;
#endif #endif
} }
@ -330,7 +342,12 @@ char *taosStrdup(const char *ptr) {
return (char *)tmp + sizeof(TdMemoryInfo); return (char *)tmp + sizeof(TdMemoryInfo);
#else #else
return tstrdup(ptr); char* p = tstrdup(ptr);
if (ptr != NULL && NULL == p) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
}
return p;
#endif #endif
} }
@ -357,8 +374,7 @@ int64_t taosMemorySize(void *ptr) {
TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char *)ptr - sizeof(TdMemoryInfo)); TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char *)ptr - sizeof(TdMemoryInfo));
ASSERT(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); ASSERT(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL);
if (pTdMemoryInfo->symbol != TD_MEMORY_SYMBOL) { if (pTdMemoryInfo->symbol != TD_MEMORY_SYMBOL) {
+return NULL; return NULL;
+
} }
return pTdMemoryInfo->memorySize; return pTdMemoryInfo->memorySize;
@ -378,7 +394,7 @@ void taosMemoryTrim(int32_t size) {
// do nothing // do nothing
return; return;
#else #else
malloc_trim(size); (void)malloc_trim(size);
#endif #endif
} }
@ -388,6 +404,13 @@ void *taosMemoryMallocAlign(uint32_t alignment, int64_t size) {
#else #else
#if defined(LINUX) #if defined(LINUX)
void *p = memalign(alignment, size); void *p = memalign(alignment, size);
if (NULL == p) {
if (ENOMEM == errno) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
} else {
terrno = TAOS_SYSTEM_ERROR(errno);
}
}
return p; return p;
#else #else
return taosMemoryMalloc(size); return taosMemoryMalloc(size);

View File

@ -69,7 +69,7 @@ uint32_t taosSafeRand(void) {
if (len < 0) { if (len < 0) {
seed = (int)taosGetTimestampSec(); seed = (int)taosGetTimestampSec();
} }
taosCloseFile(&pFile); (void)taosCloseFile(&pFile);
} }
return (uint32_t)seed; return (uint32_t)seed;

View File

@ -26,7 +26,9 @@ void taosSsleep(int32_t s) {
#ifdef WINDOWS #ifdef WINDOWS
Sleep(1000 * s); Sleep(1000 * s);
#else #else
sleep(s); while (s > 0) {
s = sleep(s);
}
#endif #endif
} }
@ -35,7 +37,7 @@ void taosMsleep(int32_t ms) {
#ifdef WINDOWS #ifdef WINDOWS
Sleep(ms); Sleep(ms);
#else #else
usleep(ms * 1000); (void)usleep(ms * 1000);
#endif #endif
} }
@ -51,6 +53,6 @@ void taosUsleep(int32_t us) {
WaitForSingleObject(timer, INFINITE); WaitForSingleObject(timer, INFINITE);
CloseHandle(timer); CloseHandle(timer);
#else #else
usleep(us); (void)usleep(us);
#endif #endif
} }

View File

@ -345,6 +345,7 @@ char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) {
} }
int32_t taosGetTimeOfDay(struct timeval *tv) { int32_t taosGetTimeOfDay(struct timeval *tv) {
int32_t code = 0;
#ifdef WINDOWS #ifdef WINDOWS
LARGE_INTEGER t; LARGE_INTEGER t;
FILETIME f; FILETIME f;
@ -359,11 +360,18 @@ int32_t taosGetTimeOfDay(struct timeval *tv) {
tv->tv_usec = (t.QuadPart % 10000000) / 10; tv->tv_usec = (t.QuadPart % 10000000) / 10;
return 0; return 0;
#else #else
return gettimeofday(tv, NULL); code = gettimeofday(tv, NULL);
return (-1 == code) ? (terrno = TAOS_SYSTEM_ERROR(errno)) : 0;
#endif #endif
} }
time_t taosTime(time_t *t) { return time(t); } time_t taosTime(time_t *t) {
time_t r = time(t);
if (r == (time_t)-1) {
terrno = TAOS_SYSTEM_ERROR(errno);
}
return r;
}
/* /*
* mktime64 - Converts date to seconds. * mktime64 - Converts date to seconds.
@ -458,7 +466,11 @@ time_t taosMktime(struct tm *timep) {
timep->tm_sec, tz); timep->tm_sec, tz);
#endif #endif
#else #else
return mktime(timep); time_t r = mktime(timep);
if (r == (time_t)-1) {
terrno = TAOS_SYSTEM_ERROR(errno);
}
return r;
#endif #endif
} }
@ -470,7 +482,7 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf) {
if (result == NULL) { if (result == NULL) {
res = localtime(timep); res = localtime(timep);
if (res == NULL && buf != NULL) { if (res == NULL && buf != NULL) {
sprintf(buf, "NaN"); (void)sprintf(buf, "NaN");
} }
return res; return res;
} }
@ -528,7 +540,7 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf) {
#else #else
res = localtime_r(timep, result); res = localtime_r(timep, result);
if (res == NULL && buf != NULL) { if (res == NULL && buf != NULL) {
sprintf(buf, "NaN"); (void)sprintf(buf, "NaN");
} }
#endif #endif
return result; return result;
@ -642,6 +654,7 @@ struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst)
int32_t taosGetTimestampSec() { return (int32_t)time(NULL); } int32_t taosGetTimestampSec() { return (int32_t)time(NULL); }
int32_t taosClockGetTime(int clock_id, struct timespec *pTS) { int32_t taosClockGetTime(int clock_id, struct timespec *pTS) {
int32_t code = 0;
#ifdef WINDOWS #ifdef WINDOWS
LARGE_INTEGER t; LARGE_INTEGER t;
FILETIME f; FILETIME f;
@ -656,6 +669,7 @@ int32_t taosClockGetTime(int clock_id, struct timespec *pTS) {
pTS->tv_nsec = (t.QuadPart % 10000000) * 100; pTS->tv_nsec = (t.QuadPart % 10000000) * 100;
return (0); return (0);
#else #else
return clock_gettime(clock_id, pTS); code = clock_gettime(clock_id, pTS);
return (-1 == code) ? (terrno = TAOS_SYSTEM_ERROR(errno)) : 0;
#endif #endif
} }

Some files were not shown because too many files have changed in this diff Show More