xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into feature/vnode

This commit is contained in:
Hongze Cheng 2021-12-21 10:05:27 +08:00
parent 816a9d9303 4e39099622
commit 2d315222d7
65 changed files with 1739 additions and 1065 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Jenkinsfile vendored
View File

@ -42,7 +42,6 @@ def pre_test(){
killall -9 gdb || echo "no gdb running" killall -9 gdb || echo "no gdb running"
killall -9 python3.8 || echo "no python program running" killall -9 python3.8 || echo "no python program running"
cd ${WKC} cd ${WKC}
git reset --hard HEAD~10 >/dev/null
''' '''
script { script {
if (env.CHANGE_TARGET == 'master') { if (env.CHANGE_TARGET == 'master') {
@ -75,11 +74,9 @@ def pre_test(){
git pull >/dev/null git pull >/dev/null
git fetch origin +refs/pull/${CHANGE_ID}/merge git fetch origin +refs/pull/${CHANGE_ID}/merge
git checkout -qf FETCH_HEAD git checkout -qf FETCH_HEAD
git clean -dfx
git clean -dfx
export TZ=Asia/Harbin export TZ=Asia/Harbin
date date
rm -rf debug
mkdir debug mkdir debug
cd debug cd debug
cmake .. > /dev/null cmake .. > /dev/null

25
include/common/taosmsg.h
View File

@ -50,9 +50,8 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONSUME, "mq-consume" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_QUERY, "mq-query" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_QUERY, "mq-query" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONNECT, "mq-connect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONNECT, "mq-connect" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_DISCONNECT, "mq-disconnect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_DISCONNECT, "mq-disconnect" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_SET, "mq-set" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_SET_CUR, "mq-set-cur" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RSP_READY, "rsp-ready" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RSP_READY, "rsp-ready" )
// message from client to mnode // message from client to mnode
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONNECT, "connect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONNECT, "connect" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_ACCT, "create-acct" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_ACCT, "create-acct" )
@ -225,6 +224,7 @@ typedef struct SBuildUseDBInput {
int32_t vgVersion; int32_t vgVersion;
} SBuildUseDBInput; } SBuildUseDBInput;
#pragma pack(push, 1) #pragma pack(push, 1)
// null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta // null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta
@ -343,7 +343,7 @@ typedef struct {
typedef struct { typedef struct {
char tableFname[TSDB_TABLE_FNAME_LEN]; char tableFname[TSDB_TABLE_FNAME_LEN];
char db[TSDB_FULL_DB_NAME_LEN]; char db[TSDB_FULL_DB_NAME_LEN];
int16_t type; /* operation type */ int16_t type; /* operation type */
int16_t numOfCols; /* number of schema */ int16_t numOfCols; /* number of schema */
int32_t tagValLen; int32_t tagValLen;
SSchema schema[]; SSchema schema[];
@ -545,8 +545,8 @@ typedef struct {
int32_t sqlstrLen; // sql query string int32_t sqlstrLen; // sql query string
int32_t prevResultLen; // previous result length int32_t prevResultLen; // previous result length
int32_t numOfOperator; int32_t numOfOperator;
int32_t tableScanOperator; // table scan operator. -1 means no scan operator int32_t tableScanOperator;// table scan operator. -1 means no scan operator
int32_t udfNum; // number of udf function int32_t udfNum; // number of udf function
int32_t udfContentOffset; int32_t udfContentOffset;
int32_t udfContentLen; int32_t udfContentLen;
SColumnInfo tableCols[]; SColumnInfo tableCols[];
@ -591,8 +591,8 @@ typedef struct {
int32_t daysToKeep0; int32_t daysToKeep0;
int32_t daysToKeep1; int32_t daysToKeep1;
int32_t daysToKeep2; int32_t daysToKeep2;
int32_t minRowsPerFileBlock; int32_t minRows;
int32_t maxRowsPerFileBlock; int32_t maxRows;
int32_t commitTime; int32_t commitTime;
int32_t fsyncPeriod; int32_t fsyncPeriod;
int8_t walLevel; int8_t walLevel;
@ -706,7 +706,7 @@ typedef struct {
SVnodeLoad data[]; SVnodeLoad data[];
} SVnodeLoads; } SVnodeLoads;
typedef struct SStatusMsg { typedef struct {
int32_t sver; int32_t sver;
int32_t dnodeId; int32_t dnodeId;
int32_t clusterId; int32_t clusterId;
@ -756,6 +756,7 @@ typedef struct {
int32_t dnodeId; int32_t dnodeId;
char db[TSDB_FULL_DB_NAME_LEN]; char db[TSDB_FULL_DB_NAME_LEN];
uint64_t dbUid; uint64_t dbUid;
int32_t vgVersion;
int32_t cacheBlockSize; int32_t cacheBlockSize;
int32_t totalBlocks; int32_t totalBlocks;
int32_t daysPerFile; int32_t daysPerFile;
@ -1005,27 +1006,35 @@ typedef struct {
// mq related // mq related
typedef struct { typedef struct {
} SMqConnectReq; } SMqConnectReq;
typedef struct { typedef struct {
} SMqConnectRsp; } SMqConnectRsp;
typedef struct { typedef struct {
} SMqDisconnectReq; } SMqDisconnectReq;
typedef struct { typedef struct {
} SMqDisconnectRsp; } SMqDisconnectRsp;
typedef struct { typedef struct {
} SMqAckReq; } SMqAckReq;
typedef struct { typedef struct {
} SMqAckRsp; } SMqAckRsp;
typedef struct { typedef struct {
} SMqResetReq; } SMqResetReq;
typedef struct { typedef struct {
} SMqResetRsp; } SMqResetRsp;
// mq related end // mq related end

4
include/common/tmsgtype.h
View File

@ -35,7 +35,7 @@ enum {
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SELECT, "select" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SELECT, "select" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_FETCH, "fetch" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_FETCH, "fetch" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_INSERT, "insert" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_INSERT, "insert" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_UPDATE_TAGS_VAL, "update-tag-val" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_UPDATE_TAG_VAL, "update-tag-val" )
// the SQL below is for mgmt node // the SQL below is for mgmt node
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MGMT, "mgmt" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MGMT, "mgmt" )
@ -54,7 +54,7 @@ enum {
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_TABLE, "alter-table" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_TABLE, "alter-table" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_DB, "alter-db" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_DB, "alter-db" )
TSDB_DEFINE_SQL_TYPE(TSDB_SQL_SYNC_DB_REPLICA, "sync db-replica") TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SYNC_DB_REPLICA, "sync db-replica")
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_MNODE, "create-mnode" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_MNODE, "create-mnode" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_MNODE, "drop-mnode" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_MNODE, "drop-mnode" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DNODE, "create-dnode" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DNODE, "create-dnode" )

192
include/dnode/vnode/tq/tq.h
View File

@ -16,86 +16,76 @@
#ifndef _TD_TQ_H_ #ifndef _TD_TQ_H_
#define _TD_TQ_H_ #define _TD_TQ_H_
#include "common.h"
#include "mallocator.h" #include "mallocator.h"
#include "os.h" #include "os.h"
#include "taoserror.h"
#include "taosmsg.h"
#include "tlist.h"
#include "tutil.h" #include "tutil.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
typedef struct TmqMsgHead { typedef struct STqMsgHead {
int32_t protoVer; int32_t protoVer;
int32_t msgType; int32_t msgType;
int64_t cgId; int64_t cgId;
int64_t clientId; int64_t clientId;
} TmqMsgHead; } STqMsgHead;
typedef struct TmqOneAck { typedef struct STqOneAck {
int64_t topicId; int64_t topicId;
int64_t consumeOffset; int64_t consumeOffset;
} TmqOneAck; } STqOneAck;
typedef struct TmqAcks { typedef struct STqAcks {
int32_t ackNum; int32_t ackNum;
// should be sorted // should be sorted
TmqOneAck acks[]; STqOneAck acks[];
} TmqAcks; } STqAcks;
// TODO: put msgs into common typedef struct STqSetCurReq {
typedef struct TmqConnectReq { STqMsgHead head;
TmqMsgHead head; int64_t topicId;
TmqAcks acks; int64_t offset;
} TmqConnectReq; } STqSetCurReq;
typedef struct TmqConnectRsp {
TmqMsgHead head;
int8_t status;
} TmqConnectRsp;
typedef struct TmqDisconnectReq {
TmqMsgHead head;
} TmqDiscconectReq;
typedef struct TmqDisconnectRsp {
TmqMsgHead head;
int8_t status;
} TmqDisconnectRsp;
typedef struct STqConsumeReq { typedef struct STqConsumeReq {
TmqMsgHead head; STqMsgHead head;
TmqAcks acks; STqAcks acks;
} STqConsumeReq; } STqConsumeReq;
typedef struct TmqMsgContent { typedef struct STqMsgContent {
int64_t topicId; int64_t topicId;
int64_t msgLen; int64_t msgLen;
char msg[]; char msg[];
} TmqMsgContent; } STqMsgContent;
typedef struct STqConsumeRsp { typedef struct STqConsumeRsp {
TmqMsgHead head; STqMsgHead head;
int64_t bodySize; int64_t bodySize;
TmqMsgContent msgs[]; STqMsgContent msgs[];
} STqConsumeRsp; } STqConsumeRsp;
typedef struct TmqSubscribeReq { typedef struct STqSubscribeReq {
TmqMsgHead head; STqMsgHead head;
int32_t topicNum; int32_t topicNum;
int64_t topic[]; int64_t topic[];
} TmqSubscribeReq; } STqSubscribeReq;
typedef struct tmqSubscribeRsp { typedef struct STqSubscribeRsp {
TmqMsgHead head; STqMsgHead head;
int64_t vgId; int64_t vgId;
char ep[TSDB_EP_LEN]; // TSDB_EP_LEN char ep[TSDB_EP_LEN]; // TSDB_EP_LEN
} TmqSubscribeRsp; } STqSubscribeRsp;
typedef struct TmqHeartbeatReq { typedef struct STqHeartbeatReq {
} TmqHeartbeatReq; } STqHeartbeatReq;
typedef struct TmqHeartbeatRsp { typedef struct STqHeartbeatRsp {
} TmqHeartbeatRsp; } STqHeartbeatRsp;
typedef struct STqTopicVhandle { typedef struct STqTopicVhandle {
int64_t topicId; int64_t topicId;
@ -108,48 +98,54 @@ typedef struct STqTopicVhandle {
#define TQ_BUFFER_SIZE 8 #define TQ_BUFFER_SIZE 8
typedef struct STqExec {
void* runtimeEnv;
SSDataBlock* (*exec)(void* runtimeEnv);
void* (*assign)(void* runtimeEnv, SSubmitBlk* inputData);
void (*clear)(void* runtimeEnv);
char* (*serialize)(struct STqExec*);
struct STqExec* (*deserialize)(char*);
} STqExec;
typedef struct STqBufferItem { typedef struct STqBufferItem {
int64_t offset; int64_t offset;
// executors are identical but not concurrent // executors are identical but not concurrent
// so there must be a copy in each item // so there must be a copy in each item
void* executor; STqExec* executor;
int64_t size; int32_t status;
void* content; int64_t size;
} STqBufferItem; void* content;
} STqMsgItem;
typedef struct STqBufferHandle { typedef struct STqTopic {
// char* topic; //c style, end with '\0' // char* topic; //c style, end with '\0'
// int64_t cgId; // int64_t cgId;
// void* ahandle; // void* ahandle;
int64_t nextConsumeOffset; int64_t nextConsumeOffset;
int64_t floatingCursor; int64_t floatingCursor;
int64_t topicId; int64_t topicId;
int32_t head; int32_t head;
int32_t tail; int32_t tail;
STqBufferItem buffer[TQ_BUFFER_SIZE]; STqMsgItem buffer[TQ_BUFFER_SIZE];
} STqBufferHandle; } STqTopic;
typedef struct STqListHandle { typedef struct STqListHandle {
STqBufferHandle bufHandle; STqTopic topic;
struct STqListHandle* next; struct STqListHandle* next;
} STqListHandle; } STqList;
typedef struct STqGroupHandle { typedef struct STqGroup {
int64_t cId; int64_t clientId;
int64_t cgId; int64_t cgId;
void* ahandle; void* ahandle;
int32_t topicNum; int32_t topicNum;
STqListHandle* head; STqList* head;
} STqGroupHandle; SList* topicList; // SList<STqTopic>
void* returnMsg; // SVReadMsg
typedef struct STqQueryExec { } STqGroup;
void* src;
STqBufferItem* dest;
void* executor;
} STqQueryExec;
typedef struct STqQueryMsg { typedef struct STqQueryMsg {
STqQueryExec* exec; STqMsgItem* item;
struct STqQueryMsg* next; struct STqQueryMsg* next;
} STqQueryMsg; } STqQueryMsg;
@ -209,15 +205,15 @@ typedef void (*FTqDelete)(void*);
#define TQ_DUP_INTXN_REWRITE 0 #define TQ_DUP_INTXN_REWRITE 0
#define TQ_DUP_INTXN_REJECT 2 #define TQ_DUP_INTXN_REJECT 2
static inline bool TqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; } static inline bool tqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; }
static inline bool TqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; } static inline bool tqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; }
static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST; static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST;
#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE #define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE
typedef struct TqMetaHandle { typedef struct STqMetaHandle {
int64_t key; int64_t key;
int64_t offset; int64_t offset;
int64_t serializedSize; int64_t serializedSize;
@ -225,23 +221,25 @@ typedef struct TqMetaHandle {
void* valueInTxn; void* valueInTxn;
} STqMetaHandle; } STqMetaHandle;
typedef struct TqMetaList { typedef struct STqMetaList {
STqMetaHandle handle; STqMetaHandle handle;
struct TqMetaList* next; struct STqMetaList* next;
// struct TqMetaList* inTxnPrev; // struct STqMetaList* inTxnPrev;
// struct TqMetaList* inTxnNext; // struct STqMetaList* inTxnNext;
struct TqMetaList* unpersistPrev; struct STqMetaList* unpersistPrev;
struct TqMetaList* unpersistNext; struct STqMetaList* unpersistNext;
} STqMetaList; } STqMetaList;
typedef struct TqMetaStore { typedef struct STqMetaStore {
STqMetaList* bucket[TQ_BUCKET_SIZE]; STqMetaList* bucket[TQ_BUCKET_SIZE];
// a table head // a table head
STqMetaList* unpersistHead; STqMetaList* unpersistHead;
// TODO:temporaral use, to be replaced by unified tfile // TODO:temporaral use, to be replaced by unified tfile
int fileFd; int fileFd;
// TODO:temporaral use, to be replaced by unified tfile // TODO:temporaral use, to be replaced by unified tfile
int idxFd; int idxFd;
char* dirPath; char* dirPath;
int32_t tqConfigFlag; int32_t tqConfigFlag;
FTqSerialize pSerializer; FTqSerialize pSerializer;
@ -250,8 +248,8 @@ typedef struct TqMetaStore {
} STqMetaStore; } STqMetaStore;
typedef struct STQ { typedef struct STQ {
// the collection of group handle // the collection of groups
// the handle of kvstore // the handle of meta kvstore
char* path; char* path;
STqCfg* tqConfig; STqCfg* tqConfig;
STqLogReader* tqLogReader; STqLogReader* tqLogReader;
@ -266,23 +264,25 @@ void tqClose(STQ*);
// void* will be replace by a msg type // void* will be replace by a msg type
int tqPushMsg(STQ*, void* msg, int64_t version); int tqPushMsg(STQ*, void* msg, int64_t version);
int tqCommit(STQ*); int tqCommit(STQ*);
int tqSetCursor(STQ*, void* msg);
int tqConsume(STQ*, STqConsumeReq*); int tqConsume(STQ*, STqConsumeReq*);
STqGroupHandle* tqGetGroupHandle(STQ*, int64_t cId); int tqSetCursor(STQ*, STqSetCurReq* pMsg);
int tqBufferSetOffset(STqTopic*, int64_t offset);
STqGroupHandle* tqOpenTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); STqTopic* tqFindTopic(STqGroup*, int64_t topicId);
int tqCloseTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId);
int tqMoveOffsetToNext(STqGroupHandle*);
int tqResetOffset(STQ*, int64_t topicId, int64_t cgId, int64_t offset);
int tqRegisterContext(STqGroupHandle*, void* ahandle);
int tqLaunchQuery(STqGroupHandle*);
int tqSendLaunchQuery(STqGroupHandle*);
int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead); STqGroup* tqGetGroup(STQ*, int64_t clientId);
const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** gHandle); STqGroup* tqOpenGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId);
int tqCloseGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId);
int tqRegisterContext(STqGroup*, void* ahandle);
int tqSendLaunchQuery(STqMsgItem*, int64_t offset);
int tqSerializeGroup(const STqGroup*, STqSerializedHead**);
const void* tqDeserializeGroup(const STqSerializedHead*, STqGroup**);
static int tqQueryExecuting(int32_t status) { return status; }
#ifdef __cplusplus #ifdef __cplusplus
} }

4
include/dnode/vnode/vnode.h
View File

@ -197,8 +197,8 @@ void *vnodeParseReq(void *buf, SVnodeReq *pReq, uint8_t type);
// int32_t daysToKeep0; // int32_t daysToKeep0;
// int32_t daysToKeep1; // int32_t daysToKeep1;
// int32_t daysToKeep2; // int32_t daysToKeep2;
// int32_t minRowsPerFileBlock; // int32_t minRows;
// int32_t maxRowsPerFileBlock; // int32_t maxRows;
// int8_t precision; // time resolution // int8_t precision; // time resolution
// int8_t compression; // int8_t compression;
// int8_t cacheLastRow; // int8_t cacheLastRow;

8
include/libs/scheduler/scheduler.h
View File

@ -50,13 +50,15 @@ typedef struct SQueryProfileSummary {
uint64_t resultSize; // generated result size in Kb. uint64_t resultSize; // generated result size in Kb.
} SQueryProfileSummary; } SQueryProfileSummary;
int32_t schedulerInit(SSchedulerCfg *cfg);
/** /**
* Process the query job, generated according to the query physical plan. * Process the query job, generated according to the query physical plan.
* This is a synchronized API, and is also thread-safety. * This is a synchronized API, and is also thread-safety.
* @param pJob * @param qnodeList Qnode address list, element is SEpAddr
* @return * @return
*/ */
int32_t scheduleQueryJob(struct SCatalog *pCatalog, void *pRpc, const SEpSet* pMgmtEps, SQueryDag* pDag, void** pJob); int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob);
int32_t scheduleFetchRows(void *pJob, void **data); int32_t scheduleFetchRows(void *pJob, void **data);
@ -70,6 +72,8 @@ int32_t scheduleCancelJob(void *pJob);
void scheduleFreeJob(void *pJob); void scheduleFreeJob(void *pJob);
void schedulerDestroy(void);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

14
include/util/taoserror.h
View File

@ -339,6 +339,20 @@ int32_t* taosGetErrno();
#define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) //"Invalid msg length") #define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) //"Invalid msg length")
#define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) //"Invalid msg type") #define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) //"Invalid msg type")
// tq
#define TSDB_CODE_TQ_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0A00) //"Invalid configuration")
#define TSDB_CODE_TQ_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0A01) //"Tq init failed")
#define TSDB_CODE_TQ_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0A02) //"No diskspace for tq")
#define TSDB_CODE_TQ_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0A03) //"No permission for disk files")
#define TSDB_CODE_TQ_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0A04) //"Data file(s) corrupted")
#define TSDB_CODE_TQ_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0A05) //"Out of memory")
#define TSDB_CODE_TQ_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0A06) //"File already exists")
#define TSDB_CODE_TQ_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0A07) //"Failed to create dir")
#define TSDB_CODE_TQ_META_NO_SUCH_KEY TAOS_DEF_ERROR_CODE(0, 0x0A08) //"Target key not found")
#define TSDB_CODE_TQ_META_KEY_NOT_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A09) //"Target key not in transaction")
#define TSDB_CODE_TQ_META_KEY_DUP_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A0A) //"Target key duplicated in transaction")
#define TSDB_CODE_TQ_GROUP_NOT_SET TAOS_DEF_ERROR_CODE(0, 0x0A0B) //"Group of corresponding client is not set by mnode")
// wal // wal
#define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) //"Unexpected generic error in wal") #define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) //"Unexpected generic error in wal")
#define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) //"WAL file is corrupted") #define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) //"WAL file is corrupted")

2
include/util/tlog.h
View File

@ -42,11 +42,11 @@ extern int32_t qDebugFlag;
extern int32_t wDebugFlag; extern int32_t wDebugFlag;
extern int32_t sDebugFlag; extern int32_t sDebugFlag;
extern int32_t tsdbDebugFlag; extern int32_t tsdbDebugFlag;
extern int32_t tqDebugFlag;
extern int32_t cqDebugFlag; extern int32_t cqDebugFlag;
extern int32_t debugFlag; extern int32_t debugFlag;
extern int32_t ctgDebugFlag; extern int32_t ctgDebugFlag;
#define DEBUG_FATAL 1U #define DEBUG_FATAL 1U
#define DEBUG_ERROR DEBUG_FATAL #define DEBUG_ERROR DEBUG_FATAL
#define DEBUG_WARN 2U #define DEBUG_WARN 2U

2
include/util/tthread.h
View File

@ -24,7 +24,7 @@ extern "C" {
#include "tdef.h" #include "tdef.h"
// create new thread // create new thread
pthread_t* taosCreateThread( void *(*__start_routine) (void *), void* param); pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param);
// destory thread // destory thread
bool taosDestoryThread(pthread_t* pthread); bool taosDestoryThread(pthread_t* pthread);
// thread running return true // thread running return true

2
source/client/src/clientImpl.c
View File

@ -153,7 +153,7 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
void* output = NULL; void* output = NULL;
int32_t outputLen = 0; int32_t outputLen = 0;
code = qParseQuerySql(pRequest->sqlstr, sqlLen, pRequest->requestId, &type, &output, &outputLen, pRequest->msgBuf, ERROR_MSG_BUF_DEFAULT_SIZE); code = qParseQuerySql(pRequest->sqlstr, sqlLen, pRequest->requestId, &type, &output, &outputLen, pRequest->msgBuf, ERROR_MSG_BUF_DEFAULT_SIZE);
if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW || type == TSDB_SQL_DROP_USER || type == TSDB_SQL_CREATE_DB) { if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW || type == TSDB_SQL_DROP_USER || type == TSDB_SQL_DROP_ACCT || type == TSDB_SQL_CREATE_DB || type == TSDB_SQL_CREATE_ACCT) {
pRequest->type = type; pRequest->type = type;
pRequest->body.param = output; pRequest->body.param = output;
pRequest->body.paramLen = outputLen; pRequest->body.paramLen = outputLen;

17
source/client/src/clientMsgHandler.c
View File

@ -407,7 +407,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
} }
// single table query error need to be handled here. // single table query error need to be handled here.
if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_UPDATE_TAG_VAL) &&
(((rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID)) || (((rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID)) ||
rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_APP_NOT_READY)) { rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_APP_NOT_READY)) {
@ -3176,6 +3176,15 @@ int32_t doBuildMsgSupp(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) {
case TSDB_SQL_CREATE_USER: case TSDB_SQL_CREATE_USER:
pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_USER; pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_USER;
break; break;
case TSDB_SQL_DROP_USER:
pMsgBody->msgType = TSDB_MSG_TYPE_DROP_USER;
break;
case TSDB_SQL_CREATE_ACCT:
pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_ACCT;
break;
case TSDB_SQL_DROP_ACCT:
pMsgBody->msgType = TSDB_MSG_TYPE_DROP_ACCT;
break;
case TSDB_SQL_CREATE_DB: { case TSDB_SQL_CREATE_DB: {
pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_DB; pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_DB;
@ -3328,7 +3337,7 @@ void initMsgHandleFp() {
tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg; tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg;
tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg; tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg;
tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg; tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg;
tscBuildMsg[TSDB_SQL_UPDATE_TAGS_VAL] = tscBuildUpdateTagMsg; tscBuildMsg[TSDB_SQL_UPDATE_TAG_VAL] = tscBuildUpdateTagMsg;
tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg; tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg;
tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg; tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg;
@ -3383,6 +3392,10 @@ void initMsgHandleFp() {
handleRequestRspFp[TSDB_SQL_CONNECT] = processConnectRsp; handleRequestRspFp[TSDB_SQL_CONNECT] = processConnectRsp;
buildRequestMsgFp[TSDB_SQL_CREATE_USER] = doBuildMsgSupp; buildRequestMsgFp[TSDB_SQL_CREATE_USER] = doBuildMsgSupp;
buildRequestMsgFp[TSDB_SQL_DROP_USER] = doBuildMsgSupp;
buildRequestMsgFp[TSDB_SQL_CREATE_ACCT] = doBuildMsgSupp;
buildRequestMsgFp[TSDB_SQL_DROP_ACCT] = doBuildMsgSupp;
buildRequestMsgFp[TSDB_SQL_SHOW] = doBuildMsgSupp; buildRequestMsgFp[TSDB_SQL_SHOW] = doBuildMsgSupp;
handleRequestRspFp[TSDB_SQL_SHOW] = processShowRsp; handleRequestRspFp[TSDB_SQL_SHOW] = processShowRsp;

1
source/client/src/tscEnv.c
View File

@ -189,7 +189,6 @@ static void doDestroyRequest(void* p) {
tfree(pRequest->pInfo); tfree(pRequest->pInfo);
if (pRequest->body.pResInfo != NULL) { if (pRequest->body.pResInfo != NULL) {
tfree(pRequest->body.pResInfo->pData);
tfree(pRequest->body.pResInfo->pMsg); tfree(pRequest->body.pResInfo->pMsg);
tfree(pRequest->body.pResInfo); tfree(pRequest->body.pResInfo);
} }

60
source/client/test/clientTests.cpp
View File

@ -40,13 +40,13 @@ TEST(testCase, driverInit_Test) {
TEST(testCase, connect_Test) { TEST(testCase, connect_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
assert(pConn != NULL); // assert(pConn != NULL);
taos_close(pConn); taos_close(pConn);
} }
TEST(testCase, create_user_Test) { TEST(testCase, create_user_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
assert(pConn != NULL); // assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'");
if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
@ -57,22 +57,35 @@ TEST(testCase, create_user_Test) {
taos_close(pConn); taos_close(pConn);
} }
//TEST(testCase, drop_user_Test) { TEST(testCase, create_account_Test) {
// TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
// assert(pConn != NULL); assert(pConn != NULL);
//
// TAOS_RES* pRes = taos_query(pConn, "drop user abc"); TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'");
// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); printf("failed to create user, reason:%s\n", taos_errstr(pRes));
// } }
//
// taos_free_result(pRes); taos_free_result(pRes);
// taos_close(pConn); taos_close(pConn);
//} }
TEST(testCase, drop_account_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "drop account aabc");
if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
printf("failed to create user, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
taos_close(pConn);
}
TEST(testCase, show_user_Test) { TEST(testCase, show_user_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
assert(pConn != NULL); // assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "show users"); TAOS_RES* pRes = taos_query(pConn, "show users");
TAOS_ROW pRow = NULL; TAOS_ROW pRow = NULL;
@ -89,10 +102,23 @@ TEST(testCase, show_user_Test) {
taos_close(pConn); taos_close(pConn);
} }
TEST(testCase, show_db_Test) { TEST(testCase, drop_user_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
assert(pConn != NULL); assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "drop user abc");
if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
printf("failed to create user, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
taos_close(pConn);
}
TEST(testCase, show_db_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "show databases"); TAOS_RES* pRes = taos_query(pConn, "show databases");
TAOS_ROW pRow = NULL; TAOS_ROW pRow = NULL;
@ -112,7 +138,7 @@ TEST(testCase, create_db_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
assert(pConn != NULL); assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "create database abc"); TAOS_RES* pRes = taos_query(pConn, "create database abc1");
TAOS_FIELD* pFields = taos_fetch_fields(pRes); TAOS_FIELD* pFields = taos_fetch_fields(pRes);
ASSERT_TRUE(pFields == NULL); ASSERT_TRUE(pFields == NULL);

2
source/dnode/mgmt/daemon/src/daemon.c
View File

@ -31,7 +31,7 @@ static struct {
} global = {0}; } global = {0};
void dmnSigintHandle(int signum, void *info, void *ctx) { void dmnSigintHandle(int signum, void *info, void *ctx) {
uError("singal:%d is received", signum); uInfo("singal:%d is received", signum);
global.stop = true; global.stop = true;
} }

2
source/dnode/mgmt/impl/src/dndTransport.c
View File

@ -44,7 +44,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONSUME] = dndProcessVnodeQueryMsg; pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONSUME] = dndProcessVnodeQueryMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONNECT] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONNECT] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_DISCONNECT] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_MQ_DISCONNECT] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_SET] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_MQ_SET_CUR] = dndProcessVnodeWriteMsg;
// msg from client to mnode // msg from client to mnode
pMgmt->msgFp[TSDB_MSG_TYPE_CONNECT] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_CONNECT] = dndProcessMnodeReadMsg;

366
source/dnode/mgmt/impl/src/dndVnodes.c
View File

@ -17,11 +17,23 @@
#include "dndVnodes.h" #include "dndVnodes.h"
#include "dndTransport.h" #include "dndTransport.h"
typedef struct {
int32_t vgId;
int32_t vgVersion;
int8_t dropped;
uint64_t dbUid;
char db[TSDB_FULL_DB_NAME_LEN];
char path[PATH_MAX + 20];
} SWrapperCfg;
typedef struct { typedef struct {
int32_t vgId; int32_t vgId;
int32_t refCount; int32_t refCount;
int32_t vgVersion;
int8_t dropped; int8_t dropped;
int8_t accessState; int8_t accessState;
uint64_t dbUid;
char *db;
char *path; char *path;
SVnode *pImpl; SVnode *pImpl;
taos_queue pWriteQ; taos_queue pWriteQ;
@ -32,13 +44,13 @@ typedef struct {
} SVnodeObj; } SVnodeObj;
typedef struct { typedef struct {
int32_t vnodeNum; int32_t vnodeNum;
int32_t opened; int32_t opened;
int32_t failed; int32_t failed;
int32_t threadIndex; int32_t threadIndex;
pthread_t *pThreadId; pthread_t thread;
SVnodeObj *pVnodes; SDnode *pDnode;
SDnode * pDnode; SWrapperCfg *pCfgs;
} SVnodeThread; } SVnodeThread;
static int32_t dndInitVnodeReadWorker(SDnode *pDnode); static int32_t dndInitVnodeReadWorker(SDnode *pDnode);
@ -73,16 +85,14 @@ void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEp
void dndProcessVnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessVnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SVnodeMsg *pMsg); static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SVnodeMsg *pMsg);
static SVnodeObj * dndAcquireVnode(SDnode *pDnode, int32_t vgId); static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId);
static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode); static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode);
static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, SVnode *pImpl); static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl);
static void dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode); static void dndCloseVnode(SDnode *pDnode, SVnodeObj *pVnode);
static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes); static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes);
static int32_t dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_t *numOfVnodes); static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_t *numOfVnodes);
static int32_t dndWriteVnodesToFile(SDnode *pDnode); static int32_t dndWriteVnodesToFile(SDnode *pDnode);
static int32_t dndCreateVnode(SDnode *pDnode, int32_t vgId, SVnodeCfg *pCfg);
static int32_t dndDropVnode(SDnode *pDnode, SVnodeObj *pVnode);
static int32_t dndOpenVnodes(SDnode *pDnode); static int32_t dndOpenVnodes(SDnode *pDnode);
static void dndCloseVnodes(SDnode *pDnode); static void dndCloseVnodes(SDnode *pDnode);
@ -126,22 +136,25 @@ static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode) {
dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount); dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount);
} }
static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, SVnode *pImpl) { static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl) {
SVnodesMgmt *pMgmt = &pDnode->vmgmt; SVnodesMgmt *pMgmt = &pDnode->vmgmt;
SVnodeObj * pVnode = calloc(1, sizeof(SVnodeObj)); SVnodeObj *pVnode = calloc(1, sizeof(SVnodeObj));
if (pVnode == NULL) { if (pVnode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
pVnode->vgId = vgId; pVnode->vgId = pCfg->vgId;
pVnode->refCount = 0; pVnode->refCount = 1;
pVnode->dropped = 0; pVnode->dropped = 0;
pVnode->accessState = TSDB_VN_ALL_ACCCESS; pVnode->accessState = TSDB_VN_ALL_ACCCESS;
pVnode->pImpl = pImpl; pVnode->pImpl = pImpl;
pVnode->vgVersion = pCfg->vgVersion;
pVnode->dbUid = pCfg->dbUid;
pVnode->db = tstrdup(pCfg->db);
pVnode->path = tstrdup(pCfg->path);
pVnode->path = tstrdup(path); if (pVnode->path == NULL || pVnode->db == NULL) {
if (pVnode->path == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
@ -167,7 +180,7 @@ static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, S
} }
taosWLockLatch(&pMgmt->latch); taosWLockLatch(&pMgmt->latch);
int32_t code = taosHashPut(pMgmt->hash, &vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *));
taosWUnLockLatch(&pMgmt->latch); taosWUnLockLatch(&pMgmt->latch);
if (code != 0) { if (code != 0) {
@ -176,7 +189,7 @@ static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, S
return code; return code;
} }
static void dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode) { static void dndCloseVnode(SDnode *pDnode, SVnodeObj *pVnode) {
SVnodesMgmt *pMgmt = &pDnode->vmgmt; SVnodesMgmt *pMgmt = &pDnode->vmgmt;
taosWLockLatch(&pMgmt->latch); taosWLockLatch(&pMgmt->latch);
taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t));
@ -195,6 +208,9 @@ static void dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode) {
dndFreeVnodeWriteQueue(pDnode, pVnode); dndFreeVnodeWriteQueue(pDnode, pVnode);
dndFreeVnodeApplyQueue(pDnode, pVnode); dndFreeVnodeApplyQueue(pDnode, pVnode);
dndFreeVnodeSyncQueue(pDnode, pVnode); dndFreeVnodeSyncQueue(pDnode, pVnode);
free(pVnode->path);
free(pVnode->db);
free(pVnode);
} }
static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) { static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) {
@ -208,16 +224,16 @@ static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) {
void *pIter = taosHashIterate(pMgmt->hash, NULL); void *pIter = taosHashIterate(pMgmt->hash, NULL);
while (pIter) { while (pIter) {
SVnodeObj **ppVnode = pIter; SVnodeObj **ppVnode = pIter;
SVnodeObj * pVnode = *ppVnode; SVnodeObj *pVnode = *ppVnode;
if (pVnode) { if (pVnode && num < size) {
int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount);
pVnodes[num] = (*ppVnode);
num++; num++;
if (num < size) { pIter = taosHashIterate(pMgmt->hash, pIter);
int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); } else {
dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); taosHashCancelIterate(pMgmt->hash, pIter);
pVnodes[num] = (*ppVnode);
}
} }
pIter = taosHashIterate(pMgmt->hash, pIter);
} }
taosRUnLockLatch(&pMgmt->latch); taosRUnLockLatch(&pMgmt->latch);
@ -226,15 +242,15 @@ static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) {
return pVnodes; return pVnodes;
} }
static int32_t dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_t *numOfVnodes) { static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) {
int32_t code = TSDB_CODE_DND_VNODE_READ_FILE_ERROR; int32_t code = TSDB_CODE_DND_VNODE_READ_FILE_ERROR;
int32_t len = 0; int32_t len = 0;
int32_t maxLen = 30000; int32_t maxLen = 30000;
char * content = calloc(1, maxLen + 1); char *content = calloc(1, maxLen + 1);
cJSON * root = NULL; cJSON *root = NULL;
FILE * fp = NULL; FILE *fp = NULL;
char file[PATH_MAX + 20] = {0}; char file[PATH_MAX + 20] = {0};
SVnodeObj *pVnodes = NULL; SWrapperCfg *pCfgs = NULL;
snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes);
@ -270,31 +286,55 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_
goto PRASE_VNODE_OVER; goto PRASE_VNODE_OVER;
} }
pVnodes = calloc(vnodesNum, sizeof(SVnodeObj)); pCfgs = calloc(vnodesNum, sizeof(SWrapperCfg));
if (pVnodes == NULL) { if (pCfgs == NULL) {
dError("failed to read %s since out of memory", file); dError("failed to read %s since out of memory", file);
goto PRASE_VNODE_OVER; goto PRASE_VNODE_OVER;
} }
for (int32_t i = 0; i < vnodesNum; ++i) { for (int32_t i = 0; i < vnodesNum; ++i) {
cJSON * vnode = cJSON_GetArrayItem(vnodes, i); cJSON *vnode = cJSON_GetArrayItem(vnodes, i);
SVnodeObj *pVnode = &pVnodes[i]; SWrapperCfg *pCfg = &pCfgs[i];
cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId"); cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId");
if (!vgId || vgId->type != cJSON_String) { if (!vgId || vgId->type != cJSON_Number) {
dError("failed to read %s since vgId not found", file); dError("failed to read %s since vgId not found", file);
goto PRASE_VNODE_OVER; goto PRASE_VNODE_OVER;
} }
pVnode->vgId = atoi(vgId->valuestring); pCfg->vgId = vgId->valueint;
snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCfg->vgId);
cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped"); cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped");
if (!dropped || dropped->type != cJSON_String) { if (!dropped || dropped->type != cJSON_Number) {
dError("failed to read %s since dropped not found", file); dError("failed to read %s since dropped not found", file);
goto PRASE_VNODE_OVER; goto PRASE_VNODE_OVER;
} }
pVnode->dropped = atoi(vnode->valuestring); pCfg->dropped = dropped->valueint;
cJSON *vgVersion = cJSON_GetObjectItem(vnode, "vgVersion");
if (!vgVersion || vgVersion->type != cJSON_Number) {
dError("failed to read %s since vgVersion not found", file);
goto PRASE_VNODE_OVER;
}
pCfg->vgVersion = vgVersion->valueint;
cJSON *dbUid = cJSON_GetObjectItem(vnode, "dbUid");
if (!dbUid || dbUid->type != cJSON_String) {
dError("failed to read %s since dbUid not found", file);
goto PRASE_VNODE_OVER;
}
pCfg->dbUid = atoll(dbUid->valuestring);
cJSON *db = cJSON_GetObjectItem(vnode, "db");
if (!db || db->type != cJSON_String) {
dError("failed to read %s since db not found", file);
goto PRASE_VNODE_OVER;
}
tstrncpy(pCfg->db, db->valuestring, TSDB_FULL_DB_NAME_LEN);
} }
*ppCfgs = pCfgs;
*numOfVnodes = vnodesNum;
code = 0; code = 0;
dInfo("succcessed to read file %s", file); dInfo("succcessed to read file %s", file);
@ -313,30 +353,35 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) {
snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes);
FILE *fp = fopen(file, "w"); FILE *fp = fopen(file, "w");
if (fp != NULL) { if (fp == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to write %s since %s", file, terrstr()); dError("failed to write %s since %s", file, terrstr());
return -1; return -1;
} }
int32_t len = 0;
int32_t maxLen = 30000;
char * content = calloc(1, maxLen + 1);
int32_t numOfVnodes = 0; int32_t numOfVnodes = 0;
SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes); SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes);
int32_t len = 0;
int32_t maxLen = 65536;
char *content = calloc(1, maxLen + 1);
len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, "{\n");
len += snprintf(content + len, maxLen - len, " \"vnodes\": [{\n"); len += snprintf(content + len, maxLen - len, " \"vnodes\": [\n");
for (int32_t i = 0; i < numOfVnodes; ++i) { for (int32_t i = 0; i < numOfVnodes; ++i) {
SVnodeObj *pVnode = pVnodes[i]; SVnodeObj *pVnode = pVnodes[i];
len += snprintf(content + len, maxLen - len, " \"vgId\": \"%d\",\n", pVnode->vgId); len += snprintf(content + len, maxLen - len, " {\n");
len += snprintf(content + len, maxLen - len, " \"dropped\": \"%d\"\n", pVnode->dropped); len += snprintf(content + len, maxLen - len, " \"vgId\": %d,\n", pVnode->vgId);
len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pVnode->dropped);
len += snprintf(content + len, maxLen - len, " \"vgVersion\": %d,\n", pVnode->vgVersion);
len += snprintf(content + len, maxLen - len, " \"dbUid\": \"%" PRIu64 "\",\n", pVnode->dbUid);
len += snprintf(content + len, maxLen - len, " \"db\": \"%s\"\n", pVnode->db);
if (i < numOfVnodes - 1) { if (i < numOfVnodes - 1) {
len += snprintf(content + len, maxLen - len, " },{\n"); len += snprintf(content + len, maxLen - len, " },\n");
} else { } else {
len += snprintf(content + len, maxLen - len, " }]\n"); len += snprintf(content + len, maxLen - len, " }\n");
} }
} }
len += snprintf(content + len, maxLen - len, " ]\n");
len += snprintf(content + len, maxLen - len, "}\n"); len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp); fwrite(content, 1, len, fp);
@ -358,74 +403,29 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) {
return taosRenameFile(file, realfile); return taosRenameFile(file, realfile);
} }
static int32_t dndCreateVnode(SDnode *pDnode, int32_t vgId, SVnodeCfg *pCfg) {
char path[PATH_MAX + 20] = {0};
snprintf(path, sizeof(path), "%s/vnode%d", pDnode->dir.vnodes, vgId);
// SVnode *pImpl = vnodeCreate(vgId, path, pCfg);
SVnode *pImpl = vnodeOpen(path, NULL);
if (pImpl == NULL) {
return -1;
}
int32_t code = dndCreateVnodeWrapper(pDnode, vgId, path, pImpl);
if (code != 0) {
vnodeClose(pImpl);
vnodeDestroy(path);
terrno = code;
return code;
}
code = dndWriteVnodesToFile(pDnode);
if (code != 0) {
vnodeClose(pImpl);
vnodeDestroy(path);
terrno = code;
return code;
}
return 0;
}
static int32_t dndDropVnode(SDnode *pDnode, SVnodeObj *pVnode) {
pVnode->dropped = 1;
if (dndWriteVnodesToFile(pDnode) != 0) {
pVnode->dropped = 0;
return -1;
}
dndDropVnodeWrapper(pDnode, pVnode);
vnodeClose(pVnode->pImpl);
vnodeDestroy(pVnode->path);
dndWriteVnodesToFile(pDnode);
return 0;
}
static void *dnodeOpenVnodeFunc(void *param) { static void *dnodeOpenVnodeFunc(void *param) {
SVnodeThread *pThread = param; SVnodeThread *pThread = param;
SDnode * pDnode = pThread->pDnode; SDnode *pDnode = pThread->pDnode;
SVnodesMgmt * pMgmt = &pDnode->vmgmt; SVnodesMgmt *pMgmt = &pDnode->vmgmt;
dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
setThreadName("open-vnodes"); setThreadName("open-vnodes");
for (int32_t v = 0; v < pThread->vnodeNum; ++v) { for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
SVnodeObj *pVnode = &pThread->pVnodes[v]; SWrapperCfg *pCfg = &pThread->pCfgs[v];
char stepDesc[TSDB_STEP_DESC_LEN] = {0}; char stepDesc[TSDB_STEP_DESC_LEN] = {0};
snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pVnode->vgId, snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pCfg->vgId,
pMgmt->openVnodes, pMgmt->totalVnodes); pMgmt->openVnodes, pMgmt->totalVnodes);
dndReportStartup(pDnode, "open-vnodes", stepDesc); dndReportStartup(pDnode, "open-vnodes", stepDesc);
char path[PATH_MAX + 20] = {0}; SVnode *pImpl = vnodeOpen(pCfg->path, NULL);
snprintf(path, sizeof(path), "%s/vnode%d", pDnode->dir.vnodes, pVnode->vgId);
SVnode *pImpl = vnodeOpen(path, NULL);
if (pImpl == NULL) { if (pImpl == NULL) {
dError("vgId:%d, failed to open vnode by thread:%d", pVnode->vgId, pThread->threadIndex); dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex);
pThread->failed++; pThread->failed++;
} else { } else {
dndCreateVnodeWrapper(pDnode, pVnode->vgId, path, pImpl); dndOpenVnode(pDnode, pCfg, pImpl);
dDebug("vgId:%d, is opened by thread:%d", pVnode->vgId, pThread->threadIndex); dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex);
pThread->opened++; pThread->opened++;
} }
@ -448,9 +448,9 @@ static int32_t dndOpenVnodes(SDnode *pDnode) {
return -1; return -1;
} }
SVnodeObj *pVnodes = NULL; SWrapperCfg *pCfgs = NULL;
int32_t numOfVnodes = 0; int32_t numOfVnodes = 0;
if (dndGetVnodesFromFile(pDnode, &pVnodes, &numOfVnodes) != 0) { if (dndGetVnodesFromFile(pDnode, &pCfgs, &numOfVnodes) != 0) {
dInfo("failed to get vnode list from disk since %s", terrstr()); dInfo("failed to get vnode list from disk since %s", terrstr());
return -1; return -1;
} }
@ -463,13 +463,14 @@ static int32_t dndOpenVnodes(SDnode *pDnode) {
SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread)); SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread));
for (int32_t t = 0; t < threadNum; ++t) { for (int32_t t = 0; t < threadNum; ++t) {
threads[t].threadIndex = t; threads[t].threadIndex = t;
threads[t].pVnodes = calloc(vnodesPerThread, sizeof(SVnodeObj)); threads[t].pDnode = pDnode;
threads[t].pCfgs = calloc(vnodesPerThread, sizeof(SWrapperCfg));
} }
for (int32_t v = 0; v < numOfVnodes; ++v) { for (int32_t v = 0; v < numOfVnodes; ++v) {
int32_t t = v % threadNum; int32_t t = v % threadNum;
SVnodeThread *pThread = &threads[t]; SVnodeThread *pThread = &threads[t];
pThread->pVnodes[pThread->vnodeNum++] = pVnodes[v]; pThread->pCfgs[pThread->vnodeNum++] = pCfgs[v];
} }
dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes); dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes);
@ -478,19 +479,25 @@ static int32_t dndOpenVnodes(SDnode *pDnode) {
SVnodeThread *pThread = &threads[t]; SVnodeThread *pThread = &threads[t];
if (pThread->vnodeNum == 0) continue; if (pThread->vnodeNum == 0) continue;
pThread->pThreadId = taosCreateThread(dnodeOpenVnodeFunc, pThread); pthread_attr_t thAttr;
if (pThread->pThreadId == NULL) { pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnodeFunc, pThread) != 0) {
dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno)); dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno));
} }
pthread_attr_destroy(&thAttr);
} }
for (int32_t t = 0; t < threadNum; ++t) { for (int32_t t = 0; t < threadNum; ++t) {
SVnodeThread *pThread = &threads[t]; SVnodeThread *pThread = &threads[t];
taosDestoryThread(pThread->pThreadId); if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) {
pThread->pThreadId = NULL; pthread_join(pThread->thread, NULL);
free(pThread->pVnodes); }
free(pThread->pCfgs);
} }
free(threads); free(threads);
free(pCfgs);
if (pMgmt->openVnodes != pMgmt->totalVnodes) { if (pMgmt->openVnodes != pMgmt->totalVnodes) {
dError("there are total vnodes:%d, opened:%d", pMgmt->totalVnodes, pMgmt->openVnodes); dError("there are total vnodes:%d, opened:%d", pMgmt->totalVnodes, pMgmt->openVnodes);
@ -508,7 +515,8 @@ static void dndCloseVnodes(SDnode *pDnode) {
SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes); SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes);
for (int32_t i = 0; i < numOfVnodes; ++i) { for (int32_t i = 0; i < numOfVnodes; ++i) {
dndDropVnodeWrapper(pDnode, pVnodes[i]); dndReleaseVnode(pDnode, pVnodes[i]);
dndCloseVnode(pDnode, pVnodes[i]);
} }
if (pVnodes != NULL) { if (pVnodes != NULL) {
@ -523,11 +531,12 @@ static void dndCloseVnodes(SDnode *pDnode) {
dInfo("total vnodes:%d are all closed", numOfVnodes); dInfo("total vnodes:%d are all closed", numOfVnodes);
} }
static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg *pCfg) { static SCreateVnodeMsg *dndParseCreateVnodeReq(SRpcMsg *rpcMsg) {
SCreateVnodeMsg *pCreate = rpcMsg->pCont; SCreateVnodeMsg *pCreate = rpcMsg->pCont;
pCreate->vgId = htonl(pCreate->vgId); pCreate->vgId = htonl(pCreate->vgId);
pCreate->dnodeId = htonl(pCreate->dnodeId); pCreate->dnodeId = htonl(pCreate->dnodeId);
pCreate->dbUid = htobe64(pCreate->dbUid); pCreate->dbUid = htobe64(pCreate->dbUid);
pCreate->vgVersion = htonl(pCreate->vgVersion);
pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize);
pCreate->totalBlocks = htonl(pCreate->totalBlocks); pCreate->totalBlocks = htonl(pCreate->totalBlocks);
pCreate->daysPerFile = htonl(pCreate->daysPerFile); pCreate->daysPerFile = htonl(pCreate->daysPerFile);
@ -544,9 +553,10 @@ static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg
pReplica->port = htons(pReplica->port); pReplica->port = htons(pReplica->port);
} }
*vgId = pCreate->vgId; return pCreate;
}
#if 0 static void dndGenerateVnodeCfg(SCreateVnodeMsg *pCreate, SVnodeCfg *pCfg) {
pCfg->wsize = pCreate->cacheBlockSize; pCfg->wsize = pCreate->cacheBlockSize;
pCfg->ssize = pCreate->cacheBlockSize; pCfg->ssize = pCreate->cacheBlockSize;
pCfg->wsize = pCreate->cacheBlockSize; pCfg->wsize = pCreate->cacheBlockSize;
@ -567,8 +577,15 @@ static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg
pCfg->walCfg.rollPeriod = 128; pCfg->walCfg.rollPeriod = 128;
pCfg->walCfg.segSize = 128; pCfg->walCfg.segSize = 128;
pCfg->walCfg.vgId = pCreate->vgId; pCfg->walCfg.vgId = pCreate->vgId;
#endif }
return 0;
static void dndGenerateWrapperCfg(SDnode *pDnode, SCreateVnodeMsg *pCreate, SWrapperCfg *pCfg) {
memcpy(pCfg->db, pCreate->db, TSDB_FULL_DB_NAME_LEN);
pCfg->dbUid = pCreate->dbUid;
pCfg->dropped = 0;
snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCreate->vgId);
pCfg->vgId = pCreate->vgId;
pCfg->vgVersion = pCreate->vgVersion;
} }
static SDropVnodeMsg *vnodeParseDropVnodeReq(SRpcMsg *rpcMsg) { static SDropVnodeMsg *vnodeParseDropVnodeReq(SRpcMsg *rpcMsg) {
@ -584,48 +601,83 @@ static SAuthVnodeMsg *vnodeParseAuthVnodeReq(SRpcMsg *rpcMsg) {
} }
static int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { static int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) {
SCreateVnodeMsg *pCreate = dndParseCreateVnodeReq(rpcMsg);
dDebug("vgId:%d, create vnode req is received", pCreate->vgId);
SVnodeCfg vnodeCfg = {0}; SVnodeCfg vnodeCfg = {0};
int32_t vgId = 0; dndGenerateVnodeCfg(pCreate, &vnodeCfg);
dndParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg); SWrapperCfg wrapperCfg = {0};
dDebug("vgId:%d, create vnode req is received", vgId); dndGenerateWrapperCfg(pDnode, pCreate, &wrapperCfg);
SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); SVnodeObj *pVnode = dndAcquireVnode(pDnode, pCreate->vgId);
if (pVnode != NULL) { if (pVnode != NULL) {
dDebug("vgId:%d, already exist, return success", vgId); dDebug("vgId:%d, already exist, return success", pCreate->vgId);
dndReleaseVnode(pDnode, pVnode); dndReleaseVnode(pDnode, pVnode);
return 0; return 0;
} }
if (dndCreateVnode(pDnode, vgId, &vnodeCfg) != 0) { SVnode *pImpl = vnodeOpen(wrapperCfg.path, NULL /*pCfg*/);
dError("vgId:%d, failed to create vnode since %s", vgId, terrstr()); if (pImpl == NULL) {
return terrno; return -1;
}
int32_t code = dndOpenVnode(pDnode, &wrapperCfg, pImpl);
if (code != 0) {
vnodeClose(pImpl);
vnodeDestroy(wrapperCfg.path);
terrno = code;
return code;
}
code = dndWriteVnodesToFile(pDnode);
if (code != 0) {
vnodeClose(pImpl);
vnodeDestroy(wrapperCfg.path);
terrno = code;
return code;
} }
return 0; return 0;
} }
static int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { static int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) {
SAlterVnodeMsg *pAlter = (SAlterVnodeMsg *)dndParseCreateVnodeReq(rpcMsg);
dDebug("vgId:%d, alter vnode req is received", pAlter->vgId);
SVnodeCfg vnodeCfg = {0}; SVnodeCfg vnodeCfg = {0};
int32_t vgId = 0; dndGenerateVnodeCfg(pAlter, &vnodeCfg);
dndParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg); SWrapperCfg wrapperCfg = {0};
dDebug("vgId:%d, alter vnode req is received", vgId); dndGenerateWrapperCfg(pDnode, pAlter, &wrapperCfg);
SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); SVnodeObj *pVnode = dndAcquireVnode(pDnode, pAlter->vgId);
if (pVnode == NULL) { if (pVnode == NULL) {
dDebug("vgId:%d, failed to alter vnode since %s", vgId, terrstr()); dDebug("vgId:%d, failed to alter vnode since %s", pAlter->vgId, terrstr());
return terrno; return terrno;
} }
if (wrapperCfg.vgVersion == pVnode->vgVersion) {
dndReleaseVnode(pDnode, pVnode);
dDebug("vgId:%d, no need to alter vnode cfg for version unchanged ", pAlter->vgId);
return 0;
}
if (vnodeAlter(pVnode->pImpl, &vnodeCfg) != 0) { if (vnodeAlter(pVnode->pImpl, &vnodeCfg) != 0) {
dError("vgId:%d, failed to alter vnode since %s", vgId, terrstr()); dError("vgId:%d, failed to alter vnode since %s", pAlter->vgId, terrstr());
dndReleaseVnode(pDnode, pVnode); dndReleaseVnode(pDnode, pVnode);
return terrno; return terrno;
} }
int32_t oldVersion = pVnode->vgVersion;
pVnode->vgVersion = wrapperCfg.vgVersion;
int32_t code = dndWriteVnodesToFile(pDnode);
if (code != 0) {
pVnode->vgVersion = oldVersion;
}
dndReleaseVnode(pDnode, pVnode); dndReleaseVnode(pDnode, pVnode);
return 0; return code;
} }
static int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { static int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) {
@ -637,14 +689,20 @@ static int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) {
SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId);
if (pVnode == NULL) { if (pVnode == NULL) {
dDebug("vgId:%d, failed to drop since %s", vgId, terrstr()); dDebug("vgId:%d, failed to drop since %s", vgId, terrstr());
return 0;
}
pVnode->dropped = 1;
if (dndWriteVnodesToFile(pDnode) != 0) {
pVnode->dropped = 0;
return terrno; return terrno;
} }
if (dndDropVnode(pDnode, pVnode) != 0) { dndReleaseVnode(pDnode, pVnode);
dError("vgId:%d, failed to drop vnode since %s", vgId, terrstr()); dndCloseVnode(pDnode, pVnode);
dndReleaseVnode(pDnode, pVnode); vnodeClose(pVnode->pImpl);
return terrno; vnodeDestroy(pVnode->path);
} dndWriteVnodesToFile(pDnode);
return 0; return 0;
} }
@ -738,12 +796,10 @@ static void dndProcessVnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) {
break; break;
} }
if (code != 0) { SRpcMsg rsp = {.code = code, .handle = pMsg->handle, .ahandle = pMsg->ahandle};
SRpcMsg rsp = {.code = code, .handle = pMsg->handle}; rpcSendResponse(&rsp);
rpcSendResponse(&rsp); rpcFreeCont(pMsg->pCont);
rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg);
taosFreeQitem(pMsg);
}
} }
static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) { static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) {
@ -756,7 +812,7 @@ static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) {
static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) { static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) {
SVnodeMsg *pMsg = vnodeInitMsg(numOfMsgs); SVnodeMsg *pMsg = vnodeInitMsg(numOfMsgs);
SRpcMsg * pRpcMsg = NULL; SRpcMsg *pRpcMsg = NULL;
for (int32_t i = 0; i < numOfMsgs; ++i) { for (int32_t i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, (void **)&pRpcMsg); taosGetQitem(qall, (void **)&pRpcMsg);
@ -1029,7 +1085,7 @@ static void dndFreeVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode) {
} }
static int32_t dndInitVnodeWriteWorker(SDnode *pDnode) { static int32_t dndInitVnodeWriteWorker(SDnode *pDnode) {
SVnodesMgmt * pMgmt = &pDnode->vmgmt; SVnodesMgmt *pMgmt = &pDnode->vmgmt;
SMWorkerPool *pPool = &pMgmt->writePool; SMWorkerPool *pPool = &pMgmt->writePool;
pPool->name = "vnode-write"; pPool->name = "vnode-write";
pPool->max = pDnode->opt.numOfCores; pPool->max = pDnode->opt.numOfCores;
@ -1137,12 +1193,12 @@ void dndGetVnodeLoads(SDnode *pDnode, SVnodeLoads *pLoads) {
pLoads->num = taosHashGetSize(pMgmt->hash); pLoads->num = taosHashGetSize(pMgmt->hash);
int32_t v = 0; int32_t v = 0;
void * pIter = taosHashIterate(pMgmt->hash, NULL); void *pIter = taosHashIterate(pMgmt->hash, NULL);
while (pIter) { while (pIter) {
SVnodeObj **ppVnode = pIter; SVnodeObj **ppVnode = pIter;
if (ppVnode == NULL || *ppVnode == NULL) continue; if (ppVnode == NULL || *ppVnode == NULL) continue;
SVnodeObj * pVnode = *ppVnode; SVnodeObj *pVnode = *ppVnode;
SVnodeLoad *pLoad = &pLoads->data[v++]; SVnodeLoad *pLoad = &pLoads->data[v++];
vnodeGetLoad(pVnode->pImpl, pLoad); vnodeGetLoad(pVnode->pImpl, pLoad);

8
source/dnode/mgmt/impl/src/dnode.c
View File

@ -176,6 +176,12 @@ SDnode *dndInit(SDnodeOpt *pOption) {
return NULL; return NULL;
} }
if (vnodeInit(1) != 0) {
dError("failed to init vnode env");
dndCleanup(pDnode);
return NULL;
}
if (dndInitDnode(pDnode) != 0) { if (dndInitDnode(pDnode) != 0) {
dError("failed to init dnode"); dError("failed to init dnode");
dndCleanup(pDnode); dndCleanup(pDnode);
@ -222,8 +228,10 @@ void dndCleanup(SDnode *pDnode) {
dndCleanupMnode(pDnode); dndCleanupMnode(pDnode);
dndCleanupVnodes(pDnode); dndCleanupVnodes(pDnode);
dndCleanupDnode(pDnode); dndCleanupDnode(pDnode);
vnodeClear();
walCleanUp(); walCleanUp();
rpcCleanup(); rpcCleanup();
dndCleanupEnv(pDnode); dndCleanupEnv(pDnode);
free(pDnode); free(pDnode);
dInfo("TDengine is cleaned up successfully"); dInfo("TDengine is cleaned up successfully");

8
source/dnode/mgmt/impl/test/db/db.cpp
View File

@ -210,8 +210,8 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) {
pReq->daysToKeep0 = htonl(3650); pReq->daysToKeep0 = htonl(3650);
pReq->daysToKeep1 = htonl(3650); pReq->daysToKeep1 = htonl(3650);
pReq->daysToKeep2 = htonl(3650); pReq->daysToKeep2 = htonl(3650);
pReq->minRowsPerFileBlock = htonl(100); pReq->minRows = htonl(100);
pReq->maxRowsPerFileBlock = htonl(4096); pReq->maxRows = htonl(4096);
pReq->commitTime = htonl(3600); pReq->commitTime = htonl(3600);
pReq->fsyncPeriod = htonl(3000); pReq->fsyncPeriod = htonl(3000);
pReq->walLevel = 1; pReq->walLevel = 1;
@ -375,8 +375,8 @@ TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) {
pReq->daysToKeep0 = htonl(3650); pReq->daysToKeep0 = htonl(3650);
pReq->daysToKeep1 = htonl(3650); pReq->daysToKeep1 = htonl(3650);
pReq->daysToKeep2 = htonl(3650); pReq->daysToKeep2 = htonl(3650);
pReq->minRowsPerFileBlock = htonl(100); pReq->minRows = htonl(100);
pReq->maxRowsPerFileBlock = htonl(4096); pReq->maxRows = htonl(4096);
pReq->commitTime = htonl(3600); pReq->commitTime = htonl(3600);
pReq->fsyncPeriod = htonl(3000); pReq->fsyncPeriod = htonl(3000);
pReq->walLevel = 1; pReq->walLevel = 1;

4
source/dnode/mgmt/impl/test/stb/stb.cpp
View File

@ -187,8 +187,8 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
pReq->daysToKeep0 = htonl(3650); pReq->daysToKeep0 = htonl(3650);
pReq->daysToKeep1 = htonl(3650); pReq->daysToKeep1 = htonl(3650);
pReq->daysToKeep2 = htonl(3650); pReq->daysToKeep2 = htonl(3650);
pReq->minRowsPerFileBlock = htonl(100); pReq->minRows = htonl(100);
pReq->maxRowsPerFileBlock = htonl(4096); pReq->maxRows = htonl(4096);
pReq->commitTime = htonl(3600); pReq->commitTime = htonl(3600);
pReq->fsyncPeriod = htonl(3000); pReq->fsyncPeriod = htonl(3000);
pReq->walLevel = 1; pReq->walLevel = 1;

2
source/dnode/mgmt/impl/test/sut/deploy.cpp
View File

@ -16,7 +16,7 @@
#include "deploy.h" #include "deploy.h"
void initLog(const char* path) { void initLog(const char* path) {
dDebugFlag = 143; dDebugFlag = 207;
vDebugFlag = 0; vDebugFlag = 0;
mDebugFlag = 207; mDebugFlag = 207;
cDebugFlag = 0; cDebugFlag = 0;

143
source/dnode/mgmt/impl/test/vgroup/vgroup.cpp
View File

@ -176,49 +176,112 @@ SServer* DndTestVgroup::pServer;
SClient* DndTestVgroup::pClient; SClient* DndTestVgroup::pClient;
int32_t DndTestVgroup::connId; int32_t DndTestVgroup::connId;
TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) {
{ {
SCreateVnodeMsg* pReq = (SCreateVnodeMsg*)rpcMallocCont(sizeof(SCreateVnodeMsg)); for (int i = 0; i < 3; ++i) {
pReq->vgId = htonl(2); SCreateVnodeMsg* pReq = (SCreateVnodeMsg*)rpcMallocCont(sizeof(SCreateVnodeMsg));
pReq->dnodeId = htonl(1); pReq->vgId = htonl(2);
strcpy(pReq->db, "1.d1"); pReq->dnodeId = htonl(1);
pReq->dbUid = htobe64(9527); strcpy(pReq->db, "1.d1");
pReq->cacheBlockSize = htonl(16); pReq->dbUid = htobe64(9527);
pReq->totalBlocks = htonl(10); pReq->vgVersion = htonl(1);
pReq->daysPerFile = htonl(10); pReq->cacheBlockSize = htonl(16);
pReq->daysToKeep0 = htonl(3650); pReq->totalBlocks = htonl(10);
pReq->daysToKeep1 = htonl(3650); pReq->daysPerFile = htonl(10);
pReq->daysToKeep2 = htonl(3650); pReq->daysToKeep0 = htonl(3650);
pReq->minRows = htonl(100); pReq->daysToKeep1 = htonl(3650);
pReq->minRows = htonl(4096); pReq->daysToKeep2 = htonl(3650);
pReq->commitTime = htonl(3600); pReq->minRows = htonl(100);
pReq->fsyncPeriod = htonl(3000); pReq->minRows = htonl(4096);
pReq->walLevel = 1; pReq->commitTime = htonl(3600);
pReq->precision = 0; pReq->fsyncPeriod = htonl(3000);
pReq->compression = 2; pReq->walLevel = 1;
pReq->replica = 1; pReq->precision = 0;
pReq->quorum = 1; pReq->compression = 2;
pReq->update = 0; pReq->replica = 1;
pReq->cacheLastRow = 0; pReq->quorum = 1;
pReq->selfIndex = 0; pReq->update = 0;
for (int r = 0; r < pReq->replica; ++r) { pReq->cacheLastRow = 0;
SReplica* pReplica = &pReq->replicas[r]; pReq->selfIndex = 0;
pReplica->id = htonl(1); for (int r = 0; r < pReq->replica; ++r) {
pReplica->port = htons(9150); SReplica* pReplica = &pReq->replicas[r];
pReplica->id = htonl(1);
pReplica->port = htons(9150);
}
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SCreateVnodeMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_VNODE_IN;
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
} }
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SCreateVnodeMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_VNODE_IN;
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
taosMsleep(1000000);
} }
} {
for (int i = 0; i < 3; ++i) {
SAlterVnodeMsg* pReq = (SAlterVnodeMsg*)rpcMallocCont(sizeof(SAlterVnodeMsg));
pReq->vgId = htonl(2);
pReq->dnodeId = htonl(1);
strcpy(pReq->db, "1.d1");
pReq->dbUid = htobe64(9527);
pReq->vgVersion = htonl(2);
pReq->cacheBlockSize = htonl(16);
pReq->totalBlocks = htonl(10);
pReq->daysPerFile = htonl(10);
pReq->daysToKeep0 = htonl(3650);
pReq->daysToKeep1 = htonl(3650);
pReq->daysToKeep2 = htonl(3650);
pReq->minRows = htonl(100);
pReq->minRows = htonl(4096);
pReq->commitTime = htonl(3600);
pReq->fsyncPeriod = htonl(3000);
pReq->walLevel = 1;
pReq->precision = 0;
pReq->compression = 2;
pReq->replica = 1;
pReq->quorum = 1;
pReq->update = 0;
pReq->cacheLastRow = 0;
pReq->selfIndex = 0;
for (int r = 0; r < pReq->replica; ++r) {
SReplica* pReplica = &pReq->replicas[r];
pReplica->id = htonl(1);
pReplica->port = htons(9150);
}
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SAlterVnodeMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_VNODE_IN;
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
}
}
{
for (int i = 0; i < 3; ++i) {
SDropVnodeMsg* pReq = (SDropVnodeMsg*)rpcMallocCont(sizeof(SDropVnodeMsg));
pReq->vgId = htonl(2);
pReq->dnodeId = htonl(1);
strcpy(pReq->db, "1.d1");
pReq->dbUid = htobe64(9527);
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SDropVnodeMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_DROP_VNODE_IN;
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
}
}
}

11
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -61,15 +61,14 @@ typedef enum {
} EAuthOp; } EAuthOp;
typedef enum { typedef enum {
TRN_STAGE_PREPARE = 1, TRN_STAGE_PREPARE = 0,
TRN_STAGE_EXECUTE = 2, TRN_STAGE_EXECUTE = 1,
TRN_STAGE_ROLLBACK = 2,
TRN_STAGE_COMMIT = 3, TRN_STAGE_COMMIT = 3,
TRN_STAGE_ROLLBACK = 4, TRN_STAGE_OVER = 4,
TRN_STAGE_RETRY = 5,
TRN_STAGE_OVER = 6,
} ETrnStage; } ETrnStage;
typedef enum { TRN_POLICY_ROLLBACK = 1, TRN_POLICY_RETRY = 2 } ETrnPolicy; typedef enum { TRN_POLICY_ROLLBACK = 0, TRN_POLICY_RETRY = 1 } ETrnPolicy;
typedef enum { typedef enum {
DND_STATUS_OFFLINE = 0, DND_STATUS_OFFLINE = 0,

1
source/dnode/mnode/impl/inc/mndInt.h
View File

@ -70,6 +70,7 @@ typedef struct SMnode {
tmr_h timer; tmr_h timer;
char *path; char *path;
SMnodeCfg cfg; SMnodeCfg cfg;
int64_t checkTime;
SSdb *pSdb; SSdb *pSdb;
SDnode *pDnode; SDnode *pDnode;
SArray *pSteps; SArray *pSteps;

21
source/dnode/mnode/impl/inc/mndTrans.h
View File

@ -22,6 +22,16 @@
extern "C" { extern "C" {
#endif #endif
typedef struct {
SEpSet epSet;
int8_t msgType;
int8_t msgSent;
int8_t msgReceived;
int32_t errCode;
int32_t contLen;
void *pCont;
} STransAction;
int32_t mndInitTrans(SMnode *pMnode); int32_t mndInitTrans(SMnode *pMnode);
void mndCleanupTrans(SMnode *pMnode); void mndCleanupTrans(SMnode *pMnode);
@ -30,12 +40,15 @@ void mndTransDrop(STrans *pTrans);
int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw);
int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw);
int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw);
int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont); int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction);
int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont); int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction);
int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans);
void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code); void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code);
char *mndTransStageStr(ETrnStage stage); void mndTransHandleActionRsp(SMnodeMsg *pMsg);
char *mndTransPolicyStr(ETrnPolicy policy);
char *mndTransStageStr(ETrnStage stage);
char *mndTransPolicyStr(ETrnPolicy policy);
#ifdef __cplusplus #ifdef __cplusplus
} }

15
source/dnode/mnode/impl/src/mndAcct.c
View File

@ -17,7 +17,8 @@
#include "mndAcct.h" #include "mndAcct.h"
#include "mndShow.h" #include "mndShow.h"
#define SDB_ACCT_VER 1 #define TSDB_ACCT_VER_NUMBER 1
#define TSDB_ACCT_RESERVE_SIZE 64
static int32_t mndCreateDefaultAcct(SMnode *pMnode); static int32_t mndCreateDefaultAcct(SMnode *pMnode);
static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct); static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct);
@ -70,7 +71,7 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) {
} }
static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) { static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, SDB_ACCT_VER, sizeof(SAcctObj)); SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, TSDB_ACCT_VER_NUMBER, sizeof(SAcctObj));
if (pRaw == NULL) return NULL; if (pRaw == NULL) return NULL;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -85,6 +86,7 @@ static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) {
SDB_SET_INT32(pRaw, dataPos, pAcct->cfg.maxStreams) SDB_SET_INT32(pRaw, dataPos, pAcct->cfg.maxStreams)
SDB_SET_INT64(pRaw, dataPos, pAcct->cfg.maxStorage) SDB_SET_INT64(pRaw, dataPos, pAcct->cfg.maxStorage)
SDB_SET_INT32(pRaw, dataPos, pAcct->cfg.accessState) SDB_SET_INT32(pRaw, dataPos, pAcct->cfg.accessState)
SDB_SET_RESERVE(pRaw, dataPos, TSDB_ACCT_RESERVE_SIZE)
SDB_SET_DATALEN(pRaw, dataPos); SDB_SET_DATALEN(pRaw, dataPos);
return pRaw; return pRaw;
@ -94,7 +96,7 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
if (sver != SDB_ACCT_VER) { if (sver != TSDB_ACCT_VER_NUMBER) {
mError("failed to decode acct since %s", terrstr()); mError("failed to decode acct since %s", terrstr());
terrno = TSDB_CODE_SDB_INVALID_DATA_VER; terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
return NULL; return NULL;
@ -116,13 +118,13 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, pRow, dataPos, &pAcct->cfg.maxStreams) SDB_GET_INT32(pRaw, pRow, dataPos, &pAcct->cfg.maxStreams)
SDB_GET_INT64(pRaw, pRow, dataPos, &pAcct->cfg.maxStorage) SDB_GET_INT64(pRaw, pRow, dataPos, &pAcct->cfg.maxStorage)
SDB_GET_INT32(pRaw, pRow, dataPos, &pAcct->cfg.accessState) SDB_GET_INT32(pRaw, pRow, dataPos, &pAcct->cfg.accessState)
SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_ACCT_RESERVE_SIZE)
return pRow; return pRow;
} }
static int32_t mndAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct) { static int32_t mndAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct) {
mTrace("acct:%s, perform insert action", pAcct->acct); mTrace("acct:%s, perform insert action", pAcct->acct);
memset(&pAcct->info, 0, sizeof(SAcctInfo));
return 0; return 0;
} }
@ -134,12 +136,9 @@ static int32_t mndAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct) {
static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct) { static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct) {
mTrace("acct:%s, perform update action", pOldAcct->acct); mTrace("acct:%s, perform update action", pOldAcct->acct);
memcpy(pOldAcct->acct, pNewAcct->acct, TSDB_USER_LEN);
pOldAcct->createdTime = pNewAcct->createdTime;
pOldAcct->updateTime = pNewAcct->updateTime; pOldAcct->updateTime = pNewAcct->updateTime;
pOldAcct->acctId = pNewAcct->acctId;
pOldAcct->status = pNewAcct->status; pOldAcct->status = pNewAcct->status;
pOldAcct->cfg = pNewAcct->cfg; memcpy(&pOldAcct->cfg, &pNewAcct->cfg, sizeof(SAcctInfo));
return 0; return 0;
} }

10
source/dnode/mnode/impl/src/mndCluster.c
View File

@ -18,7 +18,8 @@
#include "mndShow.h" #include "mndShow.h"
#include "mndTrans.h" #include "mndTrans.h"
#define SDB_CLUSTER_VER 1 #define TSDB_CLUSTER_VER_NUMBE 1
#define TSDB_CLUSTER_RESERVE_SIZE 64
static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster); static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster);
static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw); static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw);
@ -62,7 +63,7 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) {
} }
static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_CLUSTER, SDB_CLUSTER_VER, sizeof(SClusterObj)); SSdbRaw *pRaw = sdbAllocRaw(SDB_CLUSTER, TSDB_CLUSTER_VER_NUMBE, sizeof(SClusterObj));
if (pRaw == NULL) return NULL; if (pRaw == NULL) return NULL;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -70,6 +71,7 @@ static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) {
SDB_SET_INT64(pRaw, dataPos, pCluster->createdTime) SDB_SET_INT64(pRaw, dataPos, pCluster->createdTime)
SDB_SET_INT64(pRaw, dataPos, pCluster->updateTime) SDB_SET_INT64(pRaw, dataPos, pCluster->updateTime)
SDB_SET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN) SDB_SET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN)
SDB_SET_RESERVE(pRaw, dataPos, TSDB_CLUSTER_RESERVE_SIZE)
return pRaw; return pRaw;
} }
@ -78,7 +80,7 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
if (sver != SDB_CLUSTER_VER) { if (sver != TSDB_CLUSTER_VER_NUMBE) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER; terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
mError("failed to decode cluster since %s", terrstr()); mError("failed to decode cluster since %s", terrstr());
return NULL; return NULL;
@ -93,6 +95,7 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT64(pRaw, pRow, dataPos, &pCluster->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pCluster->createdTime)
SDB_GET_INT64(pRaw, pRow, dataPos, &pCluster->updateTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pCluster->updateTime)
SDB_GET_BINARY(pRaw, pRow, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN) SDB_GET_BINARY(pRaw, pRow, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN)
SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_CLUSTER_RESERVE_SIZE)
return pRow; return pRow;
} }
@ -169,6 +172,7 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg
pShow->numOfRows = 1; pShow->numOfRows = 1;
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
strcpy(pMeta->tbFname, mndShowStr(pShow->type));
return 0; return 0;
} }

321
source/dnode/mnode/impl/src/mndDb.c
View File

@ -16,13 +16,12 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "mndDb.h" #include "mndDb.h"
#include "mndDnode.h" #include "mndDnode.h"
#include "mndMnode.h"
#include "mndShow.h" #include "mndShow.h"
#include "mndTrans.h" #include "mndTrans.h"
#include "mndUser.h" #include "mndUser.h"
#include "mndVgroup.h" #include "mndVgroup.h"
#define TSDB_DB_VER_NUM 1 #define TSDB_DB_VER_NUMBER 1
#define TSDB_DB_RESERVE_SIZE 64 #define TSDB_DB_RESERVE_SIZE 64
static SSdbRaw *mndDbActionEncode(SDbObj *pDb); static SSdbRaw *mndDbActionEncode(SDbObj *pDb);
@ -66,7 +65,7 @@ int32_t mndInitDb(SMnode *pMnode) {
void mndCleanupDb(SMnode *pMnode) {} void mndCleanupDb(SMnode *pMnode) {}
static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUM, sizeof(SDbObj) + TSDB_DB_RESERVE_SIZE); SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUMBER, sizeof(SDbObj) + TSDB_DB_RESERVE_SIZE);
if (pRaw == NULL) return NULL; if (pRaw == NULL) return NULL;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -106,7 +105,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
if (sver != TSDB_DB_VER_NUM) { if (sver != TSDB_DB_VER_NUMBER) {
mError("failed to decode db since %s", terrstr()); mError("failed to decode db since %s", terrstr());
terrno = TSDB_CODE_SDB_INVALID_DATA_VER; terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
return NULL; return NULL;
@ -160,7 +159,7 @@ static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) {
static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) { static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) {
mTrace("db:%s, perform update action", pOldDb->name); mTrace("db:%s, perform update action", pOldDb->name);
pOldDb->updateTime = pNewDb->createdTime; pOldDb->updateTime = pNewDb->updateTime;
pOldDb->cfgVersion = pNewDb->cfgVersion; pOldDb->cfgVersion = pNewDb->cfgVersion;
pOldDb->vgVersion = pNewDb->vgVersion; pOldDb->vgVersion = pNewDb->vgVersion;
memcpy(&pOldDb->cfg, &pNewDb->cfg, sizeof(SDbCfg)); memcpy(&pOldDb->cfg, &pNewDb->cfg, sizeof(SDbCfg));
@ -168,8 +167,12 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) {
} }
SDbObj *mndAcquireDb(SMnode *pMnode, char *db) { SDbObj *mndAcquireDb(SMnode *pMnode, char *db) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
return sdbAcquire(pSdb, SDB_DB, db); SDbObj *pDb = sdbAcquire(pSdb, SDB_DB, db);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST;
}
return pDb;
} }
void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) {
@ -242,68 +245,74 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW;
} }
static int32_t mndSetRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
SSdbRaw *pDbRaw = mndDbActionEncode(pDb); SSdbRaw *pDbRaw = mndDbActionEncode(pDb);
if (pDbRaw == NULL || mndTransAppendRedolog(pTrans, pDbRaw) != 0) return -1; if (pDbRaw == NULL) return -1;
sdbSetRawStatus(pDbRaw, SDB_STATUS_CREATING); if (mndTransAppendRedolog(pTrans, pDbRaw) != 0) return -1;
if (sdbSetRawStatus(pDbRaw, SDB_STATUS_CREATING) != 0) return -1;
for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) {
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v);
if (pVgRaw == NULL || mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1; if (pVgRaw == NULL) return -1;
sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING); if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1;
if (sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING) != 0) return -1;
} }
return 0; return 0;
} }
static int32_t mndSetUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { static int32_t mndSetCreateDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
SSdbRaw *pDbRaw = mndDbActionEncode(pDb); SSdbRaw *pDbRaw = mndDbActionEncode(pDb);
if (pDbRaw == NULL || mndTransAppendUndolog(pTrans, pDbRaw) != 0) return -1; if (pDbRaw == NULL) return -1;
sdbSetRawStatus(pDbRaw, SDB_STATUS_DROPPED); if (mndTransAppendUndolog(pTrans, pDbRaw) != 0) return -1;
if (sdbSetRawStatus(pDbRaw, SDB_STATUS_DROPPED) != 0) return -1;
for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) {
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v);
if (pVgRaw == NULL || mndTransAppendUndolog(pTrans, pVgRaw) != 0) return -1; if (pVgRaw == NULL) return -1;
sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPED); if (mndTransAppendUndolog(pTrans, pVgRaw) != 0) return -1;
if (sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPED) != 0) return -1;
} }
return 0; return 0;
} }
static int32_t mndSetCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { static int32_t mndSetCreateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
SSdbRaw *pDbRaw = mndDbActionEncode(pDb); SSdbRaw *pDbRaw = mndDbActionEncode(pDb);
if (pDbRaw == NULL || mndTransAppendCommitlog(pTrans, pDbRaw) != 0) return -1; if (pDbRaw == NULL) return -1;
sdbSetRawStatus(pDbRaw, SDB_STATUS_READY); if (mndTransAppendCommitlog(pTrans, pDbRaw) != 0) return -1;
if (sdbSetRawStatus(pDbRaw, SDB_STATUS_READY) != 0) return -1;
for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) {
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v);
if (pVgRaw == NULL || mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; if (pVgRaw == NULL) return -1;
sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1;
if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1;
} }
return 0; return 0;
} }
static int32_t mndSetRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { for (int32_t vg = 0; vg < pDb->cfg.numOfVgroups; ++vg) {
SVgObj *pVgroup = pVgroups + v; SVgObj *pVgroup = pVgroups + vg;
for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
SVnodeGid *pVgid = pVgroup->vnodeGid + vn; STransAction action = {0};
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); SVnodeGid *pVgid = pVgroup->vnodeGid + vn;
if (pDnode == NULL) {
return -1;
}
SEpSet epset = mndGetDnodeEpset(pDnode); SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
if (pDnode == NULL) return -1;
action.epSet = mndGetDnodeEpset(pDnode);
mndReleaseDnode(pMnode, pDnode); mndReleaseDnode(pMnode, pDnode);
SCreateVnodeMsg *pMsg = mndBuildCreateVnodeMsg(pMnode, pDnode, pDb, pVgroup); SCreateVnodeMsg *pMsg = mndBuildCreateVnodeMsg(pMnode, pDnode, pDb, pVgroup);
if (pMsg == NULL) { if (pMsg == NULL) return -1;
return -1;
}
if (mndTransAppendRedoAction(pTrans, &epset, TSDB_MSG_TYPE_ALTER_VNODE_IN, sizeof(SCreateVnodeMsg), pMsg) != 0) { action.pCont = pMsg;
action.contLen = sizeof(SCreateVnodeMsg);
action.msgType = TSDB_MSG_TYPE_CREATE_VNODE_IN;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
free(pMsg); free(pMsg);
return -1; return -1;
} }
@ -313,26 +322,26 @@ static int32_t mndSetRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SV
return 0; return 0;
} }
static int32_t mndSetUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { for (int32_t vg = 0; vg < pDb->cfg.numOfVgroups; ++vg) {
SVgObj *pVgroup = pVgroups + v; SVgObj *pVgroup = pVgroups + vg;
for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
SVnodeGid *pVgid = pVgroup->vnodeGid + vn; STransAction action = {0};
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); SVnodeGid *pVgid = pVgroup->vnodeGid + vn;
if (pDnode == NULL) {
return -1;
}
SEpSet epset = mndGetDnodeEpset(pDnode); SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
if (pDnode == NULL) return -1;
action.epSet = mndGetDnodeEpset(pDnode);
mndReleaseDnode(pMnode, pDnode); mndReleaseDnode(pMnode, pDnode);
SDropVnodeMsg *pMsg = mndBuildDropVnodeMsg(pMnode, pDnode, pDb, pVgroup); SDropVnodeMsg *pMsg = mndBuildDropVnodeMsg(pMnode, pDnode, pDb, pVgroup);
if (pMsg == NULL) { if (pMsg == NULL) return -1;
return -1;
}
if (mndTransAppendUndoAction(pTrans, &epset, TSDB_MSG_TYPE_DROP_VNODE_IN, sizeof(SDropVnodeMsg), pMsg) != 0) { action.pCont = pMsg;
action.contLen = sizeof(SDropVnodeMsg);
action.msgType = TSDB_MSG_TYPE_DROP_VNODE_IN;
if (mndTransAppendUndoAction(pTrans, &action) != 0) {
free(pMsg); free(pMsg);
return -1; return -1;
} }
@ -344,14 +353,14 @@ static int32_t mndSetUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SV
static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreate, SUserObj *pUser) { static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreate, SUserObj *pUser) {
SDbObj dbObj = {0}; SDbObj dbObj = {0};
tstrncpy(dbObj.name, pCreate->db, TSDB_FULL_DB_NAME_LEN); memcpy(dbObj.name, pCreate->db, TSDB_FULL_DB_NAME_LEN);
tstrncpy(dbObj.acct, pUser->acct, TSDB_USER_LEN); memcpy(dbObj.acct, pUser->acct, TSDB_USER_LEN);
dbObj.createdTime = taosGetTimestampMs(); dbObj.createdTime = taosGetTimestampMs();
dbObj.updateTime = dbObj.createdTime; dbObj.updateTime = dbObj.createdTime;
dbObj.uid = mndGenerateUid(dbObj.name, TSDB_FULL_DB_NAME_LEN); dbObj.uid = mndGenerateUid(dbObj.name, TSDB_FULL_DB_NAME_LEN);
dbObj.hashMethod = 1;
dbObj.cfgVersion = 1; dbObj.cfgVersion = 1;
dbObj.vgVersion = 1; dbObj.vgVersion = 1;
dbObj.hashMethod = 1;
dbObj.cfg = (SDbCfg){.numOfVgroups = pCreate->numOfVgroups, dbObj.cfg = (SDbCfg){.numOfVgroups = pCreate->numOfVgroups,
.cacheBlockSize = pCreate->cacheBlockSize, .cacheBlockSize = pCreate->cacheBlockSize,
.totalBlocks = pCreate->totalBlocks, .totalBlocks = pCreate->totalBlocks,
@ -359,8 +368,8 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat
.daysToKeep0 = pCreate->daysToKeep0, .daysToKeep0 = pCreate->daysToKeep0,
.daysToKeep1 = pCreate->daysToKeep1, .daysToKeep1 = pCreate->daysToKeep1,
.daysToKeep2 = pCreate->daysToKeep2, .daysToKeep2 = pCreate->daysToKeep2,
.minRows = pCreate->minRowsPerFileBlock, .minRows = pCreate->minRows,
.maxRows = pCreate->maxRowsPerFileBlock, .maxRows = pCreate->maxRows,
.fsyncPeriod = pCreate->fsyncPeriod, .fsyncPeriod = pCreate->fsyncPeriod,
.commitTime = pCreate->commitTime, .commitTime = pCreate->commitTime,
.precision = pCreate->precision, .precision = pCreate->precision,
@ -396,29 +405,30 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat
mError("db:%s, failed to create since %s", pCreate->db, terrstr()); mError("db:%s, failed to create since %s", pCreate->db, terrstr());
goto CREATE_DB_OVER; goto CREATE_DB_OVER;
} }
mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db); mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db);
if (mndSetRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) {
mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr());
goto CREATE_DB_OVER; goto CREATE_DB_OVER;
} }
if (mndSetUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) {
mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr());
goto CREATE_DB_OVER; goto CREATE_DB_OVER;
} }
if (mndSetCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) {
mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr());
goto CREATE_DB_OVER; goto CREATE_DB_OVER;
} }
if (mndSetRedoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) { if (mndSetCreateDbRedoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) {
mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
goto CREATE_DB_OVER; goto CREATE_DB_OVER;
} }
if (mndSetUndoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) { if (mndSetCreateDbUndoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) {
mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
goto CREATE_DB_OVER; goto CREATE_DB_OVER;
} }
@ -447,8 +457,8 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) {
pCreate->daysToKeep0 = htonl(pCreate->daysToKeep0); pCreate->daysToKeep0 = htonl(pCreate->daysToKeep0);
pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1); pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1);
pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2); pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2);
pCreate->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock); pCreate->minRows = htonl(pCreate->minRows);
pCreate->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock); pCreate->maxRows = htonl(pCreate->maxRows);
pCreate->commitTime = htonl(pCreate->commitTime); pCreate->commitTime = htonl(pCreate->commitTime);
pCreate->fsyncPeriod = htonl(pCreate->fsyncPeriod); pCreate->fsyncPeriod = htonl(pCreate->fsyncPeriod);
@ -456,7 +466,7 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) {
SDbObj *pDb = mndAcquireDb(pMnode, pCreate->db); SDbObj *pDb = mndAcquireDb(pMnode, pCreate->db);
if (pDb != NULL) { if (pDb != NULL) {
sdbRelease(pMnode->pSdb, pDb); mndReleaseDb(pMnode, pDb);
if (pCreate->ignoreExist) { if (pCreate->ignoreExist) {
mDebug("db:%s, already exist, ignore exist is set", pCreate->db); mDebug("db:%s, already exist, ignore exist is set", pCreate->db);
return 0; return 0;
@ -485,57 +495,77 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) {
} }
static int32_t mndSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbMsg *pAlter) { static int32_t mndSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbMsg *pAlter) {
bool changed = false; terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED;
if (pAlter->totalBlocks >= 0 && pAlter->totalBlocks != pDb->cfg.totalBlocks) { if (pAlter->totalBlocks >= 0 && pAlter->totalBlocks != pDb->cfg.totalBlocks) {
pDb->cfg.totalBlocks = pAlter->totalBlocks; pDb->cfg.totalBlocks = pAlter->totalBlocks;
changed = true; terrno = 0;
} }
if (pAlter->daysToKeep0 >= 0 && pAlter->daysToKeep0 != pDb->cfg.daysToKeep0) { if (pAlter->daysToKeep0 >= 0 && pAlter->daysToKeep0 != pDb->cfg.daysToKeep0) {
pDb->cfg.daysToKeep0 = pAlter->daysToKeep0; pDb->cfg.daysToKeep0 = pAlter->daysToKeep0;
changed = true; terrno = 0;
} }
if (pAlter->daysToKeep1 >= 0 && pAlter->daysToKeep1 != pDb->cfg.daysToKeep1) { if (pAlter->daysToKeep1 >= 0 && pAlter->daysToKeep1 != pDb->cfg.daysToKeep1) {
pDb->cfg.daysToKeep1 = pAlter->daysToKeep1; pDb->cfg.daysToKeep1 = pAlter->daysToKeep1;
changed = true; terrno = 0;
} }
if (pAlter->daysToKeep2 >= 0 && pAlter->daysToKeep2 != pDb->cfg.daysToKeep2) { if (pAlter->daysToKeep2 >= 0 && pAlter->daysToKeep2 != pDb->cfg.daysToKeep2) {
pDb->cfg.daysToKeep2 = pAlter->daysToKeep2; pDb->cfg.daysToKeep2 = pAlter->daysToKeep2;
changed = true; terrno = 0;
} }
if (pAlter->fsyncPeriod >= 0 && pAlter->fsyncPeriod != pDb->cfg.fsyncPeriod) { if (pAlter->fsyncPeriod >= 0 && pAlter->fsyncPeriod != pDb->cfg.fsyncPeriod) {
pDb->cfg.fsyncPeriod = pAlter->fsyncPeriod; pDb->cfg.fsyncPeriod = pAlter->fsyncPeriod;
changed = true; terrno = 0;
} }
if (pAlter->walLevel >= 0 && pAlter->walLevel != pDb->cfg.walLevel) { if (pAlter->walLevel >= 0 && pAlter->walLevel != pDb->cfg.walLevel) {
pDb->cfg.walLevel = pAlter->walLevel; pDb->cfg.walLevel = pAlter->walLevel;
changed = true; terrno = 0;
} }
if (pAlter->quorum >= 0 && pAlter->quorum != pDb->cfg.quorum) { if (pAlter->quorum >= 0 && pAlter->quorum != pDb->cfg.quorum) {
pDb->cfg.quorum = pAlter->quorum; pDb->cfg.quorum = pAlter->quorum;
changed = true; terrno = 0;
} }
if (pAlter->cacheLastRow >= 0 && pAlter->cacheLastRow != pDb->cfg.cacheLastRow) { if (pAlter->cacheLastRow >= 0 && pAlter->cacheLastRow != pDb->cfg.cacheLastRow) {
pDb->cfg.cacheLastRow = pAlter->cacheLastRow; pDb->cfg.cacheLastRow = pAlter->cacheLastRow;
changed = true; terrno = 0;
} }
if (!changed) { return terrno;
terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED; }
return -1;
} static int32_t mndSetUpdateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) {
SSdbRaw *pRedoRaw = mndDbActionEncode(pNewDb);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1;
return 0; return 0;
} }
static int32_t mndSetUpdateDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) {
SSdbRaw *pUndoRaw = mndDbActionEncode(pOldDb);
if (pUndoRaw == NULL) return -1;
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
}
static int32_t mndSetUpdateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { return 0; }
static int32_t mndSetUpdateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { return 0; }
static int32_t mndSetUpdateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { return 0; }
static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pOldDb, SDbObj *pNewDb) { static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pOldDb, SDbObj *pNewDb) {
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle);
if (pTrans == NULL) { if (pTrans == NULL) {
mError("db:%s, failed to update since %s", pOldDb->name, terrstr()); mError("db:%s, failed to update since %s", pOldDb->name, terrstr());
@ -544,30 +574,41 @@ static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pOldDb, SDbO
mDebug("trans:%d, used to update db:%s", pTrans->id, pOldDb->name); mDebug("trans:%d, used to update db:%s", pTrans->id, pOldDb->name);
SSdbRaw *pRedoRaw = mndDbActionEncode(pNewDb); if (mndSetUpdateDbRedoLogs(pMnode, pTrans, pOldDb, pNewDb) != 0) {
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr());
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); goto UPDATE_DB_OVER;
mndTransDrop(pTrans);
return -1;
} }
sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
SSdbRaw *pUndoRaw = mndDbActionEncode(pOldDb); if (mndSetUpdateDbUndoLogs(pMnode, pTrans, pOldDb, pNewDb) != 0) {
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr());
mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); goto UPDATE_DB_OVER;
mndTransDrop(pTrans); }
return -1;
if (mndSetUpdateDbCommitLogs(pMnode, pTrans, pOldDb, pNewDb) != 0) {
mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr());
goto UPDATE_DB_OVER;
}
if (mndSetUpdateDbRedoActions(pMnode, pTrans, pOldDb, pNewDb) != 0) {
mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
goto UPDATE_DB_OVER;
}
if (mndSetUpdateDbUndoActions(pMnode, pTrans, pOldDb, pNewDb) != 0) {
mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
goto UPDATE_DB_OVER;
} }
sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) { if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); goto UPDATE_DB_OVER;
return -1;
} }
code = 0;
UPDATE_DB_OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
return 0; return code;
} }
static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) { static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) {
@ -610,46 +651,82 @@ static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_ACTION_IN_PROGRESS; return TSDB_CODE_MND_ACTION_IN_PROGRESS;
} }
static int32_t mndSetDropDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
SSdbRaw *pRedoRaw = mndDbActionEncode(pDb);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
return 0;
}
static int32_t mndSetDropDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
SSdbRaw *pUndoRaw = mndDbActionEncode(pDb);
if (pUndoRaw == NULL) return -1;
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
}
static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
SSdbRaw *pCommitRaw = mndDbActionEncode(pDb);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
}
static int32_t mndSetDropDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { return 0; }
static int32_t mndSetDropDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { return 0; }
static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pDb) { static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pDb) {
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle);
if (pTrans == NULL) { if (pTrans == NULL) {
mError("db:%s, failed to drop since %s", pDb->name, terrstr()); mError("db:%s, failed to drop since %s", pDb->name, terrstr());
return -1; return -1;
} }
mDebug("trans:%d, used to drop db:%s", pTrans->id, pDb->name); mDebug("trans:%d, used to drop db:%s", pTrans->id, pDb->name);
SSdbRaw *pRedoRaw = mndDbActionEncode(pDb); if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) {
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr());
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); goto DROP_DB_OVER;
mndTransDrop(pTrans);
return -1;
} }
sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING);
SSdbRaw *pUndoRaw = mndDbActionEncode(pDb); if (mndSetDropDbUndoLogs(pMnode, pTrans, pDb) != 0) {
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr());
mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); goto DROP_DB_OVER;
mndTransDrop(pTrans);
return -1;
} }
sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY);
SSdbRaw *pCommitRaw = mndDbActionEncode(pDb); if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) {
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr());
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); goto DROP_DB_OVER;
mndTransDrop(pTrans); }
return -1;
if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) {
mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
goto DROP_DB_OVER;
}
if (mndSetDropDbUndoActions(pMnode, pTrans, pDb) != 0) {
mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
goto DROP_DB_OVER;
} }
sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pMnode, pTrans) != 0) { if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); goto DROP_DB_OVER;
return -1;
} }
code = 0;
DROP_DB_OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
return 0; return code;
} }
static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) { static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) {
@ -751,29 +828,27 @@ static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) {
static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg) { static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode; SMnode *pMnode = pMsg->pMnode;
SSyncDbMsg *pSync = pMsg->rpcMsg.pCont; SSyncDbMsg *pSync = pMsg->rpcMsg.pCont;
SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db);
SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db);
if (pDb == NULL) { if (pDb == NULL) {
mError("db:%s, failed to process sync db msg since %s", pMsg->db, terrstr()); mError("db:%s, failed to process sync db msg since %s", pMsg->db, terrstr());
return -1; return -1;
} else {
mndReleaseDb(pMnode, pDb);
return 0;
} }
mndReleaseDb(pMnode, pDb);
return 0;
} }
static int32_t mndProcessCompactDbMsg(SMnodeMsg *pMsg) { static int32_t mndProcessCompactDbMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode; SMnode *pMnode = pMsg->pMnode;
SCompactDbMsg *pCompact = pMsg->rpcMsg.pCont; SCompactDbMsg *pCompact = pMsg->rpcMsg.pCont;
SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db);
SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db);
if (pDb == NULL) { if (pDb == NULL) {
mError("db:%s, failed to process compact db msg since %s", pMsg->db, terrstr()); mError("db:%s, failed to process compact db msg since %s", pMsg->db, terrstr());
return -1; return -1;
} else {
mndReleaseDb(pMnode, pDb);
return 0;
} }
mndReleaseDb(pMnode, pDb);
return 0;
} }
static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) {

32
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -18,10 +18,10 @@
#include "mndMnode.h" #include "mndMnode.h"
#include "mndShow.h" #include "mndShow.h"
#include "mndTrans.h" #include "mndTrans.h"
#include "ttime.h"
#include "tep.h" #include "tep.h"
#include "ttime.h"
#define TSDB_DNODE_VER 1 #define TSDB_DNODE_VER_NUMBER 1
#define TSDB_DNODE_RESERVE_SIZE 64 #define TSDB_DNODE_RESERVE_SIZE 64
#define TSDB_CONFIG_OPTION_LEN 16 #define TSDB_CONFIG_OPTION_LEN 16
#define TSDB_CONIIG_VALUE_LEN 48 #define TSDB_CONIIG_VALUE_LEN 48
@ -101,14 +101,14 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) {
SSdbRaw *pRaw = mndDnodeActionEncode(&dnodeObj); SSdbRaw *pRaw = mndDnodeActionEncode(&dnodeObj);
if (pRaw == NULL) return -1; if (pRaw == NULL) return -1;
sdbSetRawStatus(pRaw, SDB_STATUS_READY); if (sdbSetRawStatus(pRaw, SDB_STATUS_READY) != 0) return -1;
mDebug("dnode:%d, will be created while deploy sdb", dnodeObj.id); mDebug("dnode:%d, will be created while deploy sdb", dnodeObj.id);
return sdbWrite(pMnode->pSdb, pRaw); return sdbWrite(pMnode->pSdb, pRaw);
} }
static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) { static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, TSDB_DNODE_VER, sizeof(SDnodeObj) + TSDB_DNODE_RESERVE_SIZE); SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, TSDB_DNODE_VER_NUMBER, sizeof(SDnodeObj) + TSDB_DNODE_RESERVE_SIZE);
if (pRaw == NULL) return NULL; if (pRaw == NULL) return NULL;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -127,7 +127,7 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
if (sver != TSDB_DNODE_VER) { if (sver != TSDB_DNODE_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER; terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
mError("failed to decode dnode since %s", terrstr()); mError("failed to decode dnode since %s", terrstr());
return NULL; return NULL;
@ -150,21 +150,8 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) {
static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode) { static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode) {
mTrace("dnode:%d, perform insert action", pDnode->id); mTrace("dnode:%d, perform insert action", pDnode->id);
pDnode->rebootTime = 0;
pDnode->lastAccessTime = 0;
pDnode->accessTimes = 0;
pDnode->numOfMnodes = 0;
pDnode->numOfVnodes = 0;
pDnode->numOfQnodes = 0;
pDnode->numOfSupportMnodes = 0;
pDnode->numOfSupportVnodes = 0;
pDnode->numOfSupportQnodes = 0;
pDnode->numOfCores = 0;
pDnode->status = DND_STATUS_OFFLINE;
pDnode->offlineReason = DND_REASON_STATUS_NOT_RECEIVED; pDnode->offlineReason = DND_REASON_STATUS_NOT_RECEIVED;
snprintf(pDnode->ep, TSDB_EP_LEN, "%s:%u", pDnode->fqdn, pDnode->port); snprintf(pDnode->ep, TSDB_EP_LEN, "%s:%u", pDnode->fqdn, pDnode->port);
return 0; return 0;
} }
@ -225,7 +212,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) {
bool mndIsDnodeInReadyStatus(SMnode *pMnode, SDnodeObj *pDnode) { bool mndIsDnodeInReadyStatus(SMnode *pMnode, SDnodeObj *pDnode) {
int64_t ms = taosGetTimestampMs(); int64_t ms = taosGetTimestampMs();
int64_t interval = ABS(pDnode->lastAccessTime - ms); int64_t interval = ABS(pDnode->lastAccessTime - ms);
if (interval > 3000 * pMnode->cfg.statusInterval) { if (interval > 3500 * pMnode->cfg.statusInterval) {
return false; return false;
} }
return true; return true;
@ -267,12 +254,9 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) {
return DND_REASON_STATUS_INTERVAL_NOT_MATCH; return DND_REASON_STATUS_INTERVAL_NOT_MATCH;
} }
int64_t checkTime = 0; if ((0 != strcasecmp(pCfg->timezone, pMnode->cfg.timezone)) && (pMnode->checkTime != pCfg->checkTime)) {
char timestr[32] = "1970-01-01 00:00:00.00";
(void)taosParseTime(timestr, &checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
if ((0 != strcasecmp(pCfg->timezone, pMnode->cfg.timezone)) && (checkTime != pCfg->checkTime)) {
mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, pMnode->cfg.timezone, mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, pMnode->cfg.timezone,
pCfg->checkTime, checkTime); pCfg->checkTime, pMnode->checkTime);
return DND_REASON_TIME_ZONE_NOT_MATCH; return DND_REASON_TIME_ZONE_NOT_MATCH;
} }

9
source/dnode/mnode/impl/src/mndMnode.c
View File

@ -19,7 +19,8 @@
#include "mndShow.h" #include "mndShow.h"
#include "mndTrans.h" #include "mndTrans.h"
#define SDB_MNODE_VER 1 #define TSDB_MNODE_VER_NUMBER 1
#define TSDB_MNODE_RESERVE_SIZE 64
static int32_t mndCreateDefaultMnode(SMnode *pMnode); static int32_t mndCreateDefaultMnode(SMnode *pMnode);
static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pMnodeObj); static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pMnodeObj);
@ -97,13 +98,14 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) {
} }
static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pMnodeObj) { static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pMnodeObj) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_MNODE, SDB_MNODE_VER, sizeof(SMnodeObj)); SSdbRaw *pRaw = sdbAllocRaw(SDB_MNODE, TSDB_MNODE_VER_NUMBER, sizeof(SMnodeObj));
if (pRaw == NULL) return NULL; if (pRaw == NULL) return NULL;
int32_t dataPos = 0; int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, pMnodeObj->id); SDB_SET_INT32(pRaw, dataPos, pMnodeObj->id);
SDB_SET_INT64(pRaw, dataPos, pMnodeObj->createdTime) SDB_SET_INT64(pRaw, dataPos, pMnodeObj->createdTime)
SDB_SET_INT64(pRaw, dataPos, pMnodeObj->updateTime) SDB_SET_INT64(pRaw, dataPos, pMnodeObj->updateTime)
SDB_SET_RESERVE(pRaw, dataPos, TSDB_MNODE_RESERVE_SIZE)
return pRaw; return pRaw;
} }
@ -112,7 +114,7 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
if (sver != SDB_MNODE_VER) { if (sver != TSDB_MNODE_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER; terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
mError("failed to decode mnode since %s", terrstr()); mError("failed to decode mnode since %s", terrstr());
return NULL; return NULL;
@ -126,6 +128,7 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, pRow, dataPos, &pMnodeObj->id) SDB_GET_INT32(pRaw, pRow, dataPos, &pMnodeObj->id)
SDB_GET_INT64(pRaw, pRow, dataPos, &pMnodeObj->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pMnodeObj->createdTime)
SDB_GET_INT64(pRaw, pRow, dataPos, &pMnodeObj->updateTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pMnodeObj->updateTime)
SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_MNODE_RESERVE_SIZE)
return pRow; return pRow;
} }

208
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -17,17 +17,10 @@
#include "mndTrans.h" #include "mndTrans.h"
#include "mndSync.h" #include "mndSync.h"
#define TSDB_TRANS_VER 1 #define TSDB_TRANS_VER_NUMBER 1
#define TSDB_TRN_ARRAY_SIZE 8 #define TSDB_TRN_ARRAY_SIZE 8
#define TSDB_TRN_RESERVE_SIZE 64 #define TSDB_TRN_RESERVE_SIZE 64
typedef struct {
SEpSet epSet;
int8_t msgType;
int32_t contLen;
void *pCont;
} STransAction;
static SSdbRaw *mndTransActionEncode(STrans *pTrans); static SSdbRaw *mndTransActionEncode(STrans *pTrans);
static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw);
static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans);
@ -37,11 +30,11 @@ static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans);
static void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle); static void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle);
static void mndTransSendRpcRsp(STrans *pTrans, int32_t code); static void mndTransSendRpcRsp(STrans *pTrans, int32_t code);
static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw); static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw);
static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont); static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction);
static void mndTransDropLogs(SArray *pArray); static void mndTransDropLogs(SArray *pArray);
static void mndTransDropActions(SArray *pArray); static void mndTransDropActions(SArray *pArray);
static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray); static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray);
static int32_t mndTransExecuteActions(SMnode *pMnode, SArray *pArray); static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray);
static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans);
static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans);
static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans);
@ -100,7 +93,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
rawDataLen += (sizeof(STransAction) + pAction->contLen); rawDataLen += (sizeof(STransAction) + pAction->contLen);
} }
SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TSDB_TRANS_VER, rawDataLen); SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TSDB_TRANS_VER_NUMBER, rawDataLen);
if (pRaw == NULL) { if (pRaw == NULL) {
mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr()); mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr());
return NULL; return NULL;
@ -167,7 +160,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
return NULL; return NULL;
} }
if (sver != TSDB_TRANS_VER) { if (sver != TSDB_TRANS_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER; terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
mError("failed to get check soft ver from raw:%p since %s", pRaw, terrstr()); mError("failed to get check soft ver from raw:%p since %s", pRaw, terrstr());
return NULL; return NULL;
@ -343,10 +336,8 @@ char *mndTransStageStr(ETrnStage stage) {
return "commit"; return "commit";
case TRN_STAGE_ROLLBACK: case TRN_STAGE_ROLLBACK:
return "rollback"; return "rollback";
case TRN_STAGE_RETRY:
return "retry";
case TRN_STAGE_OVER: case TRN_STAGE_OVER:
return "stop"; return "over";
default: default:
return "undefined"; return "undefined";
} }
@ -388,7 +379,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) {
return NULL; return NULL;
} }
mDebug("trans:%d, data:%p is created", pTrans->id, pTrans); mDebug("trans:%d, is created", pTrans->id);
return pTrans; return pTrans;
} }
@ -417,7 +408,7 @@ void mndTransDrop(STrans *pTrans) {
mndTransDropActions(pTrans->redoActions); mndTransDropActions(pTrans->redoActions);
mndTransDropActions(pTrans->undoActions); mndTransDropActions(pTrans->undoActions);
mDebug("trans:%d, data:%p is dropped", pTrans->id, pTrans); // mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans);
tfree(pTrans); tfree(pTrans);
} }
@ -459,10 +450,8 @@ int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw) {
return code; return code;
} }
static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) { static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction) {
STransAction action = {.epSet = *pEpSet, .msgType = msgType, .contLen = contLen, .pCont = pCont}; void *ptr = taosArrayPush(pArray, pAction);
void *ptr = taosArrayPush(pArray, &action);
if (ptr == NULL) { if (ptr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
@ -471,16 +460,12 @@ static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgTy
return 0; return 0;
} }
int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) { int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction) {
int32_t code = mndTransAppendAction(pTrans->redoActions, pEpSet, msgType, contLen, pCont); return mndTransAppendAction(pTrans->redoActions, pAction);
mTrace("trans:%d, msg:%s len:%d append to redo actions", pTrans->id, taosMsg[msgType], contLen);
return code;
} }
int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) { int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction) {
int32_t code = mndTransAppendAction(pTrans->undoActions, pEpSet, msgType, contLen, pCont); return mndTransAppendAction(pTrans->undoActions, pAction);
mTrace("trans:%d, msg:%s len:%d append to undo actions", pTrans->id, taosMsg[msgType], contLen);
return code;
} }
int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
@ -493,7 +478,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
} }
sdbSetRawStatus(pRaw, SDB_STATUS_READY); sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mTrace("trans:%d, start sync", pTrans->id); mTrace("trans:%d, sync to other nodes", pTrans->id);
int32_t code = mndSyncPropose(pMnode, pRaw); int32_t code = mndSyncPropose(pMnode, pRaw);
if (code != 0) { if (code != 0) {
mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); mError("trans:%d, failed to sync since %s", pTrans->id, terrstr());
@ -533,7 +518,7 @@ int32_t mndTransCommit(SMnode *pMnode, STrans *pTrans) {
sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED);
if (taosArrayGetSize(pTrans->commitLogs) != 0) { if (taosArrayGetSize(pTrans->commitLogs) != 0) {
mTrace("trans:%d, start sync", pTrans->id); mTrace("trans:%d, sync to other nodes", pTrans->id);
int32_t code = mndSyncPropose(pMnode, pRaw); int32_t code = mndSyncPropose(pMnode, pRaw);
if (code != 0) { if (code != 0) {
mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); mError("trans:%d, failed to sync since %s", pTrans->id, terrstr());
@ -563,7 +548,7 @@ int32_t mndTransRollback(SMnode *pMnode, STrans *pTrans) {
} }
sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED);
mTrace("trans:%d, start sync", pTrans->id); mTrace("trans:%d, sync to other nodes", pTrans->id);
int32_t code = mndSyncPropose(pMnode, pRaw); int32_t code = mndSyncPropose(pMnode, pRaw);
if (code != 0) { if (code != 0) {
mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); mError("trans:%d, failed to sync since %s", pTrans->id, terrstr());
@ -596,6 +581,50 @@ void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code)
// todo // todo
} }
void mndTransHandleActionRsp(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
int64_t sig = (int64_t)(pMsg->rpcMsg.ahandle);
int32_t transId = (int32_t)(sig >> 32);
int32_t action = (int32_t)((sig << 32) >> 32);
STrans *pTrans = mndAcquireTrans(pMnode, transId);
if (pTrans == NULL) {
mError("trans:%d, failed to get transId from vnode rsp since %s", transId, terrstr());
goto HANDLE_ACTION_RSP_OVER;
}
SArray *pArray = NULL;
if (pTrans->stage == TRN_STAGE_EXECUTE) {
pArray = pTrans->redoActions;
} else if (pTrans->stage == TRN_STAGE_ROLLBACK) {
pArray = pTrans->undoActions;
} else {
}
if (pArray == NULL) {
mError("trans:%d, invalid trans stage:%s", transId, mndTransStageStr(pTrans->stage));
goto HANDLE_ACTION_RSP_OVER;
}
int32_t actionNum = taosArrayGetSize(pTrans->redoActions);
if (action < 0 || action > actionNum) {
mError("trans:%d, invalid action:%d", transId, action);
goto HANDLE_ACTION_RSP_OVER;
}
STransAction *pAction = taosArrayGet(pArray, action);
if (pAction != NULL) {
pAction->msgReceived = 1;
pAction->errCode = pMsg->code;
}
mDebug("trans:%d, action:%d response is received, code:0x%x", transId, action, pMsg->code);
mndTransExecute(pMnode, pTrans);
HANDLE_ACTION_RSP_OVER:
mndReleaseTrans(pMnode, pTrans);
}
static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) { static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
int32_t arraySize = taosArrayGetSize(pArray); int32_t arraySize = taosArrayGetSize(pArray);
@ -618,7 +647,7 @@ static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans) {
if (code != 0) { if (code != 0) {
mError("trans:%d, failed to execute redo logs since %s", pTrans->id, terrstr()) mError("trans:%d, failed to execute redo logs since %s", pTrans->id, terrstr())
} else { } else {
mTrace("trans:%d, execute redo logs finished", pTrans->id) mDebug("trans:%d, execute redo logs finished", pTrans->id)
} }
} }
@ -632,7 +661,7 @@ static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans) {
if (code != 0) { if (code != 0) {
mError("trans:%d, failed to execute undo logs since %s", pTrans->id, terrstr()) mError("trans:%d, failed to execute undo logs since %s", pTrans->id, terrstr())
} else { } else {
mTrace("trans:%d, execute undo logs finished", pTrans->id) mDebug("trans:%d, execute undo logs finished", pTrans->id)
} }
} }
@ -646,47 +675,70 @@ static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans) {
if (code != 0) { if (code != 0) {
mError("trans:%d, failed to execute commit logs since %s", pTrans->id, terrstr()) mError("trans:%d, failed to execute commit logs since %s", pTrans->id, terrstr())
} else { } else {
mTrace("trans:%d, execute commit logs finished", pTrans->id) mDebug("trans:%d, execute commit logs finished", pTrans->id)
} }
} }
return code; return code;
} }
static int32_t mndTransExecuteActions(SMnode *pMnode, SArray *pArray) { static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) {
#if 0 int32_t numOfActions = taosArrayGetSize(pArray);
int32_t arraySize = taosArrayGetSize(pArray); if (numOfActions == 0) return 0;
for (int32_t i = 0; i < arraySize; ++i) {
STransAction *pAction = taosArrayGet(pArray, i);
SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen}; for (int32_t action = 0; action < numOfActions; ++action) {
STransAction *pAction = taosArrayGet(pArray, action);
if (pAction == NULL) continue;
if (pAction->msgSent) continue;
int64_t signature = pTrans->id;
signature = (signature << 32);
signature += action;
SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen, .ahandle = (void *)signature};
rpcMsg.pCont = rpcMallocCont(pAction->contLen); rpcMsg.pCont = rpcMallocCont(pAction->contLen);
if (rpcMsg.pCont == NULL) { if (rpcMsg.pCont == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen); memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen);
pAction->msgSent = 1;
pAction->msgReceived = 0;
pAction->errCode = 0;
mDebug("trans:%d, action:%d is sent", pTrans->id, action);
mndSendMsgToDnode(pMnode, &pAction->epSet, &rpcMsg); mndSendMsgToDnode(pMnode, &pAction->epSet, &rpcMsg);
} }
return TSDB_CODE_MND_ACTION_IN_PROGRESS; int32_t numOfReceivedMsgs = 0;
#else int32_t errorCode = 0;
return 0; for (int32_t action = 0; action < numOfActions; ++action) {
#endif STransAction *pAction = taosArrayGet(pArray, action);
if (pAction == NULL) continue;
if (pAction->msgSent && pAction->msgReceived) {
numOfReceivedMsgs++;
if (pAction->errCode != 0) {
errorCode = pAction->errCode;
}
}
}
if (numOfReceivedMsgs == numOfActions) {
mDebug("trans:%d, all %d actions executed, code:0x%x", pTrans->id, numOfActions, errorCode);
terrno = errorCode;
return errorCode;
} else {
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
} }
static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) {
if (taosArrayGetSize(pTrans->redoActions) <= 0) return 0; return mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions);
mTrace("trans:%d, start to execute redo actions", pTrans->id);
return mndTransExecuteActions(pMnode, pTrans->redoActions);
} }
static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) {
if (taosArrayGetSize(pTrans->undoActions) <= 0) return 0; return mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions);
mTrace("trans:%d, start to execute undo actions", pTrans->id);
return mndTransExecuteActions(pMnode, pTrans->undoActions);
} }
static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) {
@ -694,7 +746,7 @@ static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) {
if (code == 0) { if (code == 0) {
pTrans->stage = TRN_STAGE_EXECUTE; pTrans->stage = TRN_STAGE_EXECUTE;
mTrace("trans:%d, stage from prepare to execute", pTrans->id); mDebug("trans:%d, stage from prepare to execute", pTrans->id);
} else { } else {
pTrans->stage = TRN_STAGE_ROLLBACK; pTrans->stage = TRN_STAGE_ROLLBACK;
mError("trans:%d, stage from prepare to rollback since %s", pTrans->id, terrstr()); mError("trans:%d, stage from prepare to rollback since %s", pTrans->id, terrstr());
@ -708,17 +760,17 @@ static int32_t mndTransPerformExecuteStage(SMnode *pMnode, STrans *pTrans) {
if (code == 0) { if (code == 0) {
pTrans->stage = TRN_STAGE_COMMIT; pTrans->stage = TRN_STAGE_COMMIT;
mTrace("trans:%d, stage from execute to commit", pTrans->id); mDebug("trans:%d, stage from execute to commit", pTrans->id);
} else if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { } else if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mTrace("trans:%d, stage keep on execute since %s", pTrans->id, terrstr(code)); mDebug("trans:%d, stage keep on execute since %s", pTrans->id, tstrerror(code));
return code; return code;
} else { } else {
if (pTrans->policy == TRN_POLICY_ROLLBACK) { if (pTrans->policy == TRN_POLICY_ROLLBACK) {
pTrans->stage = TRN_STAGE_ROLLBACK; pTrans->stage = TRN_STAGE_ROLLBACK;
mError("trans:%d, stage from execute to rollback since %s", pTrans->id, terrstr()); mError("trans:%d, stage from execute to rollback since %s", pTrans->id, terrstr());
} else { } else {
pTrans->stage = TRN_STAGE_RETRY; pTrans->stage = TRN_STAGE_EXECUTE;
mError("trans:%d, stage from execute to retry since %s", pTrans->id, terrstr()); mError("trans:%d, stage keep on execute since %s", pTrans->id, terrstr());
} }
} }
@ -726,29 +778,16 @@ static int32_t mndTransPerformExecuteStage(SMnode *pMnode, STrans *pTrans) {
} }
static int32_t mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) { static int32_t mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) {
int32_t code = mndTransExecuteCommitLogs(pMnode, pTrans); mndTransExecuteCommitLogs(pMnode, pTrans);
pTrans->stage = TRN_STAGE_OVER;
if (code == 0) { return 0;
pTrans->stage = TRN_STAGE_OVER;
mTrace("trans:%d, commit stage finished", pTrans->id);
} else {
if (pTrans->policy == TRN_POLICY_ROLLBACK) {
pTrans->stage = TRN_STAGE_ROLLBACK;
mError("trans:%d, stage from commit to rollback since %s", pTrans->id, terrstr());
} else {
pTrans->stage = TRN_STAGE_RETRY;
mError("trans:%d, stage from commit to retry since %s", pTrans->id, terrstr());
}
}
return code;
} }
static int32_t mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { static int32_t mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) {
int32_t code = mndTransExecuteUndoActions(pMnode, pTrans); int32_t code = mndTransExecuteUndoActions(pMnode, pTrans);
if (code == 0) { if (code == 0) {
mTrace("trans:%d, rollbacked", pTrans->id); mDebug("trans:%d, rollbacked", pTrans->id);
} else { } else {
pTrans->stage = TRN_STAGE_ROLLBACK; pTrans->stage = TRN_STAGE_ROLLBACK;
mError("trans:%d, stage keep on rollback since %s", pTrans->id, terrstr()); mError("trans:%d, stage keep on rollback since %s", pTrans->id, terrstr());
@ -757,20 +796,6 @@ static int32_t mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) {
return code; return code;
} }
static int32_t mndTransPerformRetryStage(SMnode *pMnode, STrans *pTrans) {
int32_t code = mndTransExecuteRedoActions(pMnode, pTrans);
if (code == 0) {
pTrans->stage = TRN_STAGE_COMMIT;
mTrace("trans:%d, stage from retry to commit", pTrans->id);
} else {
pTrans->stage = TRN_STAGE_RETRY;
mError("trans:%d, stage keep on retry since %s", pTrans->id, terrstr());
}
return code;
}
static void mndTransExecute(SMnode *pMnode, STrans *pTrans) { static void mndTransExecute(SMnode *pMnode, STrans *pTrans) {
int32_t code = 0; int32_t code = 0;
@ -785,7 +810,7 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) {
case TRN_STAGE_COMMIT: case TRN_STAGE_COMMIT:
code = mndTransCommit(pMnode, pTrans); code = mndTransCommit(pMnode, pTrans);
if (code == 0) { if (code == 0) {
code = mndTransPerformCommitStage(pMnode, pTrans); mndTransPerformCommitStage(pMnode, pTrans);
} }
break; break;
case TRN_STAGE_ROLLBACK: case TRN_STAGE_ROLLBACK:
@ -794,9 +819,6 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) {
code = mndTransRollback(pMnode, pTrans); code = mndTransRollback(pMnode, pTrans);
} }
break; break;
case TRN_STAGE_RETRY:
code = mndTransPerformRetryStage(pMnode, pTrans);
break;
default: default:
mndTransSendRpcRsp(pTrans, 0); mndTransSendRpcRsp(pTrans, 0);
return; return;

6
source/dnode/mnode/impl/src/mndUser.c
View File

@ -20,7 +20,7 @@
#include "mndTrans.h" #include "mndTrans.h"
#include "tkey.h" #include "tkey.h"
#define TSDB_USER_VER 1 #define TSDB_USER_VER_NUMBER 1
#define TSDB_USER_RESERVE_SIZE 64 #define TSDB_USER_RESERVE_SIZE 64
static int32_t mndCreateDefaultUsers(SMnode *pMnode); static int32_t mndCreateDefaultUsers(SMnode *pMnode);
@ -94,7 +94,7 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) {
} }
static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { static SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, TSDB_USER_VER, sizeof(SUserObj) + TSDB_USER_RESERVE_SIZE); SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, TSDB_USER_VER_NUMBER, sizeof(SUserObj) + TSDB_USER_RESERVE_SIZE);
if (pRaw == NULL) return NULL; if (pRaw == NULL) return NULL;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -114,7 +114,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
if (sver != TSDB_USER_VER) { if (sver != TSDB_USER_VER_NUMBER) {
mError("failed to decode user since %s", terrstr()); mError("failed to decode user since %s", terrstr());
terrno = TSDB_CODE_SDB_INVALID_DATA_VER; terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
return NULL; return NULL;

61
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -21,7 +21,7 @@
#include "mndShow.h" #include "mndShow.h"
#include "mndTrans.h" #include "mndTrans.h"
#define TSDB_VGROUP_VER_NUM 1 #define TSDB_VGROUP_VER_NUMBER 1
#define TSDB_VGROUP_RESERVE_SIZE 64 #define TSDB_VGROUP_RESERVE_SIZE 64
static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw); static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw);
@ -70,7 +70,7 @@ int32_t mndInitVgroup(SMnode *pMnode) {
void mndCleanupVgroup(SMnode *pMnode) {} void mndCleanupVgroup(SMnode *pMnode) {}
SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, TSDB_VGROUP_VER_NUM, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE); SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, TSDB_VGROUP_VER_NUMBER, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE);
if (pRaw == NULL) return NULL; if (pRaw == NULL) return NULL;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -98,7 +98,7 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
if (sver != TSDB_VGROUP_VER_NUM) { if (sver != TSDB_VGROUP_VER_NUMBER) {
mError("failed to decode vgroup since %s", terrstr()); mError("failed to decode vgroup since %s", terrstr());
terrno = TSDB_CODE_SDB_INVALID_DATA_VER; terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
return NULL; return NULL;
@ -142,14 +142,20 @@ static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNe
mTrace("vgId:%d, perform update action", pOldVgroup->vgId); mTrace("vgId:%d, perform update action", pOldVgroup->vgId);
pOldVgroup->updateTime = pNewVgroup->updateTime; pOldVgroup->updateTime = pNewVgroup->updateTime;
pOldVgroup->version = pNewVgroup->version; pOldVgroup->version = pNewVgroup->version;
pOldVgroup->hashBegin = pNewVgroup->hashBegin;
pOldVgroup->hashEnd = pNewVgroup->hashEnd;
pOldVgroup->replica = pNewVgroup->replica; pOldVgroup->replica = pNewVgroup->replica;
memcpy(pOldVgroup->vnodeGid, pNewVgroup->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); memcpy(pOldVgroup->vnodeGid, pNewVgroup->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid));
return 0; return 0;
} }
SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId) { SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
return sdbAcquire(pSdb, SDB_VGROUP, &vgId); SVgObj *pVgroup = sdbAcquire(pSdb, SDB_VGROUP, &vgId);
if (pVgroup == NULL) {
terrno = TSDB_CODE_MND_VGROUP_NOT_EXIST;
}
return pVgroup;
} }
void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) { void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) {
@ -158,16 +164,17 @@ void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) {
} }
SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) { SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) {
SCreateVnodeMsg *pCreate = malloc(sizeof(SCreateVnodeMsg)); SCreateVnodeMsg *pCreate = calloc(1, sizeof(SCreateVnodeMsg));
if (pCreate == NULL) { if (pCreate == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
pCreate->dnodeId = htonl(pDnode->id);
pCreate->vgId = htonl(pVgroup->vgId); pCreate->vgId = htonl(pVgroup->vgId);
pCreate->dnodeId = htonl(pDnode->id);
memcpy(pCreate->db, pDb->name, TSDB_FULL_DB_NAME_LEN); memcpy(pCreate->db, pDb->name, TSDB_FULL_DB_NAME_LEN);
pCreate->dbUid = htobe64(pDb->uid); pCreate->dbUid = htobe64(pDb->uid);
pCreate->vgVersion = htonl(pVgroup->version);
pCreate->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize); pCreate->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize);
pCreate->totalBlocks = htonl(pDb->cfg.totalBlocks); pCreate->totalBlocks = htonl(pDb->cfg.totalBlocks);
pCreate->daysPerFile = htonl(pDb->cfg.daysPerFile); pCreate->daysPerFile = htonl(pDb->cfg.daysPerFile);
@ -193,7 +200,6 @@ SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbOb
SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
if (pVgidDnode == NULL) { if (pVgidDnode == NULL) {
free(pCreate); free(pCreate);
terrno = TSDB_CODE_MND_APP_ERROR;
return NULL; return NULL;
} }
@ -217,7 +223,7 @@ SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbOb
} }
SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) { SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) {
SDropVnodeMsg *pDrop = malloc(sizeof(SDropVnodeMsg)); SDropVnodeMsg *pDrop = calloc(1, sizeof(SDropVnodeMsg));
if (pDrop == NULL) { if (pDrop == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
@ -269,7 +275,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
return -1; return -1;
} }
int32_t alloceVgroups = 0; int32_t allocedVgroups = 0;
int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP);
uint32_t hashMin = 0; uint32_t hashMin = 0;
uint32_t hashMax = UINT32_MAX; uint32_t hashMax = UINT32_MAX;
@ -281,7 +287,6 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
pVgroup->createdTime = taosGetTimestampMs(); pVgroup->createdTime = taosGetTimestampMs();
pVgroup->updateTime = pVgroups->createdTime; pVgroup->updateTime = pVgroups->createdTime;
pVgroup->version = 1; pVgroup->version = 1;
pVgroup->dbUid = pDb->uid;
pVgroup->hashBegin = hashMin + hashInterval * v; pVgroup->hashBegin = hashMin + hashInterval * v;
if (v == pDb->cfg.numOfVgroups - 1) { if (v == pDb->cfg.numOfVgroups - 1) {
pVgroup->hashEnd = hashMax; pVgroup->hashEnd = hashMax;
@ -290,6 +295,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
} }
memcpy(pVgroup->dbName, pDb->name, TSDB_FULL_DB_NAME_LEN); memcpy(pVgroup->dbName, pDb->name, TSDB_FULL_DB_NAME_LEN);
pVgroup->dbUid = pDb->uid;
pVgroup->replica = pDb->cfg.replications; pVgroup->replica = pDb->cfg.replications;
if (mndGetAvailableDnode(pMnode, pVgroup) != 0) { if (mndGetAvailableDnode(pMnode, pVgroup) != 0) {
@ -298,22 +304,25 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
return -1; return -1;
} }
alloceVgroups++; allocedVgroups++;
} }
*ppVgroups = pVgroups; *ppVgroups = pVgroups;
return 0; return 0;
} }
static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg) {
mndTransHandleActionRsp(pMsg);
return 0;
}
static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pReplica, int32_t *pNumOfVgroups) { static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pReplica, int32_t *pNumOfVgroups) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SDbObj *pDb = mndAcquireDb(pMnode, dbName); SDbObj *pDb = mndAcquireDb(pMnode, dbName);
if (pDb == NULL) { if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED; terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
@ -329,7 +338,7 @@ static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pRep
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break; if (pIter == NULL) break;
if (strcmp(pVgroup->dbName, dbName) == 0) { if (pVgroup->dbUid == pDb->uid) {
replica = MAX(replica, pVgroup->replica); replica = MAX(replica, pVgroup->replica);
numOfVgroups++; numOfVgroups++;
} }
@ -441,11 +450,25 @@ static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter) {
} }
static int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { static int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) {
if (dnodeId == 0) { SSdb *pSdb = pMnode->pSdb;
return 0; int32_t numOfVnodes = 0;
void *pIter = NULL;
while (1) {
SVgObj *pVgroup = NULL;
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
for (int32_t v = 0; v < pVgroup->replica; ++v) {
if (pVgroup->vnodeGid[v].dnodeId == dnodeId) {
numOfVnodes++;
}
}
sdbRelease(pSdb, pVgroup);
} }
return 0; return numOfVnodes;
} }
static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) {

3
source/dnode/mnode/impl/src/mnode.c
View File

@ -237,6 +237,9 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
return NULL; return NULL;
} }
char timestr[24] = "1970-01-01 00:00:00.00";
(void)taosParseTime(timestr, &pMnode->checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
pMnode->pSteps = taosArrayInit(24, sizeof(SMnodeStep)); pMnode->pSteps = taosArrayInit(24, sizeof(SMnodeStep));
if (pMnode->pSteps == NULL) { if (pMnode->pSteps == NULL) {
free(pMnode); free(pMnode);

4
source/dnode/mnode/sdb/src/sdbRaw.c
View File

@ -27,12 +27,12 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) {
pRaw->sver = sver; pRaw->sver = sver;
pRaw->dataLen = dataLen; pRaw->dataLen = dataLen;
mTrace("raw:%p, is created, len:%d", pRaw, dataLen); // mTrace("raw:%p, is created, len:%d", pRaw, dataLen);
return pRaw; return pRaw;
} }
void sdbFreeRaw(SSdbRaw *pRaw) { void sdbFreeRaw(SSdbRaw *pRaw) {
mTrace("raw:%p, is freed", pRaw); // mTrace("raw:%p, is freed", pRaw);
free(pRaw); free(pRaw);
} }

6
source/dnode/vnode/impl/src/vnodeWrite.c
View File

@ -16,17 +16,13 @@
#include "vnodeDef.h" #include "vnodeDef.h"
int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) { int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) {
SVnodeReq *pVnodeReq;
switch (pMsg->msgType) { switch (pMsg->msgType) {
case TSDB_MSG_TYPE_MQ_SET: case TSDB_MSG_TYPE_MQ_SET_CUR:
if (tqSetCursor(pVnode->pTq, pMsg->pCont) < 0) { if (tqSetCursor(pVnode->pTq, pMsg->pCont) < 0) {
// TODO: handle error // TODO: handle error
} }
break; break;
} }
void *pBuf = pMsg->pCont;
return 0; return 0;
} }

1
source/dnode/vnode/tq/CMakeLists.txt
View File

@ -11,6 +11,7 @@ target_link_libraries(
PUBLIC wal PUBLIC wal
PUBLIC os PUBLIC os
PUBLIC util PUBLIC util
PUBLIC common
) )
if(${BUILD_TEST}) if(${BUILD_TEST})

11
source/dnode/vnode/tq/inc/tqInt.h
View File

@ -17,11 +17,20 @@
#define _TD_TQ_INT_H_ #define _TD_TQ_INT_H_
#include "tq.h" #include "tq.h"
#include "tlog.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
extern int32_t tqDebugFlag;
#define tqFatal(...) { if (tqDebugFlag & DEBUG_FATAL) { taosPrintLog("TQ FATAL ", 255, __VA_ARGS__); }}
#define tqError(...) { if (tqDebugFlag & DEBUG_ERROR) { taosPrintLog("TQ ERROR ", 255, __VA_ARGS__); }}
#define tqWarn(...) { if (tqDebugFlag & DEBUG_WARN) { taosPrintLog("TQ WARN ", 255, __VA_ARGS__); }}
#define tqInfo(...) { if (tqDebugFlag & DEBUG_INFO) { taosPrintLog("TQ ", 255, __VA_ARGS__); }}
#define tqDebug(...) { if (tqDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); }}
#define tqTrace(...) { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); }}
// create persistent storage for meta info such as consuming offset // create persistent storage for meta info such as consuming offset
// return value > 0: cgId // return value > 0: cgId
// return value <= 0: error code // return value <= 0: error code

2
source/dnode/vnode/tq/inc/tqMetaStore.h
View File

@ -17,7 +17,7 @@
#define _TQ_META_STORE_H_ #define _TQ_META_STORE_H_
#include "os.h" #include "os.h"
#include "tq.h" #include "tqInt.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {

329
source/dnode/vnode/tq/src/tq.c
View File

@ -24,89 +24,80 @@
// handle management message // handle management message
// //
int tqGetgHandleSSize(const STqGroupHandle* gHandle); int tqGroupSSize(const STqGroup* pGroup);
int tqBufHandleSSize(); int tqTopicSSize();
int tqBufItemSSize(); int tqItemSSize();
STqGroupHandle* tqFindHandle(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { void* tqSerializeListHandle(STqList* listHandle, void* ptr);
STqGroupHandle* gHandle; void* tqSerializeTopic(STqTopic* pTopic, void* ptr);
return NULL; void* tqSerializeItem(STqMsgItem* pItem, void* ptr);
}
void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr); const void* tqDeserializeTopic(const void* pBytes, STqTopic* pTopic);
void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr); const void* tqDeserializeItem(const void* pBytes, STqMsgItem* pItem);
void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr);
const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle);
const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem);
STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac) { STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac) {
STQ* pTq = malloc(sizeof(STQ)); STQ* pTq = malloc(sizeof(STQ));
if (pTq == NULL) { if (pTq == NULL) {
// TODO: memory error terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return NULL; return NULL;
} }
pTq->path = strdup(path); pTq->path = strdup(path);
pTq->tqConfig = tqConfig; pTq->tqConfig = tqConfig;
pTq->tqLogReader = tqLogReader; pTq->tqLogReader = tqLogReader;
pTq->tqMemRef.pAlloctorFactory = allocFac; pTq->tqMemRef.pAlloctorFactory = allocFac;
// pTq->tqMemRef.pAllocator = allocFac->create(allocFac); pTq->tqMemRef.pAllocator = allocFac->create(allocFac);
if (pTq->tqMemRef.pAllocator == NULL) { if (pTq->tqMemRef.pAllocator == NULL) {
// TODO // TODO: error code of buffer pool
} }
pTq->tqMeta = pTq->tqMeta = tqStoreOpen(path, (FTqSerialize)tqSerializeGroup, (FTqDeserialize)tqDeserializeGroup, free, 0);
tqStoreOpen(path, (FTqSerialize)tqSerializeGroupHandle, (FTqDeserialize)tqDeserializeGroupHandle, free, 0);
if (pTq->tqMeta == NULL) { if (pTq->tqMeta == NULL) {
// TODO: free STQ // TODO: free STQ
return NULL; return NULL;
} }
return pTq; return pTq;
} }
void tqClose(STQ* pTq) {
void tqClose(STQ*pTq) {
// TODO // TODO
} }
static int tqProtoCheck(TmqMsgHead *pMsg) { static int tqProtoCheck(STqMsgHead* pMsg) { return pMsg->protoVer == 0; }
return pMsg->protoVer == 0;
}
static int tqAckOneTopic(STqBufferHandle* bHandle, TmqOneAck* pAck, STqQueryMsg** ppQuery) { static int tqAckOneTopic(STqTopic* pTopic, STqOneAck* pAck, STqQueryMsg** ppQuery) {
// clean old item and move forward // clean old item and move forward
int32_t consumeOffset = pAck->consumeOffset; int32_t consumeOffset = pAck->consumeOffset;
int idx = consumeOffset % TQ_BUFFER_SIZE; int idx = consumeOffset % TQ_BUFFER_SIZE;
ASSERT(bHandle->buffer[idx].content && bHandle->buffer[idx].executor); ASSERT(pTopic->buffer[idx].content && pTopic->buffer[idx].executor);
tfree(bHandle->buffer[idx].content); tfree(pTopic->buffer[idx].content);
if (1 /* TODO: need to launch new query */) { if (1 /* TODO: need to launch new query */) {
STqQueryMsg* pNewQuery = malloc(sizeof(STqQueryMsg)); STqQueryMsg* pNewQuery = malloc(sizeof(STqQueryMsg));
if (pNewQuery == NULL) { if (pNewQuery == NULL) {
// TODO: memory insufficient terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1; return -1;
} }
// TODO: lock executor // TODO: lock executor
pNewQuery->exec->executor = bHandle->buffer[idx].executor;
// TODO: read from wal and assign to src // TODO: read from wal and assign to src
pNewQuery->exec->src = 0; /*pNewQuery->exec->executor = pTopic->buffer[idx].executor;*/
pNewQuery->exec->dest = &bHandle->buffer[idx]; /*pNewQuery->exec->src = 0;*/
pNewQuery->next = *ppQuery; /*pNewQuery->exec->dest = &pTopic->buffer[idx];*/
*ppQuery = pNewQuery; /*pNewQuery->next = *ppQuery;*/
/**ppQuery = pNewQuery;*/
} }
return 0; return 0;
} }
static int tqAck(STqGroupHandle* gHandle, TmqAcks* pAcks) { static int tqAck(STqGroup* pGroup, STqAcks* pAcks) {
int32_t ackNum = pAcks->ackNum; int32_t ackNum = pAcks->ackNum;
TmqOneAck* acks = pAcks->acks; STqOneAck* acks = pAcks->acks;
// double ptr for acks and list // double ptr for acks and list
int i = 0; int i = 0;
STqListHandle* node = gHandle->head; STqList* node = pGroup->head;
int ackCnt = 0; int ackCnt = 0;
STqQueryMsg* pQuery = NULL; STqQueryMsg* pQuery = NULL;
while (i < ackNum && node->next) { while (i < ackNum && node->next) {
if (acks[i].topicId == node->next->bufHandle.topicId) { if (acks[i].topicId == node->next->topic.topicId) {
ackCnt++; ackCnt++;
tqAckOneTopic(&node->next->bufHandle, &acks[i], &pQuery); tqAckOneTopic(&node->next->topic, &acks[i], &pQuery);
} else if (acks[i].topicId < node->next->bufHandle.topicId) { } else if (acks[i].topicId < node->next->topic.topicId) {
i++; i++;
} else { } else {
node = node->next; node = node->next;
@ -118,52 +109,56 @@ static int tqAck(STqGroupHandle* gHandle, TmqAcks* pAcks) {
return ackCnt; return ackCnt;
} }
static int tqCommitTCGroup(STqGroupHandle* handle) { static int tqCommitGroup(STqGroup* pGroup) {
// persist modification into disk // persist modification into disk
return 0; return 0;
} }
int tqCreateTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroupHandle** handle) { int tqCreateGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroup** ppGroup) {
// create in disk // create in disk
STqGroupHandle* gHandle = (STqGroupHandle*)malloc(sizeof(STqGroupHandle)); STqGroup* pGroup = (STqGroup*)malloc(sizeof(STqGroup));
if (gHandle == NULL) { if (pGroup == NULL) {
// TODO // TODO
return -1; return -1;
} }
memset(gHandle, 0, sizeof(STqGroupHandle)); *ppGroup = pGroup;
memset(pGroup, 0, sizeof(STqGroup));
return 0; return 0;
} }
STqGroupHandle* tqOpenTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { STqGroup* tqOpenGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
STqGroupHandle* gHandle = tqHandleGet(pTq->tqMeta, cId); STqGroup* pGroup = tqHandleGet(pTq->tqMeta, cId);
if (gHandle == NULL) { if (pGroup == NULL) {
int code = tqCreateTCGroup(pTq, topicId, cgId, cId, &gHandle); int code = tqCreateGroup(pTq, topicId, cgId, cId, &pGroup);
if (code != 0) { if (code < 0) {
// TODO // TODO
return NULL; return NULL;
} }
tqHandleMovePut(pTq->tqMeta, cId, pGroup);
} }
ASSERT(pGroup);
// create return pGroup;
// open
return gHandle;
} }
int tqCloseTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { return 0; } int tqCloseGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
// TODO
return 0;
}
int tqDropTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { int tqDropGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
// delete from disk // delete from disk
return 0; return 0;
} }
static int tqFetch(STqGroupHandle* gHandle, void** msg) { static int tqFetch(STqGroup* pGroup, void** msg) {
STqListHandle* head = gHandle->head; STqList* head = pGroup->head;
STqListHandle* node = head; STqList* node = head;
int totSize = 0; int totSize = 0;
// TODO: make it a macro // TODO: make it a macro
int sizeLimit = 4 * 1024; int sizeLimit = 4 * 1024;
TmqMsgContent* buffer = malloc(sizeLimit); STqMsgContent* buffer = malloc(sizeLimit);
if (buffer == NULL) { if (buffer == NULL) {
// TODO:memory insufficient // TODO:memory insufficient
return -1; return -1;
@ -172,25 +167,25 @@ static int tqFetch(STqGroupHandle* gHandle, void** msg) {
// until all topic iterated or msgs over sizeLimit // until all topic iterated or msgs over sizeLimit
while (node->next) { while (node->next) {
node = node->next; node = node->next;
STqBufferHandle* bufHandle = &node->bufHandle; STqTopic* topicHandle = &node->topic;
int idx = bufHandle->nextConsumeOffset % TQ_BUFFER_SIZE; int idx = topicHandle->nextConsumeOffset % TQ_BUFFER_SIZE;
if (bufHandle->buffer[idx].content != NULL && bufHandle->buffer[idx].offset == bufHandle->nextConsumeOffset) { if (topicHandle->buffer[idx].content != NULL && topicHandle->buffer[idx].offset == topicHandle->nextConsumeOffset) {
totSize += bufHandle->buffer[idx].size; totSize += topicHandle->buffer[idx].size;
if (totSize > sizeLimit) { if (totSize > sizeLimit) {
void* ptr = realloc(buffer, totSize); void* ptr = realloc(buffer, totSize);
if (ptr == NULL) { if (ptr == NULL) {
totSize -= bufHandle->buffer[idx].size; totSize -= topicHandle->buffer[idx].size;
// TODO:memory insufficient // TODO:memory insufficient
// return msgs already copied // return msgs already copied
break; break;
} }
} }
*((int64_t*)buffer) = bufHandle->topicId; *((int64_t*)buffer) = topicHandle->topicId;
buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); buffer = POINTER_SHIFT(buffer, sizeof(int64_t));
*((int64_t*)buffer) = bufHandle->buffer[idx].size; *((int64_t*)buffer) = topicHandle->buffer[idx].size;
buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); buffer = POINTER_SHIFT(buffer, sizeof(int64_t));
memcpy(buffer, bufHandle->buffer[idx].content, bufHandle->buffer[idx].size); memcpy(buffer, topicHandle->buffer[idx].content, topicHandle->buffer[idx].size);
buffer = POINTER_SHIFT(buffer, bufHandle->buffer[idx].size); buffer = POINTER_SHIFT(buffer, topicHandle->buffer[idx].size);
if (totSize > sizeLimit) { if (totSize > sizeLimit) {
break; break;
} }
@ -199,11 +194,19 @@ static int tqFetch(STqGroupHandle* gHandle, void** msg) {
return totSize; return totSize;
} }
STqGroupHandle* tqGetGroupHandle(STQ* pTq, int64_t cId) { return NULL; } STqGroup* tqGetGroup(STQ* pTq, int64_t clientId) { return tqHandleGet(pTq->tqMeta, clientId); }
int tqLaunchQuery(STqGroupHandle* gHandle) { return 0; } int tqSendLaunchQuery(STqMsgItem* bufItem, int64_t offset) {
if (tqQueryExecuting(bufItem->status)) {
int tqSendLaunchQuery(STqGroupHandle* gHandle) { return 0; } return 0;
}
bufItem->status = 1;
// load data from wal or buffer pool
// put into exec
// send exec into non blocking queue
// when query finished, put into buffer pool
return 0;
}
/*int tqMoveOffsetToNext(TqGroupHandle* gHandle) {*/ /*int tqMoveOffsetToNext(TqGroupHandle* gHandle) {*/
/*return 0;*/ /*return 0;*/
@ -220,23 +223,96 @@ int tqCommit(STQ* pTq) {
return 0; return 0;
} }
int tqSetCursor(STQ* pTq, void* msg) { int tqBufferSetOffset(STqTopic* pTopic, int64_t offset) {
int code;
memset(pTopic->buffer, 0, sizeof(pTopic->buffer));
// launch query
for (int i = offset; i < offset + TQ_BUFFER_SIZE; i++) {
int pos = i % TQ_BUFFER_SIZE;
code = tqSendLaunchQuery(&pTopic->buffer[pos], offset);
if (code < 0) {
// TODO: error handling
}
}
// set offset
pTopic->nextConsumeOffset = offset;
pTopic->floatingCursor = offset;
return 0; return 0;
} }
int tqConsume(STQ* pTq, STqConsumeReq* pMsg) { STqTopic* tqFindTopic(STqGroup* pGroup, int64_t topicId) {
if (!tqProtoCheck((TmqMsgHead*)pMsg)) { // TODO
// proto version invalid return NULL;
return -1; }
}
int64_t clientId = pMsg->head.clientId; int tqSetCursor(STQ* pTq, STqSetCurReq* pMsg) {
STqGroupHandle* gHandle = tqGetGroupHandle(pTq, clientId); int code;
int64_t clientId = pMsg->head.clientId;
int64_t topicId = pMsg->topicId;
int64_t offset = pMsg->offset;
STqGroup* gHandle = tqGetGroup(pTq, clientId);
if (gHandle == NULL) { if (gHandle == NULL) {
// client not connect // client not connect
return -1; return -1;
} }
STqTopic* topicHandle = tqFindTopic(gHandle, topicId);
if (topicHandle == NULL) {
return -1;
}
if (pMsg->offset == topicHandle->nextConsumeOffset) {
return 0;
}
// TODO: check log last version
code = tqBufferSetOffset(topicHandle, offset);
if (code < 0) {
// set error code
return -1;
}
return 0;
}
int tqConsume(STQ* pTq, STqConsumeReq* pMsg) {
int64_t clientId = pMsg->head.clientId;
STqGroup* pGroup = tqGetGroup(pTq, clientId);
if (pGroup == NULL) {
terrno = TSDB_CODE_TQ_GROUP_NOT_SET;
return -1;
}
STqConsumeRsp* pRsp = (STqConsumeRsp*)pMsg;
int numOfMsgs = tqFetch(pGroup, (void**)&pRsp->msgs);
if (numOfMsgs < 0) {
return -1;
}
if (numOfMsgs == 0) {
// most recent data has been fetched
// enable timer for blocking wait
// once new data written during wait time
// launch query and response
}
// fetched a num of msgs, rpc response
return 0;
}
#if 0
int tqConsume(STQ* pTq, STqConsumeReq* pMsg) {
if (!tqProtoCheck((STqMsgHead*)pMsg)) {
// proto version invalid
return -1;
}
int64_t clientId = pMsg->head.clientId;
STqGroup* pGroup = tqGetGroup(pTq, clientId);
if (pGroup == NULL) {
// client not connect
return -1;
}
if (pMsg->acks.ackNum != 0) { if (pMsg->acks.ackNum != 0) {
if (tqAck(gHandle, &pMsg->acks) != 0) { if (tqAck(pGroup, &pMsg->acks) != 0) {
// ack not success // ack not success
return -1; return -1;
} }
@ -244,22 +320,23 @@ int tqConsume(STQ* pTq, STqConsumeReq* pMsg) {
STqConsumeRsp* pRsp = (STqConsumeRsp*)pMsg; STqConsumeRsp* pRsp = (STqConsumeRsp*)pMsg;
if (tqFetch(gHandle, (void**)&pRsp->msgs) <= 0) { if (tqFetch(pGroup, (void**)&pRsp->msgs) <= 0) {
// fetch error // fetch error
return -1; return -1;
} }
// judge and launch new query // judge and launch new query
if (tqLaunchQuery(gHandle)) { /*if (tqSendLaunchQuery(gHandle)) {*/
// launch query error // launch query error
return -1; /*return -1;*/
} /*}*/
return 0; return 0;
} }
#endif
int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead) { int tqSerializeGroup(const STqGroup* pGroup, STqSerializedHead** ppHead) {
// calculate size // calculate size
int sz = tqGetgHandleSSize(gHandle) + sizeof(STqSerializedHead); int sz = tqGroupSSize(pGroup) + sizeof(STqSerializedHead);
if (sz > (*ppHead)->ssize) { if (sz > (*ppHead)->ssize) {
void* tmpPtr = realloc(*ppHead, sz); void* tmpPtr = realloc(*ppHead, sz);
if (tmpPtr == NULL) { if (tmpPtr == NULL) {
@ -272,53 +349,53 @@ int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** pp
} }
void* ptr = (*ppHead)->content; void* ptr = (*ppHead)->content;
// do serialization // do serialization
*(int64_t*)ptr = gHandle->cId; *(int64_t*)ptr = pGroup->clientId;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
*(int64_t*)ptr = gHandle->cgId; *(int64_t*)ptr = pGroup->cgId;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
*(int32_t*)ptr = gHandle->topicNum; *(int32_t*)ptr = pGroup->topicNum;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
if (gHandle->topicNum > 0) { if (pGroup->topicNum > 0) {
tqSerializeListHandle(gHandle->head, ptr); tqSerializeListHandle(pGroup->head, ptr);
} }
return 0; return 0;
} }
void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr) { void* tqSerializeListHandle(STqList* listHandle, void* ptr) {
STqListHandle* node = listHandle; STqList* node = listHandle;
ASSERT(node != NULL); ASSERT(node != NULL);
while (node) { while (node) {
ptr = tqSerializeBufHandle(&node->bufHandle, ptr); ptr = tqSerializeTopic(&node->topic, ptr);
node = node->next; node = node->next;
} }
return ptr; return ptr;
} }
void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr) { void* tqSerializeTopic(STqTopic* pTopic, void* ptr) {
*(int64_t*)ptr = bufHandle->nextConsumeOffset; *(int64_t*)ptr = pTopic->nextConsumeOffset;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
*(int64_t*)ptr = bufHandle->topicId; *(int64_t*)ptr = pTopic->topicId;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
*(int32_t*)ptr = bufHandle->head; *(int32_t*)ptr = pTopic->head;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
*(int32_t*)ptr = bufHandle->tail; *(int32_t*)ptr = pTopic->tail;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
for (int i = 0; i < TQ_BUFFER_SIZE; i++) { for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
ptr = tqSerializeBufItem(&bufHandle->buffer[i], ptr); ptr = tqSerializeItem(&pTopic->buffer[i], ptr);
} }
return ptr; return ptr;
} }
void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr) { void* tqSerializeItem(STqMsgItem* bufItem, void* ptr) {
// TODO: do we need serialize this? // TODO: do we need serialize this?
// mainly for executor // mainly for executor
return ptr; return ptr;
} }
const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** ppGHandle) { const void* tqDeserializeGroup(const STqSerializedHead* pHead, STqGroup** ppGroup) {
STqGroupHandle* gHandle = *ppGHandle; STqGroup* gHandle = *ppGroup;
const void* ptr = pHead->content; const void* ptr = pHead->content;
gHandle->cId = *(int64_t*)ptr; gHandle->clientId = *(int64_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
gHandle->cgId = *(int64_t*)ptr; gHandle->cgId = *(int64_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
@ -326,63 +403,63 @@ const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHan
gHandle->topicNum = *(int32_t*)ptr; gHandle->topicNum = *(int32_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
gHandle->head = NULL; gHandle->head = NULL;
STqListHandle* node = gHandle->head; STqList* node = gHandle->head;
for (int i = 0; i < gHandle->topicNum; i++) { for (int i = 0; i < gHandle->topicNum; i++) {
if (gHandle->head == NULL) { if (gHandle->head == NULL) {
if ((node = malloc(sizeof(STqListHandle))) == NULL) { if ((node = malloc(sizeof(STqList))) == NULL) {
// TODO: error // TODO: error
return NULL; return NULL;
} }
node->next = NULL; node->next = NULL;
ptr = tqDeserializeBufHandle(ptr, &node->bufHandle); ptr = tqDeserializeTopic(ptr, &node->topic);
gHandle->head = node; gHandle->head = node;
} else { } else {
node->next = malloc(sizeof(STqListHandle)); node->next = malloc(sizeof(STqList));
if (node->next == NULL) { if (node->next == NULL) {
// TODO: error // TODO: error
return NULL; return NULL;
} }
node->next->next = NULL; node->next->next = NULL;
ptr = tqDeserializeBufHandle(ptr, &node->next->bufHandle); ptr = tqDeserializeTopic(ptr, &node->next->topic);
node = node->next; node = node->next;
} }
} }
return ptr; return ptr;
} }
const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle) { const void* tqDeserializeTopic(const void* pBytes, STqTopic* topic) {
const void* ptr = pBytes; const void* ptr = pBytes;
bufHandle->nextConsumeOffset = *(int64_t*)ptr; topic->nextConsumeOffset = *(int64_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
bufHandle->topicId = *(int64_t*)ptr; topic->topicId = *(int64_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
bufHandle->head = *(int32_t*)ptr; topic->head = *(int32_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
bufHandle->tail = *(int32_t*)ptr; topic->tail = *(int32_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
for (int i = 0; i < TQ_BUFFER_SIZE; i++) { for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
ptr = tqDeserializeBufItem(ptr, &bufHandle->buffer[i]); ptr = tqDeserializeItem(ptr, &topic->buffer[i]);
} }
return ptr; return ptr;
} }
const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem) { return pBytes; } const void* tqDeserializeItem(const void* pBytes, STqMsgItem* bufItem) { return pBytes; }
// TODO: make this a macro // TODO: make this a macro
int tqGetgHandleSSize(const STqGroupHandle* gHandle) { int tqGroupSSize(const STqGroup* gHandle) {
return sizeof(int64_t) * 2 // cId + cgId return sizeof(int64_t) * 2 // cId + cgId
+ sizeof(int32_t) // topicNum + sizeof(int32_t) // topicNum
+ gHandle->topicNum * tqBufHandleSSize(); + gHandle->topicNum * tqTopicSSize();
} }
// TODO: make this a macro // TODO: make this a macro
int tqBufHandleSSize() { int tqTopicSSize() {
return sizeof(int64_t) * 2 // nextConsumeOffset + topicId return sizeof(int64_t) * 2 // nextConsumeOffset + topicId
+ sizeof(int32_t) * 2 // head + tail + sizeof(int32_t) * 2 // head + tail
+ TQ_BUFFER_SIZE * tqBufItemSSize(); + TQ_BUFFER_SIZE * tqItemSSize();
} }
int tqBufItemSSize() { int tqItemSSize() {
// TODO: do this need serialization? // TODO: do this need serialization?
// mainly for executor // mainly for executor
return 0; return 0;

58
source/dnode/vnode/tq/src/tqMetaStore.c
View File

@ -56,6 +56,7 @@ static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) {
int offset = tqSeekLastPage(fd); int offset = tqSeekLastPage(fd);
int nBytes; int nBytes;
if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) { if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
if (nBytes == 0) { if (nBytes == 0) {
@ -71,7 +72,7 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial
int32_t tqConfigFlag) { int32_t tqConfigFlag) {
STqMetaStore* pMeta = malloc(sizeof(STqMetaStore)); STqMetaStore* pMeta = malloc(sizeof(STqMetaStore));
if (pMeta == NULL) { if (pMeta == NULL) {
// close terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return NULL; return NULL;
} }
memset(pMeta, 0, sizeof(STqMetaStore)); memset(pMeta, 0, sizeof(STqMetaStore));
@ -79,8 +80,9 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial
// concat data file name and index file name // concat data file name and index file name
size_t pathLen = strlen(path); size_t pathLen = strlen(path);
pMeta->dirPath = malloc(pathLen + 1); pMeta->dirPath = malloc(pathLen + 1);
if (pMeta->dirPath != NULL) { if (pMeta->dirPath == NULL) {
// TODO: memory insufficient terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return NULL;
} }
strcpy(pMeta->dirPath, path); strcpy(pMeta->dirPath, path);
@ -88,13 +90,14 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial
strcpy(name, path); strcpy(name, path);
if (taosDirExist(name) != 0 && taosMkDir(name) != 0) { if (taosDirExist(name) != 0 && taosMkDir(name) != 0) {
ASSERT(false); terrno = TSDB_CODE_TQ_FAILED_TO_CREATE_DIR;
tqError("failed to create dir:%s since %s ", name, terrstr());
} }
strcat(name, "/" TQ_IDX_NAME); strcat(name, "/" TQ_IDX_NAME);
int idxFd = open(name, O_RDWR | O_CREAT, 0755); int idxFd = open(name, O_RDWR | O_CREAT, 0755);
if (idxFd < 0) { if (idxFd < 0) {
ASSERT(false); terrno = TAOS_SYSTEM_ERROR(errno);
// close file tqError("failed to open file:%s since %s ", name, terrstr());
// free memory // free memory
return NULL; return NULL;
} }
@ -102,9 +105,7 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial
pMeta->idxFd = idxFd; pMeta->idxFd = idxFd;
pMeta->unpersistHead = malloc(sizeof(STqMetaList)); pMeta->unpersistHead = malloc(sizeof(STqMetaList));
if (pMeta->unpersistHead == NULL) { if (pMeta->unpersistHead == NULL) {
ASSERT(false); terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
// close file
// free memory
return NULL; return NULL;
} }
memset(pMeta->unpersistHead, 0, sizeof(STqMetaList)); memset(pMeta->unpersistHead, 0, sizeof(STqMetaList));
@ -114,7 +115,8 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial
strcat(name, "/" TQ_META_NAME); strcat(name, "/" TQ_META_NAME);
int fileFd = open(name, O_RDWR | O_CREAT, 0755); int fileFd = open(name, O_RDWR | O_CREAT, 0755);
if (fileFd < 0) { if (fileFd < 0) {
ASSERT(false); terrno = TAOS_SYSTEM_ERROR(errno);
tqError("failed to open file:%s since %s", name, terrstr());
return NULL; return NULL;
} }
@ -129,7 +131,7 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial
STqIdxPageBuf idxBuf; STqIdxPageBuf idxBuf;
STqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE); STqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE);
if (serializedObj == NULL) { if (serializedObj == NULL) {
// TODO:memory insufficient terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
} }
int idxRead; int idxRead;
int allocated = TQ_PAGE_SIZE; int allocated = TQ_PAGE_SIZE;
@ -137,14 +139,16 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial
while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) { while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) {
if (idxRead == -1) { if (idxRead == -1) {
// TODO: handle error // TODO: handle error
ASSERT(false); terrno = TAOS_SYSTEM_ERROR(errno);
tqError("failed to read tq index file since %s", terrstr());
} }
ASSERT(idxBuf.head.writeOffset == idxRead); ASSERT(idxBuf.head.writeOffset == idxRead);
// loop read every entry // loop read every entry
for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) { for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) {
STqMetaList* pNode = malloc(sizeof(STqMetaList)); STqMetaList* pNode = malloc(sizeof(STqMetaList));
if (pNode == NULL) { if (pNode == NULL) {
// TODO: free memory and return error terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
// TODO: free memory
} }
memset(pNode, 0, sizeof(STqMetaList)); memset(pNode, 0, sizeof(STqMetaList));
memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE); memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE);
@ -153,7 +157,8 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial
if (allocated < pNode->handle.serializedSize) { if (allocated < pNode->handle.serializedSize) {
void* ptr = realloc(serializedObj, pNode->handle.serializedSize); void* ptr = realloc(serializedObj, pNode->handle.serializedSize);
if (ptr == NULL) { if (ptr == NULL) {
// TODO: memory insufficient terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
// TODO: free memory
} }
serializedObj = ptr; serializedObj = ptr;
allocated = pNode->handle.serializedSize; allocated = pNode->handle.serializedSize;
@ -292,7 +297,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
STqMetaList* pNode = pHead->unpersistNext; STqMetaList* pNode = pHead->unpersistNext;
STqSerializedHead* pSHead = malloc(sizeof(STqSerializedHead)); STqSerializedHead* pSHead = malloc(sizeof(STqSerializedHead));
if (pSHead == NULL) { if (pSHead == NULL) {
// TODO: memory error terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1; return -1;
} }
pSHead->ver = TQ_SVER; pSHead->ver = TQ_SVER;
@ -403,7 +408,6 @@ static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* valu
STqMetaList* pNode = pMeta->bucket[bucketKey]; STqMetaList* pNode = pMeta->bucket[bucketKey];
while (pNode) { while (pNode) {
if (pNode->handle.key == key) { if (pNode->handle.key == key) {
// TODO: think about thread safety
if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInUse); pMeta->pDeleter(pNode->handle.valueInUse);
} }
@ -416,7 +420,7 @@ static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* valu
} }
STqMetaList* pNewNode = malloc(sizeof(STqMetaList)); STqMetaList* pNewNode = malloc(sizeof(STqMetaList));
if (pNewNode == NULL) { if (pNewNode == NULL) {
// TODO: memory error terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1; return -1;
} }
memset(pNewNode, 0, sizeof(STqMetaList)); memset(pNewNode, 0, sizeof(STqMetaList));
@ -470,10 +474,10 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va
STqMetaList* pNode = pMeta->bucket[bucketKey]; STqMetaList* pNode = pMeta->bucket[bucketKey];
while (pNode) { while (pNode) {
if (pNode->handle.key == key) { if (pNode->handle.key == key) {
// TODO: think about thread safety
if (pNode->handle.valueInTxn) { if (pNode->handle.valueInTxn) {
if (TqDupIntxnReject(pMeta->tqConfigFlag)) { if (tqDupIntxnReject(pMeta->tqConfigFlag)) {
return -2; terrno = TSDB_CODE_TQ_META_KEY_DUP_IN_TXN;
return -1;
} }
if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInTxn); pMeta->pDeleter(pNode->handle.valueInTxn);
@ -488,7 +492,7 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va
} }
STqMetaList* pNewNode = malloc(sizeof(STqMetaList)); STqMetaList* pNewNode = malloc(sizeof(STqMetaList));
if (pNewNode == NULL) { if (pNewNode == NULL) {
// TODO: memory error terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1; return -1;
} }
memset(pNewNode, 0, sizeof(STqMetaList)); memset(pNewNode, 0, sizeof(STqMetaList));
@ -505,7 +509,7 @@ int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { return
int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) { int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) {
void* vmem = malloc(vsize); void* vmem = malloc(vsize);
if (vmem == NULL) { if (vmem == NULL) {
// TODO: memory error terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1; return -1;
} }
memcpy(vmem, value, vsize); memcpy(vmem, value, vsize);
@ -535,6 +539,7 @@ int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) {
while (pNode) { while (pNode) {
if (pNode->handle.key == key) { if (pNode->handle.key == key) {
if (pNode->handle.valueInTxn == NULL) { if (pNode->handle.valueInTxn == NULL) {
terrno = TSDB_CODE_TQ_META_KEY_NOT_IN_TXN;
return -1; return -1;
} }
if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
@ -548,7 +553,8 @@ int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) {
pNode = pNode->next; pNode = pNode->next;
} }
} }
return -2; terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY;
return -1;
} }
int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) { int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) {
@ -564,12 +570,14 @@ int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) {
tqLinkUnpersist(pMeta, pNode); tqLinkUnpersist(pMeta, pNode);
return 0; return 0;
} }
terrno = TSDB_CODE_TQ_META_KEY_NOT_IN_TXN;
return -1; return -1;
} else { } else {
pNode = pNode->next; pNode = pNode->next;
} }
} }
return -2; terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY;
return -1;
} }
int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) { int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) {
@ -588,7 +596,7 @@ int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) {
pNode = pNode->next; pNode = pNode->next;
} }
} }
// no such key terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY;
return -1; return -1;
} }

115
source/dnode/vnode/tq/test/tqMetaTest.cpp
View File

@ -10,8 +10,8 @@ struct Foo {
}; };
int FooSerializer(const void* pObj, STqSerializedHead** ppHead) { int FooSerializer(const void* pObj, STqSerializedHead** ppHead) {
Foo* foo = (Foo*) pObj; Foo* foo = (Foo*)pObj;
if((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) { if ((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) {
*ppHead = (STqSerializedHead*)realloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t)); *ppHead = (STqSerializedHead*)realloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t));
(*ppHead)->ssize = sizeof(STqSerializedHead) + sizeof(int32_t); (*ppHead)->ssize = sizeof(STqSerializedHead) + sizeof(int32_t);
} }
@ -20,7 +20,7 @@ int FooSerializer(const void* pObj, STqSerializedHead** ppHead) {
} }
const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) { const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) {
if(*ppObj == NULL) { if (*ppObj == NULL) {
*ppObj = realloc(*ppObj, sizeof(int32_t)); *ppObj = realloc(*ppObj, sizeof(int32_t));
} }
Foo* pFoo = *(Foo**)ppObj; Foo* pFoo = *(Foo**)ppObj;
@ -28,28 +28,20 @@ const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) {
return NULL; return NULL;
} }
void FooDeleter(void* pObj) { void FooDeleter(void* pObj) { free(pObj); }
free(pObj);
}
class TqMetaUpdateAppendTest : public ::testing::Test { class TqMetaUpdateAppendTest : public ::testing::Test {
protected: protected:
void SetUp() override {
taosRemoveDir(pathName);
pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
ASSERT(pMeta);
}
void SetUp() override { void TearDown() override { tqStoreClose(pMeta); }
taosRemoveDir(pathName);
pMeta = tqStoreOpen(pathName,
FooSerializer, FooDeserializer, FooDeleter,
TQ_UPDATE_APPEND
);
ASSERT(pMeta);
}
void TearDown() override { STqMetaStore* pMeta;
tqStoreClose(pMeta); const char* pathName = "/tmp/tq_test";
}
TqMetaStore* pMeta;
const char* pathName = "/tmp/tq_test";
}; };
TEST_F(TqMetaUpdateAppendTest, copyPutTest) { TEST_F(TqMetaUpdateAppendTest, copyPutTest) {
@ -57,11 +49,11 @@ TEST_F(TqMetaUpdateAppendTest, copyPutTest) {
foo.a = 3; foo.a = 3;
tqHandleCopyPut(pMeta, 1, &foo, sizeof(Foo)); tqHandleCopyPut(pMeta, 1, &foo, sizeof(Foo));
Foo* pFoo = (Foo*) tqHandleGet(pMeta, 1); Foo* pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true); EXPECT_EQ(pFoo == NULL, true);
tqHandleCommit(pMeta, 1); tqHandleCommit(pMeta, 1);
pFoo = (Foo*) tqHandleGet(pMeta, 1); pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo->a, 3); EXPECT_EQ(pFoo->a, 3);
} }
@ -78,10 +70,7 @@ TEST_F(TqMetaUpdateAppendTest, persistTest) {
EXPECT_EQ(pBar == NULL, true); EXPECT_EQ(pBar == NULL, true);
tqStoreClose(pMeta); tqStoreClose(pMeta);
pMeta = tqStoreOpen(pathName, pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
FooSerializer, FooDeserializer, FooDeleter,
TQ_UPDATE_APPEND
);
ASSERT(pMeta); ASSERT(pMeta);
pBar = (Foo*)tqHandleGet(pMeta, 1); pBar = (Foo*)tqHandleGet(pMeta, 1);
@ -97,7 +86,7 @@ TEST_F(TqMetaUpdateAppendTest, uncommittedTest) {
pFoo->a = 3; pFoo->a = 3;
tqHandleMovePut(pMeta, 1, pFoo); tqHandleMovePut(pMeta, 1, pFoo);
pFoo = (Foo*) tqHandleGet(pMeta, 1); pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true); EXPECT_EQ(pFoo == NULL, true);
} }
@ -106,11 +95,11 @@ TEST_F(TqMetaUpdateAppendTest, abortTest) {
pFoo->a = 3; pFoo->a = 3;
tqHandleMovePut(pMeta, 1, pFoo); tqHandleMovePut(pMeta, 1, pFoo);
pFoo = (Foo*) tqHandleGet(pMeta, 1); pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true); EXPECT_EQ(pFoo == NULL, true);
tqHandleAbort(pMeta, 1); tqHandleAbort(pMeta, 1);
pFoo = (Foo*) tqHandleGet(pMeta, 1); pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true); EXPECT_EQ(pFoo == NULL, true);
} }
@ -119,32 +108,29 @@ TEST_F(TqMetaUpdateAppendTest, deleteTest) {
pFoo->a = 3; pFoo->a = 3;
tqHandleMovePut(pMeta, 1, pFoo); tqHandleMovePut(pMeta, 1, pFoo);
pFoo = (Foo*) tqHandleGet(pMeta, 1); pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true); EXPECT_EQ(pFoo == NULL, true);
tqHandleCommit(pMeta, 1); tqHandleCommit(pMeta, 1);
pFoo = (Foo*) tqHandleGet(pMeta, 1); pFoo = (Foo*)tqHandleGet(pMeta, 1);
ASSERT_EQ(pFoo != NULL, true); ASSERT_EQ(pFoo != NULL, true);
EXPECT_EQ(pFoo->a, 3); EXPECT_EQ(pFoo->a, 3);
tqHandleDel(pMeta, 1); tqHandleDel(pMeta, 1);
pFoo = (Foo*) tqHandleGet(pMeta, 1); pFoo = (Foo*)tqHandleGet(pMeta, 1);
ASSERT_EQ(pFoo != NULL, true); ASSERT_EQ(pFoo != NULL, true);
EXPECT_EQ(pFoo->a, 3); EXPECT_EQ(pFoo->a, 3);
tqHandleCommit(pMeta, 1); tqHandleCommit(pMeta, 1);
pFoo = (Foo*) tqHandleGet(pMeta, 1); pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true); EXPECT_EQ(pFoo == NULL, true);
tqStoreClose(pMeta); tqStoreClose(pMeta);
pMeta = tqStoreOpen(pathName, pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
FooSerializer, FooDeserializer, FooDeleter,
TQ_UPDATE_APPEND
);
ASSERT(pMeta); ASSERT(pMeta);
pFoo = (Foo*) tqHandleGet(pMeta, 1); pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true); EXPECT_EQ(pFoo == NULL, true);
} }
@ -162,10 +148,7 @@ TEST_F(TqMetaUpdateAppendTest, intxnPersist) {
EXPECT_EQ(pFoo1->a, 3); EXPECT_EQ(pFoo1->a, 3);
tqStoreClose(pMeta); tqStoreClose(pMeta);
pMeta = tqStoreOpen(pathName, pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
FooSerializer, FooDeserializer, FooDeleter,
TQ_UPDATE_APPEND
);
ASSERT(pMeta); ASSERT(pMeta);
pFoo1 = (Foo*)tqHandleGet(pMeta, 1); pFoo1 = (Foo*)tqHandleGet(pMeta, 1);
@ -177,10 +160,7 @@ TEST_F(TqMetaUpdateAppendTest, intxnPersist) {
EXPECT_EQ(pFoo1->a, 4); EXPECT_EQ(pFoo1->a, 4);
tqStoreClose(pMeta); tqStoreClose(pMeta);
pMeta = tqStoreOpen(pathName, pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
FooSerializer, FooDeserializer, FooDeleter,
TQ_UPDATE_APPEND
);
ASSERT(pMeta); ASSERT(pMeta);
pFoo1 = (Foo*)tqHandleGet(pMeta, 1); pFoo1 = (Foo*)tqHandleGet(pMeta, 1);
@ -190,13 +170,13 @@ TEST_F(TqMetaUpdateAppendTest, intxnPersist) {
TEST_F(TqMetaUpdateAppendTest, multiplePage) { TEST_F(TqMetaUpdateAppendTest, multiplePage) {
srand(0); srand(0);
std::vector<int> v; std::vector<int> v;
for(int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
v.push_back(rand()); v.push_back(rand());
Foo foo; Foo foo;
foo.a = v[i]; foo.a = v[i];
tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
} }
for(int i = 0; i < 500; i++) { for (int i = 0; i < 500; i++) {
tqHandleCommit(pMeta, i); tqHandleCommit(pMeta, i);
Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
@ -204,38 +184,34 @@ TEST_F(TqMetaUpdateAppendTest, multiplePage) {
} }
tqStoreClose(pMeta); tqStoreClose(pMeta);
pMeta = tqStoreOpen(pathName, pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
FooSerializer, FooDeserializer, FooDeleter,
TQ_UPDATE_APPEND
);
ASSERT(pMeta); ASSERT(pMeta);
for(int i = 500; i < 1000; i++) { for (int i = 500; i < 1000; i++) {
tqHandleCommit(pMeta, i); tqHandleCommit(pMeta, i);
Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
EXPECT_EQ(pFoo->a, v[i]); EXPECT_EQ(pFoo->a, v[i]);
} }
for(int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
EXPECT_EQ(pFoo->a, v[i]); EXPECT_EQ(pFoo->a, v[i]);
} }
} }
TEST_F(TqMetaUpdateAppendTest, multipleRewrite) { TEST_F(TqMetaUpdateAppendTest, multipleRewrite) {
srand(0); srand(0);
std::vector<int> v; std::vector<int> v;
for(int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
v.push_back(rand()); v.push_back(rand());
Foo foo; Foo foo;
foo.a = v[i]; foo.a = v[i];
tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
} }
for(int i = 0; i < 500; i++) { for (int i = 0; i < 500; i++) {
tqHandleCommit(pMeta, i); tqHandleCommit(pMeta, i);
v[i] = rand(); v[i] = rand();
Foo foo; Foo foo;
@ -243,25 +219,22 @@ TEST_F(TqMetaUpdateAppendTest, multipleRewrite) {
tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
} }
for(int i = 500; i < 1000; i++) { for (int i = 500; i < 1000; i++) {
v[i] = rand(); v[i] = rand();
Foo foo; Foo foo;
foo.a = v[i]; foo.a = v[i];
tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
} }
for(int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
tqHandleCommit(pMeta, i); tqHandleCommit(pMeta, i);
} }
tqStoreClose(pMeta); tqStoreClose(pMeta);
pMeta = tqStoreOpen(pathName, pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
FooSerializer, FooDeserializer, FooDeleter,
TQ_UPDATE_APPEND
);
ASSERT(pMeta); ASSERT(pMeta);
for(int i = 500; i < 1000; i++) { for (int i = 500; i < 1000; i++) {
v[i] = rand(); v[i] = rand();
Foo foo; Foo foo;
foo.a = v[i]; foo.a = v[i];
@ -269,40 +242,38 @@ TEST_F(TqMetaUpdateAppendTest, multipleRewrite) {
tqHandleCommit(pMeta, i); tqHandleCommit(pMeta, i);
} }
for(int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
EXPECT_EQ(pFoo->a, v[i]); EXPECT_EQ(pFoo->a, v[i]);
} }
} }
TEST_F(TqMetaUpdateAppendTest, dupCommit) { TEST_F(TqMetaUpdateAppendTest, dupCommit) {
srand(0); srand(0);
std::vector<int> v; std::vector<int> v;
for(int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
v.push_back(rand()); v.push_back(rand());
Foo foo; Foo foo;
foo.a = v[i]; foo.a = v[i];
tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
} }
for(int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
int ret = tqHandleCommit(pMeta, i); int ret = tqHandleCommit(pMeta, i);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
ret = tqHandleCommit(pMeta, i); ret = tqHandleCommit(pMeta, i);
EXPECT_EQ(ret, -1); EXPECT_EQ(ret, -1);
} }
for(int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
int ret = tqHandleCommit(pMeta, i); int ret = tqHandleCommit(pMeta, i);
EXPECT_EQ(ret, -1); EXPECT_EQ(ret, -1);
} }
for(int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
EXPECT_EQ(pFoo->a, v[i]); EXPECT_EQ(pFoo->a, v[i]);
} }
} }

4
source/libs/catalog/test/catalogTests.cpp
View File

@ -61,8 +61,8 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) {
pReq->daysToKeep0 = htonl(3650); pReq->daysToKeep0 = htonl(3650);
pReq->daysToKeep1 = htonl(3650); pReq->daysToKeep1 = htonl(3650);
pReq->daysToKeep2 = htonl(3650); pReq->daysToKeep2 = htonl(3650);
pReq->minRowsPerFileBlock = htonl(100); pReq->minRows = htonl(100);
pReq->maxRowsPerFileBlock = htonl(4096); pReq->maxRows = htonl(4096);
pReq->commitTime = htonl(3600); pReq->commitTime = htonl(3600);
pReq->fsyncPeriod = htonl(3000); pReq->fsyncPeriod = htonl(3000);
pReq->walLevel = 1; pReq->walLevel = 1;

2
source/libs/index/inc/indexInt.h
View File

@ -76,7 +76,7 @@ struct SIndexMultiTermQuery {
// field and key; // field and key;
typedef struct SIndexTerm { typedef struct SIndexTerm {
int64_t suid; int64_t suid;
SIndexOperOnColumn operType; // oper type, add/del/update SIndexOperOnColumn operType; // oper type, add/del/update
uint8_t colType; // term data type, str/interger/json uint8_t colType; // term data type, str/interger/json
char *colName; char *colName;

14
source/libs/index/inc/index_fst_counting_writer.h
View File

@ -34,8 +34,14 @@ typedef struct WriterCtx {
int (*flush)(struct WriterCtx *ctx); int (*flush)(struct WriterCtx *ctx);
WriterType type; WriterType type;
union { union {
int fd; struct {
void *mem; int fd;
bool readOnly;
} file;
struct {
int32_t capa;
char *buf;
} mem;
}; };
int32_t offset; int32_t offset;
int32_t limit; int32_t limit;
@ -45,7 +51,7 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len);
static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len); static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len);
static int writeCtxDoFlush(WriterCtx *ctx); static int writeCtxDoFlush(WriterCtx *ctx);
WriterCtx* writerCtxCreate(WriterType type, bool readOnly); WriterCtx* writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity);
void writerCtxDestroy(WriterCtx *w); void writerCtxDestroy(WriterCtx *w);
typedef uint32_t CheckSummer; typedef uint32_t CheckSummer;
@ -66,7 +72,7 @@ int fstCountingWriterFlush(FstCountingWriter *write);
uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write); uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write);
FstCountingWriter *fstCountingWriterCreate(void *wtr, bool readOnly); FstCountingWriter *fstCountingWriterCreate(void *wtr);
void fstCountingWriterDestroy(FstCountingWriter *w); void fstCountingWriterDestroy(FstCountingWriter *w);

62
source/libs/index/inc/index_tfile.h
View File

@ -18,23 +18,81 @@
#include "index.h" #include "index.h"
#include "indexInt.h" #include "indexInt.h"
#include "tlockfree.h" #include "tlockfree.h"
#include "tskiplist.h" #include "index_tfile.h"
#include "index_fst_counting_writer.h"
#include "index_fst.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
typedef struct IndexTFile {
typedef struct TFileCacheKey {
uint64_t suid;
uint8_t colType;
int32_t version;
const char *colName;
int32_t nColName;
} TFileCacheKey;
// table cache
// refactor to LRU cache later
typedef struct TFileCache {
SHashObj *tableCache;
int16_t capacity;
// add more param
} TFileCache;
typedef struct TFileWriter {
FstBuilder *fb;
WriterCtx *wc;
} TFileWriter;
typedef struct TFileReader {
T_REF_DECLARE() T_REF_DECLARE()
Fst *fst;
} TFileReader;
typedef struct IndexTFile {
char *path;
TFileReader *tb;
TFileWriter *tw;
} IndexTFile; } IndexTFile;
typedef struct TFileWriterOpt {
uint64_t suid;
int8_t colType;
char *colName;
int32_t nColName;
int32_t version;
} TFileWriterOpt;
typedef struct TFileReaderOpt {
uint64_t suid;
char *colName;
int32_t nColName;
} TFileReaderOpt;
// tfile cache
TFileCache *tfileCacheCreate();
void tfileCacheDestroy(TFileCache *tcache);
TFileReader* tfileCacheGet(TFileCache *tcache, TFileCacheKey *key);
void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader);
TFileWriter *tfileWriterCreate(const char *suid, const char *colName);
IndexTFile *indexTFileCreate(); IndexTFile *indexTFileCreate();
int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid);
int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result); int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result);
#ifdef __cplusplus #ifdef __cplusplus
} }

53
source/libs/index/inc/index_util.h Normal file
View File

@ -0,0 +1,53 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __INDEX_UTIL_H__
#define __INDEX_UTIL_H__
#ifdef __cplusplus
extern "C" {
#endif
#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \
do { \
memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \
buf += sizeof(key->mem); \
} while (0)
#define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \
do { \
memcpy((void *)buf, (void *)key->mem, len); \
buf += len; \
} while (0)
#define SERIALIZE_VAR_TO_BUF(buf, var, type) \
do { \
type c = var; \
assert(sizeof(var) == sizeof(type));\
memcpy((void *)buf, (void *)&c, sizeof(c)); \
buf += sizeof(c); \
} while (0)
#define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \
do { \
memcpy((void *)buf, (void *)var, len); \
buf += len;\
} while (0)
#ifdef __cplusplus
}
#endif
#endif

30
source/libs/index/src/index_cache.c
View File

@ -15,6 +15,7 @@
#include "index_cache.h" #include "index_cache.h"
#include "tcompare.h" #include "tcompare.h"
#include "index_util.h"
#define MAX_INDEX_KEY_LEN 256// test only, change later #define MAX_INDEX_KEY_LEN 256// test only, change later
@ -110,35 +111,22 @@ int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version,
if (cache == NULL) { return -1;} if (cache == NULL) { return -1;}
IndexCache *pCache = cache; IndexCache *pCache = cache;
// encode data // encode data
int32_t total = CACHE_KEY_LEN(term); int32_t total = CACHE_KEY_LEN(term);
char *buf = calloc(1, total); char *buf = calloc(1, total);
char *p = buf; char *p = buf;
memcpy(p, &total, sizeof(total)); SERIALIZE_VAR_TO_BUF(p, total,int32_t);
p += sizeof(total); SERIALIZE_VAR_TO_BUF(p, colId, int16_t);
memcpy(p, &colId, sizeof(colId)); SERIALIZE_MEM_TO_BUF(p, term, colType);
p += sizeof(colId); SERIALIZE_MEM_TO_BUF(p, term, nColVal);
SERIALIZE_STR_MEM_TO_BUF(p, term, colVal, term->nColVal);
memcpy(p, &term->colType, sizeof(term->colType)); SERIALIZE_VAR_TO_BUF(p, version, int32_t);
p += sizeof(term->colType); SERIALIZE_VAR_TO_BUF(p, uid, uint64_t);
memcpy(p, &term->nColVal, sizeof(term->nColVal)); SERIALIZE_MEM_TO_BUF(p, term, operType);
p += sizeof(term->nColVal);
memcpy(p, term->colVal, term->nColVal);
p += term->nColVal;
memcpy(p, &version, sizeof(version));
p += sizeof(version);
memcpy(p, &uid, sizeof(uid));
p += sizeof(uid);
memcpy(p, &term->operType, sizeof(term->operType));
p += sizeof(term->operType);
tSkipListPut(pCache->skiplist, (void *)buf); tSkipListPut(pCache->skiplist, (void *)buf);
return 0; return 0;

2
source/libs/index/src/index_fst.c
View File

@ -779,7 +779,7 @@ FstBuilder *fstBuilderCreate(void *w, FstType ty) {
if (NULL == b) { return b; } if (NULL == b) { return b; }
b->wrt = fstCountingWriterCreate(w, false); b->wrt = fstCountingWriterCreate(w);
b->unfinished = fstUnFinishedNodesCreate(); b->unfinished = fstUnFinishedNodesCreate();
b->registry = fstRegistryCreate(10000, 2) ; b->registry = fstRegistryCreate(10000, 2) ;
b->last = fstSliceCreate(NULL, 0); b->last = fstSliceCreate(NULL, 0);

38
source/libs/index/src/index_fst_counting_writer.c
View File

@ -23,9 +23,9 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) {
} }
if (ctx->type == TFile) { if (ctx->type == TFile) {
assert(len == tfWrite(ctx->fd, buf, len)); assert(len == tfWrite(ctx->file.fd, buf, len));
} else { } else {
memcpy(ctx->mem + ctx->offset, buf, len); memcpy(ctx->mem.buf+ ctx->offset, buf, len);
} }
ctx->offset += len; ctx->offset += len;
return len; return len;
@ -33,9 +33,9 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) {
static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) { static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) {
int nRead = 0; int nRead = 0;
if (ctx->type == TFile) { if (ctx->type == TFile) {
nRead = tfRead(ctx->fd, buf, len); nRead = tfRead(ctx->file.fd, buf, len);
} else { } else {
memcpy(buf, ctx->mem + ctx->offset, len); memcpy(buf, ctx->mem.buf + ctx->offset, len);
} }
ctx->offset += nRead; ctx->offset += nRead;
@ -44,63 +44,64 @@ static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) {
static int writeCtxDoFlush(WriterCtx *ctx) { static int writeCtxDoFlush(WriterCtx *ctx) {
if (ctx->type == TFile) { if (ctx->type == TFile) {
//tfFsync(ctx->fd); //tfFsync(ctx->fd);
//tfFlush(ctx->fd); //tfFlush(ctx->file.fd);
} else { } else {
// do nothing // do nothing
} }
return 1; return 1;
} }
WriterCtx* writerCtxCreate(WriterType type, bool readOnly) { WriterCtx* writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity) {
WriterCtx *ctx = calloc(1, sizeof(WriterCtx)); WriterCtx *ctx = calloc(1, sizeof(WriterCtx));
if (ctx == NULL) { return NULL; } if (ctx == NULL) { return NULL; }
ctx->type = type; ctx->type = type;
if (ctx->type == TFile) { if (ctx->type == TFile) {
tfInit();
// ugly code, refactor later // ugly code, refactor later
ctx->file.readOnly = readOnly;
if (readOnly == false) { if (readOnly == false) {
ctx->fd = tfOpenCreateWriteAppend(tmpFile); ctx->file.fd = tfOpenCreateWriteAppend(tmpFile);
} else { } else {
ctx->fd = tfOpenReadWrite(tmpFile); ctx->file.fd = tfOpenReadWrite(tmpFile);
} }
if (ctx->fd < 0) { if (ctx->file.fd < 0) {
indexError("open file error %d", errno); indexError("open file error %d", errno);
} }
} else if (ctx->type == TMemory) { } else if (ctx->type == TMemory) {
ctx->mem = calloc(1, DefaultMem * sizeof(uint8_t)); ctx->mem.buf = calloc(1, sizeof(char) * capacity);
ctx->mem.capa = capacity;
} }
ctx->write = writeCtxDoWrite; ctx->write = writeCtxDoWrite;
ctx->read = writeCtxDoRead; ctx->read = writeCtxDoRead;
ctx->flush = writeCtxDoFlush; ctx->flush = writeCtxDoFlush;
ctx->offset = 0; ctx->offset = 0;
ctx->limit = DefaultMem; ctx->limit = capacity;
return ctx; return ctx;
} }
void writerCtxDestroy(WriterCtx *ctx) { void writerCtxDestroy(WriterCtx *ctx) {
if (ctx->type == TMemory) { if (ctx->type == TMemory) {
free(ctx->mem); free(ctx->mem.buf);
} else { } else {
tfClose(ctx->fd); tfClose(ctx->file.fd);
tfCleanup();
} }
free(ctx); free(ctx);
} }
FstCountingWriter *fstCountingWriterCreate(void *wrt, bool readOnly) { FstCountingWriter *fstCountingWriterCreate(void *wrt) {
FstCountingWriter *cw = calloc(1, sizeof(FstCountingWriter)); FstCountingWriter *cw = calloc(1, sizeof(FstCountingWriter));
if (cw == NULL) { return NULL; } if (cw == NULL) { return NULL; }
cw->wrt = (void *)(writerCtxCreate(TFile, readOnly)); cw->wrt = wrt;
//(void *)(writerCtxCreate(TFile, readOnly));
return cw; return cw;
} }
void fstCountingWriterDestroy(FstCountingWriter *cw) { void fstCountingWriterDestroy(FstCountingWriter *cw) {
// free wrt object: close fd or free mem // free wrt object: close fd or free mem
fstCountingWriterFlush(cw); fstCountingWriterFlush(cw);
writerCtxDestroy((WriterCtx *)(cw->wrt)); //writerCtxDestroy((WriterCtx *)(cw->wrt));
free(cw); free(cw);
} }
@ -124,6 +125,7 @@ int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len)
} }
uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write) { uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write) {
return 0; return 0;
} }
int fstCountingWriterFlush(FstCountingWriter *write) { int fstCountingWriterFlush(FstCountingWriter *write) {

58
source/libs/index/src/index_tfile.c
View File

@ -14,6 +14,50 @@
*/ */
#include "index_tfile.h" #include "index_tfile.h"
#include "index_fst.h"
#include "index_util.h"
static void tfileSerialCacheKey(TFileCacheKey *key, char *buf) {
SERIALIZE_MEM_TO_BUF(buf, key, suid);
SERIALIZE_VAR_TO_BUF(buf, '_', char);
SERIALIZE_MEM_TO_BUF(buf, key, colType);
SERIALIZE_VAR_TO_BUF(buf, '_', char);
SERIALIZE_MEM_TO_BUF(buf, key, version);
SERIALIZE_VAR_TO_BUF(buf, '_', char);
SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName);
}
TFileCache *tfileCacheCreate() {
TFileCache *tcache = calloc(1, sizeof(TFileCache));
if (tcache == NULL) { return NULL; }
tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
tcache->capacity = 64;
return tcache;
}
void tfileCacheDestroy(TFileCache *tcache) {
free(tcache);
}
TFileReader *tfileCacheGet(TFileCache *tcache, TFileCacheKey *key) {
char buf[128] = {0};
tfileSerialCacheKey(key, buf);
TFileReader *reader = taosHashGet(tcache->tableCache, buf, strlen(buf));
return reader;
}
void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader) {
char buf[128] = {0};
tfileSerialCacheKey(key, buf);
taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void *));
return;
}
IndexTFile *indexTFileCreate() { IndexTFile *indexTFileCreate() {
IndexTFile *tfile = calloc(1, sizeof(IndexTFile)); IndexTFile *tfile = calloc(1, sizeof(IndexTFile));
@ -22,8 +66,22 @@ IndexTFile *indexTFileCreate() {
void IndexTFileDestroy(IndexTFile *tfile) { void IndexTFileDestroy(IndexTFile *tfile) {
free(tfile); free(tfile);
} }
int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result) { int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result) {
IndexTFile *ptfile = (IndexTFile *)tfile; IndexTFile *ptfile = (IndexTFile *)tfile;
return 0;
}
int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid) {
TFileWriterOpt wOpt = {.suid = term->suid,
.colType = term->colType,
.colName = term->colName,
.nColName= term->nColName,
.version = 1};
return 0; return 0;
} }

2
source/libs/index/test/indexTests.cc
View File

@ -45,7 +45,7 @@ class FstWriter {
class FstReadMemory { class FstReadMemory {
public: public:
FstReadMemory(size_t size) { FstReadMemory(size_t size) {
_w = fstCountingWriterCreate(NULL, true); _w = fstCountingWriterCreate(NULL);
_size = size; _size = size;
memset((void *)&_s, 0, sizeof(_s)); memset((void *)&_s, 0, sizeof(_s));
} }

3
source/libs/parser/inc/astToMsg.h
View File

@ -4,7 +4,8 @@
#include "parserInt.h" #include "parserInt.h"
#include "taosmsg.h" #include "taosmsg.h"
SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen); SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen);
SDropUserMsg* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen);
SShowMsg* buildShowMsg(SShowInfo* pShowInfo, int64_t id, char* msgBuf, int32_t msgLen); SShowMsg* buildShowMsg(SShowInfo* pShowInfo, int64_t id, char* msgBuf, int32_t msgLen);
SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, char* msgBuf, int32_t msgLen); SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, char* msgBuf, int32_t msgLen);

27
source/libs/parser/src/astToMsg.c
View File

@ -1,7 +1,7 @@
#include "parserInt.h" #include "parserInt.h"
#include "parserUtil.h" #include "parserUtil.h"
SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen) { SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen) {
SCreateUserMsg* pMsg = (SCreateUserMsg*)calloc(1, sizeof(SCreateUserMsg)); SCreateUserMsg* pMsg = (SCreateUserMsg*)calloc(1, sizeof(SCreateUserMsg));
if (pMsg == NULL) { if (pMsg == NULL) {
// tscError("0x%" PRIx64 " failed to malloc for query msg", id); // tscError("0x%" PRIx64 " failed to malloc for query msg", id);
@ -20,6 +20,24 @@ SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgB
strncpy(pMsg->pass, pUser->passwd.z, pUser->passwd.n); strncpy(pMsg->pass, pUser->passwd.z, pUser->passwd.n);
} }
*outputLen = sizeof(SUserInfo);
return pMsg;
}
SDropUserMsg* buildDropUserMsg(SSqlInfo* pInfo, int32_t *msgLen, int64_t id, char* msgBuf, int32_t msgBufLen) {
SToken* pName = taosArrayGet(pInfo->pMiscInfo->a, 0);
if (pName->n >= TSDB_USER_LEN) {
return NULL;
}
SDropUserMsg* pMsg = calloc(1, sizeof(SDropUserMsg));
if (pMsg == NULL) {
return NULL;
}
strncpy(pMsg->user, pName->z, pName->n);
*msgLen = sizeof(SDropUserMsg);
return pMsg; return pMsg;
} }
@ -89,7 +107,7 @@ static int32_t setTimePrecision(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreate
pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default
SToken* pToken = &pCreateDbInfo->precision; SToken* pToken = (SToken*) &pCreateDbInfo->precision;
if (pToken->n > 0) { if (pToken->n > 0) {
pToken->n = strdequote(pToken->z); pToken->n = strdequote(pToken->z);
@ -116,8 +134,8 @@ static void doSetDbOptions(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDb) {
pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks); pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks);
pMsg->daysPerFile = htonl(pCreateDb->daysPerFile); pMsg->daysPerFile = htonl(pCreateDb->daysPerFile);
pMsg->commitTime = htonl((int32_t)pCreateDb->commitTime); pMsg->commitTime = htonl((int32_t)pCreateDb->commitTime);
pMsg->minRowsPerFileBlock = htonl(pCreateDb->minRowsPerBlock); pMsg->minRows = htonl(pCreateDb->minRowsPerBlock);
pMsg->maxRowsPerFileBlock = htonl(pCreateDb->maxRowsPerBlock); pMsg->maxRows = htonl(pCreateDb->maxRowsPerBlock);
pMsg->fsyncPeriod = htonl(pCreateDb->fsyncPeriod); pMsg->fsyncPeriod = htonl(pCreateDb->fsyncPeriod);
pMsg->compression = pCreateDb->compressionLevel; pMsg->compression = pCreateDb->compressionLevel;
pMsg->walLevel = (char)pCreateDb->walLevel; pMsg->walLevel = (char)pCreateDb->walLevel;
@ -141,7 +159,6 @@ int32_t setDbOptions(SCreateDbMsg* pCreateDbMsg, const SCreateDbInfo* pCreateDbS
// todo configurable // todo configurable
pCreateDbMsg->numOfVgroups = htonl(2); pCreateDbMsg->numOfVgroups = htonl(2);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

8
source/libs/parser/src/astValidate.c
View File

@ -4224,7 +4224,13 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, in
} }
} }
*output = buildUserManipulationMsg(pInfo, id, msgBuf, msgBufLen); *output = buildUserManipulationMsg(pInfo, outputLen, id, msgBuf, msgBufLen);
break;
}
case TSDB_SQL_DROP_ACCT:
case TSDB_SQL_DROP_USER: {
*output = buildDropUserMsg(pInfo, outputLen, id, msgBuf, msgBufLen);
break; break;
} }

7
source/libs/parser/src/queryInfoUtil.c
View File

@ -1,10 +1,7 @@
#include "queryInfoUtil.h"
#include <function.h>
#include "astGenerator.h"
#include "function.h"
#include "os.h" #include "os.h"
#include "queryInfoUtil.h"
#include "function.h"
#include "parser.h" #include "parser.h"
#include "parserInt.h"
#include "parserUtil.h" #include "parserUtil.h"
static struct SSchema _s = { static struct SSchema _s = {

6
source/libs/parser/test/mockCatalogService.cpp
View File

@ -38,7 +38,7 @@ public:
virtual TableBuilder& setVgid(int16_t vgid) { virtual TableBuilder& setVgid(int16_t vgid) {
schema()->vgId = vgid; schema()->vgId = vgid;
meta_->vgs.emplace_back(SVgroupInfo{.vgId = vgid, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); meta_->vgs.emplace_back(SVgroupInfo{.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}});
return *this; return *this;
} }
@ -118,9 +118,9 @@ public:
meta_[db][tbname].reset(new MockTableMeta()); meta_[db][tbname].reset(new MockTableMeta());
meta_[db][tbname]->schema.reset(table.release()); meta_[db][tbname]->schema.reset(table.release());
meta_[db][tbname]->schema->uid = id_++; meta_[db][tbname]->schema->uid = id_++;
meta_[db][tbname]->vgs.emplace_back(SVgroupInfo{.vgId = vgid, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); meta_[db][tbname]->vgs.emplace_back((SVgroupInfo){.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}});
// super table // super table
meta_[db][stbname]->vgs.emplace_back(SVgroupInfo{.vgId = vgid, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); meta_[db][stbname]->vgs.emplace_back((SVgroupInfo){.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}});
} }
void showTables() const { void showTables() const {

2
source/libs/qcom/src/querymsg.c
View File

@ -294,7 +294,7 @@ void msgInit() {
tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg; tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg;
tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg; tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg;
tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg; tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg;
tscBuildMsg[TSDB_SQL_UPDATE_TAGS_VAL] = tscBuildUpdateTagMsg; tscBuildMsg[TSDB_SQL_UPDATE_TAG_VAL] = tscBuildUpdateTagMsg;
tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg; tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg;
tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg; tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg;

2
source/libs/scheduler/CMakeLists.txt
View File

@ -11,3 +11,5 @@ target_link_libraries(
scheduler scheduler
PRIVATE os util planner qcom common catalog transport PRIVATE os util planner qcom common catalog transport
) )
ADD_SUBDIRECTORY(test)

5
source/libs/scheduler/inc/schedulerInt.h
View File

@ -73,9 +73,8 @@ typedef struct SQueryJob {
SQueryProfileSummary summary; SQueryProfileSummary summary;
SEpSet dataSrcEps; SEpSet dataSrcEps;
SEpAddr resEp; SEpAddr resEp;
struct SCatalog *catalog; void *transport;
void *rpc; SArray *qnodeList;
SEpSet *mgmtEpSet;
tsem_t rspSem; tsem_t rspSem;
int32_t userFetch; int32_t userFetch;
int32_t remoteFetch; int32_t remoteFetch;

65
source/libs/scheduler/src/scheduler.c
View File

@ -58,8 +58,8 @@ int32_t schBuildTaskRalation(SQueryJob *job, SHashObj *planToTask) {
for (int32_t m = 0; m < level->taskNum; ++m) { for (int32_t m = 0; m < level->taskNum; ++m) {
SQueryTask *task = taosArrayGet(level->subTasks, m); SQueryTask *task = taosArrayGet(level->subTasks, m);
SSubplan *plan = task->plan; SSubplan *plan = task->plan;
int32_t childNum = (int32_t)taosArrayGetSize(plan->pChildern); int32_t childNum = plan->pChildern ? (int32_t)taosArrayGetSize(plan->pChildern) : 0;
int32_t parentNum = (int32_t)taosArrayGetSize(plan->pParents); int32_t parentNum = plan->pParents ? (int32_t)taosArrayGetSize(plan->pParents) : 0;
if (childNum > 0) { if (childNum > 0) {
task->children = taosArrayInit(childNum, POINTER_BYTES); task->children = taosArrayInit(childNum, POINTER_BYTES);
@ -187,13 +187,19 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) {
for (int32_t n = 0; n < levelPlanNum; ++n) { for (int32_t n = 0; n < levelPlanNum; ++n) {
SSubplan *plan = taosArrayGet(levelPlans, n); SSubplan *plan = taosArrayGet(levelPlans, n);
SQueryTask *task = taosArrayGet(level.subTasks, n); SQueryTask task = {0};
task->taskId = atomic_add_fetch_64(&schMgmt.taskId, 1); task.taskId = atomic_add_fetch_64(&schMgmt.taskId, 1);
task->plan = plan; task.plan = plan;
task->status = SCH_STATUS_NOT_START; task.status = SCH_STATUS_NOT_START;
if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &task, POINTER_BYTES)) { void *p = taosArrayPush(level.subTasks, &task);
if (NULL == p) {
qError("taosArrayPush failed");
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &p, POINTER_BYTES)) {
qError("taosHashPut failed"); qError("taosHashPut failed");
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
@ -225,20 +231,27 @@ _return:
SCH_RET(code); SCH_RET(code);
} }
int32_t schAvailableEpSet(SQueryJob *job, SEpSet *epSet) { int32_t schSetTaskExecEpSet(SQueryJob *job, SEpSet *epSet) {
if (epSet->numOfEps >= SCH_MAX_CONDIDATE_EP_NUM) { if (epSet->numOfEps >= SCH_MAX_CONDIDATE_EP_NUM) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (SCH_HAS_QNODE_IN_CLUSTER(schMgmt.cfg.clusterType)) { int32_t qnodeNum = taosArrayGetSize(job->qnodeList);
SCH_ERR_RET(catalogGetQnodeList(job->catalog, job->rpc, job->mgmtEpSet, epSet));
} else {
for (int32_t i = 0; i < job->dataSrcEps.numOfEps; ++i) {
strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i]));
epSet->port[epSet->numOfEps] = job->dataSrcEps.port[i];
++epSet->numOfEps; for (int32_t i = 0; i < qnodeNum && epSet->numOfEps < tListLen(epSet->port); ++i) {
} SEpAddr *addr = taosArrayGet(job->qnodeList, i);
strncpy(epSet->fqdn[epSet->numOfEps], addr->fqdn, sizeof(addr->fqdn));
epSet->port[epSet->numOfEps] = addr->port;
++epSet->numOfEps;
}
for (int32_t i = 0; i < job->dataSrcEps.numOfEps && epSet->numOfEps < tListLen(epSet->port); ++i) {
strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i]));
epSet->port[epSet->numOfEps] = job->dataSrcEps.port[i];
++epSet->numOfEps;
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -509,7 +522,12 @@ int32_t schLaunchTask(SQueryJob *job, SQueryTask *task) {
SCH_ERR_RET(qSubPlanToString(plan, &task->msg)); SCH_ERR_RET(qSubPlanToString(plan, &task->msg));
if (plan->execEpSet.numOfEps <= 0) { if (plan->execEpSet.numOfEps <= 0) {
SCH_ERR_RET(schAvailableEpSet(job, &plan->execEpSet)); SCH_ERR_RET(schSetTaskExecEpSet(job, &plan->execEpSet));
}
if (plan->execEpSet.numOfEps <= 0) {
SCH_TASK_ERR_LOG("invalid execEpSet num:%d", plan->execEpSet.numOfEps);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
} }
SCH_ERR_RET(schAsyncSendMsg(job, task, TSDB_MSG_TYPE_QUERY)); SCH_ERR_RET(schAsyncSendMsg(job, task, TSDB_MSG_TYPE_QUERY));
@ -548,20 +566,23 @@ int32_t schedulerInit(SSchedulerCfg *cfg) {
} }
int32_t scheduleQueryJob(struct SCatalog *pCatalog, void *pRpc, const SEpSet* pMgmtEps, SQueryDag* pDag, void** pJob) { int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob) {
if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { if (NULL == transport || NULL == transport ||NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) {
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
if (taosArrayGetSize(qnodeList) <= 0) {
qInfo("qnodeList is empty");
}
int32_t code = 0; int32_t code = 0;
SQueryJob *job = calloc(1, sizeof(SQueryJob)); SQueryJob *job = calloc(1, sizeof(SQueryJob));
if (NULL == job) { if (NULL == job) {
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
job->catalog = pCatalog; job->transport = transport;
job->rpc = pRpc; job->qnodeList = qnodeList;
job->mgmtEpSet = (SEpSet *)pMgmtEps;
SCH_ERR_JRET(schValidateAndBuildJob(pDag, job)); SCH_ERR_JRET(schValidateAndBuildJob(pDag, job));

18
source/libs/scheduler/test/CMakeLists.txt Normal file
View File

@ -0,0 +1,18 @@
MESSAGE(STATUS "build scheduler unit test")
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(schedulerTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
schedulerTest
PUBLIC os util common catalog transport gtest qcom taos planner scheduler
)
TARGET_INCLUDE_DIRECTORIES(
schedulerTest
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scheduler/"
PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/scheduler/inc"
)

104
source/libs/scheduler/test/schedulerTests.cpp
View File

@ -0,0 +1,104 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gtest/gtest.h>
#include <tglobal.h>
#include <iostream>
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#include "os.h"
#include "taos.h"
#include "tdef.h"
#include "tvariant.h"
#include "catalog.h"
#include "scheduler.h"
#include "tep.h"
#include "trpc.h"
namespace {
void mockBuildDag(SQueryDag *dag) {
uint64_t qId = 0x111111111111;
dag->queryId = qId;
dag->numOfSubplans = 2;
dag->pSubplans = taosArrayInit(dag->numOfSubplans, POINTER_BYTES);
SArray *scan = taosArrayInit(1, sizeof(SSubplan));
SArray *merge = taosArrayInit(1, sizeof(SSubplan));
SSubplan scanPlan = {0};
SSubplan mergePlan = {0};
scanPlan.id.queryId = qId;
scanPlan.id.templateId = 0x2222222222;
scanPlan.id.subplanId = 0x3333333333;
scanPlan.type = QUERY_TYPE_SCAN;
scanPlan.level = 1;
scanPlan.execEpSet.numOfEps = 1;
scanPlan.pChildern = NULL;
scanPlan.pParents = taosArrayInit(1, POINTER_BYTES);
mergePlan.id.queryId = qId;
mergePlan.id.templateId = 0x4444444444;
mergePlan.id.subplanId = 0x5555555555;
mergePlan.type = QUERY_TYPE_MERGE;
mergePlan.level = 0;
mergePlan.execEpSet.numOfEps = 1;
mergePlan.pChildern = taosArrayInit(1, POINTER_BYTES);
mergePlan.pParents = NULL;
SSubplan *mergePointer = (SSubplan *)taosArrayPush(merge, &mergePlan);
SSubplan *scanPointer = (SSubplan *)taosArrayPush(scan, &scanPlan);
taosArrayPush(mergePointer->pChildern, &scanPointer);
taosArrayPush(scanPointer->pParents, &mergePointer);
taosArrayPush(dag->pSubplans, &merge);
taosArrayPush(dag->pSubplans, &scan);
}
}
TEST(testCase, normalCase) {
void *mockPointer = (void *)0x1;
char *clusterId = "cluster1";
char *dbname = "1.db1";
char *tablename = "table1";
SVgroupInfo vgInfo = {0};
void *pJob = NULL;
SQueryDag dag = {0};
SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr));
int32_t code = schedulerInit(NULL);
ASSERT_EQ(code, 0);
mockBuildDag(&dag);
code = scheduleExecJob(mockPointer, qnodeList, &dag, &pJob);
ASSERT_EQ(code, 0);
}
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

117
source/util/src/tlog.c
View File

@ -14,21 +14,21 @@
*/ */
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h"
#include "ulog.h"
#include "tlog.h" #include "tlog.h"
#include "os.h"
#include "tnote.h" #include "tnote.h"
#include "tutil.h" #include "tutil.h"
#include "ulog.h"
#include "zlib.h" #include "zlib.h"
#define MAX_LOGLINE_SIZE (1000) #define MAX_LOGLINE_SIZE (1000)
#define MAX_LOGLINE_BUFFER_SIZE (MAX_LOGLINE_SIZE + 10) #define MAX_LOGLINE_BUFFER_SIZE (MAX_LOGLINE_SIZE + 10)
#define MAX_LOGLINE_CONTENT_SIZE (MAX_LOGLINE_SIZE - 100) #define MAX_LOGLINE_CONTENT_SIZE (MAX_LOGLINE_SIZE - 100)
#define MAX_LOGLINE_DUMP_SIZE (65 * 1024) #define MAX_LOGLINE_DUMP_SIZE (65 * 1024)
#define MAX_LOGLINE_DUMP_BUFFER_SIZE (MAX_LOGLINE_DUMP_SIZE + 10) #define MAX_LOGLINE_DUMP_BUFFER_SIZE (MAX_LOGLINE_DUMP_SIZE + 10)
#define MAX_LOGLINE_DUMP_CONTENT_SIZE (MAX_LOGLINE_DUMP_SIZE - 100) #define MAX_LOGLINE_DUMP_CONTENT_SIZE (MAX_LOGLINE_DUMP_SIZE - 100)
#define LOG_FILE_NAME_LEN 300 #define LOG_FILE_NAME_LEN 300
#define TSDB_DEFAULT_LOG_BUF_SIZE (20 * 1024 * 1024) // 20MB #define TSDB_DEFAULT_LOG_BUF_SIZE (20 * 1024 * 1024) // 20MB
#define DEFAULT_LOG_INTERVAL 25 #define DEFAULT_LOG_INTERVAL 25
@ -38,13 +38,13 @@
#define LOG_MAX_WAIT_MSEC 1000 #define LOG_MAX_WAIT_MSEC 1000
#define LOG_BUF_BUFFER(x) ((x)->buffer) #define LOG_BUF_BUFFER(x) ((x)->buffer)
#define LOG_BUF_START(x) ((x)->buffStart) #define LOG_BUF_START(x) ((x)->buffStart)
#define LOG_BUF_END(x) ((x)->buffEnd) #define LOG_BUF_END(x) ((x)->buffEnd)
#define LOG_BUF_SIZE(x) ((x)->buffSize) #define LOG_BUF_SIZE(x) ((x)->buffSize)
#define LOG_BUF_MUTEX(x) ((x)->buffMutex) #define LOG_BUF_MUTEX(x) ((x)->buffMutex)
typedef struct { typedef struct {
char * buffer; char *buffer;
int32_t buffStart; int32_t buffStart;
int32_t buffEnd; int32_t buffEnd;
int32_t buffSize; int32_t buffSize;
@ -57,18 +57,18 @@ typedef struct {
} SLogBuff; } SLogBuff;
typedef struct { typedef struct {
int32_t fileNum; int32_t fileNum;
int32_t maxLines; int32_t maxLines;
int32_t lines; int32_t lines;
int32_t flag; int32_t flag;
int32_t openInProgress; int32_t openInProgress;
pid_t pid; pid_t pid;
char logName[LOG_FILE_NAME_LEN]; char logName[LOG_FILE_NAME_LEN];
SLogBuff * logHandle; SLogBuff *logHandle;
pthread_mutex_t logMutex; pthread_mutex_t logMutex;
} SLogObj; } SLogObj;
int8_t tscEmbeddedInUtil = 0; int8_t tscEmbeddedInUtil = 0;
int32_t tsLogKeepDays = 0; int32_t tsLogKeepDays = 0;
int8_t tsAsyncLog = 1; int8_t tsAsyncLog = 1;
@ -93,19 +93,19 @@ int32_t debugFlag = 0;
int32_t sDebugFlag = 135; int32_t sDebugFlag = 135;
int32_t wDebugFlag = 135; int32_t wDebugFlag = 135;
int32_t tsdbDebugFlag = 131; int32_t tsdbDebugFlag = 131;
int32_t tqDebugFlag = 131;
int32_t cqDebugFlag = 131; int32_t cqDebugFlag = 131;
int32_t fsDebugFlag = 135; int32_t fsDebugFlag = 135;
int32_t ctgDebugFlag = 131; int32_t ctgDebugFlag = 131;
int64_t dbgEmptyW = 0; int64_t dbgEmptyW = 0;
int64_t dbgWN = 0; int64_t dbgWN = 0;
int64_t dbgSmallWN = 0; int64_t dbgSmallWN = 0;
int64_t dbgBigWN = 0; int64_t dbgBigWN = 0;
int64_t dbgWSize = 0; int64_t dbgWSize = 0;
static SLogObj tsLogObj = { .fileNum = 1 }; static SLogObj tsLogObj = {.fileNum = 1};
static void * taosAsyncOutputLog(void *param); static void *taosAsyncOutputLog(void *param);
static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen); static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen);
static SLogBuff *taosLogBuffNew(int32_t bufSize); static SLogBuff *taosLogBuffNew(int32_t bufSize);
static void taosCloseLogByFd(int32_t oldFd); static void taosCloseLogByFd(int32_t oldFd);
@ -139,8 +139,8 @@ static void taosStopLog() {
void taosCloseLog() { void taosCloseLog() {
taosStopLog(); taosStopLog();
//tsem_post(&(tsLogObj.logHandle->buffNotEmpty)); // tsem_post(&(tsLogObj.logHandle->buffNotEmpty));
taosMsleep(MAX_LOG_INTERVAL/1000); taosMsleep(MAX_LOG_INTERVAL / 1000);
if (taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) { if (taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) {
pthread_join(tsLogObj.logHandle->asyncThread, NULL); pthread_join(tsLogObj.logHandle->asyncThread, NULL);
} }
@ -308,9 +308,9 @@ static void taosGetLogFileName(char *fn) {
static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
#ifdef WINDOWS #ifdef WINDOWS
/* /*
* always set maxFileNum to 1 * always set maxFileNum to 1
* means client log filename is unique in windows * means client log filename is unique in windows
*/ */
maxFileNum = 1; maxFileNum = 1;
#endif #endif
@ -381,13 +381,14 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) {
printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB); printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", tsLogDir, tsAvailLogDirGB,
tsTotalLogDirGB);
fflush(stdout); fflush(stdout);
return; return;
} }
va_list argpointer; va_list argpointer;
char buffer[MAX_LOGLINE_BUFFER_SIZE] = { 0 }; char buffer[MAX_LOGLINE_BUFFER_SIZE] = {0};
int32_t len; int32_t len;
struct tm Tm, *ptm; struct tm Tm, *ptm;
struct timeval timeSecs; struct timeval timeSecs;
@ -434,20 +435,20 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
} }
} }
if (dflag & DEBUG_SCREEN) if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len);
taosWriteFile(1, buffer, (uint32_t)len);
if (dflag == 255) nInfo(buffer, len); if (dflag == 255) nInfo(buffer, len);
} }
void taosDumpData(unsigned char *msg, int32_t len) { void taosDumpData(unsigned char *msg, int32_t len) {
if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) {
printf("server disk:%s space remain %.3f GB, total %.1f GB, stop dump log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB); printf("server disk:%s space remain %.3f GB, total %.1f GB, stop dump log.\n", tsLogDir, tsAvailLogDirGB,
tsTotalLogDirGB);
fflush(stdout); fflush(stdout);
return; return;
} }
char temp[256]; char temp[256];
int32_t i, pos = 0, c = 0; int32_t i, pos = 0, c = 0;
for (i = 0; i < len; ++i) { for (i = 0; i < len; ++i) {
sprintf(temp + pos, "%02x ", msg[i]); sprintf(temp + pos, "%02x ", msg[i]);
@ -468,7 +469,8 @@ void taosDumpData(unsigned char *msg, int32_t len) {
void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) { void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) {
if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) {
printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB); printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", tsLogDir, tsAvailLogDirGB,
tsTotalLogDirGB);
fflush(stdout); fflush(stdout);
return; return;
} }
@ -542,7 +544,7 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize) {
tLogBuff->stop = 0; tLogBuff->stop = 0;
if (pthread_mutex_init(&LOG_BUF_MUTEX(tLogBuff), NULL) < 0) goto _err; if (pthread_mutex_init(&LOG_BUF_MUTEX(tLogBuff), NULL) < 0) goto _err;
//tsem_init(&(tLogBuff->buffNotEmpty), 0, 0); // tsem_init(&(tLogBuff->buffNotEmpty), 0, 0);
return tLogBuff; return tLogBuff;
@ -576,12 +578,12 @@ static void taosCopyLogBuffer(SLogBuff *tLogBuff, int32_t start, int32_t end, ch
} }
static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) { static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) {
int32_t start = 0; int32_t start = 0;
int32_t end = 0; int32_t end = 0;
int32_t remainSize = 0; int32_t remainSize = 0;
static int64_t lostLine = 0; static int64_t lostLine = 0;
char tmpBuf[40] = {0}; char tmpBuf[40] = {0};
int32_t tmpBufLen = 0; int32_t tmpBufLen = 0;
if (tLogBuff == NULL || tLogBuff->stop) return -1; if (tLogBuff == NULL || tLogBuff->stop) return -1;
@ -592,7 +594,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen)
remainSize = (start > end) ? (start - end - 1) : (start + LOG_BUF_SIZE(tLogBuff) - end - 1); remainSize = (start > end) ? (start - end - 1) : (start + LOG_BUF_SIZE(tLogBuff) - end - 1);
if (lostLine > 0) { if (lostLine > 0) {
sprintf(tmpBuf, "...Lost %"PRId64" lines here...\n", lostLine); sprintf(tmpBuf, "...Lost %" PRId64 " lines here...\n", lostLine);
tmpBufLen = (int32_t)strlen(tmpBuf); tmpBufLen = (int32_t)strlen(tmpBuf);
} }
@ -610,7 +612,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen)
taosCopyLogBuffer(tLogBuff, LOG_BUF_START(tLogBuff), LOG_BUF_END(tLogBuff), msg, msgLen); taosCopyLogBuffer(tLogBuff, LOG_BUF_START(tLogBuff), LOG_BUF_END(tLogBuff), msg, msgLen);
//int32_t w = atomic_sub_fetch_32(&waitLock, 1); // int32_t w = atomic_sub_fetch_32(&waitLock, 1);
/* /*
if (w <= 0 || ((remainSize - msgLen - tmpBufLen) < (LOG_BUF_SIZE(tLogBuff) * 4 /5))) { if (w <= 0 || ((remainSize - msgLen - tmpBufLen) < (LOG_BUF_SIZE(tLogBuff) * 4 /5))) {
tsem_post(&(tLogBuff->buffNotEmpty)); tsem_post(&(tLogBuff->buffNotEmpty));
@ -622,7 +624,6 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen)
pthread_mutex_unlock(&LOG_BUF_MUTEX(tLogBuff)); pthread_mutex_unlock(&LOG_BUF_MUTEX(tLogBuff));
return 0; return 0;
} }
@ -634,8 +635,8 @@ static int32_t taosGetLogRemainSize(SLogBuff *tLogBuff, int32_t start, int32_t e
static void taosWriteLog(SLogBuff *tLogBuff) { static void taosWriteLog(SLogBuff *tLogBuff) {
static int32_t lastDuration = 0; static int32_t lastDuration = 0;
int32_t remainChecked = 0; int32_t remainChecked = 0;
int32_t start, end, pollSize; int32_t start, end, pollSize;
do { do {
if (remainChecked == 0) { if (remainChecked == 0) {
@ -662,24 +663,24 @@ static void taosWriteLog(SLogBuff *tLogBuff) {
if (start < end) { if (start < end) {
taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, pollSize);
} else { } else {
int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start; int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start;
taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, tsize); taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, tsize);
taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff), end); taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff), end);
} }
dbgWN++; dbgWN++;
dbgWSize+=pollSize; dbgWSize += pollSize;
if (pollSize < tLogBuff->minBuffSize) { if (pollSize < tLogBuff->minBuffSize) {
dbgSmallWN++; dbgSmallWN++;
if (writeInterval < MAX_LOG_INTERVAL) { if (writeInterval < MAX_LOG_INTERVAL) {
writeInterval += LOG_INTERVAL_STEP; writeInterval += LOG_INTERVAL_STEP;
} }
} else if (pollSize > LOG_BUF_SIZE(tLogBuff)/3) { } else if (pollSize > LOG_BUF_SIZE(tLogBuff) / 3) {
dbgBigWN++; dbgBigWN++;
writeInterval = MIN_LOG_INTERVAL; writeInterval = MIN_LOG_INTERVAL;
} else if (pollSize > LOG_BUF_SIZE(tLogBuff)/4) { } else if (pollSize > LOG_BUF_SIZE(tLogBuff) / 4) {
if (writeInterval > MIN_LOG_INTERVAL) { if (writeInterval > MIN_LOG_INTERVAL) {
writeInterval -= LOG_INTERVAL_STEP; writeInterval -= LOG_INTERVAL_STEP;
} }
@ -698,7 +699,7 @@ static void taosWriteLog(SLogBuff *tLogBuff) {
writeInterval = MIN_LOG_INTERVAL; writeInterval = MIN_LOG_INTERVAL;
remainChecked = 1; remainChecked = 1;
}while (1); } while (1);
} }
static void *taosAsyncOutputLog(void *param) { static void *taosAsyncOutputLog(void *param) {
@ -721,8 +722,8 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) {
int32_t compressSize = 163840; int32_t compressSize = 163840;
int32_t ret = 0; int32_t ret = 0;
int32_t len = 0; int32_t len = 0;
char * data = malloc(compressSize); char *data = malloc(compressSize);
FILE * srcFp = NULL; FILE *srcFp = NULL;
gzFile dstFp = NULL; gzFile dstFp = NULL;
srcFp = fopen(srcFileName, "r"); srcFp = fopen(srcFileName, "r");

20
source/util/src/tthread.c
View File

@ -13,16 +13,16 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "os.h"
#include "tthread.h" #include "tthread.h"
#include "os.h"
#include "taoserror.h"
#include "tdef.h" #include "tdef.h"
#include "tutil.h" #include "tutil.h"
#include "ulog.h" #include "ulog.h"
#include "taoserror.h"
// create new thread // create new thread
pthread_t* taosCreateThread( void *(*__start_routine) (void *), void* param) { pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param) {
pthread_t* pthread = (pthread_t*)malloc(sizeof(pthread_t)); pthread_t* pthread = (pthread_t*)malloc(sizeof(pthread_t));
pthread_attr_t thattr; pthread_attr_t thattr;
pthread_attr_init(&thattr); pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
@ -38,8 +38,8 @@ pthread_t* taosCreateThread( void *(*__start_routine) (void *), void* param) {
// destory thread // destory thread
bool taosDestoryThread(pthread_t* pthread) { bool taosDestoryThread(pthread_t* pthread) {
if(pthread == NULL) return false; if (pthread == NULL) return false;
if(taosThreadRunning(pthread)) { if (taosThreadRunning(pthread)) {
pthread_cancel(*pthread); pthread_cancel(*pthread);
pthread_join(*pthread, NULL); pthread_join(*pthread, NULL);
} }
@ -50,12 +50,10 @@ bool taosDestoryThread(pthread_t* pthread) {
// thread running return true // thread running return true
bool taosThreadRunning(pthread_t* pthread) { bool taosThreadRunning(pthread_t* pthread) {
if(pthread == NULL) return false; if (pthread == NULL) return false;
int ret = pthread_kill(*pthread, 0); int ret = pthread_kill(*pthread, 0);
if(ret == ESRCH) if (ret == ESRCH) return false;
return false; if (ret == EINVAL) return false;
if(ret == EINVAL)
return false;
// alive // alive
return true; return true;
} }

32
tests/script/general/user/monitor.sim
View File

@ -1,32 +0,0 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
print ========== step1
system sh/cfg.sh -n dnode1 -c walLevel -v 1
system sh/cfg.sh -n dnode1 -c monitor -v 1
system sh/cfg.sh -n dnode1 -c monitorInterval -v 1
system sh/exec.sh -n dnode1 -s start
sleep 2000
sql connect
print ========== step2
#sql drop database log -x step21
# return -1
#step21:
sql drop table log.dn -x step22
# return -1
step22:
sql drop user log -x step23
# return -1
step23:
print ========== step3
sleep 2000
#sql select * from log.dn
#if $rows == 0 then
# return -1
#endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT