xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #9130 from taosdata/feature/tq 

Feature/tq
This commit is contained in:
Liu Jicong 2021-12-16 15:05:51 +08:00 committed by GitHub
parent 39de3fe0a5 652c05c6e8
commit ce321beb6c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 876 additions and 908 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

183
include/dnode/vnode/tq/tq.h
View File

@ -16,9 +16,9 @@
#ifndef _TD_TQ_H_ #ifndef _TD_TQ_H_
#define _TD_TQ_H_ #define _TD_TQ_H_
#include "mallocator.h"
#include "os.h" #include "os.h"
#include "tutil.h" #include "tutil.h"
#include "mallocator.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -97,128 +97,125 @@ typedef struct TmqHeartbeatReq {
typedef struct TmqHeartbeatRsp { typedef struct TmqHeartbeatRsp {
} TmqHeartbeatRsp; } TmqHeartbeatRsp;
typedef struct TqTopicVhandle { typedef struct STqTopicVhandle {
int64_t topicId; int64_t topicId;
// executor for filter // executor for filter
void* filterExec; void* filterExec;
// callback for mnode // callback for mnode
// trigger when vnode list associated topic change // trigger when vnode list associated topic change
void* (*mCallback)(void*, void*); void* (*mCallback)(void*, void*);
} TqTopicVhandle; } STqTopicVhandle;
#define TQ_BUFFER_SIZE 8 #define TQ_BUFFER_SIZE 8
typedef struct TqBufferItem { typedef struct STqBufferItem {
int64_t offset; int64_t offset;
// executors are identical but not concurrent // executors are identical but not concurrent
// so there must be a copy in each item // so there must be a copy in each item
void* executor; void* executor;
int64_t size; int64_t size;
void* content; void* content;
} TqBufferItem; } STqBufferItem;
typedef struct TqBufferHandle { typedef struct STqBufferHandle {
// char* topic; //c style, end with '\0' // char* topic; //c style, end with '\0'
// int64_t cgId; // int64_t cgId;
// void* ahandle; // void* ahandle;
int64_t nextConsumeOffset; int64_t nextConsumeOffset;
int64_t topicId; int64_t floatingCursor;
int32_t head; int64_t topicId;
int32_t tail; int32_t head;
TqBufferItem buffer[TQ_BUFFER_SIZE]; int32_t tail;
} TqBufferHandle; STqBufferItem buffer[TQ_BUFFER_SIZE];
} STqBufferHandle;
typedef struct TqListHandle { typedef struct STqListHandle {
TqBufferHandle bufHandle; STqBufferHandle bufHandle;
struct TqListHandle* next; struct STqListHandle* next;
} TqListHandle; } STqListHandle;
typedef struct TqGroupHandle { typedef struct STqGroupHandle {
int64_t cId; int64_t cId;
int64_t cgId; int64_t cgId;
void* ahandle; void* ahandle;
int32_t topicNum; int32_t topicNum;
TqListHandle* head; STqListHandle* head;
} TqGroupHandle; } STqGroupHandle;
typedef struct TqQueryExec { typedef struct STqQueryExec {
void* src; void* src;
TqBufferItem* dest; STqBufferItem* dest;
void* executor; void* executor;
} TqQueryExec; } STqQueryExec;
typedef struct TqQueryMsg { typedef struct STqQueryMsg {
TqQueryExec* exec; STqQueryExec* exec;
struct TqQueryMsg* next; struct STqQueryMsg* next;
} TqQueryMsg; } STqQueryMsg;
typedef struct TqLogReader { typedef struct STqLogReader {
void* logHandle; void* logHandle;
int32_t (*logRead)(void* logHandle, void** data, int64_t ver); int32_t (*logRead)(void* logHandle, void** data, int64_t ver);
int64_t (*logGetFirstVer)(void* logHandle); int64_t (*logGetFirstVer)(void* logHandle);
int64_t (*logGetSnapshotVer)(void* logHandle); int64_t (*logGetSnapshotVer)(void* logHandle);
int64_t (*logGetLastVer)(void* logHandle); int64_t (*logGetLastVer)(void* logHandle);
} TqLogReader; } STqLogReader;
typedef struct STqCfg { typedef struct STqCfg {
// TODO // TODO
} STqCfg; } STqCfg;
typedef struct TqMemRef { typedef struct STqMemRef {
SMemAllocatorFactory *pAlloctorFactory; SMemAllocatorFactory* pAlloctorFactory;
SMemAllocator *pAllocator; SMemAllocator* pAllocator;
} TqMemRef; } STqMemRef;
typedef struct TqSerializedHead { typedef struct STqSerializedHead {
int16_t ver; int16_t ver;
int16_t action; int16_t action;
int32_t checksum; int32_t checksum;
int64_t ssize; int64_t ssize;
char content[]; char content[];
} TqSerializedHead; } STqSerializedHead;
typedef int (*TqSerializeFun)(const void* pObj, TqSerializedHead** ppHead); typedef int (*FTqSerialize)(const void* pObj, STqSerializedHead** ppHead);
typedef const void* (*TqDeserializeFun)(const TqSerializedHead* pHead, void** ppObj); typedef const void* (*FTqDeserialize)(const STqSerializedHead* pHead, void** ppObj);
typedef void (*TqDeleteFun)(void*); typedef void (*FTqDelete)(void*);
#define TQ_BUCKET_MASK 0xFF #define TQ_BUCKET_MASK 0xFF
#define TQ_BUCKET_SIZE 256 #define TQ_BUCKET_SIZE 256
#define TQ_PAGE_SIZE 4096 #define TQ_PAGE_SIZE 4096
//key + offset + size // key + offset + size
#define TQ_IDX_SIZE 24 #define TQ_IDX_SIZE 24
//4096 / 24 // 4096 / 24
#define TQ_MAX_IDX_ONE_PAGE 170 #define TQ_MAX_IDX_ONE_PAGE 170
//24 * 170 // 24 * 170
#define TQ_IDX_PAGE_BODY_SIZE 4080 #define TQ_IDX_PAGE_BODY_SIZE 4080
//4096 - 4080 // 4096 - 4080
#define TQ_IDX_PAGE_HEAD_SIZE 16 #define TQ_IDX_PAGE_HEAD_SIZE 16
#define TQ_ACTION_CONST 0 #define TQ_ACTION_CONST 0
#define TQ_ACTION_INUSE 1 #define TQ_ACTION_INUSE 1
#define TQ_ACTION_INUSE_CONT 2 #define TQ_ACTION_INUSE_CONT 2
#define TQ_ACTION_INTXN 3 #define TQ_ACTION_INTXN 3
#define TQ_SVER 0 #define TQ_SVER 0
//TODO: inplace mode is not implemented // TODO: inplace mode is not implemented
#define TQ_UPDATE_INPLACE 0 #define TQ_UPDATE_INPLACE 0
#define TQ_UPDATE_APPEND 1 #define TQ_UPDATE_APPEND 1
#define TQ_DUP_INTXN_REWRITE 0 #define TQ_DUP_INTXN_REWRITE 0
#define TQ_DUP_INTXN_REJECT 2 #define TQ_DUP_INTXN_REJECT 2
static inline bool TqUpdateAppend(int32_t tqConfigFlag) { static inline bool TqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; }
return tqConfigFlag & TQ_UPDATE_APPEND;
}
static inline bool TqDupIntxnReject(int32_t tqConfigFlag) { static inline bool TqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; }
return tqConfigFlag & TQ_DUP_INTXN_REJECT;
}
static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST; static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST;
#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE
#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE
typedef struct TqMetaHandle { typedef struct TqMetaHandle {
int64_t key; int64_t key;
@ -229,41 +226,41 @@ typedef struct TqMetaHandle {
} STqMetaHandle; } STqMetaHandle;
typedef struct TqMetaList { typedef struct TqMetaList {
STqMetaHandle handle; STqMetaHandle handle;
struct TqMetaList* next; struct TqMetaList* next;
//struct TqMetaList* inTxnPrev; // struct TqMetaList* inTxnPrev;
//struct TqMetaList* inTxnNext; // struct TqMetaList* inTxnNext;
struct TqMetaList* unpersistPrev; struct TqMetaList* unpersistPrev;
struct TqMetaList* unpersistNext; struct TqMetaList* unpersistNext;
} STqMetaList; } STqMetaList;
typedef struct TqMetaStore { typedef struct TqMetaStore {
STqMetaList* bucket[TQ_BUCKET_SIZE]; STqMetaList* bucket[TQ_BUCKET_SIZE];
//a table head // a table head
STqMetaList* unpersistHead; STqMetaList* unpersistHead;
//TODO:temporaral use, to be replaced by unified tfile // TODO:temporaral use, to be replaced by unified tfile
int fileFd; int fileFd;
//TODO:temporaral use, to be replaced by unified tfile // TODO:temporaral use, to be replaced by unified tfile
int idxFd; int idxFd;
char* dirPath; char* dirPath;
int32_t tqConfigFlag; int32_t tqConfigFlag;
TqSerializeFun pSerializer; FTqSerialize pSerializer;
TqDeserializeFun pDeserializer; FTqDeserialize pDeserializer;
TqDeleteFun pDeleter; FTqDelete pDeleter;
} STqMetaStore; } STqMetaStore;
typedef struct STQ { typedef struct STQ {
// the collection of group handle // the collection of group handle
// the handle of kvstore // the handle of kvstore
char* path; char* path;
STqCfg* tqConfig; STqCfg* tqConfig;
TqLogReader* tqLogReader; STqLogReader* tqLogReader;
TqMemRef tqMemRef; STqMemRef tqMemRef;
STqMetaStore* tqMeta; STqMetaStore* tqMeta;
} STQ; } STQ;
// open in each vnode // open in each vnode
STQ* tqOpen(const char* path, STqCfg* tqConfig, TqLogReader* tqLogReader, SMemAllocatorFactory *allocFac); STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac);
void tqDestroy(STQ*); void tqDestroy(STQ*);
// void* will be replace by a msg type // void* will be replace by a msg type
@ -272,19 +269,19 @@ int tqCommit(STQ*);
int tqConsume(STQ*, TmqConsumeReq*); int tqConsume(STQ*, TmqConsumeReq*);
TqGroupHandle* tqGetGroupHandle(STQ*, int64_t cId); STqGroupHandle* tqGetGroupHandle(STQ*, int64_t cId);
TqGroupHandle* tqOpenTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); STqGroupHandle* tqOpenTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId);
int tqCloseTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); int tqCloseTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId);
int tqMoveOffsetToNext(TqGroupHandle*); int tqMoveOffsetToNext(STqGroupHandle*);
int tqResetOffset(STQ*, int64_t topicId, int64_t cgId, int64_t offset); int tqResetOffset(STQ*, int64_t topicId, int64_t cgId, int64_t offset);
int tqRegisterContext(TqGroupHandle*, void* ahandle); int tqRegisterContext(STqGroupHandle*, void* ahandle);
int tqLaunchQuery(TqGroupHandle*); int tqLaunchQuery(STqGroupHandle*);
int tqSendLaunchQuery(TqGroupHandle*); int tqSendLaunchQuery(STqGroupHandle*);
int tqSerializeGroupHandle(const TqGroupHandle* gHandle, TqSerializedHead** ppHead); int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead);
const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandle** gHandle); const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** gHandle);
#ifdef __cplusplus #ifdef __cplusplus
} }

128
include/libs/wal/wal.h
View File

@ -16,49 +16,75 @@
#define _TD_WAL_H_ #define _TD_WAL_H_
#include "os.h" #include "os.h"
#include "tarray.h"
#include "tdef.h" #include "tdef.h"
#include "tlog.h" #include "tlog.h"
#include "tarray.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
extern int32_t wDebugFlag; extern int32_t wDebugFlag;
#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); }} #define wFatal(...) \
#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); }} { \
#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", 255, __VA_ARGS__); }} if (wDebugFlag & DEBUG_FATAL) { \
#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", 255, __VA_ARGS__); }} taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); \
#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} } \
#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} }
#define wError(...) \
{ \
if (wDebugFlag & DEBUG_ERROR) { \
taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); \
} \
}
#define wWarn(...) \
{ \
if (wDebugFlag & DEBUG_WARN) { \
taosPrintLog("WAL WARN ", 255, __VA_ARGS__); \
} \
}
#define wInfo(...) \
{ \
if (wDebugFlag & DEBUG_INFO) { \
taosPrintLog("WAL ", 255, __VA_ARGS__); \
} \
}
#define wDebug(...) \
{ \
if (wDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \
} \
}
#define wTrace(...) \
{ \
if (wDebugFlag & DEBUG_TRACE) { \
taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \
} \
}
#define WAL_HEAD_VER 0 #define WAL_HEAD_VER 0
#define WAL_NOSUFFIX_LEN 20 #define WAL_NOSUFFIX_LEN 20
#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN+1) #define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1)
#define WAL_LOG_SUFFIX "log" #define WAL_LOG_SUFFIX "log"
#define WAL_INDEX_SUFFIX "idx" #define WAL_INDEX_SUFFIX "idx"
#define WAL_REFRESH_MS 1000 #define WAL_REFRESH_MS 1000
#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead)) #define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead))
#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) #define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12)
#define WAL_FILE_LEN (WAL_PATH_LEN + 32) #define WAL_FILE_LEN (WAL_PATH_LEN + 32)
#define WAL_CUR_FAILED 1 #define WAL_CUR_FAILED 1
#pragma pack(push, 1) #pragma pack(push, 1)
typedef enum { typedef enum { TAOS_WAL_NOLOG = 0, TAOS_WAL_WRITE = 1, TAOS_WAL_FSYNC = 2 } EWalType;
TAOS_WAL_NOLOG = 0,
TAOS_WAL_WRITE = 1,
TAOS_WAL_FSYNC = 2
} EWalType;
typedef struct SWalReadHead { typedef struct SWalReadHead {
int8_t headVer; int8_t headVer;
uint8_t msgType; uint8_t msgType;
int8_t reserved[2]; int8_t reserved[2];
int32_t len; int32_t len;
int64_t ingestTs; //not implemented int64_t ingestTs; // not implemented
int64_t version; int64_t version;
char body[]; char body[];
} SWalReadHead; } SWalReadHead;
typedef struct { typedef struct {
@ -68,12 +94,12 @@ typedef struct {
int32_t rollPeriod; // secs int32_t rollPeriod; // secs
int64_t retentionSize; int64_t retentionSize;
int64_t segSize; int64_t segSize;
EWalType level; // wal level EWalType level; // wal level
} SWalCfg; } SWalCfg;
typedef struct { typedef struct {
uint32_t cksumHead; uint32_t cksumHead;
uint32_t cksumBody; uint32_t cksumBody;
SWalReadHead head; SWalReadHead head;
} SWalHead; } SWalHead;
@ -89,37 +115,37 @@ typedef struct SWal {
// cfg // cfg
SWalCfg cfg; SWalCfg cfg;
int32_t fsyncSeq; int32_t fsyncSeq;
//meta // meta
SWalVer vers; SWalVer vers;
int64_t writeLogTfd; int64_t writeLogTfd;
int64_t writeIdxTfd; int64_t writeIdxTfd;
int32_t writeCur; int32_t writeCur;
SArray* fileInfoSet; SArray *fileInfoSet;
//status // status
int64_t totSize; int64_t totSize;
int64_t lastRollSeq; int64_t lastRollSeq;
//ctl // ctl
int64_t refId; int64_t refId;
pthread_mutex_t mutex; pthread_mutex_t mutex;
//path // path
char path[WAL_PATH_LEN]; char path[WAL_PATH_LEN];
//reusable write head // reusable write head
SWalHead writeHead; SWalHead writeHead;
} SWal; // WAL HANDLE } SWal; // WAL HANDLE
typedef struct SWalReadHandle { typedef struct SWalReadHandle {
SWal* pWal; SWal *pWal;
int64_t readLogTfd; int64_t readLogTfd;
int64_t readIdxTfd; int64_t readIdxTfd;
int64_t curFileFirstVer; int64_t curFileFirstVer;
int64_t curVersion; int64_t curVersion;
int64_t capacity; int64_t capacity;
int64_t status; //if cursor valid int64_t status; // if cursor valid
SWalHead* pHead; SWalHead *pHead;
} SWalReadHandle; } SWalReadHandle;
#pragma pack(pop) #pragma pack(pop)
//typedef int32_t (*FWalWrite)(void *ahandle, void *pHead); // typedef int32_t (*FWalWrite)(void *ahandle, void *pHead);
// module initialization // module initialization
int32_t walInit(); int32_t walInit();
@ -141,15 +167,15 @@ int32_t walRollback(SWal *, int64_t ver);
// notify that previous logs can be pruned safely // notify that previous logs can be pruned safely
int32_t walBeginSnapshot(SWal *, int64_t ver); int32_t walBeginSnapshot(SWal *, int64_t ver);
int32_t walEndSnapshot(SWal *); int32_t walEndSnapshot(SWal *);
//int32_t walDataCorrupted(SWal*); // int32_t walDataCorrupted(SWal*);
// read // read
SWalReadHandle* walOpenReadHandle(SWal *); SWalReadHandle *walOpenReadHandle(SWal *);
void walCloseReadHandle(SWalReadHandle *); void walCloseReadHandle(SWalReadHandle *);
int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver); int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver);
int32_t walRead(SWal *, SWalHead **, int64_t ver); int32_t walRead(SWal *, SWalHead **, int64_t ver);
//int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum); // int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum);
// lifecycle check // lifecycle check
int64_t walGetFirstVer(SWal *); int64_t walGetFirstVer(SWal *);

6
source/dnode/vnode/tq/inc/tqMetaStore.h
View File

@ -25,9 +25,9 @@ extern "C" {
STqMetaStore* tqStoreOpen(const char* path, STqMetaStore* tqStoreOpen(const char* path,
TqSerializeFun pSerializer, FTqSerialize pSerializer,
TqDeserializeFun pDeserializer, FTqDeserialize pDeserializer,
TqDeleteFun pDeleter, FTqDelete pDeleter,
int32_t tqConfigFlag int32_t tqConfigFlag
); );
int32_t tqStoreClose(STqMetaStore*); int32_t tqStoreClose(STqMetaStore*);

317
source/dnode/vnode/tq/src/tq.c
View File

@ -16,75 +16,70 @@
#include "tqInt.h" #include "tqInt.h"
#include "tqMetaStore.h" #include "tqMetaStore.h"
//static // static
//read next version data // read next version data
// //
//send to fetch queue // send to fetch queue
// //
//handle management message // handle management message
// //
int tqGetgHandleSSize(const TqGroupHandle *gHandle); int tqGetgHandleSSize(const STqGroupHandle* gHandle);
int tqBufHandleSSize(); int tqBufHandleSSize();
int tqBufItemSSize(); int tqBufItemSSize();
TqGroupHandle* tqFindHandle(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { STqGroupHandle* tqFindHandle(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
TqGroupHandle* gHandle; STqGroupHandle* gHandle;
return NULL; return NULL;
} }
void* tqSerializeListHandle(TqListHandle* listHandle, void* ptr); void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr);
void* tqSerializeBufHandle(TqBufferHandle* bufHandle, void* ptr); void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr);
void* tqSerializeBufItem(TqBufferItem* bufItem, void* ptr); void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr);
const void* tqDeserializeBufHandle(const void* pBytes, TqBufferHandle* bufHandle); const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle);
const void* tqDeserializeBufItem(const void* pBytes, TqBufferItem* bufItem); const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem);
STQ* tqOpen(const char* path, STqCfg* tqConfig, TqLogReader* tqLogReader, SMemAllocatorFactory *allocFac) { STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac) {
STQ* pTq = malloc(sizeof(STQ)); STQ* pTq = malloc(sizeof(STQ));
if(pTq == NULL) { if (pTq == NULL) {
//TODO: memory error // TODO: memory error
return NULL; return NULL;
} }
pTq->path = strdup(path); pTq->path = strdup(path);
pTq->tqConfig = tqConfig; pTq->tqConfig = tqConfig;
pTq->tqLogReader = tqLogReader; pTq->tqLogReader = tqLogReader;
pTq->tqMemRef.pAlloctorFactory = allocFac; pTq->tqMemRef.pAlloctorFactory = allocFac;
pTq->tqMemRef.pAllocator = allocFac->create(allocFac); pTq->tqMemRef.pAllocator = allocFac->create(allocFac);
if(pTq->tqMemRef.pAllocator == NULL) { if (pTq->tqMemRef.pAllocator == NULL) {
//TODO // TODO
} }
pTq->tqMeta = tqStoreOpen(path, pTq->tqMeta =
(TqSerializeFun)tqSerializeGroupHandle, tqStoreOpen(path, (FTqSerialize)tqSerializeGroupHandle, (FTqDeserialize)tqDeserializeGroupHandle, free, 0);
(TqDeserializeFun)tqDeserializeGroupHandle, if (pTq->tqMeta == NULL) {
free, // TODO: free STQ
0);
if(pTq->tqMeta == NULL) {
//TODO: free STQ
return NULL; return NULL;
} }
return pTq; return pTq;
} }
static int tqProtoCheck(TmqMsgHead *pMsg) { static int tqProtoCheck(TmqMsgHead* pMsg) { return pMsg->protoVer == 0; }
return pMsg->protoVer == 0;
}
static int tqAckOneTopic(TqBufferHandle *bHandle, TmqOneAck *pAck, TqQueryMsg** ppQuery) { static int tqAckOneTopic(STqBufferHandle* bHandle, TmqOneAck* pAck, STqQueryMsg** ppQuery) {
//clean old item and move forward // clean old item and move forward
int32_t consumeOffset = pAck->consumeOffset; int32_t consumeOffset = pAck->consumeOffset;
int idx = consumeOffset % TQ_BUFFER_SIZE; int idx = consumeOffset % TQ_BUFFER_SIZE;
ASSERT(bHandle->buffer[idx].content && bHandle->buffer[idx].executor); ASSERT(bHandle->buffer[idx].content && bHandle->buffer[idx].executor);
tfree(bHandle->buffer[idx].content); tfree(bHandle->buffer[idx].content);
if( 1 /* TODO: need to launch new query */) { if (1 /* TODO: need to launch new query */) {
TqQueryMsg* pNewQuery = malloc(sizeof(TqQueryMsg)); STqQueryMsg* pNewQuery = malloc(sizeof(STqQueryMsg));
if(pNewQuery == NULL) { if (pNewQuery == NULL) {
//TODO: memory insufficient // TODO: memory insufficient
return -1; return -1;
} }
//TODO: lock executor // TODO: lock executor
pNewQuery->exec->executor = bHandle->buffer[idx].executor; pNewQuery->exec->executor = bHandle->buffer[idx].executor;
//TODO: read from wal and assign to src // TODO: read from wal and assign to src
pNewQuery->exec->src = 0; pNewQuery->exec->src = 0;
pNewQuery->exec->dest = &bHandle->buffer[idx]; pNewQuery->exec->dest = &bHandle->buffer[idx];
pNewQuery->next = *ppQuery; pNewQuery->next = *ppQuery;
@ -93,98 +88,94 @@ static int tqAckOneTopic(TqBufferHandle *bHandle, TmqOneAck *pAck, TqQueryMsg**
return 0; return 0;
} }
static int tqAck(TqGroupHandle* gHandle, TmqAcks* pAcks) { static int tqAck(STqGroupHandle* gHandle, TmqAcks* pAcks) {
int32_t ackNum = pAcks->ackNum; int32_t ackNum = pAcks->ackNum;
TmqOneAck *acks = pAcks->acks; TmqOneAck* acks = pAcks->acks;
//double ptr for acks and list // double ptr for acks and list
int i = 0; int i = 0;
TqListHandle* node = gHandle->head; STqListHandle* node = gHandle->head;
int ackCnt = 0; int ackCnt = 0;
TqQueryMsg *pQuery = NULL; STqQueryMsg* pQuery = NULL;
while(i < ackNum && node->next) { while (i < ackNum && node->next) {
if(acks[i].topicId == node->next->bufHandle.topicId) { if (acks[i].topicId == node->next->bufHandle.topicId) {
ackCnt++; ackCnt++;
tqAckOneTopic(&node->next->bufHandle, &acks[i], &pQuery); tqAckOneTopic(&node->next->bufHandle, &acks[i], &pQuery);
} else if(acks[i].topicId < node->next->bufHandle.topicId) { } else if (acks[i].topicId < node->next->bufHandle.topicId) {
i++; i++;
} else { } else {
node = node->next; node = node->next;
} }
} }
if(pQuery) { if (pQuery) {
//post message // post message
} }
return ackCnt; return ackCnt;
} }
static int tqCommitTCGroup(TqGroupHandle* handle) { static int tqCommitTCGroup(STqGroupHandle* handle) {
//persist modification into disk // persist modification into disk
return 0; return 0;
} }
int tqCreateTCGroup(STQ *pTq, int64_t topicId, int64_t cgId, int64_t cId, TqGroupHandle** handle) { int tqCreateTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroupHandle** handle) {
//create in disk // create in disk
TqGroupHandle* gHandle = (TqGroupHandle*)malloc(sizeof(TqGroupHandle)); STqGroupHandle* gHandle = (STqGroupHandle*)malloc(sizeof(STqGroupHandle));
if(gHandle == NULL) { if (gHandle == NULL) {
//TODO // TODO
return -1; return -1;
} }
memset(gHandle, 0, sizeof(TqGroupHandle)); memset(gHandle, 0, sizeof(STqGroupHandle));
return 0; return 0;
} }
TqGroupHandle* tqOpenTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { STqGroupHandle* tqOpenTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
TqGroupHandle* gHandle = tqHandleGet(pTq->tqMeta, cId); STqGroupHandle* gHandle = tqHandleGet(pTq->tqMeta, cId);
if(gHandle == NULL) { if (gHandle == NULL) {
int code = tqCreateTCGroup(pTq, topicId, cgId, cId, &gHandle); int code = tqCreateTCGroup(pTq, topicId, cgId, cId, &gHandle);
if(code != 0) { if (code != 0) {
//TODO // TODO
return NULL; return NULL;
} }
} }
//create // create
//open // open
return gHandle; return gHandle;
} }
int tqCloseTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { int tqCloseTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { return 0; }
return 0;
}
int tqDropTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { int tqDropTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
//delete from disk // delete from disk
return 0; return 0;
} }
static int tqFetch(TqGroupHandle* gHandle, void** msg) { static int tqFetch(STqGroupHandle* gHandle, void** msg) {
TqListHandle* head = gHandle->head; STqListHandle* head = gHandle->head;
TqListHandle* node = head; STqListHandle* node = head;
int totSize = 0; int totSize = 0;
//TODO: make it a macro // TODO: make it a macro
int sizeLimit = 4 * 1024; int sizeLimit = 4 * 1024;
TmqMsgContent* buffer = malloc(sizeLimit); TmqMsgContent* buffer = malloc(sizeLimit);
if(buffer == NULL) { if (buffer == NULL) {
//TODO:memory insufficient // TODO:memory insufficient
return -1; return -1;
} }
//iterate the list to get msgs of all topics // iterate the list to get msgs of all topics
//until all topic iterated or msgs over sizeLimit // until all topic iterated or msgs over sizeLimit
while(node->next) { while (node->next) {
node = node->next; node = node->next;
TqBufferHandle* bufHandle = &node->bufHandle; STqBufferHandle* bufHandle = &node->bufHandle;
int idx = bufHandle->nextConsumeOffset % TQ_BUFFER_SIZE; int idx = bufHandle->nextConsumeOffset % TQ_BUFFER_SIZE;
if(bufHandle->buffer[idx].content != NULL && if (bufHandle->buffer[idx].content != NULL && bufHandle->buffer[idx].offset == bufHandle->nextConsumeOffset) {
bufHandle->buffer[idx].offset == bufHandle->nextConsumeOffset
) {
totSize += bufHandle->buffer[idx].size; totSize += bufHandle->buffer[idx].size;
if(totSize > sizeLimit) { if (totSize > sizeLimit) {
void *ptr = realloc(buffer, totSize); void* ptr = realloc(buffer, totSize);
if(ptr == NULL) { if (ptr == NULL) {
totSize -= bufHandle->buffer[idx].size; totSize -= bufHandle->buffer[idx].size;
//TODO:memory insufficient // TODO:memory insufficient
//return msgs already copied // return msgs already copied
break; break;
} }
} }
@ -194,7 +185,7 @@ static int tqFetch(TqGroupHandle* gHandle, void** msg) {
buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); buffer = POINTER_SHIFT(buffer, sizeof(int64_t));
memcpy(buffer, bufHandle->buffer[idx].content, bufHandle->buffer[idx].size); memcpy(buffer, bufHandle->buffer[idx].content, bufHandle->buffer[idx].size);
buffer = POINTER_SHIFT(buffer, bufHandle->buffer[idx].size); buffer = POINTER_SHIFT(buffer, bufHandle->buffer[idx].size);
if(totSize > sizeLimit) { if (totSize > sizeLimit) {
break; break;
} }
} }
@ -202,104 +193,98 @@ static int tqFetch(TqGroupHandle* gHandle, void** msg) {
return totSize; return totSize;
} }
TqGroupHandle* tqGetGroupHandle(STQ* pTq, int64_t cId) { STqGroupHandle* tqGetGroupHandle(STQ* pTq, int64_t cId) { return NULL; }
return NULL;
}
int tqLaunchQuery(TqGroupHandle* gHandle) { int tqLaunchQuery(STqGroupHandle* gHandle) { return 0; }
return 0;
}
int tqSendLaunchQuery(TqGroupHandle* gHandle) { int tqSendLaunchQuery(STqGroupHandle* gHandle) { return 0; }
return 0;
}
/*int tqMoveOffsetToNext(TqGroupHandle* gHandle) {*/ /*int tqMoveOffsetToNext(TqGroupHandle* gHandle) {*/
/*return 0;*/ /*return 0;*/
/*}*/ /*}*/
int tqPushMsg(STQ* pTq , void* p, int64_t version) { int tqPushMsg(STQ* pTq, void* p, int64_t version) {
//add reference // add reference
//judge and launch new query // judge and launch new query
return 0; return 0;
} }
int tqCommit(STQ* pTq) { int tqCommit(STQ* pTq) {
//do nothing // do nothing
return 0; return 0;
} }
int tqConsume(STQ* pTq, TmqConsumeReq* pMsg) { int tqConsume(STQ* pTq, TmqConsumeReq* pMsg) {
if(!tqProtoCheck((TmqMsgHead *)pMsg)) { if (!tqProtoCheck((TmqMsgHead*)pMsg)) {
//proto version invalid // proto version invalid
return -1; return -1;
} }
int64_t clientId = pMsg->head.clientId; int64_t clientId = pMsg->head.clientId;
TqGroupHandle *gHandle = tqGetGroupHandle(pTq, clientId); STqGroupHandle* gHandle = tqGetGroupHandle(pTq, clientId);
if(gHandle == NULL) { if (gHandle == NULL) {
//client not connect // client not connect
return -1; return -1;
} }
if(pMsg->acks.ackNum != 0) { if (pMsg->acks.ackNum != 0) {
if(tqAck(gHandle, &pMsg->acks) != 0) { if (tqAck(gHandle, &pMsg->acks) != 0) {
//ack not success // ack not success
return -1; return -1;
} }
} }
TmqConsumeRsp *pRsp = (TmqConsumeRsp*) pMsg; TmqConsumeRsp* pRsp = (TmqConsumeRsp*)pMsg;
if(tqFetch(gHandle, (void**)&pRsp->msgs) <= 0) { if (tqFetch(gHandle, (void**)&pRsp->msgs) <= 0) {
//fetch error // fetch error
return -1; return -1;
} }
//judge and launch new query // judge and launch new query
if(tqLaunchQuery(gHandle)) { if (tqLaunchQuery(gHandle)) {
//launch query error // launch query error
return -1; return -1;
} }
return 0; return 0;
} }
int tqSerializeGroupHandle(const TqGroupHandle *gHandle, TqSerializedHead** ppHead) { int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead) {
//calculate size // calculate size
int sz = tqGetgHandleSSize(gHandle) + sizeof(TqSerializedHead); int sz = tqGetgHandleSSize(gHandle) + sizeof(STqSerializedHead);
if(sz > (*ppHead)->ssize) { if (sz > (*ppHead)->ssize) {
void* tmpPtr = realloc(*ppHead, sz); void* tmpPtr = realloc(*ppHead, sz);
if(tmpPtr == NULL) { if (tmpPtr == NULL) {
free(*ppHead); free(*ppHead);
//TODO: memory err // TODO: memory err
return -1; return -1;
} }
*ppHead = tmpPtr; *ppHead = tmpPtr;
(*ppHead)->ssize = sz; (*ppHead)->ssize = sz;
} }
void* ptr = (*ppHead)->content; void* ptr = (*ppHead)->content;
//do serialization // do serialization
*(int64_t*)ptr = gHandle->cId; *(int64_t*)ptr = gHandle->cId;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
*(int64_t*)ptr = gHandle->cgId; *(int64_t*)ptr = gHandle->cgId;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
*(int32_t*)ptr = gHandle->topicNum; *(int32_t*)ptr = gHandle->topicNum;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
if(gHandle->topicNum > 0) { if (gHandle->topicNum > 0) {
tqSerializeListHandle(gHandle->head, ptr); tqSerializeListHandle(gHandle->head, ptr);
} }
return 0; return 0;
} }
void* tqSerializeListHandle(TqListHandle *listHandle, void* ptr) { void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr) {
TqListHandle *node = listHandle; STqListHandle* node = listHandle;
ASSERT(node != NULL); ASSERT(node != NULL);
while(node) { while (node) {
ptr = tqSerializeBufHandle(&node->bufHandle, ptr); ptr = tqSerializeBufHandle(&node->bufHandle, ptr);
node = node->next; node = node->next;
} }
return ptr; return ptr;
} }
void* tqSerializeBufHandle(TqBufferHandle *bufHandle, void* ptr) { void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr) {
*(int64_t*)ptr = bufHandle->nextConsumeOffset; *(int64_t*)ptr = bufHandle->nextConsumeOffset;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
*(int64_t*)ptr = bufHandle->topicId; *(int64_t*)ptr = bufHandle->topicId;
@ -308,21 +293,21 @@ void* tqSerializeBufHandle(TqBufferHandle *bufHandle, void* ptr) {
ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
*(int32_t*)ptr = bufHandle->tail; *(int32_t*)ptr = bufHandle->tail;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
for(int i = 0; i < TQ_BUFFER_SIZE; i++) { for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
ptr = tqSerializeBufItem(&bufHandle->buffer[i], ptr); ptr = tqSerializeBufItem(&bufHandle->buffer[i], ptr);
} }
return ptr; return ptr;
} }
void* tqSerializeBufItem(TqBufferItem *bufItem, void* ptr) { void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr) {
//TODO: do we need serialize this? // TODO: do we need serialize this?
//mainly for executor // mainly for executor
return ptr; return ptr;
} }
const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandle **ppGHandle) { const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** ppGHandle) {
TqGroupHandle *gHandle = *ppGHandle; STqGroupHandle* gHandle = *ppGHandle;
const void* ptr = pHead->content; const void* ptr = pHead->content;
gHandle->cId = *(int64_t*)ptr; gHandle->cId = *(int64_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
gHandle->cgId = *(int64_t*)ptr; gHandle->cgId = *(int64_t*)ptr;
@ -331,20 +316,20 @@ const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandl
gHandle->topicNum = *(int32_t*)ptr; gHandle->topicNum = *(int32_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
gHandle->head = NULL; gHandle->head = NULL;
TqListHandle *node = gHandle->head; STqListHandle* node = gHandle->head;
for(int i = 0; i < gHandle->topicNum; i++) { for (int i = 0; i < gHandle->topicNum; i++) {
if(gHandle->head == NULL) { if (gHandle->head == NULL) {
if((node = malloc(sizeof(TqListHandle))) == NULL) { if ((node = malloc(sizeof(STqListHandle))) == NULL) {
//TODO: error // TODO: error
return NULL; return NULL;
} }
node->next= NULL; node->next = NULL;
ptr = tqDeserializeBufHandle(ptr, &node->bufHandle); ptr = tqDeserializeBufHandle(ptr, &node->bufHandle);
gHandle->head = node; gHandle->head = node;
} else { } else {
node->next = malloc(sizeof(TqListHandle)); node->next = malloc(sizeof(STqListHandle));
if(node->next == NULL) { if (node->next == NULL) {
//TODO: error // TODO: error
return NULL; return NULL;
} }
node->next->next = NULL; node->next->next = NULL;
@ -355,7 +340,7 @@ const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandl
return ptr; return ptr;
} }
const void* tqDeserializeBufHandle(const void* pBytes, TqBufferHandle *bufHandle) { const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle) {
const void* ptr = pBytes; const void* ptr = pBytes;
bufHandle->nextConsumeOffset = *(int64_t*)ptr; bufHandle->nextConsumeOffset = *(int64_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
@ -365,32 +350,30 @@ const void* tqDeserializeBufHandle(const void* pBytes, TqBufferHandle *bufHandle
ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
bufHandle->tail = *(int32_t*)ptr; bufHandle->tail = *(int32_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
for(int i = 0; i < TQ_BUFFER_SIZE; i++) { for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
ptr = tqDeserializeBufItem(ptr, &bufHandle->buffer[i]); ptr = tqDeserializeBufItem(ptr, &bufHandle->buffer[i]);
} }
return ptr; return ptr;
} }
const void* tqDeserializeBufItem(const void* pBytes, TqBufferItem *bufItem) { const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem) { return pBytes; }
return pBytes;
// TODO: make this a macro
int tqGetgHandleSSize(const STqGroupHandle* gHandle) {
return sizeof(int64_t) * 2 // cId + cgId
+ sizeof(int32_t) // topicNum
+ gHandle->topicNum * tqBufHandleSSize();
} }
//TODO: make this a macro // TODO: make this a macro
int tqGetgHandleSSize(const TqGroupHandle *gHandle) {
return sizeof(int64_t) * 2 //cId + cgId
+ sizeof(int32_t) //topicNum
+ gHandle->topicNum * tqBufHandleSSize();
}
//TODO: make this a macro
int tqBufHandleSSize() { int tqBufHandleSSize() {
return sizeof(int64_t) * 2 // nextConsumeOffset + topicId return sizeof(int64_t) * 2 // nextConsumeOffset + topicId
+ sizeof(int32_t) * 2 // head + tail + sizeof(int32_t) * 2 // head + tail
+ TQ_BUFFER_SIZE * tqBufItemSSize(); + TQ_BUFFER_SIZE * tqBufItemSSize();
} }
int tqBufItemSSize() { int tqBufItemSSize() {
//TODO: do this need serialization? // TODO: do this need serialization?
//mainly for executor // mainly for executor
return 0; return 0;
} }

378
source/dnode/vnode/tq/src/tqMetaStore.c
View File

@ -13,20 +13,20 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tqMetaStore.h" #include "tqMetaStore.h"
//TODO:replace by an abstract file layer // TODO:replace by an abstract file layer
#include "osDir.h"
#include <fcntl.h> #include <fcntl.h>
#include <string.h> #include <string.h>
#include <unistd.h> #include <unistd.h>
#include "osDir.h"
#define TQ_META_NAME "tq.meta" #define TQ_META_NAME "tq.meta"
#define TQ_IDX_NAME "tq.idx" #define TQ_IDX_NAME "tq.idx"
static int32_t tqHandlePutCommitted(STqMetaStore*, int64_t key, void* value); static int32_t tqHandlePutCommitted(STqMetaStore*, int64_t key, void* value);
static void* tqHandleGetUncommitted(STqMetaStore*, int64_t key); static void* tqHandleGetUncommitted(STqMetaStore*, int64_t key);
static inline void tqLinkUnpersist(STqMetaStore *pMeta, STqMetaList* pNode) { static inline void tqLinkUnpersist(STqMetaStore* pMeta, STqMetaList* pNode) {
if(pNode->unpersistNext == NULL) { if (pNode->unpersistNext == NULL) {
pNode->unpersistNext = pMeta->unpersistHead->unpersistNext; pNode->unpersistNext = pMeta->unpersistHead->unpersistNext;
pNode->unpersistPrev = pMeta->unpersistHead; pNode->unpersistPrev = pMeta->unpersistHead;
pMeta->unpersistHead->unpersistNext->unpersistPrev = pNode; pMeta->unpersistHead->unpersistNext->unpersistPrev = pNode;
@ -41,24 +41,24 @@ static inline int tqSeekLastPage(int fd) {
return lseek(fd, curPageOffset, SEEK_SET); return lseek(fd, curPageOffset, SEEK_SET);
} }
//TODO: the struct is tightly coupled with index entry // TODO: the struct is tightly coupled with index entry
typedef struct TqIdxPageHead { typedef struct STqIdxPageHead {
int16_t writeOffset; int16_t writeOffset;
int8_t unused[14]; int8_t unused[14];
} TqIdxPageHead; } STqIdxPageHead;
typedef struct TqIdxPageBuf { typedef struct STqIdxPageBuf {
TqIdxPageHead head; STqIdxPageHead head;
char buffer[TQ_IDX_PAGE_BODY_SIZE]; char buffer[TQ_IDX_PAGE_BODY_SIZE];
} TqIdxPageBuf; } STqIdxPageBuf;
static inline int tqReadLastPage(int fd, TqIdxPageBuf* pBuf) { static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) {
int offset = tqSeekLastPage(fd); int offset = tqSeekLastPage(fd);
int nBytes; int nBytes;
if((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) { if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) {
return -1; return -1;
} }
if(nBytes == 0) { if (nBytes == 0) {
memset(pBuf, 0, TQ_PAGE_SIZE); memset(pBuf, 0, TQ_PAGE_SIZE);
pBuf->head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; pBuf->head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
} }
@ -67,28 +67,24 @@ static inline int tqReadLastPage(int fd, TqIdxPageBuf* pBuf) {
return lseek(fd, offset, SEEK_SET); return lseek(fd, offset, SEEK_SET);
} }
STqMetaStore* tqStoreOpen(const char* path, STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserialize deserializer, FTqDelete deleter,
TqSerializeFun serializer, int32_t tqConfigFlag) {
TqDeserializeFun deserializer, STqMetaStore* pMeta = malloc(sizeof(STqMetaStore));
TqDeleteFun deleter, if (pMeta == NULL) {
int32_t tqConfigFlag // close
) {
STqMetaStore* pMeta = malloc(sizeof(STqMetaStore));
if(pMeta == NULL) {
//close
return NULL; return NULL;
} }
memset(pMeta, 0, sizeof(STqMetaStore)); memset(pMeta, 0, sizeof(STqMetaStore));
//concat data file name and index file name // concat data file name and index file name
size_t pathLen = strlen(path); size_t pathLen = strlen(path);
pMeta->dirPath = malloc(pathLen+1); pMeta->dirPath = malloc(pathLen + 1);
if(pMeta->dirPath != NULL) { if (pMeta->dirPath != NULL) {
//TODO: memory insufficient // TODO: memory insufficient
} }
strcpy(pMeta->dirPath, path); strcpy(pMeta->dirPath, path);
char name[pathLen+10]; char name[pathLen + 10];
strcpy(name, path); strcpy(name, path);
if (taosDirExist(name) != 0 && taosMkDir(name) != 0) { if (taosDirExist(name) != 0 && taosMkDir(name) != 0) {
@ -96,98 +92,96 @@ STqMetaStore* tqStoreOpen(const char* path,
} }
strcat(name, "/" TQ_IDX_NAME); strcat(name, "/" TQ_IDX_NAME);
int idxFd = open(name, O_RDWR | O_CREAT, 0755); int idxFd = open(name, O_RDWR | O_CREAT, 0755);
if(idxFd < 0) { if (idxFd < 0) {
ASSERT(false); ASSERT(false);
//close file // close file
//free memory // free memory
return NULL; return NULL;
} }
pMeta->idxFd = idxFd; pMeta->idxFd = idxFd;
pMeta->unpersistHead = malloc(sizeof(STqMetaList)); pMeta->unpersistHead = malloc(sizeof(STqMetaList));
if(pMeta->unpersistHead == NULL) { if (pMeta->unpersistHead == NULL) {
ASSERT(false); ASSERT(false);
//close file // close file
//free memory // free memory
return NULL; return NULL;
} }
memset(pMeta->unpersistHead, 0, sizeof(STqMetaList)); memset(pMeta->unpersistHead, 0, sizeof(STqMetaList));
pMeta->unpersistHead->unpersistNext pMeta->unpersistHead->unpersistNext = pMeta->unpersistHead->unpersistPrev = pMeta->unpersistHead;
= pMeta->unpersistHead->unpersistPrev
= pMeta->unpersistHead;
strcpy(name, path); strcpy(name, path);
strcat(name, "/" TQ_META_NAME); strcat(name, "/" TQ_META_NAME);
int fileFd = open(name, O_RDWR | O_CREAT, 0755); int fileFd = open(name, O_RDWR | O_CREAT, 0755);
if(fileFd < 0){ if (fileFd < 0) {
ASSERT(false); ASSERT(false);
return NULL; return NULL;
} }
pMeta->fileFd = fileFd; pMeta->fileFd = fileFd;
pMeta->pSerializer = serializer; pMeta->pSerializer = serializer;
pMeta->pDeserializer = deserializer; pMeta->pDeserializer = deserializer;
pMeta->pDeleter = deleter; pMeta->pDeleter = deleter;
pMeta->tqConfigFlag = tqConfigFlag; pMeta->tqConfigFlag = tqConfigFlag;
//read idx file and load into memory // read idx file and load into memory
TqIdxPageBuf idxBuf; STqIdxPageBuf idxBuf;
TqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE); STqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE);
if(serializedObj == NULL) { if (serializedObj == NULL) {
//TODO:memory insufficient // TODO:memory insufficient
} }
int idxRead; int idxRead;
int allocated = TQ_PAGE_SIZE; int allocated = TQ_PAGE_SIZE;
bool readEnd = false; bool readEnd = false;
while((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) { while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) {
if(idxRead == -1) { if (idxRead == -1) {
//TODO: handle error // TODO: handle error
ASSERT(false); ASSERT(false);
} }
ASSERT(idxBuf.head.writeOffset == idxRead); ASSERT(idxBuf.head.writeOffset == idxRead);
//loop read every entry // loop read every entry
for(int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) { for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) {
STqMetaList *pNode = malloc(sizeof(STqMetaList)); STqMetaList* pNode = malloc(sizeof(STqMetaList));
if(pNode == NULL) { if (pNode == NULL) {
//TODO: free memory and return error // TODO: free memory and return error
} }
memset(pNode, 0, sizeof(STqMetaList)); memset(pNode, 0, sizeof(STqMetaList));
memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE); memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE);
lseek(fileFd, pNode->handle.offset, SEEK_SET); lseek(fileFd, pNode->handle.offset, SEEK_SET);
if(allocated < pNode->handle.serializedSize) { if (allocated < pNode->handle.serializedSize) {
void *ptr = realloc(serializedObj, pNode->handle.serializedSize); void* ptr = realloc(serializedObj, pNode->handle.serializedSize);
if(ptr == NULL) { if (ptr == NULL) {
//TODO: memory insufficient // TODO: memory insufficient
} }
serializedObj = ptr; serializedObj = ptr;
allocated = pNode->handle.serializedSize; allocated = pNode->handle.serializedSize;
} }
serializedObj->ssize = pNode->handle.serializedSize; serializedObj->ssize = pNode->handle.serializedSize;
if(read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) { if (read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) {
//TODO: read error // TODO: read error
} }
if(serializedObj->action == TQ_ACTION_INUSE) { if (serializedObj->action == TQ_ACTION_INUSE) {
if(serializedObj->ssize != sizeof(TqSerializedHead)) { if (serializedObj->ssize != sizeof(STqSerializedHead)) {
pMeta->pDeserializer(serializedObj, &pNode->handle.valueInUse); pMeta->pDeserializer(serializedObj, &pNode->handle.valueInUse);
} else { } else {
pNode->handle.valueInUse = TQ_DELETE_TOKEN; pNode->handle.valueInUse = TQ_DELETE_TOKEN;
} }
} else if(serializedObj->action == TQ_ACTION_INTXN) { } else if (serializedObj->action == TQ_ACTION_INTXN) {
if(serializedObj->ssize != sizeof(TqSerializedHead)) { if (serializedObj->ssize != sizeof(STqSerializedHead)) {
pMeta->pDeserializer(serializedObj, &pNode->handle.valueInTxn); pMeta->pDeserializer(serializedObj, &pNode->handle.valueInTxn);
} else { } else {
pNode->handle.valueInTxn = TQ_DELETE_TOKEN; pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
} }
} else if(serializedObj->action == TQ_ACTION_INUSE_CONT) { } else if (serializedObj->action == TQ_ACTION_INUSE_CONT) {
if(serializedObj->ssize != sizeof(TqSerializedHead)) { if (serializedObj->ssize != sizeof(STqSerializedHead)) {
pMeta->pDeserializer(serializedObj, &pNode->handle.valueInUse); pMeta->pDeserializer(serializedObj, &pNode->handle.valueInUse);
} else { } else {
pNode->handle.valueInUse = TQ_DELETE_TOKEN; pNode->handle.valueInUse = TQ_DELETE_TOKEN;
} }
TqSerializedHead* ptr = POINTER_SHIFT(serializedObj, serializedObj->ssize); STqSerializedHead* ptr = POINTER_SHIFT(serializedObj, serializedObj->ssize);
if(ptr->ssize != sizeof(TqSerializedHead)) { if (ptr->ssize != sizeof(STqSerializedHead)) {
pMeta->pDeserializer(ptr, &pNode->handle.valueInTxn); pMeta->pDeserializer(ptr, &pNode->handle.valueInTxn);
} else { } else {
pNode->handle.valueInTxn = TQ_DELETE_TOKEN; pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
@ -196,22 +190,21 @@ STqMetaStore* tqStoreOpen(const char* path,
ASSERT(0); ASSERT(0);
} }
//put into list // put into list
int bucketKey = pNode->handle.key & TQ_BUCKET_MASK; int bucketKey = pNode->handle.key & TQ_BUCKET_MASK;
STqMetaList* pBucketNode = pMeta->bucket[bucketKey]; STqMetaList* pBucketNode = pMeta->bucket[bucketKey];
if(pBucketNode == NULL) { if (pBucketNode == NULL) {
pMeta->bucket[bucketKey] = pNode; pMeta->bucket[bucketKey] = pNode;
} else if(pBucketNode->handle.key == pNode->handle.key) { } else if (pBucketNode->handle.key == pNode->handle.key) {
pNode->next = pBucketNode->next; pNode->next = pBucketNode->next;
pMeta->bucket[bucketKey] = pNode; pMeta->bucket[bucketKey] = pNode;
} else { } else {
while(pBucketNode->next && while (pBucketNode->next && pBucketNode->next->handle.key != pNode->handle.key) {
pBucketNode->next->handle.key != pNode->handle.key) { pBucketNode = pBucketNode->next;
pBucketNode = pBucketNode->next;
} }
if(pBucketNode->next) { if (pBucketNode->next) {
ASSERT(pBucketNode->next->handle.key == pNode->handle.key); ASSERT(pBucketNode->next->handle.key == pNode->handle.key);
STqMetaList *pNodeFound = pBucketNode->next; STqMetaList* pNodeFound = pBucketNode->next;
pNode->next = pNodeFound->next; pNode->next = pNodeFound->next;
pBucketNode->next = pNode; pBucketNode->next = pNode;
pBucketNode = pNodeFound; pBucketNode = pNodeFound;
@ -221,13 +214,11 @@ STqMetaStore* tqStoreOpen(const char* path,
pBucketNode = NULL; pBucketNode = NULL;
} }
} }
if(pBucketNode) { if (pBucketNode) {
if(pBucketNode->handle.valueInUse if (pBucketNode->handle.valueInUse && pBucketNode->handle.valueInUse != TQ_DELETE_TOKEN) {
&& pBucketNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pBucketNode->handle.valueInUse); pMeta->pDeleter(pBucketNode->handle.valueInUse);
} }
if(pBucketNode->handle.valueInTxn if (pBucketNode->handle.valueInTxn && pBucketNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
&& pBucketNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pBucketNode->handle.valueInTxn); pMeta->pDeleter(pBucketNode->handle.valueInTxn);
} }
free(pBucketNode); free(pBucketNode);
@ -239,23 +230,21 @@ STqMetaStore* tqStoreOpen(const char* path,
} }
int32_t tqStoreClose(STqMetaStore* pMeta) { int32_t tqStoreClose(STqMetaStore* pMeta) {
//commit data and idx // commit data and idx
tqStorePersist(pMeta); tqStorePersist(pMeta);
ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next==NULL); ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL);
close(pMeta->fileFd); close(pMeta->fileFd);
close(pMeta->idxFd); close(pMeta->idxFd);
//free memory // free memory
for(int i = 0; i < TQ_BUCKET_SIZE; i++) { for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
STqMetaList* pNode = pMeta->bucket[i]; STqMetaList* pNode = pMeta->bucket[i];
while(pNode) { while (pNode) {
ASSERT(pNode->unpersistNext == NULL); ASSERT(pNode->unpersistNext == NULL);
ASSERT(pNode->unpersistPrev == NULL); ASSERT(pNode->unpersistPrev == NULL);
if(pNode->handle.valueInTxn if (pNode->handle.valueInTxn && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
&& pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInTxn); pMeta->pDeleter(pNode->handle.valueInTxn);
} }
if(pNode->handle.valueInUse if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
&& pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInUse); pMeta->pDeleter(pNode->handle.valueInUse);
} }
STqMetaList* next = pNode->next; STqMetaList* next = pNode->next;
@ -272,17 +261,15 @@ int32_t tqStoreClose(STqMetaStore* pMeta) {
int32_t tqStoreDelete(STqMetaStore* pMeta) { int32_t tqStoreDelete(STqMetaStore* pMeta) {
close(pMeta->fileFd); close(pMeta->fileFd);
close(pMeta->idxFd); close(pMeta->idxFd);
//free memory // free memory
for(int i = 0; i < TQ_BUCKET_SIZE; i++) { for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
STqMetaList* pNode = pMeta->bucket[i]; STqMetaList* pNode = pMeta->bucket[i];
pMeta->bucket[i] = NULL; pMeta->bucket[i] = NULL;
while(pNode) { while (pNode) {
if(pNode->handle.valueInTxn if (pNode->handle.valueInTxn && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
&& pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInTxn); pMeta->pDeleter(pNode->handle.valueInTxn);
} }
if(pNode->handle.valueInUse if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
&& pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInUse); pMeta->pDeleter(pNode->handle.valueInUse);
} }
STqMetaList* next = pNode->next; STqMetaList* next = pNode->next;
@ -297,26 +284,26 @@ int32_t tqStoreDelete(STqMetaStore* pMeta) {
return 0; return 0;
} }
//TODO: wrap in tfile // TODO: wrap in tfile
int32_t tqStorePersist(STqMetaStore* pMeta) { int32_t tqStorePersist(STqMetaStore* pMeta) {
TqIdxPageBuf idxBuf; STqIdxPageBuf idxBuf;
int64_t* bufPtr = (int64_t*)idxBuf.buffer; int64_t* bufPtr = (int64_t*)idxBuf.buffer;
STqMetaList *pHead = pMeta->unpersistHead; STqMetaList* pHead = pMeta->unpersistHead;
STqMetaList *pNode = pHead->unpersistNext; STqMetaList* pNode = pHead->unpersistNext;
TqSerializedHead *pSHead = malloc(sizeof(TqSerializedHead)); STqSerializedHead* pSHead = malloc(sizeof(STqSerializedHead));
if(pSHead == NULL) { if (pSHead == NULL) {
//TODO: memory error // TODO: memory error
return -1; return -1;
} }
pSHead->ver = TQ_SVER; pSHead->ver = TQ_SVER;
pSHead->checksum = 0; pSHead->checksum = 0;
pSHead->ssize = sizeof(TqSerializedHead); pSHead->ssize = sizeof(STqSerializedHead);
int allocatedSize = sizeof(TqSerializedHead); int allocatedSize = sizeof(STqSerializedHead);
int offset = lseek(pMeta->fileFd, 0, SEEK_CUR); int offset = lseek(pMeta->fileFd, 0, SEEK_CUR);
tqReadLastPage(pMeta->idxFd, &idxBuf); tqReadLastPage(pMeta->idxFd, &idxBuf);
if(idxBuf.head.writeOffset == TQ_PAGE_SIZE) { if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) {
lseek(pMeta->idxFd, 0, SEEK_END); lseek(pMeta->idxFd, 0, SEEK_END);
memset(&idxBuf, 0, TQ_PAGE_SIZE); memset(&idxBuf, 0, TQ_PAGE_SIZE);
idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
@ -324,18 +311,18 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
bufPtr = POINTER_SHIFT(&idxBuf, idxBuf.head.writeOffset); bufPtr = POINTER_SHIFT(&idxBuf, idxBuf.head.writeOffset);
} }
while(pHead != pNode) { while (pHead != pNode) {
int nBytes = 0; int nBytes = 0;
if(pNode->handle.valueInUse) { if (pNode->handle.valueInUse) {
if(pNode->handle.valueInTxn) { if (pNode->handle.valueInTxn) {
pSHead->action = TQ_ACTION_INUSE_CONT; pSHead->action = TQ_ACTION_INUSE_CONT;
} else { } else {
pSHead->action = TQ_ACTION_INUSE; pSHead->action = TQ_ACTION_INUSE;
} }
if(pNode->handle.valueInUse == TQ_DELETE_TOKEN) { if (pNode->handle.valueInUse == TQ_DELETE_TOKEN) {
pSHead->ssize = sizeof(TqSerializedHead); pSHead->ssize = sizeof(STqSerializedHead);
} else { } else {
pMeta->pSerializer(pNode->handle.valueInUse, &pSHead); pMeta->pSerializer(pNode->handle.valueInUse, &pSHead);
} }
@ -343,10 +330,10 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
ASSERT(nBytes == pSHead->ssize); ASSERT(nBytes == pSHead->ssize);
} }
if(pNode->handle.valueInTxn) { if (pNode->handle.valueInTxn) {
pSHead->action = TQ_ACTION_INTXN; pSHead->action = TQ_ACTION_INTXN;
if(pNode->handle.valueInTxn == TQ_DELETE_TOKEN) { if (pNode->handle.valueInTxn == TQ_DELETE_TOKEN) {
pSHead->ssize = sizeof(TqSerializedHead); pSHead->ssize = sizeof(STqSerializedHead);
} else { } else {
pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead); pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead);
} }
@ -357,42 +344,39 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
pNode->handle.offset = offset; pNode->handle.offset = offset;
offset += nBytes; offset += nBytes;
//write idx file // write idx file
//TODO: endian check and convert // TODO: endian check and convert
*(bufPtr++) = pNode->handle.key; *(bufPtr++) = pNode->handle.key;
*(bufPtr++) = pNode->handle.offset; *(bufPtr++) = pNode->handle.offset;
*(bufPtr++) = (int64_t)nBytes; *(bufPtr++) = (int64_t)nBytes;
idxBuf.head.writeOffset += TQ_IDX_SIZE; idxBuf.head.writeOffset += TQ_IDX_SIZE;
if(idxBuf.head.writeOffset >= TQ_PAGE_SIZE) { if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) {
nBytes = write(pMeta->idxFd, &idxBuf, TQ_PAGE_SIZE); nBytes = write(pMeta->idxFd, &idxBuf, TQ_PAGE_SIZE);
//TODO: handle error with tfile // TODO: handle error with tfile
ASSERT(nBytes == TQ_PAGE_SIZE); ASSERT(nBytes == TQ_PAGE_SIZE);
memset(&idxBuf, 0, TQ_PAGE_SIZE); memset(&idxBuf, 0, TQ_PAGE_SIZE);
idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
bufPtr = (int64_t*)&idxBuf.buffer; bufPtr = (int64_t*)&idxBuf.buffer;
} }
//remove from unpersist list // remove from unpersist list
pHead->unpersistNext = pNode->unpersistNext; pHead->unpersistNext = pNode->unpersistNext;
pHead->unpersistNext->unpersistPrev = pHead; pHead->unpersistNext->unpersistPrev = pHead;
pNode->unpersistPrev = pNode->unpersistNext = NULL; pNode->unpersistPrev = pNode->unpersistNext = NULL;
pNode = pHead->unpersistNext; pNode = pHead->unpersistNext;
//remove from bucket // remove from bucket
if(pNode->handle.valueInUse == TQ_DELETE_TOKEN && if (pNode->handle.valueInUse == TQ_DELETE_TOKEN && pNode->handle.valueInTxn == NULL) {
pNode->handle.valueInTxn == NULL int bucketKey = pNode->handle.key & TQ_BUCKET_MASK;
) {
int bucketKey = pNode->handle.key & TQ_BUCKET_MASK;
STqMetaList* pBucketHead = pMeta->bucket[bucketKey]; STqMetaList* pBucketHead = pMeta->bucket[bucketKey];
if(pBucketHead == pNode) { if (pBucketHead == pNode) {
pMeta->bucket[bucketKey] = pNode->next; pMeta->bucket[bucketKey] = pNode->next;
} else { } else {
STqMetaList* pBucketNode = pBucketHead; STqMetaList* pBucketNode = pBucketHead;
while(pBucketNode->next != NULL while (pBucketNode->next != NULL && pBucketNode->next != pNode) {
&& pBucketNode->next != pNode) { pBucketNode = pBucketNode->next;
pBucketNode = pBucketNode->next;
} }
//impossible for pBucket->next == NULL // impossible for pBucket->next == NULL
ASSERT(pBucketNode->next == pNode); ASSERT(pBucketNode->next == pNode);
pBucketNode->next = pNode->next; pBucketNode->next = pNode->next;
} }
@ -400,46 +384,45 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
} }
} }
//write left bytes // write left bytes
free(pSHead); free(pSHead);
//TODO: write new version in tfile // TODO: write new version in tfile
if((char*)bufPtr != idxBuf.buffer) { if ((char*)bufPtr != idxBuf.buffer) {
int nBytes = write(pMeta->idxFd, &idxBuf, idxBuf.head.writeOffset); int nBytes = write(pMeta->idxFd, &idxBuf, idxBuf.head.writeOffset);
//TODO: handle error in tfile // TODO: handle error in tfile
ASSERT(nBytes == idxBuf.head.writeOffset); ASSERT(nBytes == idxBuf.head.writeOffset);
} }
//TODO: using fsync in tfile // TODO: using fsync in tfile
fsync(pMeta->idxFd); fsync(pMeta->idxFd);
fsync(pMeta->fileFd); fsync(pMeta->fileFd);
return 0; return 0;
} }
static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* value) { static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* value) {
int64_t bucketKey = key & TQ_BUCKET_MASK; int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey]; STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) { while (pNode) {
if(pNode->handle.key == key) { if (pNode->handle.key == key) {
//TODO: think about thread safety // TODO: think about thread safety
if(pNode->handle.valueInUse if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
&& pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInUse); pMeta->pDeleter(pNode->handle.valueInUse);
} }
//change pointer ownership // change pointer ownership
pNode->handle.valueInUse = value; pNode->handle.valueInUse = value;
return 0; return 0;
} else { } else {
pNode = pNode->next; pNode = pNode->next;
} }
} }
STqMetaList *pNewNode = malloc(sizeof(STqMetaList)); STqMetaList* pNewNode = malloc(sizeof(STqMetaList));
if(pNewNode == NULL) { if (pNewNode == NULL) {
//TODO: memory error // TODO: memory error
return -1; return -1;
} }
memset(pNewNode, 0, sizeof(STqMetaList)); memset(pNewNode, 0, sizeof(STqMetaList));
pNewNode->handle.key = key; pNewNode->handle.key = key;
pNewNode->handle.valueInUse = value; pNewNode->handle.valueInUse = value;
//put into unpersist list // put into unpersist list
pNewNode->unpersistPrev = pMeta->unpersistHead; pNewNode->unpersistPrev = pMeta->unpersistHead;
pNewNode->unpersistNext = pMeta->unpersistHead->unpersistNext; pNewNode->unpersistNext = pMeta->unpersistHead->unpersistNext;
pMeta->unpersistHead->unpersistNext->unpersistPrev = pNewNode; pMeta->unpersistHead->unpersistNext->unpersistPrev = pNewNode;
@ -448,12 +431,11 @@ static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* valu
} }
void* tqHandleGet(STqMetaStore* pMeta, int64_t key) { void* tqHandleGet(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK; int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey]; STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) { while (pNode) {
if(pNode->handle.key == key) { if (pNode->handle.key == key) {
if(pNode->handle.valueInUse != NULL if (pNode->handle.valueInUse != NULL && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
&& pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
return pNode->handle.valueInUse; return pNode->handle.valueInUse;
} else { } else {
return NULL; return NULL;
@ -466,12 +448,11 @@ void* tqHandleGet(STqMetaStore* pMeta, int64_t key) {
} }
void* tqHandleTouchGet(STqMetaStore* pMeta, int64_t key) { void* tqHandleTouchGet(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK; int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey]; STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) { while (pNode) {
if(pNode->handle.key == key) { if (pNode->handle.key == key) {
if(pNode->handle.valueInUse != NULL if (pNode->handle.valueInUse != NULL && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
&& pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
tqLinkUnpersist(pMeta, pNode); tqLinkUnpersist(pMeta, pNode);
return pNode->handle.valueInUse; return pNode->handle.valueInUse;
} else { } else {
@ -485,16 +466,16 @@ void* tqHandleTouchGet(STqMetaStore* pMeta, int64_t key) {
} }
static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* value) { static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* value) {
int64_t bucketKey = key & TQ_BUCKET_MASK; int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey]; STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) { while (pNode) {
if(pNode->handle.key == key) { if (pNode->handle.key == key) {
//TODO: think about thread safety // TODO: think about thread safety
if(pNode->handle.valueInTxn) { if (pNode->handle.valueInTxn) {
if(TqDupIntxnReject(pMeta->tqConfigFlag)) { if (TqDupIntxnReject(pMeta->tqConfigFlag)) {
return -2; return -2;
} }
if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInTxn); pMeta->pDeleter(pNode->handle.valueInTxn);
} }
} }
@ -505,9 +486,9 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va
pNode = pNode->next; pNode = pNode->next;
} }
} }
STqMetaList *pNewNode = malloc(sizeof(STqMetaList)); STqMetaList* pNewNode = malloc(sizeof(STqMetaList));
if(pNewNode == NULL) { if (pNewNode == NULL) {
//TODO: memory error // TODO: memory error
return -1; return -1;
} }
memset(pNewNode, 0, sizeof(STqMetaList)); memset(pNewNode, 0, sizeof(STqMetaList));
@ -519,14 +500,12 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va
return 0; return 0;
} }
int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { return tqHandlePutImpl(pMeta, key, value); }
return tqHandlePutImpl(pMeta, key, value);
}
int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) { int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) {
void *vmem = malloc(vsize); void* vmem = malloc(vsize);
if(vmem == NULL) { if (vmem == NULL) {
//TODO: memory error // TODO: memory error
return -1; return -1;
} }
memcpy(vmem, value, vsize); memcpy(vmem, value, vsize);
@ -534,12 +513,11 @@ int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vs
} }
static void* tqHandleGetUncommitted(STqMetaStore* pMeta, int64_t key) { static void* tqHandleGetUncommitted(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK; int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey]; STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) { while (pNode) {
if(pNode->handle.key == key) { if (pNode->handle.key == key) {
if(pNode->handle.valueInTxn != NULL if (pNode->handle.valueInTxn != NULL && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
&& pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
return pNode->handle.valueInTxn; return pNode->handle.valueInTxn;
} else { } else {
return NULL; return NULL;
@ -552,15 +530,14 @@ static void* tqHandleGetUncommitted(STqMetaStore* pMeta, int64_t key) {
} }
int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) { int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK; int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey]; STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) { while (pNode) {
if(pNode->handle.key == key) { if (pNode->handle.key == key) {
if(pNode->handle.valueInTxn == NULL) { if (pNode->handle.valueInTxn == NULL) {
return -1; return -1;
} }
if(pNode->handle.valueInUse if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
&& pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInUse); pMeta->pDeleter(pNode->handle.valueInUse);
} }
pNode->handle.valueInUse = pNode->handle.valueInTxn; pNode->handle.valueInUse = pNode->handle.valueInTxn;
@ -575,12 +552,12 @@ int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) {
} }
int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) { int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK; int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey]; STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) { while (pNode) {
if(pNode->handle.key == key) { if (pNode->handle.key == key) {
if(pNode->handle.valueInTxn) { if (pNode->handle.valueInTxn) {
if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInTxn); pMeta->pDeleter(pNode->handle.valueInTxn);
} }
pNode->handle.valueInTxn = NULL; pNode->handle.valueInTxn = NULL;
@ -596,13 +573,14 @@ int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) {
} }
int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) { int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK; int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey]; STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) { while (pNode) {
if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
if(pNode->handle.valueInTxn) { if (pNode->handle.valueInTxn) {
pMeta->pDeleter(pNode->handle.valueInTxn); pMeta->pDeleter(pNode->handle.valueInTxn);
} }
pNode->handle.valueInTxn = TQ_DELETE_TOKEN; pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
tqLinkUnpersist(pMeta, pNode); tqLinkUnpersist(pMeta, pNode);
return 0; return 0;
@ -610,11 +588,9 @@ int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) {
pNode = pNode->next; pNode = pNode->next;
} }
} }
//no such key // no such key
return -1; return -1;
} }
//TODO: clean deleted idx and data from persistent file // TODO: clean deleted idx and data from persistent file
int32_t tqStoreCompact(STqMetaStore *pMeta) { int32_t tqStoreCompact(STqMetaStore* pMeta) { return 0; }
return 0;
}

10
source/dnode/vnode/tq/test/tqMetaTest.cpp
View File

@ -9,17 +9,17 @@ struct Foo {
int32_t a; int32_t a;
}; };
int FooSerializer(const void* pObj, TqSerializedHead** ppHead) { int FooSerializer(const void* pObj, STqSerializedHead** ppHead) {
Foo* foo = (Foo*) pObj; Foo* foo = (Foo*) pObj;
if((*ppHead) == NULL || (*ppHead)->ssize < sizeof(TqSerializedHead) + sizeof(int32_t)) { if((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) {
*ppHead = (TqSerializedHead*)realloc(*ppHead, sizeof(TqSerializedHead) + sizeof(int32_t)); *ppHead = (STqSerializedHead*)realloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t));
(*ppHead)->ssize = sizeof(TqSerializedHead) + sizeof(int32_t); (*ppHead)->ssize = sizeof(STqSerializedHead) + sizeof(int32_t);
} }
*(int32_t*)(*ppHead)->content = foo->a; *(int32_t*)(*ppHead)->content = foo->a;
return (*ppHead)->ssize; return (*ppHead)->ssize;
} }
const void* FooDeserializer(const TqSerializedHead* pHead, void** ppObj) { const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) {
if(*ppObj == NULL) { if(*ppObj == NULL) {
*ppObj = realloc(*ppObj, sizeof(int32_t)); *ppObj = realloc(*ppObj, sizeof(int32_t));
} }

123
source/libs/wal/src/walMeta.c
View File

@ -14,27 +14,21 @@
*/ */
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "cJSON.h"
#include "os.h" #include "os.h"
#include "taoserror.h" #include "taoserror.h"
#include "tref.h"
#include "tfile.h" #include "tfile.h"
#include "cJSON.h" #include "tref.h"
#include "walInt.h" #include "walInt.h"
#include <libgen.h> #include <libgen.h>
#include <regex.h> #include <regex.h>
int64_t inline walGetFirstVer(SWal *pWal) { int64_t inline walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; }
return pWal->vers.firstVer;
}
int64_t inline walGetSnaphostVer(SWal *pWal) { int64_t inline walGetSnaphostVer(SWal* pWal) { return pWal->vers.snapshotVer; }
return pWal->vers.snapshotVer;
}
int64_t inline walGetLastVer(SWal *pWal) { int64_t inline walGetLastVer(SWal* pWal) { return pWal->vers.lastVer; }
return pWal->vers.lastVer;
}
static inline int walBuildMetaName(SWal* pWal, int metaVer, char* buf) { static inline int walBuildMetaName(SWal* pWal, int metaVer, char* buf) {
return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer);
@ -44,31 +38,30 @@ int walCheckAndRepairMeta(SWal* pWal) {
// load log files, get first/snapshot/last version info // load log files, get first/snapshot/last version info
const char* logPattern = "^[0-9]+.log$"; const char* logPattern = "^[0-9]+.log$";
const char* idxPattern = "^[0-9]+.idx$"; const char* idxPattern = "^[0-9]+.idx$";
regex_t logRegPattern; regex_t logRegPattern;
regex_t idxRegPattern; regex_t idxRegPattern;
SArray* pLogArray = taosArrayInit(8, sizeof(int64_t)); SArray* pLogArray = taosArrayInit(8, sizeof(int64_t));
regcomp(&logRegPattern, logPattern, REG_EXTENDED); regcomp(&logRegPattern, logPattern, REG_EXTENDED);
regcomp(&idxRegPattern, idxPattern, REG_EXTENDED); regcomp(&idxRegPattern, idxPattern, REG_EXTENDED);
DIR *dir = opendir(pWal->path); DIR* dir = opendir(pWal->path);
if(dir == NULL) { if (dir == NULL) {
wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno));
return -1; return -1;
} }
struct dirent* ent; struct dirent* ent;
while((ent = readdir(dir)) != NULL) { while ((ent = readdir(dir)) != NULL) {
char *name = basename(ent->d_name); char* name = basename(ent->d_name);
int code = regexec(&logRegPattern, name, 0, NULL, 0); int code = regexec(&logRegPattern, name, 0, NULL, 0);
if(code == 0) { if (code == 0) {
int64_t firstVer; int64_t firstVer;
sscanf(name, "%" PRId64 ".log", &firstVer); sscanf(name, "%" PRId64 ".log", &firstVer);
taosArrayPush(pLogArray, &firstVer); taosArrayPush(pLogArray, &firstVer);
} }
} }
// load meta // load meta
// if not match, or meta missing // if not match, or meta missing
// rebuild meta // rebuild meta
@ -85,15 +78,15 @@ int walRollFileInfo(SWal* pWal) {
int64_t ts = taosGetTimestampSec(); int64_t ts = taosGetTimestampSec();
SArray* pArray = pWal->fileInfoSet; SArray* pArray = pWal->fileInfoSet;
if(taosArrayGetSize(pArray) != 0) { if (taosArrayGetSize(pArray) != 0) {
WalFileInfo *pInfo = taosArrayGetLast(pArray); WalFileInfo* pInfo = taosArrayGetLast(pArray);
pInfo->lastVer = pWal->vers.lastVer; pInfo->lastVer = pWal->vers.lastVer;
pInfo->closeTs = ts; pInfo->closeTs = ts;
} }
//TODO: change to emplace back // TODO: change to emplace back
WalFileInfo *pNewInfo = malloc(sizeof(WalFileInfo)); WalFileInfo* pNewInfo = malloc(sizeof(WalFileInfo));
if(pNewInfo == NULL) { if (pNewInfo == NULL) {
return -1; return -1;
} }
pNewInfo->firstVer = pWal->vers.lastVer + 1; pNewInfo->firstVer = pWal->vers.lastVer + 1;
@ -109,13 +102,13 @@ int walRollFileInfo(SWal* pWal) {
char* walMetaSerialize(SWal* pWal) { char* walMetaSerialize(SWal* pWal) {
char buf[30]; char buf[30];
ASSERT(pWal->fileInfoSet); ASSERT(pWal->fileInfoSet);
int sz = pWal->fileInfoSet->size; int sz = pWal->fileInfoSet->size;
cJSON* pRoot = cJSON_CreateObject(); cJSON* pRoot = cJSON_CreateObject();
cJSON* pMeta = cJSON_CreateObject(); cJSON* pMeta = cJSON_CreateObject();
cJSON* pFiles = cJSON_CreateArray(); cJSON* pFiles = cJSON_CreateArray();
cJSON* pField; cJSON* pField;
if(pRoot == NULL || pMeta == NULL || pFiles == NULL) { if (pRoot == NULL || pMeta == NULL || pFiles == NULL) {
//TODO // TODO
return NULL; return NULL;
} }
cJSON_AddItemToObject(pRoot, "meta", pMeta); cJSON_AddItemToObject(pRoot, "meta", pMeta);
@ -130,15 +123,15 @@ char* walMetaSerialize(SWal* pWal) {
cJSON_AddItemToObject(pRoot, "files", pFiles); cJSON_AddItemToObject(pRoot, "files", pFiles);
WalFileInfo* pData = pWal->fileInfoSet->pData; WalFileInfo* pData = pWal->fileInfoSet->pData;
for(int i = 0; i < sz; i++) { for (int i = 0; i < sz; i++) {
WalFileInfo* pInfo = &pData[i]; WalFileInfo* pInfo = &pData[i];
cJSON_AddItemToArray(pFiles, pField = cJSON_CreateObject()); cJSON_AddItemToArray(pFiles, pField = cJSON_CreateObject());
if(pField == NULL) { if (pField == NULL) {
cJSON_Delete(pRoot); cJSON_Delete(pRoot);
return NULL; return NULL;
} }
//cjson only support int32_t or double // cjson only support int32_t or double
//string are used to prohibit the loss of precision // string are used to prohibit the loss of precision
sprintf(buf, "%" PRId64, pInfo->firstVer); sprintf(buf, "%" PRId64, pInfo->firstVer);
cJSON_AddStringToObject(pField, "firstVer", buf); cJSON_AddStringToObject(pField, "firstVer", buf);
sprintf(buf, "%" PRId64, pInfo->lastVer); sprintf(buf, "%" PRId64, pInfo->lastVer);
@ -171,12 +164,12 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) {
pFiles = cJSON_GetObjectItem(pRoot, "files"); pFiles = cJSON_GetObjectItem(pRoot, "files");
int sz = cJSON_GetArraySize(pFiles); int sz = cJSON_GetArraySize(pFiles);
//deserialize // deserialize
SArray* pArray = pWal->fileInfoSet; SArray* pArray = pWal->fileInfoSet;
taosArrayEnsureCap(pArray, sz); taosArrayEnsureCap(pArray, sz);
WalFileInfo *pData = pArray->pData; WalFileInfo* pData = pArray->pData;
for(int i = 0; i < sz; i++) { for (int i = 0; i < sz; i++) {
cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i);
WalFileInfo* pInfo = &pData[i]; WalFileInfo* pInfo = &pData[i];
pField = cJSON_GetObjectItem(pInfoJson, "firstVer"); pField = cJSON_GetObjectItem(pInfoJson, "firstVer");
pInfo->firstVer = atoll(cJSON_GetStringValue(pField)); pInfo->firstVer = atoll(cJSON_GetStringValue(pField));
@ -196,24 +189,24 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) {
} }
static int walFindCurMetaVer(SWal* pWal) { static int walFindCurMetaVer(SWal* pWal) {
const char * pattern = "^meta-ver[0-9]+$"; const char* pattern = "^meta-ver[0-9]+$";
regex_t walMetaRegexPattern; regex_t walMetaRegexPattern;
regcomp(&walMetaRegexPattern, pattern, REG_EXTENDED); regcomp(&walMetaRegexPattern, pattern, REG_EXTENDED);
DIR *dir = opendir(pWal->path); DIR* dir = opendir(pWal->path);
if(dir == NULL) { if (dir == NULL) {
wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno));
return -1; return -1;
} }
struct dirent* ent; struct dirent* ent;
//find existing meta-ver[x].json // find existing meta-ver[x].json
int metaVer = -1; int metaVer = -1;
while((ent = readdir(dir)) != NULL) { while ((ent = readdir(dir)) != NULL) {
char *name = basename(ent->d_name); char* name = basename(ent->d_name);
int code = regexec(&walMetaRegexPattern, name, 0, NULL, 0); int code = regexec(&walMetaRegexPattern, name, 0, NULL, 0);
if(code == 0) { if (code == 0) {
sscanf(name, "meta-ver%d", &metaVer); sscanf(name, "meta-ver%d", &metaVer);
break; break;
} }
@ -224,23 +217,23 @@ static int walFindCurMetaVer(SWal* pWal) {
} }
int walSaveMeta(SWal* pWal) { int walSaveMeta(SWal* pWal) {
int metaVer = walFindCurMetaVer(pWal); int metaVer = walFindCurMetaVer(pWal);
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
walBuildMetaName(pWal, metaVer+1, fnameStr); walBuildMetaName(pWal, metaVer + 1, fnameStr);
int metaTfd = tfOpenCreateWrite(fnameStr); int metaTfd = tfOpenCreateWrite(fnameStr);
if(metaTfd < 0) { if (metaTfd < 0) {
return -1; return -1;
} }
char* serialized = walMetaSerialize(pWal); char* serialized = walMetaSerialize(pWal);
int len = strlen(serialized); int len = strlen(serialized);
if(len != tfWrite(metaTfd, serialized, len)) { if (len != tfWrite(metaTfd, serialized, len)) {
//TODO:clean file // TODO:clean file
return -1; return -1;
} }
tfClose(metaTfd); tfClose(metaTfd);
//delete old file // delete old file
if(metaVer > -1) { if (metaVer > -1) {
walBuildMetaName(pWal, metaVer, fnameStr); walBuildMetaName(pWal, metaVer, fnameStr);
remove(fnameStr); remove(fnameStr);
} }
@ -250,30 +243,30 @@ int walSaveMeta(SWal* pWal) {
int walLoadMeta(SWal* pWal) { int walLoadMeta(SWal* pWal) {
ASSERT(pWal->fileInfoSet->size == 0); ASSERT(pWal->fileInfoSet->size == 0);
//find existing meta file // find existing meta file
int metaVer = walFindCurMetaVer(pWal); int metaVer = walFindCurMetaVer(pWal);
if(metaVer == -1) { if (metaVer == -1) {
return 0; return 0;
} }
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
walBuildMetaName(pWal, metaVer, fnameStr); walBuildMetaName(pWal, metaVer, fnameStr);
//read metafile // read metafile
struct stat statbuf; struct stat statbuf;
stat(fnameStr, &statbuf); stat(fnameStr, &statbuf);
int size = statbuf.st_size; int size = statbuf.st_size;
char* buf = malloc(size + 5); char* buf = malloc(size + 5);
if(buf == NULL) { if (buf == NULL) {
return -1; return -1;
} }
memset(buf, 0, size+5); memset(buf, 0, size + 5);
int tfd = tfOpenRead(fnameStr); int tfd = tfOpenRead(fnameStr);
if(tfRead(tfd, buf, size) != size) { if (tfRead(tfd, buf, size) != size) {
free(buf); free(buf);
return -1; return -1;
} }
//load into fileInfoSet // load into fileInfoSet
int code = walMetaDeserialize(pWal, buf); int code = walMetaDeserialize(pWal, buf);
if(code != 0) { if (code != 0) {
free(buf); free(buf);
return -1; return -1;
} }

46
source/libs/wal/src/walMgmt.c
View File

@ -14,11 +14,11 @@
*/ */
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "compare.h"
#include "os.h" #include "os.h"
#include "taoserror.h" #include "taoserror.h"
#include "tref.h"
#include "tfile.h" #include "tfile.h"
#include "compare.h" #include "tref.h"
#include "walInt.h" #include "walInt.h"
typedef struct { typedef struct {
@ -34,16 +34,14 @@ static int32_t walCreateThread();
static void walStopThread(); static void walStopThread();
static void walFreeObj(void *pWal); static void walFreeObj(void *pWal);
int64_t walGetSeq() { int64_t walGetSeq() { return (int64_t)atomic_load_32(&tsWal.seq); }
return (int64_t)atomic_load_32(&tsWal.seq);
}
int32_t walInit() { int32_t walInit() {
int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 0, 1); int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 0, 1);
if(old == 1) return 0; if (old == 1) return 0;
int code = tfInit(); int code = tfInit();
if(code != 0) { if (code != 0) {
wError("failed to init tfile since %s", tstrerror(code)); wError("failed to init tfile since %s", tstrerror(code));
atomic_store_8(&tsWal.inited, 0); atomic_store_8(&tsWal.inited, 0);
return code; return code;
@ -63,7 +61,7 @@ int32_t walInit() {
void walCleanUp() { void walCleanUp() {
int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 1, 0); int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 1, 0);
if(old == 0) { if (old == 0) {
return; return;
} }
walStopThread(); walStopThread();
@ -78,52 +76,52 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
return NULL; return NULL;
} }
//set config // set config
memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg)); memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg));
pWal->fsyncSeq = pCfg->fsyncPeriod / 1000; pWal->fsyncSeq = pCfg->fsyncPeriod / 1000;
if(pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1; if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1;
tstrncpy(pWal->path, path, sizeof(pWal->path)); tstrncpy(pWal->path, path, sizeof(pWal->path));
if(taosMkDir(pWal->path) != 0) { if (taosMkDir(pWal->path) != 0) {
wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno));
return NULL; return NULL;
} }
//open meta // open meta
walResetVer(&pWal->vers); walResetVer(&pWal->vers);
pWal->writeLogTfd = -1; pWal->writeLogTfd = -1;
pWal->writeIdxTfd = -1; pWal->writeIdxTfd = -1;
pWal->writeCur = -1; pWal->writeCur = -1;
pWal->fileInfoSet = taosArrayInit(8, sizeof(WalFileInfo)); pWal->fileInfoSet = taosArrayInit(8, sizeof(WalFileInfo));
if(pWal->fileInfoSet == NULL) { if (pWal->fileInfoSet == NULL) {
wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno)); wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno));
free(pWal); free(pWal);
return NULL; return NULL;
} }
//init status // init status
pWal->totSize = 0; pWal->totSize = 0;
pWal->lastRollSeq = -1; pWal->lastRollSeq = -1;
//init write buffer // init write buffer
memset(&pWal->writeHead, 0, sizeof(SWalHead)); memset(&pWal->writeHead, 0, sizeof(SWalHead));
pWal->writeHead.head.headVer = WAL_HEAD_VER; pWal->writeHead.head.headVer = WAL_HEAD_VER;
if(pthread_mutex_init(&pWal->mutex, NULL) < 0) { if (pthread_mutex_init(&pWal->mutex, NULL) < 0) {
taosArrayDestroy(pWal->fileInfoSet); taosArrayDestroy(pWal->fileInfoSet);
free(pWal); free(pWal);
return NULL; return NULL;
} }
pWal->refId = taosAddRef(tsWal.refSetId, pWal); pWal->refId = taosAddRef(tsWal.refSetId, pWal);
if(pWal->refId < 0) { if (pWal->refId < 0) {
pthread_mutex_destroy(&pWal->mutex); pthread_mutex_destroy(&pWal->mutex);
taosArrayDestroy(pWal->fileInfoSet); taosArrayDestroy(pWal->fileInfoSet);
free(pWal); free(pWal);
return NULL; return NULL;
} }
if(walLoadMeta(pWal) < 0 && walCheckAndRepairMeta(pWal) < 0) { if (walLoadMeta(pWal) < 0 && walCheckAndRepairMeta(pWal) < 0) {
taosRemoveRef(tsWal.refSetId, pWal->refId); taosRemoveRef(tsWal.refSetId, pWal->refId);
pthread_mutex_destroy(&pWal->mutex); pthread_mutex_destroy(&pWal->mutex);
taosArrayDestroy(pWal->fileInfoSet); taosArrayDestroy(pWal->fileInfoSet);
@ -131,11 +129,11 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
return NULL; return NULL;
} }
if(walCheckAndRepairIdx(pWal) < 0) { if (walCheckAndRepairIdx(pWal) < 0) {
} }
wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, pWal->cfg.fsyncPeriod); wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level,
pWal->cfg.fsyncPeriod);
return pWal; return pWal;
} }
@ -203,10 +201,12 @@ static void walFsyncAll() {
SWal *pWal = taosIterateRef(tsWal.refSetId, 0); SWal *pWal = taosIterateRef(tsWal.refSetId, 0);
while (pWal) { while (pWal) {
if (walNeedFsync(pWal)) { if (walNeedFsync(pWal)) {
wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq, atomic_load_32(&tsWal.seq)); wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq,
atomic_load_32(&tsWal.seq));
int32_t code = tfFsync(pWal->writeLogTfd); int32_t code = tfFsync(pWal->writeLogTfd);
if (code != 0) { if (code != 0) {
wError("vgId:%d, file:%"PRId64".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(code)); wError("vgId:%d, file:%" PRId64 ".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(code));
} }
} }
pWal = taosIterateRef(tsWal.refSetId, pWal->refId); pWal = taosIterateRef(tsWal.refSetId, pWal->refId);

101
source/libs/wal/src/walRead.c
View File

@ -13,12 +13,12 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "walInt.h"
#include "tfile.h" #include "tfile.h"
#include "walInt.h"
SWalReadHandle* walOpenReadHandle(SWal* pWal) { SWalReadHandle *walOpenReadHandle(SWal *pWal) {
SWalReadHandle *pRead = malloc(sizeof(SWalReadHandle)); SWalReadHandle *pRead = malloc(sizeof(SWalReadHandle));
if(pRead == NULL) { if (pRead == NULL) {
return NULL; return NULL;
} }
pRead->pWal = pWal; pRead->pWal = pWal;
@ -29,7 +29,7 @@ SWalReadHandle* walOpenReadHandle(SWal* pWal) {
pRead->capacity = 0; pRead->capacity = 0;
pRead->status = 0; pRead->status = 0;
pRead->pHead = malloc(sizeof(SWalHead)); pRead->pHead = malloc(sizeof(SWalHead));
if(pRead->pHead == NULL) { if (pRead->pHead == NULL) {
free(pRead); free(pRead);
return NULL; return NULL;
} }
@ -43,27 +43,25 @@ void walCloseReadHandle(SWalReadHandle *pRead) {
free(pRead); free(pRead);
} }
int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; }
return 0;
}
static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) { static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) {
int code = 0; int code = 0;
int64_t idxTfd = pRead->readIdxTfd; int64_t idxTfd = pRead->readIdxTfd;
int64_t logTfd = pRead->readLogTfd; int64_t logTfd = pRead->readLogTfd;
//seek position // seek position
int64_t offset = (ver - fileFirstVer) * sizeof(WalIdxEntry); int64_t offset = (ver - fileFirstVer) * sizeof(WalIdxEntry);
code = tfLseek(idxTfd, offset, SEEK_SET); code = tfLseek(idxTfd, offset, SEEK_SET);
if(code < 0) { if (code < 0) {
return -1; return -1;
} }
WalIdxEntry entry; WalIdxEntry entry;
if(tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) { if (tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) {
return -1; return -1;
} }
//TODO:deserialize // TODO:deserialize
ASSERT(entry.ver == ver); ASSERT(entry.ver == ver);
code = tfLseek(logTfd, entry.offset, SEEK_SET); code = tfLseek(logTfd, entry.offset, SEEK_SET);
if (code < 0) { if (code < 0) {
@ -80,13 +78,13 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
walBuildLogName(pRead->pWal, fileFirstVer, fnameStr); walBuildLogName(pRead->pWal, fileFirstVer, fnameStr);
int64_t logTfd = tfOpenRead(fnameStr); int64_t logTfd = tfOpenRead(fnameStr);
if(logTfd < 0) { if (logTfd < 0) {
return -1; return -1;
} }
walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr); walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr);
int64_t idxTfd = tfOpenRead(fnameStr); int64_t idxTfd = tfOpenRead(fnameStr);
if(idxTfd < 0) { if (idxTfd < 0) {
return -1; return -1;
} }
@ -96,76 +94,75 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
} }
static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) {
int code; int code;
SWal *pWal = pRead->pWal; SWal *pWal = pRead->pWal;
if(ver == pRead->curVersion) { if (ver == pRead->curVersion) {
return 0; return 0;
} }
if(ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) {
return -1; return -1;
} }
if(ver < pWal->vers.snapshotVer) { if (ver < pWal->vers.snapshotVer) {
} }
WalFileInfo tmpInfo; WalFileInfo tmpInfo;
tmpInfo.firstVer = ver; tmpInfo.firstVer = ver;
//bsearch in fileSet // bsearch in fileSet
WalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); WalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
ASSERT(pRet != NULL); ASSERT(pRet != NULL);
if(pRead->curFileFirstVer != pRet->firstVer) { if (pRead->curFileFirstVer != pRet->firstVer) {
code = walReadChangeFile(pRead, pRet->firstVer); code = walReadChangeFile(pRead, pRet->firstVer);
if(code < 0) { if (code < 0) {
//TODO: set error flag // TODO: set error flag
return -1; return -1;
} }
} }
code = walReadSeekFilePos(pRead, pRet->firstVer, ver); code = walReadSeekFilePos(pRead, pRet->firstVer, ver);
if(code < 0) { if (code < 0) {
return -1; return -1;
} }
pRead->curVersion = ver; pRead->curVersion = ver;
return 0; return 0;
} }
int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
int code; int code;
//TODO: check wal life // TODO: check wal life
if(pRead->curVersion != ver) { if (pRead->curVersion != ver) {
code = walReadSeekVer(pRead, ver); code = walReadSeekVer(pRead, ver);
if(code != 0) { if (code != 0) {
return -1; return -1;
} }
} }
if(!tfValid(pRead->readLogTfd)) return -1; if (!tfValid(pRead->readLogTfd)) return -1;
code = tfRead(pRead->readLogTfd, pRead->pHead, sizeof(SWalHead)); code = tfRead(pRead->readLogTfd, pRead->pHead, sizeof(SWalHead));
if(code != sizeof(SWalHead)) { if (code != sizeof(SWalHead)) {
return -1; return -1;
} }
code = walValidHeadCksum(pRead->pHead); code = walValidHeadCksum(pRead->pHead);
if(code != 0) { if (code != 0) {
return -1; return -1;
} }
if(pRead->capacity < pRead->pHead->head.len) { if (pRead->capacity < pRead->pHead->head.len) {
void* ptr = realloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len); void *ptr = realloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len);
if(ptr == NULL) { if (ptr == NULL) {
return -1; return -1;
} }
pRead->pHead = ptr; pRead->pHead = ptr;
pRead->capacity = pRead->pHead->head.len; pRead->capacity = pRead->pHead->head.len;
} }
if(pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) { if (pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) {
return -1; return -1;
} }
/*code = walValidBodyCksum(pRead->pHead);*/ /*code = walValidBodyCksum(pRead->pHead);*/
ASSERT(pRead->pHead->head.version == ver); ASSERT(pRead->pHead->head.version == ver);
if(code != 0) { if (code != 0) {
return -1; return -1;
} }
pRead->curVersion++; pRead->curVersion++;
@ -176,40 +173,40 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) {
int code; int code;
code = walSeekVer(pWal, ver); code = walSeekVer(pWal, ver);
if(code != 0) { if (code != 0) {
return code; return code;
} }
if(*ppHead == NULL) { if (*ppHead == NULL) {
void* ptr = realloc(*ppHead, sizeof(SWalHead)); void *ptr = realloc(*ppHead, sizeof(SWalHead));
if(ptr == NULL) { if (ptr == NULL) {
return -1; return -1;
} }
*ppHead = ptr; *ppHead = ptr;
} }
if(tfRead(pWal->writeLogTfd, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) { if (tfRead(pWal->writeLogTfd, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
return -1; return -1;
} }
//TODO: endian compatibility processing after read // TODO: endian compatibility processing after read
if(walValidHeadCksum(*ppHead) != 0) { if (walValidHeadCksum(*ppHead) != 0) {
return -1; return -1;
} }
void* ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len); void *ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len);
if(ptr == NULL) { if (ptr == NULL) {
free(*ppHead); free(*ppHead);
*ppHead = NULL; *ppHead = NULL;
return -1; return -1;
} }
if(tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) { if (tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) {
return -1; return -1;
} }
//TODO: endian compatibility processing after read // TODO: endian compatibility processing after read
if(walValidBodyCksum(*ppHead) != 0) { if (walValidBodyCksum(*ppHead) != 0) {
return -1; return -1;
} }
return 0; return 0;
} }
/*int32_t walReadWithFp(SWal *pWal, FWalWrite writeFp, int64_t verStart, int32_t readNum) {*/ /*int32_t walReadWithFp(SWal *pWal, FWalWrite writeFp, int64_t verStart, int32_t readNum) {*/
/*return 0;*/ /*return 0;*/
/*}*/ /*}*/

59
source/libs/wal/src/walSeek.c
View File

@ -16,8 +16,8 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "taoserror.h" #include "taoserror.h"
#include "tref.h"
#include "tfile.h" #include "tfile.h"
#include "tref.h"
#include "walInt.h" #include "walInt.h"
static int walSeekFilePos(SWal* pWal, int64_t ver) { static int walSeekFilePos(SWal* pWal, int64_t ver) {
@ -25,17 +25,17 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) {
int64_t idxTfd = pWal->writeIdxTfd; int64_t idxTfd = pWal->writeIdxTfd;
int64_t logTfd = pWal->writeLogTfd; int64_t logTfd = pWal->writeLogTfd;
//seek position // seek position
int64_t idxOff = walGetVerIdxOffset(pWal, ver); int64_t idxOff = walGetVerIdxOffset(pWal, ver);
code = tfLseek(idxTfd, idxOff, SEEK_SET); code = tfLseek(idxTfd, idxOff, SEEK_SET);
if(code != 0) { if (code != 0) {
return -1; return -1;
} }
WalIdxEntry entry; WalIdxEntry entry;
//TODO:deserialize // TODO:deserialize
code = tfRead(idxTfd, &entry, sizeof(WalIdxEntry)); code = tfRead(idxTfd, &entry, sizeof(WalIdxEntry));
if(code != 0) { if (code != 0) {
return -1; return -1;
} }
ASSERT(entry.ver == ver); ASSERT(entry.ver == ver);
@ -46,8 +46,8 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) {
return code; return code;
} }
int walChangeFileToLast(SWal *pWal) { int walChangeFileToLast(SWal* pWal) {
int64_t idxTfd, logTfd; int64_t idxTfd, logTfd;
WalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); WalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet);
ASSERT(pRet != NULL); ASSERT(pRet != NULL);
int64_t fileFirstVer = pRet->firstVer; int64_t fileFirstVer = pRet->firstVer;
@ -55,42 +55,42 @@ int walChangeFileToLast(SWal *pWal) {
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, fileFirstVer, fnameStr); walBuildIdxName(pWal, fileFirstVer, fnameStr);
idxTfd = tfOpenReadWrite(fnameStr); idxTfd = tfOpenReadWrite(fnameStr);
if(idxTfd < 0) { if (idxTfd < 0) {
return -1; return -1;
} }
walBuildLogName(pWal, fileFirstVer, fnameStr); walBuildLogName(pWal, fileFirstVer, fnameStr);
logTfd = tfOpenReadWrite(fnameStr); logTfd = tfOpenReadWrite(fnameStr);
if(logTfd < 0) { if (logTfd < 0) {
return -1; return -1;
} }
//switch file // switch file
pWal->writeIdxTfd = idxTfd; pWal->writeIdxTfd = idxTfd;
pWal->writeLogTfd = logTfd; pWal->writeLogTfd = logTfd;
return 0; return 0;
} }
int walChangeFile(SWal *pWal, int64_t ver) { int walChangeFile(SWal* pWal, int64_t ver) {
int code = 0; int code = 0;
int64_t idxTfd, logTfd; int64_t idxTfd, logTfd;
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
code = tfClose(pWal->writeLogTfd); code = tfClose(pWal->writeLogTfd);
if(code != 0) { if (code != 0) {
//TODO // TODO
return -1; return -1;
} }
code = tfClose(pWal->writeIdxTfd); code = tfClose(pWal->writeIdxTfd);
if(code != 0) { if (code != 0) {
//TODO // TODO
return -1; return -1;
} }
WalFileInfo tmpInfo; WalFileInfo tmpInfo;
tmpInfo.firstVer = ver; tmpInfo.firstVer = ver;
//bsearch in fileSet // bsearch in fileSet
WalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); WalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
ASSERT(pRet != NULL); ASSERT(pRet != NULL);
int64_t fileFirstVer = pRet->firstVer; int64_t fileFirstVer = pRet->firstVer;
//closed // closed
if(taosArrayGetLast(pWal->fileInfoSet) != pRet) { if (taosArrayGetLast(pWal->fileInfoSet) != pRet) {
walBuildIdxName(pWal, fileFirstVer, fnameStr); walBuildIdxName(pWal, fileFirstVer, fnameStr);
idxTfd = tfOpenRead(fnameStr); idxTfd = tfOpenRead(fnameStr);
walBuildLogName(pWal, fileFirstVer, fnameStr); walBuildLogName(pWal, fileFirstVer, fnameStr);
@ -107,27 +107,26 @@ int walChangeFile(SWal *pWal, int64_t ver) {
return fileFirstVer; return fileFirstVer;
} }
int walSeekVer(SWal *pWal, int64_t ver) { int walSeekVer(SWal* pWal, int64_t ver) {
int code; int code;
if(ver == pWal->vers.lastVer) { if (ver == pWal->vers.lastVer) {
return 0; return 0;
} }
if(ver > pWal->vers.lastVer|| ver < pWal->vers.firstVer) { if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) {
return -1; return -1;
} }
if(ver < pWal->vers.snapshotVer) { if (ver < pWal->vers.snapshotVer) {
} }
if(ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) { if (ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) {
code = walChangeFile(pWal, ver); code = walChangeFile(pWal, ver);
if(code != 0) { if (code != 0) {
return -1; return -1;
} }
} }
code = walSeekFilePos(pWal, ver); code = walSeekFilePos(pWal, ver);
if(code != 0) { if (code != 0) {
return -1; return -1;
} }
return 0; return 0;
} }

221
source/libs/wal/src/walWrite.c
View File

@ -24,7 +24,7 @@
int32_t walCommit(SWal *pWal, int64_t ver) { int32_t walCommit(SWal *pWal, int64_t ver) {
ASSERT(pWal->vers.commitVer >= pWal->vers.snapshotVer); ASSERT(pWal->vers.commitVer >= pWal->vers.snapshotVer);
ASSERT(pWal->vers.commitVer <= pWal->vers.lastVer); ASSERT(pWal->vers.commitVer <= pWal->vers.lastVer);
if(ver < pWal->vers.commitVer || ver > pWal->vers.lastVer) { if (ver < pWal->vers.commitVer || ver > pWal->vers.lastVer) {
return -1; return -1;
} }
pWal->vers.commitVer = ver; pWal->vers.commitVer = ver;
@ -32,57 +32,57 @@ int32_t walCommit(SWal *pWal, int64_t ver) {
} }
int32_t walRollback(SWal *pWal, int64_t ver) { int32_t walRollback(SWal *pWal, int64_t ver) {
int code; int code;
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
if(ver == pWal->vers.lastVer) { if (ver == pWal->vers.lastVer) {
return 0; return 0;
} }
if(ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) { if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) {
return -1; return -1;
} }
pthread_mutex_lock(&pWal->mutex); pthread_mutex_lock(&pWal->mutex);
//find correct file // find correct file
if(ver < walGetLastFileFirstVer(pWal)) { if (ver < walGetLastFileFirstVer(pWal)) {
//close current files // close current files
tfClose(pWal->writeIdxTfd); tfClose(pWal->writeIdxTfd);
tfClose(pWal->writeLogTfd); tfClose(pWal->writeLogTfd);
//open old files // open old files
code = walChangeFile(pWal, ver); code = walChangeFile(pWal, ver);
if(code != 0) { if (code != 0) {
return -1; return -1;
} }
//delete files // delete files
int fileSetSize = taosArrayGetSize(pWal->fileInfoSet); int fileSetSize = taosArrayGetSize(pWal->fileInfoSet);
for(int i = pWal->writeCur; i < fileSetSize; i++) { for (int i = pWal->writeCur; i < fileSetSize; i++) {
walBuildLogName(pWal, ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); walBuildLogName(pWal, ((WalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr);
remove(fnameStr); remove(fnameStr);
walBuildIdxName(pWal, ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); walBuildIdxName(pWal, ((WalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr);
remove(fnameStr); remove(fnameStr);
} }
//pop from fileInfoSet // pop from fileInfoSet
taosArraySetSize(pWal->fileInfoSet, pWal->writeCur + 1); taosArraySetSize(pWal->fileInfoSet, pWal->writeCur + 1);
} }
walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
int64_t idxTfd = tfOpenReadWrite(fnameStr); int64_t idxTfd = tfOpenReadWrite(fnameStr);
//TODO:change to deserialize function // TODO:change to deserialize function
if(idxTfd < 0) { if (idxTfd < 0) {
pthread_mutex_unlock(&pWal->mutex); pthread_mutex_unlock(&pWal->mutex);
return -1; return -1;
} }
int64_t idxOff = walGetVerIdxOffset(pWal, ver); int64_t idxOff = walGetVerIdxOffset(pWal, ver);
code = tfLseek(idxTfd, idxOff, SEEK_SET); code = tfLseek(idxTfd, idxOff, SEEK_SET);
if(code < 0) { if (code < 0) {
pthread_mutex_unlock(&pWal->mutex); pthread_mutex_unlock(&pWal->mutex);
return -1; return -1;
} }
//read idx file and get log file pos // read idx file and get log file pos
//TODO:change to deserialize function // TODO:change to deserialize function
WalIdxEntry entry; WalIdxEntry entry;
if(tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) { if (tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) {
pthread_mutex_unlock(&pWal->mutex); pthread_mutex_unlock(&pWal->mutex);
return -1; return -1;
} }
@ -90,56 +90,56 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr); walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
int64_t logTfd = tfOpenReadWrite(fnameStr); int64_t logTfd = tfOpenReadWrite(fnameStr);
if(logTfd < 0) { if (logTfd < 0) {
//TODO // TODO
pthread_mutex_unlock(&pWal->mutex); pthread_mutex_unlock(&pWal->mutex);
return -1; return -1;
} }
code = tfLseek(logTfd, entry.offset, SEEK_SET); code = tfLseek(logTfd, entry.offset, SEEK_SET);
if(code < 0) { if (code < 0) {
//TODO // TODO
pthread_mutex_unlock(&pWal->mutex); pthread_mutex_unlock(&pWal->mutex);
return -1; return -1;
} }
//validate offset // validate offset
SWalHead head; SWalHead head;
ASSERT(tfValid(logTfd)); ASSERT(tfValid(logTfd));
int size = tfRead(logTfd, &head, sizeof(SWalHead)); int size = tfRead(logTfd, &head, sizeof(SWalHead));
if(size != sizeof(SWalHead)) { if (size != sizeof(SWalHead)) {
return -1; return -1;
} }
code = walValidHeadCksum(&head); code = walValidHeadCksum(&head);
ASSERT(code == 0); ASSERT(code == 0);
if(code != 0) { if (code != 0) {
return -1; return -1;
} }
if(head.head.version != ver) { if (head.head.version != ver) {
//TODO // TODO
return -1; return -1;
} }
//truncate old files // truncate old files
code = tfFtruncate(logTfd, entry.offset); code = tfFtruncate(logTfd, entry.offset);
if(code < 0) { if (code < 0) {
return -1; return -1;
} }
code = tfFtruncate(idxTfd, idxOff); code = tfFtruncate(idxTfd, idxOff);
if(code < 0) { if (code < 0) {
return -1; return -1;
} }
pWal->vers.lastVer = ver - 1; pWal->vers.lastVer = ver - 1;
((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; ((WalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1;
((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; ((WalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset;
//unlock // unlock
pthread_mutex_unlock(&pWal->mutex); pthread_mutex_unlock(&pWal->mutex);
return 0; return 0;
} }
int32_t walBeginSnapshot(SWal* pWal, int64_t ver) { int32_t walBeginSnapshot(SWal *pWal, int64_t ver) {
pWal->vers.verInSnapshotting = ver; pWal->vers.verInSnapshotting = ver;
//check file rolling // check file rolling
if(pWal->cfg.retentionPeriod == 0) { if (pWal->cfg.retentionPeriod == 0) {
walRoll(pWal); walRoll(pWal);
} }
@ -148,54 +148,54 @@ int32_t walBeginSnapshot(SWal* pWal, int64_t ver) {
int32_t walEndSnapshot(SWal *pWal) { int32_t walEndSnapshot(SWal *pWal) {
int64_t ver = pWal->vers.verInSnapshotting; int64_t ver = pWal->vers.verInSnapshotting;
if(ver == -1) return -1; if (ver == -1) return -1;
pWal->vers.snapshotVer = ver; pWal->vers.snapshotVer = ver;
int ts = taosGetTimestampSec(); int ts = taosGetTimestampSec();
int deleteCnt = 0; int deleteCnt = 0;
int64_t newTotSize = pWal->totSize; int64_t newTotSize = pWal->totSize;
WalFileInfo tmp; WalFileInfo tmp;
tmp.firstVer = ver; tmp.firstVer = ver;
//find files safe to delete // find files safe to delete
WalFileInfo* pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE); WalFileInfo *pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE);
if(ver >= pInfo->lastVer) { if (ver >= pInfo->lastVer) {
pInfo++; pInfo++;
} }
//iterate files, until the searched result // iterate files, until the searched result
for(WalFileInfo* iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { for (WalFileInfo *iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) {
if(pWal->totSize > pWal->cfg.retentionSize || if (pWal->totSize > pWal->cfg.retentionSize || iter->closeTs + pWal->cfg.retentionPeriod > ts) {
iter->closeTs + pWal->cfg.retentionPeriod > ts) { // delete according to file size or close time
//delete according to file size or close time
deleteCnt++; deleteCnt++;
newTotSize -= iter->fileSize; newTotSize -= iter->fileSize;
} }
} }
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
//remove file // remove file
for(int i = 0; i < deleteCnt; i++) { for (int i = 0; i < deleteCnt; i++) {
WalFileInfo* pInfo = taosArrayGet(pWal->fileInfoSet, i); WalFileInfo *pInfo = taosArrayGet(pWal->fileInfoSet, i);
walBuildLogName(pWal, pInfo->firstVer, fnameStr); walBuildLogName(pWal, pInfo->firstVer, fnameStr);
remove(fnameStr); remove(fnameStr);
walBuildIdxName(pWal, pInfo->firstVer, fnameStr); walBuildIdxName(pWal, pInfo->firstVer, fnameStr);
remove(fnameStr); remove(fnameStr);
} }
//make new array, remove files // make new array, remove files
taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt); taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt);
if(taosArrayGetSize(pWal->fileInfoSet) == 0) { if (taosArrayGetSize(pWal->fileInfoSet) == 0) {
pWal->writeCur = -1; pWal->writeCur = -1;
pWal->vers.firstVer = -1; pWal->vers.firstVer = -1;
} else { } else {
pWal->vers.firstVer = ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; pWal->vers.firstVer = ((WalFileInfo *)taosArrayGet(pWal->fileInfoSet, 0))->firstVer;
} }
pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;; pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
;
pWal->totSize = newTotSize; pWal->totSize = newTotSize;
pWal->vers.verInSnapshotting = -1; pWal->vers.verInSnapshotting = -1;
//save snapshot ver, commit ver // save snapshot ver, commit ver
int code = walSaveMeta(pWal); int code = walSaveMeta(pWal);
if(code != 0) { if (code != 0) {
return -1; return -1;
} }
@ -204,41 +204,41 @@ int32_t walEndSnapshot(SWal *pWal) {
int walRoll(SWal *pWal) { int walRoll(SWal *pWal) {
int code = 0; int code = 0;
if(pWal->writeIdxTfd != -1) { if (pWal->writeIdxTfd != -1) {
code = tfClose(pWal->writeIdxTfd); code = tfClose(pWal->writeIdxTfd);
if(code != 0) { if (code != 0) {
return -1; return -1;
} }
} }
if(pWal->writeLogTfd != -1) { if (pWal->writeLogTfd != -1) {
code = tfClose(pWal->writeLogTfd); code = tfClose(pWal->writeLogTfd);
if(code != 0) { if (code != 0) {
return -1; return -1;
} }
} }
int64_t idxTfd, logTfd; int64_t idxTfd, logTfd;
//create new file // create new file
int64_t newFileFirstVersion = pWal->vers.lastVer + 1; int64_t newFileFirstVersion = pWal->vers.lastVer + 1;
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, newFileFirstVersion, fnameStr); walBuildIdxName(pWal, newFileFirstVersion, fnameStr);
idxTfd = tfOpenCreateWriteAppend(fnameStr); idxTfd = tfOpenCreateWriteAppend(fnameStr);
if(idxTfd < 0) { if (idxTfd < 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
} }
walBuildLogName(pWal, newFileFirstVersion, fnameStr); walBuildLogName(pWal, newFileFirstVersion, fnameStr);
logTfd = tfOpenCreateWriteAppend(fnameStr); logTfd = tfOpenCreateWriteAppend(fnameStr);
if(logTfd < 0) { if (logTfd < 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
} }
code = walRollFileInfo(pWal); code = walRollFileInfo(pWal);
if(code != 0) { if (code != 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
} }
//switch file // switch file
pWal->writeIdxTfd = idxTfd; pWal->writeIdxTfd = idxTfd;
pWal->writeLogTfd = logTfd; pWal->writeLogTfd = logTfd;
pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
@ -248,10 +248,10 @@ int walRoll(SWal *pWal) {
} }
static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
WalIdxEntry entry = { .ver = ver, .offset = offset }; WalIdxEntry entry = {.ver = ver, .offset = offset};
int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(WalIdxEntry)); int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(WalIdxEntry));
if(size != sizeof(WalIdxEntry)) { if (size != sizeof(WalIdxEntry)) {
//TODO truncate // TODO truncate
return -1; return -1;
} }
return 0; return 0;
@ -265,21 +265,21 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i
if (pWal->cfg.level == TAOS_WAL_NOLOG) return 0; if (pWal->cfg.level == TAOS_WAL_NOLOG) return 0;
if (index == pWal->vers.lastVer + 1) { if (index == pWal->vers.lastVer + 1) {
if(taosArrayGetSize(pWal->fileInfoSet) == 0) { if (taosArrayGetSize(pWal->fileInfoSet) == 0) {
pWal->vers.firstVer = index; pWal->vers.firstVer = index;
code = walRoll(pWal); code = walRoll(pWal);
ASSERT(code == 0); ASSERT(code == 0);
} else { } else {
int64_t passed = walGetSeq() - pWal->lastRollSeq; int64_t passed = walGetSeq() - pWal->lastRollSeq;
if(pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) { if (pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) {
walRoll(pWal); walRoll(pWal);
} else if(pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) { } else if (pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) {
walRoll(pWal); walRoll(pWal);
} }
} }
} else { } else {
//reject skip log or rewrite log // reject skip log or rewrite log
//must truncate explicitly first // must truncate explicitly first
return -1; return -1;
} }
/*if (!tfValid(pWal->writeLogTfd)) return -1;*/ /*if (!tfValid(pWal->writeLogTfd)) return -1;*/
@ -294,28 +294,30 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i
pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen);
if (tfWrite(pWal->writeLogTfd, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) { if (tfWrite(pWal->writeLogTfd, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
//ftruncate // ftruncate
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
} }
if (tfWrite(pWal->writeLogTfd, (char*)body, bodyLen) != bodyLen) { if (tfWrite(pWal->writeLogTfd, (char *)body, bodyLen) != bodyLen) {
//ftruncate // ftruncate
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
} }
code = walWriteIndex(pWal, index, offset); code = walWriteIndex(pWal, index, offset);
if(code != 0) { if (code != 0) {
//TODO // TODO
return -1; return -1;
} }
//set status // set status
pWal->vers.lastVer = index; pWal->vers.lastVer = index;
pWal->totSize += sizeof(SWalHead) + bodyLen; pWal->totSize += sizeof(SWalHead) + bodyLen;
walGetCurFileInfo(pWal)->lastVer = index; walGetCurFileInfo(pWal)->lastVer = index;
walGetCurFileInfo(pWal)->fileSize += sizeof(SWalHead) + bodyLen; walGetCurFileInfo(pWal)->fileSize += sizeof(SWalHead) + bodyLen;
pthread_mutex_unlock(&pWal->mutex); pthread_mutex_unlock(&pWal->mutex);
return code; return code;
@ -323,33 +325,34 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i
void walFsync(SWal *pWal, bool forceFsync) { void walFsync(SWal *pWal, bool forceFsync) {
if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) {
wTrace("vgId:%d, fileId:%"PRId64".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal));
if (tfFsync(pWal->writeLogTfd) < 0) { if (tfFsync(pWal->writeLogTfd) < 0) {
wError("vgId:%d, file:%"PRId64".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), strerror(errno)); wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal),
strerror(errno));
} }
} }
} }
/*static int walValidateOffset(SWal* pWal, int64_t ver) {*/ /*static int walValidateOffset(SWal* pWal, int64_t ver) {*/
/*int code = 0;*/ /*int code = 0;*/
/*SWalHead *pHead = NULL;*/ /*SWalHead *pHead = NULL;*/
/*code = (int)walRead(pWal, &pHead, ver);*/ /*code = (int)walRead(pWal, &pHead, ver);*/
/*if(pHead->head.version != ver) {*/ /*if(pHead->head.version != ver) {*/
/*return -1;*/ /*return -1;*/
/*}*/ /*}*/
/*return 0;*/ /*return 0;*/
/*}*/ /*}*/
/*static int64_t walGetOffset(SWal* pWal, int64_t ver) {*/ /*static int64_t walGetOffset(SWal* pWal, int64_t ver) {*/
/*int code = walSeekVer(pWal, ver);*/ /*int code = walSeekVer(pWal, ver);*/
/*if(code != 0) {*/ /*if(code != 0) {*/
/*return -1;*/ /*return -1;*/
/*}*/ /*}*/
/*code = walValidateOffset(pWal, ver);*/ /*code = walValidateOffset(pWal, ver);*/
/*if(code != 0) {*/ /*if(code != 0) {*/
/*return -1;*/ /*return -1;*/
/*}*/ /*}*/
/*return 0;*/ /*return 0;*/
/*}*/ /*}*/

212
source/libs/wal/test/walMetaTest.cpp
View File

@ -6,111 +6,105 @@
#include "walInt.h" #include "walInt.h"
const char* ranStr = "tvapq02tcp"; const char* ranStr = "tvapq02tcp";
const int ranStrLen = strlen(ranStr); const int ranStrLen = strlen(ranStr);
class WalCleanEnv : public ::testing::Test { class WalCleanEnv : public ::testing::Test {
protected: protected:
static void SetUpTestCase() { static void SetUpTestCase() {
int code = walInit(); int code = walInit();
ASSERT(code == 0); ASSERT(code == 0);
} }
static void TearDownTestCase() { static void TearDownTestCase() { walCleanUp(); }
walCleanUp();
}
void SetUp() override { void SetUp() override {
taosRemoveDir(pathName); taosRemoveDir(pathName);
SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg));
memset(pCfg, 0, sizeof(SWalCfg)); memset(pCfg, 0, sizeof(SWalCfg));
pCfg->rollPeriod = -1; pCfg->rollPeriod = -1;
pCfg->segSize = -1; pCfg->segSize = -1;
pCfg->retentionPeriod = 0; pCfg->retentionPeriod = 0;
pCfg->retentionSize = 0; pCfg->retentionSize = 0;
pCfg->level = TAOS_WAL_FSYNC; pCfg->level = TAOS_WAL_FSYNC;
pWal = walOpen(pathName, pCfg); pWal = walOpen(pathName, pCfg);
free(pCfg); free(pCfg);
ASSERT(pWal != NULL); ASSERT(pWal != NULL);
} }
void TearDown() override { void TearDown() override {
walClose(pWal); walClose(pWal);
pWal = NULL; pWal = NULL;
} }
SWal* pWal = NULL; SWal* pWal = NULL;
const char* pathName = "/tmp/wal_test"; const char* pathName = "/tmp/wal_test";
}; };
class WalCleanDeleteEnv : public ::testing::Test { class WalCleanDeleteEnv : public ::testing::Test {
protected: protected:
static void SetUpTestCase() { static void SetUpTestCase() {
int code = walInit(); int code = walInit();
ASSERT(code == 0); ASSERT(code == 0);
} }
static void TearDownTestCase() { static void TearDownTestCase() { walCleanUp(); }
walCleanUp();
}
void SetUp() override { void SetUp() override {
taosRemoveDir(pathName); taosRemoveDir(pathName);
SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg));
memset(pCfg, 0, sizeof(SWalCfg)); memset(pCfg, 0, sizeof(SWalCfg));
pCfg->retentionPeriod = 0; pCfg->retentionPeriod = 0;
pCfg->retentionSize = 0; pCfg->retentionSize = 0;
pCfg->level = TAOS_WAL_FSYNC; pCfg->level = TAOS_WAL_FSYNC;
pWal = walOpen(pathName, pCfg); pWal = walOpen(pathName, pCfg);
free(pCfg); free(pCfg);
ASSERT(pWal != NULL); ASSERT(pWal != NULL);
} }
void TearDown() override { void TearDown() override {
walClose(pWal); walClose(pWal);
pWal = NULL; pWal = NULL;
} }
SWal* pWal = NULL; SWal* pWal = NULL;
const char* pathName = "/tmp/wal_test"; const char* pathName = "/tmp/wal_test";
}; };
class WalKeepEnv : public ::testing::Test { class WalKeepEnv : public ::testing::Test {
protected: protected:
static void SetUpTestCase() { static void SetUpTestCase() {
int code = walInit(); int code = walInit();
ASSERT(code == 0); ASSERT(code == 0);
} }
static void TearDownTestCase() { static void TearDownTestCase() { walCleanUp(); }
walCleanUp();
}
void walResetEnv() { void walResetEnv() {
TearDown(); TearDown();
taosRemoveDir(pathName); taosRemoveDir(pathName);
SetUp(); SetUp();
} }
void SetUp() override { void SetUp() override {
SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg));
memset(pCfg, 0, sizeof(SWalCfg)); memset(pCfg, 0, sizeof(SWalCfg));
pCfg->rollPeriod = -1; pCfg->rollPeriod = -1;
pCfg->segSize = -1; pCfg->segSize = -1;
pCfg->retentionPeriod = 0; pCfg->retentionPeriod = 0;
pCfg->retentionSize = 0; pCfg->retentionSize = 0;
pCfg->level = TAOS_WAL_FSYNC; pCfg->level = TAOS_WAL_FSYNC;
pWal = walOpen(pathName, pCfg); pWal = walOpen(pathName, pCfg);
free(pCfg); free(pCfg);
ASSERT(pWal != NULL); ASSERT(pWal != NULL);
} }
void TearDown() override { void TearDown() override {
walClose(pWal); walClose(pWal);
pWal = NULL; pWal = NULL;
} }
SWal* pWal = NULL; SWal* pWal = NULL;
const char* pathName = "/tmp/wal_test"; const char* pathName = "/tmp/wal_test";
}; };
TEST_F(WalCleanEnv, createNew) { TEST_F(WalCleanEnv, createNew) {
@ -139,7 +133,7 @@ TEST_F(WalCleanEnv, serialize) {
ASSERT(code == 0); ASSERT(code == 0);
code = walRollFileInfo(pWal); code = walRollFileInfo(pWal);
ASSERT(code == 0); ASSERT(code == 0);
char*ss = walMetaSerialize(pWal); char* ss = walMetaSerialize(pWal);
printf("%s\n", ss); printf("%s\n", ss);
free(ss); free(ss);
code = walSaveMeta(pWal); code = walSaveMeta(pWal);
@ -162,11 +156,11 @@ TEST_F(WalKeepEnv, readOldMeta) {
walResetEnv(); walResetEnv();
int code; int code;
for(int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, i); ASSERT_EQ(pWal->vers.lastVer, i);
code = walWrite(pWal, i+2, i, (void*)ranStr, ranStrLen); code = walWrite(pWal, i + 2, i, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, -1); ASSERT_EQ(code, -1);
ASSERT_EQ(pWal->vers.lastVer, i); ASSERT_EQ(pWal->vers.lastVer, i);
} }
@ -182,7 +176,7 @@ TEST_F(WalKeepEnv, readOldMeta) {
int len = strlen(oldss); int len = strlen(oldss);
ASSERT_EQ(len, strlen(newss)); ASSERT_EQ(len, strlen(newss));
for(int i = 0; i < len; i++) { for (int i = 0; i < len; i++) {
EXPECT_EQ(oldss[i], newss[i]); EXPECT_EQ(oldss[i], newss[i]);
} }
free(oldss); free(oldss);
@ -191,11 +185,11 @@ TEST_F(WalKeepEnv, readOldMeta) {
TEST_F(WalCleanEnv, write) { TEST_F(WalCleanEnv, write) {
int code; int code;
for(int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, i); ASSERT_EQ(pWal->vers.lastVer, i);
code = walWrite(pWal, i+2, i, (void*)ranStr, ranStrLen); code = walWrite(pWal, i + 2, i, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, -1); ASSERT_EQ(code, -1);
ASSERT_EQ(pWal->vers.lastVer, i); ASSERT_EQ(pWal->vers.lastVer, i);
} }
@ -205,8 +199,8 @@ TEST_F(WalCleanEnv, write) {
TEST_F(WalCleanEnv, rollback) { TEST_F(WalCleanEnv, rollback) {
int code; int code;
for(int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, i); ASSERT_EQ(pWal->vers.lastVer, i);
} }
@ -223,7 +217,7 @@ TEST_F(WalCleanEnv, rollback) {
TEST_F(WalCleanDeleteEnv, roll) { TEST_F(WalCleanDeleteEnv, roll) {
int code; int code;
int i; int i;
for(i = 0; i < 100; i++) { for (i = 0; i < 100; i++) {
code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen); code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, i); ASSERT_EQ(pWal->vers.lastVer, i);
@ -231,16 +225,16 @@ TEST_F(WalCleanDeleteEnv, roll) {
ASSERT_EQ(pWal->vers.commitVer, i); ASSERT_EQ(pWal->vers.commitVer, i);
} }
walBeginSnapshot(pWal, i-1); walBeginSnapshot(pWal, i - 1);
ASSERT_EQ(pWal->vers.verInSnapshotting, i-1); ASSERT_EQ(pWal->vers.verInSnapshotting, i - 1);
walEndSnapshot(pWal); walEndSnapshot(pWal);
ASSERT_EQ(pWal->vers.snapshotVer, i-1); ASSERT_EQ(pWal->vers.snapshotVer, i - 1);
ASSERT_EQ(pWal->vers.verInSnapshotting, -1); ASSERT_EQ(pWal->vers.verInSnapshotting, -1);
code = walWrite(pWal, 5, 0, (void*)ranStr, ranStrLen); code = walWrite(pWal, 5, 0, (void*)ranStr, ranStrLen);
ASSERT_NE(code, 0); ASSERT_NE(code, 0);
for(; i < 200; i++) { for (; i < 200; i++) {
code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen); code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
code = walCommit(pWal, i); code = walCommit(pWal, i);
@ -255,36 +249,36 @@ TEST_F(WalCleanDeleteEnv, roll) {
TEST_F(WalKeepEnv, readHandleRead) { TEST_F(WalKeepEnv, readHandleRead) {
walResetEnv(); walResetEnv();
int code; int code;
SWalReadHandle* pRead = walOpenReadHandle(pWal); SWalReadHandle* pRead = walOpenReadHandle(pWal);
ASSERT(pRead != NULL); ASSERT(pRead != NULL);
int i ; int i;
for(i = 0; i < 100; i++) { for (i = 0; i < 100; i++) {
char newStr[100]; char newStr[100];
sprintf(newStr, "%s-%d", ranStr, i); sprintf(newStr, "%s-%d", ranStr, i);
int len = strlen(newStr); int len = strlen(newStr);
code = walWrite(pWal, i, 0, newStr, len); code = walWrite(pWal, i, 0, newStr, len);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
} }
for(int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
int ver = rand() % 100; int ver = rand() % 100;
code = walReadWithHandle(pRead, ver); code = walReadWithHandle(pRead, ver);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
//printf("rrbody: \n"); // printf("rrbody: \n");
//for(int i = 0; i < pRead->pHead->head.len; i++) { // for(int i = 0; i < pRead->pHead->head.len; i++) {
//printf("%d ", pRead->pHead->head.body[i]); // printf("%d ", pRead->pHead->head.body[i]);
//} //}
//printf("\n"); // printf("\n");
ASSERT_EQ(pRead->pHead->head.version, ver); ASSERT_EQ(pRead->pHead->head.version, ver);
ASSERT_EQ(pRead->curVersion, ver+1); ASSERT_EQ(pRead->curVersion, ver + 1);
char newStr[100]; char newStr[100];
sprintf(newStr, "%s-%d", ranStr, ver); sprintf(newStr, "%s-%d", ranStr, ver);
int len = strlen(newStr); int len = strlen(newStr);
ASSERT_EQ(pRead->pHead->head.len, len); ASSERT_EQ(pRead->pHead->head.len, len);
for(int j = 0; j < len; j++) { for (int j = 0; j < len; j++) {
EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]);
} }
} }