xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor wal

This commit is contained in:
Liu Jicong 2021-12-17 13:40:27 +08:00
parent 652c05c6e8
commit d037b24e05
14 changed files with 123 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/common/taosmsg.h
View File

@ -50,8 +50,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONSUME, "mq-consume" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_QUERY, "mq-query" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONNECT, "mq-connect" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_DISCONNECT, "mq-disconnect" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_ACK, "mq-ack" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_RESET, "mq-reset" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_SET, "mq-set" )
// message from client to mnode
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONNECT, "connect" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_ACCT, "create-acct" )

13
include/dnode/vnode/tq/tq.h
View File

@ -62,10 +62,10 @@ typedef struct TmqDisconnectRsp {
int8_t status;
} TmqDisconnectRsp;
typedef struct TmqConsumeReq {
typedef struct STqConsumeReq {
TmqMsgHead head;
TmqAcks acks;
} TmqConsumeReq;
} STqConsumeReq;
typedef struct TmqMsgContent {
int64_t topicId;
@ -73,11 +73,11 @@ typedef struct TmqMsgContent {
char msg[];
} TmqMsgContent;
typedef struct TmqConsumeRsp {
typedef struct STqConsumeRsp {
TmqMsgHead head;
int64_t bodySize;
TmqMsgContent msgs[];
} TmqConsumeRsp;
} STqConsumeRsp;
typedef struct TmqSubscribeReq {
TmqMsgHead head;
@ -261,13 +261,14 @@ typedef struct STQ {
// open in each vnode
STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac);
void tqDestroy(STQ*);
void tqClose(STQ*);
// void* will be replace by a msg type
int tqPushMsg(STQ*, void* msg, int64_t version);
int tqCommit(STQ*);
int tqSetCursor(STQ*, void* msg);
int tqConsume(STQ*, TmqConsumeReq*);
int tqConsume(STQ*, STqConsumeReq*);
STqGroupHandle* tqGetGroupHandle(STQ*, int64_t cId);

3
source/dnode/mgmt/impl/src/dndTransport.c
View File

@ -44,8 +44,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONSUME] = dndProcessVnodeQueryMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONNECT] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_DISCONNECT] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_ACK] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_RESET] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_SET] = dndProcessVnodeWriteMsg;
// msg from client to mnode
pMgmt->msgFp[TSDB_MSG_TYPE_CONNECT] = dndProcessMnodeReadMsg;

6
source/dnode/vnode/impl/inc/vnodeStateMgr.h
View File

@ -21,9 +21,9 @@ extern "C" {
#endif
typedef struct {
uint64_t processed;
uint64_t committed;
uint64_t applied;
int64_t processed;
int64_t committed;
int64_t applied;
} SVState;
#ifdef __cplusplus

17
source/dnode/vnode/impl/src/vnodeWrite.c
View File

@ -15,6 +15,21 @@
#include "vnodeDef.h"
int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) {
SVnodeReq *pVnodeReq;
switch (pMsg->msgType) {
case TSDB_MSG_TYPE_MQ_SET:
if (tqSetCursor(pVnode->pTq, pMsg->pCont) < 0) {
// TODO: handle error
}
break;
}
void *pBuf = pMsg->pCont;
return 0;
}
int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
SRpcMsg *pMsg;
SVnodeReq *pVnodeReq;
@ -24,7 +39,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
// ser request version
void *pBuf = pMsg->pCont;
uint64_t ver = pVnode->state.processed++;
int64_t ver = pVnode->state.processed++;
taosEncodeFixedU64(&pBuf, ver);
if (walWrite(pVnode->pWal, ver, pMsg->msgType, pMsg->pCont, pMsg->contLen) < 0) {

9
source/dnode/vnode/tq/inc/tqMetaStore.h
View File

@ -23,13 +23,8 @@
extern "C" {
#endif
STqMetaStore* tqStoreOpen(const char* path,
FTqSerialize pSerializer,
FTqDeserialize pDeserializer,
FTqDelete pDeleter,
int32_t tqConfigFlag
);
STqMetaStore* tqStoreOpen(const char* path, FTqSerialize pSerializer, FTqDeserialize pDeserializer, FTqDelete pDeleter,
int32_t tqConfigFlag);
int32_t tqStoreClose(STqMetaStore*);
// int32_t tqStoreDelete(TqMetaStore*);
// int32_t tqStoreCommitAll(TqMetaStore*);

8
source/dnode/vnode/tq/src/tq.c
View File

@ -214,7 +214,11 @@ int tqCommit(STQ* pTq) {
return 0;
}
int tqConsume(STQ* pTq, TmqConsumeReq* pMsg) {
int tqSetCursor(STQ* pTq, void* msg) {
return 0;
}
int tqConsume(STQ* pTq, STqConsumeReq* pMsg) {
if (!tqProtoCheck((TmqMsgHead*)pMsg)) {
// proto version invalid
return -1;
@ -232,7 +236,7 @@ int tqConsume(STQ* pTq, TmqConsumeReq* pMsg) {
}
}
TmqConsumeRsp* pRsp = (TmqConsumeRsp*)pMsg;
STqConsumeRsp* pRsp = (STqConsumeRsp*)pMsg;
if (tqFetch(gHandle, (void**)&pRsp->msgs) <= 0) {
// fetch error

30
source/libs/wal/inc/walInt.h
View File

@ -16,9 +16,9 @@
#ifndef _TD_WAL_INT_H_
#define _TD_WAL_INT_H_
#include "wal.h"
#include "compare.h"
#include "tchecksum.h"
#include "wal.h"
#ifdef __cplusplus
extern "C" {
@ -31,50 +31,48 @@ typedef struct WalFileInfo {
int64_t createTs;
int64_t closeTs;
int64_t fileSize;
} WalFileInfo;
} SWalFileInfo;
typedef struct WalIdxEntry {
int64_t ver;
int64_t offset;
} WalIdxEntry;
} SWalIdxEntry;
static inline int32_t compareWalFileInfo(const void* pLeft, const void* pRight) {
WalFileInfo* pInfoLeft = (WalFileInfo*)pLeft;
WalFileInfo* pInfoRight = (WalFileInfo*)pRight;
SWalFileInfo* pInfoLeft = (SWalFileInfo*)pLeft;
SWalFileInfo* pInfoRight = (SWalFileInfo*)pRight;
return compareInt64Val(&pInfoLeft->firstVer, &pInfoRight->firstVer);
}
static inline int64_t walGetLastFileSize(SWal* pWal) {
WalFileInfo* pInfo = (WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet);
SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet);
return pInfo->fileSize;
}
static inline int64_t walGetLastFileFirstVer(SWal* pWal) {
WalFileInfo* pInfo = (WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet);
SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet);
return pInfo->firstVer;
}
static inline int64_t walGetCurFileFirstVer(SWal* pWal) {
WalFileInfo* pInfo = (WalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur);
SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur);
return pInfo->firstVer;
}
static inline int64_t walGetCurFileLastVer(SWal* pWal) {
WalFileInfo* pInfo = (WalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur);
SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur);
return pInfo->firstVer;
}
static inline int64_t walGetCurFileOffset(SWal* pWal) {
WalFileInfo* pInfo = (WalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur);
SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur);
return pInfo->fileSize;
}
static inline bool walCurFileClosed(SWal* pWal) {
return taosArrayGetSize(pWal->fileInfoSet) != pWal->writeCur;
}
static inline bool walCurFileClosed(SWal* pWal) { return taosArrayGetSize(pWal->fileInfoSet) != pWal->writeCur; }
static inline WalFileInfo* walGetCurFileInfo(SWal* pWal) {
return (WalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur);
static inline SWalFileInfo* walGetCurFileInfo(SWal* pWal) {
return (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur);
}
static inline int walBuildLogName(SWal* pWal, int64_t fileFirstVer, char* buf) {
@ -106,7 +104,7 @@ static inline uint32_t walCalcBodyCksum(const void* body, uint32_t len) {
}
static inline int64_t walGetVerIdxOffset(SWal* pWal, int64_t ver) {
return (ver - walGetCurFileFirstVer(pWal)) * sizeof(WalIdxEntry);
return (ver - walGetCurFileFirstVer(pWal)) * sizeof(SWalIdxEntry);
}
static inline void walResetVer(SWalVer* pVer) {

12
source/libs/wal/src/walMeta.c
View File

@ -79,13 +79,13 @@ int walRollFileInfo(SWal* pWal) {
SArray* pArray = pWal->fileInfoSet;
if (taosArrayGetSize(pArray) != 0) {
WalFileInfo* pInfo = taosArrayGetLast(pArray);
SWalFileInfo* pInfo = taosArrayGetLast(pArray);
pInfo->lastVer = pWal->vers.lastVer;
pInfo->closeTs = ts;
}
// TODO: change to emplace back
WalFileInfo* pNewInfo = malloc(sizeof(WalFileInfo));
SWalFileInfo* pNewInfo = malloc(sizeof(SWalFileInfo));
if (pNewInfo == NULL) {
return -1;
}
@ -122,9 +122,9 @@ char* walMetaSerialize(SWal* pWal) {
cJSON_AddStringToObject(pMeta, "lastVer", buf);
cJSON_AddItemToObject(pRoot, "files", pFiles);
WalFileInfo* pData = pWal->fileInfoSet->pData;
SWalFileInfo* pData = pWal->fileInfoSet->pData;
for (int i = 0; i < sz; i++) {
WalFileInfo* pInfo = &pData[i];
SWalFileInfo* pInfo = &pData[i];
cJSON_AddItemToArray(pFiles, pField = cJSON_CreateObject());
if (pField == NULL) {
cJSON_Delete(pRoot);
@ -167,10 +167,10 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) {
// deserialize
SArray* pArray = pWal->fileInfoSet;
taosArrayEnsureCap(pArray, sz);
WalFileInfo* pData = pArray->pData;
SWalFileInfo* pData = pArray->pData;
for (int i = 0; i < sz; i++) {
cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i);
WalFileInfo* pInfo = &pData[i];
SWalFileInfo* pInfo = &pData[i];
pField = cJSON_GetObjectItem(pInfoJson, "firstVer");
pInfo->firstVer = atoll(cJSON_GetStringValue(pField));
pField = cJSON_GetObjectItem(pInfoJson, "lastVer");

2
source/libs/wal/src/walMgmt.c
View File

@ -92,7 +92,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
pWal->writeLogTfd = -1;
pWal->writeIdxTfd = -1;
pWal->writeCur = -1;
pWal->fileInfoSet = taosArrayInit(8, sizeof(WalFileInfo));
pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo));
if (pWal->fileInfoSet == NULL) {
wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno));
free(pWal);

12
source/libs/wal/src/walRead.c
View File

@ -52,13 +52,13 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i
int64_t logTfd = pRead->readLogTfd;
// seek position
int64_t offset = (ver - fileFirstVer) * sizeof(WalIdxEntry);
int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry);
code = tfLseek(idxTfd, offset, SEEK_SET);
if (code < 0) {
return -1;
}
WalIdxEntry entry;
if (tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) {
SWalIdxEntry entry;
if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
return -1;
}
// TODO:deserialize
@ -105,10 +105,10 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) {
if (ver < pWal->vers.snapshotVer) {
}
WalFileInfo tmpInfo;
SWalFileInfo tmpInfo;
tmpInfo.firstVer = ver;
// bsearch in fileSet
WalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
ASSERT(pRet != NULL);
if (pRead->curFileFirstVer != pRet->firstVer) {
code = walReadChangeFile(pRead, pRet->firstVer);
@ -159,9 +159,9 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
return -1;
}
/*code = walValidBodyCksum(pRead->pHead);*/
ASSERT(pRead->pHead->head.version == ver);
code = walValidBodyCksum(pRead->pHead);
if (code != 0) {
return -1;
}

10
source/libs/wal/src/walSeek.c
View File

@ -32,9 +32,9 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) {
if (code != 0) {
return -1;
}
WalIdxEntry entry;
SWalIdxEntry entry;
// TODO:deserialize
code = tfRead(idxTfd, &entry, sizeof(WalIdxEntry));
code = tfRead(idxTfd, &entry, sizeof(SWalIdxEntry));
if (code != 0) {
return -1;
}
@ -48,7 +48,7 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) {
int walChangeFileToLast(SWal* pWal) {
int64_t idxTfd, logTfd;
WalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet);
SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet);
ASSERT(pRet != NULL);
int64_t fileFirstVer = pRet->firstVer;
@ -83,10 +83,10 @@ int walChangeFile(SWal* pWal, int64_t ver) {
// TODO
return -1;
}
WalFileInfo tmpInfo;
SWalFileInfo tmpInfo;
tmpInfo.firstVer = ver;
// bsearch in fileSet
WalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
SWalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
ASSERT(pRet != NULL);
int64_t fileFirstVer = pRet->firstVer;
// closed

28
source/libs/wal/src/walWrite.c
View File

@ -56,9 +56,9 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
// delete files
int fileSetSize = taosArrayGetSize(pWal->fileInfoSet);
for (int i = pWal->writeCur; i < fileSetSize; i++) {
walBuildLogName(pWal, ((WalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr);
walBuildLogName(pWal, ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr);
remove(fnameStr);
walBuildIdxName(pWal, ((WalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr);
walBuildIdxName(pWal, ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr);
remove(fnameStr);
}
// pop from fileInfoSet
@ -81,8 +81,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
}
// read idx file and get log file pos
// TODO:change to deserialize function
WalIdxEntry entry;
if (tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) {
SWalIdxEntry entry;
if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
pthread_mutex_unlock(&pWal->mutex);
return -1;
}
@ -128,8 +128,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
return -1;
}
pWal->vers.lastVer = ver - 1;
((WalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1;
((WalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset;
((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1;
((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset;
// unlock
pthread_mutex_unlock(&pWal->mutex);
@ -155,15 +155,15 @@ int32_t walEndSnapshot(SWal *pWal) {
int deleteCnt = 0;
int64_t newTotSize = pWal->totSize;
WalFileInfo tmp;
SWalFileInfo tmp;
tmp.firstVer = ver;
// find files safe to delete
WalFileInfo *pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE);
SWalFileInfo *pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE);
if (ver >= pInfo->lastVer) {
pInfo++;
}
// iterate files, until the searched result
for (WalFileInfo *iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) {
for (SWalFileInfo *iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) {
if (pWal->totSize > pWal->cfg.retentionSize || iter->closeTs + pWal->cfg.retentionPeriod > ts) {
// delete according to file size or close time
deleteCnt++;
@ -173,7 +173,7 @@ int32_t walEndSnapshot(SWal *pWal) {
char fnameStr[WAL_FILE_LEN];
// remove file
for (int i = 0; i < deleteCnt; i++) {
WalFileInfo *pInfo = taosArrayGet(pWal->fileInfoSet, i);
SWalFileInfo *pInfo = taosArrayGet(pWal->fileInfoSet, i);
walBuildLogName(pWal, pInfo->firstVer, fnameStr);
remove(fnameStr);
walBuildIdxName(pWal, pInfo->firstVer, fnameStr);
@ -186,7 +186,7 @@ int32_t walEndSnapshot(SWal *pWal) {
pWal->writeCur = -1;
pWal->vers.firstVer = -1;
} else {
pWal->vers.firstVer = ((WalFileInfo *)taosArrayGet(pWal->fileInfoSet, 0))->firstVer;
pWal->vers.firstVer = ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, 0))->firstVer;
}
pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
;
@ -248,9 +248,9 @@ int walRoll(SWal *pWal) {
}
static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
WalIdxEntry entry = {.ver = ver, .offset = offset};
int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(WalIdxEntry));
if (size != sizeof(WalIdxEntry)) {
SWalIdxEntry entry = {.ver = ver, .offset = offset};
int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(SWalIdxEntry));
if (size != sizeof(SWalIdxEntry)) {
// TODO truncate
return -1;
}