xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into enh/TD-30988-3.0

This commit is contained in:
kailixu 2024-07-24 14:47:47 +08:00
parent 50c2fc567c cd2f039fd9
commit acb1d0a9cc
26 changed files with 935 additions and 961 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
include/libs/wal/wal.h
View File

@ -12,6 +12,7 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#ifndef _TD_WAL_H_ #ifndef _TD_WAL_H_
#define _TD_WAL_H_ #define _TD_WAL_H_
@ -20,19 +21,11 @@
#include "tdef.h" #include "tdef.h"
#include "tlog.h" #include "tlog.h"
#include "tmsg.h" #include "tmsg.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
// clang-format off
#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", DEBUG_DEBUG, wDebugFlag, __VA_ARGS__); }}
#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", DEBUG_TRACE, wDebugFlag, __VA_ARGS__); }}
// clang-format on
#define WAL_PROTO_VER 0 #define WAL_PROTO_VER 0
#define WAL_NOSUFFIX_LEN 20 #define WAL_NOSUFFIX_LEN 20
#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1) #define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1)
@ -131,7 +124,6 @@ typedef struct SWal {
typedef struct { typedef struct {
int64_t refId; int64_t refId;
int64_t refVer; int64_t refVer;
// int64_t refFile;
SWal *pWal; SWal *pWal;
} SWalRef; } SWalRef;
@ -143,10 +135,8 @@ typedef struct {
int8_t enableRef; int8_t enableRef;
} SWalFilterCond; } SWalFilterCond;
typedef struct SWalReader SWalReader;
// todo hide this struct // todo hide this struct
struct SWalReader { typedef struct SWalReader {
SWal *pWal; SWal *pWal;
int64_t readerId; int64_t readerId;
TdFilePtr pLogFile; TdFilePtr pLogFile;
@ -159,7 +149,7 @@ struct SWalReader {
TdThreadMutex mutex; TdThreadMutex mutex;
SWalFilterCond cond; SWalFilterCond cond;
SWalCkHead *pHead; SWalCkHead *pHead;
}; } SWalReader;
// module initialization // module initialization
int32_t walInit(); int32_t walInit();
@ -172,17 +162,9 @@ int32_t walPersist(SWal *);
void walClose(SWal *); void walClose(SWal *);
// write interfaces // write interfaces
// By assigning index by the caller, wal gurantees linearizability // By assigning index by the caller, wal gurantees linearizability
int32_t walWrite(SWal *, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen); int32_t walAppendLog(SWal *, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t bodyLen);
int32_t walWriteWithSyncInfo(SWal *, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t walFsync(SWal *, bool force);
int32_t bodyLen);
// Assign version automatically and return to caller,
// -1 will be returned for failed writes
int64_t walAppendLog(SWal *, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t bodyLen);
void walFsync(SWal *, bool force);
// apis for lifecycle management // apis for lifecycle management
int32_t walCommit(SWal *, int64_t ver); int32_t walCommit(SWal *, int64_t ver);
@ -191,17 +173,13 @@ int32_t walRollback(SWal *, int64_t ver);
int32_t walBeginSnapshot(SWal *, int64_t ver, int64_t logRetention); int32_t walBeginSnapshot(SWal *, int64_t ver, int64_t logRetention);
int32_t walEndSnapshot(SWal *); int32_t walEndSnapshot(SWal *);
int32_t walRestoreFromSnapshot(SWal *, int64_t ver); int32_t walRestoreFromSnapshot(SWal *, int64_t ver);
// for tq
int32_t walApplyVer(SWal *, int64_t ver); int32_t walApplyVer(SWal *, int64_t ver);
// int32_t walDataCorrupted(SWal*);
// wal reader // wal reader
SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond, int64_t id); SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond, int64_t id);
void walCloseReader(SWalReader *pRead); void walCloseReader(SWalReader *pRead);
void walReadReset(SWalReader *pReader); void walReadReset(SWalReader *pReader);
int32_t walReadVer(SWalReader *pRead, int64_t ver); int32_t walReadVer(SWalReader *pRead, int64_t ver);
void decryptBody(SWalCfg *cfg, SWalCkHead *pHead, int32_t plainBodyLen, const char *func);
int32_t walReaderSeekVer(SWalReader *pRead, int64_t ver); int32_t walReaderSeekVer(SWalReader *pRead, int64_t ver);
int32_t walNextValidMsg(SWalReader *pRead); int32_t walNextValidMsg(SWalReader *pRead);
int64_t walReaderGetCurrentVer(const SWalReader *pReader); int64_t walReaderGetCurrentVer(const SWalReader *pReader);
@ -216,12 +194,11 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver);
int32_t walFetchBody(SWalReader *pRead); int32_t walFetchBody(SWalReader *pRead);
int32_t walSkipFetchBody(SWalReader *pRead); int32_t walSkipFetchBody(SWalReader *pRead);
void walRefFirstVer(SWal *, SWalRef *);
void walRefLastVer(SWal *, SWalRef *);
SWalRef *walOpenRef(SWal *); SWalRef *walOpenRef(SWal *);
void walCloseRef(SWal *pWal, int64_t refId); void walCloseRef(SWal *pWal, int64_t refId);
int32_t walSetRefVer(SWalRef *, int64_t ver); int32_t walSetRefVer(SWalRef *, int64_t ver);
void walRefFirstVer(SWal *, SWalRef *);
void walRefLastVer(SWal *, SWalRef *);
// helper function for raft // helper function for raft
bool walLogExist(SWal *, int64_t ver); bool walLogExist(SWal *, int64_t ver);

6
include/util/tdigest.h
View File

@ -64,10 +64,10 @@ typedef struct TDigest {
} TDigest; } TDigest;
TDigest *tdigestNewFrom(void *pBuf, int32_t compression); TDigest *tdigestNewFrom(void *pBuf, int32_t compression);
void tdigestAdd(TDigest *t, double x, int64_t w); int32_t tdigestAdd(TDigest *t, double x, int64_t w);
void tdigestMerge(TDigest *t1, TDigest *t2); int32_t tdigestMerge(TDigest *t1, TDigest *t2);
double tdigestQuantile(TDigest *t, double q); double tdigestQuantile(TDigest *t, double q);
void tdigestCompress(TDigest *t); int32_t tdigestCompress(TDigest *t);
void tdigestFreeFrom(TDigest *t); void tdigestFreeFrom(TDigest *t);
void tdigestAutoFill(TDigest *t, int32_t compression); void tdigestAutoFill(TDigest *t, int32_t compression);

74
source/dnode/vnode/src/tq/tqMeta.c
View File

@ -37,7 +37,7 @@ int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle) {
} }
} else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
if (tEncodeI64(pEncoder, pHandle->execHandle.execTb.suid) < 0) return -1; if (tEncodeI64(pEncoder, pHandle->execHandle.execTb.suid) < 0) return -1;
if (pHandle->execHandle.execTb.qmsg != NULL){ if (pHandle->execHandle.execTb.qmsg != NULL) {
if (tEncodeCStr(pEncoder, pHandle->execHandle.execTb.qmsg) < 0) return -1; if (tEncodeCStr(pEncoder, pHandle->execHandle.execTb.qmsg) < 0) return -1;
} }
} }
@ -67,7 +67,7 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) {
} }
} else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
if (tDecodeI64(pDecoder, &pHandle->execHandle.execTb.suid) < 0) return -1; if (tDecodeI64(pDecoder, &pHandle->execHandle.execTb.suid) < 0) return -1;
if (!tDecodeIsEnd(pDecoder)){ if (!tDecodeIsEnd(pDecoder)) {
if (tDecodeCStrAlloc(pDecoder, &pHandle->execHandle.execTb.qmsg) < 0) return -1; if (tDecodeCStrAlloc(pDecoder, &pHandle->execHandle.execTb.qmsg) < 0) return -1;
} }
} }
@ -75,7 +75,7 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) {
return 0; return 0;
} }
int32_t tqMetaDecodeCheckInfo(STqCheckInfo *info, void *pVal, int32_t vLen){ int32_t tqMetaDecodeCheckInfo(STqCheckInfo* info, void* pVal, int32_t vLen) {
SDecoder decoder = {0}; SDecoder decoder = {0};
tDecoderInit(&decoder, (uint8_t*)pVal, vLen); tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
int32_t code = tDecodeSTqCheckInfo(&decoder, info); int32_t code = tDecodeSTqCheckInfo(&decoder, info);
@ -87,7 +87,7 @@ int32_t tqMetaDecodeCheckInfo(STqCheckInfo *info, void *pVal, int32_t vLen){
return code; return code;
} }
int32_t tqMetaDecodeOffsetInfo(STqOffset *info, void *pVal, int32_t vLen){ int32_t tqMetaDecodeOffsetInfo(STqOffset* info, void* pVal, int32_t vLen) {
SDecoder decoder = {0}; SDecoder decoder = {0};
tDecoderInit(&decoder, (uint8_t*)pVal, vLen); tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
int32_t code = tDecodeSTqOffset(&decoder, info); int32_t code = tDecodeSTqOffset(&decoder, info);
@ -131,7 +131,7 @@ END:
return code; return code;
} }
void* tqMetaGetOffset(STQ* pTq, const char* subkey){ void* tqMetaGetOffset(STQ* pTq, const char* subkey) {
void* data = taosHashGet(pTq->pOffset, subkey, strlen(subkey)); void* data = taosHashGet(pTq->pOffset, subkey, strlen(subkey));
if (data == NULL) { if (data == NULL) {
int vLen = 0; int vLen = 0;
@ -146,7 +146,7 @@ void* tqMetaGetOffset(STQ* pTq, const char* subkey){
return NULL; return NULL;
} }
if(taosHashPut(pTq->pOffset, subkey, strlen(subkey), &offset, sizeof(STqOffset)) != 0){ if (taosHashPut(pTq->pOffset, subkey, strlen(subkey), &offset, sizeof(STqOffset)) != 0) {
tDeleteSTqOffset(&offset); tDeleteSTqOffset(&offset);
tdbFree(data); tdbFree(data);
return NULL; return NULL;
@ -192,14 +192,15 @@ END:
return code; return code;
} }
static int tqMetaInitHandle(STQ* pTq, STqHandle* handle){ static int tqMetaInitHandle(STQ* pTq, STqHandle* handle) {
int32_t code = TDB_CODE_SUCCESS; int32_t code = TDB_CODE_SUCCESS;
SVnode* pVnode = pTq->pVnode; SVnode* pVnode = pTq->pVnode;
int32_t vgId = TD_VID(pVnode); int32_t vgId = TD_VID(pVnode);
handle->pRef = walOpenRef(pVnode->pWal); handle->pRef = walOpenRef(pVnode->pWal);
if (handle->pRef == NULL) { if (handle->pRef == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
} }
TQ_ERR_RETURN(walSetRefVer(handle->pRef, handle->snapshotVer)); TQ_ERR_RETURN(walSetRefVer(handle->pRef, handle->snapshotVer));
@ -213,8 +214,8 @@ static int tqMetaInitHandle(STQ* pTq, STqHandle* handle){
initStorageAPI(&reader.api); initStorageAPI(&reader.api);
if (handle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { if (handle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
handle->execHandle.task = handle->execHandle.task = qCreateQueueExecTaskInfo(handle->execHandle.execCol.qmsg, &reader, vgId,
qCreateQueueExecTaskInfo(handle->execHandle.execCol.qmsg, &reader, vgId, &handle->execHandle.numOfCols, handle->consumerId); &handle->execHandle.numOfCols, handle->consumerId);
if (handle->execHandle.task == NULL) { if (handle->execHandle.task == NULL) {
tqError("cannot create exec task for %s", handle->subKey); tqError("cannot create exec task for %s", handle->subKey);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -240,7 +241,7 @@ static int tqMetaInitHandle(STQ* pTq, STqHandle* handle){
} else if (handle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { } else if (handle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
handle->pWalReader = walOpenReader(pVnode->pWal, NULL, 0); handle->pWalReader = walOpenReader(pVnode->pWal, NULL, 0);
if(handle->execHandle.execTb.qmsg != NULL && strcmp(handle->execHandle.execTb.qmsg, "") != 0) { if (handle->execHandle.execTb.qmsg != NULL && strcmp(handle->execHandle.execTb.qmsg, "") != 0) {
if (nodesStringToNode(handle->execHandle.execTb.qmsg, &handle->execHandle.execTb.node) != 0) { if (nodesStringToNode(handle->execHandle.execTb.qmsg, &handle->execHandle.execTb.node) != 0) {
tqError("nodesStringToNode error in sub stable, since %s", terrstr()); tqError("nodesStringToNode error in sub stable, since %s", terrstr());
return TSDB_CODE_SCH_INTERNAL_ERROR; return TSDB_CODE_SCH_INTERNAL_ERROR;
@ -251,13 +252,15 @@ static int tqMetaInitHandle(STQ* pTq, STqHandle* handle){
handle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, handle->consumerId); handle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, handle->consumerId);
SArray* tbUidList = NULL; SArray* tbUidList = NULL;
int ret = qGetTableList(handle->execHandle.execTb.suid, pVnode, handle->execHandle.execTb.node, &tbUidList, handle->execHandle.task); int ret = qGetTableList(handle->execHandle.execTb.suid, pVnode, handle->execHandle.execTb.node, &tbUidList,
if(ret != TDB_CODE_SUCCESS) { handle->execHandle.task);
if (ret != TDB_CODE_SUCCESS) {
tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle->subKey, handle->consumerId); tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle->subKey, handle->consumerId);
taosArrayDestroy(tbUidList); taosArrayDestroy(tbUidList);
return TSDB_CODE_SCH_INTERNAL_ERROR; return TSDB_CODE_SCH_INTERNAL_ERROR;
} }
tqInfo("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pVnode->config.vgId, handle->execHandle.execTb.suid); tqInfo("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pVnode->config.vgId,
handle->execHandle.execTb.suid);
handle->execHandle.pTqReader = tqReaderOpen(pVnode); handle->execHandle.pTqReader = tqReaderOpen(pVnode);
tqReaderSetTbUidList(handle->execHandle.pTqReader, tbUidList, NULL); tqReaderSetTbUidList(handle->execHandle.pTqReader, tbUidList, NULL);
taosArrayDestroy(tbUidList); taosArrayDestroy(tbUidList);
@ -265,7 +268,7 @@ static int tqMetaInitHandle(STQ* pTq, STqHandle* handle){
return 0; return 0;
} }
static int32_t tqMetaRestoreHandle(STQ* pTq, void* pVal, int vLen, STqHandle* handle){ static int32_t tqMetaRestoreHandle(STQ* pTq, void* pVal, int vLen, STqHandle* handle) {
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
SDecoder decoder = {0}; SDecoder decoder = {0};
int32_t code = TDB_CODE_SUCCESS; int32_t code = TDB_CODE_SUCCESS;
@ -281,7 +284,7 @@ END:
return code; return code;
} }
int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){ int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle) {
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
memcpy(handle->subKey, req->subKey, TSDB_SUBSCRIBE_KEY_LEN); memcpy(handle->subKey, req->subKey, TSDB_SUBSCRIBE_KEY_LEN);
@ -289,26 +292,27 @@ int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){
handle->execHandle.subType = req->subType; handle->execHandle.subType = req->subType;
handle->fetchMeta = req->withMeta; handle->fetchMeta = req->withMeta;
if(req->subType == TOPIC_SUB_TYPE__COLUMN){ if (req->subType == TOPIC_SUB_TYPE__COLUMN) {
handle->execHandle.execCol.qmsg = taosStrdup(req->qmsg); handle->execHandle.execCol.qmsg = taosStrdup(req->qmsg);
}else if(req->subType == TOPIC_SUB_TYPE__DB){ } else if (req->subType == TOPIC_SUB_TYPE__DB) {
handle->execHandle.execDb.pFilterOutTbUid = handle->execHandle.execDb.pFilterOutTbUid =
taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
}else if(req->subType == TOPIC_SUB_TYPE__TABLE){ } else if (req->subType == TOPIC_SUB_TYPE__TABLE) {
handle->execHandle.execTb.suid = req->suid; handle->execHandle.execTb.suid = req->suid;
handle->execHandle.execTb.qmsg = taosStrdup(req->qmsg); handle->execHandle.execTb.qmsg = taosStrdup(req->qmsg);
} }
handle->snapshotVer = walGetCommittedVer(pTq->pVnode->pWal); handle->snapshotVer = walGetCommittedVer(pTq->pVnode->pWal);
if(tqMetaInitHandle(pTq, handle) < 0){ if (tqMetaInitHandle(pTq, handle) < 0) {
return -1; return -1;
} }
tqInfo("tqMetaCreateHandle %s consumer 0x%" PRIx64 " vgId:%d, snapshotVer:%" PRId64, handle->subKey, handle->consumerId, vgId, handle->snapshotVer); tqInfo("tqMetaCreateHandle %s consumer 0x%" PRIx64 " vgId:%d, snapshotVer:%" PRId64, handle->subKey,
handle->consumerId, vgId, handle->snapshotVer);
return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle)); return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle));
} }
static int32_t tqMetaTransformInfo(TDB* pMetaDB, TTB* pOld, TTB* pNew){ static int32_t tqMetaTransformInfo(TDB* pMetaDB, TTB* pOld, TTB* pNew) {
TBC* pCur = NULL; TBC* pCur = NULL;
void* pKey = NULL; void* pKey = NULL;
int kLen = 0; int kLen = 0;
@ -319,15 +323,16 @@ static int32_t tqMetaTransformInfo(TDB* pMetaDB, TTB* pOld, TTB* pNew){
int32_t code = TDB_CODE_SUCCESS; int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_GO_TO_END(tdbTbcOpen(pOld, &pCur, NULL)); TQ_ERR_GO_TO_END(tdbTbcOpen(pOld, &pCur, NULL));
TQ_ERR_GO_TO_END(tdbBegin(pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)); TQ_ERR_GO_TO_END(
tdbBegin(pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_GO_TO_END(tdbTbcMoveToFirst(pCur)); TQ_ERR_GO_TO_END(tdbTbcMoveToFirst(pCur));
while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
TQ_ERR_GO_TO_END (tdbTbUpsert(pNew, pKey, kLen, pVal, vLen, txn)); TQ_ERR_GO_TO_END(tdbTbUpsert(pNew, pKey, kLen, pVal, vLen, txn));
} }
TQ_ERR_GO_TO_END (tdbCommit(pMetaDB, txn)); TQ_ERR_GO_TO_END(tdbCommit(pMetaDB, txn));
TQ_ERR_GO_TO_END (tdbPostCommit(pMetaDB, txn)); TQ_ERR_GO_TO_END(tdbPostCommit(pMetaDB, txn));
END: END:
tdbFree(pKey); tdbFree(pKey);
@ -338,24 +343,24 @@ END:
int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle) { int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle) {
void* data = taosHashGet(pTq->pHandle, key, strlen(key)); void* data = taosHashGet(pTq->pHandle, key, strlen(key));
if(data == NULL){ if (data == NULL) {
int vLen = 0; int vLen = 0;
if (tdbTbGet(pTq->pExecStore, key, (int)strlen(key), &data, &vLen) < 0) { if (tdbTbGet(pTq->pExecStore, key, (int)strlen(key), &data, &vLen) < 0) {
tdbFree(data); tdbFree(data);
return TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; return TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST;
} }
STqHandle handle = {0}; STqHandle handle = {0};
if (tqMetaRestoreHandle(pTq, data, vLen, &handle) != 0){ if (tqMetaRestoreHandle(pTq, data, vLen, &handle) != 0) {
tdbFree(data); tdbFree(data);
tqDestroyTqHandle(&handle); tqDestroyTqHandle(&handle);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
tdbFree(data); tdbFree(data);
*pHandle = taosHashGet(pTq->pHandle, key, strlen(key)); *pHandle = taosHashGet(pTq->pHandle, key, strlen(key));
if(*pHandle == NULL){ if (*pHandle == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
}else{ } else {
*pHandle = data; *pHandle = data;
} }
return TDB_CODE_SUCCESS; return TDB_CODE_SUCCESS;
@ -372,7 +377,7 @@ END:
return code; return code;
} }
static int32_t replaceTqPath(char** path){ static int32_t replaceTqPath(char** path) {
char* tpath = NULL; char* tpath = NULL;
int32_t code = TDB_CODE_SUCCESS; int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_RETURN(tqBuildFName(&tpath, *path, TQ_SUBSCRIBE_NAME)); TQ_ERR_RETURN(tqBuildFName(&tpath, *path, TQ_SUBSCRIBE_NAME));
@ -412,10 +417,10 @@ int32_t tqMetaOpen(STQ* pTq) {
char* offsetNew = NULL; char* offsetNew = NULL;
int32_t code = TDB_CODE_SUCCESS; int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_GO_TO_END(tqBuildFName(&maindb, pTq->path, TDB_MAINDB_NAME)); TQ_ERR_GO_TO_END(tqBuildFName(&maindb, pTq->path, TDB_MAINDB_NAME));
if(!taosCheckExistFile(maindb)){ if (!taosCheckExistFile(maindb)) {
TQ_ERR_GO_TO_END(replaceTqPath(&pTq->path)); TQ_ERR_GO_TO_END(replaceTqPath(&pTq->path));
TQ_ERR_GO_TO_END(tqMetaOpenTdb(pTq)); TQ_ERR_GO_TO_END(tqMetaOpenTdb(pTq));
}else{ } else {
TQ_ERR_GO_TO_END(tqMetaTransform(pTq)); TQ_ERR_GO_TO_END(tqMetaTransform(pTq));
(void)taosRemoveFile(maindb); (void)taosRemoveFile(maindb);
} }
@ -454,6 +459,7 @@ int32_t tqMetaTransform(STQ* pTq) {
TQ_ERR_GO_TO_END(tqMetaTransformInfo(pTq->pMetaDB, pCheckStore, pTq->pCheckStore)); TQ_ERR_GO_TO_END(tqMetaTransformInfo(pTq->pMetaDB, pCheckStore, pTq->pCheckStore));
TQ_ERR_GO_TO_END(tqBuildFName(&offsetNew, pTq->path, TQ_OFFSET_NAME)); TQ_ERR_GO_TO_END(tqBuildFName(&offsetNew, pTq->path, TQ_OFFSET_NAME));
if(taosCheckExistFile(offset)) { if(taosCheckExistFile(offset)) {
if (taosCopyFile(offset, offsetNew) < 0) { if (taosCopyFile(offset, offsetNew) < 0) {
tqError("copy offset file error"); tqError("copy offset file error");
@ -466,7 +472,7 @@ END:
taosMemoryFree(offset); taosMemoryFree(offset);
taosMemoryFree(offsetNew); taosMemoryFree(offsetNew);
//return 0 always, so ignore // return 0 always, so ignore
(void)tdbTbClose(pExecStore); (void)tdbTbClose(pExecStore);
(void)tdbTbClose(pCheckStore); (void)tdbTbClose(pCheckStore);
(void)tdbClose(pMetaDB); (void)tdbClose(pMetaDB);

22
source/dnode/vnode/src/tq/tqRead.c
View File

@ -188,7 +188,7 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t
int64_t committedVer = walGetCommittedVer(pHandle->pWalReader->pWal); int64_t committedVer = walGetCommittedVer(pHandle->pWalReader->pWal);
int64_t appliedVer = walGetAppliedVer(pHandle->pWalReader->pWal); int64_t appliedVer = walGetAppliedVer(pHandle->pWalReader->pWal);
wDebug("vgId:%d, start to fetch wal, index:%" PRId64 ", last:%" PRId64 " commit:%" PRId64 ", applied:%" PRId64 tqDebug("vgId:%d, start to fetch wal, index:%" PRId64 ", last:%" PRId64 " commit:%" PRId64 ", applied:%" PRId64
", 0x%" PRIx64, ", 0x%" PRIx64,
vgId, offset, lastVer, committedVer, appliedVer, id); vgId, offset, lastVer, committedVer, appliedVer, id);
@ -240,14 +240,12 @@ END:
return code; return code;
} }
bool tqGetTablePrimaryKey(STqReader* pReader){ bool tqGetTablePrimaryKey(STqReader* pReader) { return pReader->hasPrimaryKey; }
return pReader->hasPrimaryKey;
}
void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid){ void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid) {
bool ret = false; bool ret = false;
SSchemaWrapper *schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1); SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1);
if (schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY){ if (schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) {
ret = true; ret = true;
} }
tDeleteSchemaWrapper(schema); tDeleteSchemaWrapper(schema);
@ -311,10 +309,7 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con
int32_t code = 0; int32_t code = 0;
while (1) { while (1) {
code = walNextValidMsg(pReader); TAOS_CHECK_RETURN(walNextValidMsg(pReader));
if (code != TSDB_CODE_SUCCESS) {
return code;
}
SWalCont* pCont = &pReader->pHead->head; SWalCont* pCont = &pReader->pHead->head;
int64_t ver = pCont->version; int64_t ver = pCont->version;
@ -405,7 +400,7 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) {
pReader->msg.msgStr = NULL; pReader->msg.msgStr = NULL;
int64_t elapsed = taosGetTimestampMs() - st; int64_t elapsed = taosGetTimestampMs() - st;
if(elapsed > 1000 || elapsed < 0){ if (elapsed > 1000 || elapsed < 0) {
return false; return false;
} }
@ -684,7 +679,8 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char*
SColData* pCol = taosArrayGet(pCols, sourceIdx); SColData* pCol = taosArrayGet(pCols, sourceIdx);
SColVal colVal; SColVal colVal;
tqTrace("lostdata colActual:%d, sourceIdx:%d, targetIdx:%d, numOfCols:%d, source cid:%d, dst cid:%d", colActual, sourceIdx, targetIdx, numOfCols, pCol->cid, pColData->info.colId); tqTrace("lostdata colActual:%d, sourceIdx:%d, targetIdx:%d, numOfCols:%d, source cid:%d, dst cid:%d", colActual,
sourceIdx, targetIdx, numOfCols, pCol->cid, pColData->info.colId);
if (pCol->cid < pColData->info.colId) { if (pCol->cid < pColData->info.colId) {
sourceIdx++; sourceIdx++;
} else if (pCol->cid == pColData->info.colId) { } else if (pCol->cid == pColData->info.colId) {

14
source/libs/sync/src/syncRaftLog.c
View File

@ -206,17 +206,17 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr
SSyncLogStoreData* pData = pLogStore->data; SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal; SWal* pWal = pData->pWal;
SyncIndex index = 0;
SWalSyncInfo syncMeta = {0}; SWalSyncInfo syncMeta = {0};
syncMeta.isWeek = pEntry->isWeak; syncMeta.isWeek = pEntry->isWeak;
syncMeta.seqNum = pEntry->seqNum; syncMeta.seqNum = pEntry->seqNum;
syncMeta.term = pEntry->term; syncMeta.term = pEntry->term;
int64_t tsWriteBegin = taosGetTimestampNs(); int64_t tsWriteBegin = taosGetTimestampNs();
index = walAppendLog(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen); int32_t code = walAppendLog(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen);
int64_t tsWriteEnd = taosGetTimestampNs(); int64_t tsWriteEnd = taosGetTimestampNs();
int64_t tsElapsed = tsWriteEnd - tsWriteBegin; int64_t tsElapsed = tsWriteEnd - tsWriteBegin;
if (index < 0) { if (TSDB_CODE_SUCCESS != code) {
int32_t err = terrno; int32_t err = terrno;
const char* errStr = tstrerror(err); const char* errStr = tstrerror(err);
int32_t sysErr = errno; int32_t sysErr = errno;
@ -227,9 +227,11 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr
return -1; return -1;
} }
ASSERT(pEntry->index == index); code = walFsync(pWal, forceSync);
if (TSDB_CODE_SUCCESS != code) {
walFsync(pWal, forceSync); sNError(pData->pSyncNode, "wal fsync failed since %s", tstrerror(code));
TAOS_RETURN(code);
}
sNTrace(pData->pSyncNode, "write index:%" PRId64 ", type:%s, origin type:%s, elapsed:%" PRId64, pEntry->index, sNTrace(pData->pSyncNode, "write index:%" PRId64 ", type:%s, origin type:%s, elapsed:%" PRId64, pEntry->index,
TMSG_INFO(pEntry->msgType), TMSG_INFO(pEntry->originalRpcType), tsElapsed); TMSG_INFO(pEntry->msgType), TMSG_INFO(pEntry->originalRpcType), tsElapsed);

16
source/libs/sync/test/syncRaftLogTest.cpp
View File

@ -51,8 +51,14 @@ void test2() {
SWal *pWal = walOpen(gWalPath, &walCfg); SWal *pWal = walOpen(gWalPath, &walCfg);
assert(pWal != NULL); assert(pWal != NULL);
SWalSyncInfo syncMeta = {
.isWeek = -1,
.seqNum = UINT64_MAX,
.term = UINT64_MAX,
};
for (int i = 0; i < 5; ++i) { for (int i = 0; i < 5; ++i) {
int code = walWrite(pWal, i, 100, "aa", 3); int code = walAppendLog(pWal, i, 100, syncMeta, "aa", 3);
if (code != 0) { if (code != 0) {
printf("code:%d terror:%d msg:%s i:%d \n", code, terrno, tstrerror(terrno), i); printf("code:%d terror:%d msg:%s i:%d \n", code, terrno, tstrerror(terrno), i);
assert(0); assert(0);
@ -105,10 +111,16 @@ void test4() {
SWal *pWal = walOpen(gWalPath, &walCfg); SWal *pWal = walOpen(gWalPath, &walCfg);
assert(pWal != NULL); assert(pWal != NULL);
SWalSyncInfo syncMeta = {
.isWeek = -1,
.seqNum = UINT64_MAX,
.term = UINT64_MAX,
};
walRestoreFromSnapshot(pWal, 5); walRestoreFromSnapshot(pWal, 5);
for (int i = 6; i < 10; ++i) { for (int i = 6; i < 10; ++i) {
int code = walWrite(pWal, i, 100, "aa", 3); int code = walAppendLog(pWal, i, 100, syncMeta, "aa", 3);
if (code != 0) { if (code != 0) {
printf("code:%d terror:%d msg:%s i:%d \n", code, terrno, tstrerror(terrno), i); printf("code:%d terror:%d msg:%s i:%d \n", code, terrno, tstrerror(terrno), i);
assert(0); assert(0);

2
source/libs/tdb/src/db/tdbPager.c
View File

@ -957,6 +957,8 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage
return -1; return -1;
} }
tmemory_barrier();
pPage->pPager = pPager; pPage->pPager = pPager;
TDB_UNLOCK_PAGE(pPage); TDB_UNLOCK_PAGE(pPage);

32
source/libs/wal/inc/walInt.h
View File

@ -27,6 +27,15 @@
extern "C" { extern "C" {
#endif #endif
// clang-format off
#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", DEBUG_DEBUG, wDebugFlag, __VA_ARGS__); }}
#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", DEBUG_TRACE, wDebugFlag, __VA_ARGS__); }}
// clang-format on
// meta section begin // meta section begin
typedef struct { typedef struct {
int64_t firstVer; int64_t firstVer;
@ -144,27 +153,22 @@ static inline void walResetVer(SWalVer* pVer) {
pVer->lastVer = -1; pVer->lastVer = -1;
} }
int walLoadMeta(SWal* pWal); int32_t walLoadMeta(SWal* pWal);
int walSaveMeta(SWal* pWal); int32_t walSaveMeta(SWal* pWal);
int walRemoveMeta(SWal* pWal); int32_t walRemoveMeta(SWal* pWal);
int walRollFileInfo(SWal* pWal); int32_t walRollFileInfo(SWal* pWal);
int walCheckAndRepairMeta(SWal* pWal); int32_t walCheckAndRepairMeta(SWal* pWal);
int walCheckAndRepairIdx(SWal* pWal); int32_t walCheckAndRepairIdx(SWal* pWal);
char* walMetaSerialize(SWal* pWal); int32_t walMetaSerialize(SWal* pWal, char** serialized);
int walMetaDeserialize(SWal* pWal, const char* bytes); int32_t walMetaDeserialize(SWal* pWal, const char* bytes);
// meta section end // meta section end
// seek section int32_t decryptBody(SWalCfg* cfg, SWalCkHead* pHead, int32_t plainBodyLen, const char* func);
int64_t walChangeWrite(SWal* pWal, int64_t ver);
int walInitWriteFile(SWal* pWal);
// seek section end
int64_t walGetSeq(); int64_t walGetSeq();
int walSeekWriteVer(SWal* pWal, int64_t ver);
int32_t walRollImpl(SWal* pWal);
#ifdef __cplusplus #ifdef __cplusplus
} }

321
source/libs/wal/src/walMeta.c
View File

@ -46,9 +46,13 @@ static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) {
return sprintf(buf, "%s/meta-ver.tmp", pWal->path); return sprintf(buf, "%s/meta-ver.tmp", pWal->path);
} }
static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { static FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* lastVer) {
int32_t code = 0, lino = 0;
int32_t sz = taosArrayGetSize(pWal->fileInfoSet); int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
int64_t retVer = -1;
void* ptr = NULL;
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
@ -58,8 +62,8 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE); TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE);
if (pFile == NULL) { if (pFile == NULL) {
wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno); *lastVer = retVer;
return -1; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
// ensure size as non-negative // ensure size as non-negative
@ -73,7 +77,6 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
int64_t readSize = 0; int64_t readSize = 0;
char* buf = NULL; char* buf = NULL;
int64_t offset = TMIN(pFileInfo->fileSize, fileSize); int64_t offset = TMIN(pFileInfo->fileSize, fileSize);
int64_t retVer = -1;
int64_t lastEntryBeginOffset = 0; int64_t lastEntryBeginOffset = 0;
int64_t lastEntryEndOffset = 0; int64_t lastEntryEndOffset = 0;
int64_t recordLen = 0; int64_t recordLen = 0;
@ -94,25 +97,22 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
readSize = end - offset; readSize = end - offset;
capacity = readSize + sizeof(magic); capacity = readSize + sizeof(magic);
void* ptr = taosMemoryRealloc(buf, capacity); ptr = taosMemoryRealloc(buf, capacity);
if (ptr == NULL) { if (ptr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err);
goto _err;
} }
buf = ptr; buf = ptr;
int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET); int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET);
if (ret < 0) { if (ret < 0) {
wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), offset); wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), offset);
terrno = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
goto _err;
} }
if (readSize != taosReadFile(pFile, buf, readSize)) { if (readSize != taosReadFile(pFile, buf, readSize)) {
wError("vgId:%d, failed to read file due to %s. readSize:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), wError("vgId:%d, failed to read file due to %s. readSize:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno),
readSize, fnameStr); readSize, fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
goto _err;
} }
char* candidate = NULL; char* candidate = NULL;
@ -140,7 +140,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
logContent = (SWalCkHead*)(buf + pos); logContent = (SWalCkHead*)(buf + pos);
if (walValidHeadCksum(logContent) != 0) { if (walValidHeadCksum(logContent) != 0) {
terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; code = TSDB_CODE_WAL_CHKSUM_MISMATCH;
wWarn("vgId:%d, failed to validate checksum of wal entry header. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, wWarn("vgId:%d, failed to validate checksum of wal entry header. offset:%" PRId64 ", file:%s", pWal->cfg.vgId,
offset + pos, fnameStr); offset + pos, fnameStr);
haystack = buf + pos + 1; haystack = buf + pos + 1;
@ -163,8 +163,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
capacity += extraSize; capacity += extraSize;
void* ptr = taosMemoryRealloc(buf, capacity); void* ptr = taosMemoryRealloc(buf, capacity);
if (ptr == NULL) { if (ptr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err);
goto _err;
} }
buf = ptr; buf = ptr;
} }
@ -172,21 +171,25 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
if (ret < 0) { if (ret < 0) {
wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno),
offset); offset);
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
break; break;
} }
if (extraSize != taosReadFile(pFile, buf + readSize, extraSize)) { if (extraSize != taosReadFile(pFile, buf + readSize, extraSize)) {
wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", extraSize:%" PRId64 ", file:%s", wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", extraSize:%" PRId64 ", file:%s",
pWal->cfg.vgId, strerror(errno), offset + readSize, extraSize, fnameStr); pWal->cfg.vgId, strerror(errno), offset + readSize, extraSize, fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
break; break;
} }
} }
logContent = (SWalCkHead*)(buf + pos); logContent = (SWalCkHead*)(buf + pos);
decryptBody(&pWal->cfg, logContent, logContent->head.bodyLen, __FUNCTION__); code = decryptBody(&pWal->cfg, logContent, logContent->head.bodyLen, __FUNCTION__);
if (code) {
break;
}
if (walValidBodyCksum(logContent) != 0) { if (walValidBodyCksum(logContent) != 0) {
terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; code = TSDB_CODE_WAL_CHKSUM_MISMATCH;
wWarn("vgId:%d, failed to validate checksum of wal entry body. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, wWarn("vgId:%d, failed to validate checksum of wal entry body. offset:%" PRId64 ", file:%s", pWal->cfg.vgId,
offset + pos, fnameStr); offset + pos, fnameStr);
haystack = buf + pos + 1; haystack = buf + pos + 1;
@ -211,7 +214,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
} }
if (retVer < 0) { if (retVer < 0) {
terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; code = TSDB_CODE_WAL_LOG_NOT_EXIST;
} }
// truncate file // truncate file
@ -221,30 +224,26 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
if (taosFtruncateFile(pFile, lastEntryEndOffset) < 0) { if (taosFtruncateFile(pFile, lastEntryEndOffset) < 0) {
wError("failed to truncate file due to %s. file:%s", strerror(errno), fnameStr); wError("failed to truncate file due to %s. file:%s", strerror(errno), fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
goto _err;
} }
if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pFile) < 0) { if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pFile) < 0) {
wError("failed to fsync file due to %s. file:%s", strerror(errno), fnameStr); wError("failed to fsync file due to %s. file:%s", strerror(errno), fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
goto _err;
} }
} }
pFileInfo->fileSize = lastEntryEndOffset; pFileInfo->fileSize = lastEntryEndOffset;
taosCloseFile(&pFile);
taosMemoryFree(buf);
return retVer;
_err: _err:
taosCloseFile(&pFile); taosCloseFile(&pFile);
taosMemoryFree(buf); taosMemoryFree(buf);
return -1; *lastVer = retVer;
TAOS_RETURN(code);
} }
static void walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) { static int32_t walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) {
int metaFileNum = taosArrayGetSize(metaLogList); int metaFileNum = taosArrayGetSize(metaLogList);
int actualFileNum = taosArrayGetSize(actualLogList); int actualFileNum = taosArrayGetSize(actualLogList);
int j = 0; int j = 0;
@ -270,11 +269,15 @@ static void walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) {
for (int i = 0; i < actualFileNum; i++) { for (int i = 0; i < actualFileNum; i++) {
SWalFileInfo* pFileInfo = taosArrayGet(actualLogList, i); SWalFileInfo* pFileInfo = taosArrayGet(actualLogList, i);
taosArrayPush(metaLogList, pFileInfo); if (NULL == taosArrayPush(metaLogList, pFileInfo)) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
} }
}
return TSDB_CODE_SUCCESS;
} }
void walAlignVersions(SWal* pWal) { static void walAlignVersions(SWal* pWal) {
if (pWal->vers.firstVer > pWal->vers.snapshotVer + 1) { if (pWal->vers.firstVer > pWal->vers.snapshotVer + 1) {
wWarn("vgId:%d, firstVer:%" PRId64 " is larger than snapshotVer:%" PRId64 " + 1. align with it.", pWal->cfg.vgId, wWarn("vgId:%d, firstVer:%" PRId64 " is larger than snapshotVer:%" PRId64 " + 1. align with it.", pWal->cfg.vgId,
pWal->vers.firstVer, pWal->vers.snapshotVer); pWal->vers.firstVer, pWal->vers.snapshotVer);
@ -294,7 +297,7 @@ void walAlignVersions(SWal* pWal) {
wInfo("vgId:%d, reset commitVer to %" PRId64, pWal->cfg.vgId, pWal->vers.commitVer); wInfo("vgId:%d, reset commitVer to %" PRId64, pWal->cfg.vgId, pWal->vers.commitVer);
} }
int walRepairLogFileTs(SWal* pWal, bool* updateMeta) { static int32_t walRepairLogFileTs(SWal* pWal, bool* updateMeta) {
int32_t sz = taosArrayGetSize(pWal->fileInfoSet); int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
int32_t fileIdx = -1; int32_t fileIdx = -1;
int32_t lastCloseTs = 0; int32_t lastCloseTs = 0;
@ -310,9 +313,9 @@ int walRepairLogFileTs(SWal* pWal, bool* updateMeta) {
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
int32_t mtime = 0; int32_t mtime = 0;
if (taosStatFile(fnameStr, NULL, &mtime, NULL) < 0) { if (taosStatFile(fnameStr, NULL, &mtime, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, failed to stat file due to %s, file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); wError("vgId:%d, failed to stat file due to %s, file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
if (updateMeta != NULL) *updateMeta = true; if (updateMeta != NULL) *updateMeta = true;
@ -321,10 +324,10 @@ int walRepairLogFileTs(SWal* pWal, bool* updateMeta) {
lastCloseTs = pFileInfo->closeTs; lastCloseTs = pFileInfo->closeTs;
} }
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
bool walLogEntriesComplete(const SWal* pWal) { static int32_t walLogEntriesComplete(const SWal* pWal) {
int32_t sz = taosArrayGetSize(pWal->fileInfoSet); int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
bool complete = true; bool complete = true;
int32_t fileIdx = -1; int32_t fileIdx = -1;
@ -346,13 +349,13 @@ bool walLogEntriesComplete(const SWal* pWal) {
wError("vgId:%d, WAL log entries incomplete in range [%" PRId64 ", %" PRId64 "], index:%" PRId64 wError("vgId:%d, WAL log entries incomplete in range [%" PRId64 ", %" PRId64 "], index:%" PRId64
", snaphotVer:%" PRId64, ", snaphotVer:%" PRId64,
pWal->cfg.vgId, pWal->vers.firstVer, pWal->vers.lastVer, index, pWal->vers.snapshotVer); pWal->cfg.vgId, pWal->vers.firstVer, pWal->vers.lastVer, index, pWal->vers.snapshotVer);
terrno = TSDB_CODE_WAL_LOG_INCOMPLETE; TAOS_RETURN(TSDB_CODE_WAL_LOG_INCOMPLETE);
} else {
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
return complete;
} }
int walTrimIdxFile(SWal* pWal, int32_t fileIdx) { static int32_t walTrimIdxFile(SWal* pWal, int32_t fileIdx) {
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
ASSERT(pFileInfo != NULL); ASSERT(pFileInfo != NULL);
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
@ -364,13 +367,12 @@ int walTrimIdxFile(SWal* pWal, int32_t fileIdx) {
int64_t lastEndOffset = records * sizeof(SWalIdxEntry); int64_t lastEndOffset = records * sizeof(SWalIdxEntry);
if (fileSize <= lastEndOffset) { if (fileSize <= lastEndOffset) {
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE); TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
return -1;
} }
wInfo("vgId:%d, trim idx file. file: %s, size: %" PRId64 ", offset: %" PRId64, pWal->cfg.vgId, fnameStr, fileSize, wInfo("vgId:%d, trim idx file. file: %s, size: %" PRId64 ", offset: %" PRId64, pWal->cfg.vgId, fnameStr, fileSize,
@ -378,11 +380,13 @@ int walTrimIdxFile(SWal* pWal, int32_t fileIdx) {
taosFtruncateFile(pFile, lastEndOffset); taosFtruncateFile(pFile, lastEndOffset);
taosCloseFile(&pFile); taosCloseFile(&pFile);
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int walCheckAndRepairMeta(SWal* pWal) { int32_t walCheckAndRepairMeta(SWal* pWal) {
// load log files, get first/snapshot/last version info // load log files, get first/snapshot/last version info
int32_t code = 0;
const char* logPattern = "^[0-9]+.log$"; const char* logPattern = "^[0-9]+.log$";
const char* idxPattern = "^[0-9]+.idx$"; const char* idxPattern = "^[0-9]+.idx$";
regex_t logRegPattern; regex_t logRegPattern;
@ -396,7 +400,7 @@ int walCheckAndRepairMeta(SWal* pWal) {
regfree(&logRegPattern); regfree(&logRegPattern);
regfree(&idxRegPattern); regfree(&idxRegPattern);
wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno));
return -1; TAOS_RETURN(TSDB_CODE_FAILED);
} }
SArray* actualLog = taosArrayInit(8, sizeof(SWalFileInfo)); SArray* actualLog = taosArrayInit(8, sizeof(SWalFileInfo));
@ -428,8 +432,11 @@ int walCheckAndRepairMeta(SWal* pWal) {
bool updateMeta = (metaFileNum != actualFileNum); bool updateMeta = (metaFileNum != actualFileNum);
// rebuild meta of file info // rebuild meta of file info
walRebuildFileInfoSet(pWal->fileInfoSet, actualLog); code = walRebuildFileInfoSet(pWal->fileInfoSet, actualLog);
taosArrayDestroy(actualLog); taosArrayDestroy(actualLog);
if (code) {
TAOS_RETURN(code);
}
int32_t sz = taosArrayGetSize(pWal->fileInfoSet); int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
@ -444,9 +451,9 @@ int walCheckAndRepairMeta(SWal* pWal) {
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
int32_t code = taosStatFile(fnameStr, &fileSize, NULL, NULL); int32_t code = taosStatFile(fnameStr, &fileSize, NULL, NULL);
if (code < 0) { if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("failed to stat file since %s. file:%s", terrstr(), fnameStr); wError("failed to stat file since %s. file:%s", terrstr(), fnameStr);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
if (pFileInfo->lastVer >= pFileInfo->firstVer && fileSize == pFileInfo->fileSize) { if (pFileInfo->lastVer >= pFileInfo->firstVer && fileSize == pFileInfo->fileSize) {
@ -455,16 +462,20 @@ int walCheckAndRepairMeta(SWal* pWal) {
} }
updateMeta = true; updateMeta = true;
(void)walTrimIdxFile(pWal, fileIdx); TAOS_CHECK_RETURN(walTrimIdxFile(pWal, fileIdx));
int64_t lastVer = walScanLogGetLastVer(pWal, fileIdx); int64_t lastVer = -1;
code = walScanLogGetLastVer(pWal, fileIdx, &lastVer);
if (lastVer < 0) { if (lastVer < 0) {
if (terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) { if (code != TSDB_CODE_WAL_LOG_NOT_EXIST) {
wError("failed to scan wal last ver since %s", terrstr()); wError("failed to scan wal last ver since %s", terrstr());
return -1;
TAOS_RETURN(code);
} }
// empty log file // empty log file
lastVer = pFileInfo->firstVer - 1; lastVer = pFileInfo->firstVer - 1;
code = TSDB_CODE_SUCCESS;
} }
// update lastVer // update lastVer
@ -481,45 +492,39 @@ int walCheckAndRepairMeta(SWal* pWal) {
pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer;
pWal->vers.lastVer = ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer; pWal->vers.lastVer = ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer;
} }
(void)walAlignVersions(pWal); walAlignVersions(pWal);
// repair ts of files // repair ts of files
if (walRepairLogFileTs(pWal, &updateMeta) < 0) { TAOS_CHECK_RETURN(walRepairLogFileTs(pWal, &updateMeta));
return -1;
}
// update meta file // update meta file
if (updateMeta) { if (updateMeta) {
(void)walSaveMeta(pWal); TAOS_CHECK_RETURN(walSaveMeta(pWal));
} }
if (!walLogEntriesComplete(pWal)) { TAOS_CHECK_RETURN(walLogEntriesComplete(pWal));
return -1;
}
return 0; return code;
} }
int walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) { static int32_t walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) {
if (taosLSeekFile(pLogFile, offset, SEEK_SET) < 0) { if (taosLSeekFile(pLogFile, offset, SEEK_SET) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
return -1;
} }
if (taosReadFile(pLogFile, pCkHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { if (taosReadFile(pLogFile, pCkHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) {
terrno = TAOS_SYSTEM_ERROR(errno); TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
return -1;
} }
if (walValidHeadCksum(pCkHead) != 0) { if (walValidHeadCksum(pCkHead) != 0) {
terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; TAOS_RETURN(TSDB_CODE_WAL_CHKSUM_MISMATCH);
return -1;
} }
return 0; return TSDB_CODE_SUCCESS;
} }
int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { static int32_t walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
int32_t code = 0, lino = 0;
int32_t sz = taosArrayGetSize(pWal->fileInfoSet); int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
@ -530,12 +535,12 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
if (taosStatFile(fnameStr, &fileSize, NULL, NULL) < 0 && errno != ENOENT) { if (taosStatFile(fnameStr, &fileSize, NULL, NULL) < 0 && errno != ENOENT) {
wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
if (fileSize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) { if (fileSize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) {
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
// start to repair // start to repair
@ -550,15 +555,15 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE); pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE);
if (pIdxFile == NULL) { if (pIdxFile == NULL) {
wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err; TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
} }
pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ); pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ);
if (pLogFile == NULL) { if (pLogFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr()); wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr());
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
} }
// determine the last valid entry end, i.e. offset // determine the last valid entry end, i.e. offset
@ -566,15 +571,15 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
if (taosLSeekFile(pIdxFile, offset, SEEK_SET) < 0) { if (taosLSeekFile(pIdxFile, offset, SEEK_SET) < 0) {
wError("vgId:%d, failed to seek file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), wError("vgId:%d, failed to seek file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno),
offset, fnameStr); offset, fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err; TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
} }
if (taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { if (taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno),
offset, fnameStr); offset, fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err; TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
} }
if (idxEntry.ver > pFileInfo->lastVer) { if (idxEntry.ver > pFileInfo->lastVer) {
@ -602,19 +607,19 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
// ftruncate idx file // ftruncate idx file
if (offset < fileSize) { if (offset < fileSize) {
if (taosFtruncateFile(pIdxFile, offset) < 0) { if (taosFtruncateFile(pIdxFile, offset) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, failed to ftruncate file since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(), wError("vgId:%d, failed to ftruncate file since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(),
offset, fnameStr); offset, fnameStr);
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
} }
} }
// rebuild idx file // rebuild idx file
if (taosLSeekFile(pIdxFile, 0, SEEK_END) < 0) { if (taosLSeekFile(pIdxFile, 0, SEEK_END) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, failed to seek file since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(), offset, wError("vgId:%d, failed to seek file since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(), offset,
fnameStr); fnameStr);
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
} }
int64_t count = 0; int64_t count = 0;
@ -630,23 +635,25 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
} }
idxEntry.offset += sizeof(SWalCkHead) + cryptedBodyLen; idxEntry.offset += sizeof(SWalCkHead) + cryptedBodyLen;
if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) { code = walReadLogHead(pLogFile, idxEntry.offset, &ckHead);
if (code) {
wError("vgId:%d, failed to read wal log head since %s. index:%" PRId64 ", offset:%" PRId64 ", file:%s", wError("vgId:%d, failed to read wal log head since %s. index:%" PRId64 ", offset:%" PRId64 ", file:%s",
pWal->cfg.vgId, terrstr(), idxEntry.ver, idxEntry.offset, fLogNameStr); pWal->cfg.vgId, terrstr(), idxEntry.ver, idxEntry.offset, fLogNameStr);
goto _err;
TAOS_CHECK_GOTO(code, &lino, _err);
} }
if (pWal->cfg.level != TAOS_WAL_SKIP && taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) < 0) { if (pWal->cfg.level != TAOS_WAL_SKIP && taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, failed to append file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); wError("vgId:%d, failed to append file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr);
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
} }
count++; count++;
} }
if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pIdxFile) < 0) { if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pIdxFile) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, faild to fsync file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); wError("vgId:%d, faild to fsync file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr);
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
} }
if (count > 0) { if (count > 0) {
@ -654,14 +661,15 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
pFileInfo->lastVer); pFileInfo->lastVer);
} }
(void)taosCloseFile(&pLogFile);
(void)taosCloseFile(&pIdxFile);
return 0;
_err: _err:
if (code) {
wError("vgId:%d, %s failed at line %d since %s", pWal->cfg.vgId, __func__, lino, tstrerror(code));
}
(void)taosCloseFile(&pLogFile); (void)taosCloseFile(&pLogFile);
(void)taosCloseFile(&pIdxFile); (void)taosCloseFile(&pIdxFile);
return -1;
TAOS_RETURN(code);
} }
int64_t walGetVerRetention(SWal* pWal, int64_t bytes) { int64_t walGetVerRetention(SWal* pWal, int64_t bytes) {
@ -681,19 +689,24 @@ int64_t walGetVerRetention(SWal* pWal, int64_t bytes) {
return ver + 1; return ver + 1;
} }
int walCheckAndRepairIdx(SWal* pWal) { int32_t walCheckAndRepairIdx(SWal* pWal) {
int32_t code = 0;
int32_t sz = taosArrayGetSize(pWal->fileInfoSet); int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
int32_t fileIdx = sz; int32_t fileIdx = sz;
while (--fileIdx >= 0) { while (--fileIdx >= 0) {
if (walCheckAndRepairIdxFile(pWal, fileIdx) < 0) { code = walCheckAndRepairIdxFile(pWal, fileIdx);
if (code) {
wError("vgId:%d, failed to repair idx file since %s. fileIdx:%d", pWal->cfg.vgId, terrstr(), fileIdx); wError("vgId:%d, failed to repair idx file since %s. fileIdx:%d", pWal->cfg.vgId, terrstr(), fileIdx);
return -1;
TAOS_RETURN(code);
} }
} }
return 0;
TAOS_RETURN(code);
} }
int walRollFileInfo(SWal* pWal) { int32_t walRollFileInfo(SWal* pWal) {
int64_t ts = taosGetTimestampSec(); int64_t ts = taosGetTimestampSec();
SArray* pArray = pWal->fileInfoSet; SArray* pArray = pWal->fileInfoSet;
@ -706,8 +719,7 @@ int walRollFileInfo(SWal* pWal) {
// TODO: change to emplace back // TODO: change to emplace back
SWalFileInfo* pNewInfo = taosMemoryMalloc(sizeof(SWalFileInfo)); SWalFileInfo* pNewInfo = taosMemoryMalloc(sizeof(SWalFileInfo));
if (pNewInfo == NULL) { if (pNewInfo == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
return -1;
} }
pNewInfo->firstVer = pWal->vers.lastVer + 1; pNewInfo->firstVer = pWal->vers.lastVer + 1;
pNewInfo->lastVer = -1; pNewInfo->lastVer = -1;
@ -717,10 +729,11 @@ int walRollFileInfo(SWal* pWal) {
pNewInfo->syncedOffset = 0; pNewInfo->syncedOffset = 0;
taosArrayPush(pArray, pNewInfo); taosArrayPush(pArray, pNewInfo);
taosMemoryFree(pNewInfo); taosMemoryFree(pNewInfo);
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
char* walMetaSerialize(SWal* pWal) { int32_t walMetaSerialize(SWal* pWal, char** serialized) {
char buf[30]; char buf[30];
int sz = taosArrayGetSize(pWal->fileInfoSet); int sz = taosArrayGetSize(pWal->fileInfoSet);
cJSON* pRoot = cJSON_CreateObject(); cJSON* pRoot = cJSON_CreateObject();
@ -737,8 +750,8 @@ char* walMetaSerialize(SWal* pWal) {
if (pFiles) { if (pFiles) {
cJSON_Delete(pFiles); cJSON_Delete(pFiles);
} }
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
} }
cJSON_AddItemToObject(pRoot, "meta", pMeta); cJSON_AddItemToObject(pRoot, "meta", pMeta);
sprintf(buf, "%" PRId64, pWal->vers.firstVer); sprintf(buf, "%" PRId64, pWal->vers.firstVer);
@ -757,7 +770,8 @@ char* walMetaSerialize(SWal* pWal) {
cJSON_AddItemToArray(pFiles, pField = cJSON_CreateObject()); cJSON_AddItemToArray(pFiles, pField = cJSON_CreateObject());
if (pField == NULL) { if (pField == NULL) {
cJSON_Delete(pRoot); cJSON_Delete(pRoot);
return NULL;
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
} }
// cjson only support int32_t or double // cjson only support int32_t or double
// string are used to prohibit the loss of precision // string are used to prohibit the loss of precision
@ -772,12 +786,15 @@ char* walMetaSerialize(SWal* pWal) {
sprintf(buf, "%" PRId64, pInfo->fileSize); sprintf(buf, "%" PRId64, pInfo->fileSize);
cJSON_AddStringToObject(pField, "fileSize", buf); cJSON_AddStringToObject(pField, "fileSize", buf);
} }
char* serialized = cJSON_Print(pRoot); char* pSerialized = cJSON_Print(pRoot);
cJSON_Delete(pRoot); cJSON_Delete(pRoot);
return serialized;
*serialized = pSerialized;
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int walMetaDeserialize(SWal* pWal, const char* bytes) { int32_t walMetaDeserialize(SWal* pWal, const char* bytes) {
/*A(taosArrayGetSize(pWal->fileInfoSet) == 0);*/ /*A(taosArrayGetSize(pWal->fileInfoSet) == 0);*/
cJSON *pRoot, *pMeta, *pFiles, *pInfoJson, *pField; cJSON *pRoot, *pMeta, *pFiles, *pInfoJson, *pField;
pRoot = cJSON_Parse(bytes); pRoot = cJSON_Parse(bytes);
@ -829,11 +846,11 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) {
pWal->fileInfoSet = pArray; pWal->fileInfoSet = pArray;
pWal->writeCur = sz - 1; pWal->writeCur = sz - 1;
cJSON_Delete(pRoot); cJSON_Delete(pRoot);
return 0; return TSDB_CODE_SUCCESS;
_err: _err:
cJSON_Delete(pRoot); cJSON_Delete(pRoot);
return -1; return TSDB_CODE_FAILED;
} }
static int walFindCurMetaVer(SWal* pWal) { static int walFindCurMetaVer(SWal* pWal) {
@ -866,13 +883,14 @@ static int walFindCurMetaVer(SWal* pWal) {
return metaVer; return metaVer;
} }
void walUpdateSyncedOffset(SWal* pWal) { static void walUpdateSyncedOffset(SWal* pWal) {
SWalFileInfo* pFileInfo = walGetCurFileInfo(pWal); SWalFileInfo* pFileInfo = walGetCurFileInfo(pWal);
if (pFileInfo == NULL) return; if (pFileInfo == NULL) return;
pFileInfo->syncedOffset = pFileInfo->fileSize; pFileInfo->syncedOffset = pFileInfo->fileSize;
} }
int walSaveMeta(SWal* pWal) { int32_t walSaveMeta(SWal* pWal) {
int code = 0, lino = 0;
int metaVer = walFindCurMetaVer(pWal); int metaVer = walFindCurMetaVer(pWal);
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
char tmpFnameStr[WAL_FILE_LEN]; char tmpFnameStr[WAL_FILE_LEN];
@ -881,63 +899,64 @@ int walSaveMeta(SWal* pWal) {
// fsync the idx and log file at first to ensure validity of meta // fsync the idx and log file at first to ensure validity of meta
if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pWal->pIdxFile) < 0) { if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pWal->pIdxFile) < 0) {
wError("vgId:%d, failed to sync idx file due to %s", pWal->cfg.vgId, strerror(errno)); wError("vgId:%d, failed to sync idx file due to %s", pWal->cfg.vgId, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pWal->pLogFile) < 0) { if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pWal->pLogFile) < 0) {
wError("vgId:%d, failed to sync log file due to %s", pWal->cfg.vgId, strerror(errno)); wError("vgId:%d, failed to sync log file due to %s", pWal->cfg.vgId, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
// update synced offset // update synced offset
(void)walUpdateSyncedOffset(pWal); walUpdateSyncedOffset(pWal);
// flush to a tmpfile // flush to a tmpfile
n = walBuildTmpMetaName(pWal, tmpFnameStr); n = walBuildTmpMetaName(pWal, tmpFnameStr);
if (n >= sizeof(tmpFnameStr)) { if (n >= sizeof(tmpFnameStr)) {
return -1; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
TdFilePtr pMetaFile = TdFilePtr pMetaFile =
taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pMetaFile == NULL) { if (pMetaFile == NULL) {
wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
char* serialized = walMetaSerialize(pWal); char* serialized = NULL;
TAOS_CHECK_RETURN(walMetaSerialize(pWal, &serialized));
int len = strlen(serialized); int len = strlen(serialized);
if (pWal->cfg.level != TAOS_WAL_SKIP && len != taosWriteFile(pMetaFile, serialized, len)) { if (pWal->cfg.level != TAOS_WAL_SKIP && len != taosWriteFile(pMetaFile, serialized, len)) {
wError("vgId:%d, failed to write file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); wError("vgId:%d, failed to write file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err; TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
} }
if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pMetaFile) < 0) { if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pMetaFile) < 0) {
wError("vgId:%d, failed to sync file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); wError("vgId:%d, failed to sync file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err; TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
} }
if (taosCloseFile(&pMetaFile) < 0) { if (taosCloseFile(&pMetaFile) < 0) {
wError("vgId:%d, failed to close file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); wError("vgId:%d, failed to close file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err; TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
} }
// rename it // rename it
n = walBuildMetaName(pWal, metaVer + 1, fnameStr); n = walBuildMetaName(pWal, metaVer + 1, fnameStr);
if (n >= sizeof(fnameStr)) { if (n >= sizeof(fnameStr)) {
goto _err; TAOS_CHECK_GOTO(TSDB_CODE_FAILED, &lino, _err);
} }
if (taosRenameFile(tmpFnameStr, fnameStr) < 0) { if (taosRenameFile(tmpFnameStr, fnameStr) < 0) {
wError("failed to rename file due to %s. dest:%s", strerror(errno), fnameStr); wError("failed to rename file due to %s. dest:%s", strerror(errno), fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err; TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
} }
// delete old file // delete old file
@ -945,21 +964,24 @@ int walSaveMeta(SWal* pWal) {
walBuildMetaName(pWal, metaVer, fnameStr); walBuildMetaName(pWal, metaVer, fnameStr);
taosRemoveFile(fnameStr); taosRemoveFile(fnameStr);
} }
taosMemoryFree(serialized); taosMemoryFree(serialized);
return 0; return code;
_err: _err:
taosCloseFile(&pMetaFile); taosCloseFile(&pMetaFile);
taosMemoryFree(serialized); taosMemoryFree(serialized);
return -1; return code;
} }
int walLoadMeta(SWal* pWal) { int32_t walLoadMeta(SWal* pWal) {
int32_t code = 0;
// find existing meta file // find existing meta file
int metaVer = walFindCurMetaVer(pWal); int metaVer = walFindCurMetaVer(pWal);
if (metaVer == -1) { if (metaVer == -1) {
wDebug("vgId:%d, wal find meta ver %d", pWal->cfg.vgId, metaVer); wDebug("vgId:%d, wal find meta ver %d", pWal->cfg.vgId, metaVer);
return -1;
TAOS_RETURN(TSDB_CODE_FAILED);
} }
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
walBuildMetaName(pWal, metaVer, fnameStr); walBuildMetaName(pWal, metaVer, fnameStr);
@ -969,39 +991,40 @@ int walLoadMeta(SWal* pWal) {
if (fileSize == 0) { if (fileSize == 0) {
(void)taosRemoveFile(fnameStr); (void)taosRemoveFile(fnameStr);
wDebug("vgId:%d, wal find empty meta ver %d", pWal->cfg.vgId, metaVer); wDebug("vgId:%d, wal find empty meta ver %d", pWal->cfg.vgId, metaVer);
return -1;
TAOS_RETURN(TSDB_CODE_FAILED);
} }
int size = (int)fileSize; int size = (int)fileSize;
char* buf = taosMemoryMalloc(size + 5); char* buf = taosMemoryMalloc(size + 5);
if (buf == NULL) { if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
return -1;
} }
memset(buf, 0, size + 5); memset(buf, 0, size + 5);
TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
taosMemoryFree(buf); taosMemoryFree(buf);
return -1;
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
} }
if (taosReadFile(pFile, buf, size) != size) { if (taosReadFile(pFile, buf, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFile); taosCloseFile(&pFile);
taosMemoryFree(buf); taosMemoryFree(buf);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
// load into fileInfoSet // load into fileInfoSet
int code = walMetaDeserialize(pWal, buf); code = walMetaDeserialize(pWal, buf);
if (code < 0) { if (code < 0) {
wError("failed to deserialize wal meta. file:%s", fnameStr); wError("failed to deserialize wal meta. file:%s", fnameStr);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; code = TSDB_CODE_WAL_FILE_CORRUPTED;
} }
taosCloseFile(&pFile); taosCloseFile(&pFile);
taosMemoryFree(buf); taosMemoryFree(buf);
return code;
TAOS_RETURN(code);
} }
int walRemoveMeta(SWal* pWal) { int32_t walRemoveMeta(SWal* pWal) {
int metaVer = walFindCurMetaVer(pWal); int metaVer = walFindCurMetaVer(pWal);
if (metaVer == -1) return 0; if (metaVer == -1) return 0;
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];

29
source/libs/wal/src/walMgmt.c
View File

@ -46,10 +46,11 @@ int32_t walInit() {
tsWal.refSetId = taosOpenRef(TSDB_MIN_VNODES, walFreeObj); tsWal.refSetId = taosOpenRef(TSDB_MIN_VNODES, walFreeObj);
int32_t code = walCreateThread(); int32_t code = walCreateThread();
if (code != 0) { if (TSDB_CODE_SUCCESS != code) {
wError("failed to init wal module since %s", tstrerror(code)); wError("failed to init wal module since %s", tstrerror(code));
atomic_store_8(&tsWal.inited, 0); atomic_store_8(&tsWal.inited, 0);
return code;
TAOS_RETURN(code);
} }
wInfo("wal module is initialized, rsetId:%d", tsWal.refSetId); wInfo("wal module is initialized, rsetId:%d", tsWal.refSetId);
@ -171,19 +172,20 @@ _err:
taosArrayDestroy(pWal->fileInfoSet); taosArrayDestroy(pWal->fileInfoSet);
taosHashCleanup(pWal->pRefHash); taosHashCleanup(pWal->pRefHash);
taosThreadMutexDestroy(&pWal->mutex); taosThreadMutexDestroy(&pWal->mutex);
taosMemoryFree(pWal); taosMemoryFreeClear(pWal);
pWal = NULL;
return NULL; return NULL;
} }
int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { int32_t walAlter(SWal *pWal, SWalCfg *pCfg) {
if (pWal == NULL) return TSDB_CODE_APP_ERROR; if (pWal == NULL) TAOS_RETURN(TSDB_CODE_APP_ERROR);
if (pWal->cfg.level == pCfg->level && pWal->cfg.fsyncPeriod == pCfg->fsyncPeriod && if (pWal->cfg.level == pCfg->level && pWal->cfg.fsyncPeriod == pCfg->fsyncPeriod &&
pWal->cfg.retentionPeriod == pCfg->retentionPeriod && pWal->cfg.retentionSize == pCfg->retentionSize) { pWal->cfg.retentionPeriod == pCfg->retentionPeriod && pWal->cfg.retentionSize == pCfg->retentionSize) {
wDebug("vgId:%d, walLevel:%d fsync:%d walRetentionPeriod:%d walRetentionSize:%" PRId64 " not change", wDebug("vgId:%d, walLevel:%d fsync:%d walRetentionPeriod:%d walRetentionSize:%" PRId64 " not change",
pWal->cfg.vgId, pWal->cfg.level, pWal->cfg.fsyncPeriod, pWal->cfg.retentionPeriod, pWal->cfg.retentionSize); pWal->cfg.vgId, pWal->cfg.level, pWal->cfg.fsyncPeriod, pWal->cfg.retentionPeriod, pWal->cfg.retentionSize);
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
wInfo("vgId:%d, change old walLevel:%d fsync:%d walRetentionPeriod:%d walRetentionSize:%" PRId64 wInfo("vgId:%d, change old walLevel:%d fsync:%d walRetentionPeriod:%d walRetentionSize:%" PRId64
@ -199,14 +201,17 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) {
pWal->fsyncSeq = pCfg->fsyncPeriod / 1000; pWal->fsyncSeq = pCfg->fsyncPeriod / 1000;
if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1; if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1;
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t walPersist(SWal *pWal) { int32_t walPersist(SWal *pWal) {
int32_t code = 0;
taosThreadMutexLock(&pWal->mutex); taosThreadMutexLock(&pWal->mutex);
int32_t ret = walSaveMeta(pWal); code = walSaveMeta(pWal);
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return ret;
TAOS_RETURN(code);
} }
void walClose(SWal *pWal) { void walClose(SWal *pWal) {
@ -301,14 +306,14 @@ static int32_t walCreateThread() {
if (taosThreadCreate(&tsWal.thread, &thAttr, walThreadFunc, NULL) != 0) { if (taosThreadCreate(&tsWal.thread, &thAttr, walThreadFunc, NULL) != 0) {
wError("failed to create wal thread since %s", strerror(errno)); wError("failed to create wal thread since %s", strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
taosThreadAttrDestroy(&thAttr); taosThreadAttrDestroy(&thAttr);
wDebug("wal thread is launched, thread:0x%08" PRIx64, taosGetPthreadId(tsWal.thread)); wDebug("wal thread is launched, thread:0x%08" PRIx64, taosGetPthreadId(tsWal.thread));
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
static void walStopThread() { static void walStopThread() {

268
source/libs/wal/src/walRead.c
View File

@ -26,7 +26,7 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond, int64_t id) {
} }
pReader->pWal = pWal; pReader->pWal = pWal;
pReader->readerId = (id != 0)? id:tGenIdPI64(); pReader->readerId = (id != 0) ? id : tGenIdPI64();
pReader->pIdxFile = NULL; pReader->pIdxFile = NULL;
pReader->pLogFile = NULL; pReader->pLogFile = NULL;
pReader->curVersion = -1; pReader->curVersion = -1;
@ -35,7 +35,7 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond, int64_t id) {
if (cond) { if (cond) {
pReader->cond = *cond; pReader->cond = *cond;
} else { } else {
// pReader->cond.scanUncommited = 0; // pReader->cond.scanUncommited = 0;
pReader->cond.scanNotApplied = 0; pReader->cond.scanNotApplied = 0;
pReader->cond.scanMeta = 0; pReader->cond.scanMeta = 0;
pReader->cond.enableRef = 0; pReader->cond.enableRef = 0;
@ -58,7 +58,7 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond, int64_t id) {
} }
void walCloseReader(SWalReader *pReader) { void walCloseReader(SWalReader *pReader) {
if(pReader == NULL) return; if (pReader == NULL) return;
taosCloseFile(&pReader->pIdxFile); taosCloseFile(&pReader->pIdxFile);
taosCloseFile(&pReader->pLogFile); taosCloseFile(&pReader->pLogFile);
@ -75,30 +75,24 @@ int32_t walNextValidMsg(SWalReader *pReader) {
wDebug("vgId:%d, wal start to fetch, index:%" PRId64 ", last:%" PRId64 " commit:%" PRId64 ", applied:%" PRId64, wDebug("vgId:%d, wal start to fetch, index:%" PRId64 ", last:%" PRId64 " commit:%" PRId64 ", applied:%" PRId64,
pReader->pWal->cfg.vgId, fetchVer, lastVer, committedVer, appliedVer); pReader->pWal->cfg.vgId, fetchVer, lastVer, committedVer, appliedVer);
if (fetchVer > appliedVer) { if (fetchVer > appliedVer) {
terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; TAOS_RETURN(TSDB_CODE_WAL_LOG_NOT_EXIST);
return -1;
} }
while (fetchVer <= appliedVer) { while (fetchVer <= appliedVer) {
if (walFetchHead(pReader, fetchVer) < 0) { TAOS_CHECK_RETURN(walFetchHead(pReader, fetchVer));
return -1;
}
int32_t type = pReader->pHead->head.msgType; int32_t type = pReader->pHead->head.msgType;
if (type == TDMT_VND_SUBMIT || ((type == TDMT_VND_DELETE) && (pReader->cond.deleteMsg == 1)) || if (type == TDMT_VND_SUBMIT || ((type == TDMT_VND_DELETE) && (pReader->cond.deleteMsg == 1)) ||
(IS_META_MSG(type) && pReader->cond.scanMeta)) { (IS_META_MSG(type) && pReader->cond.scanMeta)) {
int32_t code = walFetchBody(pReader); TAOS_RETURN(walFetchBody(pReader));
return (code == TSDB_CODE_SUCCESS)? 0:-1;
} else { } else {
if (walSkipFetchBody(pReader) < 0) { TAOS_CHECK_RETURN(walSkipFetchBody(pReader));
return -1;
}
fetchVer = pReader->curVersion; fetchVer = pReader->curVersion;
} }
} }
return -1; TAOS_RETURN(TSDB_CODE_FAILED);
} }
int64_t walReaderGetCurrentVer(const SWalReader *pReader) { return pReader->curVersion; } int64_t walReaderGetCurrentVer(const SWalReader *pReader) { return pReader->curVersion; }
@ -119,18 +113,18 @@ void walReaderValidVersionRange(SWalReader *pReader, int64_t *sver, int64_t *eve
*ever = pReader->cond.scanUncommited ? lastVer : committedVer; *ever = pReader->cond.scanUncommited ? lastVer : committedVer;
} }
void walReaderVerifyOffset(SWalReader *pWalReader, STqOffsetVal* pOffset){ void walReaderVerifyOffset(SWalReader *pWalReader, STqOffsetVal *pOffset) {
// if offset version is small than first version , let's seek to first version // if offset version is small than first version , let's seek to first version
taosThreadMutexLock(&pWalReader->pWal->mutex); taosThreadMutexLock(&pWalReader->pWal->mutex);
int64_t firstVer = walGetFirstVer((pWalReader)->pWal); int64_t firstVer = walGetFirstVer((pWalReader)->pWal);
taosThreadMutexUnlock(&pWalReader->pWal->mutex); taosThreadMutexUnlock(&pWalReader->pWal->mutex);
if (pOffset->version < firstVer){ if (pOffset->version < firstVer) {
pOffset->version = firstVer; pOffset->version = firstVer;
} }
} }
static int64_t walReadSeekFilePos(SWalReader *pReader, int64_t fileFirstVer, int64_t ver) { static int32_t walReadSeekFilePos(SWalReader *pReader, int64_t fileFirstVer, int64_t ver) {
int64_t ret = 0; int64_t ret = 0;
TdFilePtr pIdxTFile = pReader->pIdxFile; TdFilePtr pIdxTFile = pReader->pIdxFile;
@ -140,32 +134,34 @@ static int64_t walReadSeekFilePos(SWalReader *pReader, int64_t fileFirstVer, int
int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry); int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry);
ret = taosLSeekFile(pIdxTFile, offset, SEEK_SET); ret = taosLSeekFile(pIdxTFile, offset, SEEK_SET);
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, failed to seek idx file, index:%" PRId64 ", pos:%" PRId64 ", since %s", pReader->pWal->cfg.vgId, wError("vgId:%d, failed to seek idx file, index:%" PRId64 ", pos:%" PRId64 ", since %s", pReader->pWal->cfg.vgId,
ver, offset, terrstr()); ver, offset, terrstr());
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
SWalIdxEntry entry = {0}; SWalIdxEntry entry = {0};
if ((ret = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry))) != sizeof(SWalIdxEntry)) { if ((ret = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry))) != sizeof(SWalIdxEntry)) {
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, failed to read idx file, since %s", pReader->pWal->cfg.vgId, terrstr()); wError("vgId:%d, failed to read idx file, since %s", pReader->pWal->cfg.vgId, terrstr());
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} else { } else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
wError("vgId:%d, read idx file incompletely, read bytes %" PRId64 ", bytes should be %ld", wError("vgId:%d, read idx file incompletely, read bytes %" PRId64 ", bytes should be %ld",
pReader->pWal->cfg.vgId, ret, sizeof(SWalIdxEntry)); pReader->pWal->cfg.vgId, ret, sizeof(SWalIdxEntry));
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
} }
return -1;
} }
ret = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); ret = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET);
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, failed to seek log file, index:%" PRId64 ", pos:%" PRId64 ", since %s", pReader->pWal->cfg.vgId, wError("vgId:%d, failed to seek log file, index:%" PRId64 ", pos:%" PRId64 ", since %s", pReader->pWal->cfg.vgId,
ver, entry.offset, terrstr()); ver, entry.offset, terrstr());
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
return ret;
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
static int32_t walReadChangeFile(SWalReader *pReader, int64_t fileFirstVer) { static int32_t walReadChangeFile(SWalReader *pReader, int64_t fileFirstVer) {
@ -177,9 +173,9 @@ static int32_t walReadChangeFile(SWalReader *pReader, int64_t fileFirstVer) {
walBuildLogName(pReader->pWal, fileFirstVer, fnameStr); walBuildLogName(pReader->pWal, fileFirstVer, fnameStr);
TdFilePtr pLogFile = taosOpenFile(fnameStr, TD_FILE_READ); TdFilePtr pLogFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pLogFile == NULL) { if (pLogFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, cannot open file %s, since %s", pReader->pWal->cfg.vgId, fnameStr, terrstr()); wError("vgId:%d, cannot open file %s, since %s", pReader->pWal->cfg.vgId, fnameStr, terrstr());
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
pReader->pLogFile = pLogFile; pReader->pLogFile = pLogFile;
@ -187,19 +183,19 @@ static int32_t walReadChangeFile(SWalReader *pReader, int64_t fileFirstVer) {
walBuildIdxName(pReader->pWal, fileFirstVer, fnameStr); walBuildIdxName(pReader->pWal, fileFirstVer, fnameStr);
TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ); TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pIdxFile == NULL) { if (pIdxFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, cannot open file %s, since %s", pReader->pWal->cfg.vgId, fnameStr, terrstr()); wError("vgId:%d, cannot open file %s, since %s", pReader->pWal->cfg.vgId, fnameStr, terrstr());
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
pReader->pIdxFile = pIdxFile; pReader->pIdxFile = pIdxFile;
pReader->curFileFirstVer = fileFirstVer; pReader->curFileFirstVer = fileFirstVer;
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) { static int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) {
SWal *pWal = pReader->pWal; SWal *pWal = pReader->pWal;
// bsearch in fileSet // bsearch in fileSet
@ -208,51 +204,45 @@ int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) {
SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
if (pRet == NULL) { if (pRet == NULL) {
wError("failed to find WAL log file with ver:%" PRId64, ver); wError("failed to find WAL log file with ver:%" PRId64, ver);
terrno = TSDB_CODE_WAL_INVALID_VER;
return -1; TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER);
} }
if (pReader->curFileFirstVer != pRet->firstVer) { if (pReader->curFileFirstVer != pRet->firstVer) {
// error code was set inner // error code was set inner
if (walReadChangeFile(pReader, pRet->firstVer) < 0) { TAOS_CHECK_RETURN(walReadChangeFile(pReader, pRet->firstVer));
return -1;
}
} }
// error code was set inner // error code was set inner
if (walReadSeekFilePos(pReader, pRet->firstVer, ver) < 0) { TAOS_CHECK_RETURN(walReadSeekFilePos(pReader, pRet->firstVer, ver));
return -1;
}
wDebug("vgId:%d, wal version reset from %" PRId64 " to %" PRId64, pReader->pWal->cfg.vgId, wDebug("vgId:%d, wal version reset from %" PRId64 " to %" PRId64, pReader->pWal->cfg.vgId, pReader->curVersion, ver);
pReader->curVersion, ver);
pReader->curVersion = ver; pReader->curVersion = ver;
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t walReaderSeekVer(SWalReader *pReader, int64_t ver) { int32_t walReaderSeekVer(SWalReader *pReader, int64_t ver) {
SWal *pWal = pReader->pWal; SWal *pWal = pReader->pWal;
if (ver == pReader->curVersion) { if (ver == pReader->curVersion) {
wDebug("vgId:%d, wal index:%" PRId64 " match, no need to reset", pReader->pWal->cfg.vgId, ver); wDebug("vgId:%d, wal index:%" PRId64 " match, no need to reset", pReader->pWal->cfg.vgId, ver);
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) {
wInfo("vgId:%d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pReader->pWal->cfg.vgId, wInfo("vgId:%d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pReader->pWal->cfg.vgId,
ver, pWal->vers.firstVer, pWal->vers.lastVer); ver, pWal->vers.firstVer, pWal->vers.lastVer);
terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
return -1; TAOS_RETURN(TSDB_CODE_WAL_LOG_NOT_EXIST);
} }
if (walReadSeekVerImpl(pReader, ver) < 0) { TAOS_CHECK_RETURN(walReadSeekVerImpl(pReader, ver));
return -1;
}
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t walFetchHead(SWalReader *pRead, int64_t ver) { int32_t walFetchHead(SWalReader *pRead, int64_t ver) {
int64_t code; int64_t code;
int64_t contLen; int64_t contLen;
@ -260,14 +250,12 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver) {
// TODO: valid ver // TODO: valid ver
if (ver > pRead->pWal->vers.commitVer) { if (ver > pRead->pWal->vers.commitVer) {
return -1; TAOS_RETURN(TSDB_CODE_FAILED);
} }
if (pRead->curVersion != ver) { if (pRead->curVersion != ver) {
code = walReaderSeekVer(pRead, ver); TAOS_CHECK_RETURN(walReaderSeekVer(pRead, ver));
if (code < 0) {
return -1;
}
seeked = true; seeked = true;
} }
@ -276,53 +264,51 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver) {
if (contLen == sizeof(SWalCkHead)) { if (contLen == sizeof(SWalCkHead)) {
break; break;
} else if (contLen == 0 && !seeked) { } else if (contLen == 0 && !seeked) {
if(walReadSeekVerImpl(pRead, ver) < 0){ TAOS_CHECK_RETURN(walReadSeekVerImpl(pRead, ver));
return -1;
}
seeked = true; seeked = true;
continue; continue;
} else { } else {
if (contLen < 0) { if (contLen < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} else { } else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
} }
return -1;
} }
} }
code = walValidHeadCksum(pRead->pHead); code = walValidHeadCksum(pRead->pHead);
if (code != 0) { if (code != 0) {
wError("vgId:%d, unexpected wal log index:%" PRId64 ", since head checksum not passed, 0x%"PRIx64, pRead->pWal->cfg.vgId, ver, wError("vgId:%d, unexpected wal log index:%" PRId64 ", since head checksum not passed, 0x%" PRIx64,
pRead->readerId); pRead->pWal->cfg.vgId, ver, pRead->readerId);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1; TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
} }
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t walSkipFetchBody(SWalReader *pRead) { int32_t walSkipFetchBody(SWalReader *pRead) {
wDebug("vgId:%d, skip:%" PRId64 ", first:%" PRId64 ", commit:%" PRId64 ", last:%" PRId64 wDebug("vgId:%d, skip:%" PRId64 ", first:%" PRId64 ", commit:%" PRId64 ", last:%" PRId64 ", applied:%" PRId64
", applied:%" PRId64 ", 0x%" PRIx64, ", 0x%" PRIx64,
pRead->pWal->cfg.vgId, pRead->pHead->head.version, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer, pRead->pWal->cfg.vgId, pRead->pHead->head.version, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer,
pRead->pWal->vers.lastVer, pRead->pWal->vers.appliedVer, pRead->readerId); pRead->pWal->vers.lastVer, pRead->pWal->vers.appliedVer, pRead->readerId);
int32_t plainBodyLen = pRead->pHead->head.bodyLen; int32_t plainBodyLen = pRead->pHead->head.bodyLen;
int32_t cryptedBodyLen = plainBodyLen; int32_t cryptedBodyLen = plainBodyLen;
//TODO: dmchen emun // TODO: dmchen emun
if(pRead->pWal->cfg.encryptAlgorithm == 1){ if (pRead->pWal->cfg.encryptAlgorithm == 1) {
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen); cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
} }
int64_t code = taosLSeekFile(pRead->pLogFile, cryptedBodyLen, SEEK_CUR); int64_t code = taosLSeekFile(pRead->pLogFile, cryptedBodyLen, SEEK_CUR);
if (code < 0) { if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
return -1;
} }
pRead->curVersion++; pRead->curVersion++;
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t walFetchBody(SWalReader *pRead) { int32_t walFetchBody(SWalReader *pRead) {
@ -339,16 +325,15 @@ int32_t walFetchBody(SWalReader *pRead) {
int32_t plainBodyLen = pReadHead->bodyLen; int32_t plainBodyLen = pReadHead->bodyLen;
int32_t cryptedBodyLen = plainBodyLen; int32_t cryptedBodyLen = plainBodyLen;
//TODO: dmchen emun // TODO: dmchen emun
if(pRead->pWal->cfg.encryptAlgorithm == 1){ if (pRead->pWal->cfg.encryptAlgorithm == 1) {
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen); cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
} }
if (pRead->capacity < cryptedBodyLen) { if (pRead->capacity < cryptedBodyLen) {
SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + cryptedBodyLen); SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + cryptedBodyLen);
if (ptr == NULL) { if (ptr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
return -1;
} }
pRead->pHead = ptr; pRead->pHead = ptr;
pReadHead = &pRead->pHead->head; pReadHead = &pRead->pHead->head;
@ -357,62 +342,68 @@ int32_t walFetchBody(SWalReader *pRead) {
if (cryptedBodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, cryptedBodyLen)) { if (cryptedBodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, cryptedBodyLen)) {
if (plainBodyLen < 0) { if (plainBodyLen < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s, 0x%" PRIx64, vgId,
wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s, 0x%"PRIx64, pReadHead->version, ver, tstrerror(terrno), id);
vgId, pReadHead->version, ver, tstrerror(terrno), id);
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} else { } else {
wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since file corrupted, 0x%"PRIx64, wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64
", since file corrupted, 0x%" PRIx64,
vgId, pReadHead->version, ver, id); vgId, pReadHead->version, ver, id);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
} }
return -1;
} }
if (pReadHead->version != ver) { if (pReadHead->version != ver) {
wError("vgId:%d, wal fetch body error, index:%" PRId64 ", read request index:%" PRId64", 0x%"PRIx64, vgId, wError("vgId:%d, wal fetch body error, index:%" PRId64 ", read request index:%" PRId64 ", 0x%" PRIx64, vgId,
pReadHead->version, ver, id); pReadHead->version, ver, id);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1; TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
} }
decryptBody(&pRead->pWal->cfg, pRead->pHead, plainBodyLen, __FUNCTION__); TAOS_CHECK_RETURN(decryptBody(&pRead->pWal->cfg, pRead->pHead, plainBodyLen, __FUNCTION__));
if (walValidBodyCksum(pRead->pHead) != 0) { if (walValidBodyCksum(pRead->pHead) != 0) {
wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed, 0x%" PRIx64, vgId, ver, id); wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed, 0x%" PRIx64, vgId, ver,
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; id);
return -1;
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
} }
pRead->curVersion++; pRead->curVersion++;
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t walReadVer(SWalReader *pReader, int64_t ver) { int32_t walReadVer(SWalReader *pReader, int64_t ver) {
wDebug("vgId:%d, wal start to read index:%" PRId64, pReader->pWal->cfg.vgId, ver); wDebug("vgId:%d, wal start to read index:%" PRId64, pReader->pWal->cfg.vgId, ver);
int64_t contLen; int64_t contLen;
int32_t code; int32_t code = 0;
bool seeked = false; bool seeked = false;
if (walIsEmpty(pReader->pWal)) { if (walIsEmpty(pReader->pWal)) {
terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; TAOS_RETURN(TSDB_CODE_WAL_LOG_NOT_EXIST);
return -1;
} }
if (ver > pReader->pWal->vers.lastVer || ver < pReader->pWal->vers.firstVer) { if (ver > pReader->pWal->vers.lastVer || ver < pReader->pWal->vers.firstVer) {
wDebug("vgId:%d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pReader->pWal->cfg.vgId, wDebug("vgId:%d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pReader->pWal->cfg.vgId,
ver, pReader->pWal->vers.firstVer, pReader->pWal->vers.lastVer); ver, pReader->pWal->vers.firstVer, pReader->pWal->vers.lastVer);
terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
return -1; TAOS_RETURN(TSDB_CODE_WAL_LOG_NOT_EXIST);
} }
taosThreadMutexLock(&pReader->mutex); taosThreadMutexLock(&pReader->mutex);
if (pReader->curVersion != ver) { if (pReader->curVersion != ver) {
if (walReaderSeekVer(pReader, ver) < 0) { code = walReaderSeekVer(pReader, ver);
if (code) {
wError("vgId:%d, unexpected wal log, index:%" PRId64 ", since %s", pReader->pWal->cfg.vgId, ver, terrstr()); wError("vgId:%d, unexpected wal log, index:%" PRId64 ", since %s", pReader->pWal->cfg.vgId, ver, terrstr());
taosThreadMutexUnlock(&pReader->mutex); taosThreadMutexUnlock(&pReader->mutex);
return -1;
TAOS_RETURN(code);
} }
seeked = true; seeked = true;
} }
@ -421,22 +412,24 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
if (contLen == sizeof(SWalCkHead)) { if (contLen == sizeof(SWalCkHead)) {
break; break;
} else if (contLen == 0 && !seeked) { } else if (contLen == 0 && !seeked) {
if(walReadSeekVerImpl(pReader, ver) < 0){ code = walReadSeekVerImpl(pReader, ver);
if (code) {
taosThreadMutexUnlock(&pReader->mutex); taosThreadMutexUnlock(&pReader->mutex);
return -1;
TAOS_RETURN(code);
} }
seeked = true; seeked = true;
continue; continue;
} else { } else {
if (contLen < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
} else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
}
wError("vgId:%d, failed to read WAL record head, index:%" PRId64 ", from log file since %s", wError("vgId:%d, failed to read WAL record head, index:%" PRId64 ", from log file since %s",
pReader->pWal->cfg.vgId, ver, terrstr()); pReader->pWal->cfg.vgId, ver, terrstr());
taosThreadMutexUnlock(&pReader->mutex); taosThreadMutexUnlock(&pReader->mutex);
return -1;
if (contLen < 0) {
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} else {
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
}
} }
} }
@ -444,54 +437,57 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
if (code != 0) { if (code != 0) {
wError("vgId:%d, unexpected wal log, index:%" PRId64 ", since head checksum not passed", pReader->pWal->cfg.vgId, wError("vgId:%d, unexpected wal log, index:%" PRId64 ", since head checksum not passed", pReader->pWal->cfg.vgId,
ver); ver);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
taosThreadMutexUnlock(&pReader->mutex); taosThreadMutexUnlock(&pReader->mutex);
return -1;
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
} }
int32_t plainBodyLen = pReader->pHead->head.bodyLen; int32_t plainBodyLen = pReader->pHead->head.bodyLen;
int32_t cryptedBodyLen = plainBodyLen; int32_t cryptedBodyLen = plainBodyLen;
//TODO: dmchen emun // TODO: dmchen emun
if(pReader->pWal->cfg.encryptAlgorithm == 1){ if (pReader->pWal->cfg.encryptAlgorithm == 1) {
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen); cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
} }
if (pReader->capacity < cryptedBodyLen) { if (pReader->capacity < cryptedBodyLen) {
SWalCkHead *ptr = SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + cryptedBodyLen);
(SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + cryptedBodyLen);
if (ptr == NULL) { if (ptr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosThreadMutexUnlock(&pReader->mutex); taosThreadMutexUnlock(&pReader->mutex);
return -1;
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
} }
pReader->pHead = ptr; pReader->pHead = ptr;
pReader->capacity = cryptedBodyLen; pReader->capacity = cryptedBodyLen;
} }
if ((contLen = taosReadFile(pReader->pLogFile, pReader->pHead->head.body, cryptedBodyLen)) != if ((contLen = taosReadFile(pReader->pLogFile, pReader->pHead->head.body, cryptedBodyLen)) != cryptedBodyLen) {
cryptedBodyLen) {
if (contLen < 0)
terrno = TAOS_SYSTEM_ERROR(errno);
else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
}
wError("vgId:%d, failed to read WAL record body, index:%" PRId64 ", from log file since %s", wError("vgId:%d, failed to read WAL record body, index:%" PRId64 ", from log file since %s",
pReader->pWal->cfg.vgId, ver, terrstr()); pReader->pWal->cfg.vgId, ver, terrstr());
taosThreadMutexUnlock(&pReader->mutex); taosThreadMutexUnlock(&pReader->mutex);
return -1;
if (contLen < 0) {
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} else {
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
}
} }
if (pReader->pHead->head.version != ver) { if (pReader->pHead->head.version != ver) {
wError("vgId:%d, unexpected wal log, index:%" PRId64 ", read request index:%" PRId64, pReader->pWal->cfg.vgId, wError("vgId:%d, unexpected wal log, index:%" PRId64 ", read request index:%" PRId64, pReader->pWal->cfg.vgId,
pReader->pHead->head.version, ver); pReader->pHead->head.version, ver);
// pReader->curInvalid = 1; // pReader->curInvalid = 1;
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
taosThreadMutexUnlock(&pReader->mutex); taosThreadMutexUnlock(&pReader->mutex);
return -1;
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
} }
decryptBody(&pReader->pWal->cfg, pReader->pHead, plainBodyLen, __FUNCTION__); code = decryptBody(&pReader->pWal->cfg, pReader->pHead, plainBodyLen, __FUNCTION__);
if (code) {
taosThreadMutexUnlock(&pReader->mutex);
TAOS_RETURN(code);
}
code = walValidBodyCksum(pReader->pHead); code = walValidBodyCksum(pReader->pHead);
if (code != 0) { if (code != 0) {
@ -500,23 +496,27 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
uint32_t readCkSum = walCalcBodyCksum(pReader->pHead->head.body, plainBodyLen); uint32_t readCkSum = walCalcBodyCksum(pReader->pHead->head.body, plainBodyLen);
uint32_t logCkSum = pReader->pHead->cksumBody; uint32_t logCkSum = pReader->pHead->cksumBody;
wError("checksum written into log:%u, checksum calculated:%u", logCkSum, readCkSum); wError("checksum written into log:%u, checksum calculated:%u", logCkSum, readCkSum);
// pReader->curInvalid = 1; // pReader->curInvalid = 1;
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
taosThreadMutexUnlock(&pReader->mutex); taosThreadMutexUnlock(&pReader->mutex);
return -1;
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
} }
pReader->curVersion++; pReader->curVersion++;
taosThreadMutexUnlock(&pReader->mutex); taosThreadMutexUnlock(&pReader->mutex);
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
void decryptBody(SWalCfg* cfg, SWalCkHead* pHead, int32_t plainBodyLen, const char* func) { int32_t decryptBody(SWalCfg *cfg, SWalCkHead *pHead, int32_t plainBodyLen, const char *func) {
//TODO: dmchen emun // TODO: dmchen emun
if (cfg->encryptAlgorithm == 1) { if (cfg->encryptAlgorithm == 1) {
int32_t cryptedBodyLen = ENCRYPTED_LEN(plainBodyLen); int32_t cryptedBodyLen = ENCRYPTED_LEN(plainBodyLen);
char *newBody = taosMemoryMalloc(cryptedBodyLen); char *newBody = taosMemoryMalloc(cryptedBodyLen);
if (!newBody) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
SCryptOpts opts; SCryptOpts opts;
opts.len = cryptedBodyLen; opts.len = cryptedBodyLen;
@ -527,12 +527,14 @@ void decryptBody(SWalCfg* cfg, SWalCkHead* pHead, int32_t plainBodyLen, const ch
int32_t count = CBC_Decrypt(&opts); int32_t count = CBC_Decrypt(&opts);
//wDebug("CBC_Decrypt cryptedBodyLen:%d, plainBodyLen:%d, %s", count, plainBodyLen, func); // wDebug("CBC_Decrypt cryptedBodyLen:%d, plainBodyLen:%d, %s", count, plainBodyLen, func);
memcpy(pHead->head.body, newBody, plainBodyLen); memcpy(pHead->head.body, newBody, plainBodyLen);
taosMemoryFree(newBody); taosMemoryFree(newBody);
} }
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
void walReadReset(SWalReader *pReader) { void walReadReset(SWalReader *pReader) {

39
source/libs/wal/src/walRef.c
View File

@ -22,27 +22,39 @@
SWalRef *walOpenRef(SWal *pWal) { SWalRef *walOpenRef(SWal *pWal) {
SWalRef *pRef = taosMemoryCalloc(1, sizeof(SWalRef)); SWalRef *pRef = taosMemoryCalloc(1, sizeof(SWalRef));
if (pRef == NULL) { if (pRef == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
pRef->refId = tGenIdPI64(); pRef->refId = tGenIdPI64();
if (taosHashPut(pWal->pRefHash, &pRef->refId, sizeof(int64_t), &pRef, sizeof(void *))) {
taosMemoryFree(pRef);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pRef->refVer = -1; pRef->refVer = -1;
// pRef->refFile = -1;
pRef->pWal = pWal; pRef->pWal = pWal;
taosHashPut(pWal->pRefHash, &pRef->refId, sizeof(int64_t), &pRef, sizeof(void *));
return pRef; return pRef;
} }
void walCloseRef(SWal *pWal, int64_t refId) { void walCloseRef(SWal *pWal, int64_t refId) {
SWalRef **ppRef = taosHashGet(pWal->pRefHash, &refId, sizeof(int64_t)); SWalRef **ppRef = taosHashGet(pWal->pRefHash, &refId, sizeof(int64_t));
if (ppRef == NULL) return; if (ppRef) {
SWalRef *pRef = *ppRef; SWalRef *pRef = *ppRef;
if (pRef) { if (pRef) {
wDebug("vgId:%d, wal close ref %" PRId64 ", refId %" PRId64, pWal->cfg.vgId, pRef->refVer, pRef->refId); wDebug("vgId:%d, wal close ref %" PRId64 ", refId %" PRId64, pWal->cfg.vgId, pRef->refVer, pRef->refId);
taosMemoryFree(pRef);
} else { } else {
wDebug("vgId:%d, wal close ref null, refId %" PRId64, pWal->cfg.vgId, refId); wDebug("vgId:%d, wal close ref null, refId %" PRId64, pWal->cfg.vgId, refId);
} }
taosHashRemove(pWal->pRefHash, &refId, sizeof(int64_t));
taosMemoryFree(pRef); (void)taosHashRemove(pWal->pRefHash, &refId, sizeof(int64_t));
}
} }
int32_t walSetRefVer(SWalRef *pRef, int64_t ver) { int32_t walSetRefVer(SWalRef *pRef, int64_t ver) {
@ -52,30 +64,31 @@ int32_t walSetRefVer(SWalRef *pRef, int64_t ver) {
taosThreadMutexLock(&pWal->mutex); taosThreadMutexLock(&pWal->mutex);
if (ver < pWal->vers.firstVer || ver > pWal->vers.lastVer) { if (ver < pWal->vers.firstVer || ver > pWal->vers.lastVer) {
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return TSDB_CODE_WAL_INVALID_VER;
TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER);
} }
pRef->refVer = ver; pRef->refVer = ver;
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
} }
return TSDB_CODE_SUCCESS; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
void walRefFirstVer(SWal *pWal, SWalRef *pRef) { void walRefFirstVer(SWal *pWal, SWalRef *pRef) {
taosThreadMutexLock(&pWal->mutex); taosThreadMutexLock(&pWal->mutex);
int64_t ver = walGetFirstVer(pWal);
pRef->refVer = ver; pRef->refVer = pWal->vers.firstVer;
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
wDebug("vgId:%d, wal ref version %" PRId64 " for first", pWal->cfg.vgId, ver); wDebug("vgId:%d, wal ref version %" PRId64 " for first", pWal->cfg.vgId, pRef->refVer);
} }
void walRefLastVer(SWal *pWal, SWalRef *pRef) { void walRefLastVer(SWal *pWal, SWalRef *pRef) {
taosThreadMutexLock(&pWal->mutex); taosThreadMutexLock(&pWal->mutex);
int64_t ver = walGetLastVer(pWal);
pRef->refVer = ver; pRef->refVer = pWal->vers.lastVer;
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
wDebug("vgId:%d, wal ref version %" PRId64 " for last", pWal->cfg.vgId, ver); wDebug("vgId:%d, wal ref version %" PRId64 " for last", pWal->cfg.vgId, pRef->refVer);
} }

159
source/libs/wal/src/walSeek.c
View File

@ -1,159 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taoserror.h"
#include "tref.h"
#include "walInt.h"
#if 0
static int64_t walSeekWritePos(SWal* pWal, int64_t ver) {
int64_t code = 0;
TdFilePtr pIdxTFile = pWal->pIdxFile;
TdFilePtr pLogTFile = pWal->pLogFile;
// seek position
int64_t idxOff = walGetVerIdxOffset(pWal, ver);
code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
SWalIdxEntry entry;
// TODO:deserialize
code = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry));
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
return 0;
}
#endif
int walInitWriteFile(SWal* pWal) {
TdFilePtr pIdxTFile, pLogTFile;
SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet);
int64_t fileFirstVer = pRet->firstVer;
char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, fileFirstVer, fnameStr);
pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxTFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
walBuildLogName(pWal, fileFirstVer, fnameStr);
pLogTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pLogTFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
// switch file
pWal->pIdxFile = pIdxTFile;
pWal->pLogFile = pLogTFile;
pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
return 0;
}
int64_t walChangeWrite(SWal* pWal, int64_t ver) {
int code;
TdFilePtr pIdxTFile, pLogTFile;
char fnameStr[WAL_FILE_LEN];
if (pWal->pLogFile != NULL) {
if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pLogFile)) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
code = taosCloseFile(&pWal->pLogFile);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
}
if (pWal->pIdxFile != NULL) {
if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pIdxFile)) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
code = taosCloseFile(&pWal->pIdxFile);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
}
SWalFileInfo tmpInfo;
tmpInfo.firstVer = ver;
// bsearch in fileSet
int32_t idx = taosArraySearchIdx(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
/*A(idx != -1);*/
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, idx);
int64_t fileFirstVer = pFileInfo->firstVer;
walBuildIdxName(pWal, fileFirstVer, fnameStr);
pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxTFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
pWal->pIdxFile = NULL;
return -1;
}
walBuildLogName(pWal, fileFirstVer, fnameStr);
pLogTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pLogTFile == NULL) {
taosCloseFile(&pIdxTFile);
terrno = TAOS_SYSTEM_ERROR(errno);
pWal->pLogFile = NULL;
return -1;
}
pWal->pLogFile = pLogTFile;
pWal->pIdxFile = pIdxTFile;
pWal->writeCur = idx;
return fileFirstVer;
}
#if 0
int walSeekWriteVer(SWal* pWal, int64_t ver) {
int64_t code;
if (ver == pWal->vers.lastVer) {
return 0;
}
if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) {
terrno = TSDB_CODE_WAL_INVALID_VER;
return -1;
}
if (ver < pWal->vers.snapshotVer) {
}
if (ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) {
code = walChangeWrite(pWal, ver);
if (code != 0) {
return -1;
}
}
code = walSeekWritePos(pWal, ver);
if (code != 0) {
return -1;
}
return 0;
}
#endif

453
source/libs/wal/src/walWrite.c
View File

@ -21,6 +21,8 @@
#include "walInt.h" #include "walInt.h"
int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) {
int32_t code = 0;
taosThreadMutexLock(&pWal->mutex); taosThreadMutexLock(&pWal->mutex);
wInfo("vgId:%d, restore from snapshot, version %" PRId64, pWal->cfg.vgId, ver); wInfo("vgId:%d, restore from snapshot, version %" PRId64, pWal->cfg.vgId, ver);
@ -33,7 +35,8 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) {
if (pRef->refVer != -1 && pRef->refVer <= ver) { if (pRef->refVer != -1 && pRef->refVer <= ver) {
taosHashCancelIterate(pWal->pRefHash, pIter); taosHashCancelIterate(pWal->pRefHash, pIter);
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TSDB_CODE_FAILED);
} }
} }
@ -47,24 +50,25 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) {
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
if (taosRemoveFile(fnameStr) < 0) { if (taosRemoveFile(fnameStr) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, restore from snapshot, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); wError("vgId:%d, restore from snapshot, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr());
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
wInfo("vgId:%d, restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr); wInfo("vgId:%d, restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr);
walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
if (taosRemoveFile(fnameStr) < 0) { if (taosRemoveFile(fnameStr) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); wError("vgId:%d, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr());
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
wInfo("vgId:%d, restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr); wInfo("vgId:%d, restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr);
} }
} }
(void)walRemoveMeta(pWal);
TAOS_CHECK_RETURN(walRemoveMeta(pWal));
pWal->writeCur = -1; pWal->writeCur = -1;
pWal->totSize = 0; pWal->totSize = 0;
@ -78,25 +82,81 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) {
pWal->vers.verInSnapshotting = -1; pWal->vers.verInSnapshotting = -1;
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t walApplyVer(SWal *pWal, int64_t ver) { int32_t walApplyVer(SWal *pWal, int64_t ver) {
// TODO: error check // TODO: error check
pWal->vers.appliedVer = ver; pWal->vers.appliedVer = ver;
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t walCommit(SWal *pWal, int64_t ver) { int32_t walCommit(SWal *pWal, int64_t ver) {
if (ver < pWal->vers.commitVer) { if (ver < pWal->vers.commitVer) {
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
if (ver > pWal->vers.lastVer || pWal->vers.commitVer < pWal->vers.snapshotVer) { if (ver > pWal->vers.lastVer || pWal->vers.commitVer < pWal->vers.snapshotVer) {
terrno = TSDB_CODE_WAL_INVALID_VER; TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER);
return -1;
} }
pWal->vers.commitVer = ver; pWal->vers.commitVer = ver;
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
static int64_t walChangeWrite(SWal *pWal, int64_t ver) {
int code;
TdFilePtr pIdxTFile, pLogTFile;
char fnameStr[WAL_FILE_LEN];
if (pWal->pLogFile != NULL) {
if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pLogFile)) != 0) {
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
code = taosCloseFile(&pWal->pLogFile);
if (code != 0) {
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
}
if (pWal->pIdxFile != NULL) {
if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pIdxFile)) != 0) {
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
code = taosCloseFile(&pWal->pIdxFile);
if (code != 0) {
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
}
SWalFileInfo tmpInfo;
tmpInfo.firstVer = ver;
// bsearch in fileSet
int32_t idx = taosArraySearchIdx(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
/*A(idx != -1);*/
SWalFileInfo *pFileInfo = taosArrayGet(pWal->fileInfoSet, idx);
int64_t fileFirstVer = pFileInfo->firstVer;
walBuildIdxName(pWal, fileFirstVer, fnameStr);
pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxTFile == NULL) {
pWal->pIdxFile = NULL;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
walBuildLogName(pWal, fileFirstVer, fnameStr);
pLogTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pLogTFile == NULL) {
taosCloseFile(&pIdxTFile);
pWal->pLogFile = NULL;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
pWal->pLogFile = pLogTFile;
pWal->pIdxFile = pIdxTFile;
pWal->writeCur = idx;
return fileFirstVer;
} }
int32_t walRollback(SWal *pWal, int64_t ver) { int32_t walRollback(SWal *pWal, int64_t ver) {
@ -105,9 +165,9 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
int64_t code; int64_t code;
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
if (ver > pWal->vers.lastVer || ver <= pWal->vers.commitVer || ver <= pWal->vers.snapshotVer) { if (ver > pWal->vers.lastVer || ver <= pWal->vers.commitVer || ver <= pWal->vers.snapshotVer) {
terrno = TSDB_CODE_WAL_INVALID_VER;
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER);
} }
// find correct file // find correct file
@ -116,7 +176,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
code = walChangeWrite(pWal, ver); code = walChangeWrite(pWal, ver);
if (code < 0) { if (code < 0) {
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(code);
} }
// delete files in descending order // delete files in descending order
@ -136,22 +197,24 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
taosCloseFile(&pWal->pIdxFile); taosCloseFile(&pWal->pIdxFile);
TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND); TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND);
if (pIdxFile == NULL) { if (pIdxFile == NULL) {
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
int64_t idxOff = walGetVerIdxOffset(pWal, ver); int64_t idxOff = walGetVerIdxOffset(pWal, ver);
code = taosLSeekFile(pIdxFile, idxOff, SEEK_SET); code = taosLSeekFile(pIdxFile, idxOff, SEEK_SET);
if (code < 0) { if (code < 0) {
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
// read idx file and get log file pos // read idx file and get log file pos
SWalIdxEntry entry; SWalIdxEntry entry;
if (taosReadFile(pIdxFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { if (taosReadFile(pIdxFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr); walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
@ -160,49 +223,50 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
wDebug("vgId:%d, wal truncate file %s", pWal->cfg.vgId, fnameStr); wDebug("vgId:%d, wal truncate file %s", pWal->cfg.vgId, fnameStr);
if (pLogFile == NULL) { if (pLogFile == NULL) {
// TODO // TODO
terrno = TAOS_SYSTEM_ERROR(errno);
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
code = taosLSeekFile(pLogFile, entry.offset, SEEK_SET); code = taosLSeekFile(pLogFile, entry.offset, SEEK_SET);
if (code < 0) { if (code < 0) {
// TODO // TODO
terrno = TAOS_SYSTEM_ERROR(errno);
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
// validate offset // validate offset
SWalCkHead head; SWalCkHead head;
int64_t size = taosReadFile(pLogFile, &head, sizeof(SWalCkHead)); int64_t size = taosReadFile(pLogFile, &head, sizeof(SWalCkHead));
if (size != sizeof(SWalCkHead)) { if (size != sizeof(SWalCkHead)) {
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
code = walValidHeadCksum(&head); code = walValidHeadCksum(&head);
if (code != 0) { if (code != 0) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
} }
if (head.head.version != ver) { if (head.head.version != ver) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
} }
// truncate old files // truncate old files
code = taosFtruncateFile(pLogFile, entry.offset); code = taosFtruncateFile(pLogFile, entry.offset);
if (code < 0) { if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
code = taosFtruncateFile(pIdxFile, idxOff); code = taosFtruncateFile(pIdxFile, idxOff);
if (code < 0) { if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
pWal->vers.lastVer = ver - 1; pWal->vers.lastVer = ver - 1;
((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1;
@ -215,43 +279,98 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
if (code < 0) { if (code < 0) {
wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr()); wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr());
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(code);
} }
// unlock // unlock
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
static int32_t walRollImpl(SWal *pWal) {
int32_t code = 0, lino = 0;
if (pWal->pIdxFile != NULL) {
if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pIdxFile)) != 0) {
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
code = taosCloseFile(&pWal->pIdxFile);
if (code != 0) {
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
}
if (pWal->pLogFile != NULL) {
if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pLogFile)) != 0) {
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
code = taosCloseFile(&pWal->pLogFile);
if (code != 0) {
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
}
TdFilePtr pIdxFile, pLogFile;
// create new file
int64_t newFileFirstVer = pWal->vers.lastVer + 1;
char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, newFileFirstVer, fnameStr);
pIdxFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxFile == NULL) {
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
walBuildLogName(pWal, newFileFirstVer, fnameStr);
pLogFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
wDebug("vgId:%d, wal create new file for write:%s", pWal->cfg.vgId, fnameStr);
if (pLogFile == NULL) {
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
TAOS_CHECK_GOTO(walRollFileInfo(pWal), &lino, _exit);
// switch file
pWal->pIdxFile = pIdxFile;
pWal->pLogFile = pLogFile;
pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
pWal->lastRollSeq = walGetSeq();
TAOS_CHECK_GOTO(walSaveMeta(pWal), &lino, _exit);
_exit:
if (code) {
wError("vgId:%d, %s failed at line %d since %s", pWal->cfg.vgId, __func__, lino, tstrerror(code));
}
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
static FORCE_INLINE int32_t walCheckAndRoll(SWal *pWal) { static FORCE_INLINE int32_t walCheckAndRoll(SWal *pWal) {
if (taosArrayGetSize(pWal->fileInfoSet) == 0) { if (taosArrayGetSize(pWal->fileInfoSet) == 0) {
if (walRollImpl(pWal) < 0) { TAOS_CHECK_RETURN(walRollImpl(pWal));
return -1;
} TAOS_RETURN(TSDB_CODE_SUCCESS);
return 0;
} }
int64_t passed = walGetSeq() - pWal->lastRollSeq; int64_t passed = walGetSeq() - pWal->lastRollSeq;
if (pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) { if (pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) {
if (walRollImpl(pWal) < 0) { TAOS_CHECK_RETURN(walRollImpl(pWal));
return -1;
}
} else if (pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) { } else if (pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) {
if (walRollImpl(pWal) < 0) { TAOS_CHECK_RETURN(walRollImpl(pWal));
return -1;
}
} }
if (walGetLastFileCachedSize(pWal) > tsWalFsyncDataSizeLimit) { if (walGetLastFileCachedSize(pWal) > tsWalFsyncDataSizeLimit) {
if (walSaveMeta(pWal) < 0) { TAOS_CHECK_RETURN(walSaveMeta(pWal));
return -1;
}
} }
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t walBeginSnapshot(SWal *pWal, int64_t ver, int64_t logRetention) { int32_t walBeginSnapshot(SWal *pWal, int64_t ver, int64_t logRetention) {
int32_t code = 0;
taosThreadMutexLock(&pWal->mutex); taosThreadMutexLock(&pWal->mutex);
ASSERT(logRetention >= 0); ASSERT(logRetention >= 0);
pWal->vers.verInSnapshotting = ver; pWal->vers.verInSnapshotting = ver;
@ -262,22 +381,21 @@ int32_t walBeginSnapshot(SWal *pWal, int64_t ver, int64_t logRetention) {
pWal->cfg.vgId, ver, pWal->vers.logRetention, pWal->vers.firstVer, pWal->vers.lastVer); pWal->cfg.vgId, ver, pWal->vers.logRetention, pWal->vers.firstVer, pWal->vers.lastVer);
// check file rolling // check file rolling
if (walGetLastFileSize(pWal) != 0) { if (walGetLastFileSize(pWal) != 0) {
if (walRollImpl(pWal) < 0) { if ((code = walRollImpl(pWal)) < 0) {
wError("vgId:%d, failed to roll wal files since %s", pWal->cfg.vgId, terrstr()); wError("vgId:%d, failed to roll wal files since %s", pWal->cfg.vgId, terrstr());
goto _err; goto _exit;
} }
} }
_exit:
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return 0;
_err: TAOS_RETURN(code);
taosThreadMutexUnlock(&pWal->mutex);
return -1;
} }
int32_t walEndSnapshot(SWal *pWal) { int32_t walEndSnapshot(SWal *pWal) {
int32_t code = 0; int32_t code = 0, lino = 0;
taosThreadMutexLock(&pWal->mutex); taosThreadMutexLock(&pWal->mutex);
int64_t ver = pWal->vers.verInSnapshotting; int64_t ver = pWal->vers.verInSnapshotting;
@ -286,8 +404,7 @@ int32_t walEndSnapshot(SWal *pWal) {
pWal->cfg.vgId, ver, pWal->vers.logRetention, pWal->vers.firstVer, pWal->vers.lastVer); pWal->cfg.vgId, ver, pWal->vers.logRetention, pWal->vers.firstVer, pWal->vers.lastVer);
if (ver == -1) { if (ver == -1) {
code = -1; TAOS_CHECK_GOTO(TSDB_CODE_FAILED, &lino, _exit);
goto END;
} }
pWal->vers.snapshotVer = ver; pWal->vers.snapshotVer = ver;
@ -357,10 +474,7 @@ int32_t walEndSnapshot(SWal *pWal) {
pWal->totSize = newTotSize; pWal->totSize = newTotSize;
pWal->vers.verInSnapshotting = -1; pWal->vers.verInSnapshotting = -1;
code = walSaveMeta(pWal); TAOS_CHECK_GOTO(walSaveMeta(pWal), &lino, _exit);
if (code < 0) {
goto END;
}
// delete files // delete files
deleteCnt = taosArrayGetSize(pWal->toDeleteFiles); deleteCnt = taosArrayGetSize(pWal->toDeleteFiles);
@ -373,12 +487,12 @@ int32_t walEndSnapshot(SWal *pWal) {
walBuildLogName(pWal, pInfo->firstVer, fnameStr); walBuildLogName(pWal, pInfo->firstVer, fnameStr);
if (taosRemoveFile(fnameStr) < 0 && errno != ENOENT) { if (taosRemoveFile(fnameStr) < 0 && errno != ENOENT) {
wError("vgId:%d, failed to remove log file %s due to %s", pWal->cfg.vgId, fnameStr, strerror(errno)); wError("vgId:%d, failed to remove log file %s due to %s", pWal->cfg.vgId, fnameStr, strerror(errno));
goto END; goto _exit;
} }
walBuildIdxName(pWal, pInfo->firstVer, fnameStr); walBuildIdxName(pWal, pInfo->firstVer, fnameStr);
if (taosRemoveFile(fnameStr) < 0 && errno != ENOENT) { if (taosRemoveFile(fnameStr) < 0 && errno != ENOENT) {
wError("vgId:%d, failed to remove idx file %s due to %s", pWal->cfg.vgId, fnameStr, strerror(errno)); wError("vgId:%d, failed to remove idx file %s due to %s", pWal->cfg.vgId, fnameStr, strerror(errno));
goto END; goto _exit;
} }
} }
if (pInfo != NULL) { if (pInfo != NULL) {
@ -387,81 +501,19 @@ int32_t walEndSnapshot(SWal *pWal) {
} }
taosArrayClear(pWal->toDeleteFiles); taosArrayClear(pWal->toDeleteFiles);
END: _exit:
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return code;
}
int32_t walRollImpl(SWal *pWal) { if (code) {
int32_t code = 0; wError("vgId:%d, %s failed at line %d since %s", pWal->cfg.vgId, __func__, lino, tstrerror(code));
if (pWal->pIdxFile != NULL) {
if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pIdxFile)) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
goto END;
}
code = taosCloseFile(&pWal->pIdxFile);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
goto END;
}
} }
if (pWal->pLogFile != NULL) {
if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pLogFile)) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
goto END;
}
code = taosCloseFile(&pWal->pLogFile);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
goto END;
}
}
TdFilePtr pIdxFile, pLogFile;
// create new file
int64_t newFileFirstVer = pWal->vers.lastVer + 1;
char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, newFileFirstVer, fnameStr);
pIdxFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
code = -1;
goto END;
}
walBuildLogName(pWal, newFileFirstVer, fnameStr);
pLogFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
wDebug("vgId:%d, wal create new file for write:%s", pWal->cfg.vgId, fnameStr);
if (pLogFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
code = -1;
goto END;
}
// error code was set inner
code = walRollFileInfo(pWal);
if (code != 0) {
goto END;
}
// switch file
pWal->pIdxFile = pIdxFile;
pWal->pLogFile = pLogFile;
pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
pWal->lastRollSeq = walGetSeq();
code = walSaveMeta(pWal);
if (code < 0) {
wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr());
goto END;
}
END:
return code; return code;
} }
static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
int32_t code = 0;
SWalIdxEntry entry = {.ver = ver, .offset = offset}; SWalIdxEntry entry = {.ver = ver, .offset = offset};
SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal); SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal);
@ -472,8 +524,8 @@ static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
int64_t size = taosWriteFile(pWal->pIdxFile, &entry, sizeof(SWalIdxEntry)); int64_t size = taosWriteFile(pWal->pIdxFile, &entry, sizeof(SWalIdxEntry));
if (size != sizeof(SWalIdxEntry)) { if (size != sizeof(SWalIdxEntry)) {
wError("vgId:%d, failed to write idx entry due to %s. ver:%" PRId64, pWal->cfg.vgId, strerror(errno), ver); wError("vgId:%d, failed to write idx entry due to %s. ver:%" PRId64, pWal->cfg.vgId, strerror(errno), ver);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
// check alignment of idx entries // check alignment of idx entries
@ -484,12 +536,13 @@ static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
taosMsleep(100); taosMsleep(100);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta,
const void *body, int32_t bodyLen) { const void *body, int32_t bodyLen) {
int64_t code = 0; int32_t code = 0, lino = 0;
int32_t plainBodyLen = bodyLen; int32_t plainBodyLen = bodyLen;
int64_t offset = walGetCurFileOffset(pWal); int64_t offset = walGetCurFileOffset(pWal);
@ -509,19 +562,16 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
TMSG_INFO(msgType), pWal->writeHead.cksumHead, pWal->writeHead.cksumBody); TMSG_INFO(msgType), pWal->writeHead.cksumHead, pWal->writeHead.cksumBody);
if (pWal->cfg.level != TAOS_WAL_SKIP) { if (pWal->cfg.level != TAOS_WAL_SKIP) {
code = walWriteIndex(pWal, index, offset); TAOS_CHECK_GOTO(walWriteIndex(pWal, index, offset), &lino, _exit);
if (code < 0) {
goto END;
}
} }
if (pWal->cfg.level != TAOS_WAL_SKIP && if (pWal->cfg.level != TAOS_WAL_SKIP &&
taosWriteFile(pWal->pLogFile, &pWal->writeHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { taosWriteFile(pWal->pLogFile, &pWal->writeHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno)); strerror(errno));
code = -1;
goto END; TAOS_CHECK_GOTO(code, &lino, _exit);
} }
int32_t cyptedBodyLen = plainBodyLen; int32_t cyptedBodyLen = plainBodyLen;
@ -536,8 +586,8 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
if (newBody == NULL) { if (newBody == NULL) {
wError("vgId:%d, file:%" PRId64 ".log, failed to malloc since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), wError("vgId:%d, file:%" PRId64 ".log, failed to malloc since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno)); strerror(errno));
code = -1;
goto END; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
} }
memset(newBody, 0, cyptedBodyLen); memset(newBody, 0, cyptedBodyLen);
memcpy(newBody, body, plainBodyLen); memcpy(newBody, body, plainBodyLen);
@ -546,9 +596,10 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
if (newBodyEncrypted == NULL) { if (newBodyEncrypted == NULL) {
wError("vgId:%d, file:%" PRId64 ".log, failed to malloc since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), wError("vgId:%d, file:%" PRId64 ".log, failed to malloc since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno)); strerror(errno));
code = -1;
if (newBody != NULL) taosMemoryFreeClear(newBody); if (newBody != NULL) taosMemoryFreeClear(newBody);
goto END;
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
} }
SCryptOpts opts; SCryptOpts opts;
@ -567,15 +618,16 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
} }
if (pWal->cfg.level != TAOS_WAL_SKIP && taosWriteFile(pWal->pLogFile, (char *)buf, cyptedBodyLen) != cyptedBodyLen) { if (pWal->cfg.level != TAOS_WAL_SKIP && taosWriteFile(pWal->pLogFile, (char *)buf, cyptedBodyLen) != cyptedBodyLen) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno)); strerror(errno));
code = -1;
if (pWal->cfg.encryptAlgorithm == DND_CA_SM4) { if (pWal->cfg.encryptAlgorithm == DND_CA_SM4) {
taosMemoryFreeClear(newBody); taosMemoryFreeClear(newBody);
taosMemoryFreeClear(newBodyEncrypted); taosMemoryFreeClear(newBodyEncrypted);
} }
goto END;
TAOS_CHECK_GOTO(code, &lino, _exit);
} }
if (pWal->cfg.encryptAlgorithm == DND_CA_SM4) { if (pWal->cfg.encryptAlgorithm == DND_CA_SM4) {
@ -596,7 +648,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
return 0; return 0;
END: _exit:
// recover in a reverse order // recover in a reverse order
if (taosFtruncateFile(pWal->pLogFile, offset) < 0) { if (taosFtruncateFile(pWal->pLogFile, offset) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
@ -614,88 +666,66 @@ END:
taosMsleep(100); taosMsleep(100);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
return -1;
TAOS_RETURN(TSDB_CODE_FAILED);
} }
int64_t walAppendLog(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, static int32_t walInitWriteFile(SWal *pWal) {
TdFilePtr pIdxTFile, pLogTFile;
SWalFileInfo *pRet = taosArrayGetLast(pWal->fileInfoSet);
int64_t fileFirstVer = pRet->firstVer;
char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, fileFirstVer, fnameStr);
pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxTFile == NULL) {
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
walBuildLogName(pWal, fileFirstVer, fnameStr);
pLogTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pLogTFile == NULL) {
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
// switch file
pWal->pIdxFile = pIdxTFile;
pWal->pLogFile = pLogTFile;
pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t walAppendLog(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body,
int32_t bodyLen) { int32_t bodyLen) {
int32_t code = 0, lino = 0;
taosThreadMutexLock(&pWal->mutex); taosThreadMutexLock(&pWal->mutex);
if (index != pWal->vers.lastVer + 1) { if (index != pWal->vers.lastVer + 1) {
terrno = TSDB_CODE_WAL_INVALID_VER; TAOS_CHECK_GOTO(TSDB_CODE_WAL_INVALID_VER, &lino, _exit);
taosThreadMutexUnlock(&pWal->mutex);
return -1;
} }
if (walCheckAndRoll(pWal) < 0) { TAOS_CHECK_GOTO(walCheckAndRoll(pWal), &lino, _exit);
taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
if (pWal->pLogFile == NULL || pWal->pIdxFile == NULL || pWal->writeCur < 0) { if (pWal->pLogFile == NULL || pWal->pIdxFile == NULL || pWal->writeCur < 0) {
if (walInitWriteFile(pWal) < 0) { TAOS_CHECK_GOTO(walInitWriteFile(pWal), &lino, _exit);
taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
} }
if (walWriteImpl(pWal, index, msgType, syncMeta, body, bodyLen) < 0) { TAOS_CHECK_GOTO(walWriteImpl(pWal, index, msgType, syncMeta, body, bodyLen), &lino, _exit);
taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
taosThreadMutexUnlock(&pWal->mutex); _exit:
return index; if (code) {
} wError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
int32_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body,
int32_t bodyLen) {
int32_t code = 0;
taosThreadMutexLock(&pWal->mutex);
// concurrency control:
// if logs are write with assigned index,
// smaller index must be write before larger one
if (index != pWal->vers.lastVer + 1) {
terrno = TSDB_CODE_WAL_INVALID_VER;
taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
if (walCheckAndRoll(pWal) < 0) {
taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
if (pWal->pIdxFile == NULL || pWal->pLogFile == NULL || pWal->writeCur < 0) {
if (walInitWriteFile(pWal) < 0) {
taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
}
if (walWriteImpl(pWal, index, msgType, syncMeta, body, bodyLen) < 0) {
taosThreadMutexUnlock(&pWal->mutex);
return -1;
} }
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return code; return code;
} }
int32_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen) { int32_t walFsync(SWal *pWal, bool forceFsync) {
SWalSyncInfo syncMeta = { int32_t code = 0;
.isWeek = -1,
.seqNum = UINT64_MAX,
.term = UINT64_MAX,
};
return walWriteWithSyncInfo(pWal, index, msgType, syncMeta, body, bodyLen);
}
void walFsync(SWal *pWal, bool forceFsync) {
if (pWal->cfg.level == TAOS_WAL_SKIP) { if (pWal->cfg.level == TAOS_WAL_SKIP) {
return; return code;
} }
taosThreadMutexLock(&pWal->mutex); taosThreadMutexLock(&pWal->mutex);
@ -704,7 +734,10 @@ void walFsync(SWal *pWal, bool forceFsync) {
if (taosFsyncFile(pWal->pLogFile) < 0) { if (taosFsyncFile(pWal->pLogFile) < 0) {
wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal),
strerror(errno)); strerror(errno));
code = TAOS_SYSTEM_ERROR(errno);
} }
} }
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return code;
} }

49
source/libs/wal/test/walMetaTest.cpp
View File

@ -7,6 +7,7 @@
const char* ranStr = "tvapq02tcp"; const char* ranStr = "tvapq02tcp";
const int ranStrLen = strlen(ranStr); const int ranStrLen = strlen(ranStr);
SWalSyncInfo syncMeta = {0};
class WalCleanEnv : public ::testing::Test { class WalCleanEnv : public ::testing::Test {
protected: protected:
@ -170,7 +171,9 @@ TEST_F(WalCleanEnv, serialize) {
ASSERT(code == 0); ASSERT(code == 0);
code = walRollFileInfo(pWal); code = walRollFileInfo(pWal);
ASSERT(code == 0); ASSERT(code == 0);
char* ss = walMetaSerialize(pWal); char* ss = NULL;
code = walMetaSerialize(pWal, &ss);
ASSERT(code == 0);
printf("%s\n", ss); printf("%s\n", ss);
taosMemoryFree(ss); taosMemoryFree(ss);
code = walSaveMeta(pWal); code = walSaveMeta(pWal);
@ -193,15 +196,21 @@ TEST_F(WalKeepEnv, readOldMeta) {
walResetEnv(); walResetEnv();
int code; int code;
syncMeta.isWeek = -1;
syncMeta.seqNum = UINT64_MAX;
syncMeta.term = UINT64_MAX;
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen); code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, i); ASSERT_EQ(pWal->vers.lastVer, i);
code = walWrite(pWal, i + 2, i, (void*)ranStr, ranStrLen); code = walAppendLog(pWal, i + 2, i, syncMeta, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, -1); ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER);
ASSERT_EQ(pWal->vers.lastVer, i); ASSERT_EQ(pWal->vers.lastVer, i);
} }
char* oldss = walMetaSerialize(pWal); char* oldss = NULL;
code = walMetaSerialize(pWal, &oldss);
ASSERT(code == 0);
TearDown(); TearDown();
SetUp(); SetUp();
@ -209,7 +218,9 @@ TEST_F(WalKeepEnv, readOldMeta) {
ASSERT_EQ(pWal->vers.firstVer, 0); ASSERT_EQ(pWal->vers.firstVer, 0);
ASSERT_EQ(pWal->vers.lastVer, 9); ASSERT_EQ(pWal->vers.lastVer, 9);
char* newss = walMetaSerialize(pWal); char* newss = NULL;
code = walMetaSerialize(pWal, &newss);
ASSERT(code == 0);
int len = strlen(oldss); int len = strlen(oldss);
ASSERT_EQ(len, strlen(newss)); ASSERT_EQ(len, strlen(newss));
@ -223,11 +234,11 @@ TEST_F(WalKeepEnv, readOldMeta) {
TEST_F(WalCleanEnv, write) { TEST_F(WalCleanEnv, write) {
int code; int code;
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen); code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, i); ASSERT_EQ(pWal->vers.lastVer, i);
code = walWrite(pWal, i + 2, i, (void*)ranStr, ranStrLen); code = walAppendLog(pWal, i + 2, i, syncMeta, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, -1); ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER);
ASSERT_EQ(pWal->vers.lastVer, i); ASSERT_EQ(pWal->vers.lastVer, i);
} }
code = walSaveMeta(pWal); code = walSaveMeta(pWal);
@ -237,7 +248,7 @@ TEST_F(WalCleanEnv, write) {
TEST_F(WalCleanEnv, rollback) { TEST_F(WalCleanEnv, rollback) {
int code; int code;
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen); code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, i); ASSERT_EQ(pWal->vers.lastVer, i);
} }
@ -260,7 +271,7 @@ TEST_F(WalCleanEnv, rollback) {
TEST_F(WalCleanEnv, rollbackMultiFile) { TEST_F(WalCleanEnv, rollbackMultiFile) {
int code; int code;
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen); code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, i); ASSERT_EQ(pWal->vers.lastVer, i);
if (i == 5) { if (i == 5) {
@ -278,11 +289,11 @@ TEST_F(WalCleanEnv, rollbackMultiFile) {
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, 5); ASSERT_EQ(pWal->vers.lastVer, 5);
code = walRollback(pWal, 5); code = walRollback(pWal, 5);
ASSERT_EQ(code, -1); ASSERT_NE(code, 0);
ASSERT_EQ(pWal->vers.lastVer, 5); ASSERT_EQ(pWal->vers.lastVer, 5);
code = walWrite(pWal, 6, 6, (void*)ranStr, ranStrLen); code = walAppendLog(pWal, 6, 6, syncMeta, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, 6); ASSERT_EQ(pWal->vers.lastVer, 6);
@ -294,7 +305,7 @@ TEST_F(WalCleanDeleteEnv, roll) {
int code; int code;
int i; int i;
for (i = 0; i < 100; i++) { for (i = 0; i < 100; i++) {
code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen); code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, i); ASSERT_EQ(pWal->vers.lastVer, i);
code = walCommit(pWal, i); code = walCommit(pWal, i);
@ -307,11 +318,11 @@ TEST_F(WalCleanDeleteEnv, roll) {
ASSERT_EQ(pWal->vers.snapshotVer, i - 1); ASSERT_EQ(pWal->vers.snapshotVer, i - 1);
ASSERT_EQ(pWal->vers.verInSnapshotting, -1); ASSERT_EQ(pWal->vers.verInSnapshotting, -1);
code = walWrite(pWal, 5, 0, (void*)ranStr, ranStrLen); code = walAppendLog(pWal, 5, 0, syncMeta, (void*)ranStr, ranStrLen);
ASSERT_NE(code, 0); ASSERT_NE(code, 0);
for (; i < 200; i++) { for (; i < 200; i++) {
code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen); code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
code = walCommit(pWal, i); code = walCommit(pWal, i);
ASSERT_EQ(pWal->vers.commitVer, i); ASSERT_EQ(pWal->vers.commitVer, i);
@ -334,7 +345,7 @@ TEST_F(WalKeepEnv, readHandleRead) {
char newStr[100]; char newStr[100];
sprintf(newStr, "%s-%d", ranStr, i); sprintf(newStr, "%s-%d", ranStr, i);
int len = strlen(newStr); int len = strlen(newStr);
code = walWrite(pWal, i, 0, newStr, len); code = walAppendLog(pWal, i, 0, syncMeta, newStr, len);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
} }
for (int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
@ -370,7 +381,7 @@ TEST_F(WalRetentionEnv, repairMeta1) {
char newStr[100]; char newStr[100];
sprintf(newStr, "%s-%d", ranStr, i); sprintf(newStr, "%s-%d", ranStr, i);
int len = strlen(newStr); int len = strlen(newStr);
code = walWrite(pWal, i, 0, newStr, len); code = walAppendLog(pWal, i, 0, syncMeta, newStr, len);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
} }
@ -416,7 +427,7 @@ TEST_F(WalRetentionEnv, repairMeta1) {
char newStr[100]; char newStr[100];
sprintf(newStr, "%s-%d", ranStr, i); sprintf(newStr, "%s-%d", ranStr, i);
int len = strlen(newStr); int len = strlen(newStr);
code = walWrite(pWal, i, 0, newStr, len); code = walAppendLog(pWal, i, 0, syncMeta, newStr, len);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
} }

8
source/os/src/osMemory.c
View File

@ -266,7 +266,7 @@ void *taosMemoryMalloc(int64_t size) {
return (char *)tmp + sizeof(TdMemoryInfo); return (char *)tmp + sizeof(TdMemoryInfo);
#else #else
void* p = malloc(size); void *p = malloc(size);
if (NULL == p) { if (NULL == p) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
} }
@ -287,7 +287,7 @@ void *taosMemoryCalloc(int64_t num, int64_t size) {
return (char *)tmp + sizeof(TdMemoryInfo); return (char *)tmp + sizeof(TdMemoryInfo);
#else #else
void* p = calloc(num, size); void *p = calloc(num, size);
if (NULL == p) { if (NULL == p) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
} }
@ -317,7 +317,7 @@ void *taosMemoryRealloc(void *ptr, int64_t size) {
return (char *)tmp + sizeof(TdMemoryInfo); return (char *)tmp + sizeof(TdMemoryInfo);
#else #else
void* p = realloc(ptr, size); void *p = realloc(ptr, size);
if (size > 0 && NULL == p) { if (size > 0 && NULL == p) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
} }
@ -342,7 +342,7 @@ char *taosStrdup(const char *ptr) {
return (char *)tmp + sizeof(TdMemoryInfo); return (char *)tmp + sizeof(TdMemoryInfo);
#else #else
char* p = tstrdup(ptr); char *p = tstrdup(ptr);
if (ptr != NULL && NULL == p) { if (ptr != NULL && NULL == p) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
} }

38
source/util/src/tarray.c
View File

@ -18,7 +18,7 @@
#include "tcoding.h" #include "tcoding.h"
// todo refactor API // todo refactor API
#define BOUNDARY_SIZE 1024*1024*1024 // 1G #define BOUNDARY_SIZE 1024 * 1024 * 1024 // 1G
#define BOUNDARY_SMALL_FACTOR 1.2 #define BOUNDARY_SMALL_FACTOR 1.2
#define BOUNDARY_BIG_FACTOR 2 #define BOUNDARY_BIG_FACTOR 2
@ -52,14 +52,12 @@ SArray* taosArrayInit(size_t size, size_t elemSize) {
SArray* taosArrayInit_s(size_t elemSize, size_t initialSize) { SArray* taosArrayInit_s(size_t elemSize, size_t initialSize) {
SArray* pArray = taosMemoryMalloc(sizeof(SArray)); SArray* pArray = taosMemoryMalloc(sizeof(SArray));
if (pArray == NULL) { if (pArray == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
pArray->size = initialSize; pArray->size = initialSize;
pArray->pData = taosMemoryCalloc(initialSize, elemSize); pArray->pData = taosMemoryCalloc(initialSize, elemSize);
if (pArray->pData == NULL) { if (pArray->pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pArray); taosMemoryFree(pArray);
return NULL; return NULL;
} }
@ -75,7 +73,7 @@ static int32_t taosArrayResize(SArray* pArray) {
void* tmp = taosMemoryRealloc(pArray->pData, size * pArray->elemSize); void* tmp = taosMemoryRealloc(pArray->pData, size * pArray->elemSize);
if (tmp == NULL) { // reallocate failed, the original buffer remains if (tmp == NULL) { // reallocate failed, the original buffer remains
return -1; return terrno;
} }
pArray->pData = tmp; pArray->pData = tmp;
@ -99,7 +97,7 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) {
pArray->pData = taosMemoryRealloc(pArray->pData, tsize * pArray->elemSize); pArray->pData = taosMemoryRealloc(pArray->pData, tsize * pArray->elemSize);
if (pArray->pData == NULL) { if (pArray->pData == NULL) {
return -1; return terrno;
} }
pArray->capacity = tsize; pArray->capacity = tsize;
@ -109,10 +107,13 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) {
void* taosArrayAddBatch(SArray* pArray, const void* pData, int32_t nEles) { void* taosArrayAddBatch(SArray* pArray, const void* pData, int32_t nEles) {
if (pData == NULL) { if (pData == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL; return NULL;
} }
if (taosArrayEnsureCap(pArray, pArray->size + nEles) != 0) { int32_t code = taosArrayEnsureCap(pArray, pArray->size + nEles);
if (code) {
terrno = code;
return NULL; return NULL;
} }
@ -171,7 +172,9 @@ void* taosArrayAddAll(SArray* pArray, const SArray* pInput) {
} }
void* taosArrayReserve(SArray* pArray, int32_t num) { void* taosArrayReserve(SArray* pArray, int32_t num) {
if (taosArrayEnsureCap(pArray, pArray->size + num) != 0) { int32_t code = taosArrayEnsureCap(pArray, pArray->size + num);
if (code) {
terrno = code;
return NULL; return NULL;
} }
@ -198,7 +201,7 @@ void* taosArrayGet(const SArray* pArray, size_t index) {
} }
if (index >= pArray->size) { if (index >= pArray->size) {
uError("index is out of range, current:%" PRIzu " max:%"PRIzu, index, pArray->size); uError("index is out of range, current:%" PRIzu " max:%" PRIzu, index, pArray->size);
return NULL; return NULL;
} }
@ -230,6 +233,7 @@ size_t taosArrayGetSize(const SArray* pArray) {
void* taosArrayInsert(SArray* pArray, size_t index, const void* pData) { void* taosArrayInsert(SArray* pArray, size_t index, const void* pData) {
if (pArray == NULL || pData == NULL) { if (pArray == NULL || pData == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL; return NULL;
} }
@ -239,7 +243,6 @@ void* taosArrayInsert(SArray* pArray, size_t index, const void* pData) {
if (pArray->size >= pArray->capacity) { if (pArray->size >= pArray->capacity) {
int32_t ret = taosArrayResize(pArray); int32_t ret = taosArrayResize(pArray);
if (ret < 0) { if (ret < 0) {
return NULL; return NULL;
} }
@ -316,8 +319,10 @@ SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize) {
} }
SArray* pDst = taosArrayInit(size, elemSize); SArray* pDst = taosArrayInit(size, elemSize);
if (pDst) {
memcpy(pDst->pData, src, elemSize * size); memcpy(pDst->pData, src, elemSize * size);
pDst->size = size; pDst->size = size;
}
return pDst; return pDst;
} }
@ -333,6 +338,7 @@ SArray* taosArrayDup(const SArray* pSrc, __array_item_dup_fn_t fn) {
SArray* dst = taosArrayInit(pSrc->size, pSrc->elemSize); SArray* dst = taosArrayInit(pSrc->size, pSrc->elemSize);
if (dst) {
if (fn == NULL) { if (fn == NULL) {
memcpy(dst->pData, pSrc->pData, pSrc->elemSize * pSrc->size); memcpy(dst->pData, pSrc->pData, pSrc->elemSize * pSrc->size);
} else { } else {
@ -345,6 +351,7 @@ SArray* taosArrayDup(const SArray* pSrc, __array_item_dup_fn_t fn) {
} }
dst->size = pSrc->size; dst->size = pSrc->size;
}
return dst; return dst;
} }
@ -508,10 +515,21 @@ void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t
int32_t sz; int32_t sz;
buf = taosDecodeFixedI32(buf, &sz); buf = taosDecodeFixedI32(buf, &sz);
*pArray = taosArrayInit(sz, sizeof(void*)); *pArray = taosArrayInit(sz, sizeof(void*));
if (*pArray == NULL) {
return NULL;
}
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
void* data = taosMemoryCalloc(1, dataSz); void* data = taosMemoryCalloc(1, dataSz);
if (data == NULL) {
return NULL;
}
buf = decode(buf, data, sver); buf = decode(buf, data, sver);
taosArrayPush(*pArray, &data);
if (taosArrayPush(*pArray, &data) == NULL) {
taosMemoryFree(data);
return NULL;
}
} }
return (void*)buf; return (void*)buf;
} }

11
source/util/src/tcache.c
View File

@ -230,6 +230,7 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheNode *
static FORCE_INLINE STrashElem *doRemoveElemInTrashcan(SCacheObj *pCacheObj, STrashElem *pElem) { static FORCE_INLINE STrashElem *doRemoveElemInTrashcan(SCacheObj *pCacheObj, STrashElem *pElem) {
if (pElem->pData->signature != pElem->pData) { if (pElem->pData->signature != pElem->pData) {
uWarn("key:sig:0x%" PRIx64 " %p data has been released, ignore", (int64_t)pElem->pData->signature, pElem->pData); uWarn("key:sig:0x%" PRIx64 " %p data has been released, ignore", (int64_t)pElem->pData->signature, pElem->pData);
terrno = TSDB_CODE_INVALID_PARA;
return NULL; return NULL;
} }
@ -358,6 +359,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInMs, bool extendLi
SCacheObj *pCacheObj = (SCacheObj *)taosMemoryCalloc(1, sizeof(SCacheObj)); SCacheObj *pCacheObj = (SCacheObj *)taosMemoryCalloc(1, sizeof(SCacheObj));
if (pCacheObj == NULL) { if (pCacheObj == NULL) {
uError("failed to allocate memory, reason:%s", strerror(errno)); uError("failed to allocate memory, reason:%s", strerror(errno));
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
@ -393,6 +395,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInMs, bool extendLi
void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize,
int32_t durationMS) { int32_t durationMS) {
if (pCacheObj == NULL || pCacheObj->pEntryList == NULL || pCacheObj->deleting == 1) { if (pCacheObj == NULL || pCacheObj->pEntryList == NULL || pCacheObj->deleting == 1) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL; return NULL;
} }
@ -501,11 +504,15 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) {
} }
void *taosCacheTransferData(SCacheObj *pCacheObj, void **data) { void *taosCacheTransferData(SCacheObj *pCacheObj, void **data) {
if (pCacheObj == NULL || data == NULL || (*data) == NULL) return NULL; if (pCacheObj == NULL || data == NULL || (*data) == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
SCacheNode *ptNode = (SCacheNode *)((char *)(*data) - sizeof(SCacheNode)); SCacheNode *ptNode = (SCacheNode *)((char *)(*data) - sizeof(SCacheNode));
if (ptNode->signature != ptNode) { if (ptNode->signature != ptNode) {
uError("cache:%s, key: %p the data from cache is invalid", pCacheObj->name, ptNode); uError("cache:%s, key: %p the data from cache is invalid", pCacheObj->name, ptNode);
terrno = TSDB_CODE_INVALID_PARA;
return NULL; return NULL;
} }
@ -702,8 +709,8 @@ SCacheNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pDat
SCacheNode *pNewNode = taosMemoryCalloc(1, sizeInBytes); SCacheNode *pNewNode = taosMemoryCalloc(1, sizeInBytes);
if (pNewNode == NULL) { if (pNewNode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
uError("failed to allocate memory, reason:%s", strerror(errno)); uError("failed to allocate memory, reason:%s", strerror(errno));
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }

6
source/util/src/tdes.c
View File

@ -76,6 +76,12 @@ char* taosDesEncode(int64_t key, char* src, int32_t len) {
char* taosDesDecode(int64_t key, char* src, int32_t len) { char* taosDesDecode(int64_t key, char* src, int32_t len) {
uint8_t* keyStr = (uint8_t*)(&key); uint8_t* keyStr = (uint8_t*)(&key);
char* temp = taosMemoryCalloc(len + 8, 1); char* temp = taosMemoryCalloc(len + 8, 1);
if (!temp) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
memcpy(temp, src, len); memcpy(temp, src, len);
len += 8; len += 8;

34
source/util/src/tdigest.c
View File

@ -30,7 +30,7 @@
#include "tlog.h" #include "tlog.h"
#define INTERPOLATE(x, x0, x1) (((x) - (x0)) / ((x1) - (x0))) #define INTERPOLATE(x, x0, x1) (((x) - (x0)) / ((x1) - (x0)))
//#define INTEGRATED_LOCATION(compression, q) ((compression) * (asin(2 * (q) - 1) + M_PI / 2) / M_PI) // #define INTEGRATED_LOCATION(compression, q) ((compression) * (asin(2 * (q) - 1) + M_PI / 2) / M_PI)
#define INTEGRATED_LOCATION(compression, q) ((compression) * (asin(2 * (double)(q)-1) / M_PI + (double)1 / 2)) #define INTEGRATED_LOCATION(compression, q) ((compression) * (asin(2 * (double)(q)-1) / M_PI + (double)1 / 2))
#define FLOAT_EQ(f1, f2) (fabs((f1) - (f2)) <= FLT_EPSILON) #define FLOAT_EQ(f1, f2) (fabs((f1) - (f2)) <= FLT_EPSILON)
@ -99,16 +99,19 @@ static void mergeCentroid(SMergeArgs *args, SCentroid *merge) {
} }
} }
void tdigestCompress(TDigest *t) { int32_t tdigestCompress(TDigest *t) {
SCentroid *unmerged_centroids; SCentroid *unmerged_centroids;
int64_t unmerged_weight = 0; int64_t unmerged_weight = 0;
int32_t num_unmerged = t->num_buffered_pts; int32_t num_unmerged = t->num_buffered_pts;
int32_t i, j; int32_t i, j;
SMergeArgs args; SMergeArgs args;
if (t->num_buffered_pts <= 0) return; if (t->num_buffered_pts <= 0) return 0;
unmerged_centroids = (SCentroid *)taosMemoryMalloc(sizeof(SCentroid) * t->num_buffered_pts); unmerged_centroids = (SCentroid *)taosMemoryMalloc(sizeof(SCentroid) * t->num_buffered_pts);
if (unmerged_centroids == NULL) {
return terrno;
}
for (i = 0; i < num_unmerged; i++) { for (i = 0; i < num_unmerged; i++) {
SPt *p = t->buffered_pts + i; SPt *p = t->buffered_pts + i;
SCentroid *c = &unmerged_centroids[i]; SCentroid *c = &unmerged_centroids[i];
@ -122,6 +125,10 @@ void tdigestCompress(TDigest *t) {
taosSort(unmerged_centroids, num_unmerged, sizeof(SCentroid), cmpCentroid); taosSort(unmerged_centroids, num_unmerged, sizeof(SCentroid), cmpCentroid);
memset(&args, 0, sizeof(SMergeArgs)); memset(&args, 0, sizeof(SMergeArgs));
args.centroids = (SCentroid *)taosMemoryMalloc((size_t)(sizeof(SCentroid) * t->size)); args.centroids = (SCentroid *)taosMemoryMalloc((size_t)(sizeof(SCentroid) * t->size));
if (args.centroids == NULL) {
taosMemoryFree((void *)unmerged_centroids);
return terrno;
}
memset(args.centroids, 0, (size_t)(sizeof(SCentroid) * t->size)); memset(args.centroids, 0, (size_t)(sizeof(SCentroid) * t->size));
args.t = t; args.t = t;
@ -167,10 +174,11 @@ void tdigestCompress(TDigest *t) {
memcpy(t->centroids, args.centroids, sizeof(SCentroid) * t->num_centroids); memcpy(t->centroids, args.centroids, sizeof(SCentroid) * t->num_centroids);
taosMemoryFree((void *)args.centroids); taosMemoryFree((void *)args.centroids);
return 0;
} }
void tdigestAdd(TDigest *t, double x, int64_t w) { int32_t tdigestAdd(TDigest *t, double x, int64_t w) {
if (w == 0) return; if (w == 0) return 0;
int32_t i = t->num_buffered_pts; int32_t i = t->num_buffered_pts;
if (i > 0 && t->buffered_pts[i - 1].value == x) { if (i > 0 && t->buffered_pts[i - 1].value == x) {
@ -181,7 +189,10 @@ void tdigestAdd(TDigest *t, double x, int64_t w) {
t->num_buffered_pts++; t->num_buffered_pts++;
} }
if (t->num_buffered_pts >= t->threshold) tdigestCompress(t); if (t->num_buffered_pts >= t->threshold) {
return tdigestCompress(t);
}
return 0;
} }
#if 0 #if 0
@ -284,16 +295,21 @@ double tdigestQuantile(TDigest *t, double q) {
return t->max; return t->max;
} }
void tdigestMerge(TDigest *t1, TDigest *t2) { int32_t tdigestMerge(TDigest *t1, TDigest *t2) {
int32_t code = 0;
// SPoints // SPoints
int32_t num_pts = t2->num_buffered_pts; int32_t num_pts = t2->num_buffered_pts;
for (int32_t i = num_pts - 1; i >= 0; i--) { for (int32_t i = num_pts - 1; i >= 0; i--) {
SPt *p = t2->buffered_pts + i; SPt *p = t2->buffered_pts + i;
tdigestAdd(t1, p->value, p->weight); code = tdigestAdd(t1, p->value, p->weight);
if (code) return code;
t2->num_buffered_pts--; t2->num_buffered_pts--;
} }
// centroids // centroids
for (int32_t i = 0; i < t2->num_centroids; i++) { for (int32_t i = 0; i < t2->num_centroids; i++) {
tdigestAdd(t1, t2->centroids[i].mean, t2->centroids[i].weight); code = tdigestAdd(t1, t2->centroids[i].mean, t2->centroids[i].weight);
if (code) return code;
} }
return 0;
} }

6
source/util/src/tenv.c
View File

@ -28,7 +28,7 @@ int32_t taosEnvNameToCfgName(const char *envNameStr, char *cfgNameStr, int32_t c
// p[cfgNameMaxLen - 1] = '\0'; // p[cfgNameMaxLen - 1] = '\0';
// return strlen(cfgNameStr); // return strlen(cfgNameStr);
cfgNameStr[0] = '\0'; cfgNameStr[0] = '\0';
return -1; return TSDB_CODE_INVALID_PARA;
} }
envNameStr += 5; envNameStr += 5;
if (*envNameStr != '\0') { if (*envNameStr != '\0') {
@ -55,7 +55,7 @@ int32_t taosEnvNameToCfgName(const char *envNameStr, char *cfgNameStr, int32_t c
int32_t taosEnvToCfg(const char *envStr, char *cfgStr) { int32_t taosEnvToCfg(const char *envStr, char *cfgStr) {
if (envStr == NULL || cfgStr == NULL) { if (envStr == NULL || cfgStr == NULL) {
return -1; return TSDB_CODE_INVALID_PARA;
} }
if (cfgStr != envStr) strcpy(cfgStr, envStr); if (cfgStr != envStr) strcpy(cfgStr, envStr);
char *p = strchr(cfgStr, '='); char *p = strchr(cfgStr, '=');
@ -74,7 +74,7 @@ int32_t taosEnvToCfg(const char *envStr, char *cfgStr) {
memset(&cfgStr[cfgNameLen], ' ', p - cfgStr - cfgNameLen + 1); memset(&cfgStr[cfgNameLen], ' ', p - cfgStr - cfgNameLen + 1);
} else { } else {
*cfgStr = '\0'; *cfgStr = '\0';
return -1; return TSDB_CODE_INVALID_PARA;
} }
} }
return strlen(cfgStr); return strlen(cfgStr);

8
source/util/src/tgeosctx.c
View File

@ -13,14 +13,12 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tdef.h"
#include "tgeosctx.h" #include "tgeosctx.h"
#include "tdef.h"
static threadlocal SGeosContext tlGeosCtx = {0}; static threadlocal SGeosContext tlGeosCtx = {0};
SGeosContext* getThreadLocalGeosCtx() { SGeosContext* getThreadLocalGeosCtx() { return &tlGeosCtx; }
return &tlGeosCtx;
}
void destroyThreadLocalGeosCtx() { void destroyThreadLocalGeosCtx() {
if (tlGeosCtx.WKTReader) { if (tlGeosCtx.WKTReader) {
@ -47,7 +45,7 @@ void destroyThreadLocalGeosCtx() {
destroyRegexes(tlGeosCtx.WKTRegex, tlGeosCtx.WKTMatchData); destroyRegexes(tlGeosCtx.WKTRegex, tlGeosCtx.WKTMatchData);
} }
if(tlGeosCtx.handle) { if (tlGeosCtx.handle) {
GEOS_finish_r(tlGeosCtx.handle); GEOS_finish_r(tlGeosCtx.handle);
tlGeosCtx.handle = NULL; tlGeosCtx.handle = NULL;
} }

28
source/util/src/thash.c
View File

@ -67,7 +67,7 @@ struct SHashObj {
bool enableUpdate; // enable update bool enableUpdate; // enable update
SArray *pMemBlock; // memory block allocated for SHashEntry SArray *pMemBlock; // memory block allocated for SHashEntry
_hash_before_fn_t callbackFp; // function invoked before return the value to caller _hash_before_fn_t callbackFp; // function invoked before return the value to caller
// int64_t compTimes; // int64_t compTimes;
}; };
/* /*
@ -147,7 +147,7 @@ static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntr
uint32_t hashVal) { uint32_t hashVal) {
SHashNode *pNode = pe->next; SHashNode *pNode = pe->next;
while (pNode) { while (pNode) {
//atomic_add_fetch_64(&pHashObj->compTimes, 1); // atomic_add_fetch_64(&pHashObj->compTimes, 1);
if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) &&
pNode->removed == 0) { pNode->removed == 0) {
break; break;
@ -243,7 +243,6 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
SHashObj *pHashObj = (SHashObj *)taosMemoryMalloc(sizeof(SHashObj)); SHashObj *pHashObj = (SHashObj *)taosMemoryMalloc(sizeof(SHashObj));
if (pHashObj == NULL) { if (pHashObj == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
@ -262,7 +261,6 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
pHashObj->hashList = (SHashEntry **)taosMemoryMalloc(pHashObj->capacity * sizeof(void *)); pHashObj->hashList = (SHashEntry **)taosMemoryMalloc(pHashObj->capacity * sizeof(void *));
if (pHashObj->hashList == NULL) { if (pHashObj->hashList == NULL) {
taosMemoryFree(pHashObj); taosMemoryFree(pHashObj);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
@ -270,7 +268,6 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
if (pHashObj->pMemBlock == NULL) { if (pHashObj->pMemBlock == NULL) {
taosMemoryFree(pHashObj->hashList); taosMemoryFree(pHashObj->hashList);
taosMemoryFree(pHashObj); taosMemoryFree(pHashObj);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
@ -279,7 +276,6 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
taosArrayDestroy(pHashObj->pMemBlock); taosArrayDestroy(pHashObj->pMemBlock);
taosMemoryFree(pHashObj->hashList); taosMemoryFree(pHashObj->hashList);
taosMemoryFree(pHashObj); taosMemoryFree(pHashObj);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
@ -290,7 +286,12 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
pHashObj->hashList[i]->next = NULL; pHashObj->hashList[i]->next = NULL;
} }
taosArrayPush(pHashObj->pMemBlock, &p); if (taosArrayPush(pHashObj->pMemBlock, &p) == NULL) {
taosArrayDestroy(pHashObj->pMemBlock);
taosMemoryFree(pHashObj->hashList);
taosMemoryFree(pHashObj);
return NULL;
}
return pHashObj; return pHashObj;
} }
@ -315,8 +316,7 @@ int32_t taosHashGetSize(const SHashObj *pHashObj) {
int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const void *data, size_t size) { int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const void *data, size_t size) {
if (pHashObj == NULL || key == NULL || keyLen == 0) { if (pHashObj == NULL || key == NULL || keyLen == 0) {
terrno = TSDB_CODE_INVALID_PTR; return terrno = TSDB_CODE_INVALID_PTR;
return TSDB_CODE_INVALID_PTR;
} }
uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen);
@ -440,14 +440,12 @@ void *taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void *
*size = pNode->dataLen; *size = pNode->dataLen;
*d = taosMemoryCalloc(1, *size); *d = taosMemoryCalloc(1, *size);
if (*d == NULL) { if (*d == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
} else if (*size < pNode->dataLen) { } else if (*size < pNode->dataLen) {
*size = pNode->dataLen; *size = pNode->dataLen;
char *tmp = taosMemoryRealloc(*d, *size); char *tmp = taosMemoryRealloc(*d, *size);
if (tmp == NULL) { if (tmp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
@ -474,7 +472,7 @@ void *taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void *
int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) {
if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || key == NULL || keyLen == 0) { if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || key == NULL || keyLen == 0) {
return -1; return TSDB_CODE_INVALID_PARA;
} }
uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen);
@ -631,7 +629,10 @@ void taosHashTableResize(SHashObj *pHashObj) {
pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry)); pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry));
} }
taosArrayPush(pHashObj->pMemBlock, &p); if (taosArrayPush(pHashObj->pMemBlock, &p) == NULL) {
taosMemoryFree(p);
return;
}
pHashObj->capacity = newCapacity; pHashObj->capacity = newCapacity;
for (int32_t idx = 0; idx < pHashObj->capacity; ++idx) { for (int32_t idx = 0; idx < pHashObj->capacity; ++idx) {
@ -681,7 +682,6 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s
#endif #endif
if (pNewNode == NULL) { if (pNewNode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }

67
source/util/src/theap.c
View File

@ -188,7 +188,6 @@ void heapRemove(Heap* heap, HeapNode* node) {
void heapDequeue(Heap* heap) { heapRemove(heap, heap->min); } void heapDequeue(Heap* heap) { heapRemove(heap, heap->min); }
struct PriorityQueue { struct PriorityQueue {
SArray* container; SArray* container;
pq_comp_fn fn; pq_comp_fn fn;
@ -197,16 +196,23 @@ struct PriorityQueue {
}; };
PriorityQueue* createPriorityQueue(pq_comp_fn fn, FDelete deleteFn, void* param) { PriorityQueue* createPriorityQueue(pq_comp_fn fn, FDelete deleteFn, void* param) {
PriorityQueue* pq = (PriorityQueue*)taosMemoryCalloc(1, sizeof(PriorityQueue)); PriorityQueue* pq = (PriorityQueue*)taosMemoryCalloc(1, sizeof(PriorityQueue));
if (pq == NULL) {
return NULL;
}
pq->container = taosArrayInit(1, sizeof(PriorityQueueNode)); pq->container = taosArrayInit(1, sizeof(PriorityQueueNode));
if (pq->container == NULL) {
taosMemoryFree(pq);
return NULL;
}
pq->fn = fn; pq->fn = fn;
pq->deleteFn = deleteFn; pq->deleteFn = deleteFn;
pq->param = param; pq->param = param;
return pq; return pq;
} }
void taosPQSetFn(PriorityQueue* pq, pq_comp_fn fn) { void taosPQSetFn(PriorityQueue* pq, pq_comp_fn fn) { pq->fn = fn; }
pq->fn = fn;
}
void destroyPriorityQueue(PriorityQueue* pq) { void destroyPriorityQueue(PriorityQueue* pq) {
if (pq->deleteFn) if (pq->deleteFn)
@ -218,9 +224,9 @@ void destroyPriorityQueue(PriorityQueue* pq) {
static size_t pqParent(size_t i) { return (--i) >> 1; /* (i - 1) / 2 */ } static size_t pqParent(size_t i) { return (--i) >> 1; /* (i - 1) / 2 */ }
static size_t pqLeft(size_t i) { return (i << 1) | 1; /* i * 2 + 1 */ } static size_t pqLeft(size_t i) { return (i << 1) | 1; /* i * 2 + 1 */ }
static size_t pqRight(size_t i) { return (++i) << 1; /* (i + 1) * 2 */} static size_t pqRight(size_t i) { return (++i) << 1; /* (i + 1) * 2 */ }
static void pqSwapPQNode(PriorityQueueNode* a, PriorityQueueNode* b) { static void pqSwapPQNode(PriorityQueueNode* a, PriorityQueueNode* b) {
void * tmp = a->data; void* tmp = a->data;
a->data = b->data; a->data = b->data;
b->data = tmp; b->data = tmp;
} }
@ -288,12 +294,12 @@ static void pqRemove(PriorityQueue* pq, size_t i) {
pqUpdate(pq, i); pqUpdate(pq, i);
} }
PriorityQueueNode* taosPQTop(PriorityQueue* pq) { PriorityQueueNode* taosPQTop(PriorityQueue* pq) { return pqContainerGetEle(pq, 0); }
return pqContainerGetEle(pq, 0);
}
PriorityQueueNode* taosPQPush(PriorityQueue* pq, const PriorityQueueNode* node) { PriorityQueueNode* taosPQPush(PriorityQueue* pq, const PriorityQueueNode* node) {
taosArrayPush(pq->container, node); if (taosArrayPush(pq->container, node) == NULL) {
return NULL;
}
return pqReverseHeapify(pq, pqContainerSize(pq) - 1); return pqReverseHeapify(pq, pqContainerSize(pq) - 1);
} }
@ -310,15 +316,28 @@ struct BoundedQueue {
BoundedQueue* createBoundedQueue(uint32_t maxSize, pq_comp_fn fn, FDelete deleteFn, void* param) { BoundedQueue* createBoundedQueue(uint32_t maxSize, pq_comp_fn fn, FDelete deleteFn, void* param) {
BoundedQueue* q = (BoundedQueue*)taosMemoryCalloc(1, sizeof(BoundedQueue)); BoundedQueue* q = (BoundedQueue*)taosMemoryCalloc(1, sizeof(BoundedQueue));
if (q == NULL) {
return NULL;
}
q->queue = createPriorityQueue(fn, deleteFn, param); q->queue = createPriorityQueue(fn, deleteFn, param);
taosArrayEnsureCap(q->queue->container, maxSize + 1); if (q->queue == NULL) {
taosMemoryFree(q);
return NULL;
}
int32_t code = taosArrayEnsureCap(q->queue->container, maxSize + 1);
if (code) {
destroyPriorityQueue(q->queue);
taosMemoryFree(q);
terrno = code;
return NULL;
}
q->maxSize = maxSize; q->maxSize = maxSize;
return q; return q;
} }
void taosBQSetFn(BoundedQueue* q, pq_comp_fn fn) { void taosBQSetFn(BoundedQueue* q, pq_comp_fn fn) { taosPQSetFn(q->queue, fn); }
taosPQSetFn(q->queue, fn);
}
void destroyBoundedQueue(BoundedQueue* q) { void destroyBoundedQueue(BoundedQueue* q) {
if (!q) return; if (!q) return;
@ -343,22 +362,12 @@ PriorityQueueNode* taosBQPush(BoundedQueue* q, PriorityQueueNode* n) {
} }
} }
PriorityQueueNode* taosBQTop(BoundedQueue* q) { PriorityQueueNode* taosBQTop(BoundedQueue* q) { return taosPQTop(q->queue); }
return taosPQTop(q->queue);
}
void taosBQBuildHeap(BoundedQueue *q) { void taosBQBuildHeap(BoundedQueue* q) { pqBuildHeap(q->queue); }
pqBuildHeap(q->queue);
}
size_t taosBQMaxSize(BoundedQueue* q) { size_t taosBQMaxSize(BoundedQueue* q) { return q->maxSize; }
return q->maxSize;
}
size_t taosBQSize(BoundedQueue* q) { size_t taosBQSize(BoundedQueue* q) { return taosPQSize(q->queue); }
return taosPQSize(q->queue);
}
void taosBQPop(BoundedQueue* q) { void taosBQPop(BoundedQueue* q) { taosPQPop(q->queue); }
taosPQPop(q->queue);
}

6
source/util/src/tidpool.c
View File

@ -19,7 +19,9 @@
void *taosInitIdPool(int32_t maxId) { void *taosInitIdPool(int32_t maxId) {
id_pool_t *pIdPool = taosMemoryCalloc(1, sizeof(id_pool_t)); id_pool_t *pIdPool = taosMemoryCalloc(1, sizeof(id_pool_t));
if (pIdPool == NULL) return NULL; if (pIdPool == NULL) {
return NULL;
}
pIdPool->freeList = taosMemoryCalloc(maxId, sizeof(bool)); pIdPool->freeList = taosMemoryCalloc(maxId, sizeof(bool));
if (pIdPool->freeList == NULL) { if (pIdPool->freeList == NULL) {
@ -120,7 +122,7 @@ int32_t taosUpdateIdPool(id_pool_t *pIdPool, int32_t maxId) {
bool *idList = taosMemoryCalloc(maxId, sizeof(bool)); bool *idList = taosMemoryCalloc(maxId, sizeof(bool));
if (idList == NULL) { if (idList == NULL) {
return -1; return terrno;
} }
taosThreadMutexLock(&pIdPool->mutex); taosThreadMutexLock(&pIdPool->mutex);

27
source/util/src/tref.c
View File

@ -67,15 +67,13 @@ int32_t taosOpenRef(int32_t max, RefFp fp) {
nodeList = taosMemoryCalloc(sizeof(SRefNode *), (size_t)max); nodeList = taosMemoryCalloc(sizeof(SRefNode *), (size_t)max);
if (nodeList == NULL) { if (nodeList == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
lockedBy = taosMemoryCalloc(sizeof(int64_t), (size_t)max); lockedBy = taosMemoryCalloc(sizeof(int64_t), (size_t)max);
if (lockedBy == NULL) { if (lockedBy == NULL) {
taosMemoryFree(nodeList); taosMemoryFree(nodeList);
terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
taosThreadMutexLock(&tsRefMutex); taosThreadMutexLock(&tsRefMutex);
@ -118,8 +116,7 @@ int32_t taosCloseRef(int32_t rsetId) {
if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
uTrace("rsetId:%d is invalid, out of range", rsetId); uTrace("rsetId:%d is invalid, out of range", rsetId);
terrno = TSDB_CODE_REF_INVALID_ID; return terrno = TSDB_CODE_REF_INVALID_ID;
return -1;
} }
pSet = tsRefSetList + rsetId; pSet = tsRefSetList + rsetId;
@ -149,8 +146,7 @@ int64_t taosAddRef(int32_t rsetId, void *p) {
if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
uTrace("rsetId:%d p:%p failed to add, rsetId not valid", rsetId, p); uTrace("rsetId:%d p:%p failed to add, rsetId not valid", rsetId, p);
terrno = TSDB_CODE_REF_INVALID_ID; return terrno = TSDB_CODE_REF_INVALID_ID;
return -1;
} }
pSet = tsRefSetList + rsetId; pSet = tsRefSetList + rsetId;
@ -158,14 +154,12 @@ int64_t taosAddRef(int32_t rsetId, void *p) {
if (pSet->state != TSDB_REF_STATE_ACTIVE) { if (pSet->state != TSDB_REF_STATE_ACTIVE) {
taosDecRsetCount(pSet); taosDecRsetCount(pSet);
uTrace("rsetId:%d p:%p failed to add, not active", rsetId, p); uTrace("rsetId:%d p:%p failed to add, not active", rsetId, p);
terrno = TSDB_CODE_REF_ID_REMOVED; return terrno = TSDB_CODE_REF_ID_REMOVED;
return -1;
} }
pNode = taosMemoryCalloc(sizeof(SRefNode), 1); pNode = taosMemoryCalloc(sizeof(SRefNode), 1);
if (pNode == NULL) { if (pNode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
rid = atomic_add_fetch_64(&pSet->rid, 1); rid = atomic_add_fetch_64(&pSet->rid, 1);
@ -389,21 +383,18 @@ static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove) {
if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rsetId not valid", rsetId, rid); uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rsetId not valid", rsetId, rid);
terrno = TSDB_CODE_REF_INVALID_ID; return terrno = TSDB_CODE_REF_INVALID_ID;
return -1;
} }
if (rid <= 0) { if (rid <= 0) {
uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rid not valid", rsetId, rid); uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rid not valid", rsetId, rid);
terrno = TSDB_CODE_REF_NOT_EXIST; return terrno = TSDB_CODE_REF_NOT_EXIST;
return -1;
} }
pSet = tsRefSetList + rsetId; pSet = tsRefSetList + rsetId;
if (pSet->state == TSDB_REF_STATE_EMPTY) { if (pSet->state == TSDB_REF_STATE_EMPTY) {
uTrace("rsetId:%d rid:%" PRId64 " failed to remove, cleaned", rsetId, rid); uTrace("rsetId:%d rid:%" PRId64 " failed to remove, cleaned", rsetId, rid);
terrno = TSDB_CODE_REF_ID_REMOVED; return terrno = TSDB_CODE_REF_ID_REMOVED;
return -1;
} }
hash = rid % pSet->max; hash = rid % pSet->max;