xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into enh/TD-30988-3.0

This commit is contained in:
kailixu 2024-07-24 14:47:47 +08:00
parent 50c2fc567c cd2f039fd9
commit acb1d0a9cc
26 changed files with 935 additions and 961 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
include/libs/wal/wal.h
View File

@ -12,6 +12,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_WAL_H_
#define _TD_WAL_H_
@ -20,19 +21,11 @@
#include "tdef.h"
#include "tlog.h"
#include "tmsg.h"
#ifdef __cplusplus
extern "C" {
#endif
// clang-format off
#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", DEBUG_DEBUG, wDebugFlag, __VA_ARGS__); }}
#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", DEBUG_TRACE, wDebugFlag, __VA_ARGS__); }}
// clang-format on
#define WAL_PROTO_VER 0
#define WAL_NOSUFFIX_LEN 20
#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1)
@ -131,8 +124,7 @@ typedef struct SWal {
typedef struct {
int64_t refId;
int64_t refVer;
// int64_t refFile;
SWal *pWal;
SWal *pWal;
} SWalRef;
typedef struct {
@ -143,10 +135,8 @@ typedef struct {
int8_t enableRef;
} SWalFilterCond;
typedef struct SWalReader SWalReader;
// todo hide this struct
struct SWalReader {
typedef struct SWalReader {
SWal *pWal;
int64_t readerId;
TdFilePtr pLogFile;
@ -159,7 +149,7 @@ struct SWalReader {
TdThreadMutex mutex;
SWalFilterCond cond;
SWalCkHead *pHead;
};
} SWalReader;
// module initialization
int32_t walInit();
@ -172,17 +162,9 @@ int32_t walPersist(SWal *);
void walClose(SWal *);
// write interfaces
// By assigning index by the caller, wal gurantees linearizability
int32_t walWrite(SWal *, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen);
int32_t walWriteWithSyncInfo(SWal *, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body,
int32_t bodyLen);
// Assign version automatically and return to caller,
// -1 will be returned for failed writes
int64_t walAppendLog(SWal *, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t bodyLen);
void walFsync(SWal *, bool force);
int32_t walAppendLog(SWal *, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t bodyLen);
int32_t walFsync(SWal *, bool force);
// apis for lifecycle management
int32_t walCommit(SWal *, int64_t ver);
@ -191,17 +173,13 @@ int32_t walRollback(SWal *, int64_t ver);
int32_t walBeginSnapshot(SWal *, int64_t ver, int64_t logRetention);
int32_t walEndSnapshot(SWal *);
int32_t walRestoreFromSnapshot(SWal *, int64_t ver);
// for tq
int32_t walApplyVer(SWal *, int64_t ver);
// int32_t walDataCorrupted(SWal*);
// wal reader
SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond, int64_t id);
void walCloseReader(SWalReader *pRead);
void walReadReset(SWalReader *pReader);
int32_t walReadVer(SWalReader *pRead, int64_t ver);
void decryptBody(SWalCfg *cfg, SWalCkHead *pHead, int32_t plainBodyLen, const char *func);
int32_t walReaderSeekVer(SWalReader *pRead, int64_t ver);
int32_t walNextValidMsg(SWalReader *pRead);
int64_t walReaderGetCurrentVer(const SWalReader *pReader);
@ -216,12 +194,11 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver);
int32_t walFetchBody(SWalReader *pRead);
int32_t walSkipFetchBody(SWalReader *pRead);
void walRefFirstVer(SWal *, SWalRef *);
void walRefLastVer(SWal *, SWalRef *);
SWalRef *walOpenRef(SWal *);
void walCloseRef(SWal *pWal, int64_t refId);
int32_t walSetRefVer(SWalRef *, int64_t ver);
void walRefFirstVer(SWal *, SWalRef *);
void walRefLastVer(SWal *, SWalRef *);
// helper function for raft
bool walLogExist(SWal *, int64_t ver);

6
include/util/tdigest.h
View File

@ -64,10 +64,10 @@ typedef struct TDigest {
} TDigest;
TDigest *tdigestNewFrom(void *pBuf, int32_t compression);
void tdigestAdd(TDigest *t, double x, int64_t w);
void tdigestMerge(TDigest *t1, TDigest *t2);
int32_t tdigestAdd(TDigest *t, double x, int64_t w);
int32_t tdigestMerge(TDigest *t1, TDigest *t2);
double tdigestQuantile(TDigest *t, double q);
void tdigestCompress(TDigest *t);
int32_t tdigestCompress(TDigest *t);
void tdigestFreeFrom(TDigest *t);
void tdigestAutoFill(TDigest *t, int32_t compression);

138
source/dnode/vnode/src/tq/tqMeta.c
View File

@ -37,7 +37,7 @@ int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle) {
}
} else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
if (tEncodeI64(pEncoder, pHandle->execHandle.execTb.suid) < 0) return -1;
if (pHandle->execHandle.execTb.qmsg != NULL){
if (pHandle->execHandle.execTb.qmsg != NULL) {
if (tEncodeCStr(pEncoder, pHandle->execHandle.execTb.qmsg) < 0) return -1;
}
}
@ -67,7 +67,7 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) {
}
} else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
if (tDecodeI64(pDecoder, &pHandle->execHandle.execTb.suid) < 0) return -1;
if (!tDecodeIsEnd(pDecoder)){
if (!tDecodeIsEnd(pDecoder)) {
if (tDecodeCStrAlloc(pDecoder, &pHandle->execHandle.execTb.qmsg) < 0) return -1;
}
}
@ -75,8 +75,8 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) {
return 0;
}
int32_t tqMetaDecodeCheckInfo(STqCheckInfo *info, void *pVal, int32_t vLen){
SDecoder decoder = {0};
int32_t tqMetaDecodeCheckInfo(STqCheckInfo* info, void* pVal, int32_t vLen) {
SDecoder decoder = {0};
tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
int32_t code = tDecodeSTqCheckInfo(&decoder, info);
if (code != 0) {
@ -87,8 +87,8 @@ int32_t tqMetaDecodeCheckInfo(STqCheckInfo *info, void *pVal, int32_t vLen){
return code;
}
int32_t tqMetaDecodeOffsetInfo(STqOffset *info, void *pVal, int32_t vLen){
SDecoder decoder = {0};
int32_t tqMetaDecodeOffsetInfo(STqOffset* info, void* pVal, int32_t vLen) {
SDecoder decoder = {0};
tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
int32_t code = tDecodeSTqOffset(&decoder, info);
if (code != 0) {
@ -101,7 +101,7 @@ int32_t tqMetaDecodeOffsetInfo(STqOffset *info, void *pVal, int32_t vLen){
int32_t tqMetaSaveInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen, const void* value, int32_t vLen) {
int32_t code = TDB_CODE_SUCCESS;
TXN* txn = NULL;
TXN* txn = NULL;
TQ_ERR_GO_TO_END(tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_GO_TO_END(tdbTbUpsert(ttb, key, kLen, value, vLen, txn));
@ -117,7 +117,7 @@ END:
int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen) {
int32_t code = TDB_CODE_SUCCESS;
TXN* txn = NULL;
TXN* txn = NULL;
TQ_ERR_GO_TO_END(tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_GO_TO_END(tdbTbDelete(ttb, key, kLen, txn));
@ -131,10 +131,10 @@ END:
return code;
}
void* tqMetaGetOffset(STQ* pTq, const char* subkey){
void* tqMetaGetOffset(STQ* pTq, const char* subkey) {
void* data = taosHashGet(pTq->pOffset, subkey, strlen(subkey));
if (data == NULL) {
int vLen = 0;
int vLen = 0;
if (tdbTbGet(pTq->pOffsetStore, subkey, strlen(subkey), &data, &vLen) < 0) {
tdbFree(data);
return NULL;
@ -146,7 +146,7 @@ void* tqMetaGetOffset(STQ* pTq, const char* subkey){
return NULL;
}
if(taosHashPut(pTq->pOffset, subkey, strlen(subkey), &offset, sizeof(STqOffset)) != 0){
if (taosHashPut(pTq->pOffset, subkey, strlen(subkey), &offset, sizeof(STqOffset)) != 0) {
tDeleteSTqOffset(&offset);
tdbFree(data);
return NULL;
@ -160,9 +160,9 @@ void* tqMetaGetOffset(STQ* pTq, const char* subkey){
}
int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) {
int32_t code = TDB_CODE_SUCCESS;
int32_t vlen;
void* buf = NULL;
int32_t code = TDB_CODE_SUCCESS;
int32_t vlen;
void* buf = NULL;
SEncoder encoder;
tEncodeSize(tEncodeSTqHandle, pHandle, vlen, code);
if (code < 0) {
@ -192,14 +192,15 @@ END:
return code;
}
static int tqMetaInitHandle(STQ* pTq, STqHandle* handle){
static int tqMetaInitHandle(STQ* pTq, STqHandle* handle) {
int32_t code = TDB_CODE_SUCCESS;
SVnode* pVnode = pTq->pVnode;
int32_t vgId = TD_VID(pVnode);
handle->pRef = walOpenRef(pVnode->pWal);
if (handle->pRef == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
TQ_ERR_RETURN(walSetRefVer(handle->pRef, handle->snapshotVer));
@ -213,8 +214,8 @@ static int tqMetaInitHandle(STQ* pTq, STqHandle* handle){
initStorageAPI(&reader.api);
if (handle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
handle->execHandle.task =
qCreateQueueExecTaskInfo(handle->execHandle.execCol.qmsg, &reader, vgId, &handle->execHandle.numOfCols, handle->consumerId);
handle->execHandle.task = qCreateQueueExecTaskInfo(handle->execHandle.execCol.qmsg, &reader, vgId,
&handle->execHandle.numOfCols, handle->consumerId);
if (handle->execHandle.task == NULL) {
tqError("cannot create exec task for %s", handle->subKey);
return TSDB_CODE_OUT_OF_MEMORY;
@ -240,7 +241,7 @@ static int tqMetaInitHandle(STQ* pTq, STqHandle* handle){
} else if (handle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
handle->pWalReader = walOpenReader(pVnode->pWal, NULL, 0);
if(handle->execHandle.execTb.qmsg != NULL && strcmp(handle->execHandle.execTb.qmsg, "") != 0) {
if (handle->execHandle.execTb.qmsg != NULL && strcmp(handle->execHandle.execTb.qmsg, "") != 0) {
if (nodesStringToNode(handle->execHandle.execTb.qmsg, &handle->execHandle.execTb.node) != 0) {
tqError("nodesStringToNode error in sub stable, since %s", terrstr());
return TSDB_CODE_SCH_INTERNAL_ERROR;
@ -251,13 +252,15 @@ static int tqMetaInitHandle(STQ* pTq, STqHandle* handle){
handle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, handle->consumerId);
SArray* tbUidList = NULL;
int ret = qGetTableList(handle->execHandle.execTb.suid, pVnode, handle->execHandle.execTb.node, &tbUidList, handle->execHandle.task);
if(ret != TDB_CODE_SUCCESS) {
int ret = qGetTableList(handle->execHandle.execTb.suid, pVnode, handle->execHandle.execTb.node, &tbUidList,
handle->execHandle.task);
if (ret != TDB_CODE_SUCCESS) {
tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle->subKey, handle->consumerId);
taosArrayDestroy(tbUidList);
return TSDB_CODE_SCH_INTERNAL_ERROR;
}
tqInfo("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pVnode->config.vgId, handle->execHandle.execTb.suid);
tqInfo("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pVnode->config.vgId,
handle->execHandle.execTb.suid);
handle->execHandle.pTqReader = tqReaderOpen(pVnode);
tqReaderSetTbUidList(handle->execHandle.pTqReader, tbUidList, NULL);
taosArrayDestroy(tbUidList);
@ -265,10 +268,10 @@ static int tqMetaInitHandle(STQ* pTq, STqHandle* handle){
return 0;
}
static int32_t tqMetaRestoreHandle(STQ* pTq, void* pVal, int vLen, STqHandle* handle){
static int32_t tqMetaRestoreHandle(STQ* pTq, void* pVal, int vLen, STqHandle* handle) {
int32_t vgId = TD_VID(pTq->pVnode);
SDecoder decoder = {0};
int32_t code = TDB_CODE_SUCCESS;
int32_t code = TDB_CODE_SUCCESS;
tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
TQ_ERR_GO_TO_END(tDecodeSTqHandle(&decoder, handle));
@ -281,53 +284,55 @@ END:
return code;
}
int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){
int32_t vgId = TD_VID(pTq->pVnode);
int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle) {
int32_t vgId = TD_VID(pTq->pVnode);
memcpy(handle->subKey, req->subKey, TSDB_SUBSCRIBE_KEY_LEN);
handle->consumerId = req->newConsumerId;
handle->execHandle.subType = req->subType;
handle->fetchMeta = req->withMeta;
if(req->subType == TOPIC_SUB_TYPE__COLUMN){
if (req->subType == TOPIC_SUB_TYPE__COLUMN) {
handle->execHandle.execCol.qmsg = taosStrdup(req->qmsg);
}else if(req->subType == TOPIC_SUB_TYPE__DB){
} else if (req->subType == TOPIC_SUB_TYPE__DB) {
handle->execHandle.execDb.pFilterOutTbUid =
taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
}else if(req->subType == TOPIC_SUB_TYPE__TABLE){
} else if (req->subType == TOPIC_SUB_TYPE__TABLE) {
handle->execHandle.execTb.suid = req->suid;
handle->execHandle.execTb.qmsg = taosStrdup(req->qmsg);
}
handle->snapshotVer = walGetCommittedVer(pTq->pVnode->pWal);
if(tqMetaInitHandle(pTq, handle) < 0){
if (tqMetaInitHandle(pTq, handle) < 0) {
return -1;
}
tqInfo("tqMetaCreateHandle %s consumer 0x%" PRIx64 " vgId:%d, snapshotVer:%" PRId64, handle->subKey, handle->consumerId, vgId, handle->snapshotVer);
tqInfo("tqMetaCreateHandle %s consumer 0x%" PRIx64 " vgId:%d, snapshotVer:%" PRId64, handle->subKey,
handle->consumerId, vgId, handle->snapshotVer);
return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle));
}
static int32_t tqMetaTransformInfo(TDB* pMetaDB, TTB* pOld, TTB* pNew){
TBC* pCur = NULL;
void* pKey = NULL;
int kLen = 0;
void* pVal = NULL;
int vLen = 0;
TXN* txn = NULL;
static int32_t tqMetaTransformInfo(TDB* pMetaDB, TTB* pOld, TTB* pNew) {
TBC* pCur = NULL;
void* pKey = NULL;
int kLen = 0;
void* pVal = NULL;
int vLen = 0;
TXN* txn = NULL;
int32_t code = TDB_CODE_SUCCESS;
int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_GO_TO_END(tdbTbcOpen(pOld, &pCur, NULL));
TQ_ERR_GO_TO_END(tdbBegin(pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_GO_TO_END(
tdbBegin(pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_GO_TO_END(tdbTbcMoveToFirst(pCur));
while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
TQ_ERR_GO_TO_END (tdbTbUpsert(pNew, pKey, kLen, pVal, vLen, txn));
TQ_ERR_GO_TO_END(tdbTbUpsert(pNew, pKey, kLen, pVal, vLen, txn));
}
TQ_ERR_GO_TO_END (tdbCommit(pMetaDB, txn));
TQ_ERR_GO_TO_END (tdbPostCommit(pMetaDB, txn));
TQ_ERR_GO_TO_END(tdbCommit(pMetaDB, txn));
TQ_ERR_GO_TO_END(tdbPostCommit(pMetaDB, txn));
END:
tdbFree(pKey);
@ -338,24 +343,24 @@ END:
int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle) {
void* data = taosHashGet(pTq->pHandle, key, strlen(key));
if(data == NULL){
int vLen = 0;
if (data == NULL) {
int vLen = 0;
if (tdbTbGet(pTq->pExecStore, key, (int)strlen(key), &data, &vLen) < 0) {
tdbFree(data);
return TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST;
}
STqHandle handle = {0};
if (tqMetaRestoreHandle(pTq, data, vLen, &handle) != 0){
if (tqMetaRestoreHandle(pTq, data, vLen, &handle) != 0) {
tdbFree(data);
tqDestroyTqHandle(&handle);
return TSDB_CODE_OUT_OF_MEMORY;
}
tdbFree(data);
*pHandle = taosHashGet(pTq->pHandle, key, strlen(key));
if(*pHandle == NULL){
if (*pHandle == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}else{
} else {
*pHandle = data;
}
return TDB_CODE_SUCCESS;
@ -372,8 +377,8 @@ END:
return code;
}
static int32_t replaceTqPath(char** path){
char* tpath = NULL;
static int32_t replaceTqPath(char** path) {
char* tpath = NULL;
int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_RETURN(tqBuildFName(&tpath, *path, TQ_SUBSCRIBE_NAME));
taosMemoryFree(*path);
@ -382,12 +387,12 @@ static int32_t replaceTqPath(char** path){
}
static int32_t tqMetaRestoreCheckInfo(STQ* pTq) {
TBC* pCur = NULL;
void* pKey = NULL;
int kLen = 0;
void* pVal = NULL;
int vLen = 0;
int32_t code = 0;
TBC* pCur = NULL;
void* pKey = NULL;
int kLen = 0;
void* pVal = NULL;
int vLen = 0;
int32_t code = 0;
STqCheckInfo info = {0};
TQ_ERR_GO_TO_END(tdbTbcOpen(pTq->pCheckStore, &pCur, NULL));
@ -412,10 +417,10 @@ int32_t tqMetaOpen(STQ* pTq) {
char* offsetNew = NULL;
int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_GO_TO_END(tqBuildFName(&maindb, pTq->path, TDB_MAINDB_NAME));
if(!taosCheckExistFile(maindb)){
if (!taosCheckExistFile(maindb)) {
TQ_ERR_GO_TO_END(replaceTqPath(&pTq->path));
TQ_ERR_GO_TO_END(tqMetaOpenTdb(pTq));
}else{
} else {
TQ_ERR_GO_TO_END(tqMetaTransform(pTq));
(void)taosRemoveFile(maindb);
}
@ -435,12 +440,12 @@ END:
}
int32_t tqMetaTransform(STQ* pTq) {
int32_t code = TDB_CODE_SUCCESS;
TDB* pMetaDB = NULL;
TTB* pExecStore = NULL;
TTB* pCheckStore = NULL;
char* offsetNew = NULL;
char* offset = NULL;
int32_t code = TDB_CODE_SUCCESS;
TDB* pMetaDB = NULL;
TTB* pExecStore = NULL;
TTB* pCheckStore = NULL;
char* offsetNew = NULL;
char* offset = NULL;
TQ_ERR_GO_TO_END(tqBuildFName(&offset, pTq->path, TQ_OFFSET_NAME));
TQ_ERR_GO_TO_END(tdbOpen(pTq->path, 16 * 1024, 1, &pMetaDB, 0, 0, NULL));
@ -454,6 +459,7 @@ int32_t tqMetaTransform(STQ* pTq) {
TQ_ERR_GO_TO_END(tqMetaTransformInfo(pTq->pMetaDB, pCheckStore, pTq->pCheckStore));
TQ_ERR_GO_TO_END(tqBuildFName(&offsetNew, pTq->path, TQ_OFFSET_NAME));
if(taosCheckExistFile(offset)) {
if (taosCopyFile(offset, offsetNew) < 0) {
tqError("copy offset file error");
@ -466,7 +472,7 @@ END:
taosMemoryFree(offset);
taosMemoryFree(offsetNew);
//return 0 always, so ignore
// return 0 always, so ignore
(void)tdbTbClose(pExecStore);
(void)tdbTbClose(pCheckStore);
(void)tdbClose(pMetaDB);
@ -486,4 +492,4 @@ int32_t tqMetaClose(STQ* pTq) {
}
(void)tdbClose(pTq->pMetaDB);
return 0;
}
}

34
source/dnode/vnode/src/tq/tqRead.c
View File

@ -188,9 +188,9 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t
int64_t committedVer = walGetCommittedVer(pHandle->pWalReader->pWal);
int64_t appliedVer = walGetAppliedVer(pHandle->pWalReader->pWal);
wDebug("vgId:%d, start to fetch wal, index:%" PRId64 ", last:%" PRId64 " commit:%" PRId64 ", applied:%" PRId64
", 0x%" PRIx64,
vgId, offset, lastVer, committedVer, appliedVer, id);
tqDebug("vgId:%d, start to fetch wal, index:%" PRId64 ", last:%" PRId64 " commit:%" PRId64 ", applied:%" PRId64
", 0x%" PRIx64,
vgId, offset, lastVer, committedVer, appliedVer, id);
while (offset <= appliedVer) {
if (walFetchHead(pHandle->pWalReader, offset) < 0) {
@ -240,14 +240,12 @@ END:
return code;
}
bool tqGetTablePrimaryKey(STqReader* pReader){
return pReader->hasPrimaryKey;
}
bool tqGetTablePrimaryKey(STqReader* pReader) { return pReader->hasPrimaryKey; }
void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid){
bool ret = false;
SSchemaWrapper *schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1);
if (schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY){
void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid) {
bool ret = false;
SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1);
if (schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) {
ret = true;
}
tDeleteSchemaWrapper(schema);
@ -311,10 +309,7 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con
int32_t code = 0;
while (1) {
code = walNextValidMsg(pReader);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
TAOS_CHECK_RETURN(walNextValidMsg(pReader));
SWalCont* pCont = &pReader->pHead->head;
int64_t ver = pCont->version;
@ -374,7 +369,7 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con
}
bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) {
SWalReader* pWalReader = pReader->pWalReader;
SWalReader* pWalReader = pReader->pWalReader;
int64_t st = taosGetTimestampMs();
while (1) {
@ -405,7 +400,7 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) {
pReader->msg.msgStr = NULL;
int64_t elapsed = taosGetTimestampMs() - st;
if(elapsed > 1000 || elapsed < 0){
if (elapsed > 1000 || elapsed < 0) {
return false;
}
@ -681,10 +676,11 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char*
continue;
}
SColData* pCol = taosArrayGet(pCols, sourceIdx);
SColVal colVal;
SColData* pCol = taosArrayGet(pCols, sourceIdx);
SColVal colVal;
tqTrace("lostdata colActual:%d, sourceIdx:%d, targetIdx:%d, numOfCols:%d, source cid:%d, dst cid:%d", colActual, sourceIdx, targetIdx, numOfCols, pCol->cid, pColData->info.colId);
tqTrace("lostdata colActual:%d, sourceIdx:%d, targetIdx:%d, numOfCols:%d, source cid:%d, dst cid:%d", colActual,
sourceIdx, targetIdx, numOfCols, pCol->cid, pColData->info.colId);
if (pCol->cid < pColData->info.colId) {
sourceIdx++;
} else if (pCol->cid == pColData->info.colId) {

14
source/libs/sync/src/syncRaftLog.c
View File

@ -206,17 +206,17 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
SyncIndex index = 0;
SWalSyncInfo syncMeta = {0};
syncMeta.isWeek = pEntry->isWeak;
syncMeta.seqNum = pEntry->seqNum;
syncMeta.term = pEntry->term;
int64_t tsWriteBegin = taosGetTimestampNs();
index = walAppendLog(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen);
int32_t code = walAppendLog(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen);
int64_t tsWriteEnd = taosGetTimestampNs();
int64_t tsElapsed = tsWriteEnd - tsWriteBegin;
if (index < 0) {
if (TSDB_CODE_SUCCESS != code) {
int32_t err = terrno;
const char* errStr = tstrerror(err);
int32_t sysErr = errno;
@ -227,9 +227,11 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr
return -1;
}
ASSERT(pEntry->index == index);
walFsync(pWal, forceSync);
code = walFsync(pWal, forceSync);
if (TSDB_CODE_SUCCESS != code) {
sNError(pData->pSyncNode, "wal fsync failed since %s", tstrerror(code));
TAOS_RETURN(code);
}
sNTrace(pData->pSyncNode, "write index:%" PRId64 ", type:%s, origin type:%s, elapsed:%" PRId64, pEntry->index,
TMSG_INFO(pEntry->msgType), TMSG_INFO(pEntry->originalRpcType), tsElapsed);

16
source/libs/sync/test/syncRaftLogTest.cpp
View File

@ -51,8 +51,14 @@ void test2() {
SWal *pWal = walOpen(gWalPath, &walCfg);
assert(pWal != NULL);
SWalSyncInfo syncMeta = {
.isWeek = -1,
.seqNum = UINT64_MAX,
.term = UINT64_MAX,
};
for (int i = 0; i < 5; ++i) {
int code = walWrite(pWal, i, 100, "aa", 3);
int code = walAppendLog(pWal, i, 100, syncMeta, "aa", 3);
if (code != 0) {
printf("code:%d terror:%d msg:%s i:%d \n", code, terrno, tstrerror(terrno), i);
assert(0);
@ -105,10 +111,16 @@ void test4() {
SWal *pWal = walOpen(gWalPath, &walCfg);
assert(pWal != NULL);
SWalSyncInfo syncMeta = {
.isWeek = -1,
.seqNum = UINT64_MAX,
.term = UINT64_MAX,
};
walRestoreFromSnapshot(pWal, 5);
for (int i = 6; i < 10; ++i) {
int code = walWrite(pWal, i, 100, "aa", 3);
int code = walAppendLog(pWal, i, 100, syncMeta, "aa", 3);
if (code != 0) {
printf("code:%d terror:%d msg:%s i:%d \n", code, terrno, tstrerror(terrno), i);
assert(0);

2
source/libs/tdb/src/db/tdbPager.c
View File

@ -957,6 +957,8 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage
return -1;
}
tmemory_barrier();
pPage->pPager = pPager;
TDB_UNLOCK_PAGE(pPage);

32
source/libs/wal/inc/walInt.h
View File

@ -27,6 +27,15 @@
extern "C" {
#endif
// clang-format off
#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", DEBUG_DEBUG, wDebugFlag, __VA_ARGS__); }}
#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", DEBUG_TRACE, wDebugFlag, __VA_ARGS__); }}
// clang-format on
// meta section begin
typedef struct {
int64_t firstVer;
@ -144,27 +153,22 @@ static inline void walResetVer(SWalVer* pVer) {
pVer->lastVer = -1;
}
int walLoadMeta(SWal* pWal);
int walSaveMeta(SWal* pWal);
int walRemoveMeta(SWal* pWal);
int walRollFileInfo(SWal* pWal);
int32_t walLoadMeta(SWal* pWal);
int32_t walSaveMeta(SWal* pWal);
int32_t walRemoveMeta(SWal* pWal);
int32_t walRollFileInfo(SWal* pWal);
int walCheckAndRepairMeta(SWal* pWal);
int32_t walCheckAndRepairMeta(SWal* pWal);
int walCheckAndRepairIdx(SWal* pWal);
int32_t walCheckAndRepairIdx(SWal* pWal);
char* walMetaSerialize(SWal* pWal);
int walMetaDeserialize(SWal* pWal, const char* bytes);
int32_t walMetaSerialize(SWal* pWal, char** serialized);
int32_t walMetaDeserialize(SWal* pWal, const char* bytes);
// meta section end
// seek section
int64_t walChangeWrite(SWal* pWal, int64_t ver);
int walInitWriteFile(SWal* pWal);
// seek section end
int32_t decryptBody(SWalCfg* cfg, SWalCkHead* pHead, int32_t plainBodyLen, const char* func);
int64_t walGetSeq();
int walSeekWriteVer(SWal* pWal, int64_t ver);
int32_t walRollImpl(SWal* pWal);
#ifdef __cplusplus
}

325
source/libs/wal/src/walMeta.c
View File

@ -46,10 +46,14 @@ static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) {
return sprintf(buf, "%s/meta-ver.tmp", pWal->path);
}
static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
static FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* lastVer) {
int32_t code = 0, lino = 0;
int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
int64_t retVer = -1;
void* ptr = NULL;
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
char fnameStr[WAL_FILE_LEN];
char fnameStr[WAL_FILE_LEN];
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
int64_t fileSize = 0;
@ -58,8 +62,8 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE);
if (pFile == NULL) {
wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
*lastVer = retVer;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
// ensure size as non-negative
@ -73,7 +77,6 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
int64_t readSize = 0;
char* buf = NULL;
int64_t offset = TMIN(pFileInfo->fileSize, fileSize);
int64_t retVer = -1;
int64_t lastEntryBeginOffset = 0;
int64_t lastEntryEndOffset = 0;
int64_t recordLen = 0;
@ -94,25 +97,22 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
readSize = end - offset;
capacity = readSize + sizeof(magic);
void* ptr = taosMemoryRealloc(buf, capacity);
ptr = taosMemoryRealloc(buf, capacity);
if (ptr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err);
}
buf = ptr;
int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET);
if (ret < 0) {
wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), offset);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
}
if (readSize != taosReadFile(pFile, buf, readSize)) {
wError("vgId:%d, failed to read file due to %s. readSize:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno),
readSize, fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
}
char* candidate = NULL;
@ -140,7 +140,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
logContent = (SWalCkHead*)(buf + pos);
if (walValidHeadCksum(logContent) != 0) {
terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH;
code = TSDB_CODE_WAL_CHKSUM_MISMATCH;
wWarn("vgId:%d, failed to validate checksum of wal entry header. offset:%" PRId64 ", file:%s", pWal->cfg.vgId,
offset + pos, fnameStr);
haystack = buf + pos + 1;
@ -163,8 +163,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
capacity += extraSize;
void* ptr = taosMemoryRealloc(buf, capacity);
if (ptr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err);
}
buf = ptr;
}
@ -172,21 +171,25 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
if (ret < 0) {
wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno),
offset);
terrno = TAOS_SYSTEM_ERROR(errno);
code = TAOS_SYSTEM_ERROR(errno);
break;
}
if (extraSize != taosReadFile(pFile, buf + readSize, extraSize)) {
wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", extraSize:%" PRId64 ", file:%s",
pWal->cfg.vgId, strerror(errno), offset + readSize, extraSize, fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
code = TAOS_SYSTEM_ERROR(errno);
break;
}
}
logContent = (SWalCkHead*)(buf + pos);
decryptBody(&pWal->cfg, logContent, logContent->head.bodyLen, __FUNCTION__);
code = decryptBody(&pWal->cfg, logContent, logContent->head.bodyLen, __FUNCTION__);
if (code) {
break;
}
if (walValidBodyCksum(logContent) != 0) {
terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH;
code = TSDB_CODE_WAL_CHKSUM_MISMATCH;
wWarn("vgId:%d, failed to validate checksum of wal entry body. offset:%" PRId64 ", file:%s", pWal->cfg.vgId,
offset + pos, fnameStr);
haystack = buf + pos + 1;
@ -211,7 +214,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
}
if (retVer < 0) {
terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
code = TSDB_CODE_WAL_LOG_NOT_EXIST;
}
// truncate file
@ -221,30 +224,26 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
if (taosFtruncateFile(pFile, lastEntryEndOffset) < 0) {
wError("failed to truncate file due to %s. file:%s", strerror(errno), fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
}
if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pFile) < 0) {
wError("failed to fsync file due to %s. file:%s", strerror(errno), fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
}
}
pFileInfo->fileSize = lastEntryEndOffset;
taosCloseFile(&pFile);
taosMemoryFree(buf);
return retVer;
_err:
taosCloseFile(&pFile);
taosMemoryFree(buf);
return -1;
*lastVer = retVer;
TAOS_RETURN(code);
}
static void walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) {
static int32_t walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) {
int metaFileNum = taosArrayGetSize(metaLogList);
int actualFileNum = taosArrayGetSize(actualLogList);
int j = 0;
@ -270,11 +269,15 @@ static void walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) {
for (int i = 0; i < actualFileNum; i++) {
SWalFileInfo* pFileInfo = taosArrayGet(actualLogList, i);
taosArrayPush(metaLogList, pFileInfo);
if (NULL == taosArrayPush(metaLogList, pFileInfo)) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
}
return TSDB_CODE_SUCCESS;
}
void walAlignVersions(SWal* pWal) {
static void walAlignVersions(SWal* pWal) {
if (pWal->vers.firstVer > pWal->vers.snapshotVer + 1) {
wWarn("vgId:%d, firstVer:%" PRId64 " is larger than snapshotVer:%" PRId64 " + 1. align with it.", pWal->cfg.vgId,
pWal->vers.firstVer, pWal->vers.snapshotVer);
@ -294,7 +297,7 @@ void walAlignVersions(SWal* pWal) {
wInfo("vgId:%d, reset commitVer to %" PRId64, pWal->cfg.vgId, pWal->vers.commitVer);
}
int walRepairLogFileTs(SWal* pWal, bool* updateMeta) {
static int32_t walRepairLogFileTs(SWal* pWal, bool* updateMeta) {
int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
int32_t fileIdx = -1;
int32_t lastCloseTs = 0;
@ -310,9 +313,9 @@ int walRepairLogFileTs(SWal* pWal, bool* updateMeta) {
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
int32_t mtime = 0;
if (taosStatFile(fnameStr, NULL, &mtime, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, failed to stat file due to %s, file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
if (updateMeta != NULL) *updateMeta = true;
@ -321,10 +324,10 @@ int walRepairLogFileTs(SWal* pWal, bool* updateMeta) {
lastCloseTs = pFileInfo->closeTs;
}
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
bool walLogEntriesComplete(const SWal* pWal) {
static int32_t walLogEntriesComplete(const SWal* pWal) {
int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
bool complete = true;
int32_t fileIdx = -1;
@ -346,13 +349,13 @@ bool walLogEntriesComplete(const SWal* pWal) {
wError("vgId:%d, WAL log entries incomplete in range [%" PRId64 ", %" PRId64 "], index:%" PRId64
", snaphotVer:%" PRId64,
pWal->cfg.vgId, pWal->vers.firstVer, pWal->vers.lastVer, index, pWal->vers.snapshotVer);
terrno = TSDB_CODE_WAL_LOG_INCOMPLETE;
TAOS_RETURN(TSDB_CODE_WAL_LOG_INCOMPLETE);
} else {
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
return complete;
}
int walTrimIdxFile(SWal* pWal, int32_t fileIdx) {
static int32_t walTrimIdxFile(SWal* pWal, int32_t fileIdx) {
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
ASSERT(pFileInfo != NULL);
char fnameStr[WAL_FILE_LEN];
@ -364,13 +367,12 @@ int walTrimIdxFile(SWal* pWal, int32_t fileIdx) {
int64_t lastEndOffset = records * sizeof(SWalIdxEntry);
if (fileSize <= lastEndOffset) {
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
wInfo("vgId:%d, trim idx file. file: %s, size: %" PRId64 ", offset: %" PRId64, pWal->cfg.vgId, fnameStr, fileSize,
@ -378,11 +380,13 @@ int walTrimIdxFile(SWal* pWal, int32_t fileIdx) {
taosFtruncateFile(pFile, lastEndOffset);
taosCloseFile(&pFile);
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int walCheckAndRepairMeta(SWal* pWal) {
int32_t walCheckAndRepairMeta(SWal* pWal) {
// load log files, get first/snapshot/last version info
int32_t code = 0;
const char* logPattern = "^[0-9]+.log$";
const char* idxPattern = "^[0-9]+.idx$";
regex_t logRegPattern;
@ -396,7 +400,7 @@ int walCheckAndRepairMeta(SWal* pWal) {
regfree(&logRegPattern);
regfree(&idxRegPattern);
wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno));
return -1;
TAOS_RETURN(TSDB_CODE_FAILED);
}
SArray* actualLog = taosArrayInit(8, sizeof(SWalFileInfo));
@ -428,8 +432,11 @@ int walCheckAndRepairMeta(SWal* pWal) {
bool updateMeta = (metaFileNum != actualFileNum);
// rebuild meta of file info
walRebuildFileInfoSet(pWal->fileInfoSet, actualLog);
code = walRebuildFileInfoSet(pWal->fileInfoSet, actualLog);
taosArrayDestroy(actualLog);
if (code) {
TAOS_RETURN(code);
}
int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
@ -444,9 +451,9 @@ int walCheckAndRepairMeta(SWal* pWal) {
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
int32_t code = taosStatFile(fnameStr, &fileSize, NULL, NULL);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("failed to stat file since %s. file:%s", terrstr(), fnameStr);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
if (pFileInfo->lastVer >= pFileInfo->firstVer && fileSize == pFileInfo->fileSize) {
@ -455,16 +462,20 @@ int walCheckAndRepairMeta(SWal* pWal) {
}
updateMeta = true;
(void)walTrimIdxFile(pWal, fileIdx);
TAOS_CHECK_RETURN(walTrimIdxFile(pWal, fileIdx));
int64_t lastVer = walScanLogGetLastVer(pWal, fileIdx);
int64_t lastVer = -1;
code = walScanLogGetLastVer(pWal, fileIdx, &lastVer);
if (lastVer < 0) {
if (terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) {
if (code != TSDB_CODE_WAL_LOG_NOT_EXIST) {
wError("failed to scan wal last ver since %s", terrstr());
return -1;
TAOS_RETURN(code);
}
// empty log file
lastVer = pFileInfo->firstVer - 1;
code = TSDB_CODE_SUCCESS;
}
// update lastVer
@ -481,45 +492,39 @@ int walCheckAndRepairMeta(SWal* pWal) {
pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer;
pWal->vers.lastVer = ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer;
}
(void)walAlignVersions(pWal);
walAlignVersions(pWal);
// repair ts of files
if (walRepairLogFileTs(pWal, &updateMeta) < 0) {
return -1;
}
TAOS_CHECK_RETURN(walRepairLogFileTs(pWal, &updateMeta));
// update meta file
if (updateMeta) {
(void)walSaveMeta(pWal);
TAOS_CHECK_RETURN(walSaveMeta(pWal));
}
if (!walLogEntriesComplete(pWal)) {
return -1;
}
TAOS_CHECK_RETURN(walLogEntriesComplete(pWal));
return 0;
return code;
}
int walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) {
static int32_t walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) {
if (taosLSeekFile(pLogFile, offset, SEEK_SET) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
if (taosReadFile(pLogFile, pCkHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
if (walValidHeadCksum(pCkHead) != 0) {
terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH;
return -1;
TAOS_RETURN(TSDB_CODE_WAL_CHKSUM_MISMATCH);
}
return 0;
return TSDB_CODE_SUCCESS;
}
int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
static int32_t walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
int32_t code = 0, lino = 0;
int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
char fnameStr[WAL_FILE_LEN];
@ -530,12 +535,12 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
if (taosStatFile(fnameStr, &fileSize, NULL, NULL) < 0 && errno != ENOENT) {
wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
if (fileSize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) {
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
// start to repair
@ -550,15 +555,15 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE);
if (pIdxFile == NULL) {
wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
}
pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ);
if (pLogFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr());
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
}
// determine the last valid entry end, i.e. offset
@ -566,15 +571,15 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
if (taosLSeekFile(pIdxFile, offset, SEEK_SET) < 0) {
wError("vgId:%d, failed to seek file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno),
offset, fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
}
if (taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno),
offset, fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
}
if (idxEntry.ver > pFileInfo->lastVer) {
@ -602,19 +607,19 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
// ftruncate idx file
if (offset < fileSize) {
if (taosFtruncateFile(pIdxFile, offset) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, failed to ftruncate file since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(),
offset, fnameStr);
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
}
}
// rebuild idx file
if (taosLSeekFile(pIdxFile, 0, SEEK_END) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, failed to seek file since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(), offset,
fnameStr);
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
}
int64_t count = 0;
@ -630,23 +635,25 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
}
idxEntry.offset += sizeof(SWalCkHead) + cryptedBodyLen;
if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) {
code = walReadLogHead(pLogFile, idxEntry.offset, &ckHead);
if (code) {
wError("vgId:%d, failed to read wal log head since %s. index:%" PRId64 ", offset:%" PRId64 ", file:%s",
pWal->cfg.vgId, terrstr(), idxEntry.ver, idxEntry.offset, fLogNameStr);
goto _err;
TAOS_CHECK_GOTO(code, &lino, _err);
}
if (pWal->cfg.level != TAOS_WAL_SKIP && taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, failed to append file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr);
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
}
count++;
}
if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pIdxFile) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, faild to fsync file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr);
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
}
if (count > 0) {
@ -654,14 +661,15 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
pFileInfo->lastVer);
}
(void)taosCloseFile(&pLogFile);
(void)taosCloseFile(&pIdxFile);
return 0;
_err:
if (code) {
wError("vgId:%d, %s failed at line %d since %s", pWal->cfg.vgId, __func__, lino, tstrerror(code));
}
(void)taosCloseFile(&pLogFile);
(void)taosCloseFile(&pIdxFile);
return -1;
TAOS_RETURN(code);
}
int64_t walGetVerRetention(SWal* pWal, int64_t bytes) {
@ -681,19 +689,24 @@ int64_t walGetVerRetention(SWal* pWal, int64_t bytes) {
return ver + 1;
}
int walCheckAndRepairIdx(SWal* pWal) {
int32_t walCheckAndRepairIdx(SWal* pWal) {
int32_t code = 0;
int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
int32_t fileIdx = sz;
while (--fileIdx >= 0) {
if (walCheckAndRepairIdxFile(pWal, fileIdx) < 0) {
code = walCheckAndRepairIdxFile(pWal, fileIdx);
if (code) {
wError("vgId:%d, failed to repair idx file since %s. fileIdx:%d", pWal->cfg.vgId, terrstr(), fileIdx);
return -1;
TAOS_RETURN(code);
}
}
return 0;
TAOS_RETURN(code);
}
int walRollFileInfo(SWal* pWal) {
int32_t walRollFileInfo(SWal* pWal) {
int64_t ts = taosGetTimestampSec();
SArray* pArray = pWal->fileInfoSet;
@ -706,8 +719,7 @@ int walRollFileInfo(SWal* pWal) {
// TODO: change to emplace back
SWalFileInfo* pNewInfo = taosMemoryMalloc(sizeof(SWalFileInfo));
if (pNewInfo == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
pNewInfo->firstVer = pWal->vers.lastVer + 1;
pNewInfo->lastVer = -1;
@ -717,10 +729,11 @@ int walRollFileInfo(SWal* pWal) {
pNewInfo->syncedOffset = 0;
taosArrayPush(pArray, pNewInfo);
taosMemoryFree(pNewInfo);
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
char* walMetaSerialize(SWal* pWal) {
int32_t walMetaSerialize(SWal* pWal, char** serialized) {
char buf[30];
int sz = taosArrayGetSize(pWal->fileInfoSet);
cJSON* pRoot = cJSON_CreateObject();
@ -737,8 +750,8 @@ char* walMetaSerialize(SWal* pWal) {
if (pFiles) {
cJSON_Delete(pFiles);
}
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
cJSON_AddItemToObject(pRoot, "meta", pMeta);
sprintf(buf, "%" PRId64, pWal->vers.firstVer);
@ -757,7 +770,8 @@ char* walMetaSerialize(SWal* pWal) {
cJSON_AddItemToArray(pFiles, pField = cJSON_CreateObject());
if (pField == NULL) {
cJSON_Delete(pRoot);
return NULL;
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
// cjson only support int32_t or double
// string are used to prohibit the loss of precision
@ -772,12 +786,15 @@ char* walMetaSerialize(SWal* pWal) {
sprintf(buf, "%" PRId64, pInfo->fileSize);
cJSON_AddStringToObject(pField, "fileSize", buf);
}
char* serialized = cJSON_Print(pRoot);
char* pSerialized = cJSON_Print(pRoot);
cJSON_Delete(pRoot);
return serialized;
*serialized = pSerialized;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int walMetaDeserialize(SWal* pWal, const char* bytes) {
int32_t walMetaDeserialize(SWal* pWal, const char* bytes) {
/*A(taosArrayGetSize(pWal->fileInfoSet) == 0);*/
cJSON *pRoot, *pMeta, *pFiles, *pInfoJson, *pField;
pRoot = cJSON_Parse(bytes);
@ -829,11 +846,11 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) {
pWal->fileInfoSet = pArray;
pWal->writeCur = sz - 1;
cJSON_Delete(pRoot);
return 0;
return TSDB_CODE_SUCCESS;
_err:
cJSON_Delete(pRoot);
return -1;
return TSDB_CODE_FAILED;
}
static int walFindCurMetaVer(SWal* pWal) {
@ -866,13 +883,14 @@ static int walFindCurMetaVer(SWal* pWal) {
return metaVer;
}
void walUpdateSyncedOffset(SWal* pWal) {
static void walUpdateSyncedOffset(SWal* pWal) {
SWalFileInfo* pFileInfo = walGetCurFileInfo(pWal);
if (pFileInfo == NULL) return;
pFileInfo->syncedOffset = pFileInfo->fileSize;
}
int walSaveMeta(SWal* pWal) {
int32_t walSaveMeta(SWal* pWal) {
int code = 0, lino = 0;
int metaVer = walFindCurMetaVer(pWal);
char fnameStr[WAL_FILE_LEN];
char tmpFnameStr[WAL_FILE_LEN];
@ -881,63 +899,64 @@ int walSaveMeta(SWal* pWal) {
// fsync the idx and log file at first to ensure validity of meta
if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pWal->pIdxFile) < 0) {
wError("vgId:%d, failed to sync idx file due to %s", pWal->cfg.vgId, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pWal->pLogFile) < 0) {
wError("vgId:%d, failed to sync log file due to %s", pWal->cfg.vgId, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
// update synced offset
(void)walUpdateSyncedOffset(pWal);
walUpdateSyncedOffset(pWal);
// flush to a tmpfile
n = walBuildTmpMetaName(pWal, tmpFnameStr);
if (n >= sizeof(tmpFnameStr)) {
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
TdFilePtr pMetaFile =
taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pMetaFile == NULL) {
wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
char* serialized = walMetaSerialize(pWal);
int len = strlen(serialized);
char* serialized = NULL;
TAOS_CHECK_RETURN(walMetaSerialize(pWal, &serialized));
int len = strlen(serialized);
if (pWal->cfg.level != TAOS_WAL_SKIP && len != taosWriteFile(pMetaFile, serialized, len)) {
wError("vgId:%d, failed to write file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
}
if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pMetaFile) < 0) {
wError("vgId:%d, failed to sync file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
}
if (taosCloseFile(&pMetaFile) < 0) {
wError("vgId:%d, failed to close file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
}
// rename it
n = walBuildMetaName(pWal, metaVer + 1, fnameStr);
if (n >= sizeof(fnameStr)) {
goto _err;
TAOS_CHECK_GOTO(TSDB_CODE_FAILED, &lino, _err);
}
if (taosRenameFile(tmpFnameStr, fnameStr) < 0) {
wError("failed to rename file due to %s. dest:%s", strerror(errno), fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
}
// delete old file
@ -945,21 +964,24 @@ int walSaveMeta(SWal* pWal) {
walBuildMetaName(pWal, metaVer, fnameStr);
taosRemoveFile(fnameStr);
}
taosMemoryFree(serialized);
return 0;
return code;
_err:
taosCloseFile(&pMetaFile);
taosMemoryFree(serialized);
return -1;
return code;
}
int walLoadMeta(SWal* pWal) {
int32_t walLoadMeta(SWal* pWal) {
int32_t code = 0;
// find existing meta file
int metaVer = walFindCurMetaVer(pWal);
if (metaVer == -1) {
wDebug("vgId:%d, wal find meta ver %d", pWal->cfg.vgId, metaVer);
return -1;
TAOS_RETURN(TSDB_CODE_FAILED);
}
char fnameStr[WAL_FILE_LEN];
walBuildMetaName(pWal, metaVer, fnameStr);
@ -969,39 +991,40 @@ int walLoadMeta(SWal* pWal) {
if (fileSize == 0) {
(void)taosRemoveFile(fnameStr);
wDebug("vgId:%d, wal find empty meta ver %d", pWal->cfg.vgId, metaVer);
return -1;
TAOS_RETURN(TSDB_CODE_FAILED);
}
int size = (int)fileSize;
char* buf = taosMemoryMalloc(size + 5);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
memset(buf, 0, size + 5);
TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pFile == NULL) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
taosMemoryFree(buf);
return -1;
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
}
if (taosReadFile(pFile, buf, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFile);
taosMemoryFree(buf);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
// load into fileInfoSet
int code = walMetaDeserialize(pWal, buf);
code = walMetaDeserialize(pWal, buf);
if (code < 0) {
wError("failed to deserialize wal meta. file:%s", fnameStr);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
code = TSDB_CODE_WAL_FILE_CORRUPTED;
}
taosCloseFile(&pFile);
taosMemoryFree(buf);
return code;
TAOS_RETURN(code);
}
int walRemoveMeta(SWal* pWal) {
int32_t walRemoveMeta(SWal* pWal) {
int metaVer = walFindCurMetaVer(pWal);
if (metaVer == -1) return 0;
char fnameStr[WAL_FILE_LEN];

29
source/libs/wal/src/walMgmt.c
View File

@ -46,10 +46,11 @@ int32_t walInit() {
tsWal.refSetId = taosOpenRef(TSDB_MIN_VNODES, walFreeObj);
int32_t code = walCreateThread();
if (code != 0) {
if (TSDB_CODE_SUCCESS != code) {
wError("failed to init wal module since %s", tstrerror(code));
atomic_store_8(&tsWal.inited, 0);
return code;
TAOS_RETURN(code);
}
wInfo("wal module is initialized, rsetId:%d", tsWal.refSetId);
@ -171,19 +172,20 @@ _err:
taosArrayDestroy(pWal->fileInfoSet);
taosHashCleanup(pWal->pRefHash);
taosThreadMutexDestroy(&pWal->mutex);
taosMemoryFree(pWal);
pWal = NULL;
taosMemoryFreeClear(pWal);
return NULL;
}
int32_t walAlter(SWal *pWal, SWalCfg *pCfg) {
if (pWal == NULL) return TSDB_CODE_APP_ERROR;
if (pWal == NULL) TAOS_RETURN(TSDB_CODE_APP_ERROR);
if (pWal->cfg.level == pCfg->level && pWal->cfg.fsyncPeriod == pCfg->fsyncPeriod &&
pWal->cfg.retentionPeriod == pCfg->retentionPeriod && pWal->cfg.retentionSize == pCfg->retentionSize) {
wDebug("vgId:%d, walLevel:%d fsync:%d walRetentionPeriod:%d walRetentionSize:%" PRId64 " not change",
pWal->cfg.vgId, pWal->cfg.level, pWal->cfg.fsyncPeriod, pWal->cfg.retentionPeriod, pWal->cfg.retentionSize);
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
wInfo("vgId:%d, change old walLevel:%d fsync:%d walRetentionPeriod:%d walRetentionSize:%" PRId64
@ -199,14 +201,17 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) {
pWal->fsyncSeq = pCfg->fsyncPeriod / 1000;
if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1;
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t walPersist(SWal *pWal) {
int32_t code = 0;
taosThreadMutexLock(&pWal->mutex);
int32_t ret = walSaveMeta(pWal);
code = walSaveMeta(pWal);
taosThreadMutexUnlock(&pWal->mutex);
return ret;
TAOS_RETURN(code);
}
void walClose(SWal *pWal) {
@ -301,14 +306,14 @@ static int32_t walCreateThread() {
if (taosThreadCreate(&tsWal.thread, &thAttr, walThreadFunc, NULL) != 0) {
wError("failed to create wal thread since %s", strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
taosThreadAttrDestroy(&thAttr);
wDebug("wal thread is launched, thread:0x%08" PRIx64, taosGetPthreadId(tsWal.thread));
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
static void walStopThread() {

272
source/libs/wal/src/walRead.c
View File

@ -26,7 +26,7 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond, int64_t id) {
}
pReader->pWal = pWal;
pReader->readerId = (id != 0)? id:tGenIdPI64();
pReader->readerId = (id != 0) ? id : tGenIdPI64();
pReader->pIdxFile = NULL;
pReader->pLogFile = NULL;
pReader->curVersion = -1;
@ -35,7 +35,7 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond, int64_t id) {
if (cond) {
pReader->cond = *cond;
} else {
// pReader->cond.scanUncommited = 0;
// pReader->cond.scanUncommited = 0;
pReader->cond.scanNotApplied = 0;
pReader->cond.scanMeta = 0;
pReader->cond.enableRef = 0;
@ -58,7 +58,7 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond, int64_t id) {
}
void walCloseReader(SWalReader *pReader) {
if(pReader == NULL) return;
if (pReader == NULL) return;
taosCloseFile(&pReader->pIdxFile);
taosCloseFile(&pReader->pLogFile);
@ -75,30 +75,24 @@ int32_t walNextValidMsg(SWalReader *pReader) {
wDebug("vgId:%d, wal start to fetch, index:%" PRId64 ", last:%" PRId64 " commit:%" PRId64 ", applied:%" PRId64,
pReader->pWal->cfg.vgId, fetchVer, lastVer, committedVer, appliedVer);
if (fetchVer > appliedVer) {
terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
return -1;
TAOS_RETURN(TSDB_CODE_WAL_LOG_NOT_EXIST);
}
while (fetchVer <= appliedVer) {
if (walFetchHead(pReader, fetchVer) < 0) {
return -1;
}
TAOS_CHECK_RETURN(walFetchHead(pReader, fetchVer));
int32_t type = pReader->pHead->head.msgType;
if (type == TDMT_VND_SUBMIT || ((type == TDMT_VND_DELETE) && (pReader->cond.deleteMsg == 1)) ||
(IS_META_MSG(type) && pReader->cond.scanMeta)) {
int32_t code = walFetchBody(pReader);
return (code == TSDB_CODE_SUCCESS)? 0:-1;
TAOS_RETURN(walFetchBody(pReader));
} else {
if (walSkipFetchBody(pReader) < 0) {
return -1;
}
TAOS_CHECK_RETURN(walSkipFetchBody(pReader));
fetchVer = pReader->curVersion;
}
}
return -1;
TAOS_RETURN(TSDB_CODE_FAILED);
}
int64_t walReaderGetCurrentVer(const SWalReader *pReader) { return pReader->curVersion; }
@ -119,18 +113,18 @@ void walReaderValidVersionRange(SWalReader *pReader, int64_t *sver, int64_t *eve
*ever = pReader->cond.scanUncommited ? lastVer : committedVer;
}
void walReaderVerifyOffset(SWalReader *pWalReader, STqOffsetVal* pOffset){
void walReaderVerifyOffset(SWalReader *pWalReader, STqOffsetVal *pOffset) {
// if offset version is small than first version , let's seek to first version
taosThreadMutexLock(&pWalReader->pWal->mutex);
int64_t firstVer = walGetFirstVer((pWalReader)->pWal);
taosThreadMutexUnlock(&pWalReader->pWal->mutex);
if (pOffset->version < firstVer){
if (pOffset->version < firstVer) {
pOffset->version = firstVer;
}
}
static int64_t walReadSeekFilePos(SWalReader *pReader, int64_t fileFirstVer, int64_t ver) {
static int32_t walReadSeekFilePos(SWalReader *pReader, int64_t fileFirstVer, int64_t ver) {
int64_t ret = 0;
TdFilePtr pIdxTFile = pReader->pIdxFile;
@ -140,32 +134,34 @@ static int64_t walReadSeekFilePos(SWalReader *pReader, int64_t fileFirstVer, int
int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry);
ret = taosLSeekFile(pIdxTFile, offset, SEEK_SET);
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, failed to seek idx file, index:%" PRId64 ", pos:%" PRId64 ", since %s", pReader->pWal->cfg.vgId,
ver, offset, terrstr());
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
SWalIdxEntry entry = {0};
if ((ret = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry))) != sizeof(SWalIdxEntry)) {
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, failed to read idx file, since %s", pReader->pWal->cfg.vgId, terrstr());
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
wError("vgId:%d, read idx file incompletely, read bytes %" PRId64 ", bytes should be %ld",
pReader->pWal->cfg.vgId, ret, sizeof(SWalIdxEntry));
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
}
return -1;
}
ret = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET);
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, failed to seek log file, index:%" PRId64 ", pos:%" PRId64 ", since %s", pReader->pWal->cfg.vgId,
ver, entry.offset, terrstr());
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
return ret;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
static int32_t walReadChangeFile(SWalReader *pReader, int64_t fileFirstVer) {
@ -177,9 +173,9 @@ static int32_t walReadChangeFile(SWalReader *pReader, int64_t fileFirstVer) {
walBuildLogName(pReader->pWal, fileFirstVer, fnameStr);
TdFilePtr pLogFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pLogFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, cannot open file %s, since %s", pReader->pWal->cfg.vgId, fnameStr, terrstr());
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
pReader->pLogFile = pLogFile;
@ -187,19 +183,19 @@ static int32_t walReadChangeFile(SWalReader *pReader, int64_t fileFirstVer) {
walBuildIdxName(pReader->pWal, fileFirstVer, fnameStr);
TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pIdxFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, cannot open file %s, since %s", pReader->pWal->cfg.vgId, fnameStr, terrstr());
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
pReader->pIdxFile = pIdxFile;
pReader->curFileFirstVer = fileFirstVer;
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) {
static int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) {
SWal *pWal = pReader->pWal;
// bsearch in fileSet
@ -208,51 +204,45 @@ int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) {
SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
if (pRet == NULL) {
wError("failed to find WAL log file with ver:%" PRId64, ver);
terrno = TSDB_CODE_WAL_INVALID_VER;
return -1;
TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER);
}
if (pReader->curFileFirstVer != pRet->firstVer) {
// error code was set inner
if (walReadChangeFile(pReader, pRet->firstVer) < 0) {
return -1;
}
TAOS_CHECK_RETURN(walReadChangeFile(pReader, pRet->firstVer));
}
// error code was set inner
if (walReadSeekFilePos(pReader, pRet->firstVer, ver) < 0) {
return -1;
}
TAOS_CHECK_RETURN(walReadSeekFilePos(pReader, pRet->firstVer, ver));
wDebug("vgId:%d, wal version reset from %" PRId64 " to %" PRId64, pReader->pWal->cfg.vgId,
pReader->curVersion, ver);
wDebug("vgId:%d, wal version reset from %" PRId64 " to %" PRId64, pReader->pWal->cfg.vgId, pReader->curVersion, ver);
pReader->curVersion = ver;
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t walReaderSeekVer(SWalReader *pReader, int64_t ver) {
SWal *pWal = pReader->pWal;
if (ver == pReader->curVersion) {
wDebug("vgId:%d, wal index:%" PRId64 " match, no need to reset", pReader->pWal->cfg.vgId, ver);
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) {
wInfo("vgId:%d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pReader->pWal->cfg.vgId,
ver, pWal->vers.firstVer, pWal->vers.lastVer);
terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
return -1;
ver, pWal->vers.firstVer, pWal->vers.lastVer);
TAOS_RETURN(TSDB_CODE_WAL_LOG_NOT_EXIST);
}
if (walReadSeekVerImpl(pReader, ver) < 0) {
return -1;
}
TAOS_CHECK_RETURN(walReadSeekVerImpl(pReader, ver));
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t walFetchHead(SWalReader *pRead, int64_t ver) {
int64_t code;
int64_t contLen;
@ -260,14 +250,12 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver) {
// TODO: valid ver
if (ver > pRead->pWal->vers.commitVer) {
return -1;
TAOS_RETURN(TSDB_CODE_FAILED);
}
if (pRead->curVersion != ver) {
code = walReaderSeekVer(pRead, ver);
if (code < 0) {
return -1;
}
TAOS_CHECK_RETURN(walReaderSeekVer(pRead, ver));
seeked = true;
}
@ -276,53 +264,51 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver) {
if (contLen == sizeof(SWalCkHead)) {
break;
} else if (contLen == 0 && !seeked) {
if(walReadSeekVerImpl(pRead, ver) < 0){
return -1;
}
TAOS_CHECK_RETURN(walReadSeekVerImpl(pRead, ver));
seeked = true;
continue;
} else {
if (contLen < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
}
return -1;
}
}
code = walValidHeadCksum(pRead->pHead);
if (code != 0) {
wError("vgId:%d, unexpected wal log index:%" PRId64 ", since head checksum not passed, 0x%"PRIx64, pRead->pWal->cfg.vgId, ver,
pRead->readerId);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
wError("vgId:%d, unexpected wal log index:%" PRId64 ", since head checksum not passed, 0x%" PRIx64,
pRead->pWal->cfg.vgId, ver, pRead->readerId);
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
}
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t walSkipFetchBody(SWalReader *pRead) {
wDebug("vgId:%d, skip:%" PRId64 ", first:%" PRId64 ", commit:%" PRId64 ", last:%" PRId64
", applied:%" PRId64 ", 0x%" PRIx64,
wDebug("vgId:%d, skip:%" PRId64 ", first:%" PRId64 ", commit:%" PRId64 ", last:%" PRId64 ", applied:%" PRId64
", 0x%" PRIx64,
pRead->pWal->cfg.vgId, pRead->pHead->head.version, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer,
pRead->pWal->vers.lastVer, pRead->pWal->vers.appliedVer, pRead->readerId);
int32_t plainBodyLen = pRead->pHead->head.bodyLen;
int32_t cryptedBodyLen = plainBodyLen;
//TODO: dmchen emun
if(pRead->pWal->cfg.encryptAlgorithm == 1){
// TODO: dmchen emun
if (pRead->pWal->cfg.encryptAlgorithm == 1) {
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
}
int64_t code = taosLSeekFile(pRead->pLogFile, cryptedBodyLen, SEEK_CUR);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
pRead->curVersion++;
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t walFetchBody(SWalReader *pRead) {
@ -339,16 +325,15 @@ int32_t walFetchBody(SWalReader *pRead) {
int32_t plainBodyLen = pReadHead->bodyLen;
int32_t cryptedBodyLen = plainBodyLen;
//TODO: dmchen emun
if(pRead->pWal->cfg.encryptAlgorithm == 1){
// TODO: dmchen emun
if (pRead->pWal->cfg.encryptAlgorithm == 1) {
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
}
if (pRead->capacity < cryptedBodyLen) {
SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + cryptedBodyLen);
if (ptr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
pRead->pHead = ptr;
pReadHead = &pRead->pHead->head;
@ -357,62 +342,68 @@ int32_t walFetchBody(SWalReader *pRead) {
if (cryptedBodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, cryptedBodyLen)) {
if (plainBodyLen < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s, 0x%"PRIx64,
vgId, pReadHead->version, ver, tstrerror(terrno), id);
wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s, 0x%" PRIx64, vgId,
pReadHead->version, ver, tstrerror(terrno), id);
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} else {
wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since file corrupted, 0x%"PRIx64,
wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64
", since file corrupted, 0x%" PRIx64,
vgId, pReadHead->version, ver, id);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
}
return -1;
}
if (pReadHead->version != ver) {
wError("vgId:%d, wal fetch body error, index:%" PRId64 ", read request index:%" PRId64", 0x%"PRIx64, vgId,
wError("vgId:%d, wal fetch body error, index:%" PRId64 ", read request index:%" PRId64 ", 0x%" PRIx64, vgId,
pReadHead->version, ver, id);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
}
decryptBody(&pRead->pWal->cfg, pRead->pHead, plainBodyLen, __FUNCTION__);
TAOS_CHECK_RETURN(decryptBody(&pRead->pWal->cfg, pRead->pHead, plainBodyLen, __FUNCTION__));
if (walValidBodyCksum(pRead->pHead) != 0) {
wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed, 0x%" PRIx64, vgId, ver, id);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed, 0x%" PRIx64, vgId, ver,
id);
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
}
pRead->curVersion++;
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t walReadVer(SWalReader *pReader, int64_t ver) {
wDebug("vgId:%d, wal start to read index:%" PRId64, pReader->pWal->cfg.vgId, ver);
int64_t contLen;
int32_t code;
int32_t code = 0;
bool seeked = false;
if (walIsEmpty(pReader->pWal)) {
terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
return -1;
TAOS_RETURN(TSDB_CODE_WAL_LOG_NOT_EXIST);
}
if (ver > pReader->pWal->vers.lastVer || ver < pReader->pWal->vers.firstVer) {
wDebug("vgId:%d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pReader->pWal->cfg.vgId,
ver, pReader->pWal->vers.firstVer, pReader->pWal->vers.lastVer);
terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
return -1;
TAOS_RETURN(TSDB_CODE_WAL_LOG_NOT_EXIST);
}
taosThreadMutexLock(&pReader->mutex);
if (pReader->curVersion != ver) {
if (walReaderSeekVer(pReader, ver) < 0) {
code = walReaderSeekVer(pReader, ver);
if (code) {
wError("vgId:%d, unexpected wal log, index:%" PRId64 ", since %s", pReader->pWal->cfg.vgId, ver, terrstr());
taosThreadMutexUnlock(&pReader->mutex);
return -1;
TAOS_RETURN(code);
}
seeked = true;
}
@ -421,22 +412,24 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
if (contLen == sizeof(SWalCkHead)) {
break;
} else if (contLen == 0 && !seeked) {
if(walReadSeekVerImpl(pReader, ver) < 0){
code = walReadSeekVerImpl(pReader, ver);
if (code) {
taosThreadMutexUnlock(&pReader->mutex);
return -1;
TAOS_RETURN(code);
}
seeked = true;
continue;
} else {
if (contLen < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
} else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
}
wError("vgId:%d, failed to read WAL record head, index:%" PRId64 ", from log file since %s",
pReader->pWal->cfg.vgId, ver, terrstr());
taosThreadMutexUnlock(&pReader->mutex);
return -1;
if (contLen < 0) {
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} else {
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
}
}
}
@ -444,54 +437,57 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
if (code != 0) {
wError("vgId:%d, unexpected wal log, index:%" PRId64 ", since head checksum not passed", pReader->pWal->cfg.vgId,
ver);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
taosThreadMutexUnlock(&pReader->mutex);
return -1;
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
}
int32_t plainBodyLen = pReader->pHead->head.bodyLen;
int32_t cryptedBodyLen = plainBodyLen;
//TODO: dmchen emun
if(pReader->pWal->cfg.encryptAlgorithm == 1){
// TODO: dmchen emun
if (pReader->pWal->cfg.encryptAlgorithm == 1) {
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
}
if (pReader->capacity < cryptedBodyLen) {
SWalCkHead *ptr =
(SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + cryptedBodyLen);
SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + cryptedBodyLen);
if (ptr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosThreadMutexUnlock(&pReader->mutex);
return -1;
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
pReader->pHead = ptr;
pReader->capacity = cryptedBodyLen;
}
if ((contLen = taosReadFile(pReader->pLogFile, pReader->pHead->head.body, cryptedBodyLen)) !=
cryptedBodyLen) {
if (contLen < 0)
terrno = TAOS_SYSTEM_ERROR(errno);
else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
}
if ((contLen = taosReadFile(pReader->pLogFile, pReader->pHead->head.body, cryptedBodyLen)) != cryptedBodyLen) {
wError("vgId:%d, failed to read WAL record body, index:%" PRId64 ", from log file since %s",
pReader->pWal->cfg.vgId, ver, terrstr());
taosThreadMutexUnlock(&pReader->mutex);
return -1;
if (contLen < 0) {
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} else {
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
}
}
if (pReader->pHead->head.version != ver) {
wError("vgId:%d, unexpected wal log, index:%" PRId64 ", read request index:%" PRId64, pReader->pWal->cfg.vgId,
pReader->pHead->head.version, ver);
// pReader->curInvalid = 1;
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
// pReader->curInvalid = 1;
taosThreadMutexUnlock(&pReader->mutex);
return -1;
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
}
decryptBody(&pReader->pWal->cfg, pReader->pHead, plainBodyLen, __FUNCTION__);
code = decryptBody(&pReader->pWal->cfg, pReader->pHead, plainBodyLen, __FUNCTION__);
if (code) {
taosThreadMutexUnlock(&pReader->mutex);
TAOS_RETURN(code);
}
code = walValidBodyCksum(pReader->pHead);
if (code != 0) {
@ -500,23 +496,27 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
uint32_t readCkSum = walCalcBodyCksum(pReader->pHead->head.body, plainBodyLen);
uint32_t logCkSum = pReader->pHead->cksumBody;
wError("checksum written into log:%u, checksum calculated:%u", logCkSum, readCkSum);
// pReader->curInvalid = 1;
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
// pReader->curInvalid = 1;
taosThreadMutexUnlock(&pReader->mutex);
return -1;
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
}
pReader->curVersion++;
taosThreadMutexUnlock(&pReader->mutex);
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
void decryptBody(SWalCfg* cfg, SWalCkHead* pHead, int32_t plainBodyLen, const char* func) {
//TODO: dmchen emun
int32_t decryptBody(SWalCfg *cfg, SWalCkHead *pHead, int32_t plainBodyLen, const char *func) {
// TODO: dmchen emun
if (cfg->encryptAlgorithm == 1) {
int32_t cryptedBodyLen = ENCRYPTED_LEN(plainBodyLen);
char *newBody = taosMemoryMalloc(cryptedBodyLen);
if (!newBody) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
SCryptOpts opts;
opts.len = cryptedBodyLen;
@ -527,12 +527,14 @@ void decryptBody(SWalCfg* cfg, SWalCkHead* pHead, int32_t plainBodyLen, const ch
int32_t count = CBC_Decrypt(&opts);
//wDebug("CBC_Decrypt cryptedBodyLen:%d, plainBodyLen:%d, %s", count, plainBodyLen, func);
// wDebug("CBC_Decrypt cryptedBodyLen:%d, plainBodyLen:%d, %s", count, plainBodyLen, func);
memcpy(pHead->head.body, newBody, plainBodyLen);
taosMemoryFree(newBody);
}
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
void walReadReset(SWalReader *pReader) {

49
source/libs/wal/src/walRef.c
View File

@ -22,27 +22,39 @@
SWalRef *walOpenRef(SWal *pWal) {
SWalRef *pRef = taosMemoryCalloc(1, sizeof(SWalRef));
if (pRef == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pRef->refId = tGenIdPI64();
if (taosHashPut(pWal->pRefHash, &pRef->refId, sizeof(int64_t), &pRef, sizeof(void *))) {
taosMemoryFree(pRef);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pRef->refVer = -1;
// pRef->refFile = -1;
pRef->pWal = pWal;
taosHashPut(pWal->pRefHash, &pRef->refId, sizeof(int64_t), &pRef, sizeof(void *));
return pRef;
}
void walCloseRef(SWal *pWal, int64_t refId) {
SWalRef **ppRef = taosHashGet(pWal->pRefHash, &refId, sizeof(int64_t));
if (ppRef == NULL) return;
SWalRef *pRef = *ppRef;
if (pRef) {
wDebug("vgId:%d, wal close ref %" PRId64 ", refId %" PRId64, pWal->cfg.vgId, pRef->refVer, pRef->refId);
} else {
wDebug("vgId:%d, wal close ref null, refId %" PRId64, pWal->cfg.vgId, refId);
if (ppRef) {
SWalRef *pRef = *ppRef;
if (pRef) {
wDebug("vgId:%d, wal close ref %" PRId64 ", refId %" PRId64, pWal->cfg.vgId, pRef->refVer, pRef->refId);
taosMemoryFree(pRef);
} else {
wDebug("vgId:%d, wal close ref null, refId %" PRId64, pWal->cfg.vgId, refId);
}
(void)taosHashRemove(pWal->pRefHash, &refId, sizeof(int64_t));
}
taosHashRemove(pWal->pRefHash, &refId, sizeof(int64_t));
taosMemoryFree(pRef);
}
int32_t walSetRefVer(SWalRef *pRef, int64_t ver) {
@ -52,30 +64,31 @@ int32_t walSetRefVer(SWalRef *pRef, int64_t ver) {
taosThreadMutexLock(&pWal->mutex);
if (ver < pWal->vers.firstVer || ver > pWal->vers.lastVer) {
taosThreadMutexUnlock(&pWal->mutex);
return TSDB_CODE_WAL_INVALID_VER;
TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER);
}
pRef->refVer = ver;
taosThreadMutexUnlock(&pWal->mutex);
}
return TSDB_CODE_SUCCESS;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
void walRefFirstVer(SWal *pWal, SWalRef *pRef) {
taosThreadMutexLock(&pWal->mutex);
int64_t ver = walGetFirstVer(pWal);
pRef->refVer = ver;
pRef->refVer = pWal->vers.firstVer;
taosThreadMutexUnlock(&pWal->mutex);
wDebug("vgId:%d, wal ref version %" PRId64 " for first", pWal->cfg.vgId, ver);
wDebug("vgId:%d, wal ref version %" PRId64 " for first", pWal->cfg.vgId, pRef->refVer);
}
void walRefLastVer(SWal *pWal, SWalRef *pRef) {
taosThreadMutexLock(&pWal->mutex);
int64_t ver = walGetLastVer(pWal);
pRef->refVer = ver;
pRef->refVer = pWal->vers.lastVer;
taosThreadMutexUnlock(&pWal->mutex);
wDebug("vgId:%d, wal ref version %" PRId64 " for last", pWal->cfg.vgId, ver);
wDebug("vgId:%d, wal ref version %" PRId64 " for last", pWal->cfg.vgId, pRef->refVer);
}

159
source/libs/wal/src/walSeek.c
View File

@ -1,159 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taoserror.h"
#include "tref.h"
#include "walInt.h"
#if 0
static int64_t walSeekWritePos(SWal* pWal, int64_t ver) {
int64_t code = 0;
TdFilePtr pIdxTFile = pWal->pIdxFile;
TdFilePtr pLogTFile = pWal->pLogFile;
// seek position
int64_t idxOff = walGetVerIdxOffset(pWal, ver);
code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
SWalIdxEntry entry;
// TODO:deserialize
code = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry));
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
return 0;
}
#endif
int walInitWriteFile(SWal* pWal) {
TdFilePtr pIdxTFile, pLogTFile;
SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet);
int64_t fileFirstVer = pRet->firstVer;
char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, fileFirstVer, fnameStr);
pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxTFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
walBuildLogName(pWal, fileFirstVer, fnameStr);
pLogTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pLogTFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
// switch file
pWal->pIdxFile = pIdxTFile;
pWal->pLogFile = pLogTFile;
pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
return 0;
}
int64_t walChangeWrite(SWal* pWal, int64_t ver) {
int code;
TdFilePtr pIdxTFile, pLogTFile;
char fnameStr[WAL_FILE_LEN];
if (pWal->pLogFile != NULL) {
if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pLogFile)) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
code = taosCloseFile(&pWal->pLogFile);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
}
if (pWal->pIdxFile != NULL) {
if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pIdxFile)) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
code = taosCloseFile(&pWal->pIdxFile);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
}
SWalFileInfo tmpInfo;
tmpInfo.firstVer = ver;
// bsearch in fileSet
int32_t idx = taosArraySearchIdx(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
/*A(idx != -1);*/
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, idx);
int64_t fileFirstVer = pFileInfo->firstVer;
walBuildIdxName(pWal, fileFirstVer, fnameStr);
pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxTFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
pWal->pIdxFile = NULL;
return -1;
}
walBuildLogName(pWal, fileFirstVer, fnameStr);
pLogTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pLogTFile == NULL) {
taosCloseFile(&pIdxTFile);
terrno = TAOS_SYSTEM_ERROR(errno);
pWal->pLogFile = NULL;
return -1;
}
pWal->pLogFile = pLogTFile;
pWal->pIdxFile = pIdxTFile;
pWal->writeCur = idx;
return fileFirstVer;
}
#if 0
int walSeekWriteVer(SWal* pWal, int64_t ver) {
int64_t code;
if (ver == pWal->vers.lastVer) {
return 0;
}
if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) {
terrno = TSDB_CODE_WAL_INVALID_VER;
return -1;
}
if (ver < pWal->vers.snapshotVer) {
}
if (ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) {
code = walChangeWrite(pWal, ver);
if (code != 0) {
return -1;
}
}
code = walSeekWritePos(pWal, ver);
if (code != 0) {
return -1;
}
return 0;
}
#endif

453
source/libs/wal/src/walWrite.c
View File

@ -21,6 +21,8 @@
#include "walInt.h"
int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) {
int32_t code = 0;
taosThreadMutexLock(&pWal->mutex);
wInfo("vgId:%d, restore from snapshot, version %" PRId64, pWal->cfg.vgId, ver);
@ -33,7 +35,8 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) {
if (pRef->refVer != -1 && pRef->refVer <= ver) {
taosHashCancelIterate(pWal->pRefHash, pIter);
taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TSDB_CODE_FAILED);
}
}
@ -47,24 +50,25 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) {
char fnameStr[WAL_FILE_LEN];
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
if (taosRemoveFile(fnameStr) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, restore from snapshot, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr());
taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
wInfo("vgId:%d, restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr);
walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
if (taosRemoveFile(fnameStr) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr());
taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
wInfo("vgId:%d, restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr);
}
}
(void)walRemoveMeta(pWal);
TAOS_CHECK_RETURN(walRemoveMeta(pWal));
pWal->writeCur = -1;
pWal->totSize = 0;
@ -78,25 +82,81 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) {
pWal->vers.verInSnapshotting = -1;
taosThreadMutexUnlock(&pWal->mutex);
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t walApplyVer(SWal *pWal, int64_t ver) {
// TODO: error check
pWal->vers.appliedVer = ver;
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t walCommit(SWal *pWal, int64_t ver) {
if (ver < pWal->vers.commitVer) {
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
if (ver > pWal->vers.lastVer || pWal->vers.commitVer < pWal->vers.snapshotVer) {
terrno = TSDB_CODE_WAL_INVALID_VER;
return -1;
TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER);
}
pWal->vers.commitVer = ver;
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
static int64_t walChangeWrite(SWal *pWal, int64_t ver) {
int code;
TdFilePtr pIdxTFile, pLogTFile;
char fnameStr[WAL_FILE_LEN];
if (pWal->pLogFile != NULL) {
if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pLogFile)) != 0) {
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
code = taosCloseFile(&pWal->pLogFile);
if (code != 0) {
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
}
if (pWal->pIdxFile != NULL) {
if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pIdxFile)) != 0) {
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
code = taosCloseFile(&pWal->pIdxFile);
if (code != 0) {
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
}
SWalFileInfo tmpInfo;
tmpInfo.firstVer = ver;
// bsearch in fileSet
int32_t idx = taosArraySearchIdx(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
/*A(idx != -1);*/
SWalFileInfo *pFileInfo = taosArrayGet(pWal->fileInfoSet, idx);
int64_t fileFirstVer = pFileInfo->firstVer;
walBuildIdxName(pWal, fileFirstVer, fnameStr);
pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxTFile == NULL) {
pWal->pIdxFile = NULL;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
walBuildLogName(pWal, fileFirstVer, fnameStr);
pLogTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pLogTFile == NULL) {
taosCloseFile(&pIdxTFile);
pWal->pLogFile = NULL;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
pWal->pLogFile = pLogTFile;
pWal->pIdxFile = pIdxTFile;
pWal->writeCur = idx;
return fileFirstVer;
}
int32_t walRollback(SWal *pWal, int64_t ver) {
@ -105,9 +165,9 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
int64_t code;
char fnameStr[WAL_FILE_LEN];
if (ver > pWal->vers.lastVer || ver <= pWal->vers.commitVer || ver <= pWal->vers.snapshotVer) {
terrno = TSDB_CODE_WAL_INVALID_VER;
taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER);
}
// find correct file
@ -116,7 +176,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
code = walChangeWrite(pWal, ver);
if (code < 0) {
taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(code);
}
// delete files in descending order
@ -136,22 +197,24 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
taosCloseFile(&pWal->pIdxFile);
TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND);
if (pIdxFile == NULL) {
taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
int64_t idxOff = walGetVerIdxOffset(pWal, ver);
code = taosLSeekFile(pIdxFile, idxOff, SEEK_SET);
if (code < 0) {
taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
// read idx file and get log file pos
SWalIdxEntry entry;
if (taosReadFile(pIdxFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
@ -160,49 +223,50 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
wDebug("vgId:%d, wal truncate file %s", pWal->cfg.vgId, fnameStr);
if (pLogFile == NULL) {
// TODO
terrno = TAOS_SYSTEM_ERROR(errno);
taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
code = taosLSeekFile(pLogFile, entry.offset, SEEK_SET);
if (code < 0) {
// TODO
terrno = TAOS_SYSTEM_ERROR(errno);
taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
// validate offset
SWalCkHead head;
int64_t size = taosReadFile(pLogFile, &head, sizeof(SWalCkHead));
if (size != sizeof(SWalCkHead)) {
taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
code = walValidHeadCksum(&head);
if (code != 0) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
}
if (head.head.version != ver) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
}
// truncate old files
code = taosFtruncateFile(pLogFile, entry.offset);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
code = taosFtruncateFile(pIdxFile, idxOff);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
pWal->vers.lastVer = ver - 1;
((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1;
@ -215,43 +279,98 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
if (code < 0) {
wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr());
taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(code);
}
// unlock
taosThreadMutexUnlock(&pWal->mutex);
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
static int32_t walRollImpl(SWal *pWal) {
int32_t code = 0, lino = 0;
if (pWal->pIdxFile != NULL) {
if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pIdxFile)) != 0) {
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
code = taosCloseFile(&pWal->pIdxFile);
if (code != 0) {
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
}
if (pWal->pLogFile != NULL) {
if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pLogFile)) != 0) {
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
code = taosCloseFile(&pWal->pLogFile);
if (code != 0) {
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
}
TdFilePtr pIdxFile, pLogFile;
// create new file
int64_t newFileFirstVer = pWal->vers.lastVer + 1;
char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, newFileFirstVer, fnameStr);
pIdxFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxFile == NULL) {
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
walBuildLogName(pWal, newFileFirstVer, fnameStr);
pLogFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
wDebug("vgId:%d, wal create new file for write:%s", pWal->cfg.vgId, fnameStr);
if (pLogFile == NULL) {
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
TAOS_CHECK_GOTO(walRollFileInfo(pWal), &lino, _exit);
// switch file
pWal->pIdxFile = pIdxFile;
pWal->pLogFile = pLogFile;
pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
pWal->lastRollSeq = walGetSeq();
TAOS_CHECK_GOTO(walSaveMeta(pWal), &lino, _exit);
_exit:
if (code) {
wError("vgId:%d, %s failed at line %d since %s", pWal->cfg.vgId, __func__, lino, tstrerror(code));
}
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
static FORCE_INLINE int32_t walCheckAndRoll(SWal *pWal) {
if (taosArrayGetSize(pWal->fileInfoSet) == 0) {
if (walRollImpl(pWal) < 0) {
return -1;
}
return 0;
TAOS_CHECK_RETURN(walRollImpl(pWal));
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int64_t passed = walGetSeq() - pWal->lastRollSeq;
if (pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) {
if (walRollImpl(pWal) < 0) {
return -1;
}
TAOS_CHECK_RETURN(walRollImpl(pWal));
} else if (pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) {
if (walRollImpl(pWal) < 0) {
return -1;
}
TAOS_CHECK_RETURN(walRollImpl(pWal));
}
if (walGetLastFileCachedSize(pWal) > tsWalFsyncDataSizeLimit) {
if (walSaveMeta(pWal) < 0) {
return -1;
}
TAOS_CHECK_RETURN(walSaveMeta(pWal));
}
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t walBeginSnapshot(SWal *pWal, int64_t ver, int64_t logRetention) {
int32_t code = 0;
taosThreadMutexLock(&pWal->mutex);
ASSERT(logRetention >= 0);
pWal->vers.verInSnapshotting = ver;
@ -262,22 +381,21 @@ int32_t walBeginSnapshot(SWal *pWal, int64_t ver, int64_t logRetention) {
pWal->cfg.vgId, ver, pWal->vers.logRetention, pWal->vers.firstVer, pWal->vers.lastVer);
// check file rolling
if (walGetLastFileSize(pWal) != 0) {
if (walRollImpl(pWal) < 0) {
if ((code = walRollImpl(pWal)) < 0) {
wError("vgId:%d, failed to roll wal files since %s", pWal->cfg.vgId, terrstr());
goto _err;
goto _exit;
}
}
_exit:
taosThreadMutexUnlock(&pWal->mutex);
return 0;
_err:
taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_RETURN(code);
}
int32_t walEndSnapshot(SWal *pWal) {
int32_t code = 0;
int32_t code = 0, lino = 0;
taosThreadMutexLock(&pWal->mutex);
int64_t ver = pWal->vers.verInSnapshotting;
@ -286,8 +404,7 @@ int32_t walEndSnapshot(SWal *pWal) {
pWal->cfg.vgId, ver, pWal->vers.logRetention, pWal->vers.firstVer, pWal->vers.lastVer);
if (ver == -1) {
code = -1;
goto END;
TAOS_CHECK_GOTO(TSDB_CODE_FAILED, &lino, _exit);
}
pWal->vers.snapshotVer = ver;
@ -357,10 +474,7 @@ int32_t walEndSnapshot(SWal *pWal) {
pWal->totSize = newTotSize;
pWal->vers.verInSnapshotting = -1;
code = walSaveMeta(pWal);
if (code < 0) {
goto END;
}
TAOS_CHECK_GOTO(walSaveMeta(pWal), &lino, _exit);
// delete files
deleteCnt = taosArrayGetSize(pWal->toDeleteFiles);
@ -373,12 +487,12 @@ int32_t walEndSnapshot(SWal *pWal) {
walBuildLogName(pWal, pInfo->firstVer, fnameStr);
if (taosRemoveFile(fnameStr) < 0 && errno != ENOENT) {
wError("vgId:%d, failed to remove log file %s due to %s", pWal->cfg.vgId, fnameStr, strerror(errno));
goto END;
goto _exit;
}
walBuildIdxName(pWal, pInfo->firstVer, fnameStr);
if (taosRemoveFile(fnameStr) < 0 && errno != ENOENT) {
wError("vgId:%d, failed to remove idx file %s due to %s", pWal->cfg.vgId, fnameStr, strerror(errno));
goto END;
goto _exit;
}
}
if (pInfo != NULL) {
@ -387,81 +501,19 @@ int32_t walEndSnapshot(SWal *pWal) {
}
taosArrayClear(pWal->toDeleteFiles);
END:
_exit:
taosThreadMutexUnlock(&pWal->mutex);
return code;
}
int32_t walRollImpl(SWal *pWal) {
int32_t code = 0;
if (pWal->pIdxFile != NULL) {
if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pIdxFile)) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
goto END;
}
code = taosCloseFile(&pWal->pIdxFile);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
goto END;
}
if (code) {
wError("vgId:%d, %s failed at line %d since %s", pWal->cfg.vgId, __func__, lino, tstrerror(code));
}
if (pWal->pLogFile != NULL) {
if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pLogFile)) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
goto END;
}
code = taosCloseFile(&pWal->pLogFile);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
goto END;
}
}
TdFilePtr pIdxFile, pLogFile;
// create new file
int64_t newFileFirstVer = pWal->vers.lastVer + 1;
char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, newFileFirstVer, fnameStr);
pIdxFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
code = -1;
goto END;
}
walBuildLogName(pWal, newFileFirstVer, fnameStr);
pLogFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
wDebug("vgId:%d, wal create new file for write:%s", pWal->cfg.vgId, fnameStr);
if (pLogFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
code = -1;
goto END;
}
// error code was set inner
code = walRollFileInfo(pWal);
if (code != 0) {
goto END;
}
// switch file
pWal->pIdxFile = pIdxFile;
pWal->pLogFile = pLogFile;
pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
pWal->lastRollSeq = walGetSeq();
code = walSaveMeta(pWal);
if (code < 0) {
wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr());
goto END;
}
END:
return code;
}
static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
int32_t code = 0;
SWalIdxEntry entry = {.ver = ver, .offset = offset};
SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal);
@ -472,8 +524,8 @@ static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
int64_t size = taosWriteFile(pWal->pIdxFile, &entry, sizeof(SWalIdxEntry));
if (size != sizeof(SWalIdxEntry)) {
wError("vgId:%d, failed to write idx entry due to %s. ver:%" PRId64, pWal->cfg.vgId, strerror(errno), ver);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
// check alignment of idx entries
@ -484,12 +536,13 @@ static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
taosMsleep(100);
exit(EXIT_FAILURE);
}
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta,
const void *body, int32_t bodyLen) {
int64_t code = 0;
int32_t code = 0, lino = 0;
int32_t plainBodyLen = bodyLen;
int64_t offset = walGetCurFileOffset(pWal);
@ -509,19 +562,16 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
TMSG_INFO(msgType), pWal->writeHead.cksumHead, pWal->writeHead.cksumBody);
if (pWal->cfg.level != TAOS_WAL_SKIP) {
code = walWriteIndex(pWal, index, offset);
if (code < 0) {
goto END;
}
TAOS_CHECK_GOTO(walWriteIndex(pWal, index, offset), &lino, _exit);
}
if (pWal->cfg.level != TAOS_WAL_SKIP &&
taosWriteFile(pWal->pLogFile, &pWal->writeHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) {
terrno = TAOS_SYSTEM_ERROR(errno);
code = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
code = -1;
goto END;
TAOS_CHECK_GOTO(code, &lino, _exit);
}
int32_t cyptedBodyLen = plainBodyLen;
@ -536,8 +586,8 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
if (newBody == NULL) {
wError("vgId:%d, file:%" PRId64 ".log, failed to malloc since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
code = -1;
goto END;
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
memset(newBody, 0, cyptedBodyLen);
memcpy(newBody, body, plainBodyLen);
@ -546,9 +596,10 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
if (newBodyEncrypted == NULL) {
wError("vgId:%d, file:%" PRId64 ".log, failed to malloc since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
code = -1;
if (newBody != NULL) taosMemoryFreeClear(newBody);
goto END;
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
SCryptOpts opts;
@ -567,15 +618,16 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
}
if (pWal->cfg.level != TAOS_WAL_SKIP && taosWriteFile(pWal->pLogFile, (char *)buf, cyptedBodyLen) != cyptedBodyLen) {
terrno = TAOS_SYSTEM_ERROR(errno);
code = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
code = -1;
if (pWal->cfg.encryptAlgorithm == DND_CA_SM4) {
taosMemoryFreeClear(newBody);
taosMemoryFreeClear(newBodyEncrypted);
}
goto END;
TAOS_CHECK_GOTO(code, &lino, _exit);
}
if (pWal->cfg.encryptAlgorithm == DND_CA_SM4) {
@ -596,7 +648,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
return 0;
END:
_exit:
// recover in a reverse order
if (taosFtruncateFile(pWal->pLogFile, offset) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
@ -614,88 +666,66 @@ END:
taosMsleep(100);
exit(EXIT_FAILURE);
}
return -1;
TAOS_RETURN(TSDB_CODE_FAILED);
}
int64_t walAppendLog(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body,
static int32_t walInitWriteFile(SWal *pWal) {
TdFilePtr pIdxTFile, pLogTFile;
SWalFileInfo *pRet = taosArrayGetLast(pWal->fileInfoSet);
int64_t fileFirstVer = pRet->firstVer;
char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, fileFirstVer, fnameStr);
pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxTFile == NULL) {
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
walBuildLogName(pWal, fileFirstVer, fnameStr);
pLogTFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pLogTFile == NULL) {
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
// switch file
pWal->pIdxFile = pIdxTFile;
pWal->pLogFile = pLogTFile;
pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t walAppendLog(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body,
int32_t bodyLen) {
int32_t code = 0, lino = 0;
taosThreadMutexLock(&pWal->mutex);
if (index != pWal->vers.lastVer + 1) {
terrno = TSDB_CODE_WAL_INVALID_VER;
taosThreadMutexUnlock(&pWal->mutex);
return -1;
TAOS_CHECK_GOTO(TSDB_CODE_WAL_INVALID_VER, &lino, _exit);
}
if (walCheckAndRoll(pWal) < 0) {
taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
TAOS_CHECK_GOTO(walCheckAndRoll(pWal), &lino, _exit);
if (pWal->pLogFile == NULL || pWal->pIdxFile == NULL || pWal->writeCur < 0) {
if (walInitWriteFile(pWal) < 0) {
taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
TAOS_CHECK_GOTO(walInitWriteFile(pWal), &lino, _exit);
}
if (walWriteImpl(pWal, index, msgType, syncMeta, body, bodyLen) < 0) {
taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
TAOS_CHECK_GOTO(walWriteImpl(pWal, index, msgType, syncMeta, body, bodyLen), &lino, _exit);
taosThreadMutexUnlock(&pWal->mutex);
return index;
}
int32_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body,
int32_t bodyLen) {
int32_t code = 0;
taosThreadMutexLock(&pWal->mutex);
// concurrency control:
// if logs are write with assigned index,
// smaller index must be write before larger one
if (index != pWal->vers.lastVer + 1) {
terrno = TSDB_CODE_WAL_INVALID_VER;
taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
if (walCheckAndRoll(pWal) < 0) {
taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
if (pWal->pIdxFile == NULL || pWal->pLogFile == NULL || pWal->writeCur < 0) {
if (walInitWriteFile(pWal) < 0) {
taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
}
if (walWriteImpl(pWal, index, msgType, syncMeta, body, bodyLen) < 0) {
taosThreadMutexUnlock(&pWal->mutex);
return -1;
_exit:
if (code) {
wError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
taosThreadMutexUnlock(&pWal->mutex);
return code;
}
int32_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen) {
SWalSyncInfo syncMeta = {
.isWeek = -1,
.seqNum = UINT64_MAX,
.term = UINT64_MAX,
};
return walWriteWithSyncInfo(pWal, index, msgType, syncMeta, body, bodyLen);
}
int32_t walFsync(SWal *pWal, bool forceFsync) {
int32_t code = 0;
void walFsync(SWal *pWal, bool forceFsync) {
if (pWal->cfg.level == TAOS_WAL_SKIP) {
return;
return code;
}
taosThreadMutexLock(&pWal->mutex);
@ -704,7 +734,10 @@ void walFsync(SWal *pWal, bool forceFsync) {
if (taosFsyncFile(pWal->pLogFile) < 0) {
wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal),
strerror(errno));
code = TAOS_SYSTEM_ERROR(errno);
}
}
taosThreadMutexUnlock(&pWal->mutex);
return code;
}

53
source/libs/wal/test/walMetaTest.cpp
View File

@ -5,8 +5,9 @@
#include "walInt.h"
const char* ranStr = "tvapq02tcp";
const int ranStrLen = strlen(ranStr);
const char* ranStr = "tvapq02tcp";
const int ranStrLen = strlen(ranStr);
SWalSyncInfo syncMeta = {0};
class WalCleanEnv : public ::testing::Test {
protected:
@ -170,7 +171,9 @@ TEST_F(WalCleanEnv, serialize) {
ASSERT(code == 0);
code = walRollFileInfo(pWal);
ASSERT(code == 0);
char* ss = walMetaSerialize(pWal);
char* ss = NULL;
code = walMetaSerialize(pWal, &ss);
ASSERT(code == 0);
printf("%s\n", ss);
taosMemoryFree(ss);
code = walSaveMeta(pWal);
@ -193,15 +196,21 @@ TEST_F(WalKeepEnv, readOldMeta) {
walResetEnv();
int code;
syncMeta.isWeek = -1;
syncMeta.seqNum = UINT64_MAX;
syncMeta.term = UINT64_MAX;
for (int i = 0; i < 10; i++) {
code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen);
code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, i);
code = walWrite(pWal, i + 2, i, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, -1);
code = walAppendLog(pWal, i + 2, i, syncMeta, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER);
ASSERT_EQ(pWal->vers.lastVer, i);
}
char* oldss = walMetaSerialize(pWal);
char* oldss = NULL;
code = walMetaSerialize(pWal, &oldss);
ASSERT(code == 0);
TearDown();
SetUp();
@ -209,7 +218,9 @@ TEST_F(WalKeepEnv, readOldMeta) {
ASSERT_EQ(pWal->vers.firstVer, 0);
ASSERT_EQ(pWal->vers.lastVer, 9);
char* newss = walMetaSerialize(pWal);
char* newss = NULL;
code = walMetaSerialize(pWal, &newss);
ASSERT(code == 0);
int len = strlen(oldss);
ASSERT_EQ(len, strlen(newss));
@ -223,11 +234,11 @@ TEST_F(WalKeepEnv, readOldMeta) {
TEST_F(WalCleanEnv, write) {
int code;
for (int i = 0; i < 10; i++) {
code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen);
code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, i);
code = walWrite(pWal, i + 2, i, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, -1);
code = walAppendLog(pWal, i + 2, i, syncMeta, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER);
ASSERT_EQ(pWal->vers.lastVer, i);
}
code = walSaveMeta(pWal);
@ -237,7 +248,7 @@ TEST_F(WalCleanEnv, write) {
TEST_F(WalCleanEnv, rollback) {
int code;
for (int i = 0; i < 10; i++) {
code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen);
code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, i);
}
@ -260,7 +271,7 @@ TEST_F(WalCleanEnv, rollback) {
TEST_F(WalCleanEnv, rollbackMultiFile) {
int code;
for (int i = 0; i < 10; i++) {
code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen);
code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, i);
if (i == 5) {
@ -278,11 +289,11 @@ TEST_F(WalCleanEnv, rollbackMultiFile) {
ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, 5);
code = walRollback(pWal, 5);
ASSERT_EQ(code, -1);
ASSERT_NE(code, 0);
ASSERT_EQ(pWal->vers.lastVer, 5);
code = walWrite(pWal, 6, 6, (void*)ranStr, ranStrLen);
code = walAppendLog(pWal, 6, 6, syncMeta, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, 6);
@ -294,7 +305,7 @@ TEST_F(WalCleanDeleteEnv, roll) {
int code;
int i;
for (i = 0; i < 100; i++) {
code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen);
code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, i);
code = walCommit(pWal, i);
@ -307,11 +318,11 @@ TEST_F(WalCleanDeleteEnv, roll) {
ASSERT_EQ(pWal->vers.snapshotVer, i - 1);
ASSERT_EQ(pWal->vers.verInSnapshotting, -1);
code = walWrite(pWal, 5, 0, (void*)ranStr, ranStrLen);
code = walAppendLog(pWal, 5, 0, syncMeta, (void*)ranStr, ranStrLen);
ASSERT_NE(code, 0);
for (; i < 200; i++) {
code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen);
code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0);
code = walCommit(pWal, i);
ASSERT_EQ(pWal->vers.commitVer, i);
@ -334,7 +345,7 @@ TEST_F(WalKeepEnv, readHandleRead) {
char newStr[100];
sprintf(newStr, "%s-%d", ranStr, i);
int len = strlen(newStr);
code = walWrite(pWal, i, 0, newStr, len);
code = walAppendLog(pWal, i, 0, syncMeta, newStr, len);
ASSERT_EQ(code, 0);
}
for (int i = 0; i < 1000; i++) {
@ -370,7 +381,7 @@ TEST_F(WalRetentionEnv, repairMeta1) {
char newStr[100];
sprintf(newStr, "%s-%d", ranStr, i);
int len = strlen(newStr);
code = walWrite(pWal, i, 0, newStr, len);
code = walAppendLog(pWal, i, 0, syncMeta, newStr, len);
ASSERT_EQ(code, 0);
}
@ -416,7 +427,7 @@ TEST_F(WalRetentionEnv, repairMeta1) {
char newStr[100];
sprintf(newStr, "%s-%d", ranStr, i);
int len = strlen(newStr);
code = walWrite(pWal, i, 0, newStr, len);
code = walAppendLog(pWal, i, 0, syncMeta, newStr, len);
ASSERT_EQ(code, 0);
}

10
source/os/src/osMemory.c
View File

@ -266,7 +266,7 @@ void *taosMemoryMalloc(int64_t size) {
return (char *)tmp + sizeof(TdMemoryInfo);
#else
void* p = malloc(size);
void *p = malloc(size);
if (NULL == p) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
}
@ -287,7 +287,7 @@ void *taosMemoryCalloc(int64_t num, int64_t size) {
return (char *)tmp + sizeof(TdMemoryInfo);
#else
void* p = calloc(num, size);
void *p = calloc(num, size);
if (NULL == p) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
}
@ -317,7 +317,7 @@ void *taosMemoryRealloc(void *ptr, int64_t size) {
return (char *)tmp + sizeof(TdMemoryInfo);
#else
void* p = realloc(ptr, size);
void *p = realloc(ptr, size);
if (size > 0 && NULL == p) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
}
@ -342,12 +342,12 @@ char *taosStrdup(const char *ptr) {
return (char *)tmp + sizeof(TdMemoryInfo);
#else
char* p = tstrdup(ptr);
char *p = tstrdup(ptr);
if (ptr != NULL && NULL == p) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
}
return p;
#endif
}

62
source/util/src/tarray.c
View File

@ -18,9 +18,9 @@
#include "tcoding.h"
// todo refactor API
#define BOUNDARY_SIZE 1024*1024*1024 // 1G
#define BOUNDARY_SIZE 1024 * 1024 * 1024 // 1G
#define BOUNDARY_SMALL_FACTOR 1.2
#define BOUNDARY_BIG_FACTOR 2
#define BOUNDARY_BIG_FACTOR 2
SArray* taosArrayInit(size_t size, size_t elemSize) {
if (elemSize == 0) {
@ -52,14 +52,12 @@ SArray* taosArrayInit(size_t size, size_t elemSize) {
SArray* taosArrayInit_s(size_t elemSize, size_t initialSize) {
SArray* pArray = taosMemoryMalloc(sizeof(SArray));
if (pArray == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pArray->size = initialSize;
pArray->pData = taosMemoryCalloc(initialSize, elemSize);
if (pArray->pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pArray);
return NULL;
}
@ -75,7 +73,7 @@ static int32_t taosArrayResize(SArray* pArray) {
void* tmp = taosMemoryRealloc(pArray->pData, size * pArray->elemSize);
if (tmp == NULL) { // reallocate failed, the original buffer remains
return -1;
return terrno;
}
pArray->pData = tmp;
@ -99,7 +97,7 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) {
pArray->pData = taosMemoryRealloc(pArray->pData, tsize * pArray->elemSize);
if (pArray->pData == NULL) {
return -1;
return terrno;
}
pArray->capacity = tsize;
@ -109,10 +107,13 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) {
void* taosArrayAddBatch(SArray* pArray, const void* pData, int32_t nEles) {
if (pData == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
if (taosArrayEnsureCap(pArray, pArray->size + nEles) != 0) {
int32_t code = taosArrayEnsureCap(pArray, pArray->size + nEles);
if (code) {
terrno = code;
return NULL;
}
@ -171,7 +172,9 @@ void* taosArrayAddAll(SArray* pArray, const SArray* pInput) {
}
void* taosArrayReserve(SArray* pArray, int32_t num) {
if (taosArrayEnsureCap(pArray, pArray->size + num) != 0) {
int32_t code = taosArrayEnsureCap(pArray, pArray->size + num);
if (code) {
terrno = code;
return NULL;
}
@ -198,7 +201,7 @@ void* taosArrayGet(const SArray* pArray, size_t index) {
}
if (index >= pArray->size) {
uError("index is out of range, current:%" PRIzu " max:%"PRIzu, index, pArray->size);
uError("index is out of range, current:%" PRIzu " max:%" PRIzu, index, pArray->size);
return NULL;
}
@ -230,6 +233,7 @@ size_t taosArrayGetSize(const SArray* pArray) {
void* taosArrayInsert(SArray* pArray, size_t index, const void* pData) {
if (pArray == NULL || pData == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
@ -239,7 +243,6 @@ void* taosArrayInsert(SArray* pArray, size_t index, const void* pData) {
if (pArray->size >= pArray->capacity) {
int32_t ret = taosArrayResize(pArray);
if (ret < 0) {
return NULL;
}
@ -316,8 +319,10 @@ SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize) {
}
SArray* pDst = taosArrayInit(size, elemSize);
memcpy(pDst->pData, src, elemSize * size);
pDst->size = size;
if (pDst) {
memcpy(pDst->pData, src, elemSize * size);
pDst->size = size;
}
return pDst;
}
@ -333,18 +338,20 @@ SArray* taosArrayDup(const SArray* pSrc, __array_item_dup_fn_t fn) {
SArray* dst = taosArrayInit(pSrc->size, pSrc->elemSize);
if (fn == NULL) {
memcpy(dst->pData, pSrc->pData, pSrc->elemSize * pSrc->size);
} else {
ASSERT(pSrc->elemSize == sizeof(void*));
if (dst) {
if (fn == NULL) {
memcpy(dst->pData, pSrc->pData, pSrc->elemSize * pSrc->size);
} else {
ASSERT(pSrc->elemSize == sizeof(void*));
for (int32_t i = 0; i < pSrc->size; ++i) {
void* p = fn(taosArrayGetP(pSrc, i));
memcpy(((char*)dst->pData) + i * dst->elemSize, &p, dst->elemSize);
for (int32_t i = 0; i < pSrc->size; ++i) {
void* p = fn(taosArrayGetP(pSrc, i));
memcpy(((char*)dst->pData) + i * dst->elemSize, &p, dst->elemSize);
}
}
}
dst->size = pSrc->size;
dst->size = pSrc->size;
}
return dst;
}
@ -508,10 +515,21 @@ void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
*pArray = taosArrayInit(sz, sizeof(void*));
if (*pArray == NULL) {
return NULL;
}
for (int32_t i = 0; i < sz; i++) {
void* data = taosMemoryCalloc(1, dataSz);
if (data == NULL) {
return NULL;
}
buf = decode(buf, data, sver);
taosArrayPush(*pArray, &data);
if (taosArrayPush(*pArray, &data) == NULL) {
taosMemoryFree(data);
return NULL;
}
}
return (void*)buf;
}

11
source/util/src/tcache.c
View File

@ -230,6 +230,7 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheNode *
static FORCE_INLINE STrashElem *doRemoveElemInTrashcan(SCacheObj *pCacheObj, STrashElem *pElem) {
if (pElem->pData->signature != pElem->pData) {
uWarn("key:sig:0x%" PRIx64 " %p data has been released, ignore", (int64_t)pElem->pData->signature, pElem->pData);
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
@ -358,6 +359,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInMs, bool extendLi
SCacheObj *pCacheObj = (SCacheObj *)taosMemoryCalloc(1, sizeof(SCacheObj));
if (pCacheObj == NULL) {
uError("failed to allocate memory, reason:%s", strerror(errno));
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
@ -393,6 +395,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInMs, bool extendLi
void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize,
int32_t durationMS) {
if (pCacheObj == NULL || pCacheObj->pEntryList == NULL || pCacheObj->deleting == 1) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
@ -501,11 +504,15 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) {
}
void *taosCacheTransferData(SCacheObj *pCacheObj, void **data) {
if (pCacheObj == NULL || data == NULL || (*data) == NULL) return NULL;
if (pCacheObj == NULL || data == NULL || (*data) == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
SCacheNode *ptNode = (SCacheNode *)((char *)(*data) - sizeof(SCacheNode));
if (ptNode->signature != ptNode) {
uError("cache:%s, key: %p the data from cache is invalid", pCacheObj->name, ptNode);
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
@ -702,8 +709,8 @@ SCacheNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pDat
SCacheNode *pNewNode = taosMemoryCalloc(1, sizeInBytes);
if (pNewNode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
uError("failed to allocate memory, reason:%s", strerror(errno));
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}

6
source/util/src/tdes.c
View File

@ -76,6 +76,12 @@ char* taosDesEncode(int64_t key, char* src, int32_t len) {
char* taosDesDecode(int64_t key, char* src, int32_t len) {
uint8_t* keyStr = (uint8_t*)(&key);
char* temp = taosMemoryCalloc(len + 8, 1);
if (!temp) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
memcpy(temp, src, len);
len += 8;

34
source/util/src/tdigest.c
View File

@ -30,7 +30,7 @@
#include "tlog.h"
#define INTERPOLATE(x, x0, x1) (((x) - (x0)) / ((x1) - (x0)))
//#define INTEGRATED_LOCATION(compression, q) ((compression) * (asin(2 * (q) - 1) + M_PI / 2) / M_PI)
// #define INTEGRATED_LOCATION(compression, q) ((compression) * (asin(2 * (q) - 1) + M_PI / 2) / M_PI)
#define INTEGRATED_LOCATION(compression, q) ((compression) * (asin(2 * (double)(q)-1) / M_PI + (double)1 / 2))
#define FLOAT_EQ(f1, f2) (fabs((f1) - (f2)) <= FLT_EPSILON)
@ -99,16 +99,19 @@ static void mergeCentroid(SMergeArgs *args, SCentroid *merge) {
}
}
void tdigestCompress(TDigest *t) {
int32_t tdigestCompress(TDigest *t) {
SCentroid *unmerged_centroids;
int64_t unmerged_weight = 0;
int32_t num_unmerged = t->num_buffered_pts;
int32_t i, j;
SMergeArgs args;
if (t->num_buffered_pts <= 0) return;
if (t->num_buffered_pts <= 0) return 0;
unmerged_centroids = (SCentroid *)taosMemoryMalloc(sizeof(SCentroid) * t->num_buffered_pts);
if (unmerged_centroids == NULL) {
return terrno;
}
for (i = 0; i < num_unmerged; i++) {
SPt *p = t->buffered_pts + i;
SCentroid *c = &unmerged_centroids[i];
@ -122,6 +125,10 @@ void tdigestCompress(TDigest *t) {
taosSort(unmerged_centroids, num_unmerged, sizeof(SCentroid), cmpCentroid);
memset(&args, 0, sizeof(SMergeArgs));
args.centroids = (SCentroid *)taosMemoryMalloc((size_t)(sizeof(SCentroid) * t->size));
if (args.centroids == NULL) {
taosMemoryFree((void *)unmerged_centroids);
return terrno;
}
memset(args.centroids, 0, (size_t)(sizeof(SCentroid) * t->size));
args.t = t;
@ -167,10 +174,11 @@ void tdigestCompress(TDigest *t) {
memcpy(t->centroids, args.centroids, sizeof(SCentroid) * t->num_centroids);
taosMemoryFree((void *)args.centroids);
return 0;
}
void tdigestAdd(TDigest *t, double x, int64_t w) {
if (w == 0) return;
int32_t tdigestAdd(TDigest *t, double x, int64_t w) {
if (w == 0) return 0;
int32_t i = t->num_buffered_pts;
if (i > 0 && t->buffered_pts[i - 1].value == x) {
@ -181,7 +189,10 @@ void tdigestAdd(TDigest *t, double x, int64_t w) {
t->num_buffered_pts++;
}
if (t->num_buffered_pts >= t->threshold) tdigestCompress(t);
if (t->num_buffered_pts >= t->threshold) {
return tdigestCompress(t);
}
return 0;
}
#if 0
@ -284,16 +295,21 @@ double tdigestQuantile(TDigest *t, double q) {
return t->max;
}
void tdigestMerge(TDigest *t1, TDigest *t2) {
int32_t tdigestMerge(TDigest *t1, TDigest *t2) {
int32_t code = 0;
// SPoints
int32_t num_pts = t2->num_buffered_pts;
for (int32_t i = num_pts - 1; i >= 0; i--) {
SPt *p = t2->buffered_pts + i;
tdigestAdd(t1, p->value, p->weight);
code = tdigestAdd(t1, p->value, p->weight);
if (code) return code;
t2->num_buffered_pts--;
}
// centroids
for (int32_t i = 0; i < t2->num_centroids; i++) {
tdigestAdd(t1, t2->centroids[i].mean, t2->centroids[i].weight);
code = tdigestAdd(t1, t2->centroids[i].mean, t2->centroids[i].weight);
if (code) return code;
}
return 0;
}

6
source/util/src/tenv.c
View File

@ -28,7 +28,7 @@ int32_t taosEnvNameToCfgName(const char *envNameStr, char *cfgNameStr, int32_t c
// p[cfgNameMaxLen - 1] = '\0';
// return strlen(cfgNameStr);
cfgNameStr[0] = '\0';
return -1;
return TSDB_CODE_INVALID_PARA;
}
envNameStr += 5;
if (*envNameStr != '\0') {
@ -55,7 +55,7 @@ int32_t taosEnvNameToCfgName(const char *envNameStr, char *cfgNameStr, int32_t c
int32_t taosEnvToCfg(const char *envStr, char *cfgStr) {
if (envStr == NULL || cfgStr == NULL) {
return -1;
return TSDB_CODE_INVALID_PARA;
}
if (cfgStr != envStr) strcpy(cfgStr, envStr);
char *p = strchr(cfgStr, '=');
@ -74,7 +74,7 @@ int32_t taosEnvToCfg(const char *envStr, char *cfgStr) {
memset(&cfgStr[cfgNameLen], ' ', p - cfgStr - cfgNameLen + 1);
} else {
*cfgStr = '\0';
return -1;
return TSDB_CODE_INVALID_PARA;
}
}
return strlen(cfgStr);

8
source/util/src/tgeosctx.c
View File

@ -13,14 +13,12 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdef.h"
#include "tgeosctx.h"
#include "tdef.h"
static threadlocal SGeosContext tlGeosCtx = {0};
SGeosContext* getThreadLocalGeosCtx() {
return &tlGeosCtx;
}
SGeosContext* getThreadLocalGeosCtx() { return &tlGeosCtx; }
void destroyThreadLocalGeosCtx() {
if (tlGeosCtx.WKTReader) {
@ -47,7 +45,7 @@ void destroyThreadLocalGeosCtx() {
destroyRegexes(tlGeosCtx.WKTRegex, tlGeosCtx.WKTMatchData);
}
if(tlGeosCtx.handle) {
if (tlGeosCtx.handle) {
GEOS_finish_r(tlGeosCtx.handle);
tlGeosCtx.handle = NULL;
}

32
source/util/src/thash.c
View File

@ -67,7 +67,7 @@ struct SHashObj {
bool enableUpdate; // enable update
SArray *pMemBlock; // memory block allocated for SHashEntry
_hash_before_fn_t callbackFp; // function invoked before return the value to caller
// int64_t compTimes;
// int64_t compTimes;
};
/*
@ -147,7 +147,7 @@ static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntr
uint32_t hashVal) {
SHashNode *pNode = pe->next;
while (pNode) {
//atomic_add_fetch_64(&pHashObj->compTimes, 1);
// atomic_add_fetch_64(&pHashObj->compTimes, 1);
if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) &&
pNode->removed == 0) {
break;
@ -243,7 +243,6 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
SHashObj *pHashObj = (SHashObj *)taosMemoryMalloc(sizeof(SHashObj));
if (pHashObj == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
@ -262,7 +261,6 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
pHashObj->hashList = (SHashEntry **)taosMemoryMalloc(pHashObj->capacity * sizeof(void *));
if (pHashObj->hashList == NULL) {
taosMemoryFree(pHashObj);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
@ -270,7 +268,6 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
if (pHashObj->pMemBlock == NULL) {
taosMemoryFree(pHashObj->hashList);
taosMemoryFree(pHashObj);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
@ -279,7 +276,6 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
taosArrayDestroy(pHashObj->pMemBlock);
taosMemoryFree(pHashObj->hashList);
taosMemoryFree(pHashObj);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
@ -290,7 +286,12 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
pHashObj->hashList[i]->next = NULL;
}
taosArrayPush(pHashObj->pMemBlock, &p);
if (taosArrayPush(pHashObj->pMemBlock, &p) == NULL) {
taosArrayDestroy(pHashObj->pMemBlock);
taosMemoryFree(pHashObj->hashList);
taosMemoryFree(pHashObj);
return NULL;
}
return pHashObj;
}
@ -315,8 +316,7 @@ int32_t taosHashGetSize(const SHashObj *pHashObj) {
int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const void *data, size_t size) {
if (pHashObj == NULL || key == NULL || keyLen == 0) {
terrno = TSDB_CODE_INVALID_PTR;
return TSDB_CODE_INVALID_PTR;
return terrno = TSDB_CODE_INVALID_PTR;
}
uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen);
@ -331,7 +331,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo
// disable resize
taosHashRLock(pHashObj);
int32_t code = TSDB_CODE_SUCCESS;
int32_t code = TSDB_CODE_SUCCESS;
uint32_t slot = HASH_INDEX(hashVal, pHashObj->capacity);
SHashEntry *pe = pHashObj->hashList[slot];
@ -440,14 +440,12 @@ void *taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void *
*size = pNode->dataLen;
*d = taosMemoryCalloc(1, *size);
if (*d == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
} else if (*size < pNode->dataLen) {
*size = pNode->dataLen;
char *tmp = taosMemoryRealloc(*d, *size);
if (tmp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
@ -474,7 +472,7 @@ void *taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void *
int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) {
if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || key == NULL || keyLen == 0) {
return -1;
return TSDB_CODE_INVALID_PARA;
}
uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen);
@ -615,7 +613,7 @@ void taosHashTableResize(SHashObj *pHashObj) {
return;
}
int64_t st = taosGetTimestampUs();
int64_t st = taosGetTimestampUs();
SHashEntry **pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(SHashEntry *) * newCapacity);
if (pNewEntryList == NULL) {
// uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity);
@ -631,7 +629,10 @@ void taosHashTableResize(SHashObj *pHashObj) {
pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry));
}
taosArrayPush(pHashObj->pMemBlock, &p);
if (taosArrayPush(pHashObj->pMemBlock, &p) == NULL) {
taosMemoryFree(p);
return;
}
pHashObj->capacity = newCapacity;
for (int32_t idx = 0; idx < pHashObj->capacity; ++idx) {
@ -681,7 +682,6 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s
#endif
if (pNewNode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}

71
source/util/src/theap.c
View File

@ -188,7 +188,6 @@ void heapRemove(Heap* heap, HeapNode* node) {
void heapDequeue(Heap* heap) { heapRemove(heap, heap->min); }
struct PriorityQueue {
SArray* container;
pq_comp_fn fn;
@ -197,16 +196,23 @@ struct PriorityQueue {
};
PriorityQueue* createPriorityQueue(pq_comp_fn fn, FDelete deleteFn, void* param) {
PriorityQueue* pq = (PriorityQueue*)taosMemoryCalloc(1, sizeof(PriorityQueue));
if (pq == NULL) {
return NULL;
}
pq->container = taosArrayInit(1, sizeof(PriorityQueueNode));
if (pq->container == NULL) {
taosMemoryFree(pq);
return NULL;
}
pq->fn = fn;
pq->deleteFn = deleteFn;
pq->param = param;
return pq;
}
void taosPQSetFn(PriorityQueue* pq, pq_comp_fn fn) {
pq->fn = fn;
}
void taosPQSetFn(PriorityQueue* pq, pq_comp_fn fn) { pq->fn = fn; }
void destroyPriorityQueue(PriorityQueue* pq) {
if (pq->deleteFn)
@ -218,15 +224,15 @@ void destroyPriorityQueue(PriorityQueue* pq) {
static size_t pqParent(size_t i) { return (--i) >> 1; /* (i - 1) / 2 */ }
static size_t pqLeft(size_t i) { return (i << 1) | 1; /* i * 2 + 1 */ }
static size_t pqRight(size_t i) { return (++i) << 1; /* (i + 1) * 2 */}
static void pqSwapPQNode(PriorityQueueNode* a, PriorityQueueNode* b) {
void * tmp = a->data;
static size_t pqRight(size_t i) { return (++i) << 1; /* (i + 1) * 2 */ }
static void pqSwapPQNode(PriorityQueueNode* a, PriorityQueueNode* b) {
void* tmp = a->data;
a->data = b->data;
b->data = tmp;
}
#define pqContainerGetEle(pq, i) ((PriorityQueueNode*)taosArrayGet((pq)->container, (i)))
#define pqContainerSize(pq) (taosArrayGetSize((pq)->container))
#define pqContainerSize(pq) (taosArrayGetSize((pq)->container))
size_t taosPQSize(PriorityQueue* pq) { return pqContainerSize(pq); }
@ -288,12 +294,12 @@ static void pqRemove(PriorityQueue* pq, size_t i) {
pqUpdate(pq, i);
}
PriorityQueueNode* taosPQTop(PriorityQueue* pq) {
return pqContainerGetEle(pq, 0);
}
PriorityQueueNode* taosPQTop(PriorityQueue* pq) { return pqContainerGetEle(pq, 0); }
PriorityQueueNode* taosPQPush(PriorityQueue* pq, const PriorityQueueNode* node) {
taosArrayPush(pq->container, node);
if (taosArrayPush(pq->container, node) == NULL) {
return NULL;
}
return pqReverseHeapify(pq, pqContainerSize(pq) - 1);
}
@ -310,15 +316,28 @@ struct BoundedQueue {
BoundedQueue* createBoundedQueue(uint32_t maxSize, pq_comp_fn fn, FDelete deleteFn, void* param) {
BoundedQueue* q = (BoundedQueue*)taosMemoryCalloc(1, sizeof(BoundedQueue));
if (q == NULL) {
return NULL;
}
q->queue = createPriorityQueue(fn, deleteFn, param);
taosArrayEnsureCap(q->queue->container, maxSize + 1);
if (q->queue == NULL) {
taosMemoryFree(q);
return NULL;
}
int32_t code = taosArrayEnsureCap(q->queue->container, maxSize + 1);
if (code) {
destroyPriorityQueue(q->queue);
taosMemoryFree(q);
terrno = code;
return NULL;
}
q->maxSize = maxSize;
return q;
}
void taosBQSetFn(BoundedQueue* q, pq_comp_fn fn) {
taosPQSetFn(q->queue, fn);
}
void taosBQSetFn(BoundedQueue* q, pq_comp_fn fn) { taosPQSetFn(q->queue, fn); }
void destroyBoundedQueue(BoundedQueue* q) {
if (!q) return;
@ -343,22 +362,12 @@ PriorityQueueNode* taosBQPush(BoundedQueue* q, PriorityQueueNode* n) {
}
}
PriorityQueueNode* taosBQTop(BoundedQueue* q) {
return taosPQTop(q->queue);
}
PriorityQueueNode* taosBQTop(BoundedQueue* q) { return taosPQTop(q->queue); }
void taosBQBuildHeap(BoundedQueue *q) {
pqBuildHeap(q->queue);
}
void taosBQBuildHeap(BoundedQueue* q) { pqBuildHeap(q->queue); }
size_t taosBQMaxSize(BoundedQueue* q) {
return q->maxSize;
}
size_t taosBQMaxSize(BoundedQueue* q) { return q->maxSize; }
size_t taosBQSize(BoundedQueue* q) {
return taosPQSize(q->queue);
}
size_t taosBQSize(BoundedQueue* q) { return taosPQSize(q->queue); }
void taosBQPop(BoundedQueue* q) {
taosPQPop(q->queue);
}
void taosBQPop(BoundedQueue* q) { taosPQPop(q->queue); }

6
source/util/src/tidpool.c
View File

@ -19,7 +19,9 @@
void *taosInitIdPool(int32_t maxId) {
id_pool_t *pIdPool = taosMemoryCalloc(1, sizeof(id_pool_t));
if (pIdPool == NULL) return NULL;
if (pIdPool == NULL) {
return NULL;
}
pIdPool->freeList = taosMemoryCalloc(maxId, sizeof(bool));
if (pIdPool->freeList == NULL) {
@ -120,7 +122,7 @@ int32_t taosUpdateIdPool(id_pool_t *pIdPool, int32_t maxId) {
bool *idList = taosMemoryCalloc(maxId, sizeof(bool));
if (idList == NULL) {
return -1;
return terrno;
}
taosThreadMutexLock(&pIdPool->mutex);

27
source/util/src/tref.c
View File

@ -67,15 +67,13 @@ int32_t taosOpenRef(int32_t max, RefFp fp) {
nodeList = taosMemoryCalloc(sizeof(SRefNode *), (size_t)max);
if (nodeList == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return terrno = TSDB_CODE_OUT_OF_MEMORY;
}
lockedBy = taosMemoryCalloc(sizeof(int64_t), (size_t)max);
if (lockedBy == NULL) {
taosMemoryFree(nodeList);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return terrno = TSDB_CODE_OUT_OF_MEMORY;
}
taosThreadMutexLock(&tsRefMutex);
@ -118,8 +116,7 @@ int32_t taosCloseRef(int32_t rsetId) {
if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
uTrace("rsetId:%d is invalid, out of range", rsetId);
terrno = TSDB_CODE_REF_INVALID_ID;
return -1;
return terrno = TSDB_CODE_REF_INVALID_ID;
}
pSet = tsRefSetList + rsetId;
@ -149,8 +146,7 @@ int64_t taosAddRef(int32_t rsetId, void *p) {
if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
uTrace("rsetId:%d p:%p failed to add, rsetId not valid", rsetId, p);
terrno = TSDB_CODE_REF_INVALID_ID;
return -1;
return terrno = TSDB_CODE_REF_INVALID_ID;
}
pSet = tsRefSetList + rsetId;
@ -158,14 +154,12 @@ int64_t taosAddRef(int32_t rsetId, void *p) {
if (pSet->state != TSDB_REF_STATE_ACTIVE) {
taosDecRsetCount(pSet);
uTrace("rsetId:%d p:%p failed to add, not active", rsetId, p);
terrno = TSDB_CODE_REF_ID_REMOVED;
return -1;
return terrno = TSDB_CODE_REF_ID_REMOVED;
}
pNode = taosMemoryCalloc(sizeof(SRefNode), 1);
if (pNode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return terrno = TSDB_CODE_OUT_OF_MEMORY;
}
rid = atomic_add_fetch_64(&pSet->rid, 1);
@ -389,21 +383,18 @@ static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove) {
if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rsetId not valid", rsetId, rid);
terrno = TSDB_CODE_REF_INVALID_ID;
return -1;
return terrno = TSDB_CODE_REF_INVALID_ID;
}
if (rid <= 0) {
uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rid not valid", rsetId, rid);
terrno = TSDB_CODE_REF_NOT_EXIST;
return -1;
return terrno = TSDB_CODE_REF_NOT_EXIST;
}
pSet = tsRefSetList + rsetId;
if (pSet->state == TSDB_REF_STATE_EMPTY) {
uTrace("rsetId:%d rid:%" PRId64 " failed to remove, cleaned", rsetId, rid);
terrno = TSDB_CODE_REF_ID_REMOVED;
return -1;
return terrno = TSDB_CODE_REF_ID_REMOVED;
}
hash = rid % pSet->max;