xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(cos/multi-writing): use lasted version of data

This commit is contained in:
Minglei Jin 2024-04-25 16:37:24 +08:00
parent 3986f392b5
commit a9094d7602
2 changed files with 37 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -14,8 +14,8 @@
*/
#include "cos.h"
#include "tsdb.h"
#include "crypt.h"
#include "tsdb.h"
#include "vnd.h"
static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
@ -61,6 +61,7 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
// taosMemoryFree(pFD);
goto _exit;
}
pFD->s3File = 1;
/*
const char *object_name = taosDirEntryBaseName((char *)path);
long s3_size = 0;
@ -86,7 +87,6 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
goto _exit;
}
#else
pFD->s3File = 1;
pFD->pFD = (TdFilePtr)&pFD->s3File;
int32_t vid = 0;
sscanf(object_name, "v%df%dver%" PRId64 ".data", &vid, &pFD->fid, &pFD->cid);
@ -170,7 +170,7 @@ void tsdbCloseFile(STsdbFD **ppFD) {
}
}
static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char* encryptKey) {
static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *encryptKey) {
int32_t code = 0;
if (!pFD->pFD) {
@ -199,8 +199,8 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char* e
taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage);
if(encryptAlgorithm == DND_CA_SM4){
//if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_TSDB) == DND_CS_TSDB){
if (encryptAlgorithm == DND_CA_SM4) {
// if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_TSDB) == DND_CS_TSDB){
unsigned char PacketData[128];
int NewLen;
int32_t count = 0;
@ -210,7 +210,7 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char* e
opts.source = pFD->pBuf + count;
opts.result = PacketData;
opts.unitLen = 128;
//strncpy(opts.key, tsEncryptKey, 16);
// strncpy(opts.key, tsEncryptKey, 16);
strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN);
NewLen = CBC_Encrypt(&opts);
@ -218,7 +218,7 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char* e
memcpy(pFD->pBuf + count, PacketData, NewLen);
count += NewLen;
}
//tsdbDebug("CBC_Encrypt count:%d %s", count, __FUNCTION__);
// tsdbDebug("CBC_Encrypt count:%d %s", count, __FUNCTION__);
}
n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage);
@ -237,7 +237,7 @@ _exit:
return code;
}
static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgorithm, char* encryptKey) {
static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgorithm, char *encryptKey) {
int32_t code = 0;
// ASSERT(pgno <= pFD->szFile);
@ -297,20 +297,19 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor
}
//}
if(encryptAlgorithm == DND_CA_SM4){
//if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_TSDB) == DND_CS_TSDB){
if (encryptAlgorithm == DND_CA_SM4) {
// if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_TSDB) == DND_CS_TSDB){
unsigned char PacketData[128];
int NewLen;
int32_t count = 0;
while(count < pFD->szPage)
{
while (count < pFD->szPage) {
SCryptOpts opts = {0};
opts.len = 128;
opts.source = pFD->pBuf + count;
opts.result = PacketData;
opts.unitLen = 128;
//strncpy(opts.key, tsEncryptKey, 16);
// strncpy(opts.key, tsEncryptKey, 16);
strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN);
NewLen = CBC_Decrypt(&opts);
@ -318,7 +317,7 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor
memcpy(pFD->pBuf + count, PacketData, NewLen);
count += NewLen;
}
//tsdbDebug("CBC_Decrypt count:%d %s", count, __FUNCTION__);
// tsdbDebug("CBC_Decrypt count:%d %s", count, __FUNCTION__);
}
// check
@ -334,7 +333,7 @@ _exit:
}
int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm,
char* encryptKey) {
char *encryptKey) {
int32_t code = 0;
int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage);
@ -367,7 +366,7 @@ _exit:
}
static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm,
char* encryptKey) {
char *encryptKey) {
int32_t code = 0;
int64_t n = 0;
int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
@ -573,7 +572,7 @@ _exit:
}
int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint,
int32_t encryptAlgorithm, char* encryptKey) {
int32_t encryptAlgorithm, char *encryptKey) {
int32_t code = 0;
if (!pFD->pFD) {
code = tsdbOpenFileImpl(pFD);
@ -582,7 +581,7 @@ int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size,
}
}
if (pFD->lcn > 1 /*pFD->s3File && tsS3BlockSize < 0*/) {
if (pFD->s3File && pFD->lcn > 1 /* && tsS3BlockSize < 0*/) {
return tsdbReadFileS3(pFD, offset, pBuf, size, szHint);
} else {
return tsdbReadFileImp(pFD, offset, pBuf, size, encryptAlgorithm, encryptKey);
@ -593,20 +592,19 @@ _exit:
}
int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer *buffer, int64_t szHint,
int32_t encryptAlgorithm, char* encryptKey) {
int32_t encryptAlgorithm, char *encryptKey) {
int32_t code;
code = tBufferEnsureCapacity(buffer, buffer->size + size);
if (code) return code;
code = tsdbReadFile(pFD, offset, (uint8_t *)tBufferGetDataEnd(buffer), size, szHint,
encryptAlgorithm, encryptKey);
code = tsdbReadFile(pFD, offset, (uint8_t *)tBufferGetDataEnd(buffer), size, szHint, encryptAlgorithm, encryptKey);
if (code) return code;
buffer->size += size;
return code;
}
int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char* encryptKey) {
int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char *encryptKey) {
int32_t code = 0;
/*
if (pFD->s3File) {
@ -726,7 +724,7 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) {
// read
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err;
@ -765,7 +763,7 @@ int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) {
// read
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err;
@ -800,7 +798,7 @@ int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *m
// read
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err;
@ -895,7 +893,7 @@ int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelDa
// read
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err;
@ -937,7 +935,7 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) {
// read
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err;

4
source/dnode/vnode/src/tsdb/tsdbRetention.c
View File

@ -528,7 +528,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int
if (fdFrom == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
tsdbInfo("vgId: %d, open lcfile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, lc_size);
tsdbInfo("vgId:%d, open lcfile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, lc_size);
snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", lcn);
fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
@ -671,7 +671,7 @@ static int32_t tsdbDoS3MigrateOnFileSet(SRTNer *rtner, STFileSet *fset) {
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
int32_t lcn = fobj->f->lcn;
if (lcn < 1 && taosCheckExistFile(fobj->fname)) {
if (/*lcn < 1 && */ taosCheckExistFile(fobj->fname)) {
int32_t mtime = 0;
int64_t size = 0;
taosStatFile(fobj->fname, &size, &mtime, NULL);