xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(cos/multi-writing): use lasted version of data

This commit is contained in:
Minglei Jin 2024-04-25 16:37:24 +08:00
parent 3986f392b5
commit a9094d7602
2 changed files with 37 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -14,8 +14,8 @@
*/ */
#include "cos.h" #include "cos.h"
#include "tsdb.h"
#include "crypt.h" #include "crypt.h"
#include "tsdb.h"
#include "vnd.h" #include "vnd.h"
static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
@ -61,6 +61,7 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
// taosMemoryFree(pFD); // taosMemoryFree(pFD);
goto _exit; goto _exit;
} }
pFD->s3File = 1;
/* /*
const char *object_name = taosDirEntryBaseName((char *)path); const char *object_name = taosDirEntryBaseName((char *)path);
long s3_size = 0; long s3_size = 0;
@ -86,7 +87,6 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
goto _exit; goto _exit;
} }
#else #else
pFD->s3File = 1;
pFD->pFD = (TdFilePtr)&pFD->s3File; pFD->pFD = (TdFilePtr)&pFD->s3File;
int32_t vid = 0; int32_t vid = 0;
sscanf(object_name, "v%df%dver%" PRId64 ".data", &vid, &pFD->fid, &pFD->cid); sscanf(object_name, "v%df%dver%" PRId64 ".data", &vid, &pFD->fid, &pFD->cid);
@ -170,7 +170,7 @@ void tsdbCloseFile(STsdbFD **ppFD) {
} }
} }
static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char* encryptKey) { static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *encryptKey) {
int32_t code = 0; int32_t code = 0;
if (!pFD->pFD) { if (!pFD->pFD) {
@ -198,27 +198,27 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char* e
} }
taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage); taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage);
if(encryptAlgorithm == DND_CA_SM4){ if (encryptAlgorithm == DND_CA_SM4) {
//if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_TSDB) == DND_CS_TSDB){ // if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_TSDB) == DND_CS_TSDB){
unsigned char PacketData[128]; unsigned char PacketData[128];
int NewLen; int NewLen;
int32_t count = 0; int32_t count = 0;
while (count < pFD->szPage) { while (count < pFD->szPage) {
SCryptOpts opts = {0}; SCryptOpts opts = {0};
opts.len = 128; opts.len = 128;
opts.source = pFD->pBuf + count; opts.source = pFD->pBuf + count;
opts.result = PacketData; opts.result = PacketData;
opts.unitLen = 128; opts.unitLen = 128;
//strncpy(opts.key, tsEncryptKey, 16); // strncpy(opts.key, tsEncryptKey, 16);
strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN); strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN);
NewLen = CBC_Encrypt(&opts); NewLen = CBC_Encrypt(&opts);
memcpy(pFD->pBuf + count, PacketData, NewLen); memcpy(pFD->pBuf + count, PacketData, NewLen);
count += NewLen; count += NewLen;
} }
//tsdbDebug("CBC_Encrypt count:%d %s", count, __FUNCTION__); // tsdbDebug("CBC_Encrypt count:%d %s", count, __FUNCTION__);
} }
n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage); n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage);
@ -237,7 +237,7 @@ _exit:
return code; return code;
} }
static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgorithm, char* encryptKey) { static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgorithm, char *encryptKey) {
int32_t code = 0; int32_t code = 0;
// ASSERT(pgno <= pFD->szFile); // ASSERT(pgno <= pFD->szFile);
@ -297,20 +297,19 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor
} }
//} //}
if(encryptAlgorithm == DND_CA_SM4){ if (encryptAlgorithm == DND_CA_SM4) {
//if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_TSDB) == DND_CS_TSDB){ // if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_TSDB) == DND_CS_TSDB){
unsigned char PacketData[128]; unsigned char PacketData[128];
int NewLen; int NewLen;
int32_t count = 0; int32_t count = 0;
while(count < pFD->szPage) while (count < pFD->szPage) {
{
SCryptOpts opts = {0}; SCryptOpts opts = {0};
opts.len = 128; opts.len = 128;
opts.source = pFD->pBuf + count; opts.source = pFD->pBuf + count;
opts.result = PacketData; opts.result = PacketData;
opts.unitLen = 128; opts.unitLen = 128;
//strncpy(opts.key, tsEncryptKey, 16); // strncpy(opts.key, tsEncryptKey, 16);
strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN); strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN);
NewLen = CBC_Decrypt(&opts); NewLen = CBC_Decrypt(&opts);
@ -318,7 +317,7 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor
memcpy(pFD->pBuf + count, PacketData, NewLen); memcpy(pFD->pBuf + count, PacketData, NewLen);
count += NewLen; count += NewLen;
} }
//tsdbDebug("CBC_Decrypt count:%d %s", count, __FUNCTION__); // tsdbDebug("CBC_Decrypt count:%d %s", count, __FUNCTION__);
} }
// check // check
@ -333,8 +332,8 @@ _exit:
return code; return code;
} }
int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm, int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm,
char* encryptKey) { char *encryptKey) {
int32_t code = 0; int32_t code = 0;
int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage);
@ -366,8 +365,8 @@ _exit:
return code; return code;
} }
static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm, static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm,
char* encryptKey) { char *encryptKey) {
int32_t code = 0; int32_t code = 0;
int64_t n = 0; int64_t n = 0;
int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
@ -572,8 +571,8 @@ _exit:
return code; return code;
} }
int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint, int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint,
int32_t encryptAlgorithm, char* encryptKey) { int32_t encryptAlgorithm, char *encryptKey) {
int32_t code = 0; int32_t code = 0;
if (!pFD->pFD) { if (!pFD->pFD) {
code = tsdbOpenFileImpl(pFD); code = tsdbOpenFileImpl(pFD);
@ -582,7 +581,7 @@ int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size,
} }
} }
if (pFD->lcn > 1 /*pFD->s3File && tsS3BlockSize < 0*/) { if (pFD->s3File && pFD->lcn > 1 /* && tsS3BlockSize < 0*/) {
return tsdbReadFileS3(pFD, offset, pBuf, size, szHint); return tsdbReadFileS3(pFD, offset, pBuf, size, szHint);
} else { } else {
return tsdbReadFileImp(pFD, offset, pBuf, size, encryptAlgorithm, encryptKey); return tsdbReadFileImp(pFD, offset, pBuf, size, encryptAlgorithm, encryptKey);
@ -593,20 +592,19 @@ _exit:
} }
int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer *buffer, int64_t szHint, int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer *buffer, int64_t szHint,
int32_t encryptAlgorithm, char* encryptKey) { int32_t encryptAlgorithm, char *encryptKey) {
int32_t code; int32_t code;
code = tBufferEnsureCapacity(buffer, buffer->size + size); code = tBufferEnsureCapacity(buffer, buffer->size + size);
if (code) return code; if (code) return code;
code = tsdbReadFile(pFD, offset, (uint8_t *)tBufferGetDataEnd(buffer), size, szHint, code = tsdbReadFile(pFD, offset, (uint8_t *)tBufferGetDataEnd(buffer), size, szHint, encryptAlgorithm, encryptKey);
encryptAlgorithm, encryptKey);
if (code) return code; if (code) return code;
buffer->size += size; buffer->size += size;
return code; return code;
} }
int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char* encryptKey) { int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char *encryptKey) {
int32_t code = 0; int32_t code = 0;
/* /*
if (pFD->s3File) { if (pFD->s3File) {
@ -726,7 +724,7 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) {
// read // read
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err; if (code) goto _err;
@ -765,7 +763,7 @@ int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) {
// read // read
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err; if (code) goto _err;
@ -800,7 +798,7 @@ int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *m
// read // read
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err; if (code) goto _err;
@ -895,7 +893,7 @@ int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelDa
// read // read
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err; if (code) goto _err;
@ -937,7 +935,7 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) {
// read // read
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err; if (code) goto _err;

4
source/dnode/vnode/src/tsdb/tsdbRetention.c
View File

@ -528,7 +528,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int
if (fdFrom == NULL) code = terrno; if (fdFrom == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
tsdbInfo("vgId: %d, open lcfile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, lc_size); tsdbInfo("vgId:%d, open lcfile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, lc_size);
snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", lcn); snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", lcn);
fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
@ -671,7 +671,7 @@ static int32_t tsdbDoS3MigrateOnFileSet(SRTNer *rtner, STFileSet *fset) {
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize; int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
int32_t lcn = fobj->f->lcn; int32_t lcn = fobj->f->lcn;
if (lcn < 1 && taosCheckExistFile(fobj->fname)) { if (/*lcn < 1 && */ taosCheckExistFile(fobj->fname)) {
int32_t mtime = 0; int32_t mtime = 0;
int64_t size = 0; int64_t size = 0;
taosStatFile(fobj->fname, &size, &mtime, NULL); taosStatFile(fobj->fname, &size, &mtime, NULL);