From e429526789f4d79477508ad41b156795dd85e875 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 13 Nov 2023 12:05:55 +0800 Subject: [PATCH 01/34] refact: rename STSnapRange to STFileSetRange --- source/dnode/vnode/src/inc/tsdb.h | 14 +++++------ source/dnode/vnode/src/tsdb/tsdbFS2.c | 16 ++++++------- source/dnode/vnode/src/tsdb/tsdbFS2.h | 8 +++---- source/dnode/vnode/src/tsdb/tsdbFSet2.c | 4 ++-- source/dnode/vnode/src/tsdb/tsdbFSet2.h | 6 ++--- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 28 +++++++++++----------- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 26 ++++++++++---------- 7 files changed, 51 insertions(+), 51 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 79f9caab33..9cc6cd1132 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -681,14 +681,14 @@ struct SDelFWriter { typedef struct STFileSet STFileSet; typedef TARRAY2(STFileSet *) TFileSetArray; -typedef struct STSnapRange STSnapRange; -typedef TARRAY2(STSnapRange *) TSnapRangeArray; // disjoint snap ranges +typedef struct STFileSetRange STFileSetRange; +typedef TARRAY2(STFileSetRange *) TFileSetRangeArray; // disjoint ranges // util -int32_t tSerializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR); -int32_t tDeserializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR); -void tsdbSnapRangeArrayDestroy(TSnapRangeArray **ppSnap); -SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges); +int32_t tSerializeSnapRangeArray(void *buf, int32_t bufLen, TFileSetRangeArray *pSnapR); +int32_t tDeserializeSnapRangeArray(void *buf, int32_t bufLen, TFileSetRangeArray *pSnapR); +void tsdbFileSetRangeArrayDestroy(TFileSetRangeArray **ppSnap); +SHashObj *tsdbGetSnapRangeHash(TFileSetRangeArray *pRanges); // snap partition list typedef TARRAY2(SVersionRange) SVerRangeList; @@ -699,7 +699,7 @@ STsdbSnapPartList *tsdbSnapPartListCreate(); void tsdbSnapPartListDestroy(STsdbSnapPartList **ppList); int32_t tSerializeTsdbSnapPartList(void *buf, int32_t bufLen, STsdbSnapPartList *pList); int32_t tDeserializeTsdbSnapPartList(void *buf, int32_t bufLen, STsdbSnapPartList *pList); -int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList *pList, TSnapRangeArray **ppRanges); +int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList *pList, TFileSetRangeArray **ppRanges); enum { TSDB_SNAP_RANGE_TYP_HEAD = 0, diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 635c53bbed..df4df18dc3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -1072,7 +1072,7 @@ int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr) { return 0; } -int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRanges, TFileSetArray **fsetArr, +int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TFileSetRangeArray *pRanges, TFileSetArray **fsetArr, TFileOpArray *fopArr) { int32_t code = 0; STFileSet *fset; @@ -1096,7 +1096,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRange int64_t ever = VERSION_MAX; if (pHash) { int32_t fid = fset->fid; - STSnapRange *u = taosHashGet(pHash, &fid, sizeof(fid)); + STFileSetRange *u = taosHashGet(pHash, &fid, sizeof(fid)); if (u) { ever = u->sver - 1; } @@ -1123,7 +1123,7 @@ _out: return code; } -SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges) { +SHashObj *tsdbGetSnapRangeHash(TFileSetRangeArray *pRanges) { int32_t capacity = TARRAY2_SIZE(pRanges) * 2; SHashObj *pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); if (pHash == NULL) { @@ -1132,7 +1132,7 @@ SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges) { } for (int32_t i = 0; i < TARRAY2_SIZE(pRanges); i++) { - STSnapRange *u = TARRAY2_GET(pRanges, i); + STFileSetRange *u = TARRAY2_GET(pRanges, i); int32_t fid = u->fid; int32_t code = taosHashPut(pHash, &fid, sizeof(fid), u, sizeof(*u)); ASSERT(code == 0); @@ -1141,11 +1141,11 @@ SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges) { return pHash; } -int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pRanges, - TSnapRangeArray **fsrArr) { +int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TFileSetRangeArray *pRanges, + TFileSetRangeArray **fsrArr) { int32_t code = 0; STFileSet *fset; - STSnapRange *fsr1 = NULL; + STFileSetRange *fsr1 = NULL; SHashObj *pHash = NULL; fsrArr[0] = taosMemoryCalloc(1, sizeof(*fsrArr[0])); @@ -1170,7 +1170,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev if (pHash) { int32_t fid = fset->fid; - STSnapRange *u = taosHashGet(pHash, &fid, sizeof(fid)); + STFileSetRange *u = taosHashGet(pHash, &fid, sizeof(fid)); if (u) { sver1 = u->sver; tsdbDebug("range hash get fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index 74453126cf..8fdce9e690 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -44,12 +44,12 @@ int32_t tsdbFSCreateRefSnapshot(STFileSystem *fs, TFileSetArray **fsetArr); int32_t tsdbFSCreateRefSnapshotWithoutLock(STFileSystem *fs, TFileSetArray **fsetArr); int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr); -int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pExclude, TFileSetArray **fsetArr, +int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TFileSetRangeArray *pExclude, TFileSetArray **fsetArr, TFileOpArray *fopArr); int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr, TFileOpArray *fopArr); -int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pRanges, - TSnapRangeArray **fsrArr); -int32_t tsdbFSDestroyRefRangedSnapshot(TSnapRangeArray **fsrArr); +int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TFileSetRangeArray *pRanges, + TFileSetRangeArray **fsrArr); +int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr); // txn int64_t tsdbFSAllocEid(STFileSystem *fs); int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype); diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index 1bf886e3b0..7673299e4b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -533,7 +533,7 @@ int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_ return 0; } -int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STSnapRange **fsr) { +int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STFileSetRange **fsr) { fsr[0] = taosMemoryCalloc(1, sizeof(*fsr[0])); if (fsr[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; fsr[0]->fid = fset1->fid; @@ -575,7 +575,7 @@ int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs return 0; } -int32_t tsdbTSnapRangeClear(STSnapRange **fsr) { +int32_t tsdbTSnapRangeClear(STFileSetRange **fsr) { if (!fsr[0]) return 0; tsdbTFileSetClear(&fsr[0]->fset); diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h index 83f5b1e83c..3a6427a42c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -49,8 +49,8 @@ int32_t tsdbTFileSetRemove(STFileSet *fset); int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_t ever, STFileSet **fset, TFileOpArray *fopArr); -int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STSnapRange **fsr); -int32_t tsdbTSnapRangeClear(STSnapRange **fsr); +int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STFileSetRange **fsr); +int32_t tsdbTSnapRangeClear(STFileSetRange **fsr); // to/from json int32_t tsdbTFileSetToJson(const STFileSet *fset, cJSON *json); @@ -101,7 +101,7 @@ struct STFileSet { bool blockCommit; }; -struct STSnapRange { +struct STFileSetRange { int32_t fid; int64_t sver; int64_t ever; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index e757daa0af..9d3efcfdae 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -32,12 +32,12 @@ struct STsdbSnapReader { uint8_t* aBuf[5]; SSkmInfo skmTb[1]; - TSnapRangeArray* fsrArr; + TFileSetRangeArray* fsrArr; // context struct { int32_t fsrArrIdx; - STSnapRange* fsr; + STFileSetRange* fsr; bool isDataDone; bool isTombDone; } ctx[1]; @@ -437,14 +437,14 @@ int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type, reader[0]->ever = ever; reader[0]->type = type; - code = tsdbFSCreateRefRangedSnapshot(tsdb->pFS, sver, ever, (TSnapRangeArray*)pRanges, &reader[0]->fsrArr); + code = tsdbFSCreateRefRangedSnapshot(tsdb->pFS, sver, ever, (TFileSetRangeArray*)pRanges, &reader[0]->fsrArr); TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code), sver, ever, type); - tsdbSnapRangeArrayDestroy(&reader[0]->fsrArr); + tsdbFileSetRangeArrayDestroy(&reader[0]->fsrArr); taosMemoryFree(reader[0]); reader[0] = NULL; } else { @@ -472,7 +472,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** reader) { TARRAY2_DESTROY(reader[0]->sttReaderArr, tsdbSttFileReaderClose); tsdbDataFileReaderClose(&reader[0]->dataReader); - tsdbSnapRangeArrayDestroy(&reader[0]->fsrArr); + tsdbFileSetRangeArrayDestroy(&reader[0]->fsrArr); tDestroyTSchema(reader[0]->skmTb->pTSchema); for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->aBuf); ++i) { @@ -1061,7 +1061,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, void* pRang writer[0]->compactVersion = INT64_MAX; writer[0]->now = taosGetTimestampMs(); - code = tsdbFSCreateCopyRangedSnapshot(pTsdb->pFS, (TSnapRangeArray*)pRanges, &writer[0]->fsetArr, writer[0]->fopArr); + code = tsdbFSCreateCopyRangedSnapshot(pTsdb->pFS, (TFileSetRangeArray*)pRanges, &writer[0]->fsetArr, writer[0]->fopArr); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -1168,7 +1168,7 @@ _exit: return code; } -// snap part +// STsdbSnapPartition ===================================== static int32_t tsdbSnapPartCmprFn(STsdbSnapPartition* x, STsdbSnapPartition* y) { if (x->fid < y->fid) return -1; if (x->fid > y->fid) return 1; @@ -1183,7 +1183,7 @@ static int32_t tVersionRangeCmprFn(SVersionRange* x, SVersionRange* y) { return 0; } -static int32_t tsdbSnapRangeCmprFn(STSnapRange* x, STSnapRange* y) { +static int32_t tsdbFileSetRangeCmprFn(STFileSetRange* x, STFileSetRange* y) { if (x->fid < y->fid) return -1; if (x->fid > y->fid) return 1; return 0; @@ -1462,8 +1462,8 @@ _err: return -1; } -int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TSnapRangeArray** ppRanges) { - TSnapRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TSnapRangeArray)); +int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TFileSetRangeArray** ppRanges) { + TFileSetRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TFileSetRangeArray)); if (pDiff == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -1472,7 +1472,7 @@ int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TSnapRangeArray** STsdbSnapPartition* part; TARRAY2_FOREACH(pList, part) { - STSnapRange* r = taosMemoryCalloc(1, sizeof(STSnapRange)); + STFileSetRange* r = taosMemoryCalloc(1, sizeof(STFileSetRange)); if (r == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -1493,7 +1493,7 @@ int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TSnapRangeArray** r->sver = maxVerValid + 1; r->ever = VERSION_MAX; tsdbDebug("range diff fid:%" PRId64 ", sver:%" PRId64 ", ever:%" PRId64, part->fid, r->sver, r->ever); - int32_t code = TARRAY2_SORT_INSERT(pDiff, r, tsdbSnapRangeCmprFn); + int32_t code = TARRAY2_SORT_INSERT(pDiff, r, tsdbFileSetRangeCmprFn); ASSERT(code == 0); } ppRanges[0] = pDiff; @@ -1503,12 +1503,12 @@ int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TSnapRangeArray** _err: if (pDiff) { - tsdbSnapRangeArrayDestroy(&pDiff); + tsdbFileSetRangeArrayDestroy(&pDiff); } return -1; } -void tsdbSnapRangeArrayDestroy(TSnapRangeArray** ppSnap) { +void tsdbFileSetRangeArrayDestroy(TFileSetRangeArray** ppSnap) { if (ppSnap && ppSnap[0]) { TARRAY2_DESTROY(ppSnap[0], tsdbTSnapRangeClear); taosMemoryFree(ppSnap[0]); diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 34b508388f..f65d9085fd 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -29,7 +29,7 @@ struct SVSnapReader { SMetaSnapReader *pMetaReader; // tsdb int8_t tsdbDone; - TSnapRangeArray *pRanges; + TFileSetRangeArray *pRanges; STsdbSnapReader *pTsdbReader; // tq int8_t tqHandleDone; @@ -45,11 +45,11 @@ struct SVSnapReader { SStreamStateReader *pStreamStateReader; // rsma int8_t rsmaDone; - TSnapRangeArray *pRsmaRanges[TSDB_RETENTION_L2]; + TFileSetRangeArray *pRsmaRanges[TSDB_RETENTION_L2]; SRSmaSnapReader *pRsmaReader; }; -static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TSnapRangeArray **ppRanges) { +static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TFileSetRangeArray **ppRanges) { int32_t code = -1; STsdbSnapPartList *pList = tsdbSnapPartListCreate(); if (pList == NULL) { @@ -69,7 +69,7 @@ _out: return code; } -static TSnapRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int32_t tsdbTyp) { +static TFileSetRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int32_t tsdbTyp) { ASSERTS(sizeof(pReader->pRsmaRanges) / sizeof(pReader->pRsmaRanges[0]) == 2, "Unexpected array size"); switch (tsdbTyp) { case SNAP_DATA_TSDB: @@ -94,7 +94,7 @@ static int32_t vnodeSnapReaderDoSnapInfo(SVSnapReader *pReader, SSnapshotParam * goto _out; } - TSnapRangeArray **ppRanges = NULL; + TFileSetRangeArray **ppRanges = NULL; int32_t offset = 0; while (offset + sizeof(SSyncTLV) < datHead->len) { @@ -152,9 +152,9 @@ _err: static void vnodeSnapReaderDestroyTsdbRanges(SVSnapReader *pReader) { int32_t tsdbTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; for (int32_t j = 0; j < TSDB_RETENTION_MAX; ++j) { - TSnapRangeArray **ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, tsdbTyps[j]); + TFileSetRangeArray **ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, tsdbTyps[j]); if (ppRanges == NULL) continue; - tsdbSnapRangeArrayDestroy(ppRanges); + tsdbFileSetRangeArrayDestroy(ppRanges); } } @@ -455,7 +455,7 @@ struct SVSnapWriter { // meta SMetaSnapWriter *pMetaSnapWriter; // tsdb - TSnapRangeArray *pRanges; + TFileSetRangeArray *pRanges; STsdbSnapWriter *pTsdbSnapWriter; // tq STqSnapWriter *pTqSnapWriter; @@ -465,11 +465,11 @@ struct SVSnapWriter { SStreamTaskWriter *pStreamTaskWriter; SStreamStateWriter *pStreamStateWriter; // rsma - TSnapRangeArray *pRsmaRanges[TSDB_RETENTION_L2]; + TFileSetRangeArray *pRsmaRanges[TSDB_RETENTION_L2]; SRSmaSnapWriter *pRsmaSnapWriter; }; -TSnapRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t tsdbTyp) { +TFileSetRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t tsdbTyp) { ASSERTS(sizeof(pWriter->pRsmaRanges) / sizeof(pWriter->pRsmaRanges[0]) == 2, "Unexpected array size"); switch (tsdbTyp) { case SNAP_DATA_TSDB: @@ -494,7 +494,7 @@ static int32_t vnodeSnapWriterDoSnapInfo(SVSnapWriter *pWriter, SSnapshotParam * goto _out; } - TSnapRangeArray **ppRanges = NULL; + TFileSetRangeArray **ppRanges = NULL; int32_t offset = 0; while (offset + sizeof(SSyncTLV) < datHead->len) { @@ -576,9 +576,9 @@ _err: static void vnodeSnapWriterDestroyTsdbRanges(SVSnapWriter *pWriter) { int32_t tsdbTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; for (int32_t j = 0; j < TSDB_RETENTION_MAX; ++j) { - TSnapRangeArray **ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, tsdbTyps[j]); + TFileSetRangeArray **ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, tsdbTyps[j]); if (ppRanges == NULL) continue; - tsdbSnapRangeArrayDestroy(ppRanges); + tsdbFileSetRangeArrayDestroy(ppRanges); } } From ddcabcfa4afc3f5a46b3ab6c1e40c5647ba17f51 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 13 Nov 2023 14:39:28 +0800 Subject: [PATCH 02/34] refact: remove unused fn tSerializeSnapRangeArray --- source/dnode/vnode/src/inc/tsdb.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 9cc6cd1132..99b92ace3a 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -685,8 +685,6 @@ typedef struct STFileSetRange STFileSetRange; typedef TARRAY2(STFileSetRange *) TFileSetRangeArray; // disjoint ranges // util -int32_t tSerializeSnapRangeArray(void *buf, int32_t bufLen, TFileSetRangeArray *pSnapR); -int32_t tDeserializeSnapRangeArray(void *buf, int32_t bufLen, TFileSetRangeArray *pSnapR); void tsdbFileSetRangeArrayDestroy(TFileSetRangeArray **ppSnap); SHashObj *tsdbGetSnapRangeHash(TFileSetRangeArray *pRanges); From 3ad68bdfea9c14acf1a886a85af65e25c2f42771 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 30 Nov 2023 17:35:28 +0800 Subject: [PATCH 03/34] enh: add tsdbSnapInfo.c --- source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 source/dnode/vnode/src/tsdb/tsdbSnapInfo.c diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c new file mode 100644 index 0000000000..e69de29bb2 From b8b53a4b0e7f847dbb65c2e9f93aa4192b4047ef Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 30 Nov 2023 17:53:16 +0800 Subject: [PATCH 04/34] refact: put source code of snap info into tsdbSnapInfo.c --- source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 456 +++++++++++++++++++++ source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 436 -------------------- 2 files changed, 456 insertions(+), 436 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index e69de29bb2..e70d0eaabc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -0,0 +1,456 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" +#include "tsdbDataFileRW.h" +#include "tsdbFS2.h" +#include "tsdbFSetRW.h" +#include "tsdbIter.h" +#include "tsdbSttFileRW.h" + +// STsdbSnapPartition ===================================== +static int32_t tsdbSnapPartCmprFn(STsdbSnapPartition* x, STsdbSnapPartition* y) { + if (x->fid < y->fid) return -1; + if (x->fid > y->fid) return 1; + return 0; +} + +static int32_t tVersionRangeCmprFn(SVersionRange* x, SVersionRange* y) { + if (x->minVer < y->minVer) return -1; + if (x->minVer > y->minVer) return 1; + if (x->maxVer < y->maxVer) return -1; + if (x->maxVer > y->maxVer) return 1; + return 0; +} + +static int32_t tsdbFileSetRangeCmprFn(STFileSetRange* x, STFileSetRange* y) { + if (x->fid < y->fid) return -1; + if (x->fid > y->fid) return 1; + return 0; +} + +STsdbSnapPartition* tsdbSnapPartitionCreate() { + STsdbSnapPartition* pSP = taosMemoryCalloc(1, sizeof(STsdbSnapPartition)); + if (pSP == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) { + TARRAY2_INIT(&pSP->verRanges[i]); + } + return pSP; +} + +void tsdbSnapPartitionClear(STsdbSnapPartition** ppSP) { + if (ppSP == NULL || ppSP[0] == NULL) { + return; + } + for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) { + TARRAY2_DESTROY(&ppSP[0]->verRanges[i], NULL); + } + taosMemoryFree(ppSP[0]); + ppSP[0] = NULL; +} + +static int32_t tsdbFTypeToSRangeTyp(tsdb_ftype_t ftype) { + switch (ftype) { + case TSDB_FTYPE_HEAD: + return TSDB_SNAP_RANGE_TYP_HEAD; + case TSDB_FTYPE_DATA: + return TSDB_SNAP_RANGE_TYP_DATA; + case TSDB_FTYPE_SMA: + return TSDB_SNAP_RANGE_TYP_SMA; + case TSDB_FTYPE_TOMB: + return TSDB_SNAP_RANGE_TYP_TOMB; + case TSDB_FTYPE_STT: + return TSDB_SNAP_RANGE_TYP_STT; + } + return TSDB_SNAP_RANGE_TYP_MAX; +} + +static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbSnapPartition** ppSP) { + STsdbSnapPartition* p = tsdbSnapPartitionCreate(); + if (p == NULL) { + goto _err; + } + + p->fid = fset->fid; + + int32_t code = 0; + int32_t typ = 0; + int32_t corrupt = false; + int32_t count = 0; + for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset->farr[ftype] == NULL) continue; + typ = tsdbFTypeToSRangeTyp(ftype); + ASSERT(typ < TSDB_SNAP_RANGE_TYP_MAX); + STFile* f = fset->farr[ftype]->f; + if (f->maxVer > fset->maxVerValid) { + corrupt = true; + tsdbError("skip incomplete data file: fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 + ", ftype: %d", + fset->fid, fset->maxVerValid, f->minVer, f->maxVer, ftype); + continue; + } + count++; + SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer}; + code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); + ASSERT(code == 0); + } + + typ = TSDB_SNAP_RANGE_TYP_STT; + const SSttLvl* lvl; + TARRAY2_FOREACH(fset->lvlArr, lvl) { + STFileObj* fobj; + TARRAY2_FOREACH(lvl->fobjArr, fobj) { + STFile* f = fobj->f; + if (f->maxVer > fset->maxVerValid) { + corrupt = true; + tsdbError("skip incomplete stt file.fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 + ", ftype: %d", + fset->fid, fset->maxVerValid, f->minVer, f->maxVer, typ); + continue; + } + count++; + SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer}; + code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); + ASSERT(code == 0); + } + } + if (corrupt && count == 0) { + SVersionRange vr = {.minVer = VERSION_MIN, .maxVer = fset->maxVerValid}; + code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); + ASSERT(code == 0); + } + ppSP[0] = p; + return 0; + +_err: + tsdbSnapPartitionClear(&p); + return -1; +} + +STsdbSnapPartList* tsdbSnapPartListCreate() { + STsdbSnapPartList* pList = taosMemoryCalloc(1, sizeof(STsdbSnapPartList)); + if (pList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + TARRAY2_INIT(pList); + return pList; +} + +static STsdbSnapPartList* tsdbGetSnapPartList(STFileSystem* fs) { + STsdbSnapPartList* pList = tsdbSnapPartListCreate(); + if (pList == NULL) { + return NULL; + } + + int32_t code = 0; + taosThreadMutexLock(&fs->tsdb->mutex); + STFileSet* fset; + TARRAY2_FOREACH(fs->fSetArr, fset) { + STsdbSnapPartition* pItem = NULL; + if (tsdbTFileSetToSnapPart(fset, &pItem) < 0) { + code = -1; + break; + } + ASSERT(pItem != NULL); + code = TARRAY2_SORT_INSERT(pList, pItem, tsdbSnapPartCmprFn); + ASSERT(code == 0); + } + taosThreadMutexUnlock(&fs->tsdb->mutex); + + if (code) { + TARRAY2_DESTROY(pList, tsdbSnapPartitionClear); + taosMemoryFree(pList); + pList = NULL; + } + return pList; +} + +int32_t tTsdbSnapPartListDataLenCalc(STsdbSnapPartList* pList) { + int32_t hdrLen = sizeof(int32_t); + int32_t datLen = 0; + + int8_t msgVer = 1; + int32_t len = TARRAY2_SIZE(pList); + hdrLen += sizeof(msgVer); + hdrLen += sizeof(len); + datLen += hdrLen; + + for (int32_t u = 0; u < len; u++) { + STsdbSnapPartition* p = TARRAY2_GET(pList, u); + int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; + int32_t uItem = 0; + uItem += sizeof(STsdbSnapPartition); + uItem += sizeof(typMax); + + for (int32_t i = 0; i < typMax; i++) { + int32_t iLen = TARRAY2_SIZE(&p->verRanges[i]); + int32_t jItem = 0; + jItem += sizeof(SVersionRange); + jItem += sizeof(int64_t); + uItem += sizeof(iLen) + jItem * iLen; + } + datLen += uItem; + } + return datLen; +} + +int32_t tSerializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + int8_t reserved8 = 0; + int16_t reserved16 = 0; + int64_t reserved64 = 0; + + int8_t msgVer = 1; + int32_t len = TARRAY2_SIZE(pList); + + if (tStartEncode(&encoder) < 0) goto _err; + if (tEncodeI8(&encoder, msgVer) < 0) goto _err; + if (tEncodeI32(&encoder, len) < 0) goto _err; + + for (int32_t u = 0; u < len; u++) { + STsdbSnapPartition* p = TARRAY2_GET(pList, u); + if (tEncodeI64(&encoder, p->fid) < 0) goto _err; + if (tEncodeI8(&encoder, p->stat) < 0) goto _err; + if (tEncodeI8(&encoder, reserved8) < 0) goto _err; + if (tEncodeI16(&encoder, reserved16) < 0) goto _err; + + int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; + if (tEncodeI32(&encoder, typMax) < 0) goto _err; + + for (int32_t i = 0; i < typMax; i++) { + SVerRangeList* iList = &p->verRanges[i]; + int32_t iLen = TARRAY2_SIZE(iList); + + if (tEncodeI32(&encoder, iLen) < 0) goto _err; + for (int32_t j = 0; j < iLen; j++) { + SVersionRange r = TARRAY2_GET(iList, j); + if (tEncodeI64(&encoder, r.minVer) < 0) goto _err; + if (tEncodeI64(&encoder, r.maxVer) < 0) goto _err; + if (tEncodeI64(&encoder, reserved64) < 0) goto _err; + } + } + } + + tEndEncode(&encoder); + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; + +_err: + tEncoderClear(&encoder); + return -1; +} + +int32_t tDeserializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + int8_t reserved8 = 0; + int16_t reserved16 = 0; + int64_t reserved64 = 0; + + STsdbSnapPartition* p = NULL; + + int8_t msgVer = 0; + int32_t len = 0; + if (tStartDecode(&decoder) < 0) goto _err; + if (tDecodeI8(&decoder, &msgVer) < 0) goto _err; + if (tDecodeI32(&decoder, &len) < 0) goto _err; + + for (int32_t u = 0; u < len; u++) { + p = tsdbSnapPartitionCreate(); + if (p == NULL) goto _err; + if (tDecodeI64(&decoder, &p->fid) < 0) goto _err; + if (tDecodeI8(&decoder, &p->stat) < 0) goto _err; + if (tDecodeI8(&decoder, &reserved8) < 0) goto _err; + if (tDecodeI16(&decoder, &reserved16) < 0) goto _err; + + int32_t typMax = 0; + if (tDecodeI32(&decoder, &typMax) < 0) goto _err; + + for (int32_t i = 0; i < typMax; i++) { + SVerRangeList* iList = &p->verRanges[i]; + int32_t iLen = 0; + if (tDecodeI32(&decoder, &iLen) < 0) goto _err; + for (int32_t j = 0; j < iLen; j++) { + SVersionRange r = {0}; + if (tDecodeI64(&decoder, &r.minVer) < 0) goto _err; + if (tDecodeI64(&decoder, &r.maxVer) < 0) goto _err; + if (tDecodeI64(&decoder, &reserved64) < 0) goto _err; + TARRAY2_APPEND(iList, r); + } + } + TARRAY2_APPEND(pList, p); + p = NULL; + } + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; + +_err: + if (p) { + tsdbSnapPartitionClear(&p); + } + tDecoderClear(&decoder); + return -1; +} + +int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TFileSetRangeArray** ppRanges) { + TFileSetRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TFileSetRangeArray)); + if (pDiff == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + TARRAY2_INIT(pDiff); + + STsdbSnapPartition* part; + TARRAY2_FOREACH(pList, part) { + STFileSetRange* r = taosMemoryCalloc(1, sizeof(STFileSetRange)); + if (r == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + int64_t maxVerValid = -1; + int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; + for (int32_t i = 0; i < typMax; i++) { + SVerRangeList* iList = &part->verRanges[i]; + SVersionRange vr = {0}; + TARRAY2_FOREACH(iList, vr) { + if (vr.maxVer < vr.minVer) { + continue; + } + maxVerValid = TMAX(maxVerValid, vr.maxVer); + } + } + r->fid = part->fid; + r->sver = maxVerValid + 1; + r->ever = VERSION_MAX; + tsdbDebug("range diff fid:%" PRId64 ", sver:%" PRId64 ", ever:%" PRId64, part->fid, r->sver, r->ever); + int32_t code = TARRAY2_SORT_INSERT(pDiff, r, tsdbFileSetRangeCmprFn); + ASSERT(code == 0); + } + ppRanges[0] = pDiff; + + tsdbInfo("pDiff size:%d", TARRAY2_SIZE(pDiff)); + return 0; + +_err: + if (pDiff) { + tsdbFileSetRangeArrayDestroy(&pDiff); + } + return -1; +} + +void tsdbFileSetRangeArrayDestroy(TFileSetRangeArray** ppSnap) { + if (ppSnap && ppSnap[0]) { + TARRAY2_DESTROY(ppSnap[0], tsdbTSnapRangeClear); + taosMemoryFree(ppSnap[0]); + ppSnap[0] = NULL; + } +} + +void tsdbSnapPartListDestroy(STsdbSnapPartList** ppList) { + if (ppList == NULL || ppList[0] == NULL) return; + + TARRAY2_DESTROY(ppList[0], tsdbSnapPartitionClear); + taosMemoryFree(ppList[0]); + ppList[0] = NULL; +} + +ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) { + if (!VND_IS_RSMA(pVnode)) { + return pVnode->pTsdb->pFS->fsstate; + } + for (int32_t lvl = 0; lvl < TSDB_RETENTION_MAX; ++lvl) { + STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, lvl); + if (pTsdb && pTsdb->pFS->fsstate != TSDB_FS_STATE_NORMAL) { + return TSDB_FS_STATE_INCOMPLETE; + } + } + return TSDB_FS_STATE_NORMAL; +} + +int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { + int code = -1; + int32_t tsdbMaxCnt = (!VND_IS_RSMA(pVnode) ? 1 : TSDB_RETENTION_MAX); + int32_t subTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; + STsdbSnapPartList* pLists[TSDB_RETENTION_MAX] = {0}; + + // get part list + for (int32_t j = 0; j < tsdbMaxCnt; ++j) { + STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, j); + pLists[j] = tsdbGetSnapPartList(pTsdb->pFS); + if (pLists[j] == NULL) goto _out; + } + + // estimate bufLen and prepare + int32_t bufLen = sizeof(SSyncTLV); // typ: TDMT_SYNC_PREP_SNAPSHOT or TDMT_SYNC_PREP_SNAPSOT_REPLY + for (int32_t j = 0; j < tsdbMaxCnt; ++j) { + bufLen += sizeof(SSyncTLV); // subTyps[j] + bufLen += tTsdbSnapPartListDataLenCalc(pLists[j]); + } + + tsdbInfo("vgId:%d, allocate %d bytes for data of snapshot info.", TD_VID(pVnode), bufLen); + + void* data = taosMemoryRealloc(pSnap->data, bufLen); + if (data == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tsdbError("vgId:%d, failed to realloc memory for data of snapshot info. bytes:%d", TD_VID(pVnode), bufLen); + goto _out; + } + pSnap->data = data; + + // header + SSyncTLV* head = data; + head->len = 0; + head->typ = pSnap->type; + int32_t offset = sizeof(SSyncTLV); + int32_t tlen = 0; + + // fill snapshot info + for (int32_t j = 0; j < tsdbMaxCnt; ++j) { + // subHead + SSyncTLV* subHead = (void*)((char*)data + offset); + subHead->typ = subTyps[j]; + ASSERT(subHead->val == (char*)data + offset + sizeof(SSyncTLV)); + + if ((tlen = tSerializeTsdbSnapPartList(subHead->val, bufLen - offset - sizeof(SSyncTLV), pLists[j])) < 0) { + tsdbError("vgId:%d, failed to serialize snap partition list of tsdb %d since %s", TD_VID(pVnode), j, terrstr()); + goto _out; + } + subHead->len = tlen; + offset += sizeof(SSyncTLV) + tlen; + } + + // total length of subfields + head->len = offset - sizeof(SSyncTLV); + ASSERT(offset <= bufLen); + code = 0; + +_out: + for (int32_t j = 0; j < tsdbMaxCnt; ++j) { + if (pLists[j] == NULL) continue; + tsdbSnapPartListDestroy(&pLists[j]); + } + + return code; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 9d3efcfdae..48872404ed 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -1167,439 +1167,3 @@ _exit: } return code; } - -// STsdbSnapPartition ===================================== -static int32_t tsdbSnapPartCmprFn(STsdbSnapPartition* x, STsdbSnapPartition* y) { - if (x->fid < y->fid) return -1; - if (x->fid > y->fid) return 1; - return 0; -} - -static int32_t tVersionRangeCmprFn(SVersionRange* x, SVersionRange* y) { - if (x->minVer < y->minVer) return -1; - if (x->minVer > y->minVer) return 1; - if (x->maxVer < y->maxVer) return -1; - if (x->maxVer > y->maxVer) return 1; - return 0; -} - -static int32_t tsdbFileSetRangeCmprFn(STFileSetRange* x, STFileSetRange* y) { - if (x->fid < y->fid) return -1; - if (x->fid > y->fid) return 1; - return 0; -} - -STsdbSnapPartition* tsdbSnapPartitionCreate() { - STsdbSnapPartition* pSP = taosMemoryCalloc(1, sizeof(STsdbSnapPartition)); - if (pSP == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) { - TARRAY2_INIT(&pSP->verRanges[i]); - } - return pSP; -} - -void tsdbSnapPartitionClear(STsdbSnapPartition** ppSP) { - if (ppSP == NULL || ppSP[0] == NULL) { - return; - } - for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) { - TARRAY2_DESTROY(&ppSP[0]->verRanges[i], NULL); - } - taosMemoryFree(ppSP[0]); - ppSP[0] = NULL; -} - -static int32_t tsdbFTypeToSRangeTyp(tsdb_ftype_t ftype) { - switch (ftype) { - case TSDB_FTYPE_HEAD: - return TSDB_SNAP_RANGE_TYP_HEAD; - case TSDB_FTYPE_DATA: - return TSDB_SNAP_RANGE_TYP_DATA; - case TSDB_FTYPE_SMA: - return TSDB_SNAP_RANGE_TYP_SMA; - case TSDB_FTYPE_TOMB: - return TSDB_SNAP_RANGE_TYP_TOMB; - case TSDB_FTYPE_STT: - return TSDB_SNAP_RANGE_TYP_STT; - } - return TSDB_SNAP_RANGE_TYP_MAX; -} - -static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbSnapPartition** ppSP) { - STsdbSnapPartition* p = tsdbSnapPartitionCreate(); - if (p == NULL) { - goto _err; - } - - p->fid = fset->fid; - - int32_t code = 0; - int32_t typ = 0; - int32_t corrupt = false; - int32_t count = 0; - for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { - if (fset->farr[ftype] == NULL) continue; - typ = tsdbFTypeToSRangeTyp(ftype); - ASSERT(typ < TSDB_SNAP_RANGE_TYP_MAX); - STFile* f = fset->farr[ftype]->f; - if (f->maxVer > fset->maxVerValid) { - corrupt = true; - tsdbError("skip incomplete data file: fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 - ", ftype: %d", - fset->fid, fset->maxVerValid, f->minVer, f->maxVer, ftype); - continue; - } - count++; - SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer}; - code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); - ASSERT(code == 0); - } - - typ = TSDB_SNAP_RANGE_TYP_STT; - const SSttLvl* lvl; - TARRAY2_FOREACH(fset->lvlArr, lvl) { - STFileObj* fobj; - TARRAY2_FOREACH(lvl->fobjArr, fobj) { - STFile* f = fobj->f; - if (f->maxVer > fset->maxVerValid) { - corrupt = true; - tsdbError("skip incomplete stt file.fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 - ", ftype: %d", - fset->fid, fset->maxVerValid, f->minVer, f->maxVer, typ); - continue; - } - count++; - SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer}; - code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); - ASSERT(code == 0); - } - } - if (corrupt && count == 0) { - SVersionRange vr = {.minVer = VERSION_MIN, .maxVer = fset->maxVerValid}; - code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); - ASSERT(code == 0); - } - ppSP[0] = p; - return 0; - -_err: - tsdbSnapPartitionClear(&p); - return -1; -} - -STsdbSnapPartList* tsdbSnapPartListCreate() { - STsdbSnapPartList* pList = taosMemoryCalloc(1, sizeof(STsdbSnapPartList)); - if (pList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - TARRAY2_INIT(pList); - return pList; -} - -static STsdbSnapPartList* tsdbGetSnapPartList(STFileSystem* fs) { - STsdbSnapPartList* pList = tsdbSnapPartListCreate(); - if (pList == NULL) { - return NULL; - } - - int32_t code = 0; - taosThreadMutexLock(&fs->tsdb->mutex); - STFileSet* fset; - TARRAY2_FOREACH(fs->fSetArr, fset) { - STsdbSnapPartition* pItem = NULL; - if (tsdbTFileSetToSnapPart(fset, &pItem) < 0) { - code = -1; - break; - } - ASSERT(pItem != NULL); - code = TARRAY2_SORT_INSERT(pList, pItem, tsdbSnapPartCmprFn); - ASSERT(code == 0); - } - taosThreadMutexUnlock(&fs->tsdb->mutex); - - if (code) { - TARRAY2_DESTROY(pList, tsdbSnapPartitionClear); - taosMemoryFree(pList); - pList = NULL; - } - return pList; -} - -int32_t tTsdbSnapPartListDataLenCalc(STsdbSnapPartList* pList) { - int32_t hdrLen = sizeof(int32_t); - int32_t datLen = 0; - - int8_t msgVer = 1; - int32_t len = TARRAY2_SIZE(pList); - hdrLen += sizeof(msgVer); - hdrLen += sizeof(len); - datLen += hdrLen; - - for (int32_t u = 0; u < len; u++) { - STsdbSnapPartition* p = TARRAY2_GET(pList, u); - int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; - int32_t uItem = 0; - uItem += sizeof(STsdbSnapPartition); - uItem += sizeof(typMax); - - for (int32_t i = 0; i < typMax; i++) { - int32_t iLen = TARRAY2_SIZE(&p->verRanges[i]); - int32_t jItem = 0; - jItem += sizeof(SVersionRange); - jItem += sizeof(int64_t); - uItem += sizeof(iLen) + jItem * iLen; - } - datLen += uItem; - } - return datLen; -} - -int32_t tSerializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - - int8_t reserved8 = 0; - int16_t reserved16 = 0; - int64_t reserved64 = 0; - - int8_t msgVer = 1; - int32_t len = TARRAY2_SIZE(pList); - - if (tStartEncode(&encoder) < 0) goto _err; - if (tEncodeI8(&encoder, msgVer) < 0) goto _err; - if (tEncodeI32(&encoder, len) < 0) goto _err; - - for (int32_t u = 0; u < len; u++) { - STsdbSnapPartition* p = TARRAY2_GET(pList, u); - if (tEncodeI64(&encoder, p->fid) < 0) goto _err; - if (tEncodeI8(&encoder, p->stat) < 0) goto _err; - if (tEncodeI8(&encoder, reserved8) < 0) goto _err; - if (tEncodeI16(&encoder, reserved16) < 0) goto _err; - - int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; - if (tEncodeI32(&encoder, typMax) < 0) goto _err; - - for (int32_t i = 0; i < typMax; i++) { - SVerRangeList* iList = &p->verRanges[i]; - int32_t iLen = TARRAY2_SIZE(iList); - - if (tEncodeI32(&encoder, iLen) < 0) goto _err; - for (int32_t j = 0; j < iLen; j++) { - SVersionRange r = TARRAY2_GET(iList, j); - if (tEncodeI64(&encoder, r.minVer) < 0) goto _err; - if (tEncodeI64(&encoder, r.maxVer) < 0) goto _err; - if (tEncodeI64(&encoder, reserved64) < 0) goto _err; - } - } - } - - tEndEncode(&encoder); - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; - -_err: - tEncoderClear(&encoder); - return -1; -} - -int32_t tDeserializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - int8_t reserved8 = 0; - int16_t reserved16 = 0; - int64_t reserved64 = 0; - - STsdbSnapPartition* p = NULL; - - int8_t msgVer = 0; - int32_t len = 0; - if (tStartDecode(&decoder) < 0) goto _err; - if (tDecodeI8(&decoder, &msgVer) < 0) goto _err; - if (tDecodeI32(&decoder, &len) < 0) goto _err; - - for (int32_t u = 0; u < len; u++) { - p = tsdbSnapPartitionCreate(); - if (p == NULL) goto _err; - if (tDecodeI64(&decoder, &p->fid) < 0) goto _err; - if (tDecodeI8(&decoder, &p->stat) < 0) goto _err; - if (tDecodeI8(&decoder, &reserved8) < 0) goto _err; - if (tDecodeI16(&decoder, &reserved16) < 0) goto _err; - - int32_t typMax = 0; - if (tDecodeI32(&decoder, &typMax) < 0) goto _err; - - for (int32_t i = 0; i < typMax; i++) { - SVerRangeList* iList = &p->verRanges[i]; - int32_t iLen = 0; - if (tDecodeI32(&decoder, &iLen) < 0) goto _err; - for (int32_t j = 0; j < iLen; j++) { - SVersionRange r = {0}; - if (tDecodeI64(&decoder, &r.minVer) < 0) goto _err; - if (tDecodeI64(&decoder, &r.maxVer) < 0) goto _err; - if (tDecodeI64(&decoder, &reserved64) < 0) goto _err; - TARRAY2_APPEND(iList, r); - } - } - TARRAY2_APPEND(pList, p); - p = NULL; - } - - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; - -_err: - if (p) { - tsdbSnapPartitionClear(&p); - } - tDecoderClear(&decoder); - return -1; -} - -int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TFileSetRangeArray** ppRanges) { - TFileSetRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TFileSetRangeArray)); - if (pDiff == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - TARRAY2_INIT(pDiff); - - STsdbSnapPartition* part; - TARRAY2_FOREACH(pList, part) { - STFileSetRange* r = taosMemoryCalloc(1, sizeof(STFileSetRange)); - if (r == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - int64_t maxVerValid = -1; - int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; - for (int32_t i = 0; i < typMax; i++) { - SVerRangeList* iList = &part->verRanges[i]; - SVersionRange vr = {0}; - TARRAY2_FOREACH(iList, vr) { - if (vr.maxVer < vr.minVer) { - continue; - } - maxVerValid = TMAX(maxVerValid, vr.maxVer); - } - } - r->fid = part->fid; - r->sver = maxVerValid + 1; - r->ever = VERSION_MAX; - tsdbDebug("range diff fid:%" PRId64 ", sver:%" PRId64 ", ever:%" PRId64, part->fid, r->sver, r->ever); - int32_t code = TARRAY2_SORT_INSERT(pDiff, r, tsdbFileSetRangeCmprFn); - ASSERT(code == 0); - } - ppRanges[0] = pDiff; - - tsdbInfo("pDiff size:%d", TARRAY2_SIZE(pDiff)); - return 0; - -_err: - if (pDiff) { - tsdbFileSetRangeArrayDestroy(&pDiff); - } - return -1; -} - -void tsdbFileSetRangeArrayDestroy(TFileSetRangeArray** ppSnap) { - if (ppSnap && ppSnap[0]) { - TARRAY2_DESTROY(ppSnap[0], tsdbTSnapRangeClear); - taosMemoryFree(ppSnap[0]); - ppSnap[0] = NULL; - } -} - -void tsdbSnapPartListDestroy(STsdbSnapPartList** ppList) { - if (ppList == NULL || ppList[0] == NULL) return; - - TARRAY2_DESTROY(ppList[0], tsdbSnapPartitionClear); - taosMemoryFree(ppList[0]); - ppList[0] = NULL; -} - -ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) { - if (!VND_IS_RSMA(pVnode)) { - return pVnode->pTsdb->pFS->fsstate; - } - for (int32_t lvl = 0; lvl < TSDB_RETENTION_MAX; ++lvl) { - STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, lvl); - if (pTsdb && pTsdb->pFS->fsstate != TSDB_FS_STATE_NORMAL) { - return TSDB_FS_STATE_INCOMPLETE; - } - } - return TSDB_FS_STATE_NORMAL; -} - -int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { - int code = -1; - int32_t tsdbMaxCnt = (!VND_IS_RSMA(pVnode) ? 1 : TSDB_RETENTION_MAX); - int32_t subTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; - STsdbSnapPartList* pLists[TSDB_RETENTION_MAX] = {0}; - - for (int32_t j = 0; j < tsdbMaxCnt; ++j) { - STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, j); - pLists[j] = tsdbGetSnapPartList(pTsdb->pFS); - if (pLists[j] == NULL) goto _out; - } - - // estimate bufLen and prepare - int32_t bufLen = sizeof(SSyncTLV); // typ: TDMT_SYNC_PREP_SNAPSHOT or TDMT_SYNC_PREP_SNAPSOT_REPLY - for (int32_t j = 0; j < tsdbMaxCnt; ++j) { - bufLen += sizeof(SSyncTLV); // subTyps[j] - bufLen += tTsdbSnapPartListDataLenCalc(pLists[j]); - } - - tsdbInfo("vgId:%d, allocate %d bytes for data of snapshot info.", TD_VID(pVnode), bufLen); - - void* data = taosMemoryRealloc(pSnap->data, bufLen); - if (data == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tsdbError("vgId:%d, failed to realloc memory for data of snapshot info. bytes:%d", TD_VID(pVnode), bufLen); - goto _out; - } - pSnap->data = data; - - // header - SSyncTLV* head = data; - head->len = 0; - head->typ = pSnap->type; - int32_t offset = sizeof(SSyncTLV); - int32_t tlen = 0; - - // fill snapshot info - for (int32_t j = 0; j < tsdbMaxCnt; ++j) { - if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { - } - - // subHead - SSyncTLV* subHead = (void*)((char*)data + offset); - subHead->typ = subTyps[j]; - ASSERT(subHead->val == (char*)data + offset + sizeof(SSyncTLV)); - - if ((tlen = tSerializeTsdbSnapPartList(subHead->val, bufLen - offset - sizeof(SSyncTLV), pLists[j])) < 0) { - tsdbError("vgId:%d, failed to serialize snap partition list of tsdb %d since %s", TD_VID(pVnode), j, terrstr()); - goto _out; - } - subHead->len = tlen; - offset += sizeof(SSyncTLV) + tlen; - } - - head->len = offset - sizeof(SSyncTLV); - ASSERT(offset <= bufLen); - code = 0; - -_out: - for (int32_t j = 0; j < tsdbMaxCnt; ++j) { - if (pLists[j] == NULL) continue; - tsdbSnapPartListDestroy(&pLists[j]); - } - - return code; -} \ No newline at end of file From 4487eeacb58eebdb2cc897a32e9c7dafab653e6a Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 30 Nov 2023 19:09:32 +0800 Subject: [PATCH 05/34] refact: improve funcs dealing with SnapInfo for vnode snap reader and writer --- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 100 ++++++++++++--------- 1 file changed, 60 insertions(+), 40 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index f65d9085fd..941660f776 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -16,6 +16,26 @@ #include "tsdb.h" #include "vnd.h" +static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TFileSetRangeArray **ppRanges) { + int32_t code = -1; + STsdbSnapPartList *pList = tsdbSnapPartListCreate(); + if (pList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _out; + } + if (tDeserializeTsdbSnapPartList(buf, bufLen, pList) < 0) { + terrno = TSDB_CODE_INVALID_DATA_FMT; + goto _out; + } + if (tsdbSnapPartListToRangeDiff(pList, ppRanges) < 0) { + goto _out; + } + code = 0; +_out: + tsdbSnapPartListDestroy(&pList); + return code; +} + // SVSnapReader ======================================================== struct SVSnapReader { SVnode *pVnode; @@ -49,26 +69,6 @@ struct SVSnapReader { SRSmaSnapReader *pRsmaReader; }; -static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TFileSetRangeArray **ppRanges) { - int32_t code = -1; - STsdbSnapPartList *pList = tsdbSnapPartListCreate(); - if (pList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _out; - } - if (tDeserializeTsdbSnapPartList(buf, bufLen, pList) < 0) { - terrno = TSDB_CODE_INVALID_DATA_FMT; - goto _out; - } - if (tsdbSnapPartListToRangeDiff(pList, ppRanges) < 0) { - goto _out; - } - code = 0; -_out: - tsdbSnapPartListDestroy(&pList); - return code; -} - static TFileSetRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int32_t tsdbTyp) { ASSERTS(sizeof(pReader->pRsmaRanges) / sizeof(pReader->pRsmaRanges[0]) == 2, "Unexpected array size"); switch (tsdbTyp) { @@ -83,7 +83,7 @@ static TFileSetRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, } } -static int32_t vnodeSnapReaderDoSnapInfo(SVSnapReader *pReader, SSnapshotParam *pParam) { +static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotParam *pParam) { SVnode *pVnode = pReader->pVnode; int32_t code = -1; @@ -102,14 +102,24 @@ static int32_t vnodeSnapReaderDoSnapInfo(SVSnapReader *pReader, SSnapshotParam * offset += sizeof(SSyncTLV) + subField->len; void *buf = subField->val; int32_t bufLen = subField->len; - ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, subField->typ); - if (ppRanges == NULL) { - vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); - goto _out; - } - if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { - vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); - goto _out; + + switch (subField->typ) { + case SNAP_DATA_TSDB: + case SNAP_DATA_RSMA1: + case SNAP_DATA_RSMA2: { + ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, subField->typ); + if (ppRanges == NULL) { + vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); + goto _out; + } + if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { + vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + } break; + default: + vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ); + goto _out; } } } @@ -135,7 +145,7 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader pReader->ever = ever; // snapshot info - if (vnodeSnapReaderDoSnapInfo(pReader, pParam) < 0) { + if (vnodeSnapReaderDealWithSnapInfo(pReader, pParam) < 0) { goto _err; } @@ -483,7 +493,7 @@ TFileSetRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t } } -static int32_t vnodeSnapWriterDoSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *pParam) { +static int32_t vnodeSnapWriterDealWithSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *pParam) { SVnode *pVnode = pWriter->pVnode; int32_t code = -1; @@ -502,14 +512,24 @@ static int32_t vnodeSnapWriterDoSnapInfo(SVSnapWriter *pWriter, SSnapshotParam * offset += sizeof(SSyncTLV) + subField->len; void *buf = subField->val; int32_t bufLen = subField->len; - ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, subField->typ); - if (ppRanges == NULL) { - vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); - goto _out; - } - if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { - vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); - goto _out; + + switch (subField->typ) { + case SNAP_DATA_TSDB: + case SNAP_DATA_RSMA1: + case SNAP_DATA_RSMA2: { + ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, subField->typ); + if (ppRanges == NULL) { + vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); + goto _out; + } + if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { + vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + } break; + default: + vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ); + goto _out; } } } @@ -558,7 +578,7 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter pWriter->commitID = ++pVnode->state.commitID; // snapshot info - if (vnodeSnapWriterDoSnapInfo(pWriter, pParam) < 0) { + if (vnodeSnapWriterDealWithSnapInfo(pWriter, pParam) < 0) { goto _err; } From 20b5cf8d49e98ff15c69d849fdfca07c69f53595 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 30 Nov 2023 20:30:59 +0800 Subject: [PATCH 06/34] refact: remove redundent includes from tsdbSnapInfo.c --- source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index e70d0eaabc..5aaa88511a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -14,11 +14,7 @@ */ #include "tsdb.h" -#include "tsdbDataFileRW.h" #include "tsdbFS2.h" -#include "tsdbFSetRW.h" -#include "tsdbIter.h" -#include "tsdbSttFileRW.h" // STsdbSnapPartition ===================================== static int32_t tsdbSnapPartCmprFn(STsdbSnapPartition* x, STsdbSnapPartition* y) { From ed916b1a80385223491f25e53a033e51bfd7e3fc Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 30 Nov 2023 21:02:26 +0800 Subject: [PATCH 07/34] refact: rename tsdb snap partition to fset partition --- source/dnode/vnode/src/inc/tsdb.h | 16 +++--- source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 65 +++++++++++----------- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 8 +-- 3 files changed, 45 insertions(+), 44 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 99b92ace3a..374753fcde 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -690,14 +690,14 @@ SHashObj *tsdbGetSnapRangeHash(TFileSetRangeArray *pRanges); // snap partition list typedef TARRAY2(SVersionRange) SVerRangeList; -typedef struct STsdbSnapPartition STsdbSnapPartition; -typedef TARRAY2(STsdbSnapPartition *) STsdbSnapPartList; +typedef struct STsdbFSetPartition STsdbFSetPartition; +typedef TARRAY2(STsdbFSetPartition *) STsdbFSetPartList; // util -STsdbSnapPartList *tsdbSnapPartListCreate(); -void tsdbSnapPartListDestroy(STsdbSnapPartList **ppList); -int32_t tSerializeTsdbSnapPartList(void *buf, int32_t bufLen, STsdbSnapPartList *pList); -int32_t tDeserializeTsdbSnapPartList(void *buf, int32_t bufLen, STsdbSnapPartList *pList); -int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList *pList, TFileSetRangeArray **ppRanges); +STsdbFSetPartList *tsdbFSetPartListCreate(); +void tsdbFSetPartListDestroy(STsdbFSetPartList **ppList); +int32_t tSerializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList); +int32_t tDeserializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList); +int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList *pList, TFileSetRangeArray **ppRanges); enum { TSDB_SNAP_RANGE_TYP_HEAD = 0, @@ -708,7 +708,7 @@ enum { TSDB_SNAP_RANGE_TYP_MAX, }; -struct STsdbSnapPartition { +struct STsdbFSetPartition { int64_t fid; int8_t stat; SVerRangeList verRanges[TSDB_SNAP_RANGE_TYP_MAX]; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index 5aaa88511a..c73d75030d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -16,8 +16,8 @@ #include "tsdb.h" #include "tsdbFS2.h" -// STsdbSnapPartition ===================================== -static int32_t tsdbSnapPartCmprFn(STsdbSnapPartition* x, STsdbSnapPartition* y) { +// STsdbFSetPartition ===================================== +static int32_t tsdbFSetPartCmprFn(STsdbFSetPartition* x, STsdbFSetPartition* y) { if (x->fid < y->fid) return -1; if (x->fid > y->fid) return 1; return 0; @@ -37,8 +37,8 @@ static int32_t tsdbFileSetRangeCmprFn(STFileSetRange* x, STFileSetRange* y) { return 0; } -STsdbSnapPartition* tsdbSnapPartitionCreate() { - STsdbSnapPartition* pSP = taosMemoryCalloc(1, sizeof(STsdbSnapPartition)); +STsdbFSetPartition* tsdbFSetPartitionCreate() { + STsdbFSetPartition* pSP = taosMemoryCalloc(1, sizeof(STsdbFSetPartition)); if (pSP == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -49,7 +49,7 @@ STsdbSnapPartition* tsdbSnapPartitionCreate() { return pSP; } -void tsdbSnapPartitionClear(STsdbSnapPartition** ppSP) { +void tsdbFSetPartitionClear(STsdbFSetPartition** ppSP) { if (ppSP == NULL || ppSP[0] == NULL) { return; } @@ -76,8 +76,8 @@ static int32_t tsdbFTypeToSRangeTyp(tsdb_ftype_t ftype) { return TSDB_SNAP_RANGE_TYP_MAX; } -static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbSnapPartition** ppSP) { - STsdbSnapPartition* p = tsdbSnapPartitionCreate(); +static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbFSetPartition** ppSP) { + STsdbFSetPartition* p = tsdbFSetPartitionCreate(); if (p == NULL) { goto _err; } @@ -134,12 +134,12 @@ static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbSnapPartition** ppSP return 0; _err: - tsdbSnapPartitionClear(&p); + tsdbFSetPartitionClear(&p); return -1; } -STsdbSnapPartList* tsdbSnapPartListCreate() { - STsdbSnapPartList* pList = taosMemoryCalloc(1, sizeof(STsdbSnapPartList)); +STsdbFSetPartList* tsdbFSetPartListCreate() { + STsdbFSetPartList* pList = taosMemoryCalloc(1, sizeof(STsdbFSetPartList)); if (pList == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -148,8 +148,8 @@ STsdbSnapPartList* tsdbSnapPartListCreate() { return pList; } -static STsdbSnapPartList* tsdbGetSnapPartList(STFileSystem* fs) { - STsdbSnapPartList* pList = tsdbSnapPartListCreate(); +static STsdbFSetPartList* tsdbGetSnapPartList(STFileSystem* fs) { + STsdbFSetPartList* pList = tsdbFSetPartListCreate(); if (pList == NULL) { return NULL; } @@ -158,26 +158,26 @@ static STsdbSnapPartList* tsdbGetSnapPartList(STFileSystem* fs) { taosThreadMutexLock(&fs->tsdb->mutex); STFileSet* fset; TARRAY2_FOREACH(fs->fSetArr, fset) { - STsdbSnapPartition* pItem = NULL; + STsdbFSetPartition* pItem = NULL; if (tsdbTFileSetToSnapPart(fset, &pItem) < 0) { code = -1; break; } ASSERT(pItem != NULL); - code = TARRAY2_SORT_INSERT(pList, pItem, tsdbSnapPartCmprFn); + code = TARRAY2_SORT_INSERT(pList, pItem, tsdbFSetPartCmprFn); ASSERT(code == 0); } taosThreadMutexUnlock(&fs->tsdb->mutex); if (code) { - TARRAY2_DESTROY(pList, tsdbSnapPartitionClear); + TARRAY2_DESTROY(pList, tsdbFSetPartitionClear); taosMemoryFree(pList); pList = NULL; } return pList; } -int32_t tTsdbSnapPartListDataLenCalc(STsdbSnapPartList* pList) { +int32_t tTsdbFSetPartListDataLenCalc(STsdbFSetPartList* pList) { int32_t hdrLen = sizeof(int32_t); int32_t datLen = 0; @@ -188,10 +188,10 @@ int32_t tTsdbSnapPartListDataLenCalc(STsdbSnapPartList* pList) { datLen += hdrLen; for (int32_t u = 0; u < len; u++) { - STsdbSnapPartition* p = TARRAY2_GET(pList, u); + STsdbFSetPartition* p = TARRAY2_GET(pList, u); int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; int32_t uItem = 0; - uItem += sizeof(STsdbSnapPartition); + uItem += sizeof(STsdbFSetPartition); uItem += sizeof(typMax); for (int32_t i = 0; i < typMax; i++) { @@ -206,7 +206,7 @@ int32_t tTsdbSnapPartListDataLenCalc(STsdbSnapPartList* pList) { return datLen; } -int32_t tSerializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) { +int32_t tSerializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* pList) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -222,7 +222,7 @@ int32_t tSerializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* if (tEncodeI32(&encoder, len) < 0) goto _err; for (int32_t u = 0; u < len; u++) { - STsdbSnapPartition* p = TARRAY2_GET(pList, u); + STsdbFSetPartition* p = TARRAY2_GET(pList, u); if (tEncodeI64(&encoder, p->fid) < 0) goto _err; if (tEncodeI8(&encoder, p->stat) < 0) goto _err; if (tEncodeI8(&encoder, reserved8) < 0) goto _err; @@ -255,7 +255,7 @@ _err: return -1; } -int32_t tDeserializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) { +int32_t tDeserializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* pList) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -263,7 +263,7 @@ int32_t tDeserializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartLis int16_t reserved16 = 0; int64_t reserved64 = 0; - STsdbSnapPartition* p = NULL; + STsdbFSetPartition* p = NULL; int8_t msgVer = 0; int32_t len = 0; @@ -272,7 +272,7 @@ int32_t tDeserializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartLis if (tDecodeI32(&decoder, &len) < 0) goto _err; for (int32_t u = 0; u < len; u++) { - p = tsdbSnapPartitionCreate(); + p = tsdbFSetPartitionCreate(); if (p == NULL) goto _err; if (tDecodeI64(&decoder, &p->fid) < 0) goto _err; if (tDecodeI8(&decoder, &p->stat) < 0) goto _err; @@ -304,13 +304,13 @@ int32_t tDeserializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartLis _err: if (p) { - tsdbSnapPartitionClear(&p); + tsdbFSetPartitionClear(&p); } tDecoderClear(&decoder); return -1; } -int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TFileSetRangeArray** ppRanges) { +int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList* pList, TFileSetRangeArray** ppRanges) { TFileSetRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TFileSetRangeArray)); if (pDiff == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -318,7 +318,7 @@ int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TFileSetRangeArray } TARRAY2_INIT(pDiff); - STsdbSnapPartition* part; + STsdbFSetPartition* part; TARRAY2_FOREACH(pList, part) { STFileSetRange* r = taosMemoryCalloc(1, sizeof(STFileSetRange)); if (r == NULL) { @@ -364,10 +364,10 @@ void tsdbFileSetRangeArrayDestroy(TFileSetRangeArray** ppSnap) { } } -void tsdbSnapPartListDestroy(STsdbSnapPartList** ppList) { +void tsdbFSetPartListDestroy(STsdbFSetPartList** ppList) { if (ppList == NULL || ppList[0] == NULL) return; - TARRAY2_DESTROY(ppList[0], tsdbSnapPartitionClear); + TARRAY2_DESTROY(ppList[0], tsdbFSetPartitionClear); taosMemoryFree(ppList[0]); ppList[0] = NULL; } @@ -389,7 +389,7 @@ int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { int code = -1; int32_t tsdbMaxCnt = (!VND_IS_RSMA(pVnode) ? 1 : TSDB_RETENTION_MAX); int32_t subTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; - STsdbSnapPartList* pLists[TSDB_RETENTION_MAX] = {0}; + STsdbFSetPartList* pLists[TSDB_RETENTION_MAX] = {0}; // get part list for (int32_t j = 0; j < tsdbMaxCnt; ++j) { @@ -402,7 +402,7 @@ int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { int32_t bufLen = sizeof(SSyncTLV); // typ: TDMT_SYNC_PREP_SNAPSHOT or TDMT_SYNC_PREP_SNAPSOT_REPLY for (int32_t j = 0; j < tsdbMaxCnt; ++j) { bufLen += sizeof(SSyncTLV); // subTyps[j] - bufLen += tTsdbSnapPartListDataLenCalc(pLists[j]); + bufLen += tTsdbFSetPartListDataLenCalc(pLists[j]); } tsdbInfo("vgId:%d, allocate %d bytes for data of snapshot info.", TD_VID(pVnode), bufLen); @@ -429,7 +429,7 @@ int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { subHead->typ = subTyps[j]; ASSERT(subHead->val == (char*)data + offset + sizeof(SSyncTLV)); - if ((tlen = tSerializeTsdbSnapPartList(subHead->val, bufLen - offset - sizeof(SSyncTLV), pLists[j])) < 0) { + if ((tlen = tSerializeTsdbFSetPartList(subHead->val, bufLen - offset - sizeof(SSyncTLV), pLists[j])) < 0) { tsdbError("vgId:%d, failed to serialize snap partition list of tsdb %d since %s", TD_VID(pVnode), j, terrstr()); goto _out; } @@ -445,8 +445,9 @@ int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { _out: for (int32_t j = 0; j < tsdbMaxCnt; ++j) { if (pLists[j] == NULL) continue; - tsdbSnapPartListDestroy(&pLists[j]); + tsdbFSetPartListDestroy(&pLists[j]); } return code; } + diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 941660f776..bb3bd59971 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -18,21 +18,21 @@ static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TFileSetRangeArray **ppRanges) { int32_t code = -1; - STsdbSnapPartList *pList = tsdbSnapPartListCreate(); + STsdbFSetPartList *pList = tsdbFSetPartListCreate(); if (pList == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _out; } - if (tDeserializeTsdbSnapPartList(buf, bufLen, pList) < 0) { + if (tDeserializeTsdbFSetPartList(buf, bufLen, pList) < 0) { terrno = TSDB_CODE_INVALID_DATA_FMT; goto _out; } - if (tsdbSnapPartListToRangeDiff(pList, ppRanges) < 0) { + if (tsdbFSetPartListToRangeDiff(pList, ppRanges) < 0) { goto _out; } code = 0; _out: - tsdbSnapPartListDestroy(&pList); + tsdbFSetPartListDestroy(&pList); return code; } From 962febef02bea1c4c0ed1635e3d0c6cdc1ade0a4 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 1 Dec 2023 16:05:14 +0800 Subject: [PATCH 08/34] refact: rename tsdb snap range to tsdb fset range --- source/dnode/vnode/src/inc/tsdb.h | 29 ++++++------- source/dnode/vnode/src/tsdb/tsdbFS2.c | 48 +++++++++++----------- source/dnode/vnode/src/tsdb/tsdbFS2.h | 2 +- source/dnode/vnode/src/tsdb/tsdbFSet2.c | 14 ++++++- source/dnode/vnode/src/tsdb/tsdbFSet2.h | 4 +- source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 13 ++---- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 +- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 4 +- 8 files changed, 61 insertions(+), 57 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 374753fcde..b0cf6ecf5b 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -681,24 +681,14 @@ struct SDelFWriter { typedef struct STFileSet STFileSet; typedef TARRAY2(STFileSet *) TFileSetArray; +// fset range typedef struct STFileSetRange STFileSetRange; typedef TARRAY2(STFileSetRange *) TFileSetRangeArray; // disjoint ranges -// util -void tsdbFileSetRangeArrayDestroy(TFileSetRangeArray **ppSnap); -SHashObj *tsdbGetSnapRangeHash(TFileSetRangeArray *pRanges); - -// snap partition list -typedef TARRAY2(SVersionRange) SVerRangeList; -typedef struct STsdbFSetPartition STsdbFSetPartition; -typedef TARRAY2(STsdbFSetPartition *) STsdbFSetPartList; -// util -STsdbFSetPartList *tsdbFSetPartListCreate(); -void tsdbFSetPartListDestroy(STsdbFSetPartList **ppList); -int32_t tSerializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList); -int32_t tDeserializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList); -int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList *pList, TFileSetRangeArray **ppRanges); +int32_t tsdbTFileSetRangeClear(STFileSetRange **fsr); +int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray **ppArr); +// fset partition enum { TSDB_SNAP_RANGE_TYP_HEAD = 0, TSDB_SNAP_RANGE_TYP_DATA, @@ -708,12 +698,23 @@ enum { TSDB_SNAP_RANGE_TYP_MAX, }; +typedef TARRAY2(SVersionRange) SVerRangeList; + struct STsdbFSetPartition { int64_t fid; int8_t stat; SVerRangeList verRanges[TSDB_SNAP_RANGE_TYP_MAX]; }; +typedef struct STsdbFSetPartition STsdbFSetPartition; +typedef TARRAY2(STsdbFSetPartition *) STsdbFSetPartList; + +STsdbFSetPartList *tsdbFSetPartListCreate(); +void tsdbFSetPartListDestroy(STsdbFSetPartList **ppList); +int32_t tSerializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList); +int32_t tDeserializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList); +int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList *pList, TFileSetRangeArray **ppRanges); + // snap read struct STsdbReadSnap { SMemTable *pMem; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index df4df18dc3..ab52f5799d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -1072,6 +1072,24 @@ int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr) { return 0; } +static SHashObj *tsdbFSetRangeArrayToHash(TFileSetRangeArray *pRanges) { + int32_t capacity = TARRAY2_SIZE(pRanges) * 2; + SHashObj *pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); + if (pHash == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + for (int32_t i = 0; i < TARRAY2_SIZE(pRanges); i++) { + STFileSetRange *u = TARRAY2_GET(pRanges, i); + int32_t fid = u->fid; + int32_t code = taosHashPut(pHash, &fid, sizeof(fid), u, sizeof(*u)); + ASSERT(code == 0); + tsdbDebug("range diff hash fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever); + } + return pHash; +} + int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TFileSetRangeArray *pRanges, TFileSetArray **fsetArr, TFileOpArray *fopArr) { int32_t code = 0; @@ -1084,7 +1102,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TFileSetRangeArray *pRa TARRAY2_INIT(fsetArr[0]); if (pRanges) { - pHash = tsdbGetSnapRangeHash(pRanges); + pHash = tsdbFSetRangeArrayToHash(pRanges); if (pHash == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _out; @@ -1123,24 +1141,6 @@ _out: return code; } -SHashObj *tsdbGetSnapRangeHash(TFileSetRangeArray *pRanges) { - int32_t capacity = TARRAY2_SIZE(pRanges) * 2; - SHashObj *pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); - if (pHash == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - for (int32_t i = 0; i < TARRAY2_SIZE(pRanges); i++) { - STFileSetRange *u = TARRAY2_GET(pRanges, i); - int32_t fid = u->fid; - int32_t code = taosHashPut(pHash, &fid, sizeof(fid), u, sizeof(*u)); - ASSERT(code == 0); - tsdbDebug("range diff hash fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever); - } - return pHash; -} - int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TFileSetRangeArray *pRanges, TFileSetRangeArray **fsrArr) { int32_t code = 0; @@ -1156,7 +1156,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev tsdbInfo("pRanges size:%d", (pRanges == NULL ? 0 : TARRAY2_SIZE(pRanges))); if (pRanges) { - pHash = tsdbGetSnapRangeHash(pRanges); + pHash = tsdbFSetRangeArrayToHash(pRanges); if (pHash == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _out; @@ -1184,7 +1184,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev tsdbDebug("fsrArr:%p, fid:%d, sver:%" PRId64 ", ever:%" PRId64, fsrArr, fset->fid, sver1, ever1); - code = tsdbTSnapRangeInitRef(fs->tsdb, fset, sver1, ever1, &fsr1); + code = tsdbTFileSetRangeInitRef(fs->tsdb, fset, sver1, ever1, &fsr1); if (code) break; code = TARRAY2_APPEND(fsrArr[0], fsr1); @@ -1195,8 +1195,8 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev taosThreadMutexUnlock(&fs->tsdb->mutex); if (code) { - tsdbTSnapRangeClear(&fsr1); - TARRAY2_DESTROY(fsrArr[0], tsdbTSnapRangeClear); + tsdbTFileSetRangeClear(&fsr1); + TARRAY2_DESTROY(fsrArr[0], tsdbTFileSetRangeClear); fsrArr[0] = NULL; } @@ -1206,4 +1206,4 @@ _out: pHash = NULL; } return code; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index 8fdce9e690..3960ca908a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -50,7 +50,7 @@ int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr, TFileOpArray *f int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TFileSetRangeArray *pRanges, TFileSetRangeArray **fsrArr); int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr); -// txn +// txn int64_t tsdbFSAllocEid(STFileSystem *fs); int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype); int32_t tsdbFSEditCommit(STFileSystem *fs); diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index 7673299e4b..e088f54930 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -533,7 +533,8 @@ int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_ return 0; } -int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STFileSetRange **fsr) { +int32_t tsdbTFileSetRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, + STFileSetRange **fsr) { fsr[0] = taosMemoryCalloc(1, sizeof(*fsr[0])); if (fsr[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; fsr[0]->fid = fset1->fid; @@ -575,7 +576,7 @@ int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs return 0; } -int32_t tsdbTSnapRangeClear(STFileSetRange **fsr) { +int32_t tsdbTFileSetRangeClear(STFileSetRange **fsr) { if (!fsr[0]) return 0; tsdbTFileSetClear(&fsr[0]->fset); @@ -584,6 +585,15 @@ int32_t tsdbTSnapRangeClear(STFileSetRange **fsr) { return 0; } +int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray** ppArr) { + if (ppArr && ppArr[0]) { + TARRAY2_DESTROY(ppArr[0], tsdbTFileSetRangeClear); + taosMemoryFree(ppArr[0]); + ppArr[0] = NULL; + } + return 0; +} + int32_t tsdbTFileSetClear(STFileSet **fset) { if (!fset[0]) return 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h index 3a6427a42c..0951a28f4e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -49,8 +49,8 @@ int32_t tsdbTFileSetRemove(STFileSet *fset); int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_t ever, STFileSet **fset, TFileOpArray *fopArr); -int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STFileSetRange **fsr); -int32_t tsdbTSnapRangeClear(STFileSetRange **fsr); +int32_t tsdbTFileSetRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, + STFileSetRange **fsr); // to/from json int32_t tsdbTFileSetToJson(const STFileSet *fset, cJSON *json); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index c73d75030d..9d741ab2d3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -16,7 +16,7 @@ #include "tsdb.h" #include "tsdbFS2.h" -// STsdbFSetPartition ===================================== +// fset partition static int32_t tsdbFSetPartCmprFn(STsdbFSetPartition* x, STsdbFSetPartition* y) { if (x->fid < y->fid) return -1; if (x->fid > y->fid) return 1; @@ -138,6 +138,7 @@ _err: return -1; } +// fset partition list STsdbFSetPartList* tsdbFSetPartListCreate() { STsdbFSetPartList* pList = taosMemoryCalloc(1, sizeof(STsdbFSetPartList)); if (pList == NULL) { @@ -351,19 +352,11 @@ int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList* pList, TFileSetRangeArray _err: if (pDiff) { - tsdbFileSetRangeArrayDestroy(&pDiff); + tsdbTFileSetRangeArrayDestroy(&pDiff); } return -1; } -void tsdbFileSetRangeArrayDestroy(TFileSetRangeArray** ppSnap) { - if (ppSnap && ppSnap[0]) { - TARRAY2_DESTROY(ppSnap[0], tsdbTSnapRangeClear); - taosMemoryFree(ppSnap[0]); - ppSnap[0] = NULL; - } -} - void tsdbFSetPartListDestroy(STsdbFSetPartList** ppList) { if (ppList == NULL || ppList[0] == NULL) return; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 48872404ed..f890353320 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -444,7 +444,7 @@ _exit: if (code) { tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code), sver, ever, type); - tsdbFileSetRangeArrayDestroy(&reader[0]->fsrArr); + tsdbTFileSetRangeArrayDestroy(&reader[0]->fsrArr); taosMemoryFree(reader[0]); reader[0] = NULL; } else { @@ -472,7 +472,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** reader) { TARRAY2_DESTROY(reader[0]->sttReaderArr, tsdbSttFileReaderClose); tsdbDataFileReaderClose(&reader[0]->dataReader); - tsdbFileSetRangeArrayDestroy(&reader[0]->fsrArr); + tsdbTFileSetRangeArrayDestroy(&reader[0]->fsrArr); tDestroyTSchema(reader[0]->skmTb->pTSchema); for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->aBuf); ++i) { diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index bb3bd59971..21c858709b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -164,7 +164,7 @@ static void vnodeSnapReaderDestroyTsdbRanges(SVSnapReader *pReader) { for (int32_t j = 0; j < TSDB_RETENTION_MAX; ++j) { TFileSetRangeArray **ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, tsdbTyps[j]); if (ppRanges == NULL) continue; - tsdbFileSetRangeArrayDestroy(ppRanges); + tsdbTFileSetRangeArrayDestroy(ppRanges); } } @@ -598,7 +598,7 @@ static void vnodeSnapWriterDestroyTsdbRanges(SVSnapWriter *pWriter) { for (int32_t j = 0; j < TSDB_RETENTION_MAX; ++j) { TFileSetRangeArray **ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, tsdbTyps[j]); if (ppRanges == NULL) continue; - tsdbFileSetRangeArrayDestroy(ppRanges); + tsdbTFileSetRangeArrayDestroy(ppRanges); } } From 3550347f0c9b1283701e7230c4d84c60a21097b3 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 1 Dec 2023 16:29:48 +0800 Subject: [PATCH 09/34] refact: use tsdbFSDestroyCopyRangedSnapshot and tsdbFSDestroyRefRangedSnapshot --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 4 ++++ source/dnode/vnode/src/tsdb/tsdbFS2.h | 2 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index ab52f5799d..e933b3a7dc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -1141,6 +1141,8 @@ _out: return code; } +int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr) { return tsdbFSDestroyCopySnapshot(fsetArr); } + int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TFileSetRangeArray *pRanges, TFileSetRangeArray **fsrArr) { int32_t code = 0; @@ -1207,3 +1209,5 @@ _out: } return code; } + +int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr) { return tsdbTFileSetRangeArrayDestroy(fsrArr); } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index 3960ca908a..714bf5bf16 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -46,7 +46,7 @@ int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr); int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TFileSetRangeArray *pExclude, TFileSetArray **fsetArr, TFileOpArray *fopArr); -int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr, TFileOpArray *fopArr); +int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr); int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TFileSetRangeArray *pRanges, TFileSetRangeArray **fsrArr); int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index f890353320..8f5394a9bc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -472,7 +472,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** reader) { TARRAY2_DESTROY(reader[0]->sttReaderArr, tsdbSttFileReaderClose); tsdbDataFileReaderClose(&reader[0]->dataReader); - tsdbTFileSetRangeArrayDestroy(&reader[0]->fsrArr); + tsdbFSDestroyRefRangedSnapshot(&reader[0]->fsrArr); tDestroyTSchema(reader[0]->skmTb->pTSchema); for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->aBuf); ++i) { @@ -1125,7 +1125,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** writer, int8_t rollback) { tsdbDataFileReaderClose(&writer[0]->ctx->dataReader); TARRAY2_DESTROY(writer[0]->fopArr, NULL); - tsdbFSDestroyCopySnapshot(&writer[0]->fsetArr); + tsdbFSDestroyCopyRangedSnapshot(&writer[0]->fsetArr); for (int32_t i = 0; i < ARRAY_SIZE(writer[0]->aBuf); ++i) { tFree(writer[0]->aBuf[i]); From ed39c9a57253be3ee7e5ffb5f8278414feb8dc64 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 1 Dec 2023 17:21:27 +0800 Subject: [PATCH 10/34] refact: rename snap range to fset range in tsdbSnapInfo.c --- source/dnode/vnode/src/inc/tsdb.h | 14 +- source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 179 +++++++++++---------- 2 files changed, 98 insertions(+), 95 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index b0cf6ecf5b..a5203353db 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -690,12 +690,12 @@ int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray **ppArr); // fset partition enum { - TSDB_SNAP_RANGE_TYP_HEAD = 0, - TSDB_SNAP_RANGE_TYP_DATA, - TSDB_SNAP_RANGE_TYP_SMA, - TSDB_SNAP_RANGE_TYP_TOMB, - TSDB_SNAP_RANGE_TYP_STT, - TSDB_SNAP_RANGE_TYP_MAX, + TSDB_FSET_RANGE_TYP_HEAD = 0, + TSDB_FSET_RANGE_TYP_DATA, + TSDB_FSET_RANGE_TYP_SMA, + TSDB_FSET_RANGE_TYP_TOMB, + TSDB_FSET_RANGE_TYP_STT, + TSDB_FSET_RANGE_TYP_MAX, }; typedef TARRAY2(SVersionRange) SVerRangeList; @@ -703,7 +703,7 @@ typedef TARRAY2(SVersionRange) SVerRangeList; struct STsdbFSetPartition { int64_t fid; int8_t stat; - SVerRangeList verRanges[TSDB_SNAP_RANGE_TYP_MAX]; + SVerRangeList verRanges[TSDB_FSET_RANGE_TYP_MAX]; }; typedef struct STsdbFSetPartition STsdbFSetPartition; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index 9d741ab2d3..8afbe187a4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -31,7 +31,7 @@ static int32_t tVersionRangeCmprFn(SVersionRange* x, SVersionRange* y) { return 0; } -static int32_t tsdbFileSetRangeCmprFn(STFileSetRange* x, STFileSetRange* y) { +static int32_t tsdbTFileSetRangeCmprFn(STFileSetRange* x, STFileSetRange* y) { if (x->fid < y->fid) return -1; if (x->fid > y->fid) return 1; return 0; @@ -43,7 +43,7 @@ STsdbFSetPartition* tsdbFSetPartitionCreate() { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) { + for (int32_t i = 0; i < TSDB_FSET_RANGE_TYP_MAX; i++) { TARRAY2_INIT(&pSP->verRanges[i]); } return pSP; @@ -53,30 +53,30 @@ void tsdbFSetPartitionClear(STsdbFSetPartition** ppSP) { if (ppSP == NULL || ppSP[0] == NULL) { return; } - for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) { + for (int32_t i = 0; i < TSDB_FSET_RANGE_TYP_MAX; i++) { TARRAY2_DESTROY(&ppSP[0]->verRanges[i], NULL); } taosMemoryFree(ppSP[0]); ppSP[0] = NULL; } -static int32_t tsdbFTypeToSRangeTyp(tsdb_ftype_t ftype) { +static int32_t tsdbFTypeToFRangeType(tsdb_ftype_t ftype) { switch (ftype) { case TSDB_FTYPE_HEAD: - return TSDB_SNAP_RANGE_TYP_HEAD; + return TSDB_FSET_RANGE_TYP_HEAD; case TSDB_FTYPE_DATA: - return TSDB_SNAP_RANGE_TYP_DATA; + return TSDB_FSET_RANGE_TYP_DATA; case TSDB_FTYPE_SMA: - return TSDB_SNAP_RANGE_TYP_SMA; + return TSDB_FSET_RANGE_TYP_SMA; case TSDB_FTYPE_TOMB: - return TSDB_SNAP_RANGE_TYP_TOMB; + return TSDB_FSET_RANGE_TYP_TOMB; case TSDB_FTYPE_STT: - return TSDB_SNAP_RANGE_TYP_STT; + return TSDB_FSET_RANGE_TYP_STT; } - return TSDB_SNAP_RANGE_TYP_MAX; + return TSDB_FSET_RANGE_TYP_MAX; } -static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbFSetPartition** ppSP) { +static int32_t tsdbTFileSetToFSetPartition(STFileSet* fset, STsdbFSetPartition** ppSP) { STsdbFSetPartition* p = tsdbFSetPartitionCreate(); if (p == NULL) { goto _err; @@ -90,8 +90,8 @@ static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbFSetPartition** ppSP int32_t count = 0; for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { if (fset->farr[ftype] == NULL) continue; - typ = tsdbFTypeToSRangeTyp(ftype); - ASSERT(typ < TSDB_SNAP_RANGE_TYP_MAX); + typ = tsdbFTypeToFRangeType(ftype); + ASSERT(typ < TSDB_FSET_RANGE_TYP_MAX); STFile* f = fset->farr[ftype]->f; if (f->maxVer > fset->maxVerValid) { corrupt = true; @@ -106,7 +106,7 @@ static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbFSetPartition** ppSP ASSERT(code == 0); } - typ = TSDB_SNAP_RANGE_TYP_STT; + typ = TSDB_FSET_RANGE_TYP_STT; const SSttLvl* lvl; TARRAY2_FOREACH(fset->lvlArr, lvl) { STFileObj* fobj; @@ -149,35 +149,61 @@ STsdbFSetPartList* tsdbFSetPartListCreate() { return pList; } -static STsdbFSetPartList* tsdbGetSnapPartList(STFileSystem* fs) { - STsdbFSetPartList* pList = tsdbFSetPartListCreate(); - if (pList == NULL) { - return NULL; - } +void tsdbFSetPartListDestroy(STsdbFSetPartList** ppList) { + if (ppList == NULL || ppList[0] == NULL) return; - int32_t code = 0; - taosThreadMutexLock(&fs->tsdb->mutex); - STFileSet* fset; - TARRAY2_FOREACH(fs->fSetArr, fset) { - STsdbFSetPartition* pItem = NULL; - if (tsdbTFileSetToSnapPart(fset, &pItem) < 0) { - code = -1; - break; - } - ASSERT(pItem != NULL); - code = TARRAY2_SORT_INSERT(pList, pItem, tsdbFSetPartCmprFn); - ASSERT(code == 0); - } - taosThreadMutexUnlock(&fs->tsdb->mutex); - - if (code) { - TARRAY2_DESTROY(pList, tsdbFSetPartitionClear); - taosMemoryFree(pList); - pList = NULL; - } - return pList; + TARRAY2_DESTROY(ppList[0], tsdbFSetPartitionClear); + taosMemoryFree(ppList[0]); + ppList[0] = NULL; } +int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList* pList, TFileSetRangeArray** ppRanges) { + TFileSetRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TFileSetRangeArray)); + if (pDiff == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + TARRAY2_INIT(pDiff); + + STsdbFSetPartition* part; + TARRAY2_FOREACH(pList, part) { + STFileSetRange* r = taosMemoryCalloc(1, sizeof(STFileSetRange)); + if (r == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + int64_t maxVerValid = -1; + int32_t typMax = TSDB_FSET_RANGE_TYP_MAX; + for (int32_t i = 0; i < typMax; i++) { + SVerRangeList* iList = &part->verRanges[i]; + SVersionRange vr = {0}; + TARRAY2_FOREACH(iList, vr) { + if (vr.maxVer < vr.minVer) { + continue; + } + maxVerValid = TMAX(maxVerValid, vr.maxVer); + } + } + r->fid = part->fid; + r->sver = maxVerValid + 1; + r->ever = VERSION_MAX; + tsdbDebug("range diff fid:%" PRId64 ", sver:%" PRId64 ", ever:%" PRId64, part->fid, r->sver, r->ever); + int32_t code = TARRAY2_SORT_INSERT(pDiff, r, tsdbTFileSetRangeCmprFn); + ASSERT(code == 0); + } + ppRanges[0] = pDiff; + + tsdbInfo("pDiff size:%d", TARRAY2_SIZE(pDiff)); + return 0; + +_err: + if (pDiff) { + tsdbTFileSetRangeArrayDestroy(&pDiff); + } + return -1; +} + +// serialization int32_t tTsdbFSetPartListDataLenCalc(STsdbFSetPartList* pList) { int32_t hdrLen = sizeof(int32_t); int32_t datLen = 0; @@ -190,7 +216,7 @@ int32_t tTsdbFSetPartListDataLenCalc(STsdbFSetPartList* pList) { for (int32_t u = 0; u < len; u++) { STsdbFSetPartition* p = TARRAY2_GET(pList, u); - int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; + int32_t typMax = TSDB_FSET_RANGE_TYP_MAX; int32_t uItem = 0; uItem += sizeof(STsdbFSetPartition); uItem += sizeof(typMax); @@ -229,7 +255,7 @@ int32_t tSerializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* if (tEncodeI8(&encoder, reserved8) < 0) goto _err; if (tEncodeI16(&encoder, reserved16) < 0) goto _err; - int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; + int32_t typMax = TSDB_FSET_RANGE_TYP_MAX; if (tEncodeI32(&encoder, typMax) < 0) goto _err; for (int32_t i = 0; i < typMax; i++) { @@ -311,58 +337,34 @@ _err: return -1; } -int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList* pList, TFileSetRangeArray** ppRanges) { - TFileSetRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TFileSetRangeArray)); - if (pDiff == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; +// fs state +static STsdbFSetPartList* tsdbSnapGetFSetPartList(STFileSystem* fs) { + STsdbFSetPartList* pList = tsdbFSetPartListCreate(); + if (pList == NULL) { + return NULL; } - TARRAY2_INIT(pDiff); - STsdbFSetPartition* part; - TARRAY2_FOREACH(pList, part) { - STFileSetRange* r = taosMemoryCalloc(1, sizeof(STFileSetRange)); - if (r == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + int32_t code = 0; + taosThreadMutexLock(&fs->tsdb->mutex); + STFileSet* fset; + TARRAY2_FOREACH(fs->fSetArr, fset) { + STsdbFSetPartition* pItem = NULL; + if (tsdbTFileSetToFSetPartition(fset, &pItem) < 0) { + code = -1; + break; } - int64_t maxVerValid = -1; - int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; - for (int32_t i = 0; i < typMax; i++) { - SVerRangeList* iList = &part->verRanges[i]; - SVersionRange vr = {0}; - TARRAY2_FOREACH(iList, vr) { - if (vr.maxVer < vr.minVer) { - continue; - } - maxVerValid = TMAX(maxVerValid, vr.maxVer); - } - } - r->fid = part->fid; - r->sver = maxVerValid + 1; - r->ever = VERSION_MAX; - tsdbDebug("range diff fid:%" PRId64 ", sver:%" PRId64 ", ever:%" PRId64, part->fid, r->sver, r->ever); - int32_t code = TARRAY2_SORT_INSERT(pDiff, r, tsdbFileSetRangeCmprFn); + ASSERT(pItem != NULL); + code = TARRAY2_SORT_INSERT(pList, pItem, tsdbFSetPartCmprFn); ASSERT(code == 0); } - ppRanges[0] = pDiff; + taosThreadMutexUnlock(&fs->tsdb->mutex); - tsdbInfo("pDiff size:%d", TARRAY2_SIZE(pDiff)); - return 0; - -_err: - if (pDiff) { - tsdbTFileSetRangeArrayDestroy(&pDiff); + if (code) { + TARRAY2_DESTROY(pList, tsdbFSetPartitionClear); + taosMemoryFree(pList); + pList = NULL; } - return -1; -} - -void tsdbFSetPartListDestroy(STsdbFSetPartList** ppList) { - if (ppList == NULL || ppList[0] == NULL) return; - - TARRAY2_DESTROY(ppList[0], tsdbFSetPartitionClear); - taosMemoryFree(ppList[0]); - ppList[0] = NULL; + return pList; } ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) { @@ -378,6 +380,7 @@ ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) { return TSDB_FS_STATE_NORMAL; } +// description int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { int code = -1; int32_t tsdbMaxCnt = (!VND_IS_RSMA(pVnode) ? 1 : TSDB_RETENTION_MAX); @@ -387,7 +390,7 @@ int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { // get part list for (int32_t j = 0; j < tsdbMaxCnt; ++j) { STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, j); - pLists[j] = tsdbGetSnapPartList(pTsdb->pFS); + pLists[j] = tsdbSnapGetFSetPartList(pTsdb->pFS); if (pLists[j] == NULL) goto _out; } From f68804322c2e6ea54d7843102169e5d30153e0c0 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 4 Dec 2023 16:56:20 +0800 Subject: [PATCH 11/34] refact: improve code of tsdbSnapGetDetails as tsdbSnapPrepDescription --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 133 ++++++++++++++------- source/dnode/vnode/src/vnd/vnodeSync.c | 2 +- 3 files changed, 89 insertions(+), 48 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index a5203353db..dc3aa418b4 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -1041,7 +1041,7 @@ typedef enum { // utils ETsdbFsState tsdbSnapGetFsState(SVnode *pVnode); -int32_t tsdbSnapGetDetails(SVnode *pVnode, SSnapshot *pSnap); +int32_t tsdbSnapPrepDescription(SVnode *pVnode, SSnapshot *pSnap); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index 8afbe187a4..573ba48774 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -381,69 +381,110 @@ ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) { } // description -int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { - int code = -1; - int32_t tsdbMaxCnt = (!VND_IS_RSMA(pVnode) ? 1 : TSDB_RETENTION_MAX); - int32_t subTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; - STsdbFSetPartList* pLists[TSDB_RETENTION_MAX] = {0}; +typedef struct STsdbPartitionInfo { + int32_t vgId; + int32_t tsdbMaxCnt; + int32_t subTyps[TSDB_RETENTION_MAX]; + STsdbFSetPartList* pLists[TSDB_RETENTION_MAX]; +} STsdbPartitionInfo; - // get part list - for (int32_t j = 0; j < tsdbMaxCnt; ++j) { +static int32_t tsdbPartitionInfoInit(SVnode* pVnode, STsdbPartitionInfo* pInfo) { + int32_t subTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; + pInfo->vgId = TD_VID(pVnode); + pInfo->tsdbMaxCnt = (!VND_IS_RSMA(pVnode) ? 1 : TSDB_RETENTION_MAX); + + ASSERT(sizeof(pInfo->subTyps) == sizeof(subTyps)); + memcpy(pInfo->subTyps, (char*)subTyps, sizeof(subTyps)); + + // fset partition list + memset(pInfo->pLists, 0, sizeof(pInfo->pLists[0]) * TSDB_RETENTION_MAX); + for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) { STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, j); - pLists[j] = tsdbSnapGetFSetPartList(pTsdb->pFS); - if (pLists[j] == NULL) goto _out; + pInfo->pLists[j] = tsdbSnapGetFSetPartList(pTsdb->pFS); + if (pInfo->pLists[j] == NULL) return -1; } + return 0; +} - // estimate bufLen and prepare - int32_t bufLen = sizeof(SSyncTLV); // typ: TDMT_SYNC_PREP_SNAPSHOT or TDMT_SYNC_PREP_SNAPSOT_REPLY - for (int32_t j = 0; j < tsdbMaxCnt; ++j) { - bufLen += sizeof(SSyncTLV); // subTyps[j] - bufLen += tTsdbFSetPartListDataLenCalc(pLists[j]); +static void tsdbPartitionInfoClear(STsdbPartitionInfo* pInfo) { + for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) { + if (pInfo->pLists[j] == NULL) continue; + tsdbFSetPartListDestroy(&pInfo->pLists[j]); } +} - tsdbInfo("vgId:%d, allocate %d bytes for data of snapshot info.", TD_VID(pVnode), bufLen); +static int32_t tsdbPartitionInfoEstSize(STsdbPartitionInfo* pInfo) { + int32_t dataLen = 0; + for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) { + dataLen += sizeof(SSyncTLV); // subTyps[j] + dataLen += tTsdbFSetPartListDataLenCalc(pInfo->pLists[j]); + } + return dataLen; +} - void* data = taosMemoryRealloc(pSnap->data, bufLen); +static int32_t tsdbPartitionInfoSerialize(STsdbPartitionInfo* pInfo, uint8_t* buf, int32_t bufLen, int32_t* offset) { + int32_t tlen = 0; + for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) { + SSyncTLV* pSubHead = (void*)((char*)buf + offset[0]); + int32_t valOffset = offset[0] + sizeof(*pSubHead); + ASSERT(pSubHead->val == (char*)buf + valOffset); + if ((tlen = tSerializeTsdbFSetPartList(pSubHead->val, bufLen - valOffset, pInfo->pLists[j])) < 0) { + tsdbError("vgId:%d, failed to serialize fset partition list of tsdb %d since %s", pInfo->vgId, j, terrstr()); + return -1; + } + pSubHead->typ = pInfo->subTyps[j]; + pSubHead->len = tlen; + offset[0] += sizeof(*pSubHead) + tlen; + } + return 0; +} + +int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size) { + void* data = taosMemoryRealloc(pSnap->data, size); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pSnap->data = data; + return 0; +} + +int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { + ASSERT(pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY); + STsdbPartitionInfo partitionInfo = {0}; + int code = -1; + STsdbPartitionInfo* pInfo = &partitionInfo; + + if (tsdbPartitionInfoInit(pVnode, pInfo) != 0) { + goto _out; + } + + // info data realloc + int32_t bufLen = sizeof(SSyncTLV); + bufLen += tsdbPartitionInfoEstSize(pInfo); + if (syncSnapInfoDataRealloc(pSnap, bufLen) != 0) { tsdbError("vgId:%d, failed to realloc memory for data of snapshot info. bytes:%d", TD_VID(pVnode), bufLen); goto _out; } - pSnap->data = data; - // header - SSyncTLV* head = data; - head->len = 0; - head->typ = pSnap->type; - int32_t offset = sizeof(SSyncTLV); - int32_t tlen = 0; + // serialization + SSyncTLV* pHead = pSnap->data; + pHead->typ = pSnap->type; - // fill snapshot info - for (int32_t j = 0; j < tsdbMaxCnt; ++j) { - // subHead - SSyncTLV* subHead = (void*)((char*)data + offset); - subHead->typ = subTyps[j]; - ASSERT(subHead->val == (char*)data + offset + sizeof(SSyncTLV)); - - if ((tlen = tSerializeTsdbFSetPartList(subHead->val, bufLen - offset - sizeof(SSyncTLV), pLists[j])) < 0) { - tsdbError("vgId:%d, failed to serialize snap partition list of tsdb %d since %s", TD_VID(pVnode), j, terrstr()); - goto _out; - } - subHead->len = tlen; - offset += sizeof(SSyncTLV) + tlen; + int32_t offset = 0; + if (tsdbPartitionInfoSerialize(pInfo, pHead->val, bufLen - sizeof(*pHead), &offset) != 0) { + tsdbError("vgId:%d, failed to serialize tsdb partition info since %s", TD_VID(pVnode), terrstr()); + goto _out; } - // total length of subfields - head->len = offset - sizeof(SSyncTLV); - ASSERT(offset <= bufLen); + // set header of info data + ASSERT(sizeof(*pHead) + offset <= bufLen); + pHead->len = offset; + + tsdbInfo("vgId:%d, tsdb snap info prepared. type:%s, val length:%d", TD_VID(pVnode), TMSG_INFO(pHead->typ), + pHead->len); code = 0; - _out: - for (int32_t j = 0; j < tsdbMaxCnt; ++j) { - if (pLists[j] == NULL) continue; - tsdbFSetPartListDestroy(&pLists[j]); - } - + tsdbPartitionInfoClear(pInfo); return code; } - diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 817d5124a2..5871a60c9e 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -804,7 +804,7 @@ int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnap) { } if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { - code = tsdbSnapGetDetails(pVnode, pSnap); + code = tsdbSnapPrepDescription(pVnode, pSnap); } return code; } From 1a9b08fa0874c39ddadb4285e1f54b32a53c2400 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 4 Dec 2023 17:05:25 +0800 Subject: [PATCH 12/34] refact: relocate func syncSnapInfoDataRealloc --- include/libs/sync/sync.h | 3 +++ source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 10 ---------- source/libs/sync/src/syncUtil.c | 10 ++++++++++ 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index ece1e40585..a428a9ae6a 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -289,6 +289,9 @@ const char* syncStr(ESyncState state); int32_t syncNodeGetConfig(int64_t rid, SSyncCfg *cfg); +// util +int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index 573ba48774..1662567247 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -439,16 +439,6 @@ static int32_t tsdbPartitionInfoSerialize(STsdbPartitionInfo* pInfo, uint8_t* bu return 0; } -int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size) { - void* data = taosMemoryRealloc(pSnap->data, size); - if (data == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pSnap->data = data; - return 0; -} - int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { ASSERT(pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY); STsdbPartitionInfo partitionInfo = {0}; diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 06847c081c..2ce56af946 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -487,3 +487,13 @@ void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteRepl sNInfo(pSyncNode, "send sync-request-vote-reply to dnode:%d {term:%" PRId64 ", grant:%d}, %s", DID(&pMsg->destId), pMsg->term, pMsg->voteGranted, s); } + +int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size) { + void* data = taosMemoryRealloc(pSnap->data, size); + if (data == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pSnap->data = data; + return 0; +} From de9c9d4f205f8f30e29c1d47b4be5f6aa4474772 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 4 Dec 2023 21:05:18 +0800 Subject: [PATCH 13/34] refact: improve code of tsdbPartitionInfoSerialize --- source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 36 +++++++++++++--------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index 1662567247..65ee1a7db3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -16,6 +16,8 @@ #include "tsdb.h" #include "tsdbFS2.h" +#define TSDB_SNAP_MSG_VER 1 + // fset partition static int32_t tsdbFSetPartCmprFn(STsdbFSetPartition* x, STsdbFSetPartition* y) { if (x->fid < y->fid) return -1; @@ -241,7 +243,7 @@ int32_t tSerializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* int16_t reserved16 = 0; int64_t reserved64 = 0; - int8_t msgVer = 1; + int8_t msgVer = TSDB_SNAP_MSG_VER; int32_t len = TARRAY2_SIZE(pList); if (tStartEncode(&encoder) < 0) goto _err; @@ -296,6 +298,7 @@ int32_t tDeserializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartLis int32_t len = 0; if (tStartDecode(&decoder) < 0) goto _err; if (tDecodeI8(&decoder, &msgVer) < 0) goto _err; + if (msgVer != TSDB_SNAP_MSG_VER) goto _err; if (tDecodeI32(&decoder, &len) < 0) goto _err; for (int32_t u = 0; u < len; u++) { @@ -422,11 +425,12 @@ static int32_t tsdbPartitionInfoEstSize(STsdbPartitionInfo* pInfo) { return dataLen; } -static int32_t tsdbPartitionInfoSerialize(STsdbPartitionInfo* pInfo, uint8_t* buf, int32_t bufLen, int32_t* offset) { +static int32_t tsdbPartitionInfoSerialize(STsdbPartitionInfo* pInfo, uint8_t* buf, int32_t bufLen) { int32_t tlen = 0; + int32_t offset = 0; for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) { - SSyncTLV* pSubHead = (void*)((char*)buf + offset[0]); - int32_t valOffset = offset[0] + sizeof(*pSubHead); + SSyncTLV* pSubHead = (void*)((char*)buf + offset); + int32_t valOffset = offset + sizeof(*pSubHead); ASSERT(pSubHead->val == (char*)buf + valOffset); if ((tlen = tSerializeTsdbFSetPartList(pSubHead->val, bufLen - valOffset, pInfo->pLists[j])) < 0) { tsdbError("vgId:%d, failed to serialize fset partition list of tsdb %d since %s", pInfo->vgId, j, terrstr()); @@ -434,9 +438,9 @@ static int32_t tsdbPartitionInfoSerialize(STsdbPartitionInfo* pInfo, uint8_t* bu } pSubHead->typ = pInfo->subTyps[j]; pSubHead->len = tlen; - offset[0] += sizeof(*pSubHead) + tlen; + offset += sizeof(*pSubHead) + tlen; } - return 0; + return offset; } int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { @@ -450,26 +454,30 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { } // info data realloc - int32_t bufLen = sizeof(SSyncTLV); + const int32_t headLen = sizeof(SSyncTLV); + int32_t bufLen = headLen; bufLen += tsdbPartitionInfoEstSize(pInfo); if (syncSnapInfoDataRealloc(pSnap, bufLen) != 0) { - tsdbError("vgId:%d, failed to realloc memory for data of snapshot info. bytes:%d", TD_VID(pVnode), bufLen); + tsdbError("vgId:%d, failed to realloc memory for data of snap info. bytes:%d", TD_VID(pVnode), bufLen); goto _out; } // serialization - SSyncTLV* pHead = pSnap->data; - pHead->typ = pSnap->type; + char* buf = (void*)pSnap->data; + int32_t offset = headLen; + int32_t tlen = 0; - int32_t offset = 0; - if (tsdbPartitionInfoSerialize(pInfo, pHead->val, bufLen - sizeof(*pHead), &offset) != 0) { + if ((tlen = tsdbPartitionInfoSerialize(pInfo, buf + offset, bufLen - offset)) < 0) { tsdbError("vgId:%d, failed to serialize tsdb partition info since %s", TD_VID(pVnode), terrstr()); goto _out; } + offset += tlen; + ASSERT(offset <= bufLen); // set header of info data - ASSERT(sizeof(*pHead) + offset <= bufLen); - pHead->len = offset; + SSyncTLV* pHead = pSnap->data; + pHead->typ = pSnap->type; + pHead->len = offset - headLen; tsdbInfo("vgId:%d, tsdb snap info prepared. type:%s, val length:%d", TD_VID(pVnode), TMSG_INFO(pHead->typ), pHead->len); From d4add073cc676d9ab52de5d24422d7e88a2f9704 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 3 Nov 2023 16:42:41 +0800 Subject: [PATCH 14/34] refact: improve code with syncSnapSendRsp --- source/libs/sync/inc/syncReplication.h | 4 ++ source/libs/sync/src/syncSnapshot.c | 70 ++++++++++---------------- 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index 04456b2454..ecd2b5163e 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -56,6 +56,10 @@ int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode); int32_t syncNodeSendAppendEntries(SSyncNode* pNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg); +int32_t syncSnapSendMsg(SSyncSnapshotSender* pSender, int32_t seq, void* pBlock, int32_t len, int32_t typ); +int32_t syncSnapSendRsp(SSyncSnapshotReceiver* pReceiver, SyncSnapshotSend* pMsg, void* pBlock, int32_t len, + int32_t typ, int32_t code); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 95952c960e..e1471ad3d9 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -23,8 +23,6 @@ #include "syncReplication.h" #include "syncUtil.h" -int32_t syncSnapSendMsg(SSyncSnapshotSender *pSender, int32_t seq, void *pBlock, int32_t len, int32_t typ); - static void syncSnapBufferReset(SSyncSnapBuffer *pBuf) { taosThreadMutexLock(&pBuf->mutex); for (int64_t i = pBuf->start; i < pBuf->end; ++i) { @@ -153,7 +151,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { pSender->lastSendTime = taosGetTimestampMs(); pSender->finish = false; - // Get full snapshot info + // Get snapshot info SSyncNode *pSyncNode = pSender->pSyncNode; SSnapshot snapInfo = {.type = TDMT_SYNC_PREP_SNAPSHOT}; if (pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapInfo) != 0) { @@ -161,11 +159,10 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { goto _out; } - int dataLen = 0; void *pData = snapInfo.data; - int32_t type = 0; + int32_t type = (pData) ? snapInfo.type : 0; + int32_t dataLen = 0; if (pData) { - type = snapInfo.type; SSyncTLV *datHead = pData; if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT) { sSError(pSender, "unexpected data typ in data of snapshot info. typ: %d", datHead->typ); @@ -688,24 +685,23 @@ _START_RECEIVER: snapshotReceiverStart(pReceiver, pMsg); // set start-time same with sender -_SEND_REPLY: - // build msg - ; // make complier happy +_SEND_REPLY:; SSnapshot snapInfo = {.type = TDMT_SYNC_PREP_SNAPSHOT_REPLY}; int32_t dataLen = 0; - if (pMsg->dataLen > 0) { + if (pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT) { void *data = taosMemoryCalloc(1, pMsg->dataLen); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; code = terrno; goto _out; } - memcpy(data, pMsg->data, pMsg->dataLen); snapInfo.data = data; data = NULL; - pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapInfo); + memcpy(snapInfo.data, pMsg->data, pMsg->dataLen); + // exchange snap info + pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapInfo); SSyncTLV *datHead = snapInfo.data; if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { sRError(pReceiver, "unexpected data typ in data of snapshot info. typ: %d", datHead->typ); @@ -715,29 +711,16 @@ _SEND_REPLY: dataLen = sizeof(SSyncTLV) + datHead->len; } - SRpcMsg rpcMsg = {0}; - if (syncBuildSnapshotSendRsp(&rpcMsg, dataLen, pSyncNode->vgId) != 0) { - sRError(pReceiver, "snapshot receiver failed to build resp since %s", terrstr()); + // send response + void *pData = snapInfo.data; + int32_t type = (pData) ? snapInfo.type : 0; + + if (syncSnapSendRsp(pReceiver, pMsg, pData, dataLen, type, code) != 0) { code = terrno; goto _out; } - SyncSnapshotRsp *pRspMsg = rpcMsg.pCont; - pRspMsg->srcId = pSyncNode->myRaftId; - pRspMsg->destId = pMsg->srcId; - pRspMsg->term = raftStoreGetTerm(pSyncNode); - pRspMsg->lastIndex = pMsg->lastIndex; - pRspMsg->lastTerm = pMsg->lastTerm; - pRspMsg->startTime = pMsg->startTime; - pRspMsg->ack = pMsg->seq; // receiver maybe already closed - pRspMsg->code = code; - pRspMsg->snapBeginIndex = syncNodeGetSnapBeginIndex(pSyncNode); - - if (snapInfo.data) { - pRspMsg->payloadType = snapInfo.type; - memcpy(pRspMsg->data, snapInfo.data, dataLen); - - // save snapshot info + if (pData) { SSnapshotParam *pParam = &pReceiver->snapshotParam; void *data = taosMemoryRealloc(pParam->data, dataLen); if (data == NULL) { @@ -748,15 +731,10 @@ _SEND_REPLY: goto _out; } pParam->data = data; + data = NULL; memcpy(pParam->data, snapInfo.data, dataLen); } - // send msg - if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) { - sRError(pReceiver, "failed to send resp since %s", terrstr()); - code = terrno; - } - _out: if (snapInfo.data) { taosMemoryFree(snapInfo.data); @@ -820,11 +798,12 @@ _SEND_REPLY: return code; } -static int32_t syncSnapSendRsp(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg, int32_t code) { +int32_t syncSnapSendRsp(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg, void *pBlock, int32_t blockLen, + int32_t type, int32_t code) { SSyncNode *pSyncNode = pReceiver->pSyncNode; // build msg SRpcMsg rpcMsg = {0}; - if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId)) { + if (syncBuildSnapshotSendRsp(&rpcMsg, blockLen, pSyncNode->vgId)) { sRError(pReceiver, "failed to build snapshot receiver resp since %s", terrstr()); return -1; } @@ -832,13 +811,18 @@ static int32_t syncSnapSendRsp(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSen SyncSnapshotRsp *pRspMsg = rpcMsg.pCont; pRspMsg->srcId = pSyncNode->myRaftId; pRspMsg->destId = pMsg->srcId; - pRspMsg->term = raftStoreGetTerm(pSyncNode); + pRspMsg->term = pMsg->term; pRspMsg->lastIndex = pMsg->lastIndex; pRspMsg->lastTerm = pMsg->lastTerm; pRspMsg->startTime = pMsg->startTime; pRspMsg->ack = pMsg->seq; pRspMsg->code = code; pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start; + pRspMsg->payloadType = type; + + if (pBlock != NULL && blockLen > 0) { + memcpy(pRspMsg->data, pBlock, blockLen); + } // send msg if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) { @@ -872,7 +856,7 @@ static int32_t syncSnapBufferRecv(SSyncSnapshotReceiver *pReceiver, SyncSnapshot ppMsg[0] = NULL; pRcvBuf->end = TMAX(pMsg->seq + 1, pRcvBuf->end); } else if (pMsg->seq < pRcvBuf->start) { - syncSnapSendRsp(pReceiver, pMsg, code); + syncSnapSendRsp(pReceiver, pMsg, NULL, 0, 0, code); goto _out; } @@ -892,7 +876,7 @@ static int32_t syncSnapBufferRecv(SSyncSnapshotReceiver *pReceiver, SyncSnapshot } } pRcvBuf->start = seq + 1; - syncSnapSendRsp(pReceiver, pRcvBuf->entries[seq % pRcvBuf->size], code); + syncSnapSendRsp(pReceiver, pRcvBuf->entries[seq % pRcvBuf->size], NULL, 0, 0, code); pRcvBuf->entryDeleteCb(pRcvBuf->entries[seq % pRcvBuf->size]); pRcvBuf->entries[seq % pRcvBuf->size] = NULL; if (code) goto _out; @@ -915,7 +899,7 @@ static int32_t syncNodeOnSnapshotReceive(SSyncNode *pSyncNode, SyncSnapshotSend if (snapshotReceiverSignatureCmp(pReceiver, pMsg) != 0) { terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; sRError(pReceiver, "failed to receive snapshot data since %s.", terrstr()); - return syncSnapSendRsp(pReceiver, pMsg, terrno); + return syncSnapSendRsp(pReceiver, pMsg, NULL, 0, 0, terrno); } return syncSnapBufferRecv(pReceiver, ppMsg); From 41fe39de3edbbda0eb3c276e176dd447c817e579 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 3 Nov 2023 17:19:17 +0800 Subject: [PATCH 15/34] refact: improve code with syncNodeExchangeSnapInfo --- source/libs/sync/src/syncSnapshot.c | 83 +++++++++++++++++------------ 1 file changed, 48 insertions(+), 35 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index e1471ad3d9..a315b91791 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -641,6 +641,50 @@ SyncIndex syncNodeGetSnapBeginIndex(SSyncNode *ths) { return snapStart; } +static int32_t syncNodeExchangeSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg, + SSnapshot *pInfo) { + ASSERT(pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT); + int32_t code = 0; + + // copy snap info from leader + void *data = taosMemoryCalloc(1, pMsg->dataLen); + if (data == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; + goto _out; + } + pInfo->data = data; + data = NULL; + memcpy(pInfo->data, pMsg->data, pMsg->dataLen); + + // exchange snap info + pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, pInfo); + SSyncTLV *datHead = pInfo->data; + if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { + sRError(pReceiver, "unexpected data typ in data of snapshot info. typ: %d", datHead->typ); + code = TSDB_CODE_INVALID_DATA_FMT; + goto _out; + } + int32_t dataLen = sizeof(SSyncTLV) + datHead->len; + + // save exchanged snap info + SSnapshotParam *pParam = &pReceiver->snapshotParam; + data = taosMemoryRealloc(pParam->data, dataLen); + if (data == NULL) { + sError("vgId:%d, failed to realloc memory for snapshot prep due to %s. dataLen:%d", pSyncNode->vgId, + strerror(errno), dataLen); + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; + goto _out; + } + pParam->data = data; + data = NULL; + memcpy(pParam->data, pInfo->data, dataLen); + +_out: + return code; +} + static int32_t syncNodeOnSnapshotPrep(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; int64_t timeNow = taosGetTimestampMs(); @@ -683,58 +727,27 @@ _START_RECEIVER: snapshotReceiverStop(pReceiver); } - snapshotReceiverStart(pReceiver, pMsg); // set start-time same with sender + snapshotReceiverStart(pReceiver, pMsg); _SEND_REPLY:; SSnapshot snapInfo = {.type = TDMT_SYNC_PREP_SNAPSHOT_REPLY}; int32_t dataLen = 0; if (pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT) { - void *data = taosMemoryCalloc(1, pMsg->dataLen); - if (data == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = terrno; + if (syncNodeExchangeSnapInfo(pSyncNode, pReceiver, pMsg, &snapInfo) != 0) { goto _out; } - snapInfo.data = data; - data = NULL; - memcpy(snapInfo.data, pMsg->data, pMsg->dataLen); - - // exchange snap info - pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapInfo); SSyncTLV *datHead = snapInfo.data; - if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { - sRError(pReceiver, "unexpected data typ in data of snapshot info. typ: %d", datHead->typ); - code = TSDB_CODE_INVALID_DATA_FMT; - goto _out; - } dataLen = sizeof(SSyncTLV) + datHead->len; } // send response - void *pData = snapInfo.data; - int32_t type = (pData) ? snapInfo.type : 0; - - if (syncSnapSendRsp(pReceiver, pMsg, pData, dataLen, type, code) != 0) { + int32_t type = (snapInfo.data) ? snapInfo.type : 0; + if (syncSnapSendRsp(pReceiver, pMsg, snapInfo.data, dataLen, type, code) != 0) { code = terrno; goto _out; } - if (pData) { - SSnapshotParam *pParam = &pReceiver->snapshotParam; - void *data = taosMemoryRealloc(pParam->data, dataLen); - if (data == NULL) { - sError("vgId:%d, failed to realloc memory for snapshot prep due to %s. dataLen:%d", pSyncNode->vgId, - strerror(errno), dataLen); - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = terrno; - goto _out; - } - pParam->data = data; - data = NULL; - memcpy(pParam->data, snapInfo.data, dataLen); - } - _out: if (snapInfo.data) { taosMemoryFree(snapInfo.data); From 030f3db4d604ae89a151ba5e270a4d38c806bd67 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 3 Nov 2023 17:38:33 +0800 Subject: [PATCH 16/34] refact: improve code of syncNodeOnSnapshotBegin with syncSnapSendRsp --- source/libs/sync/src/syncSnapshot.c | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index a315b91791..25b620854e 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -784,27 +784,8 @@ _SEND_REPLY: code = terrno; } - // build msg - SRpcMsg rpcMsg = {0}; - if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId) != 0) { - sRError(pReceiver, "failed to build snapshot receiver resp since %s", terrstr()); - return -1; - } - - SyncSnapshotRsp *pRspMsg = rpcMsg.pCont; - pRspMsg->srcId = pSyncNode->myRaftId; - pRspMsg->destId = pMsg->srcId; - pRspMsg->term = raftStoreGetTerm(pSyncNode); - pRspMsg->lastIndex = pMsg->lastIndex; - pRspMsg->lastTerm = pMsg->lastTerm; - pRspMsg->startTime = pMsg->startTime; - pRspMsg->ack = pReceiver->ack; // receiver maybe already closed - pRspMsg->code = code; - pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start; - - // send msg - if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) { - sRError(pReceiver, "failed to send snapshot receiver resp since %s", terrstr()); + // send response + if (syncSnapSendRsp(pReceiver, pMsg, NULL, 0, 0, code) != 0) { return -1; } From 081c83710ebde03067d5dd496b90a21fe299086d Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 9 Nov 2023 18:21:15 +0800 Subject: [PATCH 17/34] enh: save a copy of snapshot info in syncNodeOnSnapshotPrepRsp --- source/libs/sync/src/syncSnapshot.c | 79 +++++++++++++---------------- 1 file changed, 36 insertions(+), 43 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 25b620854e..1e3614857e 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -121,6 +121,11 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { if (pSender->pSndBuf) { syncSnapBufferDestroy(&pSender->pSndBuf); } + + if (pSender->snapshotParam.data) { + taosMemoryFree(pSender->snapshotParam.data); + pSender->snapshotParam.data = NULL; + } // free sender taosMemoryFree(pSender); } @@ -344,9 +349,6 @@ _out:; return code; } -// return 0, start ok -// return 1, last snapshot finish ok -// return -1, error int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { SSyncSnapshotSender *pSender = syncNodeGetSnapshotSender(pSyncNode, pDestId); if (pSender == NULL) { @@ -377,6 +379,7 @@ int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { return 0; } +// receiver SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId) { bool condition = (pSyncNode->pFsm->FpSnapshotStartWrite != NULL) && (pSyncNode->pFsm->FpSnapshotStopWrite != NULL) && (pSyncNode->pFsm->FpSnapshotDoWrite != NULL); @@ -506,8 +509,6 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *p sRInfo(pReceiver, "snapshot receiver start, from dnode:%d.", DID(&pReceiver->fromId)); } -// just set start = false -// FpSnapshotStopWrite should not be called void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { sRDebug(pReceiver, "snapshot receiver stop, not apply, writer:%p", pReceiver->pWriter); @@ -528,7 +529,6 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { syncSnapBufferReset(pReceiver->pRcvBuf); } -// when recv last snapshot block, apply data into snapshot static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) { int32_t code = 0; if (pReceiver->pWriter != NULL) { @@ -587,8 +587,6 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap return 0; } -// apply data block -// update progress static int32_t snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) { if (pMsg->seq != pReceiver->ack + 1) { sRError(pReceiver, "snapshot receiver invalid seq, ack:%d seq:%d", pReceiver->ack, pMsg->seq); @@ -641,8 +639,8 @@ SyncIndex syncNodeGetSnapBeginIndex(SSyncNode *ths) { return snapStart; } -static int32_t syncNodeExchangeSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg, - SSnapshot *pInfo) { +static int32_t syncSnapReceiverExchgSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotReceiver *pReceiver, + SyncSnapshotSend *pMsg, SSnapshot *pInfo) { ASSERT(pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT); int32_t code = 0; @@ -734,7 +732,7 @@ _SEND_REPLY:; SSnapshot snapInfo = {.type = TDMT_SYNC_PREP_SNAPSHOT_REPLY}; int32_t dataLen = 0; if (pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT) { - if (syncNodeExchangeSnapInfo(pSyncNode, pReceiver, pMsg, &snapInfo) != 0) { + if (syncSnapReceiverExchgSnapInfo(pSyncNode, pReceiver, pMsg, &snapInfo) != 0) { goto _out; } SSyncTLV *datHead = snapInfo.data; @@ -949,26 +947,6 @@ _SEND_REPLY:; return code; } -// receiver on message -// -// condition 1, recv SYNC_SNAPSHOT_SEQ_PREP -// if receiver already start -// if sender.start-time > receiver.start-time, restart receiver(reply snapshot start) -// if sender.start-time = receiver.start-time, maybe duplicate msg -// if sender.start-time < receiver.start-time, ignore -// else -// waiting for clock match -// start receiver(reply snapshot start) -// -// condition 2, recv SYNC_SNAPSHOT_SEQ_BEGIN -// a. create writer with -// -// condition 3, recv SYNC_SNAPSHOT_SEQ_END, finish receiver(apply snapshot data, update commit index, maybe reconfig) -// -// condition 4, recv SYNC_SNAPSHOT_SEQ_FORCE_CLOSE, force close -// -// condition 5, got data, update ack -// int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { SyncSnapshotSend **ppMsg = (SyncSnapshotSend **)&pRpcMsg->pCont; SyncSnapshotSend *pMsg = ppMsg[0]; @@ -1052,6 +1030,32 @@ _out:; return code; } +static int32_t syncSnapSenderExchgSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { + ASSERT(pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT_REPLY); + + SSyncTLV *datHead = (void *)pMsg->data; + if (datHead->typ != pMsg->payloadType) { + sSError(pSender, "unexpected data type in data of SyncSnapshotRsp. typ: %d", datHead->typ); + terrno = TSDB_CODE_INVALID_DATA_FMT; + return -1; + } + int32_t dataLen = sizeof(SSyncTLV) + datHead->len; + + SSnapshotParam *pParam = &pSender->snapshotParam; + void *data = taosMemoryRealloc(pParam->data, dataLen); + if (data == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + memcpy(data, pMsg->data, dataLen); + + pParam->data = data; + data = NULL; + sSInfo(pSender, "data of snapshot param. len: %d", datHead->len); + return 0; +} + +// sender static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { SSnapshot snapshot = {0}; pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); @@ -1068,14 +1072,9 @@ static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSend // start reader if (pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { - SSyncTLV *datHead = (void *)pMsg->data; - if (datHead->typ != pMsg->payloadType) { - sSError(pSender, "unexpected data type in data of SyncSnapshotRsp. typ: %d", datHead->typ); - terrno = TSDB_CODE_INVALID_DATA_FMT; + if (syncSnapSenderExchgSnapInfo(pSyncNode, pSender, pMsg) != 0) { return -1; } - pSender->snapshotParam.data = (void *)pMsg->data; - sSInfo(pSender, "data of snapshot param. len: %d", datHead->len); } int32_t code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &pSender->snapshotParam, &pSender->pReader); @@ -1160,12 +1159,6 @@ _out: return code; } -// sender on message -// -// condition 1 sender receives SYNC_SNAPSHOT_SEQ_END, close sender -// condition 2 sender receives ack, set seq = ack + 1, send msg from seq -// condition 3 sender receives error msg, just print error log -// int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { SyncSnapshotRsp **ppMsg = (SyncSnapshotRsp **)&pRpcMsg->pCont; SyncSnapshotRsp *pMsg = ppMsg[0]; From fe258e226d67f6c65cda967ead94cd530fd3aadf Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 9 Nov 2023 18:27:07 +0800 Subject: [PATCH 18/34] feat: add data type of SNAP_DATA_RAW for snap replication --- source/dnode/vnode/src/inc/vnodeInt.h | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 7ed0b5103f..2c051ea642 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -531,6 +531,7 @@ enum { SNAP_DATA_STREAM_STATE = 11, SNAP_DATA_STREAM_STATE_BACKEND = 12, SNAP_DATA_TQ_CHECKINFO = 13, + SNAP_DATA_RAW = 14, }; struct SSnapDataHdr { From e05914119ed8515df37af53eca1d6a2a1747310f Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 10 Nov 2023 15:27:06 +0800 Subject: [PATCH 19/34] feat: add file tsdbSnapshotRAW.c --- source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c new file mode 100644 index 0000000000..e69de29bb2 From 61fbf089cc6f8a2929ed31c1decddb172e740679 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 10 Nov 2023 15:28:30 +0800 Subject: [PATCH 20/34] enh: add cscope.files into .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 704b2e7415..08e3d57717 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ CMakeSettings.json cmake-build-debug/ cmake-build-release/ cscope.out +cscope.files .DS_Store debug/ release/ From da646b5a5b158839d9f445eb6cc12d457d8c252c Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 13 Nov 2023 16:00:37 +0800 Subject: [PATCH 21/34] feat: add file tsdbDataFileRAW.c --- source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c | 0 source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c create mode 100644 source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c new file mode 100644 index 0000000000..e69de29bb2 diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h new file mode 100644 index 0000000000..e69de29bb2 From a4c504169d1a64589ae6b29a8df79592911038d4 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 15 Nov 2023 15:49:11 +0800 Subject: [PATCH 22/34] feat: add file tsdbFSetRAW.c --- source/dnode/vnode/src/tsdb/tsdbFSetRAW.c | 0 source/dnode/vnode/src/tsdb/tsdbFSetRAW.h | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 source/dnode/vnode/src/tsdb/tsdbFSetRAW.c create mode 100644 source/dnode/vnode/src/tsdb/tsdbFSetRAW.h diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c new file mode 100644 index 0000000000..e69de29bb2 diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.h b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.h new file mode 100644 index 0000000000..e69de29bb2 From 6c419423de9c2b7f3319da17e8d70994e851fbd2 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 20 Nov 2023 20:19:46 +0800 Subject: [PATCH 23/34] feat: impl tsdb snapshot reader and writer for raw files --- source/dnode/vnode/src/inc/vnodeInt.h | 11 + source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c | 212 +++++++ source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h | 127 ++++ source/dnode/vnode/src/tsdb/tsdbFSetRAW.c | 173 ++++++ source/dnode/vnode/src/tsdb/tsdbFSetRAW.h | 45 ++ source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 586 ++++++++++++++++++ source/dnode/vnode/src/vnd/vnodeSnapshot.c | 17 + 7 files changed, 1171 insertions(+) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 2c051ea642..50a28357e5 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -65,6 +65,8 @@ typedef struct SMetaSnapReader SMetaSnapReader; typedef struct SMetaSnapWriter SMetaSnapWriter; typedef struct STsdbSnapReader STsdbSnapReader; typedef struct STsdbSnapWriter STsdbSnapWriter; +typedef struct STsdbSnapRAWReader STsdbSnapRAWReader; +typedef struct STsdbSnapRAWWriter STsdbSnapRAWWriter; typedef struct STqSnapReader STqSnapReader; typedef struct STqSnapWriter STqSnapWriter; typedef struct STqOffsetReader STqOffsetReader; @@ -313,6 +315,15 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, void* pRang int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, SSnapDataHdr* pHdr); int32_t tsdbSnapWriterPrepareClose(STsdbSnapWriter* pWriter); int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback); +// STsdbSnapRAWReader ======================================== +int32_t tsdbSnapRAWReaderOpen(STsdb* pTsdb, int64_t ever, int8_t type, STsdbSnapRAWReader** ppReader); +int32_t tsdbSnapRAWReaderClose(STsdbSnapRAWReader** ppReader); +int32_t tsdbSnapRAWRead(STsdbSnapRAWReader* pReader, uint8_t** ppData); +// STsdbSnapRAWWriter ======================================== +int32_t tsdbSnapRAWWriterOpen(STsdb* pTsdb, int64_t ever, STsdbSnapRAWWriter** ppWriter); +int32_t tsdbSnapRAWWrite(STsdbSnapRAWWriter* pWriter, SSnapDataHdr* pHdr); +int32_t tsdbSnapRAWWriterPrepareClose(STsdbSnapRAWWriter* pWriter); +int32_t tsdbSnapRAWWriterClose(STsdbSnapRAWWriter** ppWriter, int8_t rollback); // STqSnapshotReader == int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** ppReader); int32_t tqSnapReaderClose(STqSnapReader** ppReader); diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c index e69de29bb2..0c3d890966 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDataFileRAW.h" + +// SDataFileRAWReader ============================================= +int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderConfig *config, + SDataFileRAWReader **reader) { + int32_t code = 0; + int32_t lino = 0; + + reader[0] = taosMemoryCalloc(1, sizeof(SDataFileRAWReader)); + if (reader[0] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + reader[0]->config[0] = config[0]; + + if (fname) { + if (fname) { + code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + char fname1[TSDB_FILENAME_LEN]; + tsdbTFileName(config->tsdb, &config->file, fname1); + code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileRAWReaderClose(SDataFileRAWReader **reader) { + if (reader[0] == NULL) return 0; + + if (reader[0]->fd) { + tsdbCloseFile(&reader[0]->fd); + } + + taosMemoryFree(reader[0]); + reader[0] = NULL; + return 0; +} + +int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlockHeader *bHdr) { + int32_t code = 0; + int32_t lino = 0; + + bHdr->file.type = reader->config->file.type; + bHdr->file.fid = reader->config->file.fid; + bHdr->file.cid = reader->config->file.cid; + bHdr->file.size = reader->config->file.size; + bHdr->file.minVer = reader->config->file.minVer; + bHdr->file.maxVer = reader->config->file.maxVer; + bHdr->file.stt->level = reader->config->file.stt->level; + + int64_t size = TMIN(bHdr->dataLength, reader->config->file.size - reader->ctx->offset); + ASSERT(size > 0); + bHdr->dataLength = 0; + bHdr->offset = reader->ctx->offset; + + code = tsdbReadFile(reader->fd, bHdr->offset, bHdr->data, size); + TSDB_CHECK_CODE(code, lino, _exit); + + bHdr->dataLength = size; +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +// SDataFileRAWWriter ============================================= +int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataFileRAWWriter **writer) { + writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); + if (!writer[0]) return TSDB_CODE_OUT_OF_MEMORY; + + writer[0]->config[0] = config[0]; + return 0; +} + +static int32_t tsdbDataFileRAWWriterCloseAbort(SDataFileRAWWriter *writer) { + ASSERT(0); + return 0; +} + +static int32_t tsdbDataFileRAWWriterDoClose(SDataFileRAWWriter *writer) { return 0; } + +int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + writer->file = writer->config->file; + writer->ctx->offset = 0; + + writer->ctx->opened = true; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFileOpArray *opArr) { + int32_t code = 0; + int32_t lino = 0; + STFileOp op; + + op = (STFileOp){ + .optype = TSDB_FOP_CREATE, + .fid = writer->config->fid, + .nf = writer->file, + }; + code = TARRAY2_APPEND(opArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + + if (writer->fd) { + code = tsdbFsyncFile(writer->fd); + TSDB_CHECK_CODE(code, lino, _exit); + tsdbCloseFile(&writer->fd); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileRAWWriterOpenDataFD(SDataFileRAWWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + char fname[TSDB_FILENAME_LEN]; + int32_t flag = TD_FILE_READ | TD_FILE_WRITE; + + if (writer->file.size == 0) { + flag |= (TD_FILE_CREATE | TD_FILE_TRUNC); + } + + tsdbTFileName(writer->config->tsdb, &writer->file, fname); + code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr) { + if (writer[0] == NULL) return 0; + + int32_t code = 0; + int32_t lino = 0; + + if (writer[0]->ctx->opened) { + if (abort) { + code = tsdbDataFileRAWWriterCloseAbort(writer[0]); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbDataFileRAWWriterCloseCommit(writer[0], opArr); + TSDB_CHECK_CODE(code, lino, _exit); + } + tsdbDataFileRAWWriterDoClose(writer[0]); + } + taosMemoryFree(writer[0]); + writer[0] = NULL; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer[0]->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *pDataBlock) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->file.size += pDataBlock->dataLength; + writer->ctx->offset += pDataBlock->dataLength; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h index e69de29bb2..49f80b0be5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tarray2.h" +#include "tsdbDef.h" +#include "tsdbFSet2.h" +#include "tsdbFile2.h" +#include "tsdbUtil2.h" + +#ifndef _TSDB_DATA_FILE_RAW_H +#define _TSDB_DATA_FILE_RAW_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define TSDB_SNAP_RAW_PAYLOAD_SIZE (4096 * 1024) +#if 0 +struct SDataRAWBlock { + int8_t *data; + int64_t size; +}; + +int32_t tsdbDataRAWBlockReset(SDataRAWBlock *pBlock); +int32_t tsdbDataRAWBlockAlloc(SDataRawBlock *pBlock); +void tsdbDataRAWBlockFree(SDataRAWBlock *pBlock); +#endif + +// STsdbDataRAWBlockHeader ======================================= +typedef struct STsdbDataRAWBlockHeader { + struct { + int32_t type; + int64_t fid; + int64_t cid; + int64_t size; + int64_t minVer; + int64_t maxVer; + union { + struct { + int32_t level; + } stt[1]; + }; + } file; + + int64_t offset; + int64_t dataLength; + uint8_t data[0]; +} STsdbDataRAWBlockHeader; + +// SDataFileRAWReader ============================================= +typedef struct SDataFileRAWReaderConfig { + STsdb *tsdb; + int32_t szPage; + + STFile file; +} SDataFileRAWReaderConfig; + +typedef struct SDataFileRAWReader { + SDataFileRAWReaderConfig config[1]; + + struct { + bool opened; + int64_t offset; + } ctx[1]; + + STsdbFD *fd; +} SDataFileRAWReader; + +typedef TARRAY2(SDataFileRAWReader *) SDataFileRAWReaderArray; + +int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderConfig *config, + SDataFileRAWReader **reader); +int32_t tsdbDataFileRAWReaderClose(SDataFileRAWReader **reader); + +int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlockHeader *bHdr); + +// SDataFileRAWWriter ============================================= +typedef struct SDataFileRAWWriterConfig { + STsdb *tsdb; + int32_t szPage; + + SDiskID did; + int64_t fid; + int64_t cid; + int32_t level; + + STFile file; +} SDataFileRAWWriterConfig; + +typedef struct SDataFileRAWWriter { + SDataFileRAWWriterConfig config[1]; + + struct { + bool opened; + int64_t offset; + } ctx[1]; + + STFile file; + STsdbFD *fd; +} SDataFileRAWWriter; + +typedef struct SDataFileRAWWriter SDataFileRAWWriter; + +int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataFileRAWWriter **writer); +int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr); + +int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer); +int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *bHdr); +int32_t tsdbDataFileRAWFlush(SDataFileRAWWriter *writer); + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_DATA_FILE_RAW_H*/ diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c index e69de29bb2..d9cd419ef9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbFSetRAW.h" + +// SFSetRAWWriter ================================================== +typedef struct SFSetRAWWriter { + SFSetRAWWriterConfig config[1]; + + struct { + TFileOpArray fopArr[1]; + STFile file; + int64_t offset; + } ctx[1]; + + // writer + SDataFileRAWWriter *dataWriter; +} SFSetRAWWriter; + +int32_t tsdbFSetRAWWriterOpen(SFSetRAWWriterConfig *config, SFSetRAWWriter **writer) { + int32_t code = 0; + int32_t lino = 0; + + writer[0] = taosMemoryCalloc(1, sizeof(SFSetRAWWriter)); + if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + writer[0]->config[0] = config[0]; + + TARRAY2_INIT(writer[0]->ctx->fopArr); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbFSetRAWWriterFinish(SFSetRAWWriter *writer, TFileOpArray *fopArr) { + int32_t code = 0; + int32_t lino = 0; + + STsdb *tsdb = writer->config->tsdb; + + STFileOp op; + TARRAY2_FOREACH(writer->ctx->fopArr, op) { + code = TARRAY2_APPEND(fopArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + } + + TARRAY2_CLEAR(writer->ctx->fopArr, NULL); +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr) { + int32_t code = 0; + int32_t lino = 0; + + SDataFileRAWWriterConfig config = { + .tsdb = writer->config->tsdb, + .szPage = writer->config->szPage, + .did = writer->config->did, + .cid = writer->config->cid, + .level = writer->config->level, + + .file = + { + .type = bHdr->file.type, + .did = writer->config->did, + .cid = writer->config->cid, + .size = bHdr->file.size, + .minVer = bHdr->file.minVer, + .maxVer = bHdr->file.maxVer, + .stt = {{ + .level = bHdr->file.stt->level, + }}, + }, + }; + + code = tsdbDataFileRAWWriterOpen(&config, &writer->dataWriter); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbFSetRAWWriteFileDataEnd(SFSetRAWWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbDataFileRAWWriterClose(&writer->dataWriter, false, writer->ctx->fopArr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFSetRAWWriterClose(SFSetRAWWriter **writer, bool abort, TFileOpArray *fopArr) { + if (writer[0] == NULL) return 0; + + int32_t code = 0; + int32_t lino = 0; + + STsdb *tsdb = writer[0]->config->tsdb; + + // end + code = tsdbFSetRAWWriteFileDataEnd(writer[0]); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileRAWWriterClose(&writer[0]->dataWriter, abort, writer[0]->ctx->fopArr); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFSetRAWWriterFinish(writer[0], fopArr); + TSDB_CHECK_CODE(code, lino, _exit); + // free + TARRAY2_DESTROY(writer[0]->ctx->fopArr, NULL); + taosMemoryFree(writer[0]); + writer[0] = NULL; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(writer->ctx->offset >= 0 && writer->ctx->offset <= writer->ctx->file.size); + + if (writer->ctx->offset == writer->ctx->file.size) { + code = tsdbFSetRAWWriteFileDataEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFSetRAWWriteFileDataBegin(writer, bHdr); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbDataFileRAWWriteBlockData(writer->dataWriter, bHdr); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->offset += bHdr->dataLength; + ASSERT(writer->ctx->offset == writer->dataWriter->ctx->offset); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.h b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.h index e69de29bb2..205c785e99 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDataFileRAW.h" + +#ifndef _TSDB_FSET_RAW_H +#define _TSDB_FSET_RAW_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SFSetRAWWriterConfig { + STsdb *tsdb; + int32_t szPage; + + SDiskID did; + int64_t fid; + int64_t cid; + int32_t level; +} SFSetRAWWriterConfig; + +typedef struct SFSetRAWWriter SFSetRAWWriter; + +int32_t tsdbFSetRAWWriterOpen(SFSetRAWWriterConfig *config, SFSetRAWWriter **writer); +int32_t tsdbFSetRAWWriterClose(SFSetRAWWriter **writer, bool abort, TFileOpArray *fopArr); +int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr); + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_FSET_RAW_H*/ diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index e69de29bb2..2bcd00bfec 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -0,0 +1,586 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" +#include "tsdbDataFileRAW.h" +#include "tsdbFS2.h" +#include "tsdbFSetRAW.h" + +// reader + +typedef struct SDataFileRAWReaderIter { + int32_t count; + int32_t idx; + int64_t offset; + int64_t size; +} SDataFileRAWReaderIter; + +typedef struct STsdbSnapRAWReader { + STsdb* tsdb; + int64_t ever; + int8_t type; + + TFileSetArray* fsetArr; + + // context + struct { + int32_t fsetArrIdx; + STFileSet* fset; + bool isDataDone; + } ctx[1]; + + // reader + SDataFileRAWReaderArray dataReaderArr[1]; + + // iter + SDataFileRAWReaderIter dataIter[1]; +} STsdbSnapRAWReader; + +int32_t tsdbSnapRAWReaderOpen(STsdb* tsdb, int64_t ever, int8_t type, STsdbSnapRAWReader** reader) { + int32_t code = 0; + int32_t lino = 0; + + reader[0] = taosMemoryCalloc(1, sizeof(STsdbSnapRAWReader)); + if (reader[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + reader[0]->tsdb = tsdb; + reader[0]->ever = ever; + reader[0]->type = type; + + code = tsdbFSCreateRefSnapshot(tsdb->pFS, &reader[0]->fsetArr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), + __func__, lino, tstrerror(code), 0, ever, type); + tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr); + taosMemoryFree(reader[0]); + reader[0] = NULL; + } else { + tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), 0, + ever, type); + } + return code; +} + +int32_t tsdbSnapRAWReaderClose(STsdbSnapRAWReader** reader) { + if (reader[0] == NULL) return 0; + + int32_t code = 0; + int32_t lino = 0; + + STsdb* tsdb = reader[0]->tsdb; + + tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr); + taosMemoryFree(reader[0]); + reader[0] = NULL; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); + } + return code; +} + +static int32_t tsdbSnapRAWReadFileSetOpenReader(STsdbSnapRAWReader* reader) { + int32_t code = 0; + int32_t lino = 0; + + // data + for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ftype++) { + if (reader->ctx->fset->farr[ftype] == NULL) { + continue; + } + STFileObj* fobj = reader->ctx->fset->farr[ftype]; + SDataFileRAWReader* dataReader; + SDataFileRAWReaderConfig config = { + .tsdb = reader->tsdb, + .szPage = reader->tsdb->pVnode->config.tsdbPageSize, + .file = fobj->f[0], + }; + code = tsdbDataFileRAWReaderOpen(NULL, &config, &dataReader); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(reader->dataReaderArr, dataReader); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // stt + SSttLvl* lvl; + TARRAY2_FOREACH(reader->ctx->fset->lvlArr, lvl) { + STFileObj* fobj; + TARRAY2_FOREACH(lvl->fobjArr, fobj) { + SDataFileRAWReader* dataReader; + SDataFileRAWReaderConfig config = { + .tsdb = reader->tsdb, + .szPage = reader->tsdb->pVnode->config.tsdbPageSize, + .file = fobj->f[0], + }; + code = tsdbDataFileRAWReaderOpen(NULL, &config, &dataReader); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(reader->dataReaderArr, dataReader); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + +static int32_t tsdbSnapRAWReadFileSetCloseReader(STsdbSnapRAWReader* reader) { + int32_t code = 0; + int32_t lino = 0; + + TARRAY2_CLEAR(reader->dataReaderArr, tsdbDataFileRAWReaderClose); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + +static int32_t tsdbSnapRAWReadFileSetOpenIter(STsdbSnapRAWReader* reader) { + int32_t code = 0; + int32_t lino = 0; + + reader->dataIter->count = TARRAY2_SIZE(reader->dataReaderArr); + reader->dataIter->idx = -1; + reader->dataIter->offset = 0; + reader->dataIter->size = 0; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + +static int32_t tsdbSnapRAWReadFileSetCloseIter(STsdbSnapRAWReader* reader) { + reader->dataIter->count = 0; + reader->dataIter->idx = 0; + reader->dataIter->offset = 0; + reader->dataIter->size = 0; + return 0; +} + +static int32_t tsdbSnapRAWReadNext(STsdbSnapRAWReader* reader, STsdbDataRAWBlockHeader* bHdr) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(reader->dataIter->offset <= reader->dataIter->size); + ASSERT(reader->dataIter->idx <= reader->dataIter->count); + + if (reader->dataIter->offset == reader->dataIter->size && reader->dataIter->idx < reader->dataIter->count) { + reader->dataIter->idx++; + } + if (reader->dataIter->idx == reader->dataIter->count) { + return 0; + } + + SDataFileRAWReader* dataReader = TARRAY2_GET(reader->dataReaderArr, reader->dataIter->idx); + code = tsdbDataFileRAWReadBlockData(dataReader, bHdr); + TSDB_CHECK_CODE(code, lino, _exit); + + reader->dataIter->offset += bHdr->dataLength; +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + +static int32_t tsdbSnapRAWReadData(STsdbSnapRAWReader* reader, SSnapDataHdr** data) { + int32_t code = 0; + int32_t lino = 0; + + void* pBuf = taosMemoryCalloc(1, sizeof(SSnapDataHdr) + sizeof(STsdbDataRAWBlockHeader) + TSDB_SNAP_RAW_PAYLOAD_SIZE); + if (pBuf == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + SSnapDataHdr* pHdr = pBuf; + pHdr->type = reader->type; + STsdbDataRAWBlockHeader* pData = (void*)pHdr->data; + pData->dataLength = TSDB_SNAP_RAW_PAYLOAD_SIZE; + + code = tsdbSnapRAWReadNext(reader, pData); + TSDB_CHECK_CODE(code, lino, _exit); + + ASSERT(pData->dataLength > 0 && pData->dataLength <= TSDB_SNAP_RAW_PAYLOAD_SIZE); + pHdr->size = sizeof(STsdbDataRAWBlockHeader) + pData->dataLength; + +_exit: + if (code) { + taosMemoryFree(pBuf); + pBuf = NULL; + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + data[0] = pBuf; + return code; +} + +static int32_t tsdbSnapRAWReadBegin(STsdbSnapRAWReader* reader) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(reader->ctx->fset == NULL); + + if (reader->ctx->fsetArrIdx < TARRAY2_SIZE(reader->fsetArr)) { + reader->ctx->fset = TARRAY2_GET(reader->fsetArr, reader->ctx->fsetArrIdx++); + reader->ctx->isDataDone = false; + + code = tsdbSnapRAWReadFileSetOpenReader(reader); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbSnapRAWReadFileSetOpenIter(reader); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + +static int32_t tsdbSnapRAWReadEnd(STsdbSnapRAWReader* reader) { + tsdbSnapRAWReadFileSetCloseIter(reader); + tsdbSnapRAWReadFileSetCloseReader(reader); + reader->ctx->fset = NULL; + return 0; +} + +int32_t tsdbSnapRAWRead(STsdbSnapRAWReader* reader, uint8_t** data) { + int32_t code = 0; + int32_t lino = 0; + + data[0] = NULL; + + for (;;) { + if (reader->ctx->fset == NULL) { + code = tsdbSnapRAWReadBegin(reader); + TSDB_CHECK_CODE(code, lino, _exit); + + if (reader->ctx->fset == NULL) { + break; + } + } + + if (!reader->ctx->isDataDone) { + code = tsdbSnapRAWReadData(reader, (SSnapDataHdr**)data); + TSDB_CHECK_CODE(code, lino, _exit); + if (data[0]) { + goto _exit; + } else { + reader->ctx->isDataDone = true; + } + } + + code = tsdbSnapRAWReadEnd(reader); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } else { + tsdbDebug("vgId:%d %s done", TD_VID(reader->tsdb->pVnode), __func__); + } + return code; +} + +// writer +struct STsdbSnapRAWWriter { + STsdb* tsdb; + int64_t sver; + int64_t ever; + int32_t minutes; + int8_t precision; + int32_t minRow; + int32_t maxRow; + int8_t cmprAlg; + int64_t commitID; + int32_t szPage; + int64_t compactVersion; + int64_t now; + + TFileSetArray* fsetArr; + TFileOpArray fopArr[1]; + + struct { + bool fsetWriteBegin; + int32_t fid; + STFileSet* fset; + SDiskID did; + int64_t cid; + int64_t level; + + // writer + SFSetRAWWriter* fsetWriter; + } ctx[1]; +}; + +int32_t tsdbSnapRAWWriterOpen(STsdb* pTsdb, int64_t ever, STsdbSnapRAWWriter** writer) { + int32_t code = 0; + int32_t lino = 0; + + // disable background tasks + tsdbFSDisableBgTask(pTsdb->pFS); + + // start to write + writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); + if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + writer[0]->tsdb = pTsdb; + writer[0]->ever = ever; + writer[0]->minutes = pTsdb->keepCfg.days; + writer[0]->precision = pTsdb->keepCfg.precision; + writer[0]->minRow = pTsdb->pVnode->config.tsdbCfg.minRows; + writer[0]->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; + writer[0]->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; + writer[0]->commitID = tsdbFSAllocEid(pTsdb->pFS); + writer[0]->szPage = pTsdb->pVnode->config.tsdbPageSize; + writer[0]->compactVersion = INT64_MAX; + writer[0]->now = taosGetTimestampMs(); + + code = tsdbFSCreateCopySnapshot(pTsdb->pFS, &writer[0]->fsetArr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbInfo("vgId:%d %s done, sver:%" PRId64 " ever:%" PRId64, TD_VID(pTsdb->pVnode), __func__, 0, ever); + } + return code; +} + +static int32_t tsdbSnapRAWWriteFileSetOpenIter(STsdbSnapRAWWriter* writer) { + int32_t code = 0; + int32_t lino = 0; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbSnapRAWWriteFileSetCloseIter(STsdbSnapRAWWriter* writer) { return 0; } + +static int32_t tsdbSnapRAWWriteFileSetOpenWriter(STsdbSnapRAWWriter* writer) { + int32_t code = 0; + int32_t lino = 0; + + SFSetRAWWriterConfig config = { + .tsdb = writer->tsdb, + .szPage = writer->szPage, + .fid = writer->ctx->fid, + .cid = writer->commitID, + .did = writer->ctx->did, + .level = writer->ctx->level, + }; + + code = tsdbFSetRAWWriterOpen(&config, &writer->ctx->fsetWriter); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbSnapRAWWriteFileSetCloseWriter(STsdbSnapRAWWriter* writer) { + return tsdbFSetRAWWriterClose(&writer->ctx->fsetWriter, 0, writer->fopArr); +} + +static int32_t tsdbSnapRAWWriteFileSetBegin(STsdbSnapRAWWriter* writer, int32_t fid) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(writer->ctx->fsetWriteBegin == false); + + STFileSet* fset = &(STFileSet){.fid = fid}; + + writer->ctx->fid = fid; + STFileSet** fsetPtr = TARRAY2_SEARCH(writer->fsetArr, &fset, tsdbTFileSetCmprFn, TD_EQ); + writer->ctx->fset = (fsetPtr == NULL) ? NULL : *fsetPtr; + + int32_t level = tsdbFidLevel(fid, &writer->tsdb->keepCfg, taosGetTimestampSec()); + if (tfsAllocDisk(writer->tsdb->pVnode->pTfs, level, &writer->ctx->did)) { + code = TSDB_CODE_NO_AVAIL_DISK; + TSDB_CHECK_CODE(code, lino, _exit); + } + tfsMkdirRecurAt(writer->tsdb->pVnode->pTfs, writer->tsdb->path, writer->ctx->did); + + code = tsdbSnapRAWWriteFileSetOpenWriter(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->level = level; + writer->ctx->fsetWriteBegin = true; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbSnapRAWWriteFileSetEnd(STsdbSnapRAWWriter* writer) { + if (!writer->ctx->fsetWriteBegin) return 0; + + int32_t code = 0; + int32_t lino = 0; + + // close write + code = tsdbSnapRAWWriteFileSetCloseWriter(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->fsetWriteBegin = false; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbSnapRAWWriterPrepareClose(STsdbSnapRAWWriter* writer) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbSnapRAWWriteFileSetEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFSEditBegin(writer->tsdb->pFS, writer->fopArr, TSDB_FEDIT_COMMIT); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d %s done", TD_VID(writer->tsdb->pVnode), __func__); + } + return code; +} + +int32_t tsdbSnapRAWWriterClose(STsdbSnapRAWWriter** writer, int8_t rollback) { + if (writer[0] == NULL) return 0; + + int32_t code = 0; + int32_t lino = 0; + + STsdb* tsdb = writer[0]->tsdb; + + if (rollback) { + code = tsdbFSEditAbort(writer[0]->tsdb->pFS); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + taosThreadMutexLock(&writer[0]->tsdb->mutex); + + code = tsdbFSEditCommit(writer[0]->tsdb->pFS); + if (code) { + taosThreadMutexUnlock(&writer[0]->tsdb->mutex); + TSDB_CHECK_CODE(code, lino, _exit); + } + + writer[0]->tsdb->pFS->fsstate = TSDB_FS_STATE_NORMAL; + + taosThreadMutexUnlock(&writer[0]->tsdb->mutex); + } + tsdbFSEnableBgTask(tsdb->pFS); + + TARRAY2_DESTROY(writer[0]->fopArr, NULL); + tsdbFSDestroyCopySnapshot(&writer[0]->fsetArr); + + taosMemoryFree(writer[0]); + writer[0] = NULL; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } else { + tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); + } + return code; +} + +static int32_t tsdbSnapRAWWriteTimeSeriesData(STsdbSnapRAWWriter* writer, STsdbDataRAWBlockHeader* bHdr) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFSetRAWWriteBlockData(writer->ctx->fsetWriter, bHdr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbSnapRAWWriteData(STsdbSnapRAWWriter* writer, SSnapDataHdr* hdr) { + int32_t code = 0; + int32_t lino = 0; + + STsdbDataRAWBlockHeader* bHdr = (void*)hdr->data; + int32_t fid = bHdr->file.fid; + if (!writer->ctx->fsetWriteBegin || fid != writer->ctx->fid) { + code = tsdbSnapRAWWriteFileSetEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbSnapRAWWriteFileSetBegin(writer, fid); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbSnapRAWWriteTimeSeriesData(writer, bHdr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbSnapRAWWrite(STsdbSnapRAWWriter* writer, SSnapDataHdr* hdr) { + ASSERT(hdr->type == SNAP_DATA_RAW); + + int32_t code = 0; + int32_t lino = 0; + + code = tsdbSnapRAWWriteData(writer, hdr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s, type:%d index:%" PRId64 " size:%" PRId64, + TD_VID(writer->tsdb->pVnode), __func__, lino, tstrerror(code), hdr->type, hdr->index, hdr->size); + } else { + tsdbDebug("vgId:%d %s done, type:%d index:%" PRId64 " size:%" PRId64, TD_VID(writer->tsdb->pVnode), __func__, + hdr->type, hdr->index, hdr->size); + } + return code; +} diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 21c858709b..1a6b1a7a6c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -51,6 +51,10 @@ struct SVSnapReader { int8_t tsdbDone; TFileSetRangeArray *pRanges; STsdbSnapReader *pTsdbReader; + // tsdb raw + int8_t tsdbRawDone; + STsdbSnapRAWReader *pTsdbRawReader; + // tq int8_t tqHandleDone; STqSnapReader *pTqSnapReader; @@ -467,6 +471,8 @@ struct SVSnapWriter { // tsdb TFileSetRangeArray *pRanges; STsdbSnapWriter *pTsdbSnapWriter; + // tsdb raw + STsdbSnapRAWWriter *pTsdbSnapRAWWriter; // tq STqSnapWriter *pTqSnapWriter; STqOffsetWriter *pTqOffsetWriter; @@ -772,6 +778,17 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pHdr); if (code) goto _err; } break; + case SNAP_DATA_RAW: { + // tsdb + if (pWriter->pTsdbSnapRAWWriter == NULL) { + ASSERT(pWriter->sver == 0); + code = tsdbSnapRAWWriterOpen(pVnode->pTsdb, pWriter->ever, &pWriter->pTsdbSnapRAWWriter); + if (code) goto _err; + } + + code = tsdbSnapRAWWrite(pWriter->pTsdbSnapRAWWriter, pHdr); + if (code) goto _err; + } break; case SNAP_DATA_TQ_HANDLE: { // tq handle if (pWriter->pTqSnapWriter == NULL) { From a504007ade887f559c3ad2486d7a82de09651b5f Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 24 Nov 2023 16:13:53 +0800 Subject: [PATCH 24/34] refact: alloc just enough memory for snap data in tsdbSnapRAWReadData --- source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c | 24 +++---- source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 65 ++++++++++++------- 2 files changed, 49 insertions(+), 40 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c index 0c3d890966..aaaaa56b0e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c @@ -60,27 +60,21 @@ int32_t tsdbDataFileRAWReaderClose(SDataFileRAWReader **reader) { return 0; } -int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlockHeader *bHdr) { +int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlockHeader *pBlock) { int32_t code = 0; int32_t lino = 0; - bHdr->file.type = reader->config->file.type; - bHdr->file.fid = reader->config->file.fid; - bHdr->file.cid = reader->config->file.cid; - bHdr->file.size = reader->config->file.size; - bHdr->file.minVer = reader->config->file.minVer; - bHdr->file.maxVer = reader->config->file.maxVer; - bHdr->file.stt->level = reader->config->file.stt->level; + pBlock->file.type = reader->config->file.type; + pBlock->file.fid = reader->config->file.fid; + pBlock->file.cid = reader->config->file.cid; + pBlock->file.size = reader->config->file.size; + pBlock->file.minVer = reader->config->file.minVer; + pBlock->file.maxVer = reader->config->file.maxVer; + pBlock->file.stt->level = reader->config->file.stt->level; - int64_t size = TMIN(bHdr->dataLength, reader->config->file.size - reader->ctx->offset); - ASSERT(size > 0); - bHdr->dataLength = 0; - bHdr->offset = reader->ctx->offset; - - code = tsdbReadFile(reader->fd, bHdr->offset, bHdr->data, size); + code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength); TSDB_CHECK_CODE(code, lino, _exit); - bHdr->dataLength = size; _exit: if (code) { TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index 2bcd00bfec..14c52c0bfb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -183,52 +183,54 @@ static int32_t tsdbSnapRAWReadFileSetCloseIter(STsdbSnapRAWReader* reader) { return 0; } -static int32_t tsdbSnapRAWReadNext(STsdbSnapRAWReader* reader, STsdbDataRAWBlockHeader* bHdr) { +static int64_t tsdbSnapRAWReadPeek(SDataFileRAWReader* reader) { + int64_t size = TMIN(reader->config->file.size - reader->ctx->offset, TSDB_SNAP_RAW_PAYLOAD_SIZE); + return size; +} + +static int32_t tsdbSnapRAWReadNext(STsdbSnapRAWReader* reader, SSnapDataHdr** ppData) { int32_t code = 0; int32_t lino = 0; + ppData[0] = NULL; ASSERT(reader->dataIter->offset <= reader->dataIter->size); ASSERT(reader->dataIter->idx <= reader->dataIter->count); + // dataReader if (reader->dataIter->offset == reader->dataIter->size && reader->dataIter->idx < reader->dataIter->count) { reader->dataIter->idx++; } if (reader->dataIter->idx == reader->dataIter->count) { return 0; } - + int8_t type = reader->type; SDataFileRAWReader* dataReader = TARRAY2_GET(reader->dataReaderArr, reader->dataIter->idx); - code = tsdbDataFileRAWReadBlockData(dataReader, bHdr); - TSDB_CHECK_CODE(code, lino, _exit); - reader->dataIter->offset += bHdr->dataLength; -_exit: - if (code) { - TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); - } - return code; -} + // prepare + int64_t dataLength = tsdbSnapRAWReadPeek(dataReader); + ASSERT(dataLength > 0); -static int32_t tsdbSnapRAWReadData(STsdbSnapRAWReader* reader, SSnapDataHdr** data) { - int32_t code = 0; - int32_t lino = 0; - - void* pBuf = taosMemoryCalloc(1, sizeof(SSnapDataHdr) + sizeof(STsdbDataRAWBlockHeader) + TSDB_SNAP_RAW_PAYLOAD_SIZE); + void* pBuf = taosMemoryCalloc(1, sizeof(SSnapDataHdr) + sizeof(STsdbDataRAWBlockHeader) + dataLength); if (pBuf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _exit); } - SSnapDataHdr* pHdr = pBuf; - pHdr->type = reader->type; - STsdbDataRAWBlockHeader* pData = (void*)pHdr->data; - pData->dataLength = TSDB_SNAP_RAW_PAYLOAD_SIZE; + pHdr->type = type; + pHdr->size = sizeof(STsdbDataRAWBlockHeader) + dataLength; - code = tsdbSnapRAWReadNext(reader, pData); + STsdbDataRAWBlockHeader* pBlock = (void*)pHdr->data; + pBlock->offset = dataReader->ctx->offset; + pBlock->dataLength = dataLength; + + // read + code = tsdbDataFileRAWReadBlockData(dataReader, pBlock); TSDB_CHECK_CODE(code, lino, _exit); - ASSERT(pData->dataLength > 0 && pData->dataLength <= TSDB_SNAP_RAW_PAYLOAD_SIZE); - pHdr->size = sizeof(STsdbDataRAWBlockHeader) + pData->dataLength; + // finish + reader->dataIter->offset += pBlock->dataLength; + ppData[0] = pBuf; + ASSERT(reader->dataIter->offset <= reader->dataIter->size); _exit: if (code) { @@ -236,7 +238,20 @@ _exit: pBuf = NULL; TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); } - data[0] = pBuf; + return code; +} + +static int32_t tsdbSnapRAWReadData(STsdbSnapRAWReader* reader, uint8_t** ppData) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbSnapRAWReadNext(reader, (SSnapDataHdr**)ppData); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } return code; } @@ -288,7 +303,7 @@ int32_t tsdbSnapRAWRead(STsdbSnapRAWReader* reader, uint8_t** data) { } if (!reader->ctx->isDataDone) { - code = tsdbSnapRAWReadData(reader, (SSnapDataHdr**)data); + code = tsdbSnapRAWReadData(reader, data); TSDB_CHECK_CODE(code, lino, _exit); if (data[0]) { goto _exit; From ae60e1f810e411273b59a0c8b0fad324cc37404a Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 24 Nov 2023 16:15:33 +0800 Subject: [PATCH 25/34] refact: add tsdbSnapRAWReaderIterNext --- source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index 14c52c0bfb..7010e6ad27 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -23,8 +23,6 @@ typedef struct SDataFileRAWReaderIter { int32_t count; int32_t idx; - int64_t offset; - int64_t size; } SDataFileRAWReaderIter; typedef struct STsdbSnapRAWReader { @@ -164,9 +162,7 @@ static int32_t tsdbSnapRAWReadFileSetOpenIter(STsdbSnapRAWReader* reader) { int32_t lino = 0; reader->dataIter->count = TARRAY2_SIZE(reader->dataReaderArr); - reader->dataIter->idx = -1; - reader->dataIter->offset = 0; - reader->dataIter->size = 0; + reader->dataIter->idx = 0; _exit: if (code) { @@ -178,8 +174,6 @@ _exit: static int32_t tsdbSnapRAWReadFileSetCloseIter(STsdbSnapRAWReader* reader) { reader->dataIter->count = 0; reader->dataIter->idx = 0; - reader->dataIter->offset = 0; - reader->dataIter->size = 0; return 0; } @@ -188,23 +182,30 @@ static int64_t tsdbSnapRAWReadPeek(SDataFileRAWReader* reader) { return size; } +static SDataFileRAWReader* tsdbSnapRAWReaderIterNext(STsdbSnapRAWReader* reader) { + ASSERT(reader->dataIter->idx <= reader->dataIter->count); + + while (reader->dataIter->idx < reader->dataIter->count) { + SDataFileRAWReader* dataReader = TARRAY2_GET(reader->dataReaderArr, reader->dataIter->idx); + ASSERT(dataReader); + if (dataReader->ctx->offset < dataReader->config->file.size) { + return dataReader; + } + reader->dataIter->idx++; + } + return NULL; +} + static int32_t tsdbSnapRAWReadNext(STsdbSnapRAWReader* reader, SSnapDataHdr** ppData) { int32_t code = 0; int32_t lino = 0; + int8_t type = reader->type; ppData[0] = NULL; - ASSERT(reader->dataIter->offset <= reader->dataIter->size); - ASSERT(reader->dataIter->idx <= reader->dataIter->count); - - // dataReader - if (reader->dataIter->offset == reader->dataIter->size && reader->dataIter->idx < reader->dataIter->count) { - reader->dataIter->idx++; - } - if (reader->dataIter->idx == reader->dataIter->count) { + SDataFileRAWReader* dataReader = tsdbSnapRAWReaderIterNext(reader); + if (dataReader == NULL) { return 0; } - int8_t type = reader->type; - SDataFileRAWReader* dataReader = TARRAY2_GET(reader->dataReaderArr, reader->dataIter->idx); // prepare int64_t dataLength = tsdbSnapRAWReadPeek(dataReader); @@ -219,18 +220,18 @@ static int32_t tsdbSnapRAWReadNext(STsdbSnapRAWReader* reader, SSnapDataHdr** pp pHdr->type = type; pHdr->size = sizeof(STsdbDataRAWBlockHeader) + dataLength; + // read STsdbDataRAWBlockHeader* pBlock = (void*)pHdr->data; pBlock->offset = dataReader->ctx->offset; pBlock->dataLength = dataLength; - // read code = tsdbDataFileRAWReadBlockData(dataReader, pBlock); TSDB_CHECK_CODE(code, lino, _exit); // finish - reader->dataIter->offset += pBlock->dataLength; + dataReader->ctx->offset += pBlock->dataLength; + ASSERT(dataReader->ctx->offset <= dataReader->config->file.size); ppData[0] = pBuf; - ASSERT(reader->dataIter->offset <= reader->dataIter->size); _exit: if (code) { From 1dc9019baa66e30d88497ab993c7948a0b1e70b0 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 24 Nov 2023 16:28:39 +0800 Subject: [PATCH 26/34] feat: add tsdbSnapRAWRead into vnodeSnapRead --- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 26 ++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 1a6b1a7a6c..cb3d346df5 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -52,8 +52,8 @@ struct SVSnapReader { TFileSetRangeArray *pRanges; STsdbSnapReader *pTsdbReader; // tsdb raw - int8_t tsdbRawDone; - STsdbSnapRAWReader *pTsdbRawReader; + int8_t tsdbRAWDone; + STsdbSnapRAWReader *pTsdbRAWReader; // tq int8_t tqHandleDone; @@ -299,6 +299,28 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) } } + if (!pReader->tsdbRAWDone) { + // open if not + if (pReader->pTsdbRAWReader == NULL) { + ASSERT(pReader->sver == 0); + code = tsdbSnapRAWReaderOpen(pReader->pVnode->pTsdb, pReader->ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader); + if (code) goto _err; + } + + code = tsdbSnapRAWRead(pReader->pTsdbRAWReader, ppData); + if (code) { + goto _err; + } else { + if (*ppData) { + goto _exit; + } else { + pReader->tsdbRAWDone = 1; + code = tsdbSnapRAWReaderClose(&pReader->pTsdbRAWReader); + if (code) goto _err; + } + } + } + // TQ ================ vInfo("vgId:%d tq transform start", vgId); if (!pReader->tqHandleDone) { From f1362669722690e5a70eaf3672b44f06f896c85b Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 24 Nov 2023 20:04:39 +0800 Subject: [PATCH 27/34] feat: support snap replication by file blocks --- source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c | 66 ++++++++++++------- source/dnode/vnode/src/tsdb/tsdbFSetRAW.c | 5 ++ source/dnode/vnode/src/vnd/vnodeSnapshot.c | 11 ++++ 3 files changed, 57 insertions(+), 25 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c index aaaaa56b0e..4a72fcfda8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c @@ -83,12 +83,26 @@ _exit: } // SDataFileRAWWriter ============================================= -int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataFileRAWWriter **writer) { - writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); - if (!writer[0]) return TSDB_CODE_OUT_OF_MEMORY; +int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataFileRAWWriter **ppWriter) { + int32_t code = 0; + int32_t lino = 0; - writer[0]->config[0] = config[0]; - return 0; + SDataFileRAWWriter *writer = taosMemoryCalloc(1, sizeof(SDataFileRAWWriter)); + if (!writer) return TSDB_CODE_OUT_OF_MEMORY; + + writer->config[0] = config[0]; + + code = tsdbDataFileRAWWriterDoOpen(writer); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + taosMemoryFree(writer); + writer = NULL; + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + ppWriter[0] = writer; + return code; } static int32_t tsdbDataFileRAWWriterCloseAbort(SDataFileRAWWriter *writer) { @@ -98,28 +112,13 @@ static int32_t tsdbDataFileRAWWriterCloseAbort(SDataFileRAWWriter *writer) { static int32_t tsdbDataFileRAWWriterDoClose(SDataFileRAWWriter *writer) { return 0; } -int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer) { - int32_t code = 0; - int32_t lino = 0; - - writer->file = writer->config->file; - writer->ctx->offset = 0; - - writer->ctx->opened = true; - -_exit: - if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); - } - return code; -} - static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFileOpArray *opArr) { int32_t code = 0; int32_t lino = 0; - STFileOp op; + ASSERT(writer->ctx->offset == writer->file.size); + ASSERT(writer->config->fid == writer->file.fid); - op = (STFileOp){ + STFileOp op = (STFileOp){ .optype = TSDB_FOP_CREATE, .fid = writer->config->fid, .nf = writer->file, @@ -147,7 +146,7 @@ static int32_t tsdbDataFileRAWWriterOpenDataFD(SDataFileRAWWriter *writer) { char fname[TSDB_FILENAME_LEN]; int32_t flag = TD_FILE_READ | TD_FILE_WRITE; - if (writer->file.size == 0) { + if (writer->ctx->offset == 0) { flag |= (TD_FILE_CREATE | TD_FILE_TRUNC); } @@ -162,6 +161,24 @@ _exit: return code; } +int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + writer->file = writer->config->file; + writer->ctx->offset = 0; + + code = tsdbDataFileRAWWriterOpenDataFD(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->opened = true; +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr) { if (writer[0] == NULL) return 0; @@ -195,7 +212,6 @@ int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDat code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength); TSDB_CHECK_CODE(code, lino, _exit); - writer->file.size += pDataBlock->dataLength; writer->ctx->offset += pDataBlock->dataLength; _exit: diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c index d9cd419ef9..03c12502d5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c @@ -74,6 +74,7 @@ static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRA SDataFileRAWWriterConfig config = { .tsdb = writer->config->tsdb, .szPage = writer->config->szPage, + .fid = bHdr->file.fid, .did = writer->config->did, .cid = writer->config->cid, .level = writer->config->level, @@ -81,6 +82,7 @@ static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRA .file = { .type = bHdr->file.type, + .fid = bHdr->file.fid, .did = writer->config->did, .cid = writer->config->cid, .size = bHdr->file.size, @@ -92,6 +94,9 @@ static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRA }, }; + writer->ctx->offset = 0; + writer->ctx->file = config.file; + code = tsdbDataFileRAWWriterOpen(&config, &writer->dataWriter); TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index cb3d346df5..2a8484bcd2 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -277,6 +277,8 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) } // TSDB ============== + pReader->tsdbDone = true; + if (!pReader->tsdbDone) { // open if not if (pReader->pTsdbReader == NULL) { @@ -641,6 +643,10 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * tsdbSnapWriterPrepareClose(pWriter->pTsdbSnapWriter); } + if (pWriter->pTsdbSnapRAWWriter) { + tsdbSnapRAWWriterPrepareClose(pWriter->pTsdbSnapRAWWriter); + } + if (pWriter->pRsmaSnapWriter) { rsmaSnapWriterPrepareClose(pWriter->pRsmaSnapWriter); } @@ -677,6 +683,11 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * if (code) goto _exit; } + if (pWriter->pTsdbSnapRAWWriter) { + code = tsdbSnapRAWWriterClose(&pWriter->pTsdbSnapRAWWriter, rollback); + if (code) goto _exit; + } + if (pWriter->pTqSnapWriter) { code = tqSnapWriterClose(&pWriter->pTqSnapWriter, rollback); if (code) goto _exit; From 6c4c0242fd4c7a9565ecf5d4dbf29a3cc96746e4 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 27 Nov 2023 11:16:09 +0800 Subject: [PATCH 28/34] refact: adjust logging format in tsdbSnapRAWReaderOpen --- source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index 7010e6ad27..ff121a3d30 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -62,14 +62,13 @@ int32_t tsdbSnapRAWReaderOpen(STsdb* tsdb, int64_t ever, int8_t type, STsdbSnapR _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), - __func__, lino, tstrerror(code), 0, ever, type); + tsdbError("vgId:%d %s failed at line %d since %s, sver:0, ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), __func__, + lino, tstrerror(code), ever, type); tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr); taosMemoryFree(reader[0]); reader[0] = NULL; } else { - tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), 0, - ever, type); + tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:0, ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), ever, type); } return code; } @@ -387,7 +386,7 @@ _exit: if (code) { tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } else { - tsdbInfo("vgId:%d %s done, sver:%" PRId64 " ever:%" PRId64, TD_VID(pTsdb->pVnode), __func__, 0, ever); + tsdbInfo("vgId:%d %s done, sver:0, ever:%" PRId64, TD_VID(pTsdb->pVnode), __func__, ever); } return code; } From ef34176e37f76202931a918cc4c1de7d95a3011f Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 30 Nov 2023 16:54:17 +0800 Subject: [PATCH 29/34] fix: close data file readers in tsbSnapRAWReaderClose --- source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index ff121a3d30..c3503a0cd6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -18,6 +18,8 @@ #include "tsdbFS2.h" #include "tsdbFSetRAW.h" +static int32_t tsdbSnapRAWReadFileSetCloseReader(STsdbSnapRAWReader* reader); + // reader typedef struct SDataFileRAWReaderIter { @@ -81,6 +83,7 @@ int32_t tsdbSnapRAWReaderClose(STsdbSnapRAWReader** reader) { STsdb* tsdb = reader[0]->tsdb; + tsdbSnapRAWReadFileSetCloseReader(reader[0]); tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr); taosMemoryFree(reader[0]); reader[0] = NULL; From 52672657c17f56c90d0e1fccbf35347baea1c16c Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 5 Dec 2023 11:36:58 +0800 Subject: [PATCH 30/34] feat: toggle tsdb snap replication mode by snap info handshake ahead of time --- source/dnode/vnode/src/inc/tsdb.h | 14 +++ source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 139 +++++++++++++++++++++ source/dnode/vnode/src/vnd/vnodeSnapshot.c | 34 ++++- 3 files changed, 183 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index dc3aa418b4..95982abfbe 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -715,6 +715,20 @@ int32_t tSerializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFS int32_t tDeserializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList); int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList *pList, TFileSetRangeArray **ppRanges); +// snap rep format +typedef enum ETsdbRepFmt { + TSDB_SNAP_REP_FMT_DEFAULT = 0, + TSDB_SNAP_REP_FMT_RAW, + TSDB_SNAP_REP_FMT_HYBRID, +} ETsdbRepFmt; + +typedef struct STsdbRepOpts { + ETsdbRepFmt format; +} STsdbRepOpts; + +int32_t tSerializeTsdbRepOpts(void *buf, int32_t bufLen, STsdbRepOpts *pInfo); +int32_t tDeserializeTsdbRepOpts(void *buf, int32_t bufLen, STsdbRepOpts *pInfo); + // snap read struct STsdbReadSnap { SMemTable *pMem; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index 65ee1a7db3..9dae9bdd36 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -443,6 +443,126 @@ static int32_t tsdbPartitionInfoSerialize(STsdbPartitionInfo* pInfo, uint8_t* bu return offset; } +// tsdb replication opts +static int32_t tTsdbRepOptsDataLenCalc(STsdbRepOpts* pInfo) { + int32_t hdrLen = sizeof(int32_t); + int32_t datLen = 0; + + int8_t msgVer = 0; + int64_t reserved64 = 0; + int16_t format = 0; + hdrLen += sizeof(msgVer); + datLen += hdrLen; + datLen += sizeof(format); + datLen += sizeof(reserved64); + datLen += sizeof(*pInfo); + return datLen; +} + +int32_t tSerializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + int64_t reserved64 = 0; + int8_t msgVer = TSDB_SNAP_MSG_VER; + + if (tStartEncode(&encoder) < 0) goto _err; + if (tEncodeI8(&encoder, msgVer) < 0) goto _err; + int16_t format = pOpts->format; + if (tEncodeI16(&encoder, format) < 0) goto _err; + if (tEncodeI64(&encoder, reserved64) < 0) goto _err; + + tEndEncode(&encoder); + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; + +_err: + tEncoderClear(&encoder); + return -1; +} + +int32_t tDeserializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + int64_t reserved64 = 0; + int8_t msgVer = 0; + + if (tStartDecode(&decoder) < 0) goto _err; + if (tDecodeI8(&decoder, &msgVer) < 0) goto _err; + if (msgVer != TSDB_SNAP_MSG_VER) goto _err; + int16_t format = 0; + if (tDecodeI16(&decoder, &format) < 0) goto _err; + pOpts->format = format; + if (tDecodeI64(&decoder, &reserved64) < 0) goto _err; + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; + +_err: + tDecoderClear(&decoder); + return -1; +} + +static int32_t tsdbRepOptsEstSize(STsdbRepOpts* pOpts) { + int32_t dataLen = 0; + dataLen += sizeof(SSyncTLV); + dataLen += tTsdbRepOptsDataLenCalc(pOpts); + return dataLen; +} + +static int32_t tsdbRepOptsSerialize(STsdbRepOpts* pOpts, void* buf, int32_t bufLen) { + SSyncTLV* pSubHead = buf; + int32_t offset = 0; + int32_t tlen = 0; + if ((tlen = tSerializeTsdbRepOpts(pSubHead->val, bufLen, pOpts)) < 0) { + return -1; + } + pSubHead->typ = SNAP_DATA_RAW; + pSubHead->len = tlen; + offset += sizeof(*pSubHead) + tlen; + return offset; +} + +// snap info +static int32_t tsdbSnapPrepDealWithSnapInfo(SVnode* pVnode, SSnapshot* pSnap, STsdbRepOpts* pInfo) { + if (!pSnap->data) return 0; + int32_t code = -1; + + SSyncTLV* pHead = (void*)pSnap->data; + int32_t offset = 0; + + while (offset + sizeof(*pHead) < pHead->len) { + SSyncTLV* pField = (void*)(pHead->val + offset); + offset += sizeof(*pField) + pField->len; + void* buf = pField->val; + int32_t bufLen = pField->len; + + switch (pField->typ) { + case SNAP_DATA_TSDB: + case SNAP_DATA_RSMA1: + case SNAP_DATA_RSMA2: { + } break; + case SNAP_DATA_RAW: { + if (tDeserializeTsdbRepOpts(buf, bufLen, pInfo) < 0) { + terrno = TSDB_CODE_INVALID_DATA_FMT; + tsdbError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + } break; + default: + tsdbError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), pField->typ); + goto _out; + } + } + + code = 0; +_out: + return code; +} + int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { ASSERT(pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY); STsdbPartitionInfo partitionInfo = {0}; @@ -453,10 +573,22 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { goto _out; } + // deal with snap info for reply + STsdbRepOpts opts = {.format = TSDB_SNAP_REP_FMT_RAW}; + if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { + STsdbRepOpts leaderOpts = {0}; + if (tsdbSnapPrepDealWithSnapInfo(pVnode, pSnap, &leaderOpts) < 0) { + tsdbError("vgId:%d, failed to deal with snap info for reply since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + opts.format = TMIN(opts.format, leaderOpts.format); + } + // info data realloc const int32_t headLen = sizeof(SSyncTLV); int32_t bufLen = headLen; bufLen += tsdbPartitionInfoEstSize(pInfo); + bufLen += tsdbRepOptsEstSize(&opts); if (syncSnapInfoDataRealloc(pSnap, bufLen) != 0) { tsdbError("vgId:%d, failed to realloc memory for data of snap info. bytes:%d", TD_VID(pVnode), bufLen); goto _out; @@ -474,6 +606,13 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { offset += tlen; ASSERT(offset <= bufLen); + if ((tlen = tsdbRepOptsSerialize(&opts, buf + offset, bufLen - offset)) < 0) { + tsdbError("vgId:%d, failed to serialize tsdb rep opts since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + offset += tlen; + ASSERT(offset <= bufLen); + // set header of info data SSyncTLV* pHead = pSnap->data; pHead->typ = pSnap->type; diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 2a8484bcd2..438fa35713 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -92,14 +92,16 @@ static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotP int32_t code = -1; if (pParam->data) { + // decode SSyncTLV *datHead = (void *)pParam->data; if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { terrno = TSDB_CODE_INVALID_DATA_FMT; goto _out; } + STsdbRepOpts tsdbOpts = {0}; TFileSetRangeArray **ppRanges = NULL; - int32_t offset = 0; + int32_t offset = 0; while (offset + sizeof(SSyncTLV) < datHead->len) { SSyncTLV *subField = (void *)(datHead->val + offset); @@ -121,13 +123,30 @@ static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotP goto _out; } } break; + case SNAP_DATA_RAW: { + if (tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts) < 0) { + vError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + } break; default: vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ); goto _out; } } - } + // toggle snap replication mode + vInfo("vgId:%d, vnode snap reader supported tsdb rep of format:%d", TD_VID(pVnode), tsdbOpts.format); + if (pReader->sver == 0 && tsdbOpts.format == TSDB_SNAP_REP_FMT_RAW) { + pReader->tsdbDone = true; + } else { + pReader->tsdbRAWDone = true; + } + + ASSERT(pReader->tsdbDone != pReader->tsdbRAWDone); + vInfo("vgId:%d, vnode snap writer enabled replication mode: %s", TD_VID(pVnode), + (pReader->tsdbDone ? "raw" : "normal")); + } code = 0; _out: return code; @@ -277,8 +296,6 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) } // TSDB ============== - pReader->tsdbDone = true; - if (!pReader->tsdbDone) { // open if not if (pReader->pTsdbReader == NULL) { @@ -534,6 +551,7 @@ static int32_t vnodeSnapWriterDealWithSnapInfo(SVSnapWriter *pWriter, SSnapshotP goto _out; } + STsdbRepOpts tsdbOpts = {0}; TFileSetRangeArray **ppRanges = NULL; int32_t offset = 0; @@ -557,11 +575,19 @@ static int32_t vnodeSnapWriterDealWithSnapInfo(SVSnapWriter *pWriter, SSnapshotP goto _out; } } break; + case SNAP_DATA_RAW: { + if (tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts) < 0) { + vError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + } break; default: vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ); goto _out; } } + + vInfo("vgId:%d, vnode snap writer supported tsdb rep of format:%d", TD_VID(pVnode), tsdbOpts.format); } code = 0; From 28e1d836628a71de0e23f731f03e7f649d079bce Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 6 Dec 2023 16:38:09 +0800 Subject: [PATCH 31/34] fix: adjust tsdbReadFile call in tsdbDataFileRAWReadBlockData --- source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c index 4a72fcfda8..3f448379c9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c @@ -72,7 +72,7 @@ int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlo pBlock->file.maxVer = reader->config->file.maxVer; pBlock->file.stt->level = reader->config->file.stt->level; - code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength); + code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0); TSDB_CHECK_CODE(code, lino, _exit); _exit: From 5cdf2b0b041755c3316512ee40be667c9a734634 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 6 Dec 2023 16:40:04 +0800 Subject: [PATCH 32/34] fixup: remove call to obsolete funcs of BgTask in tsdbSnapRAWWriter open and close --- source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index c3503a0cd6..6b61fcc324 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -363,9 +363,6 @@ int32_t tsdbSnapRAWWriterOpen(STsdb* pTsdb, int64_t ever, STsdbSnapRAWWriter** w int32_t code = 0; int32_t lino = 0; - // disable background tasks - tsdbFSDisableBgTask(pTsdb->pFS); - // start to write writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; @@ -528,7 +525,6 @@ int32_t tsdbSnapRAWWriterClose(STsdbSnapRAWWriter** writer, int8_t rollback) { taosThreadMutexUnlock(&writer[0]->tsdb->mutex); } - tsdbFSEnableBgTask(tsdb->pFS); TARRAY2_DESTROY(writer[0]->fopArr, NULL); tsdbFSDestroyCopySnapshot(&writer[0]->fsetArr); From ebd3b697451e4eaa40e70e8084438ffaedbcfa29 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 6 Dec 2023 17:51:40 +0800 Subject: [PATCH 33/34] enh: control playload size of tsdb snap replication with TSDB_SNAP_DATA_PAYLOAD_SIZE --- include/util/tdef.h | 1 + source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h | 12 ------------ source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 2 +- source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 3 +-- 4 files changed, 3 insertions(+), 15 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 1a440c7268..51b0b63da2 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -288,6 +288,7 @@ typedef enum ELogicConditionType { #define TSDB_CONN_ACTIVE_KEY_LEN 255 #define TSDB_DEFAULT_PKT_SIZE 65480 // same as RPC_MAX_UDP_SIZE +#define TSDB_SNAP_DATA_PAYLOAD_SIZE (1 * 1024 * 1024) #define TSDB_PAYLOAD_SIZE TSDB_DEFAULT_PKT_SIZE #define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h index 49f80b0be5..d765671698 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h @@ -26,18 +26,6 @@ extern "C" { #endif -#define TSDB_SNAP_RAW_PAYLOAD_SIZE (4096 * 1024) -#if 0 -struct SDataRAWBlock { - int8_t *data; - int64_t size; -}; - -int32_t tsdbDataRAWBlockReset(SDataRAWBlock *pBlock); -int32_t tsdbDataRAWBlockAlloc(SDataRawBlock *pBlock); -void tsdbDataRAWBlockFree(SDataRAWBlock *pBlock); -#endif - // STsdbDataRAWBlockHeader ======================================= typedef struct STsdbDataRAWBlockHeader { struct { diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 8f5394a9bc..6aff1c2930 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -331,7 +331,7 @@ static int32_t tsdbSnapReadTimeSeriesData(STsdbSnapReader* reader, uint8_t** dat if (!(reader->blockData->nRow % 16)) { int64_t nData = tBlockDataSize(reader->blockData); - if (nData >= 1 * 1024 * 1024) { + if (nData >= TSDB_SNAP_DATA_PAYLOAD_SIZE) { break; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index 6b61fcc324..462afcbec3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -21,7 +21,6 @@ static int32_t tsdbSnapRAWReadFileSetCloseReader(STsdbSnapRAWReader* reader); // reader - typedef struct SDataFileRAWReaderIter { int32_t count; int32_t idx; @@ -180,7 +179,7 @@ static int32_t tsdbSnapRAWReadFileSetCloseIter(STsdbSnapRAWReader* reader) { } static int64_t tsdbSnapRAWReadPeek(SDataFileRAWReader* reader) { - int64_t size = TMIN(reader->config->file.size - reader->ctx->offset, TSDB_SNAP_RAW_PAYLOAD_SIZE); + int64_t size = TMIN(reader->config->file.size - reader->ctx->offset, TSDB_SNAP_DATA_PAYLOAD_SIZE); return size; } From 1255b46469b274682c5dc216d4a40b65897521e6 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 7 Dec 2023 14:49:31 +0800 Subject: [PATCH 34/34] fix: destroy dataReaderArr properly in tsdbSnapRAWReaderClose --- source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 2 +- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 4 ++++ source/libs/sync/src/syncSnapshot.c | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index 462afcbec3..b7c22aa0e9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -82,7 +82,7 @@ int32_t tsdbSnapRAWReaderClose(STsdbSnapRAWReader** reader) { STsdb* tsdb = reader[0]->tsdb; - tsdbSnapRAWReadFileSetCloseReader(reader[0]); + TARRAY2_DESTROY(reader[0]->dataReaderArr, tsdbDataFileRAWReaderClose); tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr); taosMemoryFree(reader[0]); reader[0] = NULL; diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 438fa35713..ed1dcc64c9 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -203,6 +203,10 @@ void vnodeSnapReaderClose(SVSnapReader *pReader) { tsdbSnapReaderClose(&pReader->pTsdbReader); } + if (pReader->pTsdbRAWReader) { + tsdbSnapRAWReaderClose(&pReader->pTsdbRAWReader); + } + if (pReader->pMetaReader) { metaSnapReaderClose(&pReader->pMetaReader); } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 1e3614857e..93e81fd8e2 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -1108,7 +1108,7 @@ static int32_t syncSnapBufferSend(SSyncSnapshotSender *pSender, SyncSnapshotRsp goto _out; } - if (pSender->pReader == NULL || pSender->finish) { + if (pSender->pReader == NULL || pSender->finish || !snapshotSenderIsStart(pSender)) { code = terrno = TSDB_CODE_SYN_INTERNAL_ERROR; goto _out; }