xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

tsdb2: adapt cache with tsdb2 API

This commit is contained in:
Minglei Jin 2023-07-17 08:50:54 +08:00
parent 145c43e346
commit 2b21c5e0f6
6 changed files with 412 additions and 677 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
source/dnode/vnode/src/inc/tsdb.h
View File

@ -368,20 +368,20 @@ typedef struct {
} SCacheFlushState; } SCacheFlushState;
struct STsdb { struct STsdb {
char *path; char *path;
SVnode *pVnode; SVnode *pVnode;
STsdbKeepCfg keepCfg; STsdbKeepCfg keepCfg;
TdThreadRwlock rwLock; TdThreadRwlock rwLock;
SMemTable *mem; SMemTable *mem;
SMemTable *imem; SMemTable *imem;
STsdbFS fs; // old STsdbFS fs; // old
SLRUCache *lruCache; SLRUCache *lruCache;
SCacheFlushState flushState; SCacheFlushState flushState;
TdThreadMutex lruMutex; TdThreadMutex lruMutex;
SLRUCache *biCache; SLRUCache *biCache;
TdThreadMutex biMutex; TdThreadMutex biMutex;
struct STFileSystem *pFS; // new struct STFileSystem *pFS; // new
SRocksCache rCache; SRocksCache rCache;
}; };
struct TSDBKEY { struct TSDBKEY {
@ -659,12 +659,19 @@ struct SDelFWriter {
uint8_t *aBuf[1]; uint8_t *aBuf[1];
}; };
#include "tarray2.h"
//#include "tsdbFS2.h"
// struct STFileSet;
typedef struct STFileSet STFileSet;
typedef TARRAY2(STFileSet *) TFileSetArray;
struct STsdbReadSnap { struct STsdbReadSnap {
SMemTable *pMem; SMemTable *pMem;
SQueryNode *pNode; SQueryNode *pNode;
SMemTable *pIMem; SMemTable *pIMem;
SQueryNode *pINode; SQueryNode *pINode;
STsdbFS fs; TFileSetArray *pfSetArray;
STsdbFS fs;
}; };
struct SDataFWriter { struct SDataFWriter {
@ -777,7 +784,7 @@ struct SDiskDataBuilder {
typedef struct SLDataIter { typedef struct SLDataIter {
SRBTreeNode node; SRBTreeNode node;
SSttBlk *pSttBlk; SSttBlk *pSttBlk;
int32_t iStt; // for debug purpose int32_t iStt; // for debug purpose
int8_t backward; int8_t backward;
int32_t iSttBlk; int32_t iSttBlk;
int32_t iRow; int32_t iRow;
@ -796,22 +803,22 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead
bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter *pLDataIter); bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter *pLDataIter);
typedef struct { typedef struct {
int8_t backward; int8_t backward;
STsdb *pTsdb; STsdb *pTsdb;
uint64_t suid; uint64_t suid;
uint64_t uid; uint64_t uid;
STimeWindow timewindow; STimeWindow timewindow;
SVersionRange verRange; SVersionRange verRange;
bool strictTimeRange; bool strictTimeRange;
SArray *pSttFileBlockIterArray; SArray *pSttFileBlockIterArray;
void *pCurrentFileset; void *pCurrentFileset;
STSchema *pSchema; STSchema *pSchema;
int16_t *pCols; int16_t *pCols;
int32_t numOfCols; int32_t numOfCols;
void *pReader; void *pReader;
void *idstr; void *idstr;
} SMergeTreeConf; } SMergeTreeConf;
int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf* pConf); int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf);
void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter);
bool tMergeTreeNext(SMergeTree *pMTree); bool tMergeTreeNext(SMergeTree *pMTree);
@ -827,29 +834,28 @@ void *destroySttBlockReader(SArray *pLDataIterArray, int64_t *block
// tsdbCache ============================================================================================== // tsdbCache ==============================================================================================
typedef struct SCacheRowsReader { typedef struct SCacheRowsReader {
STsdb *pTsdb; STsdb *pTsdb;
SVersionRange verRange; SVersionRange verRange;
TdThreadMutex readerMutex; TdThreadMutex readerMutex;
SVnode *pVnode; SVnode *pVnode;
STSchema *pSchema; STSchema *pSchema;
STSchema *pCurrSchema; STSchema *pCurrSchema;
uint64_t uid; uint64_t uid;
uint64_t suid; uint64_t suid;
char **transferBuf; // todo remove it soon char **transferBuf; // todo remove it soon
int32_t numOfCols; int32_t numOfCols;
SArray *pCidList; SArray *pCidList;
int32_t *pSlotIds; int32_t *pSlotIds;
int32_t type; int32_t type;
int32_t tableIndex; // currently returned result tables int32_t tableIndex; // currently returned result tables
STableKeyInfo *pTableList; // table id list STableKeyInfo *pTableList; // table id list
int32_t numOfTables; int32_t numOfTables;
SSttBlockLoadInfo *pLoadInfo; SArray *pLDataIterArray;
SLDataIter *pDataIter; STsdbReadSnap *pReadSnap;
STsdbReadSnap *pReadSnap; SDataFReader *pDataFReader;
SDataFReader *pDataFReader; SDataFReader *pDataFReaderLast;
SDataFReader *pDataFReaderLast; char *idstr;
const char *idstr; int64_t lastTs;
int64_t lastTs;
} SCacheRowsReader; } SCacheRowsReader;
typedef struct { typedef struct {

916
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

File diff suppressed because it is too large Load Diff

30
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -124,7 +124,10 @@ int32_t tsdbReuseCacherowsReader(void* reader, void* pTableIdList, int32_t numOf
pReader->numOfTables = numOfTables; pReader->numOfTables = numOfTables;
pReader->lastTs = INT64_MIN; pReader->lastTs = INT64_MIN;
resetLastBlockLoadInfo(pReader->pLoadInfo); int64_t blocks;
double elapse;
pReader->pLDataIterArray = destroySttBlockReader(pReader->pLDataIterArray, &blocks, &elapse);
pReader->pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -178,14 +181,8 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList,
SVnodeCfg* pCfg = &((SVnode*)pVnode)->config; SVnodeCfg* pCfg = &((SVnode*)pVnode)->config;
int32_t numOfStt = pCfg->sttTrigger; int32_t numOfStt = pCfg->sttTrigger;
p->pLoadInfo = tCreateLastBlockLoadInfo(p->pSchema, NULL, 0, numOfStt); p->pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
if (p->pLoadInfo == NULL) { if (p->pLDataIterArray == NULL) {
tsdbCacherowsReaderClose(p);
return TSDB_CODE_OUT_OF_MEMORY;
}
p->pDataIter = taosMemoryCalloc(pCfg->sttTrigger, sizeof(SLDataIter));
if (p->pDataIter == NULL) {
tsdbCacherowsReaderClose(p); tsdbCacherowsReaderClose(p);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
@ -214,10 +211,11 @@ void* tsdbCacherowsReaderClose(void* pReader) {
taosMemoryFree(p->pSchema); taosMemoryFree(p->pSchema);
} }
taosMemoryFree(p->pDataIter);
taosMemoryFree(p->pCurrSchema); taosMemoryFree(p->pCurrSchema);
destroyLastBlockLoadInfo(p->pLoadInfo); int64_t loadBlocks = 0;
double elapse = 0;
destroySttBlockReader(p->pLDataIterArray, &loadBlocks, &elapse);
taosMemoryFree((void*)p->idstr); taosMemoryFree((void*)p->idstr);
taosThreadMutexDestroy(&p->readerMutex); taosThreadMutexDestroy(&p->readerMutex);
@ -298,7 +296,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
} }
taosThreadMutexLock(&pr->readerMutex); taosThreadMutexLock(&pr->readerMutex);
code = tsdbTakeReadSnap((STsdbReader*)pr, tsdbCacheQueryReseek, &pr->pReadSnap); code = tsdbTakeReadSnap2((STsdbReader*)pr, tsdbCacheQueryReseek, &pr->pReadSnap);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _end; goto _end;
} }
@ -427,8 +425,12 @@ _end:
tsdbDataFReaderClose(&pr->pDataFReaderLast); tsdbDataFReaderClose(&pr->pDataFReaderLast);
tsdbDataFReaderClose(&pr->pDataFReader); tsdbDataFReaderClose(&pr->pDataFReader);
resetLastBlockLoadInfo(pr->pLoadInfo); int64_t loadBlocks = 0;
tsdbUntakeReadSnap((STsdbReader*)pr, pr->pReadSnap, true); double elapse = 0;
pr->pLDataIterArray = destroySttBlockReader(pr->pLDataIterArray, &loadBlocks, &elapse);
pr->pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
tsdbUntakeReadSnap2((STsdbReader*)pr, pr->pReadSnap, true);
taosThreadMutexUnlock(&pr->readerMutex); taosThreadMutexUnlock(&pr->readerMutex);
if (pRes != NULL) { if (pRes != NULL) {

2
source/dnode/vnode/src/tsdb/tsdbFS2.h
View File

@ -25,7 +25,7 @@ extern "C" {
/* Exposed Handle */ /* Exposed Handle */
typedef struct STFileSystem STFileSystem; typedef struct STFileSystem STFileSystem;
typedef struct STFSBgTask STFSBgTask; typedef struct STFSBgTask STFSBgTask;
typedef TARRAY2(STFileSet *) TFileSetArray; // typedef TARRAY2(STFileSet *) TFileSetArray;
typedef enum { typedef enum {
TSDB_FEDIT_COMMIT = 1, // TSDB_FEDIT_COMMIT = 1, //

16
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -18,9 +18,9 @@
#include "tsdbDataFileRW.h" #include "tsdbDataFileRW.h"
#include "tsdbFS2.h" #include "tsdbFS2.h"
#include "tsdbMerge.h" #include "tsdbMerge.h"
#include "tsdbReadUtil.h"
#include "tsdbUtil2.h" #include "tsdbUtil2.h"
#include "tsimplehash.h" #include "tsimplehash.h"
#include "tsdbReadUtil.h"
#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC)
#define getCurrentKeyInLastBlock(_r) ((_r)->currentKey) #define getCurrentKeyInLastBlock(_r) ((_r)->currentKey)
@ -100,7 +100,7 @@ static int32_t updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInf
i += 1; i += 1;
j += 1; j += 1;
} else if (pTCol->colId < pSupInfo->colId[j]) { // do nothing } else if (pTCol->colId < pSupInfo->colId[j]) { // do nothing
i += 1; i += 1;
} else { } else {
return TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
@ -3815,7 +3815,7 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) {
SReaderStatus* pStatus = &pReader->status; SReaderStatus* pStatus = &pReader->status;
SDataBlockIter* pBlockIter = &pStatus->blockIter; SDataBlockIter* pBlockIter = &pStatus->blockIter;
initFilesetIterator(&pStatus->fileIter, pReader->status.pfSetArray, pReader); initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader);
resetDataBlockIterator(&pStatus->blockIter, pReader->info.order); resetDataBlockIterator(&pStatus->blockIter, pReader->info.order);
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -3842,7 +3842,6 @@ static void freeSchemaFunc(void* param) {
static void clearSharedPtr(STsdbReader* p) { static void clearSharedPtr(STsdbReader* p) {
p->status.pTableMap = NULL; p->status.pTableMap = NULL;
p->status.uidList.tableUidList = NULL; p->status.uidList.tableUidList = NULL;
p->status.pfSetArray = NULL;
p->info.pSchema = NULL; p->info.pSchema = NULL;
p->pReadSnap = NULL; p->pReadSnap = NULL;
p->pSchemaMap = NULL; p->pSchemaMap = NULL;
@ -3851,7 +3850,8 @@ static void clearSharedPtr(STsdbReader* p) {
static void setSharedPtr(STsdbReader* pDst, const STsdbReader* pSrc) { static void setSharedPtr(STsdbReader* pDst, const STsdbReader* pSrc) {
pDst->status.pTableMap = pSrc->status.pTableMap; pDst->status.pTableMap = pSrc->status.pTableMap;
pDst->status.uidList = pSrc->status.uidList; pDst->status.uidList = pSrc->status.uidList;
pDst->status.pfSetArray = pSrc->status.pfSetArray; // pDst->status.pfSetArray = pSrc->status.pfSetArray;
pDst->pReadSnap->pfSetArray = pSrc->pReadSnap->pfSetArray;
pDst->info.pSchema = pSrc->info.pSchema; pDst->info.pSchema = pSrc->info.pSchema;
pDst->pSchemaMap = pSrc->pSchemaMap; pDst->pSchemaMap = pSrc->pSchemaMap;
pDst->pReadSnap = pSrc->pReadSnap; pDst->pReadSnap = pSrc->pReadSnap;
@ -4633,7 +4633,7 @@ int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) {
int32_t numOfTables = tSimpleHashGetSize(pStatus->pTableMap); int32_t numOfTables = tSimpleHashGetSize(pStatus->pTableMap);
initFilesetIterator(&pStatus->fileIter, pReader->status.pfSetArray, pReader); initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader);
resetDataBlockIterator(pBlockIter, pReader->info.order); resetDataBlockIterator(pBlockIter, pReader->info.order);
resetTableListIndex(&pReader->status); resetTableListIndex(&pReader->status);
@ -4886,7 +4886,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs
} }
// fs // fs
code = tsdbFSCreateRefSnapshot(pTsdb->pFS, &pReader->status.pfSetArray); code = tsdbFSCreateRefSnapshot(pTsdb->pFS, &pSnap->pfSetArray);
if (code) { if (code) {
taosThreadRwlockUnlock(&pTsdb->rwLock); taosThreadRwlockUnlock(&pTsdb->rwLock);
goto _exit; goto _exit;
@ -4929,7 +4929,7 @@ void tsdbUntakeReadSnap2(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proact
if (pSnap->pINode) taosMemoryFree(pSnap->pINode); if (pSnap->pINode) taosMemoryFree(pSnap->pINode);
taosMemoryFree(pSnap); taosMemoryFree(pSnap);
tsdbFSDestroyRefSnapshot(&pReader->status.pfSetArray); tsdbFSDestroyRefSnapshot(&pSnap->pfSetArray);
} }
tsdbTrace("vgId:%d, untake read snapshot", TD_VID(pTsdb->pVnode)); tsdbTrace("vgId:%d, untake read snapshot", TD_VID(pTsdb->pVnode));
} }

3
source/dnode/vnode/src/tsdb/tsdbReadUtil.h
View File

@ -20,8 +20,8 @@
extern "C" { extern "C" {
#endif #endif
#include "tsdbUtil2.h"
#include "tsdbDataFileRW.h" #include "tsdbDataFileRW.h"
#include "tsdbUtil2.h"
#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC)
@ -196,7 +196,6 @@ typedef struct SReaderStatus {
SArray* pLDataIterArray; SArray* pLDataIterArray;
SRowMerger merger; SRowMerger merger;
SColumnInfoData* pPrimaryTsCol; // primary time stamp output col info data SColumnInfoData* pPrimaryTsCol; // primary time stamp output col info data
TFileSetArray* pfSetArray;
} SReaderStatus; } SReaderStatus;
struct STsdbReader { struct STsdbReader {