xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

pagedbuf

This commit is contained in:
Shengliang Guan 2022-02-28 14:15:44 +08:00
parent 6834200db3
commit ca7e62d9da
2 changed files with 130 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
include/util/tpagedbuf.h
View File

@ -13,18 +13,17 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TPAGEDBUF_H
#define TDENGINE_TPAGEDBUF_H
#ifndef _TD_UTIL_PAGEDBUF_H_
#define _TD_UTIL_PAGEDBUF_H_
#include "thash.h"
#include "tlist.h"
#include "tlockfree.h"
#ifdef __cplusplus
extern "C" {
#endif
#include "tlist.h"
#include "thash.h"
#include "os.h"
#include "tlockfree.h"
typedef struct SArray* SIDList;
typedef struct SPageInfo SPageInfo;
typedef struct SDiskbasedBuf SDiskbasedBuf;
@ -55,7 +54,8 @@ typedef struct SDiskbasedBufStatis {
* @param handle
* @return
*/
int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir);
int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId,
const char* dir);
/**
*
@ -168,4 +168,4 @@ SDiskbasedBufStatis getDBufStatis(const SDiskbasedBuf* pBuf);
}
#endif
#endif // TDENGINE_TPAGEDBUF_H
#endif // _TD_UTIL_PAGEDBUF_H_

59
source/util/src/tpagedbuf.c
View File

@ -1,8 +1,24 @@
#include "tlog.h"
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "tpagedbuf.h"
#include "taoserror.h"
#include "tcompression.h"
#include "thash.h"
#include "tlog.h"
#define GET_DATA_PAYLOAD(_p) ((char*)(_p)->pData + POINTER_BYTES)
#define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages)
@ -52,7 +68,8 @@ typedef struct SDiskbasedBuf {
static void printStatisData(const SDiskbasedBuf* pBuf);
int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir) {
int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId,
const char* dir) {
*pBuf = calloc(1, sizeof(SDiskbasedBuf));
SDiskbasedBuf* pResBuf = *pBuf;
@ -86,7 +103,8 @@ static void printStatisData(const SDiskbasedBuf* pBuf);
pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t));
// qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId, pResBuf->pageSize,
// qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId,
// pResBuf->pageSize,
// pResBuf->inMemPages, pResBuf->path);
return TSDB_CODE_SUCCESS;
@ -121,7 +139,8 @@ static char* doDecompressData(void* data, int32_t srcSize, int32_t *dst, SDiskba
return data;
}
*dst = tsDecompressString(data, srcSize, 1, pBuf->assistBuf, pBuf->pageSize+sizeof(SFilePage), ONE_STAGE_COMP, NULL, 0);
*dst = tsDecompressString(data, srcSize, 1, pBuf->assistBuf, pBuf->pageSize + sizeof(SFilePage), ONE_STAGE_COMP, NULL,
0);
if (*dst > 0) {
memcpy(data, pBuf->assistBuf, *dst);
}
@ -496,7 +515,9 @@ void destroyResultBuf(SDiskbasedBuf* pBuf) {
printStatisData(pBuf);
if (pBuf->pFile != NULL) {
uDebug("Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f Kb, %"PRIx64"\n",
uDebug(
"Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page "
"size:%.2f Kb, %" PRIx64 "\n",
pBuf->totalBufSize / 1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0,
listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->qId);
@ -508,9 +529,11 @@ void destroyResultBuf(SDiskbasedBuf* pBuf) {
// print the statistics information
{
SDiskbasedBufStatis* ps = &pBuf->statis;
uDebug("Get/Release pages:%d/%d, flushToDisk:%.2f Kb (%d Pages), loadFromDisk:%.2f Kb (%d Pages), avgPageSize:%.2f Kb\n"
, ps->getPages, ps->releasePages, ps->flushBytes/1024.0f, ps->flushPages, ps->loadBytes/1024.0f, ps->loadPages
, ps->loadBytes/(1024.0 * ps->loadPages));
uDebug(
"Get/Release pages:%d/%d, flushToDisk:%.2f Kb (%d Pages), loadFromDisk:%.2f Kb (%d Pages), avgPageSize:%.2f "
"Kb\n",
ps->getPages, ps->releasePages, ps->flushBytes / 1024.0f, ps->flushPages, ps->loadBytes / 1024.0f,
ps->loadPages, ps->loadBytes / (1024.0 * ps->loadPages));
}
remove(pBuf->path);
@ -549,17 +572,11 @@ int32_t getPageId(const SPageInfo* pPgInfo) {
return pPgInfo->pageId;
}
int32_t getBufPageSize(const SDiskbasedBuf* pBuf) {
return pBuf->pageSize;
}
int32_t getBufPageSize(const SDiskbasedBuf* pBuf) { return pBuf->pageSize; }
int32_t getNumOfInMemBufPages(const SDiskbasedBuf* pBuf) {
return pBuf->inMemPages;
}
int32_t getNumOfInMemBufPages(const SDiskbasedBuf* pBuf) { return pBuf->inMemPages; }
bool isAllDataInMemBuf(const SDiskbasedBuf* pBuf) {
return pBuf->fileSize == 0;
}
bool isAllDataInMemBuf(const SDiskbasedBuf* pBuf) { return pBuf->fileSize == 0; }
void setBufPageDirty(SFilePage* pPage, bool dirty) {
int32_t offset = offsetof(SPageInfo, pData); // todo extract method
@ -569,13 +586,9 @@ void setBufPageDirty(SFilePage* pPage, bool dirty) {
ppi->dirty = dirty;
}
void printStatisBeforeClose(SDiskbasedBuf* pBuf) {
pBuf->printStatis = true;
}
void printStatisBeforeClose(SDiskbasedBuf* pBuf) { pBuf->printStatis = true; }
SDiskbasedBufStatis getDBufStatis(const SDiskbasedBuf* pBuf) {
return pBuf->statis;
}
SDiskbasedBufStatis getDBufStatis(const SDiskbasedBuf* pBuf) { return pBuf->statis; }
void printStatisData(const SDiskbasedBuf* pBuf) {
if (!pBuf->printStatis) {