xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

pagedbuf

This commit is contained in:
Shengliang Guan 2022-02-28 14:15:44 +08:00
parent 6834200db3
commit ca7e62d9da
2 changed files with 130 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
include/util/tpagedbuf.h
View File

@ -13,18 +13,17 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#ifndef TDENGINE_TPAGEDBUF_H #ifndef _TD_UTIL_PAGEDBUF_H_
#define TDENGINE_TPAGEDBUF_H #define _TD_UTIL_PAGEDBUF_H_
#include "thash.h"
#include "tlist.h"
#include "tlockfree.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#include "tlist.h"
#include "thash.h"
#include "os.h"
#include "tlockfree.h"
typedef struct SArray* SIDList; typedef struct SArray* SIDList;
typedef struct SPageInfo SPageInfo; typedef struct SPageInfo SPageInfo;
typedef struct SDiskbasedBuf SDiskbasedBuf; typedef struct SDiskbasedBuf SDiskbasedBuf;
@ -55,7 +54,8 @@ typedef struct SDiskbasedBufStatis {
* @param handle * @param handle
* @return * @return
*/ */
int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir); int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId,
const char* dir);
/** /**
* *
@ -168,4 +168,4 @@ SDiskbasedBufStatis getDBufStatis(const SDiskbasedBuf* pBuf);
} }
#endif #endif
#endif // TDENGINE_TPAGEDBUF_H #endif // _TD_UTIL_PAGEDBUF_H_

59
source/util/src/tpagedbuf.c
View File

@ -1,8 +1,24 @@
#include "tlog.h" /*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "tpagedbuf.h" #include "tpagedbuf.h"
#include "taoserror.h" #include "taoserror.h"
#include "tcompression.h" #include "tcompression.h"
#include "thash.h" #include "thash.h"
#include "tlog.h"
#define GET_DATA_PAYLOAD(_p) ((char*)(_p)->pData + POINTER_BYTES) #define GET_DATA_PAYLOAD(_p) ((char*)(_p)->pData + POINTER_BYTES)
#define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages) #define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages)
@ -52,7 +68,8 @@ typedef struct SDiskbasedBuf {
static void printStatisData(const SDiskbasedBuf* pBuf); static void printStatisData(const SDiskbasedBuf* pBuf);
int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir) { int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId,
const char* dir) {
*pBuf = calloc(1, sizeof(SDiskbasedBuf)); *pBuf = calloc(1, sizeof(SDiskbasedBuf));
SDiskbasedBuf* pResBuf = *pBuf; SDiskbasedBuf* pResBuf = *pBuf;
@ -86,7 +103,8 @@ static void printStatisData(const SDiskbasedBuf* pBuf);
pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t)); pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t));
// qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId, pResBuf->pageSize, // qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId,
// pResBuf->pageSize,
// pResBuf->inMemPages, pResBuf->path); // pResBuf->inMemPages, pResBuf->path);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -121,7 +139,8 @@ static char* doDecompressData(void* data, int32_t srcSize, int32_t *dst, SDiskba
return data; return data;
} }
*dst = tsDecompressString(data, srcSize, 1, pBuf->assistBuf, pBuf->pageSize+sizeof(SFilePage), ONE_STAGE_COMP, NULL, 0); *dst = tsDecompressString(data, srcSize, 1, pBuf->assistBuf, pBuf->pageSize + sizeof(SFilePage), ONE_STAGE_COMP, NULL,
0);
if (*dst > 0) { if (*dst > 0) {
memcpy(data, pBuf->assistBuf, *dst); memcpy(data, pBuf->assistBuf, *dst);
} }
@ -496,7 +515,9 @@ void destroyResultBuf(SDiskbasedBuf* pBuf) {
printStatisData(pBuf); printStatisData(pBuf);
if (pBuf->pFile != NULL) { if (pBuf->pFile != NULL) {
uDebug("Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f Kb, %"PRIx64"\n", uDebug(
"Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page "
"size:%.2f Kb, %" PRIx64 "\n",
pBuf->totalBufSize / 1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0, pBuf->totalBufSize / 1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0,
listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->qId); listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->qId);
@ -508,9 +529,11 @@ void destroyResultBuf(SDiskbasedBuf* pBuf) {
// print the statistics information // print the statistics information
{ {
SDiskbasedBufStatis* ps = &pBuf->statis; SDiskbasedBufStatis* ps = &pBuf->statis;
uDebug("Get/Release pages:%d/%d, flushToDisk:%.2f Kb (%d Pages), loadFromDisk:%.2f Kb (%d Pages), avgPageSize:%.2f Kb\n" uDebug(
, ps->getPages, ps->releasePages, ps->flushBytes/1024.0f, ps->flushPages, ps->loadBytes/1024.0f, ps->loadPages "Get/Release pages:%d/%d, flushToDisk:%.2f Kb (%d Pages), loadFromDisk:%.2f Kb (%d Pages), avgPageSize:%.2f "
, ps->loadBytes/(1024.0 * ps->loadPages)); "Kb\n",
ps->getPages, ps->releasePages, ps->flushBytes / 1024.0f, ps->flushPages, ps->loadBytes / 1024.0f,
ps->loadPages, ps->loadBytes / (1024.0 * ps->loadPages));
} }
remove(pBuf->path); remove(pBuf->path);
@ -549,17 +572,11 @@ int32_t getPageId(const SPageInfo* pPgInfo) {
return pPgInfo->pageId; return pPgInfo->pageId;
} }
int32_t getBufPageSize(const SDiskbasedBuf* pBuf) { int32_t getBufPageSize(const SDiskbasedBuf* pBuf) { return pBuf->pageSize; }
return pBuf->pageSize;
}
int32_t getNumOfInMemBufPages(const SDiskbasedBuf* pBuf) { int32_t getNumOfInMemBufPages(const SDiskbasedBuf* pBuf) { return pBuf->inMemPages; }
return pBuf->inMemPages;
}
bool isAllDataInMemBuf(const SDiskbasedBuf* pBuf) { bool isAllDataInMemBuf(const SDiskbasedBuf* pBuf) { return pBuf->fileSize == 0; }
return pBuf->fileSize == 0;
}
void setBufPageDirty(SFilePage* pPage, bool dirty) { void setBufPageDirty(SFilePage* pPage, bool dirty) {
int32_t offset = offsetof(SPageInfo, pData); // todo extract method int32_t offset = offsetof(SPageInfo, pData); // todo extract method
@ -569,13 +586,9 @@ void setBufPageDirty(SFilePage* pPage, bool dirty) {
ppi->dirty = dirty; ppi->dirty = dirty;
} }
void printStatisBeforeClose(SDiskbasedBuf* pBuf) { void printStatisBeforeClose(SDiskbasedBuf* pBuf) { pBuf->printStatis = true; }
pBuf->printStatis = true;
}
SDiskbasedBufStatis getDBufStatis(const SDiskbasedBuf* pBuf) { SDiskbasedBufStatis getDBufStatis(const SDiskbasedBuf* pBuf) { return pBuf->statis; }
return pBuf->statis;
}
void printStatisData(const SDiskbasedBuf* pBuf) { void printStatisData(const SDiskbasedBuf* pBuf) {
if (!pBuf->printStatis) { if (!pBuf->printStatis) {