[td-225] update resbuf
This commit is contained in:
parent
f72183fea5
commit
9a9ea692ed
|
@ -20,42 +20,56 @@
|
||||||
extern "C" {
|
extern "C" {
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#include <tlist.h>
|
||||||
#include "hash.h"
|
#include "hash.h"
|
||||||
#include "os.h"
|
#include "os.h"
|
||||||
#include "qExtbuffer.h"
|
#include "qExtbuffer.h"
|
||||||
|
#include "tlockfree.h"
|
||||||
|
|
||||||
typedef struct SArray* SIDList;
|
typedef struct SArray* SIDList;
|
||||||
|
|
||||||
typedef struct SPageInfo {
|
typedef struct SPageDiskInfo {
|
||||||
int32_t pageId;
|
|
||||||
int32_t offset;
|
int32_t offset;
|
||||||
int32_t lengthOnDisk;
|
int32_t length;
|
||||||
|
} SPageDiskInfo;
|
||||||
|
|
||||||
|
typedef struct SPageInfo {
|
||||||
|
int32_t pageId;
|
||||||
|
SPageDiskInfo info;
|
||||||
|
void* pData;
|
||||||
|
T_REF_DECLARE();
|
||||||
} SPageInfo;
|
} SPageInfo;
|
||||||
|
|
||||||
|
typedef struct SFreeListItem {
|
||||||
|
int32_t offset;
|
||||||
|
int32_t len;
|
||||||
|
} SFreeListItem;
|
||||||
|
|
||||||
typedef struct SDiskbasedResultBuf {
|
typedef struct SDiskbasedResultBuf {
|
||||||
int32_t numOfRowsPerPage;
|
int32_t numOfRowsPerPage;
|
||||||
int32_t numOfPages;
|
int32_t numOfPages;
|
||||||
int64_t totalBufSize;
|
int64_t totalBufSize;
|
||||||
int32_t fd;
|
// int32_t fd;
|
||||||
// FILE* file;
|
FILE* file;
|
||||||
int32_t allocateId; // allocated page id
|
int32_t allocateId; // allocated page id
|
||||||
int32_t incStep; // minimum allocated pages
|
// int32_t incStep; // minimum allocated pages
|
||||||
void* pBuf; // mmap buffer pointer
|
void* pBuf; // mmap buffer pointer
|
||||||
char* path; // file path
|
char* path; // file path
|
||||||
int32_t pageSize; // current used page size
|
int32_t pageSize; // current used page size
|
||||||
int32_t inMemPages; // numOfPages that are allocated in memory
|
int32_t inMemPages; // numOfPages that are allocated in memory
|
||||||
SHashObj* idsTable; // id hash table
|
SHashObj* idsTable; // id hash table
|
||||||
SIDList list; // for each id, there is a page id list
|
SHashObj* all;
|
||||||
|
SList* pPageList;
|
||||||
void* iBuf; // inmemory buf
|
|
||||||
void* handle; // for debug purpose
|
void* handle; // for debug purpose
|
||||||
void* emptyDummyIdList; // dummy id list
|
void* emptyDummyIdList; // dummy id list
|
||||||
bool comp;
|
bool comp;
|
||||||
|
SArray* pFree; // free area in file
|
||||||
|
int32_t nextPos; // next page flush position
|
||||||
} SDiskbasedResultBuf;
|
} SDiskbasedResultBuf;
|
||||||
|
|
||||||
#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L)
|
#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L)
|
||||||
#define DEFAULT_INMEM_BUF_PAGES 10
|
#define DEFAULT_INMEM_BUF_PAGES 10
|
||||||
|
#define PAGE_INFO_INITIALIZER (SPageDiskInfo){-1, -1}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* create disk-based result buffer
|
* create disk-based result buffer
|
||||||
|
@ -65,7 +79,7 @@ typedef struct SDiskbasedResultBuf {
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t numOfPages, int32_t rowSize, int32_t pagesize,
|
int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t numOfPages, int32_t rowSize, int32_t pagesize,
|
||||||
int32_t inMemPages, const void* handle);
|
int32_t inMemPages, const void* handle);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
*
|
*
|
||||||
|
@ -97,13 +111,10 @@ SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId);
|
||||||
* @param id
|
* @param id
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
static FORCE_INLINE tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) {
|
tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id);
|
||||||
if (id < pResultBuf->inMemPages) {
|
|
||||||
return (tFilePage*) ((char*) pResultBuf->iBuf + id * pResultBuf->pageSize);
|
void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page);
|
||||||
} else {
|
|
||||||
return (tFilePage*) ((char*) pResultBuf->pBuf + (id - pResultBuf->inMemPages) * pResultBuf->pageSize);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
/**
|
/**
|
||||||
* get the total buffer size in the format of disk file
|
* get the total buffer size in the format of disk file
|
||||||
* @param pResultBuf
|
* @param pResultBuf
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
#include "qResultbuf.h"
|
#include "qResultbuf.h"
|
||||||
|
#include <stddef.h>
|
||||||
#include "hash.h"
|
#include "hash.h"
|
||||||
#include "qExtbuffer.h"
|
#include "qExtbuffer.h"
|
||||||
#include "queryLog.h"
|
#include "queryLog.h"
|
||||||
|
@ -14,29 +15,26 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t nu
|
||||||
}
|
}
|
||||||
|
|
||||||
pResBuf->pageSize = pagesize;
|
pResBuf->pageSize = pagesize;
|
||||||
pResBuf->numOfPages = inMemPages; // all pages are in buffer in the first place
|
pResBuf->numOfPages = 0; // all pages are in buffer in the first place
|
||||||
pResBuf->inMemPages = inMemPages;
|
pResBuf->inMemPages = inMemPages;
|
||||||
assert(inMemPages <= numOfPages);
|
assert(inMemPages <= numOfPages);
|
||||||
|
|
||||||
pResBuf->numOfRowsPerPage = (pagesize - sizeof(tFilePage)) / rowSize;
|
pResBuf->numOfRowsPerPage = (pagesize - sizeof(tFilePage)) / rowSize;
|
||||||
|
|
||||||
pResBuf->totalBufSize = pResBuf->numOfPages * pagesize;
|
pResBuf->totalBufSize = pResBuf->numOfPages * pagesize;
|
||||||
pResBuf->incStep = 4;
|
|
||||||
pResBuf->allocateId = -1;
|
pResBuf->allocateId = -1;
|
||||||
|
|
||||||
// todo opt perf by on demand create in memory buffer
|
pResBuf->pPageList = tdListNew(POINTER_BYTES);
|
||||||
pResBuf->iBuf = calloc(pResBuf->inMemPages, pResBuf->pageSize);
|
|
||||||
|
|
||||||
// init id hash table
|
// init id hash table
|
||||||
pResBuf->idsTable = taosHashInit(numOfPages, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
|
pResBuf->idsTable = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
|
||||||
pResBuf->list = taosArrayInit(numOfPages, POINTER_BYTES);
|
pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
|
||||||
|
|
||||||
char path[PATH_MAX] = {0};
|
char path[PATH_MAX] = {0};
|
||||||
getTmpfilePath("qbuf", path);
|
getTmpfilePath("qbuf", path);
|
||||||
pResBuf->path = strdup(path);
|
pResBuf->path = strdup(path);
|
||||||
|
|
||||||
pResBuf->fd = FD_INITIALIZER;
|
pResBuf->file = NULL;
|
||||||
pResBuf->pBuf = NULL;
|
|
||||||
pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t));
|
pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t));
|
||||||
|
|
||||||
qDebug("QInfo:%p create resBuf for output, page size:%d, initial pages:%d, %" PRId64 "bytes", handle,
|
qDebug("QInfo:%p create resBuf for output, page size:%d, initial pages:%d, %" PRId64 "bytes", handle,
|
||||||
|
@ -53,133 +51,258 @@ int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->tota
|
||||||
#define FILE_SIZE_ON_DISK(_r) (NUM_OF_PAGES_ON_DISK(_r) * (_r)->pageSize)
|
#define FILE_SIZE_ON_DISK(_r) (NUM_OF_PAGES_ON_DISK(_r) * (_r)->pageSize)
|
||||||
|
|
||||||
static int32_t createDiskResidesBuf(SDiskbasedResultBuf* pResultBuf) {
|
static int32_t createDiskResidesBuf(SDiskbasedResultBuf* pResultBuf) {
|
||||||
pResultBuf->fd = open(pResultBuf->path, O_CREAT | O_RDWR, 0666);
|
// pResultBuf->fd = open(pResultBuf->path, O_CREAT | O_RDWR, 0666);
|
||||||
// pResultBuf->file = fopen(pResultBuf->path, "w");
|
pResultBuf->file = fopen(pResultBuf->path, "w");
|
||||||
if (!FD_VALID(pResultBuf->fd)) {
|
if (pResultBuf->file == NULL) {
|
||||||
qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno));
|
qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno));
|
||||||
return TAOS_SYSTEM_ERROR(errno);
|
return TAOS_SYSTEM_ERROR(errno);
|
||||||
}
|
}
|
||||||
|
|
||||||
assert(pResultBuf->numOfPages == pResultBuf->inMemPages);
|
|
||||||
pResultBuf->numOfPages += pResultBuf->incStep;
|
|
||||||
|
|
||||||
int32_t ret = ftruncate(pResultBuf->fd, NUM_OF_PAGES_ON_DISK(pResultBuf) * pResultBuf->pageSize);
|
|
||||||
if (ret != TSDB_CODE_SUCCESS) {
|
|
||||||
qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno));
|
|
||||||
return TAOS_SYSTEM_ERROR(errno);
|
|
||||||
}
|
|
||||||
|
|
||||||
pResultBuf->pBuf = mmap(NULL, FILE_SIZE_ON_DISK(pResultBuf), PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0);
|
|
||||||
|
|
||||||
if (pResultBuf->pBuf == MAP_FAILED) {
|
|
||||||
qError("QInfo:%p failed to map temp file: %s. %s", pResultBuf->handle, pResultBuf->path, strerror(errno));
|
|
||||||
return TAOS_SYSTEM_ERROR(errno);
|
|
||||||
}
|
|
||||||
|
|
||||||
pResultBuf->totalBufSize = pResultBuf->numOfPages * pResultBuf->pageSize;
|
|
||||||
return TSDB_CODE_SUCCESS;
|
return TSDB_CODE_SUCCESS;
|
||||||
}
|
}
|
||||||
|
|
||||||
static int32_t extendDiskFileSize(SDiskbasedResultBuf* pResultBuf, int32_t incNumOfPages) {
|
static char* doCompressData(void* data, int32_t srcSize, int32_t *dst) { // do nothing
|
||||||
assert(pResultBuf->numOfPages * pResultBuf->pageSize == pResultBuf->totalBufSize);
|
*dst = srcSize;
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int32_t allocatePositionInFile(SDiskbasedResultBuf* pResultBuf, size_t size) {
|
||||||
|
if (pResultBuf->pFree == NULL) {
|
||||||
|
return pResultBuf->nextPos;
|
||||||
|
} else { //todo speed up the search procedure
|
||||||
|
size_t num = taosArrayGetSize(pResultBuf->pFree);
|
||||||
|
|
||||||
|
int32_t offset = -1;
|
||||||
|
|
||||||
|
for(int32_t i = 0; i < num; ++i) {
|
||||||
|
SFreeListItem* pi = taosArrayGet(pResultBuf->pFree, i);
|
||||||
|
if (pi->len >= size) {
|
||||||
|
offset = pi->offset;
|
||||||
|
pi->offset += size;
|
||||||
|
pi->len -= size;
|
||||||
|
|
||||||
|
return offset;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// no available recycle space, allocate new area in file
|
||||||
|
return pResultBuf->nextPos;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
|
||||||
|
assert(T_REF_VAL_GET(pg) == 0);
|
||||||
|
|
||||||
|
int32_t size = -1;
|
||||||
|
char* t = doCompressData(pg->pData + POINTER_BYTES, pResultBuf->pageSize, &size);
|
||||||
|
|
||||||
|
// this page is flushed to disk for the first time
|
||||||
|
if (pg->info.offset == -1) {
|
||||||
|
int32_t offset = allocatePositionInFile(pResultBuf, size);
|
||||||
|
pResultBuf->nextPos += size;
|
||||||
|
|
||||||
|
fseek(pResultBuf->file, offset, SEEK_SET);
|
||||||
|
fwrite(t, size, 1, pResultBuf->file);
|
||||||
|
} else {
|
||||||
|
if (pg->info.length < size) { // length becomes greater, current space is not enough, allocate new place.
|
||||||
|
//1. add current space to free list
|
||||||
|
taosArrayPush(pResultBuf->pFree, &pg->info);
|
||||||
|
|
||||||
|
//2. allocate new position, and update the info
|
||||||
|
int32_t offset = allocatePositionInFile(pResultBuf, size);
|
||||||
|
pResultBuf->nextPos += size;
|
||||||
|
|
||||||
|
//3. write to disk.
|
||||||
|
fseek(pResultBuf->file, offset, SEEK_SET);
|
||||||
|
fwrite(t, size, 1, pResultBuf->file);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static int32_t flushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
|
||||||
int32_t ret = TSDB_CODE_SUCCESS;
|
int32_t ret = TSDB_CODE_SUCCESS;
|
||||||
|
assert(pResultBuf->numOfPages * pResultBuf->pageSize == pResultBuf->totalBufSize && pResultBuf->numOfPages >= pResultBuf->inMemPages);
|
||||||
|
|
||||||
if (pResultBuf->pBuf == NULL) {
|
if (pResultBuf->pBuf == NULL) {
|
||||||
assert(!FD_VALID(pResultBuf->fd));
|
assert(pResultBuf->file == NULL);
|
||||||
|
|
||||||
if ((ret = createDiskResidesBuf(pResultBuf)) != TSDB_CODE_SUCCESS) {
|
if ((ret = createDiskResidesBuf(pResultBuf)) != TSDB_CODE_SUCCESS) {
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
ret = munmap(pResultBuf->pBuf, FILE_SIZE_ON_DISK(pResultBuf));
|
|
||||||
pResultBuf->numOfPages += incNumOfPages;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* disk-based output buffer is exhausted, try to extend the disk-based buffer, the available disk space may
|
|
||||||
* be insufficient
|
|
||||||
*/
|
|
||||||
ret = ftruncate(pResultBuf->fd, NUM_OF_PAGES_ON_DISK(pResultBuf) * pResultBuf->pageSize);
|
|
||||||
if (ret != TSDB_CODE_SUCCESS) {
|
|
||||||
// dError("QInfo:%p failed to create intermediate result output file:%s. %s", pQInfo, pSupporter->extBufFile,
|
|
||||||
// strerror(errno));
|
|
||||||
return TSDB_CODE_QRY_NO_DISKSPACE;
|
|
||||||
}
|
|
||||||
|
|
||||||
pResultBuf->totalBufSize = pResultBuf->numOfPages * pResultBuf->pageSize;
|
|
||||||
pResultBuf->pBuf = mmap(NULL, FILE_SIZE_ON_DISK(pResultBuf), PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0);
|
|
||||||
|
|
||||||
if (pResultBuf->pBuf == MAP_FAILED) {
|
|
||||||
// dError("QInfo:%p failed to map temp file: %s. %s", pQInfo, pSupporter->extBufFile, strerror(errno));
|
|
||||||
return TSDB_CODE_QRY_OUT_OF_MEMORY;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
doFlushPageToDisk(pResultBuf, pg);
|
||||||
return TSDB_CODE_SUCCESS;
|
return TSDB_CODE_SUCCESS;
|
||||||
}
|
}
|
||||||
|
|
||||||
#define NO_AVAILABLE_PAGES(_b) ((_b)->allocateId == (_b)->numOfPages - 1)
|
#define NO_AVAILABLE_PAGES(_b) ((_b)->numOfPages >= (_b)->inMemPages)
|
||||||
|
|
||||||
static FORCE_INLINE int32_t getGroupIndex(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
|
static SIDList addNewGroup(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
|
||||||
assert(pResultBuf != NULL);
|
assert(taosHashGet(pResultBuf->idsTable, (const char*) &groupId, sizeof(int32_t)) == NULL);
|
||||||
|
|
||||||
char* p = taosHashGet(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t));
|
SArray* pa = taosArrayInit(1, sizeof(SPageInfo));
|
||||||
|
int32_t ret = taosHashPut(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t), &pa, POINTER_BYTES);
|
||||||
|
assert(ret == 0);
|
||||||
|
|
||||||
|
return pa;
|
||||||
|
}
|
||||||
|
|
||||||
|
static SPageInfo* registerPage(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) {
|
||||||
|
SIDList list = NULL;
|
||||||
|
|
||||||
|
char** p = taosHashGet(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t));
|
||||||
if (p == NULL) { // it is a new group id
|
if (p == NULL) { // it is a new group id
|
||||||
return -1;
|
list = addNewGroup(pResultBuf, groupId);
|
||||||
|
} else {
|
||||||
|
list = (SIDList) (*p);
|
||||||
}
|
}
|
||||||
|
|
||||||
int32_t slot = GET_INT32_VAL(p);
|
pResultBuf->numOfPages += 1;
|
||||||
assert(slot >= 0 && slot < taosHashGetSize(pResultBuf->idsTable));
|
|
||||||
|
|
||||||
return slot;
|
SPageInfo ppi = { .info = PAGE_INFO_INITIALIZER, .pageId = pageId, };
|
||||||
}
|
return taosArrayPush(list, &ppi);
|
||||||
|
|
||||||
static int32_t addNewGroupId(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
|
|
||||||
int32_t num = getNumOfResultBufGroupId(pResultBuf); // the num is the newest allocated group id slot
|
|
||||||
taosHashPut(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t), &num, sizeof(int32_t));
|
|
||||||
|
|
||||||
SArray* pa = taosArrayInit(1, sizeof(int32_t));
|
|
||||||
taosArrayPush(pResultBuf->list, &pa);
|
|
||||||
|
|
||||||
assert(taosArrayGetSize(pResultBuf->list) == taosHashGetSize(pResultBuf->idsTable));
|
|
||||||
return num;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void registerPageId(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) {
|
|
||||||
int32_t slot = getGroupIndex(pResultBuf, groupId);
|
|
||||||
if (slot < 0) {
|
|
||||||
slot = addNewGroupId(pResultBuf, groupId);
|
|
||||||
}
|
|
||||||
|
|
||||||
SIDList pList = taosArrayGetP(pResultBuf->list, slot);
|
|
||||||
taosArrayPush(pList, &pageId);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) {
|
tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) {
|
||||||
if (NO_AVAILABLE_PAGES(pResultBuf)) {
|
if (NO_AVAILABLE_PAGES(pResultBuf)) {
|
||||||
if (extendDiskFileSize(pResultBuf, pResultBuf->incStep) != TSDB_CODE_SUCCESS) {
|
// get the last page in linked list
|
||||||
return NULL;
|
SListIter iter = {0};
|
||||||
|
tdListInitIter(pResultBuf->pPageList, &iter, TD_LIST_BACKWARD);
|
||||||
|
|
||||||
|
SListNode* pn = NULL;
|
||||||
|
while((pn = tdListNext(&iter)) != NULL) {
|
||||||
|
assert(pn != NULL);
|
||||||
|
if (T_REF_VAL_GET(*(SPageInfo**)pn->data) == 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// all pages are referenced by user, try to allocate new space
|
||||||
|
if (pn == NULL) {
|
||||||
|
int32_t prev = pResultBuf->inMemPages;
|
||||||
|
pResultBuf->inMemPages = pResultBuf->inMemPages * 1.5;
|
||||||
|
|
||||||
|
qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pResultBuf, prev,
|
||||||
|
pResultBuf->inMemPages, pResultBuf->pageSize);
|
||||||
|
} else {
|
||||||
|
tdListPopNode(pResultBuf->pPageList, pn);
|
||||||
|
if (flushPageToDisk(pResultBuf, *(SPageInfo**)pn->data) != TSDB_CODE_SUCCESS) {
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// register new id in this group
|
// register new id in this group
|
||||||
*pageId = (++pResultBuf->allocateId);
|
*pageId = (++pResultBuf->allocateId);
|
||||||
registerPageId(pResultBuf, groupId, *pageId);
|
|
||||||
|
|
||||||
// clear memory for the new page
|
// register page id info
|
||||||
tFilePage* page = getResBufPage(pResultBuf, *pageId);
|
SPageInfo* pi = registerPage(pResultBuf, groupId, *pageId);
|
||||||
memset(page, 0, pResultBuf->pageSize);
|
|
||||||
|
// add to LRU list
|
||||||
return page;
|
assert(listNEles(pResultBuf->pPageList) < pResultBuf->inMemPages);
|
||||||
|
tdListPrepend(pResultBuf->pPageList, &pi);
|
||||||
|
|
||||||
|
// add to hash map
|
||||||
|
taosHashPut(pResultBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES);
|
||||||
|
|
||||||
|
// allocate buf
|
||||||
|
pi->pData = calloc(1, pResultBuf->pageSize + POINTER_BYTES);
|
||||||
|
pResultBuf->totalBufSize += pResultBuf->pageSize;
|
||||||
|
|
||||||
|
T_REF_INC(pi); // add ref count
|
||||||
|
((void**)pi->pData)[0] = pi;
|
||||||
|
|
||||||
|
return pi->pData + POINTER_BYTES;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) {
|
||||||
|
assert(pResultBuf != NULL && id >= 0);
|
||||||
|
|
||||||
|
SPageInfo** pi = taosHashGet(pResultBuf->all, &id, sizeof(int32_t));
|
||||||
|
assert(pi != NULL && *pi != NULL);
|
||||||
|
|
||||||
|
if ((*pi)->pData != NULL) { // it is in memory
|
||||||
|
// no need to update the LRU list
|
||||||
|
if (pResultBuf->numOfPages == 1) {
|
||||||
|
return (*pi)->pData + POINTER_BYTES;
|
||||||
|
}
|
||||||
|
|
||||||
|
SListNode* pnode = NULL; // todo speed up
|
||||||
|
|
||||||
|
SListIter iter = {0};
|
||||||
|
tdListInitIter(pResultBuf->pPageList, &iter, TD_LIST_FORWARD);
|
||||||
|
|
||||||
|
while((pnode = tdListNext(&iter)) != NULL) {
|
||||||
|
SPageInfo** pInfo = (SPageInfo**) pnode->data;
|
||||||
|
|
||||||
|
// remove it and add it into the front of linked-list
|
||||||
|
if ((*pInfo)->pageId == id) {
|
||||||
|
tdListPopNode(pResultBuf->pPageList, pnode);
|
||||||
|
tdListPrependNode(pResultBuf->pPageList, pnode);
|
||||||
|
T_REF_INC(*(SPageInfo**)pnode->data);
|
||||||
|
|
||||||
|
return ((*(SPageInfo**)pnode->data)->pData + POINTER_BYTES);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else { // not in memory
|
||||||
|
// choose the be flushed page
|
||||||
|
// get the last page in linked list
|
||||||
|
SListIter iter1 = {0};
|
||||||
|
tdListInitIter(pResultBuf->pPageList, &iter1, TD_LIST_BACKWARD);
|
||||||
|
|
||||||
|
SListNode* pn = NULL;
|
||||||
|
while((pn = tdListNext(&iter1)) != NULL) {
|
||||||
|
assert(pn != NULL);
|
||||||
|
if (T_REF_VAL_GET(*(SPageInfo**)pn->data) == 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// all pages are referenced by user, try to allocate new space
|
||||||
|
if (pn == NULL) {
|
||||||
|
pResultBuf->inMemPages = pResultBuf->inMemPages * 1.5;
|
||||||
|
assert(0);
|
||||||
|
return NULL;
|
||||||
|
} else {
|
||||||
|
tdListPopNode(pResultBuf->pPageList, pn);
|
||||||
|
if (flushPageToDisk(pResultBuf, *(SPageInfo**)pn->data) != TSDB_CODE_SUCCESS) {
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
char* buf = (*(SPageInfo**)pn->data)->pData;
|
||||||
|
(*(SPageInfo**)pn->data)->pData = NULL;
|
||||||
|
|
||||||
|
// load file in disk
|
||||||
|
fseek(pResultBuf->file, (*pi)->info.offset, SEEK_SET);
|
||||||
|
fread(buf, (*pi)->info.length, 1, pResultBuf->file);
|
||||||
|
|
||||||
|
(*pi)->pData = buf;
|
||||||
|
return (*pi)->pData;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page) {
|
||||||
|
assert(pResultBuf != NULL && page != NULL);
|
||||||
|
char* p = (char*) page - POINTER_BYTES;
|
||||||
|
|
||||||
|
SPageInfo* ppi = ((SPageInfo**) p)[0];
|
||||||
|
|
||||||
|
assert(T_REF_VAL_GET(ppi) > 0);
|
||||||
|
T_REF_DEC(ppi);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; }
|
int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; }
|
||||||
|
|
||||||
SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
|
SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
|
||||||
int32_t slot = getGroupIndex(pResultBuf, groupId);
|
assert(pResultBuf != NULL);
|
||||||
if (slot < 0) {
|
|
||||||
|
char** p = taosHashGet(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t));
|
||||||
|
if (p == NULL) { // it is a new group id
|
||||||
return pResultBuf->emptyDummyIdList;
|
return pResultBuf->emptyDummyIdList;
|
||||||
} else {
|
} else {
|
||||||
return taosArrayGetP(pResultBuf->list, slot);
|
return (SArray*) (*p);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -188,12 +311,11 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (FD_VALID(pResultBuf->fd)) {
|
if (pResultBuf->file != NULL) {
|
||||||
qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, file created:%s, file size:%d", handle,
|
qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, file created:%s, file size:%d", handle,
|
||||||
pResultBuf->totalBufSize, pResultBuf->path, FILE_SIZE_ON_DISK(pResultBuf));
|
pResultBuf->totalBufSize, pResultBuf->path, FILE_SIZE_ON_DISK(pResultBuf));
|
||||||
|
|
||||||
close(pResultBuf->fd);
|
fclose(pResultBuf->file);
|
||||||
munmap(pResultBuf->pBuf, FILE_SIZE_ON_DISK(pResultBuf));
|
|
||||||
pResultBuf->pBuf = NULL;
|
pResultBuf->pBuf = NULL;
|
||||||
} else {
|
} else {
|
||||||
qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, no file created", handle,
|
qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, no file created", handle,
|
||||||
|
@ -203,17 +325,16 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) {
|
||||||
unlink(pResultBuf->path);
|
unlink(pResultBuf->path);
|
||||||
tfree(pResultBuf->path);
|
tfree(pResultBuf->path);
|
||||||
|
|
||||||
size_t size = taosArrayGetSize(pResultBuf->list);
|
// size_t size = taosArrayGetSize(pResultBuf->list);
|
||||||
for (int32_t i = 0; i < size; ++i) {
|
// for (int32_t i = 0; i < size; ++i) {
|
||||||
SArray* pa = taosArrayGetP(pResultBuf->list, i);
|
// SArray* pa = taosArrayGetP(pResultBuf->list, i);
|
||||||
taosArrayDestroy(pa);
|
// taosArrayDestroy(pa);
|
||||||
}
|
// }
|
||||||
|
|
||||||
taosArrayDestroy(pResultBuf->list);
|
tdListFree(pResultBuf->pPageList);
|
||||||
taosArrayDestroy(pResultBuf->emptyDummyIdList);
|
taosArrayDestroy(pResultBuf->emptyDummyIdList);
|
||||||
taosHashCleanup(pResultBuf->idsTable);
|
taosHashCleanup(pResultBuf->idsTable);
|
||||||
|
|
||||||
tfree(pResultBuf->iBuf);
|
|
||||||
tfree(pResultBuf);
|
tfree(pResultBuf);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -18,13 +18,35 @@ void simpleTest() {
|
||||||
tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
|
tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||||
ASSERT_TRUE(pBufPage != NULL);
|
ASSERT_TRUE(pBufPage != NULL);
|
||||||
|
|
||||||
ASSERT_EQ(getNumOfRowsPerPage(pResultBuf), (16384L - sizeof(int64_t))/64);
|
ASSERT_EQ(getResBufSize(pResultBuf), 1024);
|
||||||
ASSERT_EQ(getResBufSize(pResultBuf), 1000*16384L);
|
|
||||||
|
|
||||||
SIDList list = getDataBufPagesIdList(pResultBuf, groupId);
|
SIDList list = getDataBufPagesIdList(pResultBuf, groupId);
|
||||||
ASSERT_EQ(taosArrayGetSize(list), 1);
|
ASSERT_EQ(taosArrayGetSize(list), 1);
|
||||||
ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1);
|
ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1);
|
||||||
|
|
||||||
|
releaseResBufPage(pResultBuf, pBufPage);
|
||||||
|
|
||||||
|
tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||||
|
|
||||||
|
tFilePage* t = getResBufPage(pResultBuf, pageId);
|
||||||
|
assert(t == pBufPage1);
|
||||||
|
|
||||||
|
tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||||
|
tFilePage* t1 = getResBufPage(pResultBuf, pageId);
|
||||||
|
assert(t1 == pBufPage2);
|
||||||
|
|
||||||
|
tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||||
|
tFilePage* t2 = getResBufPage(pResultBuf, pageId);
|
||||||
|
assert(t2 == pBufPage3);
|
||||||
|
|
||||||
|
tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||||
|
tFilePage* t3 = getResBufPage(pResultBuf, pageId);
|
||||||
|
assert(t3 == pBufPage4);
|
||||||
|
|
||||||
|
tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||||
|
tFilePage* t4 = getResBufPage(pResultBuf, pageId);
|
||||||
|
assert(t4 == pBufPage5);
|
||||||
|
|
||||||
destroyResultBuf(pResultBuf, NULL);
|
destroyResultBuf(pResultBuf, NULL);
|
||||||
}
|
}
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
|
@ -76,6 +76,7 @@ int tdListPrepend(SList *list, void *data) {
|
||||||
SListNode *node = (SListNode *)malloc(sizeof(SListNode) + list->eleSize);
|
SListNode *node = (SListNode *)malloc(sizeof(SListNode) + list->eleSize);
|
||||||
if (node == NULL) return -1;
|
if (node == NULL) return -1;
|
||||||
|
|
||||||
|
node->next = node->prev = NULL;
|
||||||
memcpy((void *)(node->data), data, list->eleSize);
|
memcpy((void *)(node->data), data, list->eleSize);
|
||||||
tdListPrependNode(list, node);
|
tdListPrependNode(list, node);
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue