From d5e4fc3201f9f3cba195b4b9b64ef8746940385d Mon Sep 17 00:00:00 2001 From: hzcheng Date: Fri, 20 Mar 2020 23:02:42 +0800 Subject: [PATCH] TD-34 --- src/util/inc/tlist.h | 7 +-- src/util/src/tlist.c | 57 +++++++++++------- src/vnode/tsdb/inc/tsdbCache.h | 50 +++++++--------- src/vnode/tsdb/src/tsdbCache.c | 103 +++++++++++++++++++++++++++++---- src/vnode/tsdb/src/tsdbMain.c | 4 +- 5 files changed, 154 insertions(+), 67 deletions(-) diff --git a/src/util/inc/tlist.h b/src/util/inc/tlist.h index cc99e183ed..2ea536ea3e 100644 --- a/src/util/inc/tlist.h +++ b/src/util/inc/tlist.h @@ -19,10 +19,7 @@ extern "C" { #endif -typedef enum { - TD_LIST_FORWARD, - TD_LIST_BACKWARD -} TD_LIST_DIRECTION_T; +typedef enum { TD_LIST_FORWARD, TD_LIST_BACKWARD } TD_LIST_DIRECTION_T; typedef struct _list_node { struct _list_node *next; @@ -52,6 +49,8 @@ typedef struct { SList * tdListNew(int eleSize); void tdListFree(SList *list); void tdListEmpty(SList *list); +void tdListPrependNode(SList *list, SListNode *node); +void tdListAppendNode(SList *list, SListNode *node); int tdListPrepend(SList *list, void *data); int tdListAppend(SList *list, void *data); SListNode *tdListPopHead(SList *list); diff --git a/src/util/src/tlist.c b/src/util/src/tlist.c index a6cb98df5f..f11a5481f6 100644 --- a/src/util/src/tlist.c +++ b/src/util/src/tlist.c @@ -43,10 +43,7 @@ void tdListFree(SList *list) { free(list); } -int tdListPrepend(SList *list, void *data) { - SListNode *node = (SListNode *)malloc(sizeof(SListNode) + list->eleSize); - if (node == NULL) return -1; - +void tdListPrependNode(SList *list, SListNode *node) { if (list->head == NULL) { list->head = node; list->tail = node; @@ -57,12 +54,9 @@ int tdListPrepend(SList *list, void *data) { list->head = node; } list->numOfEles++; - return 0; } -int tdListAppend(SList *list, void *data) { - SListNode *node = (SListNode *)malloc(sizeof(SListNode) + list->eleSize); - if (node == NULL) return -1; +void tdListAppendNode(SList *list, SListNode *node) { if (list->head == NULL) { list->head = node; list->tail = node; @@ -74,6 +68,25 @@ int tdListAppend(SList *list, void *data) { } list->numOfEles++; +} + +int tdListPrepend(SList *list, void *data) { + SListNode *node = (SListNode *)malloc(sizeof(SListNode) + list->eleSize); + if (node == NULL) return -1; + + memcpy((void *)(node->data), data, list->eleSize); + tdListPrependNode(list, node); + + return 0; +} + +int tdListAppend(SList *list, void *data) { + SListNode *node = (SListNode *)malloc(sizeof(SListNode) + list->eleSize); + if (node == NULL) return -1; + + memcpy((void *)(node->data), data, list->eleSize); + tdListAppendNode(list, node); + return 0; } @@ -104,22 +117,22 @@ SListNode *tdListPopTail(SList *list) { } SListNode *tdListPopNode(SList *list, SListNode *node) { - if (list->head == node) { - list->head = node->next; - } - if (list->tail == node) { - list->tail = node->prev; - } + if (list->head == node) { + list->head = node->next; + } + if (list->tail == node) { + list->tail = node->prev; + } - if (node->prev != NULL) { - node->prev->next = node->next; - } - if (node->next != NULL) { - node->next->prev = node->prev; - } - list->numOfEles--; + if (node->prev != NULL) { + node->prev->next = node->next; + } + if (node->next != NULL) { + node->next->prev = node->prev; + } + list->numOfEles--; - return node; + return node; } void tdListNodeGetData(SList *list, SListNode *node, void *target) { memcpy(node->data, target, list->eleSize); } diff --git a/src/vnode/tsdb/inc/tsdbCache.h b/src/vnode/tsdb/inc/tsdbCache.h index 1821505eae..3bffa1c6a9 100644 --- a/src/vnode/tsdb/inc/tsdbCache.h +++ b/src/vnode/tsdb/inc/tsdbCache.h @@ -17,45 +17,39 @@ #include -// #include "cache.h" +#include "tlist.h" #ifdef __cplusplus extern "C" { #endif -#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16*1024*1024 /* 16M */ +#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 * 1024 * 1024 /* 16M */ typedef struct { - int64_t skey; // start key - int64_t ekey; // end key - int32_t numOfRows; // numOfRows -} STableCacheInfo; + int blockId; + int offset; + int remain; + int padding; + char data[]; +} STsdbCacheBlock; -typedef struct _tsdb_cache_block { - char * pData; - STableCacheInfo * pTableInfo; - struct _tsdb_cache_block *prev; - struct _tsdb_cache_block *next; -} STSDBCacheBlock; +typedef struct { + int64_t index; + SList * memPool; +} STsdbCachePool; -// Use a doublely linked list to implement this -typedef struct STSDBCache { - // Number of blocks the cache is allocated - int32_t numOfBlocks; - STSDBCacheBlock *cacheList; - void * current; +typedef struct { + int maxBytes; + int cacheBlockSize; + STsdbCachePool pool; + STsdbCacheBlock *curBlock; + SList * mem; + SList * imem; } STsdbCache; -// ---- Operation on STSDBCacheBlock -#define TSDB_CACHE_BLOCK_DATA(pBlock) ((pBlock)->pData) -#define TSDB_CACHE_AVAIL_SPACE(pBlock) ((char *)((pBlock)->pTableInfo) - ((pBlock)->pData)) -#define TSDB_TABLE_INFO_OF_CACHE(pBlock, tableId) ((pBlock)->pTableInfo)[tableId] -#define TSDB_NEXT_CACHE_BLOCK(pBlock) ((pBlock)->next) -#define TSDB_PREV_CACHE_BLOCK(pBlock) ((pBlock)->prev) - -STsdbCache *tsdbInitCache(int64_t maxSize); -int32_t tsdbFreeCache(STsdbCache *pCache); -void * tsdbAllocFromCache(STsdbCache *pCache, int64_t bytes); +STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize); +void tsdbFreeCache(STsdbCache *pCache); +void * tsdbAllocFromCache(STsdbCache *pCache, int bytes); #ifdef __cplusplus } diff --git a/src/vnode/tsdb/src/tsdbCache.c b/src/vnode/tsdb/src/tsdbCache.c index 165c561b5d..4baedf55a3 100644 --- a/src/vnode/tsdb/src/tsdbCache.c +++ b/src/vnode/tsdb/src/tsdbCache.c @@ -16,22 +16,103 @@ #include "tsdbCache.h" -STsdbCache *tsdbInitCache(int64_t maxSize) { - STsdbCache *pCacheHandle = (STsdbCache *)malloc(sizeof(STsdbCache)); - if (pCacheHandle == NULL) { - // TODO : deal with the error - return NULL; +static int tsdbAllocBlockFromPool(STsdbCache *pCache); +static void tsdbFreeBlockList(SList *list); + +STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize) { + STsdbCache *pCache = (STsdbCache *)calloc(1, sizeof(STsdbCache)); + if (pCache == NULL) return NULL; + + pCache->maxBytes = maxBytes; + pCache->cacheBlockSize = cacheBlockSize; + + int nBlocks = maxBytes / cacheBlockSize + 1; + if (nBlocks <= 1) nBlocks = 2; + + STsdbCachePool *pPool = &(pCache->pool); + pPool->index = 0; + pPool->memPool = tdListNew(sizeof(STsdbCacheBlock *)); + if (pPool->memPool == NULL) goto _err; + + for (int i = 0; i < nBlocks; i++) { + STsdbCacheBlock *pBlock = (STsdbCacheBlock *)malloc(sizeof(STsdbCacheBlock) + cacheBlockSize); + if (pBlock == NULL) { + goto _err; + } + pBlock->offset = 0; + pBlock->remain = cacheBlockSize; + tdListAppend(pPool->memPool, (void *)(&pBlock)); } - return pCacheHandle; + pCache->mem = tdListNew(sizeof(STsdbCacheBlock *)); + if (pCache->mem == NULL) goto _err; + + pCache->imem = tdListNew(sizeof(STsdbCacheBlock *)); + if (pCache->imem == NULL) goto _err; + + return pCache; + +_err: + tsdbFreeCache(pCache); + return NULL; } -int32_t tsdbFreeCache(STsdbCache *pHandle) { return 0; } +void tsdbFreeCache(STsdbCache *pCache) { + tsdbFreeBlockList(pCache->imem); + tsdbFreeBlockList(pCache->mem); + tsdbFreeBlockList(pCache->pool.memPool); + free(pCache); +} -void *tsdbAllocFromCache(STsdbCache *pCache, int64_t bytes) { - // TODO: implement here - void *ptr = malloc(bytes); - if (ptr == NULL) return NULL; +void *tsdbAllocFromCache(STsdbCache *pCache, int bytes) { + if (pCache == NULL) return NULL; + if (bytes > pCache->cacheBlockSize) return NULL; + + if (isListEmpty(pCache->imem)) { + if (tsdbAllocBlockFromPool(pCache) < 0) { + // TODO: deal with the error + } + } + + if (pCache->curBlock->remain < bytes) { + if (tsdbAllocBlockFromPool(pCache) < 0) { + // TODO: deal with the error + } + } + + void *ptr = (void *)(pCache->curBlock->data + pCache->curBlock->offset); + pCache->curBlock->offset += bytes; + pCache->curBlock->remain -= bytes; return ptr; +} + +static void tsdbFreeBlockList(SList *list) { + if (list == NULL) return; + SListNode * node = NULL; + STsdbCacheBlock *pBlock = NULL; + while ((node = tdListPopHead(list)) != NULL) { + tdListNodeGetData(list, node, (void *)(&pBlock)); + free(pBlock); + listNodeFree(node); + } + tdListFree(list); +} + +static int tsdbAllocBlockFromPool(STsdbCache *pCache) { + STsdbCachePool *pPool = &(pCache->pool); + if (listNEles(pPool->memPool) == 0) return -1; + + SListNode *node = tdListPopHead(pPool->memPool); + + STsdbCacheBlock *pBlock = NULL; + tdListNodeGetData(pPool->memPool, node, (void *)(&pBlock)); + pBlock->blockId = pPool->index++; + pBlock->offset = 0; + pBlock->remain = pCache->cacheBlockSize; + + tdListAppendNode(pPool->memPool, node); + pCache->curBlock = pBlock; + + return 0; } \ No newline at end of file diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 80a9ae4631..87cd23eb92 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -163,7 +163,7 @@ tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO pRepo->tsdbMeta = pMeta; // Initialize cache - STsdbCache *pCache = tsdbInitCache(pCfg->maxCacheSize); + STsdbCache *pCache = tsdbInitCache(pCfg->maxCacheSize, -1); if (pCache == NULL) { free(pRepo->rootDir); tsdbFreeMeta(pRepo->tsdbMeta); @@ -244,7 +244,7 @@ tsdb_repo_t *tsdbOpenRepo(char *tsdbDir) { return NULL; } - pRepo->tsdbCache = tsdbInitCache(pRepo->config.maxCacheSize); + pRepo->tsdbCache = tsdbInitCache(pRepo->config.maxCacheSize, -1); if (pRepo->tsdbCache == NULL) { tsdbFreeMeta(pRepo->tsdbMeta); free(pRepo->rootDir);