xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #9752 from taosdata/feature/vnode 

Feature/vnode
This commit is contained in:
Hongze Cheng 2022-01-12 10:06:35 +08:00 committed by GitHub
parent 0384fb14a2 8b5e6b6895
commit 3cfaaf6b20
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 303 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
include/libs/tkv/tkv.h
View File

@ -1,64 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TKV_H_
#define _TD_TKV_H_
#if 0
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
// Types exported
typedef struct STkvDb STkvDb;
typedef struct STkvOpts STkvOpts;
typedef struct STkvCache STkvCache;
typedef struct STkvReadOpts STkvReadOpts;
typedef struct STkvWriteOpts STkvWriteOpts;
// DB operations
STkvDb *tkvOpen(const STkvOpts *options, const char *path);
void tkvClose(STkvDb *db);
void tkvPut(STkvDb *db, const STkvWriteOpts *, const char *key, size_t keylen, const char *val, size_t vallen);
char * tkvGet(STkvDb *db, const STkvReadOpts *, const char *key, size_t keylen, size_t *vallen);
void tkvCommit(STkvDb *db);
// DB options
STkvOpts *tkvOptsCreate();
void tkvOptsDestroy(STkvOpts *);
void tkvOptionsSetCache(STkvOpts *, STkvCache *);
void tkvOptsSetCreateIfMissing(STkvOpts *, unsigned char);
// DB cache
typedef enum { TKV_LRU_CACHE = 0, TKV_LFU_CACHE = 1 } ETkvCacheType;
STkvCache *tkvCacheCreate(size_t capacity, ETkvCacheType type);
void tkvCacheDestroy(STkvCache *);
// STkvReadOpts
STkvReadOpts *tkvReadOptsCreate();
void tkvReadOptsDestroy(STkvReadOpts *);
// STkvWriteOpts
STkvWriteOpts *tkvWriteOptsCreate();
void tkvWriteOptsDestroy(STkvWriteOpts *);
#ifdef __cplusplus
}
#endif
#endif
#endif /*_TD_TKV_H_*/

22
include/util/tarray.h
View File

@ -23,6 +23,19 @@ extern "C" {
#include "os.h"
#include "talgo.h"
#if 0
#define TARRAY(TYPE) \
struct { \
int32_t tarray_size_; \
int32_t tarray_neles_; \
struct TYPE* td_array_data_; \
}
#define TARRAY_SIZE(ARRAY) (ARRAY)->tarray_size_
#define TARRAY_NELES(ARRAY) (ARRAY)->tarray_neles_
#define TARRAY_ELE_AT(ARRAY, IDX) ((ARRAY)->td_array_data_ + idx)
#endif
#define TARRAY_MIN_SIZE 8
#define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize))
#define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize)
@ -57,7 +70,7 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t tsize);
* @param nEles
* @return
*/
void *taosArrayAddBatch(SArray *pArray, const void *pData, int nEles);
void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles);
/**
*
@ -65,7 +78,7 @@ void *taosArrayAddBatch(SArray *pArray, const void *pData, int nEles);
* @param pData position array list
* @param numOfElems the number of removed position
*/
void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfElems);
void taosArrayRemoveBatch(SArray* pArray, const int32_t* pData, int32_t numOfElems);
/**
*
@ -73,7 +86,7 @@ void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfEle
* @param comparFn
* @param fp
*/
void taosArrayRemoveDuplicate(SArray *pArray, __compar_fn_t comparFn, void (*fp)(void*));
void taosArrayRemoveDuplicate(SArray* pArray, __compar_fn_t comparFn, void (*fp)(void*));
/**
* add all element from the source array list into the destination
@ -242,7 +255,6 @@ int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t
*/
char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int flags);
/**
* sort the pointer data in the array
* @param pArray
@ -251,7 +263,7 @@ char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t
* @return
*/
void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void *param);
void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param);
#ifdef __cplusplus
}

10
source/dnode/vnode/impl/src/vnodeWrite.c
View File

@ -83,7 +83,17 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) {
// TODO: handle error
}
if (pCreateTbReq->type == TD_SUPER_TABLE) {
free(pCreateTbReq->stbCfg.pSchema);
free(pCreateTbReq->stbCfg.pTagSchema);
} else if (pCreateTbReq->type == TD_CHILD_TABLE) {
free(pCreateTbReq->ctbCfg.pTag);
} else {
free(pCreateTbReq->ntbCfg.pSchema);
}
}
taosArrayDestroy(vCreateTbBatchReq.pArray);
break;
case TDMT_VND_DROP_STB:
case TDMT_VND_DROP_TABLE:

10
source/libs/tkv/CMakeLists.txt
View File

@ -1,11 +1,17 @@
aux_source_directory(src TKV_SRC)
add_library(tkv STATIC ${TKV_SRC})
# target_include_directories(
# tkv
# PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tkv"
# PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
# )
target_include_directories(
tkv
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tkv"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc"
PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/src/inc"
)
target_link_libraries(
tkv
PUBLIC os
PUBLIC util
)

39
source/libs/tkv/inc/tkv.h Normal file
View File

@ -0,0 +1,39 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TKV_H_
#define _TD_TKV_H_
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
// Forward declaration
typedef struct TDB TDB;
typedef struct TDB_ENV TDB_ENV;
// SKey
typedef struct {
void * bdata;
uint32_t size;
} TDB_KEY, TDB_VALUE;
#ifdef __cplusplus
}
#endif
#endif /*_TD_TKV_H_*/

39
source/libs/tkv/src/inc/tkvBufPool.h Normal file
View File

@ -0,0 +1,39 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TKV_BUF_POOL_H_
#define _TD_TKV_BUF_POOL_H_
#include "tkvPage.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct STkvBufPool STkvBufPool;
int tbpOpen(STkvBufPool **ppTkvBufPool);
int tbpClose(STkvBufPool *pTkvBufPool);
STkvPage *tbpNewPage(STkvBufPool *pTkvBufPool);
int tbpDelPage(STkvBufPool *pTkvBufPool);
STkvPage *tbpFetchPage(STkvBufPool *pTkvBufPool, pgid_t pgid);
int tbpUnpinPage(STkvBufPool *pTkvBufPool, pgid_t pgid);
void tbpFlushPages(STkvBufPool *pTkvBufPool);
#ifdef __cplusplus
}
#endif
#endif /*_TD_TKV_BUF_POOL_H_*/

31
source/libs/tkv/src/inc/tkvDB.h Normal file
View File

@ -0,0 +1,31 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TKV_DB_H_
#define _TD_TKV_DB_H_
#ifdef __cplusplus
extern "C" {
#endif
struct TDB {
// TODO
};
#ifdef __cplusplus
}
#endif
#endif /*_TD_TKV_DB_H_*/

9
source/libs/tkv/inc/tkvDef.h → source/libs/tkv/src/inc/tkvDef.h
View File

@ -26,6 +26,15 @@ extern "C" {
typedef int32_t pgid_t;
#define TKV_IVLD_PGID ((pgid_t)-1)
// framd_id_t
typedef int32_t frame_id_t;
// pgsize_t
typedef int32_t pgsize_t;
#define TKV_MIN_PGSIZE 512
#define TKV_MAX_PGSIZE 16384
#define TKV_IS_PGSIZE_VLD(s) (((s) >= TKV_MIN_PGSIZE) && (TKV_MAX_PGSIZE <= TKV_MAX_PGSIZE))
#ifdef __cplusplus
}
#endif

13
source/libs/tkv/inc/tDiskMgr.h → source/libs/tkv/src/inc/tkvDiskMgr.h
View File

@ -24,13 +24,14 @@ extern "C" {
#include "tkvDef.h"
typedef struct SDiskMgr SDiskMgr;
typedef struct STkvDiskMgr STkvDiskMgr;
int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize);
int tdmClose(SDiskMgr *pDiskMgr);
int tdmReadPage(SDiskMgr *pDiskMgr, pgid_t pgid, void *pData);
int tdmWritePage(SDiskMgr *pDiskMgr, pgid_t pgid, const void *pData);
int32_t tdmAllocPage(SDiskMgr *pDiskMgr);
int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize);
int tdmClose(STkvDiskMgr *pDiskMgr);
int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData);
int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData);
int tdmFlush(STkvDiskMgr *pDiskMgr);
pgid_t tdmAllocPage(STkvDiskMgr *pDiskMgr);
#ifdef __cplusplus
}

31
source/libs/tkv/src/inc/tkvEnv.h Normal file
View File

@ -0,0 +1,31 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TKV_ENV_H_
#define _TD_TKV_ENV_H_
#ifdef __cplusplus
extern "C" {
#endif
struct TDB_ENV {
char *homeDir;
};
#ifdef __cplusplus
}
#endif
#endif /*_TD_TKV_ENV_H_*/

20
source/libs/tkv/inc/tPage.h → source/libs/tkv/src/inc/tkvPage.h
View File

@ -17,22 +17,30 @@
#define _TD_TKV_PAGE_H_
#include "os.h"
#include "tkvDef.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct STkvPage {
pgid_t pgid;
int32_t pinCount;
bool idDirty;
char* pData;
} STkvPage;
typedef struct {
uint16_t dbver;
uint16_t pgsize;
uint32_t cksm;
} SPgHdr;
} STkvPgHdr;
typedef struct {
SPgHdr chdr;
uint16_t used; // number of used slots
uint16_t loffset; // the offset of the starting location of the last slot used
} SSlottedPgHdr;
// typedef struct {
// SPgHdr chdr;
// uint16_t used; // number of used slots
// uint16_t loffset; // the offset of the starting location of the last slot used
// } SSlottedPgHdr;
#ifdef __cplusplus
}

47
source/libs/tkv/src/tDiskMgr.c
View File

@ -13,37 +13,62 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tDiskMgr.h"
#include "tkvDiskMgr.h"
struct SDiskMgr {
const char *fname;
struct STkvDiskMgr {
char * fname;
uint16_t pgsize;
FileFd fd;
int32_t npgid;
pgid_t npgid;
};
#define PAGE_OFFSET(PGID, PGSIZE) ((PGID) * (PGSIZE))
int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) {
// TODO
int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) {
STkvDiskMgr *pDiskMgr;
pDiskMgr = malloc(sizeof(*pDiskMgr));
if (pDiskMgr == NULL) {
return -1;
}
pDiskMgr->fname = strdup(fname);
if (pDiskMgr->fname == NULL) {
free(pDiskMgr);
return -1;
}
pDiskMgr->pgsize = pgsize;
pDiskMgr->fd = open(fname, O_CREAT | O_RDWR, 0755);
if (pDiskMgr->fd < 0) {
free(pDiskMgr->fname);
free(pDiskMgr);
return -1;
}
*ppDiskMgr = pDiskMgr;
return 0;
}
int tdmClose(SDiskMgr *pDiskMgr) {
// TODO
int tdmClose(STkvDiskMgr *pDiskMgr) {
close(pDiskMgr->fd);
free(pDiskMgr->fname);
free(pDiskMgr);
return 0;
}
int tdmReadPage(SDiskMgr *pDiskMgr, pgid_t pgid, void *pData) {
int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData) {
taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET);
taosReadFile(pDiskMgr->fd, pData, pDiskMgr->pgsize);
return 0;
}
int tdmWritePage(SDiskMgr *pDiskMgr, pgid_t pgid, const void *pData) {
int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData) {
taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET);
taosWriteFile(pDiskMgr->fd, pData, pDiskMgr->pgsize);
return 0;
}
int32_t tdmAllocPage(SDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; }
int tdmFlush(STkvDiskMgr *pDiskMgr) { return taosFsyncFile(pDiskMgr->fd); }
int32_t tdmAllocPage(STkvDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; }

54
source/libs/tkv/src/tkvBufPool.c Normal file
View File

@ -0,0 +1,54 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "thash.h"
#include "tlist.h"
#include "tkvBufPool.h"
#include "tkvDiskMgr.h"
#include "tkvPage.h"
struct SFrameIdWrapper {
TD_SLIST_NODE(SFrameIdWrapper);
frame_id_t id;
};
struct STkvBufPool {
STkvPage* pages;
STkvDiskMgr* pDiskMgr;
SHashObj* pgTb; // page_id_t --> frame_id_t
TD_SLIST(SFrameIdWrapper) freeList;
pthread_mutex_t mutex;
};
typedef struct STkvLRUReplacer {
} STkvLRUReplacer;
typedef struct STkvLFUReplacer {
} STkvLFUReplacer;
typedef struct STkvCLKReplacer {
} STkvCLKReplacer;
typedef enum { TKV_LRU_REPLACER = 0, TKV_LFU_REPLACER, TVK_CLK_REPLACER } tkv_replacer_t;
typedef struct STkvReplacer {
tkv_replacer_t type;
union {
STkvLRUReplacer lruRep;
STkvLFUReplacer lfuRep;
STkvCLKReplacer clkRep;
};
} STkvReplacer;