From f70c550359c254ae481244aae705e5d9900fe3ff Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 7 Jan 2022 10:45:33 +0000 Subject: [PATCH 1/3] more --- include/libs/tkv/tkv.h | 2 + source/libs/tkv/CMakeLists.txt | 8 +- source/libs/tkv/inc/{tkvDef.h => tDiskMgr.h} | 39 +--- .../libs/tkv/inc/{tkvRocksdb.h => tkvMacro.h} | 6 +- .../libs/tkv/src/{tkvRocksdb.c => tDiskMgr.c} | 16 +- source/libs/tkv/src/tkv.c | 171 ------------------ 6 files changed, 29 insertions(+), 213 deletions(-) rename source/libs/tkv/inc/{tkvDef.h => tDiskMgr.h} (60%) rename source/libs/tkv/inc/{tkvRocksdb.h => tkvMacro.h} (89%) rename source/libs/tkv/src/{tkvRocksdb.c => tDiskMgr.c} (71%) delete mode 100644 source/libs/tkv/src/tkv.c diff --git a/include/libs/tkv/tkv.h b/include/libs/tkv/tkv.h index edade1dc36..98194f090c 100644 --- a/include/libs/tkv/tkv.h +++ b/include/libs/tkv/tkv.h @@ -16,6 +16,7 @@ #ifndef _TD_TKV_H_ #define _TD_TKV_H_ +#if 0 #include "os.h" #ifdef __cplusplus @@ -59,4 +60,5 @@ void tkvWriteOptsDestroy(STkvWriteOpts *); } #endif +#endif #endif /*_TD_TKV_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/CMakeLists.txt b/source/libs/tkv/CMakeLists.txt index 529b2703f7..0620e12f55 100644 --- a/source/libs/tkv/CMakeLists.txt +++ b/source/libs/tkv/CMakeLists.txt @@ -8,10 +8,4 @@ target_include_directories( target_link_libraries( tkv PUBLIC os -) -if(${BUILD_WITH_ROCKSDB}) - target_link_libraries( - tkv - PUBLIC rocksdb - ) -endif(${BUILD_WITH_ROCKSDB}) +) \ No newline at end of file diff --git a/source/libs/tkv/inc/tkvDef.h b/source/libs/tkv/inc/tDiskMgr.h similarity index 60% rename from source/libs/tkv/inc/tkvDef.h rename to source/libs/tkv/inc/tDiskMgr.h index 7433b1a4dd..43f5bca582 100644 --- a/source/libs/tkv/inc/tkvDef.h +++ b/source/libs/tkv/inc/tDiskMgr.h @@ -13,46 +13,23 @@ * along with this program. If not, see . */ -#ifndef _TD_TKV_DEF_H_ -#define _TD_TKV_DEF_H_ - -#ifdef USE_ROCKSDB -#include -#endif +#ifndef _TD_TDISK_MGR_H_ +#define _TD_TDISK_MGR_H_ #ifdef __cplusplus extern "C" { #endif -struct STkvDb { -#ifdef USE_ROCKSDB - rocksdb_t *db; -#endif -}; -struct STkvOpts { -#ifdef USE_ROCKSDB - rocksdb_options_t *opts; -#endif -}; +#include "os.h" -struct STkvCache { - // TODO -}; +typedef struct SDiskMgr SDiskMgr; -struct STkvReadOpts { -#ifdef USE_ROCKSDB - rocksdb_readoptions_t *ropts; -#endif -}; - -struct STkvWriteOpts { -#ifdef USE_ROCKSDB - rocksdb_writeoptions_t *wopts; -#endif -}; +int tdmReadPage(int32_t pgid, char *pData); +int tdmWritePage(int32_t pgid, const char *pData); +int32_t tdmAllocPage(SDiskMgr *pDiskMgr); #ifdef __cplusplus } #endif -#endif /*_TD_TKV_DEF_H_*/ \ No newline at end of file +#endif /*_TD_TDISK_MGR_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/inc/tkvRocksdb.h b/source/libs/tkv/inc/tkvMacro.h similarity index 89% rename from source/libs/tkv/inc/tkvRocksdb.h rename to source/libs/tkv/inc/tkvMacro.h index 658deb335f..6e3458c021 100644 --- a/source/libs/tkv/inc/tkvRocksdb.h +++ b/source/libs/tkv/inc/tkvMacro.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_TVK_ROCKSDB_H_ -#define _TD_TVK_ROCKSDB_H_ +#ifndef _TD_TKV_MACRO_H_ +#define _TD_TKV_MACRO_H_ #ifdef __cplusplus extern "C" { @@ -24,4 +24,4 @@ extern "C" { } #endif -#endif /*_TD_TVK_ROCKSDB_H_*/ \ No newline at end of file +#endif /*_TD_TKV_MACRO_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/tkvRocksdb.c b/source/libs/tkv/src/tDiskMgr.c similarity index 71% rename from source/libs/tkv/src/tkvRocksdb.c rename to source/libs/tkv/src/tDiskMgr.c index 6dea4a4e57..916eaf66d6 100644 --- a/source/libs/tkv/src/tkvRocksdb.c +++ b/source/libs/tkv/src/tDiskMgr.c @@ -11,4 +11,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "tDiskMgr.h" + +struct SDiskMgr { + const char *fname; + uint16_t pgsize; + int fd; +}; + +int tdmReadPage(int32_t pgid, char *pData) {} + +int tdmWritePage(int32_t pgid, const char *pData) {} + +int32_t tdmAllocPage(SDiskMgr *pDiskMgr) {} \ No newline at end of file diff --git a/source/libs/tkv/src/tkv.c b/source/libs/tkv/src/tkv.c deleted file mode 100644 index 0c6f896d56..0000000000 --- a/source/libs/tkv/src/tkv.c +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tkv.h" -#include "tkvDef.h" - -static pthread_once_t isInit = PTHREAD_ONCE_INIT; -static STkvReadOpts defaultReadOpts; -static STkvWriteOpts defaultWriteOpts; - -static void tkvInit(); - -STkvDb *tkvOpen(const STkvOpts *options, const char *path) { - pthread_once(&isInit, tkvInit); - STkvDb *pDb = NULL; - - pDb = (STkvDb *)malloc(sizeof(*pDb)); - if (pDb == NULL) { - return NULL; - } - -#ifdef USE_ROCKSDB - char *err = NULL; - - pDb->db = rocksdb_open(options->opts, path, &err); - // TODO: check err -#endif - - return pDb; -} - -void tkvClose(STkvDb *pDb) { - if (pDb) { -#ifdef USE_ROCKSDB - rocksdb_close(pDb->db); -#endif - free(pDb); - } -} - -void tkvPut(STkvDb *pDb, const STkvWriteOpts *pwopts, const char *key, size_t keylen, const char *val, size_t vallen) { -#ifdef USE_ROCKSDB - char *err = NULL; - rocksdb_put(pDb->db, pwopts ? pwopts->wopts : defaultWriteOpts.wopts, key, keylen, val, vallen, &err); - // TODO: check error -#endif -} - -char *tkvGet(STkvDb *pDb, const STkvReadOpts *propts, const char *key, size_t keylen, size_t *vallen) { - char *ret = NULL; - -#ifdef USE_ROCKSDB - char *err = NULL; - ret = rocksdb_get(pDb->db, propts ? propts->ropts : defaultReadOpts.ropts, key, keylen, vallen, &err); - // TODD: check error -#endif - - return ret; -} - -STkvOpts *tkvOptsCreate() { - STkvOpts *pOpts = NULL; - - pOpts = (STkvOpts *)malloc(sizeof(*pOpts)); - if (pOpts == NULL) { - return NULL; - } - -#ifdef USE_ROCKSDB - pOpts->opts = rocksdb_options_create(); - // TODO: check error -#endif - - return pOpts; -} - -void tkvOptsDestroy(STkvOpts *pOpts) { - if (pOpts) { -#ifdef USE_ROCKSDB - rocksdb_options_destroy(pOpts->opts); -#endif - free(pOpts); - } -} - -void tkvOptionsSetCache(STkvOpts *popts, STkvCache *pCache) { - // TODO -} - -void tkvOptsSetCreateIfMissing(STkvOpts *pOpts, unsigned char c) { -#ifdef USE_ROCKSDB - rocksdb_options_set_create_if_missing(pOpts->opts, c); -#endif -} - -STkvReadOpts *tkvReadOptsCreate() { - STkvReadOpts *pReadOpts = NULL; - - pReadOpts = (STkvReadOpts *)malloc(sizeof(*pReadOpts)); - if (pReadOpts == NULL) { - return NULL; - } - -#ifdef USE_ROCKSDB - pReadOpts->ropts = rocksdb_readoptions_create(); -#endif - - return pReadOpts; -} - -void tkvReadOptsDestroy(STkvReadOpts *pReadOpts) { - if (pReadOpts) { -#ifdef USE_ROCKSDB - rocksdb_readoptions_destroy(pReadOpts->ropts); -#endif - free(pReadOpts); - } -} - -STkvWriteOpts *tkvWriteOptsCreate() { - STkvWriteOpts *pWriteOpts = NULL; - - pWriteOpts = (STkvWriteOpts *)malloc(sizeof(*pWriteOpts)); - if (pWriteOpts == NULL) { - return NULL; - } - -#ifdef USE_ROCKSDB - pWriteOpts->wopts = rocksdb_writeoptions_create(); -#endif - - return pWriteOpts; -} - -void tkvWriteOptsDestroy(STkvWriteOpts *pWriteOpts) { - if (pWriteOpts) { -#ifdef USE_ROCKSDB - rocksdb_writeoptions_destroy(pWriteOpts->wopts); -#endif - free(pWriteOpts); - } -} - -/* ------------------------ STATIC METHODS ------------------------ */ -static void tkvInit() { -#ifdef USE_ROCKSDB - defaultReadOpts.ropts = rocksdb_readoptions_create(); - defaultWriteOpts.wopts = rocksdb_writeoptions_create(); - rocksdb_writeoptions_disable_WAL(defaultWriteOpts.wopts, true); - -#endif -} - -static void tkvClear() { -#ifdef USE_ROCKSDB - rocksdb_readoptions_destroy(defaultReadOpts.ropts); - rocksdb_writeoptions_destroy(defaultWriteOpts.wopts); -#endif -} From 0439c5257565fa873452cf87fc9a0ef68631d93b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 7 Jan 2022 10:54:54 +0000 Subject: [PATCH 2/3] more --- source/libs/tkv/inc/tDiskMgr.h | 4 ++-- source/libs/tkv/src/tDiskMgr.c | 19 +++++++++++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/source/libs/tkv/inc/tDiskMgr.h b/source/libs/tkv/inc/tDiskMgr.h index 43f5bca582..ff6e0b6ab7 100644 --- a/source/libs/tkv/inc/tDiskMgr.h +++ b/source/libs/tkv/inc/tDiskMgr.h @@ -24,8 +24,8 @@ extern "C" { typedef struct SDiskMgr SDiskMgr; -int tdmReadPage(int32_t pgid, char *pData); -int tdmWritePage(int32_t pgid, const char *pData); +int tdmReadPage(SDiskMgr *pDiskMgr, int32_t pgid, void *pData); +int tdmWritePage(SDiskMgr *pDiskMgr, int32_t pgid, const void *pData); int32_t tdmAllocPage(SDiskMgr *pDiskMgr); #ifdef __cplusplus diff --git a/source/libs/tkv/src/tDiskMgr.c b/source/libs/tkv/src/tDiskMgr.c index 916eaf66d6..dfdfd1f96b 100644 --- a/source/libs/tkv/src/tDiskMgr.c +++ b/source/libs/tkv/src/tDiskMgr.c @@ -18,11 +18,22 @@ struct SDiskMgr { const char *fname; uint16_t pgsize; - int fd; + FileFd fd; + int32_t npgid; }; -int tdmReadPage(int32_t pgid, char *pData) {} +#define PAGE_OFFSET(PGID, PGSIZE) ((PGID) * (PGSIZE)) -int tdmWritePage(int32_t pgid, const char *pData) {} +int tdmReadPage(SDiskMgr *pDiskMgr, int32_t pgid, void *pData) { + taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET); + taosReadFile(pDiskMgr->fd, pData, pDiskMgr->pgsize); + return 0; +} -int32_t tdmAllocPage(SDiskMgr *pDiskMgr) {} \ No newline at end of file +int tdmWritePage(SDiskMgr *pDiskMgr, int32_t pgid, const void *pData) { + taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET); + taosWriteFile(pDiskMgr->fd, pData, pDiskMgr->pgsize); + return 0; +} + +int32_t tdmAllocPage(SDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; } \ No newline at end of file From eb7151f73e5dfc3bbc934e1e58dd36b9dfa5b2c5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 7 Jan 2022 10:57:39 +0000 Subject: [PATCH 3/3] more --- include/os/osFile.h | 2 +- source/os/src/osFile.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/os/osFile.h b/include/os/osFile.h index 8d03759d82..2b0abc60ae 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -54,7 +54,7 @@ int32_t taosFtruncateFile(FileFd fd, int64_t length); int32_t taosFsyncFile(FileFd fd); int64_t taosReadFile(FileFd fd, void *buf, int64_t count); -int64_t taosWriteFile(FileFd fd, void *buf, int64_t count); +int64_t taosWriteFile(FileFd fd, const void *buf, int64_t count); void taosCloseFile(FileFd fd); diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index bdf3a53dcb..60155c8fb9 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -120,7 +120,7 @@ int64_t taosReadFile(FileFd fd, void *buf, int64_t count) { return count; } -int64_t taosWriteFile(FileFd fd, void *buf, int64_t n) { +int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) { int64_t nleft = n; int64_t nwritten = 0; char * tbuf = (char *)buf;