From d89aae08a2b4f01914f3e5c1decb0a78ca6f7b83 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 8 Oct 2021 10:38:22 +0800 Subject: [PATCH 01/24] more --- include/libs/tkv/tkv.h | 59 +++++++------------------ include/server/vnode/meta/meta.h | 36 ++++++++++++--- include/server/vnode/tsdb/tsdb.h | 75 ++++++++++++++++++++++---------- 3 files changed, 98 insertions(+), 72 deletions(-) diff --git a/include/libs/tkv/tkv.h b/include/libs/tkv/tkv.h index 73c0156bd1..c31d655e43 100644 --- a/include/libs/tkv/tkv.h +++ b/include/libs/tkv/tkv.h @@ -20,52 +20,25 @@ extern "C" { #endif -typedef struct tkv_db_s tkv_db_t; +// Types exported +typedef struct STkvDb STkvDb; +typedef struct STkvOptions STkvOptions; +typedef struct STkvCache STkvCache; -typedef struct { - /* data */ -} tkv_key_t; +// DB operations +STkvDb *tkvOpen(const STkvOptions *options, const char *path); +void tkvClose(STkvDb *db); +void tkvPut(STkvDb *db, void * /*TODO*/); -typedef struct { - bool pinned; - int64_t ref; // TODO: use util library - // TODO: add a RW latch here - uint64_t offset; - void * pObj; -} tkv_obj_t; +// DB options +STkvOptions *tkvOptionsCreate(); +void tkvOptionsDestroy(STkvOptions *); +void tkvOptionsSetCache(STkvOptions *, STkvCache *); -typedef int (*tkv_key_comp_fn_t)(const tkv_key_t *, const tkv_key_t *); -typedef void (*tkv_get_key_fn_t)(const tkv_obj_t *, tkv_key_t *); -typedef int (*tkv_obj_encode_fn_t)(void **buf, void *pObj); -typedef void *(*tkv_obj_decode_fn_t)(void *buf, void **pObj); -typedef int (*tkv_obj_comp_fn_t)(const tkv_obj_t *, const tkv_obj_t *); -typedef void (*tkv_obj_destroy_fn_t)(void *); - -typedef struct { - uint64_t memLimit; - tkv_get_key_fn_t getKey; - tkv_obj_encode_fn_t encode; - tkv_obj_decode_fn_t decode; - tkv_obj_comp_fn_t compare; - tkv_obj_destroy_fn_t destroy; -} tkv_db_option_t; - -tkv_db_t * tkvOpenDB(char *dir, tkv_db_option_t *); -int tkvCloseDB(tkv_db_t *); -int tkvPut(tkv_db_t *, tkv_obj_t *); -int tkvPutBatch(tkv_db_t *, tkv_obj_t **, int); // TODO: use array here -const tkv_obj_t *tkvGet(tkv_key_t *); -int tkvGetBatch(tkv_db_t *, tkv_key_t **, int, tkv_obj_t **); // TODO: use array here -int tkvDrop(tkv_db_t *, tkv_key_t *); -int tkvDropBatch(tkv_db_t *, tkv_key_t **, int); // TODO: use array here -int tkvCommit(tkv_db_t *, void * /*TODO*/); - -typedef struct { -} tkv_db_iter_t; - -tkv_db_iter_t * tkvIterNew(tkv_db_t *); -void tkvIterFree(tkv_db_iter_t *); -const tkv_obj_t *tkvIterNext(tkv_db_iter_t *); +// DB cache +typedef enum { TKV_LRU_CACHE = 0, TKV_LFU_CACHE = 1 } ETkvCacheType; +STkvCache *tkvCacheCreate(size_t capacity, ETkvCacheType type); +void tkvCacheDestroy(STkvCache *); #ifdef __cplusplus } diff --git a/include/server/vnode/meta/meta.h b/include/server/vnode/meta/meta.h index ca7cba9b0c..259fd1fbd0 100644 --- a/include/server/vnode/meta/meta.h +++ b/include/server/vnode/meta/meta.h @@ -18,16 +18,42 @@ #include "taosMsg.h" +#include "os.h" + #ifdef __cplusplus extern "C" { #endif -typedef struct SMeta SMeta; +typedef uint64_t tuid_t; -int metaCreateTable(SMeta *pMeta, SCreateTableReq *pReq); -int metaDropTable(SMeta *pMeta, SDropTableReq *pReq); -int metaAlterTable(SMeta *pMeta, SAlterTableReq *pReq); -int metaCommit(SMeta *pMeta); +// Types exported +typedef struct SMeta SMeta; +typedef struct SMetaOptions SMetaOptions; +typedef struct SMetaQueryHandle SMetaQueryHandle; +typedef struct SMetaQueryOptions SMetaQueryOptions; + +// SMeta operations +int metaCreate(const char *path); +int metaDestroy(const char *path); +SMeta *metaOpen(SMetaOptions *); +void metaClose(SMeta *); +int metaCreateTable(SMeta *, void *); +int metaDropTable(SMeta *, uint64_t tuid_t); +int metaAlterTable(SMeta *, void *); +int metaCommit(SMeta *); + +// Options +SMetaOptions *metaOptionsCreate(); +void metaOptionsDestroy(SMetaOptions *); +void metaOptionsSetCache(SMetaOptions *, size_t capacity); + +// SMetaQueryHandle +SMetaQueryHandle *metaQueryHandleCreate(SMetaQueryOptions *); +void metaQueryHandleDestroy(SMetaQueryHandle *); + +// SMetaQueryOptions +SMetaQueryOptions *metaQueryOptionsCreate(); +void metaQueryOptionsDestroy(SMetaQueryOptions *); #ifdef __cplusplus } diff --git a/include/server/vnode/tsdb/tsdb.h b/include/server/vnode/tsdb/tsdb.h index 968bac2fa2..8b8877ff1d 100644 --- a/include/server/vnode/tsdb/tsdb.h +++ b/include/server/vnode/tsdb/tsdb.h @@ -23,33 +23,60 @@ extern "C" { #endif -typedef struct STsdb STsdb; -typedef struct { - int32_t id; // TODO: use a global definition - int32_t days; - int32_t keep; - int32_t keep1; - int32_t keep2; - int32_t minRows; - int32_t maxRows; - int8_t precision; - int8_t update; -} STsdbCfg; +// Types exported +typedef struct STsdb STsdb; +typedef struct STsdbOptions STsdbOptions; +typedef struct STsdbSMAOptions STsdbSMAOptions; // SMA stands for Small Materialized Aggregation +typedef struct STsdbReadOptions STsdbReadOptions; +typedef struct STsdbSnapshot STsdbSnapshot; +typedef struct STsdbQueryHandle STsdbQueryHandle; -// Module init and clear -int tsdbInit(); -int tsdbClear(); +// DB operations +int tsdbCreate(const char *path); +int tsdbDestroy(const char *path); +STsdb *tsdbOpen(const STsdbOptions *options); +void tsdbClose(STsdb *); +int tsdbReset(STsdb *, const STsdbOptions *); +int tsdbInsert(STsdb *, SSubmitReq *, SSubmitRsp *); +int tsdbCommit(STsdb *); +int tsdbCompact(STsdb *); -// Repository operations -int tsdbCreateRepo(int id); -int tsdbDropRepo(int id); -STsdb *tsdbOpenRepo(STsdbCfg *pCfg); -int tsdbCloseRepo(STsdb *pTsdb); -int tsdbForceCloseRepo(STsdb *pTsdb); +// Options +STsdbOptions *tsdbOptionsCreate(); +void tsdbOptionsDestroy(STsdbOptions *); +void tsdbOptionsSetId(STsdbOptions *, int id); +void tsdbOptionsSetHoursPerFile(STsdbOptions *, int hours); +void tsdbOptionsSetRetention(STsdbOptions *, int keep, int keep1, int keep2); +void tsdbOptionsSetMinAndMaxRows(STsdbOptions *, int minRows, int maxRows); +void tsdbOptionsSetPrecision(STsdbOptions *, int); +void tsdbOptionsSetCache(STsdbOptions *, int); +typedef enum { TSDB_NO_UPDATE = 0, TSDB_WHOLE_ROW_UPDATE = 1, TSDB_PARTIAL_ROW_UPDATE = 2 } ETsdbUpdateType; +void tsdbOptionsSetUpdate(STsdbOptions *, ETsdbUpdateType); +void tsdbOptionsSetSMA(STsdbOptions *, STsdbSMAOptions *); -// Data commit -int tsdbInsert(STsdb *pTsdb, SSubmitReq *pMsg); -int tsdbCommit(STsdb *pTsdb); +// STsdbSMAOptions +STsdbSMAOptions *tsdbSMAOptionsCreate(); +void tsdbSMAOptionsDestroy(STsdbSMAOptions *); +void tsdbSMAOptionsSetFuncs(STsdbSMAOptions *, SArray * /*Array of function to perform on each block*/); +void tsdbSMAOptionsSetIntervals(STsdbSMAOptions *, SArray *); +void tsdbSMAOptionsSetColTypes(STsdbSMAOptions *, SArray *); + +// STsdbQueryHandle +STsdbQueryHandle *tsdbQueryHandleCreate(STsdb *, STsdbReadOptions *); +void tsdbQueryHandleDestroy(STsdbQueryHandle *); +void tsdbResetQueryHandle(STsdbQueryHandle *, STsdbReadOptions *); +bool tsdbNextDataBlock(STsdbQueryHandle *); +void tsdbGetDataBlockInfo(STsdbQueryHandle *, SDataBlockInfo *); +void tsdbGetDataBlockStatisInfo(STsdbQueryHandle *, SDataStatis **); + +// STsdbReadOptions +STsdbReadOptions *tsdbReadOptionsCreate(); +void tsdbReadOptionsDestroy(STsdbReadOptions *); +void tsdbReadOptionsSetSnapshot(STsdbReadOptions *, STsdbSnapshot *); + +// STsdbSnapshot +STsdbSnapshot *tsdbSnapshotCreate(STsdb *); +void tsdbSnapshotDestroy(STsdbSnapshot *); #ifdef __cplusplus } From 5fc96f8ccf93fd468989ca52f11d5a088335d3ae Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 9 Oct 2021 10:17:40 +0800 Subject: [PATCH 02/24] refact --- cmake/leveldb_CMakeLists.txt.in | 13 +++++++++++++ cmake/rocksdb_CMakeLists.txt.in | 11 +++++++++++ include/server/vnode/tsdb/tsdb.h | 10 +++++----- source/server/vnode/meta/src/meta.c | 20 +------------------- source/server/vnode/tsdb/src/tsdb.c | 2 +- 5 files changed, 31 insertions(+), 25 deletions(-) create mode 100644 cmake/leveldb_CMakeLists.txt.in create mode 100644 cmake/rocksdb_CMakeLists.txt.in diff --git a/cmake/leveldb_CMakeLists.txt.in b/cmake/leveldb_CMakeLists.txt.in new file mode 100644 index 0000000000..7392f3d245 --- /dev/null +++ b/cmake/leveldb_CMakeLists.txt.in @@ -0,0 +1,13 @@ + +# leveldb +ExternalProject_Add(leveldb + GIT_REPOSITORY https://github.com/taosdata-contrib/leveldb.git + GIT_TAG master + SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/leveldb" + BINARY_DIR "" + #BUILD_IN_SOURCE TRUE + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) \ No newline at end of file diff --git a/cmake/rocksdb_CMakeLists.txt.in b/cmake/rocksdb_CMakeLists.txt.in new file mode 100644 index 0000000000..bacc48ecac --- /dev/null +++ b/cmake/rocksdb_CMakeLists.txt.in @@ -0,0 +1,11 @@ + +# rocksdb +ExternalProject_Add(rocksdb + GIT_REPOSITORY https://github.com/taosdata-contrib/rocksdb.git + GIT_TAG v6.23.3 + SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/rocksdb" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" + ) \ No newline at end of file diff --git a/include/server/vnode/tsdb/tsdb.h b/include/server/vnode/tsdb/tsdb.h index a25cb47985..09cebc7bb6 100644 --- a/include/server/vnode/tsdb/tsdb.h +++ b/include/server/vnode/tsdb/tsdb.h @@ -57,17 +57,17 @@ void tsdbOptionsSetSMA(STsdbOptions *, STsdbSMAOptions *); // STsdbSMAOptions STsdbSMAOptions *tsdbSMAOptionsCreate(); void tsdbSMAOptionsDestroy(STsdbSMAOptions *); -void tsdbSMAOptionsSetFuncs(STsdbSMAOptions *, SArray * /*Array of function to perform on each block*/); -void tsdbSMAOptionsSetIntervals(STsdbSMAOptions *, SArray *); -void tsdbSMAOptionsSetColTypes(STsdbSMAOptions *, SArray *); +// void tsdbSMAOptionsSetFuncs(STsdbSMAOptions *, SArray * /*Array of function to perform on each block*/); +// void tsdbSMAOptionsSetIntervals(STsdbSMAOptions *, SArray *); +// void tsdbSMAOptionsSetColTypes(STsdbSMAOptions *, SArray *); // STsdbQueryHandle STsdbQueryHandle *tsdbQueryHandleCreate(STsdb *, STsdbReadOptions *); void tsdbQueryHandleDestroy(STsdbQueryHandle *); void tsdbResetQueryHandle(STsdbQueryHandle *, STsdbReadOptions *); bool tsdbNextDataBlock(STsdbQueryHandle *); -void tsdbGetDataBlockInfo(STsdbQueryHandle *, SDataBlockInfo *); -void tsdbGetDataBlockStatisInfo(STsdbQueryHandle *, SDataStatis **); +// void tsdbGetDataBlockInfo(STsdbQueryHandle *, SDataBlockInfo *); +// void tsdbGetDataBlockStatisInfo(STsdbQueryHandle *, SDataStatis **); // STsdbReadOptions STsdbReadOptions *tsdbReadOptionsCreate(); diff --git a/source/server/vnode/meta/src/meta.c b/source/server/vnode/meta/src/meta.c index 7d8ec11494..d05fa1754f 100644 --- a/source/server/vnode/meta/src/meta.c +++ b/source/server/vnode/meta/src/meta.c @@ -15,22 +15,4 @@ #include "meta.h" -int metaCreateTable(SMeta *pMeta, SCreateTableReq *pReq) { - // TODO - return 0; -} - -int metaDropTable(SMeta *pMeta, SDropTableReq *pReq) { - // TODO - return 0; -} - -int metaAlterTable(SMeta *pMeta, SAlterTableReq *pReq) { - // TODO - return 0; -} - -int metaCommit(SMeta *pMeta) { - // TODO - return 0; -} \ No newline at end of file +int metaCommit(SMeta *meta) { return 0; } \ No newline at end of file diff --git a/source/server/vnode/tsdb/src/tsdb.c b/source/server/vnode/tsdb/src/tsdb.c index b79d6a850e..56637ec74e 100644 --- a/source/server/vnode/tsdb/src/tsdb.c +++ b/source/server/vnode/tsdb/src/tsdb.c @@ -15,5 +15,5 @@ #include "tsdb.h" -int tsdbInsert(STsdb *pTsdb, SSubmitReq *pMsg) { return 0; } +int tsdbInsert(STsdb *tsdb, SSubmitReq *pReq, SSubmitRsp *pRsp) { return 0; } int tsdbCommit(STsdb *pTsdb) { return 0; } \ No newline at end of file From 181dfe81def452a2671428bb187e2ae94cfad80f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 9 Oct 2021 13:20:42 +0800 Subject: [PATCH 03/24] add leveldb and rocksdb deps --- CMakeLists.txt | 12 ++++++++++++ deps/CMakeLists.txt | 25 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index ccad9e7a2d..190b693321 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,18 @@ cat("${CMAKE_SUPPORT_DIR}/zlib_CMakeLists.txt.in" ${DEPS_TMP_FILE}) ## cJson cat("${CMAKE_SUPPORT_DIR}/cjson_CMakeLists.txt.in" ${DEPS_TMP_FILE}) +## leveldb +option(BUILD_WITH_LEVELDB "If build with leveldb" ON) +if(${BUILD_WITH_LEVELDB}) + cat("${CMAKE_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${DEPS_TMP_FILE}) +endif(${BUILD_WITH_LEVELDB}) + +## rocksdb +option(BUILD_WITH_ROCKSDB "If build with rocksdb" ON) +if(${BUILD_WITH_ROCKSDB}) + cat("${CMAKE_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${DEPS_TMP_FILE}) +endif(${BUILD_WITH_ROCKSDB}) + ## download dependencies configure_file(${DEPS_TMP_FILE} "${CMAKE_SOURCE_DIR}/deps/deps-download/CMakeLists.txt") execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index 410e0c564e..a872de5fb8 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -1,7 +1,9 @@ +# googletest if(${BUILD_TEST}) add_subdirectory(googletest) endif(${BUILD_TEST}) +# cJson add_subdirectory(cJson) target_include_directories( cjson @@ -9,15 +11,38 @@ target_include_directories( PUBLIC $ ) +# lz4 add_subdirectory(lz4/build/cmake) target_include_directories( lz4_static PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib ) +# zlib add_subdirectory(zlib) target_include_directories( zlib PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/zlib ) + +# leveldb +if(${BUILD_WITH_LEVELDB}) +option(LEVELDB_BUILD_TESTS "" OFF) + add_subdirectory(leveldb) + target_include_directories( + leveldb + PUBLIC $ + ) +endif(${BUILD_WITH_LEVELDB}) + +# rocksdb +if(${BUILD_WITH_ROCKSDB}) + option(WITH_TESTS "" OFF) + option(WITH_BENCHMARK_TOOLS "" OFF) + add_subdirectory(rocksdb) + target_include_directories( + rocksdb + PUBLIC $ + ) +endif(${BUILD_WITH_ROCKSDB}) From 7ec37add5c35ca110e2b97ba0e34db1d400fdaaf Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 9 Oct 2021 13:35:24 +0800 Subject: [PATCH 04/24] supress CMP0048 warning --- CMakeLists.txt | 4 ++-- cmake/EnableCMP0048.txt.in | 1 + deps/CMakeLists.txt | 5 +++++ 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 cmake/EnableCMP0048.txt.in diff --git a/CMakeLists.txt b/CMakeLists.txt index 190b693321..65d1e133d7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,13 +36,13 @@ cat("${CMAKE_SUPPORT_DIR}/zlib_CMakeLists.txt.in" ${DEPS_TMP_FILE}) cat("${CMAKE_SUPPORT_DIR}/cjson_CMakeLists.txt.in" ${DEPS_TMP_FILE}) ## leveldb -option(BUILD_WITH_LEVELDB "If build with leveldb" ON) +option(BUILD_WITH_LEVELDB "If build with leveldb" OFF) if(${BUILD_WITH_LEVELDB}) cat("${CMAKE_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${DEPS_TMP_FILE}) endif(${BUILD_WITH_LEVELDB}) ## rocksdb -option(BUILD_WITH_ROCKSDB "If build with rocksdb" ON) +option(BUILD_WITH_ROCKSDB "If build with rocksdb" OFF) if(${BUILD_WITH_ROCKSDB}) cat("${CMAKE_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${DEPS_TMP_FILE}) endif(${BUILD_WITH_ROCKSDB}) diff --git a/cmake/EnableCMP0048.txt.in b/cmake/EnableCMP0048.txt.in new file mode 100644 index 0000000000..1b59188fd7 --- /dev/null +++ b/cmake/EnableCMP0048.txt.in @@ -0,0 +1 @@ +cmake_policy(SET CMP0048 NEW) \ No newline at end of file diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index a872de5fb8..50225da82b 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -4,12 +4,15 @@ if(${BUILD_TEST}) endif(${BUILD_TEST}) # cJson +# see https://stackoverflow.com/questions/37582508/silence-cmp0048-warnings-in-vendored-projects +set(CMAKE_PROJECT_INCLUDE_BEFORE "${CMAKE_SUPPORT_DIR}/EnableCMP0048.txt.in") add_subdirectory(cJson) target_include_directories( cjson # see https://stackoverflow.com/questions/25676277/cmake-target-include-directories-prints-an-error-when-i-try-to-add-the-source PUBLIC $ ) +unset(CMAKE_PROJECT_INCLUDE_BEFORE) # lz4 add_subdirectory(lz4/build/cmake) @@ -19,12 +22,14 @@ target_include_directories( ) # zlib +set(CMAKE_PROJECT_INCLUDE_BEFORE "${CMAKE_SUPPORT_DIR}/EnableCMP0048.txt.in") add_subdirectory(zlib) target_include_directories( zlib PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/zlib ) +unset(CMAKE_PROJECT_INCLUDE_BEFORE) # leveldb if(${BUILD_WITH_LEVELDB}) From f762d037911d448c22db37d87cb6d7c2e6541346 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 9 Oct 2021 13:50:09 +0800 Subject: [PATCH 05/24] build cjson static library --- deps/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index 50225da82b..e28651824a 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -6,6 +6,9 @@ endif(${BUILD_TEST}) # cJson # see https://stackoverflow.com/questions/37582508/silence-cmp0048-warnings-in-vendored-projects set(CMAKE_PROJECT_INCLUDE_BEFORE "${CMAKE_SUPPORT_DIR}/EnableCMP0048.txt.in") +option(ENABLE_CJSON_TEST "Enable building cJSON test" OFF) +option(CJSON_OVERRIDE_BUILD_SHARED_LIBS "Override BUILD_SHARED_LIBS with CJSON_BUILD_SHARED_LIBS" ON) +option(CJSON_BUILD_SHARED_LIBS "Overrides BUILD_SHARED_LIBS if CJSON_OVERRIDE_BUILD_SHARED_LIBS is enabled" OFF) add_subdirectory(cJson) target_include_directories( cjson From a7e15c32c1cf3531371a79ea09244e2a95d8c551 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 9 Oct 2021 16:38:56 +0800 Subject: [PATCH 06/24] finish vma --- source/server/vnode/inc/vnodeMemAllocator.h | 29 ++++- source/server/vnode/src/vnodeMemAllocator.c | 112 +++++++++++++++++++- 2 files changed, 139 insertions(+), 2 deletions(-) diff --git a/source/server/vnode/inc/vnodeMemAllocator.h b/source/server/vnode/inc/vnodeMemAllocator.h index 9b3d27776d..eabccc689e 100644 --- a/source/server/vnode/inc/vnodeMemAllocator.h +++ b/source/server/vnode/inc/vnodeMemAllocator.h @@ -16,12 +16,39 @@ #ifndef _TD_VNODE_MEM_ALLOCATOR_H_ #define _TD_VNODE_MEM_ALLOCATOR_H_ -#include "amalloc.h" +#include "os.h" #ifdef __cplusplus extern "C" { #endif +typedef struct SVMANode SVMANode; +typedef struct SVnodeMemAllocator SVnodeMemAllocator; + +SVnodeMemAllocator *VMACreate(size_t size /* base size */, size_t ssize /* step size */, + size_t threshold /* threshold size when full*/); +void VMADestroy(SVnodeMemAllocator *pvma); +void VMAReset(SVnodeMemAllocator *pvma); +void * VMAMalloc(SVnodeMemAllocator *pvma, size_t size); +void VMAFree(SVnodeMemAllocator *pvma, void *ptr); +bool VMAIsFull(SVnodeMemAllocator *pvma); + +// ------------------ FOR TEST ONLY ------------------ +struct SVMANode { + struct SVMANode *prev; + size_t tsize; + size_t used; + char data[]; +}; + +struct SVnodeMemAllocator { + bool full; // if allocator is full + size_t threshold; // threshold; + size_t ssize; // step size to allocate + SVMANode *inuse; // inuse node to allocate + SVMANode node; // basic node to use +}; + #ifdef __cplusplus } #endif diff --git a/source/server/vnode/src/vnodeMemAllocator.c b/source/server/vnode/src/vnodeMemAllocator.c index 6dea4a4e57..29909df491 100644 --- a/source/server/vnode/src/vnodeMemAllocator.c +++ b/source/server/vnode/src/vnodeMemAllocator.c @@ -11,4 +11,114 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "vnodeMemAllocator.h" + +#define VMA_IS_FULL(pvma) \ + (((pvma)->inuse != &((pvma)->node)) || ((pvma)->inuse->tsize - (pvma)->inuse->used < (pvma)->threshold)) + +static SVMANode *VMANodeNew(size_t size); +static void VMANodeFree(SVMANode *node); + +SVnodeMemAllocator *VMACreate(size_t size, size_t ssize, size_t threshold) { + SVnodeMemAllocator *pvma = NULL; + + if (size < threshold) { + return NULL; + } + + pvma = (SVnodeMemAllocator *)malloc(sizeof(*pvma) + size); + if (pvma) { + pvma->full = false; + pvma->threshold = threshold; + pvma->ssize = ssize; + pvma->inuse = &(pvma->node); + + pvma->inuse->prev = NULL; + pvma->inuse->tsize = size; + pvma->inuse->used = 0; + } + + return pvma; +} + +void VMADestroy(SVnodeMemAllocator *pvma) { + if (pvma) { + VMAReset(pvma); + free(pvma); + } +} + +void VMAReset(SVnodeMemAllocator *pvma) { + while (pvma->inuse != &(pvma->node)) { + SVMANode *node = pvma->inuse; + pvma->inuse = node->prev; + VMANodeFree(node); + } + + pvma->inuse->used = 0; + pvma->full = false; +} + +void *VMAMalloc(SVnodeMemAllocator *pvma, size_t size) { + void * ptr = NULL; + size_t tsize = size + sizeof(size_t); + + if (pvma->inuse->tsize - pvma->inuse->used < tsize) { + SVMANode *pNode = VMANodeNew(MAX(pvma->ssize, tsize)); + if (pNode == NULL) { + return NULL; + } + + pNode->prev = pvma->inuse; + pvma->inuse = pNode; + } + + ptr = pvma->inuse->data + pvma->inuse->used; + pvma->inuse->used += tsize; + *(size_t *)ptr = size; + ptr = POINTER_SHIFT(ptr, sizeof(size_t)); + + pvma->full = VMA_IS_FULL(pvma); + + return ptr; +} + +void VMAFree(SVnodeMemAllocator *pvma, void *ptr) { + if (ptr) { + size_t size = *(size_t *)POINTER_SHIFT(ptr, -sizeof(size_t)); + if (POINTER_SHIFT(ptr, size) == pvma->inuse->data + pvma->inuse->used) { + pvma->inuse->used -= (size + sizeof(size_t)); + + if ((pvma->inuse->used == 0) && (pvma->inuse != &(pvma->node))) { + SVMANode *node = pvma->inuse; + pvma->inuse = node->prev; + VMANodeFree(node); + } + + pvma->full = VMA_IS_FULL(pvma); + } + } +} + +bool VMAIsFull(SVnodeMemAllocator *pvma) { return pvma->full; } + +static SVMANode *VMANodeNew(size_t size) { + SVMANode *node = NULL; + + node = (SVMANode *)malloc(sizeof(*node) + size); + if (node) { + node->prev = NULL; + node->tsize = size; + node->used = 0; + } + + return node; +} + +static void VMANodeFree(SVMANode *node) { + if (node) { + free(node); + } +} \ No newline at end of file From 0eb6d29210b85261724cf52aad3b7fa03b211b20 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 9 Oct 2021 16:42:44 +0800 Subject: [PATCH 07/24] refact --- source/server/vnode/inc/vnodeMemAllocator.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/server/vnode/inc/vnodeMemAllocator.h b/source/server/vnode/inc/vnodeMemAllocator.h index eabccc689e..ca00abc70a 100644 --- a/source/server/vnode/inc/vnodeMemAllocator.h +++ b/source/server/vnode/inc/vnodeMemAllocator.h @@ -22,7 +22,6 @@ extern "C" { #endif -typedef struct SVMANode SVMANode; typedef struct SVnodeMemAllocator SVnodeMemAllocator; SVnodeMemAllocator *VMACreate(size_t size /* base size */, size_t ssize /* step size */, @@ -34,12 +33,12 @@ void VMAFree(SVnodeMemAllocator *pvma, void *ptr); bool VMAIsFull(SVnodeMemAllocator *pvma); // ------------------ FOR TEST ONLY ------------------ -struct SVMANode { +typedef struct SVMANode { struct SVMANode *prev; size_t tsize; size_t used; char data[]; -}; +} SVMANode; struct SVnodeMemAllocator { bool full; // if allocator is full From 34f7051eec85f5643bf6f15ed74710bcd8f3dfa4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 9 Oct 2021 17:13:40 +0800 Subject: [PATCH 08/24] add VMATest --- source/server/vnode/CMakeLists.txt | 7 ++++++- source/server/vnode/test/CMakeLists.txt | 9 +++++++++ source/server/vnode/test/vnodeMemAllocatorTest.cpp | 8 ++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 source/server/vnode/test/CMakeLists.txt create mode 100644 source/server/vnode/test/vnodeMemAllocatorTest.cpp diff --git a/source/server/vnode/CMakeLists.txt b/source/server/vnode/CMakeLists.txt index 03421b2628..20b50bf244 100644 --- a/source/server/vnode/CMakeLists.txt +++ b/source/server/vnode/CMakeLists.txt @@ -15,4 +15,9 @@ target_link_libraries( PUBLIC meta PUBLIC tq PUBLIC tsdb -) \ No newline at end of file +) + +# test +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) diff --git a/source/server/vnode/test/CMakeLists.txt b/source/server/vnode/test/CMakeLists.txt new file mode 100644 index 0000000000..fad366e6ef --- /dev/null +++ b/source/server/vnode/test/CMakeLists.txt @@ -0,0 +1,9 @@ +# vnodeMemAllocatorTest +add_executable(VMATest "") +target_sources(VMATest + PRIVATE + "../src/vnodeMemAllocator.c" + "vnodeMemAllocatorTest.cpp" +) +target_include_directories(VMATest PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../inc") +target_link_libraries(VMATest os gtest_main) \ No newline at end of file diff --git a/source/server/vnode/test/vnodeMemAllocatorTest.cpp b/source/server/vnode/test/vnodeMemAllocatorTest.cpp new file mode 100644 index 0000000000..4de717d80d --- /dev/null +++ b/source/server/vnode/test/vnodeMemAllocatorTest.cpp @@ -0,0 +1,8 @@ +#include +#include + +#include "vnodeMemAllocator.h" + +TEST(VMATest, basic_create_and_destroy_test) { + std::cout << "Hello, this is VMA test" << std::endl; +} \ No newline at end of file From 5787ff052d30e4b6f31a52e51d2ba2e2f9651488 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 11 Oct 2021 09:54:36 +0800 Subject: [PATCH 09/24] add test --- .../server/vnode/test/vnodeMemAllocatorTest.cpp | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/source/server/vnode/test/vnodeMemAllocatorTest.cpp b/source/server/vnode/test/vnodeMemAllocatorTest.cpp index 4de717d80d..f2d07e6aa5 100644 --- a/source/server/vnode/test/vnodeMemAllocatorTest.cpp +++ b/source/server/vnode/test/vnodeMemAllocatorTest.cpp @@ -4,5 +4,19 @@ #include "vnodeMemAllocator.h" TEST(VMATest, basic_create_and_destroy_test) { - std::cout << "Hello, this is VMA test" << std::endl; + SVnodeMemAllocator *vma = VMACreate(1024, 512, 64); + EXPECT_TRUE(vma != nullptr); + EXPECT_EQ(vma->full, false); + EXPECT_EQ(vma->ssize, 512); + EXPECT_EQ(vma->threshold, 64); + EXPECT_EQ(vma->inuse->tsize, 1024); + VMADestroy(vma); + + vma = VMACreate(1024, 512, 1024); + EXPECT_TRUE(vma != nullptr); + VMADestroy(vma); + + vma = VMACreate(1024, 512, 1025); + EXPECT_TRUE(vma == nullptr); + VMADestroy(vma); } \ No newline at end of file From b50676b6c03bfa6b275f072141b97edcbb6b1633 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 11 Oct 2021 15:41:49 +0800 Subject: [PATCH 10/24] refact --- CMakeLists.txt | 2 +- deps/CMakeLists.txt | 1 + include/common/trow.h | 7 +++ source/server/vnode/CMakeLists.txt | 2 +- source/server/vnode/meta/CMakeLists.txt | 9 ++- source/server/vnode/meta/src/meta.c | 66 ++++++++++++++++++++ source/server/vnode/meta/test/CMakeLists.txt | 18 ++++++ source/server/vnode/meta/test/metaTests.cpp | 8 +++ 8 files changed, 109 insertions(+), 4 deletions(-) create mode 100644 source/server/vnode/meta/test/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 65d1e133d7..f27f5f1672 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,7 +42,7 @@ if(${BUILD_WITH_LEVELDB}) endif(${BUILD_WITH_LEVELDB}) ## rocksdb -option(BUILD_WITH_ROCKSDB "If build with rocksdb" OFF) +option(BUILD_WITH_ROCKSDB "If build with rocksdb" ON) if(${BUILD_WITH_ROCKSDB}) cat("${CMAKE_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${DEPS_TMP_FILE}) endif(${BUILD_WITH_ROCKSDB}) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index e28651824a..05c154af86 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -48,6 +48,7 @@ endif(${BUILD_WITH_LEVELDB}) if(${BUILD_WITH_ROCKSDB}) option(WITH_TESTS "" OFF) option(WITH_BENCHMARK_TOOLS "" OFF) + option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF) add_subdirectory(rocksdb) target_include_directories( rocksdb diff --git a/include/common/trow.h b/include/common/trow.h index be4b7af32a..6094425bbf 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -20,6 +20,13 @@ extern "C" { #endif +typedef struct SRow SRow; + +#define rowType(r) +#define rowLen(r) +#define rowVersion(r) +#define rowNCols(r) + #ifdef __cplusplus } #endif diff --git a/source/server/vnode/CMakeLists.txt b/source/server/vnode/CMakeLists.txt index 20b50bf244..5e11e45567 100644 --- a/source/server/vnode/CMakeLists.txt +++ b/source/server/vnode/CMakeLists.txt @@ -3,7 +3,7 @@ add_subdirectory(tq) add_subdirectory(tsdb) aux_source_directory(src VNODE_SRC) -add_library(vnode ${VNODE_SRC}) +add_library(vnode STATIC ${VNODE_SRC}) target_include_directories( vnode PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode" diff --git a/source/server/vnode/meta/CMakeLists.txt b/source/server/vnode/meta/CMakeLists.txt index 832e13a155..113bcd5d6f 100644 --- a/source/server/vnode/meta/CMakeLists.txt +++ b/source/server/vnode/meta/CMakeLists.txt @@ -1,5 +1,5 @@ aux_source_directory(src META_SRC) -add_library(meta ${META_SRC}) +add_library(meta STATIC ${META_SRC}) target_include_directories( meta PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode/meta" @@ -8,4 +8,9 @@ target_include_directories( target_link_libraries( meta PUBLIC common -) \ No newline at end of file +) +target_link_libraries(meta PUBLIC rocksdb) + +# if(${BUILD_TEST}) +# add_subdirectory(test) +# endif(${BUILD_TEST}) diff --git a/source/server/vnode/meta/src/meta.c b/source/server/vnode/meta/src/meta.c index d05fa1754f..f35de5eb9e 100644 --- a/source/server/vnode/meta/src/meta.c +++ b/source/server/vnode/meta/src/meta.c @@ -13,6 +13,72 @@ * along with this program. If not, see . */ +#include + +#include "thash.h" +#include "tlist.h" +#include "tlockfree.h" +#include "ttypes.h" + #include "meta.h" +typedef struct STable { + uint64_t uid; + tstr * name; + uint64_t suid; + SArray * schema; +} STable; + +typedef struct STableObj { + bool pin; + uint64_t ref; + SRWLatch latch; + uint64_t offset; + SList * ctbList; // child table list + STable * pTable; +} STableObj; + +struct SMeta { + pthread_rwlock_t rwLock; + + SHashObj * pTableObjHash; // uid --> STableObj + SList * stbList; // super table list + rocksdb_t *tbnameDb; // tbname --> uid + rocksdb_t *tagDb; // uid --> tag + rocksdb_t *schemaDb; + size_t totalUsed; +}; + +SMeta *metaOpen(SMetaOptions *options) { + SMeta *pMeta = NULL; + char * err = NULL; + + pMeta = (SMeta *)calloc(1, sizeof(*pMeta)); + if (pMeta == NULL) { + return NULL; + } + + pthread_rwlock_init(&(pMeta->rwLock), NULL); + + pMeta->pTableObjHash = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + + pMeta->stbList = tdListNew(sizeof(STableObj *)); + + // Open tbname DB + rocksdb_options_t *tbnameDbOptions = rocksdb_options_create(); + pMeta->tbnameDb = rocksdb_open(tbnameDbOptions, "tbname_uid_db", &err); + + // Open tag DB + pMeta->tagDb = rocksdb_open(tbnameDbOptions, "uid_tag_db", &err); + + // Open schema DB + pMeta->schemaDb = rocksdb_open(tbnameDbOptions, "schema_db", &err); + + return pMeta; +} + +void metaClose(SMeta *pMeta) { + // TODO +} + int metaCommit(SMeta *meta) { return 0; } \ No newline at end of file diff --git a/source/server/vnode/meta/test/CMakeLists.txt b/source/server/vnode/meta/test/CMakeLists.txt new file mode 100644 index 0000000000..ca82d3fb83 --- /dev/null +++ b/source/server/vnode/meta/test/CMakeLists.txt @@ -0,0 +1,18 @@ +add_executable(metaTest "") +target_sources(metaTest + PRIVATE + "../src/meta.c" + "metaTests.cpp" +) +target_include_directories(metaTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/server/vnode/meta" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_link_libraries(metaTest + os + util + common + rocksdb + gtest_main +) \ No newline at end of file diff --git a/source/server/vnode/meta/test/metaTests.cpp b/source/server/vnode/meta/test/metaTests.cpp index e69de29bb2..c62ae0aa02 100644 --- a/source/server/vnode/meta/test/metaTests.cpp +++ b/source/server/vnode/meta/test/metaTests.cpp @@ -0,0 +1,8 @@ +#include +#include + +#include "meta.h" + +TEST(MetaTest, meta_open_test) { + std::cout << "Hello META!" << std::endl; +} \ No newline at end of file From 700cd741055151a6d57109f850b1ac73d3c47b75 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 11 Oct 2021 16:24:10 +0800 Subject: [PATCH 11/24] make compile --- source/server/vnode/inc/vnodeInt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/server/vnode/inc/vnodeInt.h b/source/server/vnode/inc/vnodeInt.h index 5f07c5819c..b2512b2892 100644 --- a/source/server/vnode/inc/vnodeInt.h +++ b/source/server/vnode/inc/vnodeInt.h @@ -79,7 +79,7 @@ typedef struct { void * cq; // continuous query int32_t dbCfgVersion; int32_t vgCfgVersion; - STsdbCfg tsdbCfg; + // STsdbCfg tsdbCfg; #if 0 SSyncCfg syncCfg; #endif From 1b6c7a5923b683a29752d42dcfb073728f5caed3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 11 Oct 2021 18:36:40 +0800 Subject: [PATCH 12/24] make compile --- include/common/tcompare.h | 36 +++ include/common/ttypes.h | 6 +- include/util/{tcompare.h => compare.h} | 19 +- include/util/tdef.h | 27 -- include/util/types.h | 88 ++++++ source/common/src/tcompare.c | 384 +---------------------- source/server/CMakeLists.txt | 1 + source/server/vnode/meta/CMakeLists.txt | 6 +- source/util/CMakeLists.txt | 2 +- source/util/src/compare.c | 398 ++++++++++++++++++++++++ source/{common => util}/src/terror.c | 0 source/util/src/thashutil.c | 3 +- source/util/src/tskiplist.c | 2 +- source/util/src/tutil.c | 21 -- 14 files changed, 544 insertions(+), 449 deletions(-) create mode 100644 include/common/tcompare.h rename include/util/{tcompare.h => compare.h} (79%) create mode 100644 include/util/types.h create mode 100644 source/util/src/compare.c rename source/{common => util}/src/terror.c (100%) diff --git a/include/common/tcompare.h b/include/common/tcompare.h new file mode 100644 index 0000000000..8476a79e92 --- /dev/null +++ b/include/common/tcompare.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TCOMPARE_H_ +#define _TD_TCOMPARE_H_ + +#include "compare.h" +#include "ttypes.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t compareStrPatternComp(const void* pLeft, const void* pRight); +int32_t compareWStrPatternComp(const void* pLeft, const void* pRight); +__compar_fn_t getComparFunc(int32_t type, int32_t optr); +__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order); +int32_t doCompare(const char* a, const char* b, int32_t type, size_t size); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TCOMPARE_H_*/ \ No newline at end of file diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 6fe6e11d05..991bb5c6be 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -6,10 +6,10 @@ extern "C" { #endif #include "taosdef.h" +#include "types.h" // ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR typedef int32_t VarDataOffsetT; -typedef int16_t VarDataLenT; // maxVarDataLen: 32767 typedef uint16_t TDRowLenT; // not including overhead: 0 ~ 65535 typedef uint32_t TDRowTLenT; // total length, including overhead @@ -30,11 +30,7 @@ typedef struct { } SNCharNullT; #pragma pack(pop) -#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) - -#define varDataLen(v) ((VarDataLenT *)(v))[0] #define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) -#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE)) #define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v)) #define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE)) #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len)) diff --git a/include/util/tcompare.h b/include/util/compare.h similarity index 79% rename from include/util/tcompare.h rename to include/util/compare.h index 4b2c583b4b..461552ca7b 100644 --- a/include/util/tcompare.h +++ b/include/util/compare.h @@ -46,11 +46,6 @@ int patternMatch(const char *pattern, const char *str, size_t size, const SPatte int WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo); -int32_t doCompare(const char* a, const char* b, int32_t type, size_t size); - -__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order); - -__compar_fn_t getComparFunc(int32_t type, int32_t optr); int32_t taosArrayCompareString(const void* a, const void* b); @@ -82,12 +77,22 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight); int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight); int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight); -int32_t compareStrPatternComp(const void* pLeft, const void* pRight); int32_t compareStrRegexComp(const void* pLeft, const void* pRight); int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight); int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight); int32_t compareFindItemInSet(const void *pLeft, const void* pRight); -int32_t compareWStrPatternComp(const void* pLeft, const void* pRight); +int32_t compareInt8ValDesc(const void *pLeft, const void *pRight); +int32_t compareInt16ValDesc(const void* pLeft, const void* pRight); +int32_t compareInt32ValDesc(const void* pLeft, const void* pRight); +int32_t compareInt64ValDesc(const void* pLeft, const void* pRight); +int32_t compareFloatValDesc(const void* pLeft, const void* pRight); +int32_t compareDoubleValDesc(const void* pLeft, const void* pRight); +int32_t compareUint8ValDesc(const void* pLeft, const void* pRight); +int32_t compareUint16ValDesc(const void* pLeft, const void* pRight); +int32_t compareUint32ValDesc(const void* pLeft, const void* pRight); +int32_t compareUint64ValDesc(const void* pLeft, const void* pRight); +int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight); +int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight); #ifdef __cplusplus } diff --git a/include/util/tdef.h b/include/util/tdef.h index a24d2bc911..008c9215ba 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -110,33 +110,6 @@ do { \ (src) = (void *)((char *)src + sizeof(type));\ } while(0) -#define GET_INT8_VAL(x) (*(int8_t *)(x)) -#define GET_INT16_VAL(x) (*(int16_t *)(x)) -#define GET_INT32_VAL(x) (*(int32_t *)(x)) -#define GET_INT64_VAL(x) (*(int64_t *)(x)) -#define GET_UINT8_VAL(x) (*(uint8_t*) (x)) -#define GET_UINT16_VAL(x) (*(uint16_t *)(x)) -#define GET_UINT32_VAL(x) (*(uint32_t *)(x)) -#define GET_UINT64_VAL(x) (*(uint64_t *)(x)) - -#ifdef _TD_ARM_32 - float taos_align_get_float(const char* pBuf); - double taos_align_get_double(const char* pBuf); - - #define GET_FLOAT_VAL(x) taos_align_get_float(x) - #define GET_DOUBLE_VAL(x) taos_align_get_double(x) - #define SET_FLOAT_VAL(x, y) { float z = (float)(y); (*(int32_t*) x = *(int32_t*)(&z)); } - #define SET_DOUBLE_VAL(x, y) { double z = (double)(y); (*(int64_t*) x = *(int64_t*)(&z)); } - #define SET_FLOAT_PTR(x, y) { (*(int32_t*) x = *(int32_t*)y); } - #define SET_DOUBLE_PTR(x, y) { (*(int64_t*) x = *(int64_t*)y); } -#else - #define GET_FLOAT_VAL(x) (*(float *)(x)) - #define GET_DOUBLE_VAL(x) (*(double *)(x)) - #define SET_FLOAT_VAL(x, y) { (*(float *)(x)) = (float)(y); } - #define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y); } - #define SET_FLOAT_PTR(x, y) { (*(float *)(x)) = (*(float *)(y)); } - #define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); } -#endif // TODO: check if below is necessary #define TSDB_RELATION_INVALID 0 diff --git a/include/util/types.h b/include/util/types.h new file mode 100644 index 0000000000..35cb614f6f --- /dev/null +++ b/include/util/types.h @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TYPES_H_ +#define _TD_TYPES_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define GET_INT8_VAL(x) (*(int8_t *)(x)) +#define GET_INT16_VAL(x) (*(int16_t *)(x)) +#define GET_INT32_VAL(x) (*(int32_t *)(x)) +#define GET_INT64_VAL(x) (*(int64_t *)(x)) +#define GET_UINT8_VAL(x) (*(uint8_t*) (x)) +#define GET_UINT16_VAL(x) (*(uint16_t *)(x)) +#define GET_UINT32_VAL(x) (*(uint32_t *)(x)) +#define GET_UINT64_VAL(x) (*(uint64_t *)(x)) + +static FORCE_INLINE float taos_align_get_float(const char *pBuf) { +#if __STDC_VERSION__ >= 201112L + static_assert(sizeof(float) == sizeof(uint32_t), "sizeof(float) must equal to sizeof(uint32_t)"); +#else + assert(sizeof(float) == sizeof(uint32_t)); +#endif + float fv = 0; + memcpy(&fv, pBuf, sizeof(fv)); // in ARM, return *((const float*)(pBuf)) may cause problem + return fv; +} + +static FORCE_INLINE double taos_align_get_double(const char *pBuf) { +#if __STDC_VERSION__ >= 201112L + static_assert(sizeof(double) == sizeof(uint64_t), "sizeof(double) must equal to sizeof(uint64_t)"); +#else + assert(sizeof(double) == sizeof(uint64_t)); +#endif + double dv = 0; + memcpy(&dv, pBuf, sizeof(dv)); // in ARM, return *((const double*)(pBuf)) may cause problem + return dv; +} + +// #ifdef _TD_ARM_32 +// float taos_align_get_float(const char* pBuf); +// double taos_align_get_double(const char* pBuf); + +// #define GET_FLOAT_VAL(x) taos_align_get_float(x) +// #define GET_DOUBLE_VAL(x) taos_align_get_double(x) +// #define SET_FLOAT_VAL(x, y) { float z = (float)(y); (*(int32_t*) x = *(int32_t*)(&z)); } +// #define SET_DOUBLE_VAL(x, y) { double z = (double)(y); (*(int64_t*) x = *(int64_t*)(&z)); } +// #define SET_FLOAT_PTR(x, y) { (*(int32_t*) x = *(int32_t*)y); } +// #define SET_DOUBLE_PTR(x, y) { (*(int64_t*) x = *(int64_t*)y); } +// #else + #define GET_FLOAT_VAL(x) (*(float *)(x)) + #define GET_DOUBLE_VAL(x) (*(double *)(x)) + #define SET_FLOAT_VAL(x, y) { (*(float *)(x)) = (float)(y); } + #define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y); } + #define SET_FLOAT_PTR(x, y) { (*(float *)(x)) = (*(float *)(y)); } + #define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); } +// #endif + +typedef int16_t VarDataLenT; // maxVarDataLen: 32767 +#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) + +#define varDataLen(v) ((VarDataLenT *)(v))[0] +#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE)) + +typedef int32_t VarDataOffsetT; +typedef int16_t VarDataLenT; // maxVarDataLen: 32767 + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TYPES_H_*/ \ No newline at end of file diff --git a/source/common/src/tcompare.c b/source/common/src/tcompare.c index e8e9ae3d68..3c0e6df735 100644 --- a/source/common/src/tcompare.c +++ b/source/common/src/tcompare.c @@ -12,334 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#define _BSD_SOURCE -#define _GNU_SOURCE -#define _XOPEN_SOURCE -#define _DEFAULT_SOURCE -#include "os.h" #include "tcompare.h" -#include "ulog.h" -#include "thash.h" -#include "regex.h" -#include "ttypes.h" - -int32_t setCompareBytes1(const void *pLeft, const void *pRight) { - return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; -} - -int32_t setCompareBytes2(const void *pLeft, const void *pRight) { - return NULL != taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0; -} - -int32_t setCompareBytes4(const void *pLeft, const void *pRight) { - return NULL != taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0; -} - -int32_t setCompareBytes8(const void *pLeft, const void *pRight) { - return NULL != taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0; -} - -int32_t compareInt8Val(const void *pLeft, const void *pRight) { - int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight); - if (left > right) return 1; - if (left < right) return -1; - return 0; -} - -int32_t compareInt8ValDesc(const void *pLeft, const void *pRight) { - return compareInt8Val(pRight, pLeft); -} - -int32_t compareInt16Val(const void *pLeft, const void *pRight) { - int16_t left = GET_INT16_VAL(pLeft), right = GET_INT16_VAL(pRight); - if (left > right) return 1; - if (left < right) return -1; - return 0; -} - -int32_t compareInt16ValDesc(const void* pLeft, const void* pRight) { - return compareInt16Val(pRight, pLeft); -} - -int32_t compareInt32Val(const void *pLeft, const void *pRight) { - int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight); - if (left > right) return 1; - if (left < right) return -1; - return 0; -} - -int32_t compareInt32ValDesc(const void* pLeft, const void* pRight) { - return compareInt32Val(pRight, pLeft); -} - -int32_t compareInt64Val(const void *pLeft, const void *pRight) { - int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight); - if (left > right) return 1; - if (left < right) return -1; - return 0; -} - -int32_t compareInt64ValDesc(const void* pLeft, const void* pRight) { - return compareInt64Val(pRight, pLeft); -} - -int32_t compareUint32Val(const void *pLeft, const void *pRight) { - uint32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight); - if (left > right) return 1; - if (left < right) return -1; - return 0; -} - -int32_t compareUint32ValDesc(const void* pLeft, const void* pRight) { - return compareUint32Val(pRight, pLeft); -} - -int32_t compareUint64Val(const void *pLeft, const void *pRight) { - uint64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight); - if (left > right) return 1; - if (left < right) return -1; - return 0; -} - -int32_t compareUint64ValDesc(const void* pLeft, const void* pRight) { - return compareUint64Val(pRight, pLeft); -} - -int32_t compareUint16Val(const void *pLeft, const void *pRight) { - uint16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight); - if (left > right) return 1; - if (left < right) return -1; - return 0; -} - -int32_t compareUint16ValDesc(const void* pLeft, const void* pRight) { - return compareUint16Val(pRight, pLeft); -} - -int32_t compareUint8Val(const void* pLeft, const void* pRight) { - uint8_t left = GET_UINT8_VAL(pLeft), right = GET_UINT8_VAL(pRight); - if (left > right) return 1; - if (left < right) return -1; - return 0; -} - -int32_t compareUint8ValDesc(const void* pLeft, const void* pRight) { - return compareUint8Val(pRight, pLeft); -} - -int32_t compareFloatVal(const void *pLeft, const void *pRight) { - float p1 = GET_FLOAT_VAL(pLeft); - float p2 = GET_FLOAT_VAL(pRight); - - if (isnan(p1) && isnan(p2)) { - return 0; - } - - if (isnan(p1)) { - return -1; - } - - if (isnan(p2)) { - return 1; - } - if (FLT_EQUAL(p1, p2)) { - return 0; - } - return FLT_GREATER(p1, p2) ? 1: -1; -} - -int32_t compareFloatValDesc(const void* pLeft, const void* pRight) { - return compareFloatVal(pRight, pLeft); -} - -int32_t compareDoubleVal(const void *pLeft, const void *pRight) { - double p1 = GET_DOUBLE_VAL(pLeft); - double p2 = GET_DOUBLE_VAL(pRight); - - if (isnan(p1) && isnan(p2)) { - return 0; - } - - if (isnan(p1)) { - return -1; - } - - if (isnan(p2)) { - return 1; - } - if (FLT_EQUAL(p1, p2)) { - return 0; - } - return FLT_GREATER(p1, p2) ? 1: -1; -} - -int32_t compareDoubleValDesc(const void* pLeft, const void* pRight) { - return compareDoubleVal(pRight, pLeft); -} - -int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) { - int32_t len1 = varDataLen(pLeft); - int32_t len2 = varDataLen(pRight); - - if (len1 != len2) { - return len1 > len2? 1:-1; - } else { - int32_t ret = strncmp(varDataVal(pLeft), varDataVal(pRight), len1); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1:-1; - } - } -} - -int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight) { - return compareLenPrefixedStr(pRight, pLeft); -} - -int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) { - int32_t len1 = varDataLen(pLeft); - int32_t len2 = varDataLen(pRight); - - if (len1 != len2) { - return len1 > len2? 1:-1; - } else { - int32_t ret = memcmp((wchar_t*) pLeft, (wchar_t*) pRight, len1); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } - } -} - -int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight) { - return compareLenPrefixedWStr(pRight, pLeft); -} - -/* - * Compare two strings - * TSDB_MATCH: Match - * TSDB_NOMATCH: No match - * TSDB_NOWILDCARDMATCH: No match in spite of having * or % wildcards. - * Like matching rules: - * '%': Matches zero or more characters - * '_': Matches one character - * - */ -int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) { - char c, c1; - - int32_t i = 0; - int32_t j = 0; - int32_t o = 0; - int32_t m = 0; - - while ((c = patterStr[i++]) != 0) { - if (c == pInfo->matchAll) { /* Match "*" */ - - while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) { - if (c == pInfo->matchOne) { - if (j > size || str[j++] == 0) { - // empty string, return not match - return TSDB_PATTERN_NOWILDCARDMATCH; - } else { - ++o; - } - } - } - - if (c == 0) { - return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */ - } - - char next[3] = {toupper(c), tolower(c), 0}; - m = o; - while (1) { - size_t n = strcspn(str + m, next); - str += m + n; - - if (str[0] == 0 || (n >= size)) { - break; - } - - int32_t ret = patternMatch(&patterStr[i], ++str, size - n - 1, pInfo); - if (ret != TSDB_PATTERN_NOMATCH) { - return ret; - } - m = 0; - } - return TSDB_PATTERN_NOWILDCARDMATCH; - } - - c1 = str[j++]; - ++o; - - if (j <= size) { - if (c == '\\' && patterStr[i] == '_' && c1 == '_') { i++; continue; } - if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) { - continue; - } - } - - return TSDB_PATTERN_NOMATCH; - } - - return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; -} - -int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) { - wchar_t c, c1; - wchar_t matchOne = L'_'; // "_" - wchar_t matchAll = L'%'; // "%" - - int32_t i = 0; - int32_t j = 0; - - while ((c = patterStr[i++]) != 0) { - if (c == matchAll) { /* Match "%" */ - - while ((c = patterStr[i++]) == matchAll || c == matchOne) { - if (c == matchOne && (j >= size || str[j++] == 0)) { - return TSDB_PATTERN_NOWILDCARDMATCH; - } - } - if (c == 0) { - return TSDB_PATTERN_MATCH; - } - - wchar_t accept[3] = {towupper(c), towlower(c), 0}; - while (1) { - size_t n = wcscspn(str, accept); - - str += n; - if (str[0] == 0 || (n >= size)) { - break; - } - - int32_t ret = WCSPatternMatch(&patterStr[i], ++str, size - n - 1, pInfo); - if (ret != TSDB_PATTERN_NOMATCH) { - return ret; - } - } - - return TSDB_PATTERN_NOWILDCARDMATCH; - } - - c1 = str[j++]; - - if (j <= size) { - if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) { - continue; - } - } - - return TSDB_PATTERN_NOMATCH; - } - - return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; -} int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { SPatternCompareInfo pInfo = {'%', '_'}; @@ -359,62 +33,6 @@ int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } -int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight) { - return compareStrRegexComp(pLeft, pRight); -} - -int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight) { - return compareStrRegexComp(pLeft, pRight) ? 0 : 1; -} - -int32_t compareStrRegexComp(const void* pLeft, const void* pRight) { - size_t sz = varDataLen(pRight); - char *pattern = malloc(sz + 1); - memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); - pattern[sz] = 0; - - sz = varDataLen(pLeft); - char *str = malloc(sz + 1); - memcpy(str, varDataVal(pLeft), sz); - str[sz] = 0; - - int errCode = 0; - regex_t regex; - char msgbuf[256] = {0}; - - int cflags = REG_EXTENDED; - if ((errCode = regcomp(®ex, pattern, cflags)) != 0) { - regerror(errCode, ®ex, msgbuf, sizeof(msgbuf)); - uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf); - regfree(®ex); - free(str); - free(pattern); - return 1; - } - - errCode = regexec(®ex, str, 0, NULL, 0); - if (errCode != 0 && errCode != REG_NOMATCH) { - regerror(errCode, ®ex, msgbuf, sizeof(msgbuf)); - uDebug("Failed to match %s with pattern %s, reason %s", str, pattern, msgbuf) - } - int32_t result = (errCode == 0) ? 0 : 1; - regfree(®ex); - free(str); - free(pattern); - return result; -} - -int32_t taosArrayCompareString(const void* a, const void* b) { - const char* x = *(const char**)a; - const char* y = *(const char**)b; - - return compareLenPrefixedStr(x, y); -} - -int32_t compareFindItemInSet(const void *pLeft, const void* pRight) { - return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0; -} - int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { SPatternCompareInfo pInfo = {'%', '_'}; @@ -600,4 +218,4 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { } } } -} +} \ No newline at end of file diff --git a/source/server/CMakeLists.txt b/source/server/CMakeLists.txt index 1f6da54ebe..8afa650d36 100644 --- a/source/server/CMakeLists.txt +++ b/source/server/CMakeLists.txt @@ -8,4 +8,5 @@ add_executable(taosd ${TAOSD_SRC}) target_link_libraries( taosd PUBLIC dnode + PUBLIC util ) \ No newline at end of file diff --git a/source/server/vnode/meta/CMakeLists.txt b/source/server/vnode/meta/CMakeLists.txt index 113bcd5d6f..94bf9581d3 100644 --- a/source/server/vnode/meta/CMakeLists.txt +++ b/source/server/vnode/meta/CMakeLists.txt @@ -11,6 +11,6 @@ target_link_libraries( ) target_link_libraries(meta PUBLIC rocksdb) -# if(${BUILD_TEST}) -# add_subdirectory(test) -# endif(${BUILD_TEST}) +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index 09b877ea8e..7c42afcc51 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -1,5 +1,5 @@ aux_source_directory(src UTIL_SRC) -add_library(util ${UTIL_SRC}) +add_library(util STATIC ${UTIL_SRC}) target_include_directories( util PUBLIC "${CMAKE_SOURCE_DIR}/include/util" diff --git a/source/util/src/compare.c b/source/util/src/compare.c new file mode 100644 index 0000000000..f2d320fde0 --- /dev/null +++ b/source/util/src/compare.c @@ -0,0 +1,398 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#define _BSD_SOURCE +#define _GNU_SOURCE +#define _XOPEN_SOURCE +#define _DEFAULT_SOURCE + +#include "os.h" +#include "types.h" +#include "compare.h" +#include "ulog.h" +#include "thash.h" +#include "regex.h" + +int32_t setCompareBytes1(const void *pLeft, const void *pRight) { + return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; +} + +int32_t setCompareBytes2(const void *pLeft, const void *pRight) { + return NULL != taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0; +} + +int32_t setCompareBytes4(const void *pLeft, const void *pRight) { + return NULL != taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0; +} + +int32_t setCompareBytes8(const void *pLeft, const void *pRight) { + return NULL != taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0; +} + +int32_t compareInt8Val(const void *pLeft, const void *pRight) { + int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; +} + +int32_t compareInt8ValDesc(const void *pLeft, const void *pRight) { + return compareInt8Val(pRight, pLeft); +} + +int32_t compareInt16Val(const void *pLeft, const void *pRight) { + int16_t left = GET_INT16_VAL(pLeft), right = GET_INT16_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; +} + +int32_t compareInt16ValDesc(const void* pLeft, const void* pRight) { + return compareInt16Val(pRight, pLeft); +} + +int32_t compareInt32Val(const void *pLeft, const void *pRight) { + int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; +} + +int32_t compareInt32ValDesc(const void* pLeft, const void* pRight) { + return compareInt32Val(pRight, pLeft); +} + +int32_t compareInt64Val(const void *pLeft, const void *pRight) { + int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; +} + +int32_t compareInt64ValDesc(const void* pLeft, const void* pRight) { + return compareInt64Val(pRight, pLeft); +} + +int32_t compareUint32Val(const void *pLeft, const void *pRight) { + uint32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; +} + +int32_t compareUint32ValDesc(const void* pLeft, const void* pRight) { + return compareUint32Val(pRight, pLeft); +} + +int32_t compareUint64Val(const void *pLeft, const void *pRight) { + uint64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; +} + +int32_t compareUint64ValDesc(const void* pLeft, const void* pRight) { + return compareUint64Val(pRight, pLeft); +} + +int32_t compareUint16Val(const void *pLeft, const void *pRight) { + uint16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; +} + +int32_t compareUint16ValDesc(const void* pLeft, const void* pRight) { + return compareUint16Val(pRight, pLeft); +} + +int32_t compareUint8Val(const void* pLeft, const void* pRight) { + uint8_t left = GET_UINT8_VAL(pLeft), right = GET_UINT8_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; +} + +int32_t compareUint8ValDesc(const void* pLeft, const void* pRight) { + return compareUint8Val(pRight, pLeft); +} + +int32_t compareFloatVal(const void *pLeft, const void *pRight) { + float p1 = GET_FLOAT_VAL(pLeft); + float p2 = GET_FLOAT_VAL(pRight); + + if (isnan(p1) && isnan(p2)) { + return 0; + } + + if (isnan(p1)) { + return -1; + } + + if (isnan(p2)) { + return 1; + } + if (FLT_EQUAL(p1, p2)) { + return 0; + } + return FLT_GREATER(p1, p2) ? 1: -1; +} + +int32_t compareFloatValDesc(const void* pLeft, const void* pRight) { + return compareFloatVal(pRight, pLeft); +} + +int32_t compareDoubleVal(const void *pLeft, const void *pRight) { + double p1 = GET_DOUBLE_VAL(pLeft); + double p2 = GET_DOUBLE_VAL(pRight); + + if (isnan(p1) && isnan(p2)) { + return 0; + } + + if (isnan(p1)) { + return -1; + } + + if (isnan(p2)) { + return 1; + } + if (FLT_EQUAL(p1, p2)) { + return 0; + } + return FLT_GREATER(p1, p2) ? 1: -1; +} + +int32_t compareDoubleValDesc(const void* pLeft, const void* pRight) { + return compareDoubleVal(pRight, pLeft); +} + +int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) { + int32_t len1 = varDataLen(pLeft); + int32_t len2 = varDataLen(pRight); + + if (len1 != len2) { + return len1 > len2? 1:-1; + } else { + int32_t ret = strncmp(varDataVal(pLeft), varDataVal(pRight), len1); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1:-1; + } + } +} + +int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight) { + return compareLenPrefixedStr(pRight, pLeft); +} + +int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) { + int32_t len1 = varDataLen(pLeft); + int32_t len2 = varDataLen(pRight); + + if (len1 != len2) { + return len1 > len2? 1:-1; + } else { + int32_t ret = memcmp((wchar_t*) pLeft, (wchar_t*) pRight, len1); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } + } +} + +int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight) { + return compareLenPrefixedWStr(pRight, pLeft); +} + +/* + * Compare two strings + * TSDB_MATCH: Match + * TSDB_NOMATCH: No match + * TSDB_NOWILDCARDMATCH: No match in spite of having * or % wildcards. + * Like matching rules: + * '%': Matches zero or more characters + * '_': Matches one character + * + */ +int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) { + char c, c1; + + int32_t i = 0; + int32_t j = 0; + int32_t o = 0; + int32_t m = 0; + + while ((c = patterStr[i++]) != 0) { + if (c == pInfo->matchAll) { /* Match "*" */ + + while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) { + if (c == pInfo->matchOne) { + if (j > size || str[j++] == 0) { + // empty string, return not match + return TSDB_PATTERN_NOWILDCARDMATCH; + } else { + ++o; + } + } + } + + if (c == 0) { + return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */ + } + + char next[3] = {toupper(c), tolower(c), 0}; + m = o; + while (1) { + size_t n = strcspn(str + m, next); + str += m + n; + + if (str[0] == 0 || (n >= size)) { + break; + } + + int32_t ret = patternMatch(&patterStr[i], ++str, size - n - 1, pInfo); + if (ret != TSDB_PATTERN_NOMATCH) { + return ret; + } + m = 0; + } + return TSDB_PATTERN_NOWILDCARDMATCH; + } + + c1 = str[j++]; + ++o; + + if (j <= size) { + if (c == '\\' && patterStr[i] == '_' && c1 == '_') { i++; continue; } + if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) { + continue; + } + } + + return TSDB_PATTERN_NOMATCH; + } + + return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; +} + +int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) { + wchar_t c, c1; + wchar_t matchOne = L'_'; // "_" + wchar_t matchAll = L'%'; // "%" + + int32_t i = 0; + int32_t j = 0; + + while ((c = patterStr[i++]) != 0) { + if (c == matchAll) { /* Match "%" */ + + while ((c = patterStr[i++]) == matchAll || c == matchOne) { + if (c == matchOne && (j >= size || str[j++] == 0)) { + return TSDB_PATTERN_NOWILDCARDMATCH; + } + } + if (c == 0) { + return TSDB_PATTERN_MATCH; + } + + wchar_t accept[3] = {towupper(c), towlower(c), 0}; + while (1) { + size_t n = wcscspn(str, accept); + + str += n; + if (str[0] == 0 || (n >= size)) { + break; + } + + int32_t ret = WCSPatternMatch(&patterStr[i], ++str, size - n - 1, pInfo); + if (ret != TSDB_PATTERN_NOMATCH) { + return ret; + } + } + + return TSDB_PATTERN_NOWILDCARDMATCH; + } + + c1 = str[j++]; + + if (j <= size) { + if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) { + continue; + } + } + + return TSDB_PATTERN_NOMATCH; + } + + return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; +} + +int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight) { + return compareStrRegexComp(pLeft, pRight); +} + +int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight) { + return compareStrRegexComp(pLeft, pRight) ? 0 : 1; +} + +int32_t compareStrRegexComp(const void* pLeft, const void* pRight) { + size_t sz = varDataLen(pRight); + char *pattern = malloc(sz + 1); + memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); + pattern[sz] = 0; + + sz = varDataLen(pLeft); + char *str = malloc(sz + 1); + memcpy(str, varDataVal(pLeft), sz); + str[sz] = 0; + + int errCode = 0; + regex_t regex; + char msgbuf[256] = {0}; + + int cflags = REG_EXTENDED; + if ((errCode = regcomp(®ex, pattern, cflags)) != 0) { + regerror(errCode, ®ex, msgbuf, sizeof(msgbuf)); + uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf); + regfree(®ex); + free(str); + free(pattern); + return 1; + } + + errCode = regexec(®ex, str, 0, NULL, 0); + if (errCode != 0 && errCode != REG_NOMATCH) { + regerror(errCode, ®ex, msgbuf, sizeof(msgbuf)); + uDebug("Failed to match %s with pattern %s, reason %s", str, pattern, msgbuf) + } + int32_t result = (errCode == 0) ? 0 : 1; + regfree(®ex); + free(str); + free(pattern); + return result; +} + +int32_t taosArrayCompareString(const void* a, const void* b) { + const char* x = *(const char**)a; + const char* y = *(const char**)b; + + return compareLenPrefixedStr(x, y); +} + +int32_t compareFindItemInSet(const void *pLeft, const void* pRight) { + return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0; +} diff --git a/source/common/src/terror.c b/source/util/src/terror.c similarity index 100% rename from source/common/src/terror.c rename to source/util/src/terror.c diff --git a/source/util/src/thashutil.c b/source/util/src/thashutil.c index 83f85bfdbc..7deb4d5a78 100644 --- a/source/util/src/thashutil.c +++ b/source/util/src/thashutil.c @@ -15,8 +15,9 @@ #include "os.h" #include "thash.h" -#include "tcompare.h" +#include "compare.h" #include "tdef.h" +#include "types.h" #define ROTL32(x, r) ((x) << (r) | (x) >> (32u - (r))) diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index d02b148863..00263d7bce 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -15,7 +15,7 @@ */ #include "tskiplist.h" #include "os.h" -#include "tcompare.h" +#include "compare.h" #include "ulog.h" #include "tutil.h" diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index c5027af7c7..22e378d067 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -450,24 +450,3 @@ char *taosIpStr(uint32_t ipInt) { return ipStr; } -FORCE_INLINE float taos_align_get_float(const char* pBuf) { -#if __STDC_VERSION__ >= 201112L - static_assert(sizeof(float) == sizeof(uint32_t), "sizeof(float) must equal to sizeof(uint32_t)"); -#else - assert(sizeof(float) == sizeof(uint32_t)); -#endif - float fv = 0; - memcpy(&fv, pBuf, sizeof(fv)); // in ARM, return *((const float*)(pBuf)) may cause problem - return fv; -} - -FORCE_INLINE double taos_align_get_double(const char* pBuf) { -#if __STDC_VERSION__ >= 201112L - static_assert(sizeof(double) == sizeof(uint64_t), "sizeof(double) must equal to sizeof(uint64_t)"); -#else - assert(sizeof(double) == sizeof(uint64_t)); -#endif - double dv = 0; - memcpy(&dv, pBuf, sizeof(dv)); // in ARM, return *((const double*)(pBuf)) may cause problem - return dv; -} From bab6e54b1382de33a04a83f9d9ad66b4f8cf3d5b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 11 Oct 2021 23:52:52 +0800 Subject: [PATCH 13/24] partial SRow code --- include/common/trow.h | 48 +++++++++++-- source/common/src/trow.c | 77 ++++++++++++++++++++- source/server/vnode/meta/test/metaTests.cpp | 1 + 3 files changed, 120 insertions(+), 6 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index 6094425bbf..e699031d7a 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -16,16 +16,54 @@ #ifndef _TD_COMMON_ROW_H_ #define _TD_COMMON_ROW_H_ +#include "os.h" + #ifdef __cplusplus extern "C" { #endif -typedef struct SRow SRow; +// types +typedef void * SRow; +typedef struct SRowBatch SRowBatch; +typedef struct SRowBuilder SRowBuilder; +typedef struct SRowBatchIter SRowBatchIter; +typedef struct SRowBatchBuilder SRowBatchBuilder; -#define rowType(r) -#define rowLen(r) -#define rowVersion(r) -#define rowNCols(r) +// SRow +#define ROW_HEADER_SIZE (sizeof(uint8_t) + 2 * sizeof(uint16_t) + sizeof(uint64_t)) +#define rowType(r) (*(uint8_t *)(r)) // row type +#define rowLen(r) (*(uint16_t *)POINTER_SHIFT(r, sizeof(uint8_t))) // row length +#define rowSVer(r) \ + (*(uint16_t *)POINTER_SHIFT(r, sizeof(uint8_t) + sizeof(uint16_t))) // row schema version, only for SDataRow +#define rowNCols(r) rowSVer(r) // only for SKVRow +#define rowVer(r) (*(uint64_t)POINTER_SHIFT(r, sizeof(uint8_t) + 2 * sizeof(uint16_t))) // row version +#define rowCopy(dest, r) memcpy((dest), r, rowLen(r)) + +static FORCE_INLINE SRow rowDup(SRow row) { + SRow r = malloc(rowLen(row)); + if (r == NULL) { + return NULL; + } + + rowCopy(r, row); + + return r; +} + +// SRowBatch + +// SRowBuilder +SRowBuilder *rowBuilderCreate(); +void rowBuilderDestroy(SRowBuilder *); + +// SRowBatchIter +SRowBatchIter *rowBatchIterCreate(SRowBatch *); +void rowBatchIterDestroy(SRowBatchIter *); +const SRow rowBatchIterNext(SRowBatchIter *); + +// SRowBatchBuilder +SRowBatchBuilder *rowBatchBuilderCreate(); +void rowBatchBuilderDestroy(SRowBatchBuilder *); #ifdef __cplusplus } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 6dea4a4e57..cf1b0eceff 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -11,4 +11,79 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "trow.h" + +/* ------------ Structures ---------- */ +struct SRowBatch { + int32_t compress : 1; // if batch row is compressed + int32_t nrows : 31; // number of rows + int32_t tlen; // total length (including `nrows` and `tlen`) + char rows[]; +}; + +struct SRowBuilder { + // TODO +}; + +struct SRowBatchIter { + int32_t counter; // row counter + SRowBatch *rb; // row batch to iter + SRow nrow; // next row +}; + +struct SRowBatchBuilder { + // TODO +}; + +/* ------------ Methods ---------- */ + +// SRowBuilder +SRowBuilder *rowBuilderCreate() { + SRowBuilder *pRowBuilder = NULL; + // TODO + + return pRowBuilder; +} + +void rowBuilderDestroy(SRowBuilder *pRowBuilder) { + if (pRowBuilder) { + free(pRowBuilder); + } +} + +// SRowBatchIter +SRowBatchIter *rowBatchIterCreate(SRowBatch *pRowBatch) { + SRowBatchIter *pRowBatchIter = (SRowBatchIter *)malloc(sizeof(*pRowBatchIter)); + if (pRowBatchIter == NULL) { + return NULL; + } + + pRowBatchIter->counter = 0; + pRowBatchIter->rb = pRowBatch; + pRowBatchIter->nrow = pRowBatch->rows; + + return pRowBatchIter; +}; + +void rowBatchIterDestroy(SRowBatchIter *pRowBatchIter) { + if (pRowBatchIter) { + free(pRowBatchIter); + } +} + +const SRow rowBatchIterNext(SRowBatchIter *pRowBatchIter) { + SRow r = NULL; + if (pRowBatchIter->counter < pRowBatchIter->rb->nrows) { + r = pRowBatchIter->nrow; + pRowBatchIter->counter += 1; + pRowBatchIter->nrow = (SRow)POINTER_SHIFT(r, rowLen(r)); + } + + return r; +} + +// SRowBatchBuilder +SRowBatchBuilder *rowBatchBuilderCreate(); +void rowBatchBuilderDestroy(SRowBatchBuilder *); \ No newline at end of file diff --git a/source/server/vnode/meta/test/metaTests.cpp b/source/server/vnode/meta/test/metaTests.cpp index c62ae0aa02..ab204455f2 100644 --- a/source/server/vnode/meta/test/metaTests.cpp +++ b/source/server/vnode/meta/test/metaTests.cpp @@ -4,5 +4,6 @@ #include "meta.h" TEST(MetaTest, meta_open_test) { + metaOpen(NULL); std::cout << "Hello META!" << std::endl; } \ No newline at end of file From 557905d0068c97ec557558a2699034b543a260df Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Oct 2021 11:15:11 +0800 Subject: [PATCH 14/24] more progress --- source/server/vnode/meta/src/meta.c | 33 +++++++++++++++++---- source/server/vnode/meta/test/metaTests.cpp | 7 +++-- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/source/server/vnode/meta/src/meta.c b/source/server/vnode/meta/src/meta.c index f35de5eb9e..ff8186d1d7 100644 --- a/source/server/vnode/meta/src/meta.c +++ b/source/server/vnode/meta/src/meta.c @@ -22,6 +22,7 @@ #include "meta.h" +/* -------------------- Structures -------------------- */ typedef struct STable { uint64_t uid; tstr * name; @@ -46,9 +47,12 @@ struct SMeta { rocksdb_t *tbnameDb; // tbname --> uid rocksdb_t *tagDb; // uid --> tag rocksdb_t *schemaDb; + rocksdb_t *tagIdx; size_t totalUsed; }; +/* -------------------- Methods -------------------- */ + SMeta *metaOpen(SMetaOptions *options) { SMeta *pMeta = NULL; char * err = NULL; @@ -64,21 +68,40 @@ SMeta *metaOpen(SMetaOptions *options) { pMeta->stbList = tdListNew(sizeof(STableObj *)); + // Options + rocksdb_options_t *dbOptions = rocksdb_options_create(); + rocksdb_options_set_create_if_missing(dbOptions, 1); + + taosMkDir("meta"); + // Open tbname DB - rocksdb_options_t *tbnameDbOptions = rocksdb_options_create(); - pMeta->tbnameDb = rocksdb_open(tbnameDbOptions, "tbname_uid_db", &err); + pMeta->tbnameDb = rocksdb_open(dbOptions, "meta/tbname_uid_db", &err); // Open tag DB - pMeta->tagDb = rocksdb_open(tbnameDbOptions, "uid_tag_db", &err); + pMeta->tagDb = rocksdb_open(dbOptions, "meta/uid_tag_db", &err); // Open schema DB - pMeta->schemaDb = rocksdb_open(tbnameDbOptions, "schema_db", &err); + pMeta->schemaDb = rocksdb_open(dbOptions, "meta/schema_db", &err); + + // Open tag index + pMeta->tagIdx = rocksdb_open(dbOptions, "meta/tag_idx_db", &err); + + rocksdb_options_destroy(dbOptions); return pMeta; } void metaClose(SMeta *pMeta) { - // TODO + if (pMeta) { + rocksdb_close(pMeta->tagIdx); + rocksdb_close(pMeta->schemaDb); + rocksdb_close(pMeta->tagDb); + rocksdb_close(pMeta->tbnameDb); + + tdListFree(pMeta->stbList); + taosHashCleanup(pMeta->pTableObjHash); + pthread_rwlock_destroy(&(pMeta->rwLock)); + } } int metaCommit(SMeta *meta) { return 0; } \ No newline at end of file diff --git a/source/server/vnode/meta/test/metaTests.cpp b/source/server/vnode/meta/test/metaTests.cpp index ab204455f2..caf77f720a 100644 --- a/source/server/vnode/meta/test/metaTests.cpp +++ b/source/server/vnode/meta/test/metaTests.cpp @@ -4,6 +4,9 @@ #include "meta.h" TEST(MetaTest, meta_open_test) { - metaOpen(NULL); - std::cout << "Hello META!" << std::endl; + SMeta *meta = metaOpen(NULL); + std::cout << "Meta is opened!" << std::endl; + + metaClose(meta); + std::cout << "Meta is closed!" << std::endl; } \ No newline at end of file From be63e61e1664bcb80050096f4432cdb88f908d1c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Oct 2021 11:29:22 +0800 Subject: [PATCH 15/24] more --- source/server/vnode/meta/test/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/server/vnode/meta/test/CMakeLists.txt b/source/server/vnode/meta/test/CMakeLists.txt index ca82d3fb83..ce00dc0a66 100644 --- a/source/server/vnode/meta/test/CMakeLists.txt +++ b/source/server/vnode/meta/test/CMakeLists.txt @@ -15,4 +15,9 @@ target_link_libraries(metaTest common rocksdb gtest_main +) +enable_testing() +add_test( + NAME meta_test + COMMAND metaTest ) \ No newline at end of file From 98ae04e7e62872b258a72f46167e90c555834f03 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Oct 2021 11:41:27 +0800 Subject: [PATCH 16/24] more --- include/server/vnode/meta/meta.h | 2 +- source/os/src/osDir.c | 2 +- source/server/vnode/meta/src/meta.c | 2 ++ source/server/vnode/meta/test/metaTests.cpp | 3 +++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/include/server/vnode/meta/meta.h b/include/server/vnode/meta/meta.h index 4167dedde6..bfa11b9c39 100644 --- a/include/server/vnode/meta/meta.h +++ b/include/server/vnode/meta/meta.h @@ -34,7 +34,7 @@ typedef struct SMetaQueryOptions SMetaQueryOptions; // SMeta operations int metaCreate(const char *path); -int metaDestroy(const char *path); +void metaDestroy(const char *path); SMeta *metaOpen(SMetaOptions *); void metaClose(SMeta *); int metaCreateTable(SMeta *, void *); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 0526dffe9e..839db811fd 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -48,7 +48,7 @@ void taosRemoveDir(char *dirname) { taosRemoveDir(filename); } else { (void)remove(filename); - printf("file:%s is removed", filename); + printf("file:%s is removed\n", filename); } } diff --git a/source/server/vnode/meta/src/meta.c b/source/server/vnode/meta/src/meta.c index ff8186d1d7..e945e6eac4 100644 --- a/source/server/vnode/meta/src/meta.c +++ b/source/server/vnode/meta/src/meta.c @@ -104,4 +104,6 @@ void metaClose(SMeta *pMeta) { } } +void metaDestroy(const char *path) { taosRemoveDir(path); } + int metaCommit(SMeta *meta) { return 0; } \ No newline at end of file diff --git a/source/server/vnode/meta/test/metaTests.cpp b/source/server/vnode/meta/test/metaTests.cpp index caf77f720a..fcc1220273 100644 --- a/source/server/vnode/meta/test/metaTests.cpp +++ b/source/server/vnode/meta/test/metaTests.cpp @@ -9,4 +9,7 @@ TEST(MetaTest, meta_open_test) { metaClose(meta); std::cout << "Meta is closed!" << std::endl; + + metaDestroy("meta"); + std::cout << "Meta is destroyed!" << std::endl; } \ No newline at end of file From 9728ec02fe1508732c38a092201707e546128f5c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Oct 2021 13:42:12 +0800 Subject: [PATCH 17/24] refact --- include/common/schema.h | 76 ----------------------------- include/server/vnode/meta/meta.h | 27 +++++----- source/server/vnode/meta/src/meta.c | 13 ++++- 3 files changed, 26 insertions(+), 90 deletions(-) diff --git a/include/common/schema.h b/include/common/schema.h index ccc91f09ff..5e9057520b 100644 --- a/include/common/schema.h +++ b/include/common/schema.h @@ -16,86 +16,10 @@ #ifndef _TD_COMMON_SCHEMA_H_ #define _TD_COMMON_SCHEMA_H_ -#include "os.h" - #ifdef __cplusplus extern "C" { #endif -// ----------------- TSDB COLUMN DEFINITION -typedef struct { - int8_t type; // Column type - int16_t colId; // column ID - int16_t bytes; // column bytes (restore to int16_t in case of misuse) - uint16_t offset; // point offset in SDataRow after the header part. -} STColumn; - -#define colType(col) ((col)->type) -#define colColId(col) ((col)->colId) -#define colBytes(col) ((col)->bytes) -#define colOffset(col) ((col)->offset) - -#define colSetType(col, t) (colType(col) = (t)) -#define colSetColId(col, id) (colColId(col) = (id)) -#define colSetBytes(col, b) (colBytes(col) = (b)) -#define colSetOffset(col, o) (colOffset(col) = (o)) - -// ----------------- TSDB SCHEMA DEFINITION -typedef struct { - int version; // version - int numOfCols; // Number of columns appended - int tlen; // maximum length of a SDataRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + // - // (bytes)) - uint16_t flen; // First part length in a SDataRow after the header part - uint16_t vlen; // pure value part length, excluded the overhead (bytes only) - STColumn columns[]; -} STSchema; - -#define schemaNCols(s) ((s)->numOfCols) -#define schemaVersion(s) ((s)->version) -#define schemaTLen(s) ((s)->tlen) -#define schemaFLen(s) ((s)->flen) -#define schemaVLen(s) ((s)->vlen) -#define schemaColAt(s, i) ((s)->columns + i) -#define tdFreeSchema(s) tfree((s)) - -STSchema *tdDupSchema(STSchema *pSchema); -int tdEncodeSchema(void **buf, STSchema *pSchema); -void * tdDecodeSchema(void *buf, STSchema **pRSchema); - -static FORCE_INLINE int comparColId(const void *key1, const void *key2) { - if (*(int16_t *)key1 > ((STColumn *)key2)->colId) { - return 1; - } else if (*(int16_t *)key1 < ((STColumn *)key2)->colId) { - return -1; - } else { - return 0; - } -} - -static FORCE_INLINE STColumn *tdGetColOfID(STSchema *pSchema, int16_t colId) { - void *ptr = bsearch(&colId, (void *)pSchema->columns, schemaNCols(pSchema), sizeof(STColumn), comparColId); - if (ptr == NULL) return NULL; - return (STColumn *)ptr; -} - -// ----------------- SCHEMA BUILDER DEFINITION -typedef struct { - int tCols; - int nCols; - int tlen; - uint16_t flen; - uint16_t vlen; - int version; - STColumn *columns; -} STSchemaBuilder; - -int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); -void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); -void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); -int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes); -STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); - #ifdef __cplusplus } #endif diff --git a/include/server/vnode/meta/meta.h b/include/server/vnode/meta/meta.h index bfa11b9c39..f639ed243f 100644 --- a/include/server/vnode/meta/meta.h +++ b/include/server/vnode/meta/meta.h @@ -27,33 +27,34 @@ extern "C" { typedef uint64_t tuid_t; // Types exported -typedef struct SMeta SMeta; -typedef struct SMetaOptions SMetaOptions; -typedef struct SMetaQueryHandle SMetaQueryHandle; -typedef struct SMetaQueryOptions SMetaQueryOptions; +typedef struct SMeta SMeta; +typedef struct SMetaOpts SMetaOpts; +typedef struct SMetaQueryHandle SMetaQueryHandle; +typedef struct SMetaQueryOpts SMetaQueryOpts; +typedef struct STableOpts STableOpts; // SMeta operations int metaCreate(const char *path); void metaDestroy(const char *path); -SMeta *metaOpen(SMetaOptions *); +SMeta *metaOpen(SMetaOpts *); void metaClose(SMeta *); -int metaCreateTable(SMeta *, void *); +int metaCreateTable(SMeta *, STableOpts *); int metaDropTable(SMeta *, uint64_t tuid_t); int metaAlterTable(SMeta *, void *); int metaCommit(SMeta *); // Options -SMetaOptions *metaOptionsCreate(); -void metaOptionsDestroy(SMetaOptions *); -void metaOptionsSetCache(SMetaOptions *, size_t capacity); +SMetaOpts *metaOptionsCreate(); +void metaOptionsDestroy(SMetaOpts *); +void metaOptionsSetCache(SMetaOpts *, size_t capacity); // SMetaQueryHandle -SMetaQueryHandle *metaQueryHandleCreate(SMetaQueryOptions *); +SMetaQueryHandle *metaQueryHandleCreate(SMetaQueryOpts *); void metaQueryHandleDestroy(SMetaQueryHandle *); -// SMetaQueryOptions -SMetaQueryOptions *metaQueryOptionsCreate(); -void metaQueryOptionsDestroy(SMetaQueryOptions *); +// SMetaQueryOpts +SMetaQueryOpts *metaQueryOptionsCreate(); +void metaQueryOptionsDestroy(SMetaQueryOpts *); #ifdef __cplusplus } diff --git a/source/server/vnode/meta/src/meta.c b/source/server/vnode/meta/src/meta.c index e945e6eac4..a035fef205 100644 --- a/source/server/vnode/meta/src/meta.c +++ b/source/server/vnode/meta/src/meta.c @@ -51,9 +51,15 @@ struct SMeta { size_t totalUsed; }; +struct STableOpts { + int8_t type; + char * name; + STSchema *pSchema; +}; + /* -------------------- Methods -------------------- */ -SMeta *metaOpen(SMetaOptions *options) { +SMeta *metaOpen(SMetaOpts *options) { SMeta *pMeta = NULL; char * err = NULL; @@ -104,6 +110,11 @@ void metaClose(SMeta *pMeta) { } } +int metaCreateTable(SMeta *pMeta, STableOpts *pTableOpts) { + // TODO + return 0; +} + void metaDestroy(const char *path) { taosRemoveDir(path); } int metaCommit(SMeta *meta) { return 0; } \ No newline at end of file From 512ad36e62b5a29d919bc00f0cdfd2ca9416e4a9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Oct 2021 13:49:12 +0800 Subject: [PATCH 18/24] refact --- include/common/{schema.h => tschema.h} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename include/common/{schema.h => tschema.h} (100%) diff --git a/include/common/schema.h b/include/common/tschema.h similarity index 100% rename from include/common/schema.h rename to include/common/tschema.h From b45884c15135ad04b4eb2bc5e094667d168f7e6f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Oct 2021 14:45:59 +0800 Subject: [PATCH 19/24] more --- include/server/vnode/meta/meta.h | 10 ++ source/server/vnode/meta/inc/metaUid.h | 34 ++++++ source/server/vnode/meta/src/meta.c | 106 +++++++++++++++++-- source/server/vnode/meta/src/metaUid.c | 23 ++++ source/server/vnode/meta/test/CMakeLists.txt | 1 + 5 files changed, 164 insertions(+), 10 deletions(-) create mode 100644 source/server/vnode/meta/inc/metaUid.h create mode 100644 source/server/vnode/meta/src/metaUid.c diff --git a/include/server/vnode/meta/meta.h b/include/server/vnode/meta/meta.h index f639ed243f..9797ed8168 100644 --- a/include/server/vnode/meta/meta.h +++ b/include/server/vnode/meta/meta.h @@ -56,6 +56,16 @@ void metaQueryHandleDestroy(SMetaQueryHandle *); SMetaQueryOpts *metaQueryOptionsCreate(); void metaQueryOptionsDestroy(SMetaQueryOpts *); +// STableOpts +void metaTableOptsInit(int8_t type, const char *name, const STSchema *pSchema); + +/* -------------------------------- Hided implementations -------------------------------- */ +struct STableOpts { + int8_t type; + char * name; + STSchema *pSchema; +}; + #ifdef __cplusplus } #endif diff --git a/source/server/vnode/meta/inc/metaUid.h b/source/server/vnode/meta/inc/metaUid.h new file mode 100644 index 0000000000..b01492cbf7 --- /dev/null +++ b/source/server/vnode/meta/inc/metaUid.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_META_UID_H_ +#define _TD_META_UID_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef uint64_t tb_uid_t; +tb_uid_t metaGenerateUid(); + +#define IVLD_TB_UID 0 + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_META_UID_H_*/ \ No newline at end of file diff --git a/source/server/vnode/meta/src/meta.c b/source/server/vnode/meta/src/meta.c index a035fef205..9cf9d7ba78 100644 --- a/source/server/vnode/meta/src/meta.c +++ b/source/server/vnode/meta/src/meta.c @@ -21,12 +21,14 @@ #include "ttypes.h" #include "meta.h" +#include "metaUid.h" /* -------------------- Structures -------------------- */ + typedef struct STable { - uint64_t uid; - tstr * name; - uint64_t suid; + tb_uid_t uid; + char * name; + tb_uid_t suid; SArray * schema; } STable; @@ -51,11 +53,8 @@ struct SMeta { size_t totalUsed; }; -struct STableOpts { - int8_t type; - char * name; - STSchema *pSchema; -}; +static STable * metaTableNew(tb_uid_t uid, const char *name, int32_t sver); +static STableObj *metaTableObjNew(); /* -------------------- Methods -------------------- */ @@ -111,10 +110,97 @@ void metaClose(SMeta *pMeta) { } int metaCreateTable(SMeta *pMeta, STableOpts *pTableOpts) { - // TODO + size_t vallen; + char * err = NULL; + rocksdb_readoptions_t * ropt; + STableObj * pTableObj = NULL; + rocksdb_writeoptions_t *wopt; + + // Check if table already exists + ropt = rocksdb_readoptions_create(); + + char *uidStr = rocksdb_get(pMeta->tbnameDb, ropt, pTableOpts->name, strlen(pTableOpts->name), &vallen, &err); + if (uidStr != NULL) { + // Has duplicate named table + return -1; + } + + rocksdb_readoptions_destroy(ropt); + + // Create table obj + pTableObj = metaTableObjNew(); + if (pTableObj == NULL) { + // TODO + return -1; + } + + // Create table object + pTableObj->pTable = metaTableNew(metaGenerateUid(), pTableOpts->name, schemaVersion(pTableOpts->pSchema)); + if (pTableObj->pTable == NULL) { + // TODO + } + + pthread_rwlock_rdlock(&pMeta->rwLock); + + taosHashPut(pMeta->pTableObjHash, &(pTableObj->pTable->uid), sizeof(tb_uid_t), &pTableObj, sizeof(pTableObj)); + + wopt = rocksdb_writeoptions_create(); + + rocksdb_put(pMeta->tbnameDb, wopt, pTableOpts->name, strlen(pTableOpts->name), &pTableObj->pTable->uid, + sizeof(tb_uid_t), &err); + rocksdb_put(pMeta->schemaDb, wopt, pTableOpts->name, strlen(pTableOpts->name), &pTableObj->pTable->uid, + sizeof(tb_uid_t), &err); + + rocksdb_writeoptions_destroy(wopt); + + pthread_rwlock_unlock(&pMeta->rwLock); + return 0; } void metaDestroy(const char *path) { taosRemoveDir(path); } -int metaCommit(SMeta *meta) { return 0; } \ No newline at end of file +int metaCommit(SMeta *meta) { return 0; } + +/* -------------------- Static Methods -------------------- */ + +static STable *metaTableNew(tb_uid_t uid, const char *name, int32_t sver) { + STable *pTable = NULL; + + pTable = (STable *)malloc(sizeof(*pTable)); + if (pTable == NULL) { + // TODO + return NULL; + } + + pTable->schema = taosArrayInit(0, sizeof(int32_t)); + if (pTable->schema == NULL) { + // TODO + return NULL; + } + + pTable->uid = uid; + pTable->name = strdup(name); + pTable->suid = IVLD_TB_UID; + taosArrayPush(pTable->schema, &sver); + + return pTable; +} + +static STableObj *metaTableObjNew() { + STableObj *pTableObj = NULL; + + pTableObj = (STableObj *)malloc(sizeof(*pTableObj)); + if (pTableObj == NULL) { + return NULL; + } + + pTableObj->pin = true; + pTableObj->ref = 1; + taosInitRWLatch(&(pTableObj->latch)); + pTableObj->offset = UINT64_MAX; + pTableObj->ctbList = NULL; + pTableObj->pTable = NULL; + + return pTableObj; +} \ No newline at end of file diff --git a/source/server/vnode/meta/src/metaUid.c b/source/server/vnode/meta/src/metaUid.c new file mode 100644 index 0000000000..4662969a2e --- /dev/null +++ b/source/server/vnode/meta/src/metaUid.c @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "metaUid.h" + +static tb_uid_t nuid = IVLD_TB_UID; + +tb_uid_t metaGenerateUid() { + // TODO: need a more complex UID generator + return ++nuid; +} \ No newline at end of file diff --git a/source/server/vnode/meta/test/CMakeLists.txt b/source/server/vnode/meta/test/CMakeLists.txt index ce00dc0a66..ee16a28687 100644 --- a/source/server/vnode/meta/test/CMakeLists.txt +++ b/source/server/vnode/meta/test/CMakeLists.txt @@ -2,6 +2,7 @@ add_executable(metaTest "") target_sources(metaTest PRIVATE "../src/meta.c" + "../src/metaUid.c" "metaTests.cpp" ) target_include_directories(metaTest From 87dcc7fd92bd2dd1a03f58de9a585569d746ae80 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Oct 2021 15:11:44 +0800 Subject: [PATCH 20/24] make compile and run test --- include/server/vnode/meta/meta.h | 2 +- source/server/vnode/meta/src/meta.c | 19 +++++++++++++-- source/server/vnode/meta/test/metaTests.cpp | 26 +++++++++++++++++++-- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/include/server/vnode/meta/meta.h b/include/server/vnode/meta/meta.h index 9797ed8168..3afe01511b 100644 --- a/include/server/vnode/meta/meta.h +++ b/include/server/vnode/meta/meta.h @@ -57,7 +57,7 @@ SMetaQueryOpts *metaQueryOptionsCreate(); void metaQueryOptionsDestroy(SMetaQueryOpts *); // STableOpts -void metaTableOptsInit(int8_t type, const char *name, const STSchema *pSchema); +void metaTableOptsInit(STableOpts *, int8_t type, const char *name, const STSchema *pSchema); /* -------------------------------- Hided implementations -------------------------------- */ struct STableOpts { diff --git a/source/server/vnode/meta/src/meta.c b/source/server/vnode/meta/src/meta.c index 9cf9d7ba78..b3d33c7b87 100644 --- a/source/server/vnode/meta/src/meta.c +++ b/source/server/vnode/meta/src/meta.c @@ -146,10 +146,19 @@ int metaCreateTable(SMeta *pMeta, STableOpts *pTableOpts) { wopt = rocksdb_writeoptions_create(); + // Add to tbname db rocksdb_put(pMeta->tbnameDb, wopt, pTableOpts->name, strlen(pTableOpts->name), &pTableObj->pTable->uid, sizeof(tb_uid_t), &err); - rocksdb_put(pMeta->schemaDb, wopt, pTableOpts->name, strlen(pTableOpts->name), &pTableObj->pTable->uid, - sizeof(tb_uid_t), &err); + + // Add to schema db + char id[12]; + char buf[256]; + void *pBuf = buf; + *(tb_uid_t *)id = pTableObj->pTable->uid; + *(int32_t *)(id + sizeof(tb_uid_t)) = schemaVersion(pTableOpts->pSchema); + int size = tdEncodeSchema(&pBuf, pTableOpts->pSchema); + + rocksdb_put(pMeta->schemaDb, wopt, id, 12, buf, size, &err); rocksdb_writeoptions_destroy(wopt); @@ -162,6 +171,12 @@ void metaDestroy(const char *path) { taosRemoveDir(path); } int metaCommit(SMeta *meta) { return 0; } +void metaTableOptsInit(STableOpts *pTableOpts, int8_t type, const char *name, const STSchema *pSchema) { + pTableOpts->type = type; + pTableOpts->name = strdup(name); + pTableOpts->pSchema = tdDupSchema(pSchema); +} + /* -------------------- Static Methods -------------------- */ static STable *metaTableNew(tb_uid_t uid, const char *name, int32_t sver) { diff --git a/source/server/vnode/meta/test/metaTests.cpp b/source/server/vnode/meta/test/metaTests.cpp index fcc1220273..f81302fc91 100644 --- a/source/server/vnode/meta/test/metaTests.cpp +++ b/source/server/vnode/meta/test/metaTests.cpp @@ -1,15 +1,37 @@ #include +#include #include #include "meta.h" TEST(MetaTest, meta_open_test) { + // Open Meta SMeta *meta = metaOpen(NULL); std::cout << "Meta is opened!" << std::endl; + // Create tables + STableOpts tbOpts; + char tbname[128]; + STSchema * pSchema; + STSchemaBuilder sb; + tdInitTSchemaBuilder(&sb, 0); + for (size_t i = 0; i < 10; i++) { + tdAddColToSchema(&sb, TSDB_DATA_TYPE_TIMESTAMP, i, 8); + } + pSchema = tdGetSchemaFromBuilder(&sb); + tdDestroyTSchemaBuilder(&sb); + for (size_t i = 0; i < 1000000; i++) { + sprintf(tbname, "tb%ld", i); + metaTableOptsInit(&tbOpts, 0, tbname, pSchema); + + metaCreateTable(meta, &tbOpts); + } + + // Close Meta metaClose(meta); std::cout << "Meta is closed!" << std::endl; - metaDestroy("meta"); - std::cout << "Meta is destroyed!" << std::endl; + // // Destroy Meta + // metaDestroy("meta"); + // std::cout << "Meta is destroyed!" << std::endl; } \ No newline at end of file From 8287a34ccb1230970be4a16c5fb4cea4c6a904ab Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Oct 2021 15:23:36 +0800 Subject: [PATCH 21/24] more progress --- source/server/vnode/meta/src/meta.c | 7 ++++--- source/server/vnode/meta/test/metaTests.cpp | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/source/server/vnode/meta/src/meta.c b/source/server/vnode/meta/src/meta.c index b3d33c7b87..000b2ca9b0 100644 --- a/source/server/vnode/meta/src/meta.c +++ b/source/server/vnode/meta/src/meta.c @@ -145,19 +145,20 @@ int metaCreateTable(SMeta *pMeta, STableOpts *pTableOpts) { taosHashPut(pMeta->pTableObjHash, &(pTableObj->pTable->uid), sizeof(tb_uid_t), &pTableObj, sizeof(pTableObj)); wopt = rocksdb_writeoptions_create(); + rocksdb_writeoptions_disable_WAL(wopt, 1); // Add to tbname db rocksdb_put(pMeta->tbnameDb, wopt, pTableOpts->name, strlen(pTableOpts->name), &pTableObj->pTable->uid, sizeof(tb_uid_t), &err); // Add to schema db - char id[12]; - char buf[256]; + char id[12]; + char buf[256]; void *pBuf = buf; *(tb_uid_t *)id = pTableObj->pTable->uid; *(int32_t *)(id + sizeof(tb_uid_t)) = schemaVersion(pTableOpts->pSchema); int size = tdEncodeSchema(&pBuf, pTableOpts->pSchema); - + rocksdb_put(pMeta->schemaDb, wopt, id, 12, buf, size, &err); rocksdb_writeoptions_destroy(wopt); diff --git a/source/server/vnode/meta/test/metaTests.cpp b/source/server/vnode/meta/test/metaTests.cpp index f81302fc91..47ca49b70a 100644 --- a/source/server/vnode/meta/test/metaTests.cpp +++ b/source/server/vnode/meta/test/metaTests.cpp @@ -31,7 +31,7 @@ TEST(MetaTest, meta_open_test) { metaClose(meta); std::cout << "Meta is closed!" << std::endl; - // // Destroy Meta - // metaDestroy("meta"); - // std::cout << "Meta is destroyed!" << std::endl; + // Destroy Meta + metaDestroy("meta"); + std::cout << "Meta is destroyed!" << std::endl; } \ No newline at end of file From 59de14eaa48dd95d3002275897b888ed0f9fb904 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Oct 2021 16:15:44 +0800 Subject: [PATCH 22/24] wrap tkv --- CMakeLists.txt | 2 +- include/libs/tkv/tkv.h | 29 +++++--- source/libs/tkv/CMakeLists.txt | 4 ++ source/libs/tkv/src/tkv.c | 69 +++++++++++++++++++- source/server/vnode/meta/CMakeLists.txt | 2 +- source/server/vnode/meta/src/meta.c | 65 +++++++++--------- source/server/vnode/meta/test/CMakeLists.txt | 2 +- 7 files changed, 127 insertions(+), 46 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f27f5f1672..65d1e133d7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,7 +42,7 @@ if(${BUILD_WITH_LEVELDB}) endif(${BUILD_WITH_LEVELDB}) ## rocksdb -option(BUILD_WITH_ROCKSDB "If build with rocksdb" ON) +option(BUILD_WITH_ROCKSDB "If build with rocksdb" OFF) if(${BUILD_WITH_ROCKSDB}) cat("${CMAKE_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${DEPS_TMP_FILE}) endif(${BUILD_WITH_ROCKSDB}) diff --git a/include/libs/tkv/tkv.h b/include/libs/tkv/tkv.h index c31d655e43..d9cabc1e40 100644 --- a/include/libs/tkv/tkv.h +++ b/include/libs/tkv/tkv.h @@ -16,30 +16,43 @@ #ifndef _TD_TKV_H_ #define _TD_TKV_H_ +#include "os.h" + #ifdef __cplusplus extern "C" { #endif // Types exported -typedef struct STkvDb STkvDb; -typedef struct STkvOptions STkvOptions; -typedef struct STkvCache STkvCache; +typedef struct STkvDb STkvDb; +typedef struct STkvOpts STkvOpts; +typedef struct STkvCache STkvCache; +typedef struct STkvReadOpts STkvReadOpts; +typedef struct STkvWriteOpts STkvWriteOpts; // DB operations -STkvDb *tkvOpen(const STkvOptions *options, const char *path); +STkvDb *tkvOpen(const STkvOpts *options, const char *path); void tkvClose(STkvDb *db); -void tkvPut(STkvDb *db, void * /*TODO*/); +void tkvPut(STkvDb *db, STkvWriteOpts *, char *key, size_t keylen, char *val, size_t vallen); +char * tkvGet(STkvDb *db, STkvReadOpts *, char *key, size_t keylen, size_t *vallen); // DB options -STkvOptions *tkvOptionsCreate(); -void tkvOptionsDestroy(STkvOptions *); -void tkvOptionsSetCache(STkvOptions *, STkvCache *); +STkvOpts *tkvOptionsCreate(); +void tkvOptionsDestroy(STkvOpts *); +void tkvOptionsSetCache(STkvOpts *, STkvCache *); // DB cache typedef enum { TKV_LRU_CACHE = 0, TKV_LFU_CACHE = 1 } ETkvCacheType; STkvCache *tkvCacheCreate(size_t capacity, ETkvCacheType type); void tkvCacheDestroy(STkvCache *); +// STkvReadOpts +STkvReadOpts *tkvReadOptsCreate(); +void tkvReadOptsDestroy(STkvReadOpts *); + +// STkvWriteOpts +STkvWriteOpts *tkvWriteOptsCreate(); +void tkvWriteOptsDestroy(STkvWriteOpts *); + #ifdef __cplusplus } #endif diff --git a/source/libs/tkv/CMakeLists.txt b/source/libs/tkv/CMakeLists.txt index bec359d75d..6ad553b1ca 100644 --- a/source/libs/tkv/CMakeLists.txt +++ b/source/libs/tkv/CMakeLists.txt @@ -4,4 +4,8 @@ target_include_directories( tkv PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tkv" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +target_link_libraries( + tkv + PUBLIC os ) \ No newline at end of file diff --git a/source/libs/tkv/src/tkv.c b/source/libs/tkv/src/tkv.c index 6dea4a4e57..9c78c23db4 100644 --- a/source/libs/tkv/src/tkv.c +++ b/source/libs/tkv/src/tkv.c @@ -11,4 +11,71 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "tkv.h" + +struct STkvDb { + // TODO +}; +struct STkvOpts { + // TODO +}; +struct STkvCache { + // TODO +}; +struct STkvReadOpts { + // TODO +}; +struct STkvWriteOpts { + // TODO +}; + +STkvDb *tkvOpen(const STkvOpts *options, const char *path) { + // TODO + return NULL; +} + +void tkvClose(STkvDb *db) { + // TODO +} + +void tkvPut(STkvDb *db, STkvWriteOpts *pwopts, char *key, size_t keylen, char *val, size_t vallen) { + // TODO +} + +char *tkvGet(STkvDb *db, STkvReadOpts *propts, char *key, size_t keylen, size_t *vallen) { + // TODO + return NULL; +} + +STkvOpts *tkvOptionsCreate() { + // TODO + return NULL; +} + +void tkvOptionsDestroy(STkvOpts *popts) { + // TODO +} + +void tkvOptionsSetCache(STkvOpts *popts, STkvCache *pCache) { + // TODO +} + +STkvReadOpts *tkvReadOptsCreate() { + // TODO + return NULL; +} + +void tkvReadOptsDestroy(STkvReadOpts *propts) { + // TODO +} + +STkvWriteOpts *tkvWriteOptsCreate() { + // TODO + return NULL; +} + +void tkvWriteOptsDestroy(STkvWriteOpts *pwopts) { + // TODO +} \ No newline at end of file diff --git a/source/server/vnode/meta/CMakeLists.txt b/source/server/vnode/meta/CMakeLists.txt index 94bf9581d3..0de78074ee 100644 --- a/source/server/vnode/meta/CMakeLists.txt +++ b/source/server/vnode/meta/CMakeLists.txt @@ -8,8 +8,8 @@ target_include_directories( target_link_libraries( meta PUBLIC common + PUBLIC tkv ) -target_link_libraries(meta PUBLIC rocksdb) if(${BUILD_TEST}) add_subdirectory(test) diff --git a/source/server/vnode/meta/src/meta.c b/source/server/vnode/meta/src/meta.c index 000b2ca9b0..c63aa31b30 100644 --- a/source/server/vnode/meta/src/meta.c +++ b/source/server/vnode/meta/src/meta.c @@ -13,8 +13,7 @@ * along with this program. If not, see . */ -#include - +#include "tkv.h" #include "thash.h" #include "tlist.h" #include "tlockfree.h" @@ -44,13 +43,13 @@ typedef struct STableObj { struct SMeta { pthread_rwlock_t rwLock; - SHashObj * pTableObjHash; // uid --> STableObj - SList * stbList; // super table list - rocksdb_t *tbnameDb; // tbname --> uid - rocksdb_t *tagDb; // uid --> tag - rocksdb_t *schemaDb; - rocksdb_t *tagIdx; - size_t totalUsed; + SHashObj *pTableObjHash; // uid --> STableObj + SList * stbList; // super table list + STkvDb * tbnameDb; // tbname --> uid + STkvDb * tagDb; // uid --> tag + STkvDb * schemaDb; + STkvDb * tagIdx; + size_t totalUsed; }; static STable * metaTableNew(tb_uid_t uid, const char *name, int32_t sver); @@ -74,34 +73,33 @@ SMeta *metaOpen(SMetaOpts *options) { pMeta->stbList = tdListNew(sizeof(STableObj *)); // Options - rocksdb_options_t *dbOptions = rocksdb_options_create(); - rocksdb_options_set_create_if_missing(dbOptions, 1); + STkvOpts *dbOptions = tkvOptionsCreate(); taosMkDir("meta"); // Open tbname DB - pMeta->tbnameDb = rocksdb_open(dbOptions, "meta/tbname_uid_db", &err); + pMeta->tbnameDb = tkvOpen(dbOptions, "meta/tbname_uid_db"); // Open tag DB - pMeta->tagDb = rocksdb_open(dbOptions, "meta/uid_tag_db", &err); + pMeta->tagDb = tkvOpen(dbOptions, "meta/uid_tag_db"); // Open schema DB - pMeta->schemaDb = rocksdb_open(dbOptions, "meta/schema_db", &err); + pMeta->schemaDb = tkvOpen(dbOptions, "meta/schema_db"); // Open tag index - pMeta->tagIdx = rocksdb_open(dbOptions, "meta/tag_idx_db", &err); + pMeta->tagIdx = tkvOpen(dbOptions, "meta/tag_idx_db"); - rocksdb_options_destroy(dbOptions); + tkvOptionsDestroy(dbOptions); return pMeta; } void metaClose(SMeta *pMeta) { if (pMeta) { - rocksdb_close(pMeta->tagIdx); - rocksdb_close(pMeta->schemaDb); - rocksdb_close(pMeta->tagDb); - rocksdb_close(pMeta->tbnameDb); + tkvClose(pMeta->tagIdx); + tkvClose(pMeta->schemaDb); + tkvClose(pMeta->tagDb); + tkvClose(pMeta->tbnameDb); tdListFree(pMeta->stbList); taosHashCleanup(pMeta->pTableObjHash); @@ -110,22 +108,21 @@ void metaClose(SMeta *pMeta) { } int metaCreateTable(SMeta *pMeta, STableOpts *pTableOpts) { - size_t vallen; - char * err = NULL; - rocksdb_readoptions_t * ropt; - STableObj * pTableObj = NULL; - rocksdb_writeoptions_t *wopt; + size_t vallen; + STkvReadOpts *ropt; + STableObj * pTableObj = NULL; + STkvWriteOpts *wopt; // Check if table already exists - ropt = rocksdb_readoptions_create(); + ropt = tkvReadOptsCreate(); - char *uidStr = rocksdb_get(pMeta->tbnameDb, ropt, pTableOpts->name, strlen(pTableOpts->name), &vallen, &err); + char *uidStr = tkvGet(pMeta->tbnameDb, ropt, pTableOpts->name, strlen(pTableOpts->name), &vallen); if (uidStr != NULL) { // Has duplicate named table return -1; } - rocksdb_readoptions_destroy(ropt); + tkvReadOptsDestroy(ropt); // Create table obj pTableObj = metaTableObjNew(); @@ -144,12 +141,12 @@ int metaCreateTable(SMeta *pMeta, STableOpts *pTableOpts) { taosHashPut(pMeta->pTableObjHash, &(pTableObj->pTable->uid), sizeof(tb_uid_t), &pTableObj, sizeof(pTableObj)); - wopt = rocksdb_writeoptions_create(); - rocksdb_writeoptions_disable_WAL(wopt, 1); + wopt = tkvWriteOptsCreate(); + // rocksdb_writeoptions_disable_WAL(wopt, 1); // Add to tbname db - rocksdb_put(pMeta->tbnameDb, wopt, pTableOpts->name, strlen(pTableOpts->name), &pTableObj->pTable->uid, - sizeof(tb_uid_t), &err); + tkvPut(pMeta->tbnameDb, wopt, pTableOpts->name, strlen(pTableOpts->name), (char *)&pTableObj->pTable->uid, + sizeof(tb_uid_t)); // Add to schema db char id[12]; @@ -159,9 +156,9 @@ int metaCreateTable(SMeta *pMeta, STableOpts *pTableOpts) { *(int32_t *)(id + sizeof(tb_uid_t)) = schemaVersion(pTableOpts->pSchema); int size = tdEncodeSchema(&pBuf, pTableOpts->pSchema); - rocksdb_put(pMeta->schemaDb, wopt, id, 12, buf, size, &err); + tkvPut(pMeta->schemaDb, wopt, id, 12, buf, size); - rocksdb_writeoptions_destroy(wopt); + tkvWriteOptsDestroy(wopt); pthread_rwlock_unlock(&pMeta->rwLock); diff --git a/source/server/vnode/meta/test/CMakeLists.txt b/source/server/vnode/meta/test/CMakeLists.txt index ee16a28687..bca02c2907 100644 --- a/source/server/vnode/meta/test/CMakeLists.txt +++ b/source/server/vnode/meta/test/CMakeLists.txt @@ -14,8 +14,8 @@ target_link_libraries(metaTest os util common - rocksdb gtest_main + tkv ) enable_testing() add_test( From 2d8aadff21d756ab37433713d667a0e6a8f99fcb Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 12 Oct 2021 16:30:40 +0800 Subject: [PATCH 23/24] put taosGetTimestampSec in osTime --- source/common/src/ttime.c | 2 -- source/os/src/osTime.c | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 0f15dcc13b..0aa2aef4be 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -94,8 +94,6 @@ static int32_t (*parseLocaltimeFp[]) (char* timestr, int64_t* time, int32_t time parseLocaltimeWithDst }; -int32_t taosGetTimestampSec() { return (int32_t)time(NULL); } - int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t day_light) { /* parse datatime string in with tz */ if (strnchr(timestr, 'T', len, false) != NULL) { diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index bf0585e86d..7c655c0251 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -63,4 +63,6 @@ FORCE_INLINE int32_t taosGetTimeOfDay(struct timeval *tv) { return gettimeofday(tv, NULL); } +int32_t taosGetTimestampSec() { return (int32_t)time(NULL); } + #endif From 8ce34a9096a1f7f2e0a78f5eb54198b97bd8a43a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Oct 2021 16:37:44 +0800 Subject: [PATCH 24/24] fix some compile warning --- include/common/tdataformat.h | 2 +- include/os/osDir.h | 2 +- source/common/src/tdataformat.c | 2 +- source/os/src/osDir.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index c4321eb9df..2eef9db064 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -87,7 +87,7 @@ typedef struct { #define schemaColAt(s, i) ((s)->columns + i) #define tdFreeSchema(s) tfree((s)) -STSchema *tdDupSchema(STSchema *pSchema); +STSchema *tdDupSchema(const STSchema *pSchema); int tdEncodeSchema(void **buf, STSchema *pSchema); void * tdDecodeSchema(void *buf, STSchema **pRSchema); diff --git a/include/os/osDir.h b/include/os/osDir.h index 32733753a8..3ee3be2c10 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -20,7 +20,7 @@ extern "C" { #endif -void taosRemoveDir(char *dirname); +void taosRemoveDir(const char *dirname); bool taosDirExist(char *dirname); bool taosMkDir(char *dirname); void taosRemoveOldFiles(char *dirname, int32_t keepDays); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 6e76e3a8d0..94d93b651a 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -48,7 +48,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { /** * Duplicate the schema and return a new object */ -STSchema *tdDupSchema(STSchema *pSchema) { +STSchema *tdDupSchema(const STSchema *pSchema) { int tlen = sizeof(STSchema) + sizeof(STColumn) * schemaNCols(pSchema); STSchema *tSchema = (STSchema *)malloc(tlen); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 839db811fd..17ab88edf6 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -34,7 +34,7 @@ #include #include -void taosRemoveDir(char *dirname) { +void taosRemoveDir(const char *dirname) { DIR *dir = opendir(dirname); if (dir == NULL) return;