From 3347de549f7ad4cb8404aef1a1d9667df318b264 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 26 Sep 2021 10:44:50 +0800 Subject: [PATCH 01/10] refact --- include/server/vnode/memtable/memTable.h | 0 include/server/vnode/tsdb/tsdb.h | 20 ++++++++++++++++++-- source/server/vnode/tsdb/CMakeLists.txt | 4 ++++ 3 files changed, 22 insertions(+), 2 deletions(-) delete mode 100644 include/server/vnode/memtable/memTable.h diff --git a/include/server/vnode/memtable/memTable.h b/include/server/vnode/memtable/memTable.h deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/include/server/vnode/tsdb/tsdb.h b/include/server/vnode/tsdb/tsdb.h index 8bca3590c0..988669324e 100644 --- a/include/server/vnode/tsdb/tsdb.h +++ b/include/server/vnode/tsdb/tsdb.h @@ -24,16 +24,32 @@ extern "C" { typedef struct STsdb STsdb; typedef struct { + int32_t id; // TODO: use a global definition + int32_t days; + int32_t keep; + int32_t keep1; + int32_t keep2; + int32_t minRows; + int32_t maxRows; + int8_t precision; + int8_t update; } STsdbCfg; -int tsdbInit(int nthreads); -int tsdbClear(); +// Module init and clear +int tsdbInit(); +int tsdbClear(); + +// Repository operations int tsdbCreateRepo(int id); int tsdbDropRepo(int id); STsdb *tsdbOpenRepo(STsdbCfg *pCfg); int tsdbCloseRepo(STsdb *pTsdb); int tsdbForceCloseRepo(STsdb *pTsdb); +// Data commit +int tsdbInsert(STsdb *pTsdb, SSubmitMsg *pMsg); +int tsdbCommit(STsdb *pTsdb); + #ifdef __cplusplus } #endif diff --git a/source/server/vnode/tsdb/CMakeLists.txt b/source/server/vnode/tsdb/CMakeLists.txt index 19ad9e1591..3c1dc5810b 100644 --- a/source/server/vnode/tsdb/CMakeLists.txt +++ b/source/server/vnode/tsdb/CMakeLists.txt @@ -4,4 +4,8 @@ target_include_directories( tsdb PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode/tsdb" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +target_include_directories( + tsdb + PRIVATE os ) \ No newline at end of file From 6c3b627e54bcd5a9460f11ff2c2a9999ea2394a6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 27 Sep 2021 10:49:36 +0800 Subject: [PATCH 02/10] refact --- include/common/taosMsg.h | 55 +++++++++++++++++ include/server/vnode/meta/meta.h | 4 ++ include/server/vnode/tq/tq.h | 5 ++ include/server/vnode/tsdb/tsdb.h | 3 +- source/server/vnode/CMakeLists.txt | 10 +-- source/server/vnode/inc/vnodeCommit.h | 29 +++++++++ source/server/vnode/inc/vnodeInt.h | 11 ++++ source/server/vnode/inc/vnodeMemAllocator.h | 27 ++++++++ source/server/vnode/inc/vnodeWrite.h | 21 +++++++ source/server/vnode/src/vnodeCommit.c | 53 ++++++++++++++++ source/server/vnode/src/vnodeMemAllocator.c | 14 +++++ source/server/vnode/src/vnodeWrite.c | 68 +++++++++++++++++++++ source/server/vnode/tsdb/CMakeLists.txt | 1 + 13 files changed, 296 insertions(+), 5 deletions(-) create mode 100644 include/common/taosMsg.h create mode 100644 source/server/vnode/inc/vnodeCommit.h create mode 100644 source/server/vnode/inc/vnodeMemAllocator.h create mode 100644 source/server/vnode/inc/vnodeWrite.h create mode 100644 source/server/vnode/src/vnodeCommit.c create mode 100644 source/server/vnode/src/vnodeMemAllocator.c create mode 100644 source/server/vnode/src/vnodeWrite.c diff --git a/include/common/taosMsg.h b/include/common/taosMsg.h new file mode 100644 index 0000000000..0d083a4ca5 --- /dev/null +++ b/include/common/taosMsg.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TAOS_MSG_H_ +#define _TD_TAOS_MSG_H_ + +typedef struct { + /* data */ +} SSubmitReq; + +typedef struct { + /* data */ +} SSubmitRsp; + +typedef struct { + /* data */ +} SSubmitReqReader; + +typedef struct { + /* data */ +} SCreateTableReq; + +typedef struct { + /* data */ +} SCreateTableRsp; + +typedef struct { + /* data */ +} SDropTableReq; + +typedef struct { + /* data */ +} SDropTableRsp; + +typedef struct { + /* data */ +} SAlterTableReq; + +typedef struct { + /* data */ +} SAlterTableRsp; + +#endif /*_TD_TAOS_MSG_H_*/ \ No newline at end of file diff --git a/include/server/vnode/meta/meta.h b/include/server/vnode/meta/meta.h index c584c03de0..2addcdc6c3 100644 --- a/include/server/vnode/meta/meta.h +++ b/include/server/vnode/meta/meta.h @@ -20,6 +20,10 @@ extern "C" { #endif +typedef struct SMeta SMeta; + +int metaCommit(SMeta *pMeta); + #ifdef __cplusplus } #endif diff --git a/include/server/vnode/tq/tq.h b/include/server/vnode/tq/tq.h index 77d6cdf2c9..4c626a1e25 100644 --- a/include/server/vnode/tq/tq.h +++ b/include/server/vnode/tq/tq.h @@ -20,6 +20,11 @@ extern "C" { #endif +typedef struct STQ STQ; + +int tqPushMsg(void *); +int tqCommit(STQ *pTQ); + #ifdef __cplusplus } #endif diff --git a/include/server/vnode/tsdb/tsdb.h b/include/server/vnode/tsdb/tsdb.h index 988669324e..968bac2fa2 100644 --- a/include/server/vnode/tsdb/tsdb.h +++ b/include/server/vnode/tsdb/tsdb.h @@ -17,6 +17,7 @@ #define _TD_TSDB_H_ #include "os.h" +#include "taosMsg.h" #ifdef __cplusplus extern "C" { @@ -47,7 +48,7 @@ int tsdbCloseRepo(STsdb *pTsdb); int tsdbForceCloseRepo(STsdb *pTsdb); // Data commit -int tsdbInsert(STsdb *pTsdb, SSubmitMsg *pMsg); +int tsdbInsert(STsdb *pTsdb, SSubmitReq *pMsg); int tsdbCommit(STsdb *pTsdb); #ifdef __cplusplus diff --git a/source/server/vnode/CMakeLists.txt b/source/server/vnode/CMakeLists.txt index 75d24ed64e..d5778f9344 100644 --- a/source/server/vnode/CMakeLists.txt +++ b/source/server/vnode/CMakeLists.txt @@ -9,9 +9,11 @@ target_include_directories( PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode" private "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_include_directories( +target_link_libraries( vnode - PRIVATE meta - PRIVATE tq - PRIVATE tsdb + PUBLIC meta + PUBLIC tq + PUBLIC tsdb + PRIVATE os + PRIVATE common ) \ No newline at end of file diff --git a/source/server/vnode/inc/vnodeCommit.h b/source/server/vnode/inc/vnodeCommit.h new file mode 100644 index 0000000000..d37c61008c --- /dev/null +++ b/source/server/vnode/inc/vnodeCommit.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_VNODE_COMMIT_H_ +#define _TD_VNODE_COMMIT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int vnodeAsyncCommit(SVnode *pVnode); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_VNODE_COMMIT_H_*/ \ No newline at end of file diff --git a/source/server/vnode/inc/vnodeInt.h b/source/server/vnode/inc/vnodeInt.h index 855eb66c8a..79ff0bc76a 100644 --- a/source/server/vnode/inc/vnodeInt.h +++ b/source/server/vnode/inc/vnodeInt.h @@ -16,10 +16,21 @@ #ifndef _TD_VNODE_INT_H_ #define _TD_VNODE_INT_H_ +#include "tq.h" +#include "tsdb.h" +#include "meta.h" + #ifdef __cplusplus extern "C" { #endif +typedef struct SVnode { + SMeta *pMeta; + STsdb *pTsdb; + STQ * pTQ; + void * allocator; // TODO +} SVnode; + #ifdef __cplusplus } #endif diff --git a/source/server/vnode/inc/vnodeMemAllocator.h b/source/server/vnode/inc/vnodeMemAllocator.h new file mode 100644 index 0000000000..f17af4a5a2 --- /dev/null +++ b/source/server/vnode/inc/vnodeMemAllocator.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_VNODE_MEM_ALLOCATOR_H_ +#define _TD_VNODE_MEM_ALLOCATOR_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_VNODE_MEM_ALLOCATOR_H_*/ \ No newline at end of file diff --git a/source/server/vnode/inc/vnodeWrite.h b/source/server/vnode/inc/vnodeWrite.h new file mode 100644 index 0000000000..b4430cd255 --- /dev/null +++ b/source/server/vnode/inc/vnodeWrite.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_VNODE_WRITE_H_ +#define _TD_VNODE_WRITE_H_ + +int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp); + +#endif /*_TD_VNODE_WRITE_H_*/ \ No newline at end of file diff --git a/source/server/vnode/src/vnodeCommit.c b/source/server/vnode/src/vnodeCommit.c new file mode 100644 index 0000000000..3200411f4d --- /dev/null +++ b/source/server/vnode/src/vnodeCommit.c @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "vnodeInt.h" + +static int vnodeStartCommit(SVnode *pVnode); +static int vnodeEndCommit(SVnode *pVnode); + +int vnodeAsyncCommit(SVnode *pVnode) { + if (vnodeStartCommit(pVnode) < 0) { + // TODO + } + + if (tqCommit(pVnode->pTQ) < 0) { + // TODO + } + + if (metaCommit(pVnode->pMeta) < 0) { + // TODO + } + + if (tsdbCommit(pVnode->pTsdb) < 0) { + // TODO + } + + if (vnodeEndCommit(pVnode) < 0) { + // TODO + } + + return 0; +} + +static int vnodeStartCommit(SVnode *pVnode) { + // TODO + return 0; +} + +static int vnodeEndCommit(SVnode *pVnode) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/server/vnode/src/vnodeMemAllocator.c b/source/server/vnode/src/vnodeMemAllocator.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/server/vnode/src/vnodeMemAllocator.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/server/vnode/src/vnodeWrite.c b/source/server/vnode/src/vnodeWrite.c new file mode 100644 index 0000000000..d6c97132d8 --- /dev/null +++ b/source/server/vnode/src/vnodeWrite.c @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "vnodeInt.h" + +int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp) { + // TODO: Check inputs + +#if 1 + void *pMem = NULL; + if ((pMem = aMalloc(pVnode->allocator, REQ_SIZE(pReq))) == NULL) { + // No more memory to allocate, schedule an async commit + // and continue + vnodeAsyncCommit(pVnode); + + // Reset allocator and allocat more + vnodeResetAllocator(pVnode); + pMem = aMalloc(pVnode->allocator, REQ_SIZE(pReq)); + if (pMem == NULL) { + // TODO: handle the error + } + } + + // TODO: if SSubmitReq is compressed or encoded, we need to decode the request + memcpy(pMem, pReq, REQ_SIZE(pReq)); + + if (tqPushMsg((SSubmitReq *)pReq) < 0) { + // TODO: handle error + } + + SSubmitReqReader reader; + taosInitSubmitReqReader(&reader, (SSubmitReq *)pMem); + + if (tsdbInsertData((SSubmitReq *)pMem) < 0) { + // TODO: handler error + } + +#endif + + return 0; +} + +int vnodeProcessCreateTableReq(SVnode *pVnode, SCreateTableReq *pReq, SCreateTableRsp *pRsp) { + // TODO + return 0; +} + +int vnodeProcessDropTableReq(SVnode *pVnode, SDropTableReq *pReq, SDropTableRsp *pRsp) { + // TODO + return 0; +} + +int vnodeProcessAlterTableReq(SVnode *pVnode, SDropTableReq *pReq, SDropTableRsp *pRsp) { + // TODO + return 0; +} diff --git a/source/server/vnode/tsdb/CMakeLists.txt b/source/server/vnode/tsdb/CMakeLists.txt index 3c1dc5810b..4a1e154cd5 100644 --- a/source/server/vnode/tsdb/CMakeLists.txt +++ b/source/server/vnode/tsdb/CMakeLists.txt @@ -8,4 +8,5 @@ target_include_directories( target_include_directories( tsdb PRIVATE os + PRIVATE common ) \ No newline at end of file From e7b97be7cf6575d689b1f357d447c01a8320d077 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 27 Sep 2021 11:42:12 +0800 Subject: [PATCH 03/10] refact --- include/util/amalloc.h | 47 +++++++++++++++++++++ include/util/tlockfree.h | 2 +- source/server/vnode/CMakeLists.txt | 5 ++- source/server/vnode/inc/vnodeInt.h | 9 ++-- source/server/vnode/inc/vnodeMemAllocator.h | 2 + source/server/vnode/src/vnodeWrite.c | 6 +-- 6 files changed, 61 insertions(+), 10 deletions(-) create mode 100644 include/util/amalloc.h diff --git a/include/util/amalloc.h b/include/util/amalloc.h new file mode 100644 index 0000000000..944d27513e --- /dev/null +++ b/include/util/amalloc.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_AMALLOC_H_ +#define _TD_AMALLOC_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Interfaces to implement +typedef struct { + void *(*malloc)(void *, size_t size); + void *(*calloc)(void *, size_t nmemb, size_t size); + void (*free)(void *ptr, size_t size); // Do we need to set size in the allocated memory? + void *(*realloc)(void *ptr, size_t size); +} SMemAllocatorIf; + +typedef struct { + void * impl; + SMemAllocatorIf interface; +} SMemAllocator; + +#define amalloc(allocator, size) (*((allocator)->interface.malloc))((allocator)->impl, size) +#define acalloc(allocator, nmemb, size) (*((allocator)->interface.calloc))((allocator)->impl, nmemb, size) +#define arealloc(allocator, ptr, size) (*((allocator)->interface.realloc))((allocator)->impl, ptr, size) +#define afree(allocator, ptr, size) (*((allocator)->interface.free))((allocator)->impl, ptr, size) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_AMALLOC_H_*/ \ No newline at end of file diff --git a/include/util/tlockfree.h b/include/util/tlockfree.h index e960b601ca..4660dd1cbd 100644 --- a/include/util/tlockfree.h +++ b/include/util/tlockfree.h @@ -71,7 +71,7 @@ typedef void (*_ref_fn_t)(const void* pObj); // single writer multiple reader lock -typedef int32_t SRWLatch; +typedef volatile int32_t SRWLatch; void taosInitRWLatch(SRWLatch *pLatch); void taosWLockLatch(SRWLatch *pLatch); diff --git a/source/server/vnode/CMakeLists.txt b/source/server/vnode/CMakeLists.txt index d5778f9344..73605adc3f 100644 --- a/source/server/vnode/CMakeLists.txt +++ b/source/server/vnode/CMakeLists.txt @@ -14,6 +14,7 @@ target_link_libraries( PUBLIC meta PUBLIC tq PUBLIC tsdb - PRIVATE os - PRIVATE common + PUBLIC os + PUBLIC common + PUBLIC util ) \ No newline at end of file diff --git a/source/server/vnode/inc/vnodeInt.h b/source/server/vnode/inc/vnodeInt.h index 79ff0bc76a..3d218df55e 100644 --- a/source/server/vnode/inc/vnodeInt.h +++ b/source/server/vnode/inc/vnodeInt.h @@ -16,6 +16,7 @@ #ifndef _TD_VNODE_INT_H_ #define _TD_VNODE_INT_H_ +#include "amalloc.h" #include "tq.h" #include "tsdb.h" #include "meta.h" @@ -25,10 +26,10 @@ extern "C" { #endif typedef struct SVnode { - SMeta *pMeta; - STsdb *pTsdb; - STQ * pTQ; - void * allocator; // TODO + SMeta * pMeta; + STsdb * pTsdb; + STQ * pTQ; + SMemAllocator *allocator; } SVnode; #ifdef __cplusplus diff --git a/source/server/vnode/inc/vnodeMemAllocator.h b/source/server/vnode/inc/vnodeMemAllocator.h index f17af4a5a2..9b3d27776d 100644 --- a/source/server/vnode/inc/vnodeMemAllocator.h +++ b/source/server/vnode/inc/vnodeMemAllocator.h @@ -16,6 +16,8 @@ #ifndef _TD_VNODE_MEM_ALLOCATOR_H_ #define _TD_VNODE_MEM_ALLOCATOR_H_ +#include "amalloc.h" + #ifdef __cplusplus extern "C" { #endif diff --git a/source/server/vnode/src/vnodeWrite.c b/source/server/vnode/src/vnodeWrite.c index d6c97132d8..0163dd71f1 100644 --- a/source/server/vnode/src/vnodeWrite.c +++ b/source/server/vnode/src/vnodeWrite.c @@ -20,14 +20,14 @@ int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp) { #if 1 void *pMem = NULL; - if ((pMem = aMalloc(pVnode->allocator, REQ_SIZE(pReq))) == NULL) { + if ((pMem = amalloc(pVnode->allocator, REQ_SIZE(pReq))) == NULL) { // No more memory to allocate, schedule an async commit // and continue vnodeAsyncCommit(pVnode); // Reset allocator and allocat more vnodeResetAllocator(pVnode); - pMem = aMalloc(pVnode->allocator, REQ_SIZE(pReq)); + pMem = amalloc(pVnode->allocator, REQ_SIZE(pReq)); if (pMem == NULL) { // TODO: handle the error } @@ -43,7 +43,7 @@ int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp) { SSubmitReqReader reader; taosInitSubmitReqReader(&reader, (SSubmitReq *)pMem); - if (tsdbInsertData((SSubmitReq *)pMem) < 0) { + if (tsdbInsert(pVnode->pTsdb, (SSubmitReq *)pMem) < 0) { // TODO: handler error } From 7f4551db610b1c2d9c0763b6f730e6c7baff895f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 27 Sep 2021 14:44:18 +0800 Subject: [PATCH 04/10] refact --- include/libs/tkv/tkv.h | 35 ++++++++++++++++++++++++ include/server/vnode/meta/meta.h | 5 ++++ source/server/vnode/meta/CMakeLists.txt | 4 +++ source/server/vnode/meta/inc/metaInt.h | 4 +++ source/server/vnode/meta/src/meta.c | 36 +++++++++++++++++++++++++ source/server/vnode/src/vnodeWrite.c | 5 +--- 6 files changed, 85 insertions(+), 4 deletions(-) diff --git a/include/libs/tkv/tkv.h b/include/libs/tkv/tkv.h index 6072525f88..ef865e894b 100644 --- a/include/libs/tkv/tkv.h +++ b/include/libs/tkv/tkv.h @@ -20,6 +20,41 @@ extern "C" { #endif +typedef struct tkv_db_s tkv_db_t; + + +typedef struct { + /* data */ +} tkv_key_t; + +typedef struct { + bool pinned; + int64_t ref; // TODO: use util library + // TODO: add a RW latch here + void *pObj; +} tkv_obj_t; + + +typedef int (*tkv_key_comp_fn_t)(const tkv_key_t *, const tkv_key_t *); +typedef void (*tkv_get_key_fn_t)(const tkv_obj_t *, tkv_key_t *); +typedef int (*tkv_obj_encode_fn_t)(void **buf, void *pObj); +typedef void *(*tkv_obj_decode_fn_t)(void *buf, void **pObj); + +typedef struct { + uint64_t memLimit : 63; + tkv_get_key_fn_t getKey; + tkv_obj_encode_fn_t encode; + tkv_obj_decode_fn_t decode; +} tkv_db_option_t; + +tkv_db_t * tkvOpenDB(char *dir, tkv_db_option_t *); +int tkvCloseDB(tkv_db_t *); +int tkvPut(tkv_db_t *, tkv_obj_t *); +int tkvPutBatch(tkv_db_t *, tkv_obj_t **, int); +tkv_obj_t *tkvGet(tkv_key_t *); +int tkvGetBatch(tkv_db_t *, tkv_key_t **, int, tkv_obj_t **); +int tkvCommit(tkv_db_t *, void * /*TODO*/); + #ifdef __cplusplus } #endif diff --git a/include/server/vnode/meta/meta.h b/include/server/vnode/meta/meta.h index 2addcdc6c3..ca7cba9b0c 100644 --- a/include/server/vnode/meta/meta.h +++ b/include/server/vnode/meta/meta.h @@ -16,12 +16,17 @@ #ifndef _TD_META_H_ #define _TD_META_H_ +#include "taosMsg.h" + #ifdef __cplusplus extern "C" { #endif typedef struct SMeta SMeta; +int metaCreateTable(SMeta *pMeta, SCreateTableReq *pReq); +int metaDropTable(SMeta *pMeta, SDropTableReq *pReq); +int metaAlterTable(SMeta *pMeta, SAlterTableReq *pReq); int metaCommit(SMeta *pMeta); #ifdef __cplusplus diff --git a/source/server/vnode/meta/CMakeLists.txt b/source/server/vnode/meta/CMakeLists.txt index 7ce192ef3e..832e13a155 100644 --- a/source/server/vnode/meta/CMakeLists.txt +++ b/source/server/vnode/meta/CMakeLists.txt @@ -4,4 +4,8 @@ target_include_directories( meta PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode/meta" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +target_link_libraries( + meta + PUBLIC common ) \ No newline at end of file diff --git a/source/server/vnode/meta/inc/metaInt.h b/source/server/vnode/meta/inc/metaInt.h index 8b9adfca89..549ac829ab 100644 --- a/source/server/vnode/meta/inc/metaInt.h +++ b/source/server/vnode/meta/inc/metaInt.h @@ -20,6 +20,10 @@ extern "C" { #endif +struct { + tkv_db_t db; +} SMeta; + #ifdef __cplusplus } #endif diff --git a/source/server/vnode/meta/src/meta.c b/source/server/vnode/meta/src/meta.c index e69de29bb2..7d8ec11494 100644 --- a/source/server/vnode/meta/src/meta.c +++ b/source/server/vnode/meta/src/meta.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "meta.h" + +int metaCreateTable(SMeta *pMeta, SCreateTableReq *pReq) { + // TODO + return 0; +} + +int metaDropTable(SMeta *pMeta, SDropTableReq *pReq) { + // TODO + return 0; +} + +int metaAlterTable(SMeta *pMeta, SAlterTableReq *pReq) { + // TODO + return 0; +} + +int metaCommit(SMeta *pMeta) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/server/vnode/src/vnodeWrite.c b/source/server/vnode/src/vnodeWrite.c index 0163dd71f1..a3aff78d02 100644 --- a/source/server/vnode/src/vnodeWrite.c +++ b/source/server/vnode/src/vnodeWrite.c @@ -18,7 +18,6 @@ int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp) { // TODO: Check inputs -#if 1 void *pMem = NULL; if ((pMem = amalloc(pVnode->allocator, REQ_SIZE(pReq))) == NULL) { // No more memory to allocate, schedule an async commit @@ -47,8 +46,6 @@ int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp) { // TODO: handler error } -#endif - return 0; } @@ -62,7 +59,7 @@ int vnodeProcessDropTableReq(SVnode *pVnode, SDropTableReq *pReq, SDropTableRsp return 0; } -int vnodeProcessAlterTableReq(SVnode *pVnode, SDropTableReq *pReq, SDropTableRsp *pRsp) { +int vnodeProcessAlterTableReq(SVnode *pVnode, SAlterTableReq *pReq, SAlterTableRsp *pRsp) { // TODO return 0; } From 5e95ab2e6ceca8daeb7b091f2a99c5a29e80c9dd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 27 Sep 2021 15:14:33 +0800 Subject: [PATCH 05/10] refact --- include/libs/tkv/tkv.h | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/include/libs/tkv/tkv.h b/include/libs/tkv/tkv.h index ef865e894b..96585cfd80 100644 --- a/include/libs/tkv/tkv.h +++ b/include/libs/tkv/tkv.h @@ -22,7 +22,6 @@ extern "C" { typedef struct tkv_db_s tkv_db_t; - typedef struct { /* data */ } tkv_key_t; @@ -31,29 +30,33 @@ typedef struct { bool pinned; int64_t ref; // TODO: use util library // TODO: add a RW latch here - void *pObj; + uint64_t offset; + void * pObj; } tkv_obj_t; - typedef int (*tkv_key_comp_fn_t)(const tkv_key_t *, const tkv_key_t *); typedef void (*tkv_get_key_fn_t)(const tkv_obj_t *, tkv_key_t *); typedef int (*tkv_obj_encode_fn_t)(void **buf, void *pObj); typedef void *(*tkv_obj_decode_fn_t)(void *buf, void **pObj); +typedef int (*tkv_obj_comp_fn_t)(const tkv_obj_t *, const tkv_obj_t *); typedef struct { - uint64_t memLimit : 63; + uint64_t memLimit; tkv_get_key_fn_t getKey; tkv_obj_encode_fn_t encode; tkv_obj_decode_fn_t decode; + tkv_obj_comp_fn_t compare; } tkv_db_option_t; -tkv_db_t * tkvOpenDB(char *dir, tkv_db_option_t *); -int tkvCloseDB(tkv_db_t *); -int tkvPut(tkv_db_t *, tkv_obj_t *); -int tkvPutBatch(tkv_db_t *, tkv_obj_t **, int); -tkv_obj_t *tkvGet(tkv_key_t *); -int tkvGetBatch(tkv_db_t *, tkv_key_t **, int, tkv_obj_t **); -int tkvCommit(tkv_db_t *, void * /*TODO*/); +tkv_db_t * tkvOpenDB(char *dir, tkv_db_option_t *); +int tkvCloseDB(tkv_db_t *); +int tkvPut(tkv_db_t *, tkv_obj_t *); +int tkvPutBatch(tkv_db_t *, tkv_obj_t **, int); +const tkv_obj_t *tkvGet(tkv_key_t *); +int tkvGetBatch(tkv_db_t *, tkv_key_t **, int, tkv_obj_t **); +int tkvDrop(tkv_db_t *, tkv_key_t *); +int tkvCommit(tkv_db_t *, void * /*TODO*/); +// TODO: iter function #ifdef __cplusplus } From ff2b8e0602288da0e3747e33bc8f32fb41dffed1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 27 Sep 2021 15:39:25 +0800 Subject: [PATCH 06/10] refact --- include/libs/tkv/tkv.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/include/libs/tkv/tkv.h b/include/libs/tkv/tkv.h index 96585cfd80..22da145237 100644 --- a/include/libs/tkv/tkv.h +++ b/include/libs/tkv/tkv.h @@ -56,7 +56,13 @@ const tkv_obj_t *tkvGet(tkv_key_t *); int tkvGetBatch(tkv_db_t *, tkv_key_t **, int, tkv_obj_t **); int tkvDrop(tkv_db_t *, tkv_key_t *); int tkvCommit(tkv_db_t *, void * /*TODO*/); -// TODO: iter function + +typedef struct { +} tkv_db_iter_t; + +tkv_db_iter_t * tkvIterNew(tkv_db_t *); +void tkvIterFree(tkv_db_iter_t *); +const tkv_obj_t *tkvIterNext(tkv_db_iter_t *); #ifdef __cplusplus } From 7b6002bd94c6ad6af8a8c87604585960bce34320 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 27 Sep 2021 16:02:55 +0800 Subject: [PATCH 07/10] refact --- include/libs/tkv/tkv.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/libs/tkv/tkv.h b/include/libs/tkv/tkv.h index 22da145237..3f0e2833b8 100644 --- a/include/libs/tkv/tkv.h +++ b/include/libs/tkv/tkv.h @@ -51,10 +51,11 @@ typedef struct { tkv_db_t * tkvOpenDB(char *dir, tkv_db_option_t *); int tkvCloseDB(tkv_db_t *); int tkvPut(tkv_db_t *, tkv_obj_t *); -int tkvPutBatch(tkv_db_t *, tkv_obj_t **, int); +int tkvPutBatch(tkv_db_t *, tkv_obj_t **, int); // TODO: use array here const tkv_obj_t *tkvGet(tkv_key_t *); -int tkvGetBatch(tkv_db_t *, tkv_key_t **, int, tkv_obj_t **); +int tkvGetBatch(tkv_db_t *, tkv_key_t **, int, tkv_obj_t **); // TODO: use array here int tkvDrop(tkv_db_t *, tkv_key_t *); +int tkvDropBatch(tkv_db_t *, tkv_key_t **, int); // TODO: use array here int tkvCommit(tkv_db_t *, void * /*TODO*/); typedef struct { From 773db063d2aea42e983c53c96655ae4361b57bbc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 27 Sep 2021 16:05:18 +0800 Subject: [PATCH 08/10] refact --- include/libs/tkv/tkv.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/include/libs/tkv/tkv.h b/include/libs/tkv/tkv.h index 3f0e2833b8..73c0156bd1 100644 --- a/include/libs/tkv/tkv.h +++ b/include/libs/tkv/tkv.h @@ -39,13 +39,15 @@ typedef void (*tkv_get_key_fn_t)(const tkv_obj_t *, tkv_key_t *); typedef int (*tkv_obj_encode_fn_t)(void **buf, void *pObj); typedef void *(*tkv_obj_decode_fn_t)(void *buf, void **pObj); typedef int (*tkv_obj_comp_fn_t)(const tkv_obj_t *, const tkv_obj_t *); +typedef void (*tkv_obj_destroy_fn_t)(void *); typedef struct { - uint64_t memLimit; - tkv_get_key_fn_t getKey; - tkv_obj_encode_fn_t encode; - tkv_obj_decode_fn_t decode; - tkv_obj_comp_fn_t compare; + uint64_t memLimit; + tkv_get_key_fn_t getKey; + tkv_obj_encode_fn_t encode; + tkv_obj_decode_fn_t decode; + tkv_obj_comp_fn_t compare; + tkv_obj_destroy_fn_t destroy; } tkv_db_option_t; tkv_db_t * tkvOpenDB(char *dir, tkv_db_option_t *); From eec3543c093fe65d8b625faa117fa1f3eec13083 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 27 Sep 2021 17:08:02 +0800 Subject: [PATCH 09/10] refact --- include/util/amalloc.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/include/util/amalloc.h b/include/util/amalloc.h index 944d27513e..6d2869f719 100644 --- a/include/util/amalloc.h +++ b/include/util/amalloc.h @@ -35,10 +35,14 @@ typedef struct { SMemAllocatorIf interface; } SMemAllocator; -#define amalloc(allocator, size) (*((allocator)->interface.malloc))((allocator)->impl, size) -#define acalloc(allocator, nmemb, size) (*((allocator)->interface.calloc))((allocator)->impl, nmemb, size) -#define arealloc(allocator, ptr, size) (*((allocator)->interface.realloc))((allocator)->impl, ptr, size) -#define afree(allocator, ptr, size) (*((allocator)->interface.free))((allocator)->impl, ptr, size) +#define amalloc(allocator, size) \ + ((allocator) ? (*((allocator)->interface.malloc))((allocator)->impl, (size)) : malloc(size)) +#define acalloc(allocator, nmemb, size) \ + ((allocator) ? (*((allocator)->interface.calloc))((allocator)->impl, (nmemb), (size)) : calloc((nmemb), (size))) +#define arealloc(allocator, ptr, size) \ + ((allocator) ? (*((allocator)->interface.realloc))((allocator)->impl, (ptr), (size)) : realloc((ptr), (size))) +#define afree(allocator, ptr, size) \ + ((allocator) ? (*((allocator)->interface.free))((allocator)->impl, (ptr), (size)) : free(ptr)) #ifdef __cplusplus } From 4e306a79cdc88a6f30c251add13a68ffc4ed44c2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 30 Sep 2021 13:37:52 +0800 Subject: [PATCH 10/10] use https for deps download --- cmake/cjson_CMakeLists.txt.in | 2 +- cmake/gtest_CMakeLists.txt.in | 2 +- cmake/lz4_CMakeLists.txt.in | 2 +- cmake/zlib_CMakeLists.txt.in | 2 +- source/server/vnode/src/vnodeWrite.c | 2 ++ 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cmake/cjson_CMakeLists.txt.in b/cmake/cjson_CMakeLists.txt.in index 9995d1f8d8..6c285d1a7e 100644 --- a/cmake/cjson_CMakeLists.txt.in +++ b/cmake/cjson_CMakeLists.txt.in @@ -1,7 +1,7 @@ # cjson ExternalProject_Add(cjson - GIT_REPOSITORY git@github.com:taosdata-contrib/cJSON.git + GIT_REPOSITORY https://github.com/taosdata-contrib/cJSON.git GIT_TAG v1.7.15 SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/cJson" BINARY_DIR "" diff --git a/cmake/gtest_CMakeLists.txt.in b/cmake/gtest_CMakeLists.txt.in index 655870e0c9..e54806ae3b 100644 --- a/cmake/gtest_CMakeLists.txt.in +++ b/cmake/gtest_CMakeLists.txt.in @@ -1,7 +1,7 @@ # googletest ExternalProject_Add(googletest - GIT_REPOSITORY git@github.com:taosdata-contrib/googletest.git + GIT_REPOSITORY https://github.com/taosdata-contrib/googletest.git GIT_TAG release-1.11.0 SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/googletest" BINARY_DIR "" diff --git a/cmake/lz4_CMakeLists.txt.in b/cmake/lz4_CMakeLists.txt.in index 5dba4f2ab1..4b25d48e86 100644 --- a/cmake/lz4_CMakeLists.txt.in +++ b/cmake/lz4_CMakeLists.txt.in @@ -1,7 +1,7 @@ # lz4 ExternalProject_Add(lz4 - GIT_REPOSITORY git@github.com:taosdata-contrib/lz4.git + GIT_REPOSITORY https://github.com/taosdata-contrib/lz4.git GIT_TAG v1.9.3 SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/lz4" BINARY_DIR "" diff --git a/cmake/zlib_CMakeLists.txt.in b/cmake/zlib_CMakeLists.txt.in index c997fc7983..2176486519 100644 --- a/cmake/zlib_CMakeLists.txt.in +++ b/cmake/zlib_CMakeLists.txt.in @@ -1,7 +1,7 @@ # zlib ExternalProject_Add(zlib - GIT_REPOSITORY git@github.com:taosdata-contrib/zlib.git + GIT_REPOSITORY https://github.com/taosdata-contrib/zlib.git GIT_TAG v1.2.11 SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/zlib" BINARY_DIR "" diff --git a/source/server/vnode/src/vnodeWrite.c b/source/server/vnode/src/vnodeWrite.c index a3aff78d02..d4ef4ab337 100644 --- a/source/server/vnode/src/vnodeWrite.c +++ b/source/server/vnode/src/vnodeWrite.c @@ -18,6 +18,7 @@ int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp) { // TODO: Check inputs +#if 0 void *pMem = NULL; if ((pMem = amalloc(pVnode->allocator, REQ_SIZE(pReq))) == NULL) { // No more memory to allocate, schedule an async commit @@ -45,6 +46,7 @@ int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp) { if (tsdbInsert(pVnode->pTsdb, (SSubmitReq *)pMem) < 0) { // TODO: handler error } +#endif return 0; }