diff --git a/cmake/cjson_CMakeLists.txt.in b/cmake/cjson_CMakeLists.txt.in index 9995d1f8d8..6c285d1a7e 100644 --- a/cmake/cjson_CMakeLists.txt.in +++ b/cmake/cjson_CMakeLists.txt.in @@ -1,7 +1,7 @@ # cjson ExternalProject_Add(cjson - GIT_REPOSITORY git@github.com:taosdata-contrib/cJSON.git + GIT_REPOSITORY https://github.com/taosdata-contrib/cJSON.git GIT_TAG v1.7.15 SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/cJson" BINARY_DIR "" diff --git a/cmake/gtest_CMakeLists.txt.in b/cmake/gtest_CMakeLists.txt.in index 655870e0c9..e54806ae3b 100644 --- a/cmake/gtest_CMakeLists.txt.in +++ b/cmake/gtest_CMakeLists.txt.in @@ -1,7 +1,7 @@ # googletest ExternalProject_Add(googletest - GIT_REPOSITORY git@github.com:taosdata-contrib/googletest.git + GIT_REPOSITORY https://github.com/taosdata-contrib/googletest.git GIT_TAG release-1.11.0 SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/googletest" BINARY_DIR "" diff --git a/cmake/lz4_CMakeLists.txt.in b/cmake/lz4_CMakeLists.txt.in index 5dba4f2ab1..4b25d48e86 100644 --- a/cmake/lz4_CMakeLists.txt.in +++ b/cmake/lz4_CMakeLists.txt.in @@ -1,7 +1,7 @@ # lz4 ExternalProject_Add(lz4 - GIT_REPOSITORY git@github.com:taosdata-contrib/lz4.git + GIT_REPOSITORY https://github.com/taosdata-contrib/lz4.git GIT_TAG v1.9.3 SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/lz4" BINARY_DIR "" diff --git a/cmake/zlib_CMakeLists.txt.in b/cmake/zlib_CMakeLists.txt.in index c997fc7983..2176486519 100644 --- a/cmake/zlib_CMakeLists.txt.in +++ b/cmake/zlib_CMakeLists.txt.in @@ -1,7 +1,7 @@ # zlib ExternalProject_Add(zlib - GIT_REPOSITORY git@github.com:taosdata-contrib/zlib.git + GIT_REPOSITORY https://github.com/taosdata-contrib/zlib.git GIT_TAG v1.2.11 SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/zlib" BINARY_DIR "" diff --git a/include/common/taosMsg.h b/include/common/taosMsg.h new file mode 100644 index 0000000000..0d083a4ca5 --- /dev/null +++ b/include/common/taosMsg.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TAOS_MSG_H_ +#define _TD_TAOS_MSG_H_ + +typedef struct { + /* data */ +} SSubmitReq; + +typedef struct { + /* data */ +} SSubmitRsp; + +typedef struct { + /* data */ +} SSubmitReqReader; + +typedef struct { + /* data */ +} SCreateTableReq; + +typedef struct { + /* data */ +} SCreateTableRsp; + +typedef struct { + /* data */ +} SDropTableReq; + +typedef struct { + /* data */ +} SDropTableRsp; + +typedef struct { + /* data */ +} SAlterTableReq; + +typedef struct { + /* data */ +} SAlterTableRsp; + +#endif /*_TD_TAOS_MSG_H_*/ \ No newline at end of file diff --git a/include/libs/tkv/tkv.h b/include/libs/tkv/tkv.h index 6072525f88..73c0156bd1 100644 --- a/include/libs/tkv/tkv.h +++ b/include/libs/tkv/tkv.h @@ -20,6 +20,53 @@ extern "C" { #endif +typedef struct tkv_db_s tkv_db_t; + +typedef struct { + /* data */ +} tkv_key_t; + +typedef struct { + bool pinned; + int64_t ref; // TODO: use util library + // TODO: add a RW latch here + uint64_t offset; + void * pObj; +} tkv_obj_t; + +typedef int (*tkv_key_comp_fn_t)(const tkv_key_t *, const tkv_key_t *); +typedef void (*tkv_get_key_fn_t)(const tkv_obj_t *, tkv_key_t *); +typedef int (*tkv_obj_encode_fn_t)(void **buf, void *pObj); +typedef void *(*tkv_obj_decode_fn_t)(void *buf, void **pObj); +typedef int (*tkv_obj_comp_fn_t)(const tkv_obj_t *, const tkv_obj_t *); +typedef void (*tkv_obj_destroy_fn_t)(void *); + +typedef struct { + uint64_t memLimit; + tkv_get_key_fn_t getKey; + tkv_obj_encode_fn_t encode; + tkv_obj_decode_fn_t decode; + tkv_obj_comp_fn_t compare; + tkv_obj_destroy_fn_t destroy; +} tkv_db_option_t; + +tkv_db_t * tkvOpenDB(char *dir, tkv_db_option_t *); +int tkvCloseDB(tkv_db_t *); +int tkvPut(tkv_db_t *, tkv_obj_t *); +int tkvPutBatch(tkv_db_t *, tkv_obj_t **, int); // TODO: use array here +const tkv_obj_t *tkvGet(tkv_key_t *); +int tkvGetBatch(tkv_db_t *, tkv_key_t **, int, tkv_obj_t **); // TODO: use array here +int tkvDrop(tkv_db_t *, tkv_key_t *); +int tkvDropBatch(tkv_db_t *, tkv_key_t **, int); // TODO: use array here +int tkvCommit(tkv_db_t *, void * /*TODO*/); + +typedef struct { +} tkv_db_iter_t; + +tkv_db_iter_t * tkvIterNew(tkv_db_t *); +void tkvIterFree(tkv_db_iter_t *); +const tkv_obj_t *tkvIterNext(tkv_db_iter_t *); + #ifdef __cplusplus } #endif diff --git a/include/server/vnode/memtable/memTable.h b/include/server/vnode/memtable/memTable.h deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/include/server/vnode/meta/meta.h b/include/server/vnode/meta/meta.h index c584c03de0..ca7cba9b0c 100644 --- a/include/server/vnode/meta/meta.h +++ b/include/server/vnode/meta/meta.h @@ -16,10 +16,19 @@ #ifndef _TD_META_H_ #define _TD_META_H_ +#include "taosMsg.h" + #ifdef __cplusplus extern "C" { #endif +typedef struct SMeta SMeta; + +int metaCreateTable(SMeta *pMeta, SCreateTableReq *pReq); +int metaDropTable(SMeta *pMeta, SDropTableReq *pReq); +int metaAlterTable(SMeta *pMeta, SAlterTableReq *pReq); +int metaCommit(SMeta *pMeta); + #ifdef __cplusplus } #endif diff --git a/include/server/vnode/tq/tq.h b/include/server/vnode/tq/tq.h index 77d6cdf2c9..4c626a1e25 100644 --- a/include/server/vnode/tq/tq.h +++ b/include/server/vnode/tq/tq.h @@ -20,6 +20,11 @@ extern "C" { #endif +typedef struct STQ STQ; + +int tqPushMsg(void *); +int tqCommit(STQ *pTQ); + #ifdef __cplusplus } #endif diff --git a/include/server/vnode/tsdb/tsdb.h b/include/server/vnode/tsdb/tsdb.h index 8bca3590c0..968bac2fa2 100644 --- a/include/server/vnode/tsdb/tsdb.h +++ b/include/server/vnode/tsdb/tsdb.h @@ -17,6 +17,7 @@ #define _TD_TSDB_H_ #include "os.h" +#include "taosMsg.h" #ifdef __cplusplus extern "C" { @@ -24,16 +25,32 @@ extern "C" { typedef struct STsdb STsdb; typedef struct { + int32_t id; // TODO: use a global definition + int32_t days; + int32_t keep; + int32_t keep1; + int32_t keep2; + int32_t minRows; + int32_t maxRows; + int8_t precision; + int8_t update; } STsdbCfg; -int tsdbInit(int nthreads); -int tsdbClear(); +// Module init and clear +int tsdbInit(); +int tsdbClear(); + +// Repository operations int tsdbCreateRepo(int id); int tsdbDropRepo(int id); STsdb *tsdbOpenRepo(STsdbCfg *pCfg); int tsdbCloseRepo(STsdb *pTsdb); int tsdbForceCloseRepo(STsdb *pTsdb); +// Data commit +int tsdbInsert(STsdb *pTsdb, SSubmitReq *pMsg); +int tsdbCommit(STsdb *pTsdb); + #ifdef __cplusplus } #endif diff --git a/include/util/amalloc.h b/include/util/amalloc.h new file mode 100644 index 0000000000..6d2869f719 --- /dev/null +++ b/include/util/amalloc.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_AMALLOC_H_ +#define _TD_AMALLOC_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Interfaces to implement +typedef struct { + void *(*malloc)(void *, size_t size); + void *(*calloc)(void *, size_t nmemb, size_t size); + void (*free)(void *ptr, size_t size); // Do we need to set size in the allocated memory? + void *(*realloc)(void *ptr, size_t size); +} SMemAllocatorIf; + +typedef struct { + void * impl; + SMemAllocatorIf interface; +} SMemAllocator; + +#define amalloc(allocator, size) \ + ((allocator) ? (*((allocator)->interface.malloc))((allocator)->impl, (size)) : malloc(size)) +#define acalloc(allocator, nmemb, size) \ + ((allocator) ? (*((allocator)->interface.calloc))((allocator)->impl, (nmemb), (size)) : calloc((nmemb), (size))) +#define arealloc(allocator, ptr, size) \ + ((allocator) ? (*((allocator)->interface.realloc))((allocator)->impl, (ptr), (size)) : realloc((ptr), (size))) +#define afree(allocator, ptr, size) \ + ((allocator) ? (*((allocator)->interface.free))((allocator)->impl, (ptr), (size)) : free(ptr)) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_AMALLOC_H_*/ \ No newline at end of file diff --git a/include/util/tlockfree.h b/include/util/tlockfree.h index 82c9607ff4..fec346cd02 100644 --- a/include/util/tlockfree.h +++ b/include/util/tlockfree.h @@ -70,7 +70,7 @@ typedef void (*_ref_fn_t)(const void* pObj); // single writer multiple reader lock -typedef int32_t SRWLatch; +typedef volatile int32_t SRWLatch; void taosInitRWLatch(SRWLatch *pLatch); void taosWLockLatch(SRWLatch *pLatch); diff --git a/source/server/vnode/inc/vnodeCommit.h b/source/server/vnode/inc/vnodeCommit.h new file mode 100644 index 0000000000..d37c61008c --- /dev/null +++ b/source/server/vnode/inc/vnodeCommit.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_VNODE_COMMIT_H_ +#define _TD_VNODE_COMMIT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int vnodeAsyncCommit(SVnode *pVnode); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_VNODE_COMMIT_H_*/ \ No newline at end of file diff --git a/source/server/vnode/inc/vnodeInt.h b/source/server/vnode/inc/vnodeInt.h index 12568d21b8..545d376f49 100644 --- a/source/server/vnode/inc/vnodeInt.h +++ b/source/server/vnode/inc/vnodeInt.h @@ -16,14 +16,26 @@ #ifndef _TD_VNODE_INT_H_ #define _TD_VNODE_INT_H_ + +#include "os.h" +#include "amalloc.h" +#include "meta.h" +#include "taosmsg.h" +#include "tq.h" +#include "trpc.h" +#include "tsdb.h" +#include "vnode.h" + #ifdef __cplusplus extern "C" { #endif -#include "os.h" -#include "taosmsg.h" -#include "trpc.h" -#include "vnode.h" +typedef struct SVnode { + SMeta * pMeta; + STsdb * pTsdb; + STQ * pTQ; + SMemAllocator *allocator; +} SVnode; #ifdef __cplusplus } diff --git a/source/server/vnode/inc/vnodeMemAllocator.h b/source/server/vnode/inc/vnodeMemAllocator.h new file mode 100644 index 0000000000..9b3d27776d --- /dev/null +++ b/source/server/vnode/inc/vnodeMemAllocator.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_VNODE_MEM_ALLOCATOR_H_ +#define _TD_VNODE_MEM_ALLOCATOR_H_ + +#include "amalloc.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_VNODE_MEM_ALLOCATOR_H_*/ \ No newline at end of file diff --git a/source/server/vnode/inc/vnodeWrite.h b/source/server/vnode/inc/vnodeWrite.h new file mode 100644 index 0000000000..b4430cd255 --- /dev/null +++ b/source/server/vnode/inc/vnodeWrite.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_VNODE_WRITE_H_ +#define _TD_VNODE_WRITE_H_ + +int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp); + +#endif /*_TD_VNODE_WRITE_H_*/ \ No newline at end of file diff --git a/source/server/vnode/meta/CMakeLists.txt b/source/server/vnode/meta/CMakeLists.txt index 7ce192ef3e..832e13a155 100644 --- a/source/server/vnode/meta/CMakeLists.txt +++ b/source/server/vnode/meta/CMakeLists.txt @@ -4,4 +4,8 @@ target_include_directories( meta PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode/meta" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +target_link_libraries( + meta + PUBLIC common ) \ No newline at end of file diff --git a/source/server/vnode/meta/inc/metaInt.h b/source/server/vnode/meta/inc/metaInt.h index 8b9adfca89..549ac829ab 100644 --- a/source/server/vnode/meta/inc/metaInt.h +++ b/source/server/vnode/meta/inc/metaInt.h @@ -20,6 +20,10 @@ extern "C" { #endif +struct { + tkv_db_t db; +} SMeta; + #ifdef __cplusplus } #endif diff --git a/source/server/vnode/meta/src/meta.c b/source/server/vnode/meta/src/meta.c index e69de29bb2..7d8ec11494 100644 --- a/source/server/vnode/meta/src/meta.c +++ b/source/server/vnode/meta/src/meta.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "meta.h" + +int metaCreateTable(SMeta *pMeta, SCreateTableReq *pReq) { + // TODO + return 0; +} + +int metaDropTable(SMeta *pMeta, SDropTableReq *pReq) { + // TODO + return 0; +} + +int metaAlterTable(SMeta *pMeta, SAlterTableReq *pReq) { + // TODO + return 0; +} + +int metaCommit(SMeta *pMeta) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/server/vnode/src/vnodeCommit.c b/source/server/vnode/src/vnodeCommit.c new file mode 100644 index 0000000000..3200411f4d --- /dev/null +++ b/source/server/vnode/src/vnodeCommit.c @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "vnodeInt.h" + +static int vnodeStartCommit(SVnode *pVnode); +static int vnodeEndCommit(SVnode *pVnode); + +int vnodeAsyncCommit(SVnode *pVnode) { + if (vnodeStartCommit(pVnode) < 0) { + // TODO + } + + if (tqCommit(pVnode->pTQ) < 0) { + // TODO + } + + if (metaCommit(pVnode->pMeta) < 0) { + // TODO + } + + if (tsdbCommit(pVnode->pTsdb) < 0) { + // TODO + } + + if (vnodeEndCommit(pVnode) < 0) { + // TODO + } + + return 0; +} + +static int vnodeStartCommit(SVnode *pVnode) { + // TODO + return 0; +} + +static int vnodeEndCommit(SVnode *pVnode) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/server/vnode/src/vnodeMemAllocator.c b/source/server/vnode/src/vnodeMemAllocator.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/server/vnode/src/vnodeMemAllocator.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/server/vnode/src/vnodeWrite.c b/source/server/vnode/src/vnodeWrite.c new file mode 100644 index 0000000000..d4ef4ab337 --- /dev/null +++ b/source/server/vnode/src/vnodeWrite.c @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "vnodeInt.h" + +int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp) { + // TODO: Check inputs + +#if 0 + void *pMem = NULL; + if ((pMem = amalloc(pVnode->allocator, REQ_SIZE(pReq))) == NULL) { + // No more memory to allocate, schedule an async commit + // and continue + vnodeAsyncCommit(pVnode); + + // Reset allocator and allocat more + vnodeResetAllocator(pVnode); + pMem = amalloc(pVnode->allocator, REQ_SIZE(pReq)); + if (pMem == NULL) { + // TODO: handle the error + } + } + + // TODO: if SSubmitReq is compressed or encoded, we need to decode the request + memcpy(pMem, pReq, REQ_SIZE(pReq)); + + if (tqPushMsg((SSubmitReq *)pReq) < 0) { + // TODO: handle error + } + + SSubmitReqReader reader; + taosInitSubmitReqReader(&reader, (SSubmitReq *)pMem); + + if (tsdbInsert(pVnode->pTsdb, (SSubmitReq *)pMem) < 0) { + // TODO: handler error + } +#endif + + return 0; +} + +int vnodeProcessCreateTableReq(SVnode *pVnode, SCreateTableReq *pReq, SCreateTableRsp *pRsp) { + // TODO + return 0; +} + +int vnodeProcessDropTableReq(SVnode *pVnode, SDropTableReq *pReq, SDropTableRsp *pRsp) { + // TODO + return 0; +} + +int vnodeProcessAlterTableReq(SVnode *pVnode, SAlterTableReq *pReq, SAlterTableRsp *pRsp) { + // TODO + return 0; +} diff --git a/source/server/vnode/tsdb/CMakeLists.txt b/source/server/vnode/tsdb/CMakeLists.txt index 19ad9e1591..4a1e154cd5 100644 --- a/source/server/vnode/tsdb/CMakeLists.txt +++ b/source/server/vnode/tsdb/CMakeLists.txt @@ -4,4 +4,9 @@ target_include_directories( tsdb PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode/tsdb" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +target_include_directories( + tsdb + PRIVATE os + PRIVATE common ) \ No newline at end of file