From 3347de549f7ad4cb8404aef1a1d9667df318b264 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 26 Sep 2021 10:44:50 +0800 Subject: [PATCH 01/44] refact --- include/server/vnode/memtable/memTable.h | 0 include/server/vnode/tsdb/tsdb.h | 20 ++++++++++++++++++-- source/server/vnode/tsdb/CMakeLists.txt | 4 ++++ 3 files changed, 22 insertions(+), 2 deletions(-) delete mode 100644 include/server/vnode/memtable/memTable.h diff --git a/include/server/vnode/memtable/memTable.h b/include/server/vnode/memtable/memTable.h deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/include/server/vnode/tsdb/tsdb.h b/include/server/vnode/tsdb/tsdb.h index 8bca3590c0..988669324e 100644 --- a/include/server/vnode/tsdb/tsdb.h +++ b/include/server/vnode/tsdb/tsdb.h @@ -24,16 +24,32 @@ extern "C" { typedef struct STsdb STsdb; typedef struct { + int32_t id; // TODO: use a global definition + int32_t days; + int32_t keep; + int32_t keep1; + int32_t keep2; + int32_t minRows; + int32_t maxRows; + int8_t precision; + int8_t update; } STsdbCfg; -int tsdbInit(int nthreads); -int tsdbClear(); +// Module init and clear +int tsdbInit(); +int tsdbClear(); + +// Repository operations int tsdbCreateRepo(int id); int tsdbDropRepo(int id); STsdb *tsdbOpenRepo(STsdbCfg *pCfg); int tsdbCloseRepo(STsdb *pTsdb); int tsdbForceCloseRepo(STsdb *pTsdb); +// Data commit +int tsdbInsert(STsdb *pTsdb, SSubmitMsg *pMsg); +int tsdbCommit(STsdb *pTsdb); + #ifdef __cplusplus } #endif diff --git a/source/server/vnode/tsdb/CMakeLists.txt b/source/server/vnode/tsdb/CMakeLists.txt index 19ad9e1591..3c1dc5810b 100644 --- a/source/server/vnode/tsdb/CMakeLists.txt +++ b/source/server/vnode/tsdb/CMakeLists.txt @@ -4,4 +4,8 @@ target_include_directories( tsdb PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode/tsdb" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +target_include_directories( + tsdb + PRIVATE os ) \ No newline at end of file From 6c3b627e54bcd5a9460f11ff2c2a9999ea2394a6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 27 Sep 2021 10:49:36 +0800 Subject: [PATCH 02/44] refact --- include/common/taosMsg.h | 55 +++++++++++++++++ include/server/vnode/meta/meta.h | 4 ++ include/server/vnode/tq/tq.h | 5 ++ include/server/vnode/tsdb/tsdb.h | 3 +- source/server/vnode/CMakeLists.txt | 10 +-- source/server/vnode/inc/vnodeCommit.h | 29 +++++++++ source/server/vnode/inc/vnodeInt.h | 11 ++++ source/server/vnode/inc/vnodeMemAllocator.h | 27 ++++++++ source/server/vnode/inc/vnodeWrite.h | 21 +++++++ source/server/vnode/src/vnodeCommit.c | 53 ++++++++++++++++ source/server/vnode/src/vnodeMemAllocator.c | 14 +++++ source/server/vnode/src/vnodeWrite.c | 68 +++++++++++++++++++++ source/server/vnode/tsdb/CMakeLists.txt | 1 + 13 files changed, 296 insertions(+), 5 deletions(-) create mode 100644 include/common/taosMsg.h create mode 100644 source/server/vnode/inc/vnodeCommit.h create mode 100644 source/server/vnode/inc/vnodeMemAllocator.h create mode 100644 source/server/vnode/inc/vnodeWrite.h create mode 100644 source/server/vnode/src/vnodeCommit.c create mode 100644 source/server/vnode/src/vnodeMemAllocator.c create mode 100644 source/server/vnode/src/vnodeWrite.c diff --git a/include/common/taosMsg.h b/include/common/taosMsg.h new file mode 100644 index 0000000000..0d083a4ca5 --- /dev/null +++ b/include/common/taosMsg.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TAOS_MSG_H_ +#define _TD_TAOS_MSG_H_ + +typedef struct { + /* data */ +} SSubmitReq; + +typedef struct { + /* data */ +} SSubmitRsp; + +typedef struct { + /* data */ +} SSubmitReqReader; + +typedef struct { + /* data */ +} SCreateTableReq; + +typedef struct { + /* data */ +} SCreateTableRsp; + +typedef struct { + /* data */ +} SDropTableReq; + +typedef struct { + /* data */ +} SDropTableRsp; + +typedef struct { + /* data */ +} SAlterTableReq; + +typedef struct { + /* data */ +} SAlterTableRsp; + +#endif /*_TD_TAOS_MSG_H_*/ \ No newline at end of file diff --git a/include/server/vnode/meta/meta.h b/include/server/vnode/meta/meta.h index c584c03de0..2addcdc6c3 100644 --- a/include/server/vnode/meta/meta.h +++ b/include/server/vnode/meta/meta.h @@ -20,6 +20,10 @@ extern "C" { #endif +typedef struct SMeta SMeta; + +int metaCommit(SMeta *pMeta); + #ifdef __cplusplus } #endif diff --git a/include/server/vnode/tq/tq.h b/include/server/vnode/tq/tq.h index 77d6cdf2c9..4c626a1e25 100644 --- a/include/server/vnode/tq/tq.h +++ b/include/server/vnode/tq/tq.h @@ -20,6 +20,11 @@ extern "C" { #endif +typedef struct STQ STQ; + +int tqPushMsg(void *); +int tqCommit(STQ *pTQ); + #ifdef __cplusplus } #endif diff --git a/include/server/vnode/tsdb/tsdb.h b/include/server/vnode/tsdb/tsdb.h index 988669324e..968bac2fa2 100644 --- a/include/server/vnode/tsdb/tsdb.h +++ b/include/server/vnode/tsdb/tsdb.h @@ -17,6 +17,7 @@ #define _TD_TSDB_H_ #include "os.h" +#include "taosMsg.h" #ifdef __cplusplus extern "C" { @@ -47,7 +48,7 @@ int tsdbCloseRepo(STsdb *pTsdb); int tsdbForceCloseRepo(STsdb *pTsdb); // Data commit -int tsdbInsert(STsdb *pTsdb, SSubmitMsg *pMsg); +int tsdbInsert(STsdb *pTsdb, SSubmitReq *pMsg); int tsdbCommit(STsdb *pTsdb); #ifdef __cplusplus diff --git a/source/server/vnode/CMakeLists.txt b/source/server/vnode/CMakeLists.txt index 75d24ed64e..d5778f9344 100644 --- a/source/server/vnode/CMakeLists.txt +++ b/source/server/vnode/CMakeLists.txt @@ -9,9 +9,11 @@ target_include_directories( PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode" private "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_include_directories( +target_link_libraries( vnode - PRIVATE meta - PRIVATE tq - PRIVATE tsdb + PUBLIC meta + PUBLIC tq + PUBLIC tsdb + PRIVATE os + PRIVATE common ) \ No newline at end of file diff --git a/source/server/vnode/inc/vnodeCommit.h b/source/server/vnode/inc/vnodeCommit.h new file mode 100644 index 0000000000..d37c61008c --- /dev/null +++ b/source/server/vnode/inc/vnodeCommit.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_VNODE_COMMIT_H_ +#define _TD_VNODE_COMMIT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int vnodeAsyncCommit(SVnode *pVnode); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_VNODE_COMMIT_H_*/ \ No newline at end of file diff --git a/source/server/vnode/inc/vnodeInt.h b/source/server/vnode/inc/vnodeInt.h index 855eb66c8a..79ff0bc76a 100644 --- a/source/server/vnode/inc/vnodeInt.h +++ b/source/server/vnode/inc/vnodeInt.h @@ -16,10 +16,21 @@ #ifndef _TD_VNODE_INT_H_ #define _TD_VNODE_INT_H_ +#include "tq.h" +#include "tsdb.h" +#include "meta.h" + #ifdef __cplusplus extern "C" { #endif +typedef struct SVnode { + SMeta *pMeta; + STsdb *pTsdb; + STQ * pTQ; + void * allocator; // TODO +} SVnode; + #ifdef __cplusplus } #endif diff --git a/source/server/vnode/inc/vnodeMemAllocator.h b/source/server/vnode/inc/vnodeMemAllocator.h new file mode 100644 index 0000000000..f17af4a5a2 --- /dev/null +++ b/source/server/vnode/inc/vnodeMemAllocator.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_VNODE_MEM_ALLOCATOR_H_ +#define _TD_VNODE_MEM_ALLOCATOR_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_VNODE_MEM_ALLOCATOR_H_*/ \ No newline at end of file diff --git a/source/server/vnode/inc/vnodeWrite.h b/source/server/vnode/inc/vnodeWrite.h new file mode 100644 index 0000000000..b4430cd255 --- /dev/null +++ b/source/server/vnode/inc/vnodeWrite.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_VNODE_WRITE_H_ +#define _TD_VNODE_WRITE_H_ + +int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp); + +#endif /*_TD_VNODE_WRITE_H_*/ \ No newline at end of file diff --git a/source/server/vnode/src/vnodeCommit.c b/source/server/vnode/src/vnodeCommit.c new file mode 100644 index 0000000000..3200411f4d --- /dev/null +++ b/source/server/vnode/src/vnodeCommit.c @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "vnodeInt.h" + +static int vnodeStartCommit(SVnode *pVnode); +static int vnodeEndCommit(SVnode *pVnode); + +int vnodeAsyncCommit(SVnode *pVnode) { + if (vnodeStartCommit(pVnode) < 0) { + // TODO + } + + if (tqCommit(pVnode->pTQ) < 0) { + // TODO + } + + if (metaCommit(pVnode->pMeta) < 0) { + // TODO + } + + if (tsdbCommit(pVnode->pTsdb) < 0) { + // TODO + } + + if (vnodeEndCommit(pVnode) < 0) { + // TODO + } + + return 0; +} + +static int vnodeStartCommit(SVnode *pVnode) { + // TODO + return 0; +} + +static int vnodeEndCommit(SVnode *pVnode) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/server/vnode/src/vnodeMemAllocator.c b/source/server/vnode/src/vnodeMemAllocator.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/server/vnode/src/vnodeMemAllocator.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/server/vnode/src/vnodeWrite.c b/source/server/vnode/src/vnodeWrite.c new file mode 100644 index 0000000000..d6c97132d8 --- /dev/null +++ b/source/server/vnode/src/vnodeWrite.c @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "vnodeInt.h" + +int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp) { + // TODO: Check inputs + +#if 1 + void *pMem = NULL; + if ((pMem = aMalloc(pVnode->allocator, REQ_SIZE(pReq))) == NULL) { + // No more memory to allocate, schedule an async commit + // and continue + vnodeAsyncCommit(pVnode); + + // Reset allocator and allocat more + vnodeResetAllocator(pVnode); + pMem = aMalloc(pVnode->allocator, REQ_SIZE(pReq)); + if (pMem == NULL) { + // TODO: handle the error + } + } + + // TODO: if SSubmitReq is compressed or encoded, we need to decode the request + memcpy(pMem, pReq, REQ_SIZE(pReq)); + + if (tqPushMsg((SSubmitReq *)pReq) < 0) { + // TODO: handle error + } + + SSubmitReqReader reader; + taosInitSubmitReqReader(&reader, (SSubmitReq *)pMem); + + if (tsdbInsertData((SSubmitReq *)pMem) < 0) { + // TODO: handler error + } + +#endif + + return 0; +} + +int vnodeProcessCreateTableReq(SVnode *pVnode, SCreateTableReq *pReq, SCreateTableRsp *pRsp) { + // TODO + return 0; +} + +int vnodeProcessDropTableReq(SVnode *pVnode, SDropTableReq *pReq, SDropTableRsp *pRsp) { + // TODO + return 0; +} + +int vnodeProcessAlterTableReq(SVnode *pVnode, SDropTableReq *pReq, SDropTableRsp *pRsp) { + // TODO + return 0; +} diff --git a/source/server/vnode/tsdb/CMakeLists.txt b/source/server/vnode/tsdb/CMakeLists.txt index 3c1dc5810b..4a1e154cd5 100644 --- a/source/server/vnode/tsdb/CMakeLists.txt +++ b/source/server/vnode/tsdb/CMakeLists.txt @@ -8,4 +8,5 @@ target_include_directories( target_include_directories( tsdb PRIVATE os + PRIVATE common ) \ No newline at end of file From e7b97be7cf6575d689b1f357d447c01a8320d077 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 27 Sep 2021 11:42:12 +0800 Subject: [PATCH 03/44] refact --- include/util/amalloc.h | 47 +++++++++++++++++++++ include/util/tlockfree.h | 2 +- source/server/vnode/CMakeLists.txt | 5 ++- source/server/vnode/inc/vnodeInt.h | 9 ++-- source/server/vnode/inc/vnodeMemAllocator.h | 2 + source/server/vnode/src/vnodeWrite.c | 6 +-- 6 files changed, 61 insertions(+), 10 deletions(-) create mode 100644 include/util/amalloc.h diff --git a/include/util/amalloc.h b/include/util/amalloc.h new file mode 100644 index 0000000000..944d27513e --- /dev/null +++ b/include/util/amalloc.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_AMALLOC_H_ +#define _TD_AMALLOC_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Interfaces to implement +typedef struct { + void *(*malloc)(void *, size_t size); + void *(*calloc)(void *, size_t nmemb, size_t size); + void (*free)(void *ptr, size_t size); // Do we need to set size in the allocated memory? + void *(*realloc)(void *ptr, size_t size); +} SMemAllocatorIf; + +typedef struct { + void * impl; + SMemAllocatorIf interface; +} SMemAllocator; + +#define amalloc(allocator, size) (*((allocator)->interface.malloc))((allocator)->impl, size) +#define acalloc(allocator, nmemb, size) (*((allocator)->interface.calloc))((allocator)->impl, nmemb, size) +#define arealloc(allocator, ptr, size) (*((allocator)->interface.realloc))((allocator)->impl, ptr, size) +#define afree(allocator, ptr, size) (*((allocator)->interface.free))((allocator)->impl, ptr, size) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_AMALLOC_H_*/ \ No newline at end of file diff --git a/include/util/tlockfree.h b/include/util/tlockfree.h index e960b601ca..4660dd1cbd 100644 --- a/include/util/tlockfree.h +++ b/include/util/tlockfree.h @@ -71,7 +71,7 @@ typedef void (*_ref_fn_t)(const void* pObj); // single writer multiple reader lock -typedef int32_t SRWLatch; +typedef volatile int32_t SRWLatch; void taosInitRWLatch(SRWLatch *pLatch); void taosWLockLatch(SRWLatch *pLatch); diff --git a/source/server/vnode/CMakeLists.txt b/source/server/vnode/CMakeLists.txt index d5778f9344..73605adc3f 100644 --- a/source/server/vnode/CMakeLists.txt +++ b/source/server/vnode/CMakeLists.txt @@ -14,6 +14,7 @@ target_link_libraries( PUBLIC meta PUBLIC tq PUBLIC tsdb - PRIVATE os - PRIVATE common + PUBLIC os + PUBLIC common + PUBLIC util ) \ No newline at end of file diff --git a/source/server/vnode/inc/vnodeInt.h b/source/server/vnode/inc/vnodeInt.h index 79ff0bc76a..3d218df55e 100644 --- a/source/server/vnode/inc/vnodeInt.h +++ b/source/server/vnode/inc/vnodeInt.h @@ -16,6 +16,7 @@ #ifndef _TD_VNODE_INT_H_ #define _TD_VNODE_INT_H_ +#include "amalloc.h" #include "tq.h" #include "tsdb.h" #include "meta.h" @@ -25,10 +26,10 @@ extern "C" { #endif typedef struct SVnode { - SMeta *pMeta; - STsdb *pTsdb; - STQ * pTQ; - void * allocator; // TODO + SMeta * pMeta; + STsdb * pTsdb; + STQ * pTQ; + SMemAllocator *allocator; } SVnode; #ifdef __cplusplus diff --git a/source/server/vnode/inc/vnodeMemAllocator.h b/source/server/vnode/inc/vnodeMemAllocator.h index f17af4a5a2..9b3d27776d 100644 --- a/source/server/vnode/inc/vnodeMemAllocator.h +++ b/source/server/vnode/inc/vnodeMemAllocator.h @@ -16,6 +16,8 @@ #ifndef _TD_VNODE_MEM_ALLOCATOR_H_ #define _TD_VNODE_MEM_ALLOCATOR_H_ +#include "amalloc.h" + #ifdef __cplusplus extern "C" { #endif diff --git a/source/server/vnode/src/vnodeWrite.c b/source/server/vnode/src/vnodeWrite.c index d6c97132d8..0163dd71f1 100644 --- a/source/server/vnode/src/vnodeWrite.c +++ b/source/server/vnode/src/vnodeWrite.c @@ -20,14 +20,14 @@ int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp) { #if 1 void *pMem = NULL; - if ((pMem = aMalloc(pVnode->allocator, REQ_SIZE(pReq))) == NULL) { + if ((pMem = amalloc(pVnode->allocator, REQ_SIZE(pReq))) == NULL) { // No more memory to allocate, schedule an async commit // and continue vnodeAsyncCommit(pVnode); // Reset allocator and allocat more vnodeResetAllocator(pVnode); - pMem = aMalloc(pVnode->allocator, REQ_SIZE(pReq)); + pMem = amalloc(pVnode->allocator, REQ_SIZE(pReq)); if (pMem == NULL) { // TODO: handle the error } @@ -43,7 +43,7 @@ int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp) { SSubmitReqReader reader; taosInitSubmitReqReader(&reader, (SSubmitReq *)pMem); - if (tsdbInsertData((SSubmitReq *)pMem) < 0) { + if (tsdbInsert(pVnode->pTsdb, (SSubmitReq *)pMem) < 0) { // TODO: handler error } From 7f4551db610b1c2d9c0763b6f730e6c7baff895f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 27 Sep 2021 14:44:18 +0800 Subject: [PATCH 04/44] refact --- include/libs/tkv/tkv.h | 35 ++++++++++++++++++++++++ include/server/vnode/meta/meta.h | 5 ++++ source/server/vnode/meta/CMakeLists.txt | 4 +++ source/server/vnode/meta/inc/metaInt.h | 4 +++ source/server/vnode/meta/src/meta.c | 36 +++++++++++++++++++++++++ source/server/vnode/src/vnodeWrite.c | 5 +--- 6 files changed, 85 insertions(+), 4 deletions(-) diff --git a/include/libs/tkv/tkv.h b/include/libs/tkv/tkv.h index 6072525f88..ef865e894b 100644 --- a/include/libs/tkv/tkv.h +++ b/include/libs/tkv/tkv.h @@ -20,6 +20,41 @@ extern "C" { #endif +typedef struct tkv_db_s tkv_db_t; + + +typedef struct { + /* data */ +} tkv_key_t; + +typedef struct { + bool pinned; + int64_t ref; // TODO: use util library + // TODO: add a RW latch here + void *pObj; +} tkv_obj_t; + + +typedef int (*tkv_key_comp_fn_t)(const tkv_key_t *, const tkv_key_t *); +typedef void (*tkv_get_key_fn_t)(const tkv_obj_t *, tkv_key_t *); +typedef int (*tkv_obj_encode_fn_t)(void **buf, void *pObj); +typedef void *(*tkv_obj_decode_fn_t)(void *buf, void **pObj); + +typedef struct { + uint64_t memLimit : 63; + tkv_get_key_fn_t getKey; + tkv_obj_encode_fn_t encode; + tkv_obj_decode_fn_t decode; +} tkv_db_option_t; + +tkv_db_t * tkvOpenDB(char *dir, tkv_db_option_t *); +int tkvCloseDB(tkv_db_t *); +int tkvPut(tkv_db_t *, tkv_obj_t *); +int tkvPutBatch(tkv_db_t *, tkv_obj_t **, int); +tkv_obj_t *tkvGet(tkv_key_t *); +int tkvGetBatch(tkv_db_t *, tkv_key_t **, int, tkv_obj_t **); +int tkvCommit(tkv_db_t *, void * /*TODO*/); + #ifdef __cplusplus } #endif diff --git a/include/server/vnode/meta/meta.h b/include/server/vnode/meta/meta.h index 2addcdc6c3..ca7cba9b0c 100644 --- a/include/server/vnode/meta/meta.h +++ b/include/server/vnode/meta/meta.h @@ -16,12 +16,17 @@ #ifndef _TD_META_H_ #define _TD_META_H_ +#include "taosMsg.h" + #ifdef __cplusplus extern "C" { #endif typedef struct SMeta SMeta; +int metaCreateTable(SMeta *pMeta, SCreateTableReq *pReq); +int metaDropTable(SMeta *pMeta, SDropTableReq *pReq); +int metaAlterTable(SMeta *pMeta, SAlterTableReq *pReq); int metaCommit(SMeta *pMeta); #ifdef __cplusplus diff --git a/source/server/vnode/meta/CMakeLists.txt b/source/server/vnode/meta/CMakeLists.txt index 7ce192ef3e..832e13a155 100644 --- a/source/server/vnode/meta/CMakeLists.txt +++ b/source/server/vnode/meta/CMakeLists.txt @@ -4,4 +4,8 @@ target_include_directories( meta PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode/meta" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +target_link_libraries( + meta + PUBLIC common ) \ No newline at end of file diff --git a/source/server/vnode/meta/inc/metaInt.h b/source/server/vnode/meta/inc/metaInt.h index 8b9adfca89..549ac829ab 100644 --- a/source/server/vnode/meta/inc/metaInt.h +++ b/source/server/vnode/meta/inc/metaInt.h @@ -20,6 +20,10 @@ extern "C" { #endif +struct { + tkv_db_t db; +} SMeta; + #ifdef __cplusplus } #endif diff --git a/source/server/vnode/meta/src/meta.c b/source/server/vnode/meta/src/meta.c index e69de29bb2..7d8ec11494 100644 --- a/source/server/vnode/meta/src/meta.c +++ b/source/server/vnode/meta/src/meta.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "meta.h" + +int metaCreateTable(SMeta *pMeta, SCreateTableReq *pReq) { + // TODO + return 0; +} + +int metaDropTable(SMeta *pMeta, SDropTableReq *pReq) { + // TODO + return 0; +} + +int metaAlterTable(SMeta *pMeta, SAlterTableReq *pReq) { + // TODO + return 0; +} + +int metaCommit(SMeta *pMeta) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/server/vnode/src/vnodeWrite.c b/source/server/vnode/src/vnodeWrite.c index 0163dd71f1..a3aff78d02 100644 --- a/source/server/vnode/src/vnodeWrite.c +++ b/source/server/vnode/src/vnodeWrite.c @@ -18,7 +18,6 @@ int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp) { // TODO: Check inputs -#if 1 void *pMem = NULL; if ((pMem = amalloc(pVnode->allocator, REQ_SIZE(pReq))) == NULL) { // No more memory to allocate, schedule an async commit @@ -47,8 +46,6 @@ int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp) { // TODO: handler error } -#endif - return 0; } @@ -62,7 +59,7 @@ int vnodeProcessDropTableReq(SVnode *pVnode, SDropTableReq *pReq, SDropTableRsp return 0; } -int vnodeProcessAlterTableReq(SVnode *pVnode, SDropTableReq *pReq, SDropTableRsp *pRsp) { +int vnodeProcessAlterTableReq(SVnode *pVnode, SAlterTableReq *pReq, SAlterTableRsp *pRsp) { // TODO return 0; } From 5e95ab2e6ceca8daeb7b091f2a99c5a29e80c9dd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 27 Sep 2021 15:14:33 +0800 Subject: [PATCH 05/44] refact --- include/libs/tkv/tkv.h | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/include/libs/tkv/tkv.h b/include/libs/tkv/tkv.h index ef865e894b..96585cfd80 100644 --- a/include/libs/tkv/tkv.h +++ b/include/libs/tkv/tkv.h @@ -22,7 +22,6 @@ extern "C" { typedef struct tkv_db_s tkv_db_t; - typedef struct { /* data */ } tkv_key_t; @@ -31,29 +30,33 @@ typedef struct { bool pinned; int64_t ref; // TODO: use util library // TODO: add a RW latch here - void *pObj; + uint64_t offset; + void * pObj; } tkv_obj_t; - typedef int (*tkv_key_comp_fn_t)(const tkv_key_t *, const tkv_key_t *); typedef void (*tkv_get_key_fn_t)(const tkv_obj_t *, tkv_key_t *); typedef int (*tkv_obj_encode_fn_t)(void **buf, void *pObj); typedef void *(*tkv_obj_decode_fn_t)(void *buf, void **pObj); +typedef int (*tkv_obj_comp_fn_t)(const tkv_obj_t *, const tkv_obj_t *); typedef struct { - uint64_t memLimit : 63; + uint64_t memLimit; tkv_get_key_fn_t getKey; tkv_obj_encode_fn_t encode; tkv_obj_decode_fn_t decode; + tkv_obj_comp_fn_t compare; } tkv_db_option_t; -tkv_db_t * tkvOpenDB(char *dir, tkv_db_option_t *); -int tkvCloseDB(tkv_db_t *); -int tkvPut(tkv_db_t *, tkv_obj_t *); -int tkvPutBatch(tkv_db_t *, tkv_obj_t **, int); -tkv_obj_t *tkvGet(tkv_key_t *); -int tkvGetBatch(tkv_db_t *, tkv_key_t **, int, tkv_obj_t **); -int tkvCommit(tkv_db_t *, void * /*TODO*/); +tkv_db_t * tkvOpenDB(char *dir, tkv_db_option_t *); +int tkvCloseDB(tkv_db_t *); +int tkvPut(tkv_db_t *, tkv_obj_t *); +int tkvPutBatch(tkv_db_t *, tkv_obj_t **, int); +const tkv_obj_t *tkvGet(tkv_key_t *); +int tkvGetBatch(tkv_db_t *, tkv_key_t **, int, tkv_obj_t **); +int tkvDrop(tkv_db_t *, tkv_key_t *); +int tkvCommit(tkv_db_t *, void * /*TODO*/); +// TODO: iter function #ifdef __cplusplus } From ff2b8e0602288da0e3747e33bc8f32fb41dffed1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 27 Sep 2021 15:39:25 +0800 Subject: [PATCH 06/44] refact --- include/libs/tkv/tkv.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/include/libs/tkv/tkv.h b/include/libs/tkv/tkv.h index 96585cfd80..22da145237 100644 --- a/include/libs/tkv/tkv.h +++ b/include/libs/tkv/tkv.h @@ -56,7 +56,13 @@ const tkv_obj_t *tkvGet(tkv_key_t *); int tkvGetBatch(tkv_db_t *, tkv_key_t **, int, tkv_obj_t **); int tkvDrop(tkv_db_t *, tkv_key_t *); int tkvCommit(tkv_db_t *, void * /*TODO*/); -// TODO: iter function + +typedef struct { +} tkv_db_iter_t; + +tkv_db_iter_t * tkvIterNew(tkv_db_t *); +void tkvIterFree(tkv_db_iter_t *); +const tkv_obj_t *tkvIterNext(tkv_db_iter_t *); #ifdef __cplusplus } From 7b6002bd94c6ad6af8a8c87604585960bce34320 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 27 Sep 2021 16:02:55 +0800 Subject: [PATCH 07/44] refact --- include/libs/tkv/tkv.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/libs/tkv/tkv.h b/include/libs/tkv/tkv.h index 22da145237..3f0e2833b8 100644 --- a/include/libs/tkv/tkv.h +++ b/include/libs/tkv/tkv.h @@ -51,10 +51,11 @@ typedef struct { tkv_db_t * tkvOpenDB(char *dir, tkv_db_option_t *); int tkvCloseDB(tkv_db_t *); int tkvPut(tkv_db_t *, tkv_obj_t *); -int tkvPutBatch(tkv_db_t *, tkv_obj_t **, int); +int tkvPutBatch(tkv_db_t *, tkv_obj_t **, int); // TODO: use array here const tkv_obj_t *tkvGet(tkv_key_t *); -int tkvGetBatch(tkv_db_t *, tkv_key_t **, int, tkv_obj_t **); +int tkvGetBatch(tkv_db_t *, tkv_key_t **, int, tkv_obj_t **); // TODO: use array here int tkvDrop(tkv_db_t *, tkv_key_t *); +int tkvDropBatch(tkv_db_t *, tkv_key_t **, int); // TODO: use array here int tkvCommit(tkv_db_t *, void * /*TODO*/); typedef struct { From 773db063d2aea42e983c53c96655ae4361b57bbc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 27 Sep 2021 16:05:18 +0800 Subject: [PATCH 08/44] refact --- include/libs/tkv/tkv.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/include/libs/tkv/tkv.h b/include/libs/tkv/tkv.h index 3f0e2833b8..73c0156bd1 100644 --- a/include/libs/tkv/tkv.h +++ b/include/libs/tkv/tkv.h @@ -39,13 +39,15 @@ typedef void (*tkv_get_key_fn_t)(const tkv_obj_t *, tkv_key_t *); typedef int (*tkv_obj_encode_fn_t)(void **buf, void *pObj); typedef void *(*tkv_obj_decode_fn_t)(void *buf, void **pObj); typedef int (*tkv_obj_comp_fn_t)(const tkv_obj_t *, const tkv_obj_t *); +typedef void (*tkv_obj_destroy_fn_t)(void *); typedef struct { - uint64_t memLimit; - tkv_get_key_fn_t getKey; - tkv_obj_encode_fn_t encode; - tkv_obj_decode_fn_t decode; - tkv_obj_comp_fn_t compare; + uint64_t memLimit; + tkv_get_key_fn_t getKey; + tkv_obj_encode_fn_t encode; + tkv_obj_decode_fn_t decode; + tkv_obj_comp_fn_t compare; + tkv_obj_destroy_fn_t destroy; } tkv_db_option_t; tkv_db_t * tkvOpenDB(char *dir, tkv_db_option_t *); From eec3543c093fe65d8b625faa117fa1f3eec13083 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 27 Sep 2021 17:08:02 +0800 Subject: [PATCH 09/44] refact --- include/util/amalloc.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/include/util/amalloc.h b/include/util/amalloc.h index 944d27513e..6d2869f719 100644 --- a/include/util/amalloc.h +++ b/include/util/amalloc.h @@ -35,10 +35,14 @@ typedef struct { SMemAllocatorIf interface; } SMemAllocator; -#define amalloc(allocator, size) (*((allocator)->interface.malloc))((allocator)->impl, size) -#define acalloc(allocator, nmemb, size) (*((allocator)->interface.calloc))((allocator)->impl, nmemb, size) -#define arealloc(allocator, ptr, size) (*((allocator)->interface.realloc))((allocator)->impl, ptr, size) -#define afree(allocator, ptr, size) (*((allocator)->interface.free))((allocator)->impl, ptr, size) +#define amalloc(allocator, size) \ + ((allocator) ? (*((allocator)->interface.malloc))((allocator)->impl, (size)) : malloc(size)) +#define acalloc(allocator, nmemb, size) \ + ((allocator) ? (*((allocator)->interface.calloc))((allocator)->impl, (nmemb), (size)) : calloc((nmemb), (size))) +#define arealloc(allocator, ptr, size) \ + ((allocator) ? (*((allocator)->interface.realloc))((allocator)->impl, (ptr), (size)) : realloc((ptr), (size))) +#define afree(allocator, ptr, size) \ + ((allocator) ? (*((allocator)->interface.free))((allocator)->impl, (ptr), (size)) : free(ptr)) #ifdef __cplusplus } From 26a96ba78d31a8f91c173888fb9caffc5872d106 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Sep 2021 13:48:55 +0800 Subject: [PATCH 10/44] [TD-10430] header file for the dnode module --- include/server/dnode/dnode.h | 43 +++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/include/server/dnode/dnode.h b/include/server/dnode/dnode.h index fae60495e9..711b6d92a5 100644 --- a/include/server/dnode/dnode.h +++ b/include/server/dnode/dnode.h @@ -20,56 +20,59 @@ extern "C" { #endif -struct SRpcMsg; -struct SRpcEpSet; - /** * Initialize and start the dnode module - * - * @return Error Code + * + * @return Instance of dnode module */ -int32_t dnodeInit(); +struct Dnode *dnodeCreateInstance(); /** * Stop and cleanup dnode module + * + * @param dnode Instance of dnode module */ -void dnodeCleanup(); +void dnodeCleanupInstance(struct Dnode *dnode); /** * Send messages to other dnodes, such as create vnode message. * + * @param dnode The instance of Dnode module * @param epSet The endpoint list of the dnodes. * @param rpcMsg Message to be sent. */ -void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg); +void dnodeSendMsgToDnode(struct Dnode *dnode, struct SRpcEpSet *epSet, struct SRpcMsg *rpcMsg); /** * Send messages to mnode, such as config message. * + * @param dnode The instance of dnode module * @param rpcMsg Message to be sent. */ -void dnodeSendMsgToMnode(SRpcMsg *rpcMsg); +void dnodeSendMsgToMnode(struct Dnode *dnode, struct SRpcMsg *rpcMsg); + +/** + * Send redirect message to dnode or shell. + * + * @param dnode The instance of dnode module + * @param rpcMsg Message to be sent. + * @param forShell Used to identify whether to send to shell or dnode. + */ +void dnodeSendRedirectMsg(struct Dnode *dnode, struct SRpcMsg *rpcMsg, bool forShell); /** * Get the corresponding endpoint information from dnodeId. * - * @param dnodeId + * @param dnode The instance of Dnode module + * @param dnodeId The id ot dnode * @param ep The endpoint of dnode * @param fqdn The fqdn of dnode * @param port The port of dnode */ -void dnodeGetDnodeEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); - -/** - * Report to dnode the start-up steps of other modules - * - * @param name Name of the start-up phase. - * @param desc Description of the start-up phase. - */ -void dnodeReportStartup(char *name, char *desc); +void dnodeGetDnodeEp(struct Dnode *dnode, int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); #ifdef __cplusplus } #endif -#endif /*_TD_DNODE_H_*/ \ No newline at end of file +#endif /*_TD_DNODE_H_*/ From 409e9c9f0551f177fd4be6a82868a563fee95933 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Sep 2021 13:55:53 +0800 Subject: [PATCH 11/44] [TD-10430] move main.c outof dnode module --- source/server/CMakeLists.txt | 3 ++- source/server/dnode/CMakeLists.txt | 7 +++++++ source/server/{dnode => }/main.c | 0 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 source/server/dnode/CMakeLists.txt rename source/server/{dnode => }/main.c (100%) diff --git a/source/server/CMakeLists.txt b/source/server/CMakeLists.txt index ab88d50191..d80c7876db 100644 --- a/source/server/CMakeLists.txt +++ b/source/server/CMakeLists.txt @@ -1,8 +1,9 @@ add_subdirectory(mnode) add_subdirectory(vnode) add_subdirectory(qnode) +add_subdirectory(dnode) -aux_source_directory(dnode TAOSD_SRC) +aux_source_directory(. TAOSD_SRC) add_executable(taosd ${TAOSD_SRC}) target_link_libraries( taosd diff --git a/source/server/dnode/CMakeLists.txt b/source/server/dnode/CMakeLists.txt new file mode 100644 index 0000000000..db80bf727d --- /dev/null +++ b/source/server/dnode/CMakeLists.txt @@ -0,0 +1,7 @@ +aux_source_directory(src DNODE_SRC) +add_library(dnode ${DNODE_SRC}) +target_include_directories( + dnode + PUBLIC "${CMAKE_SOURCE_DIR}/include/server/dnode" + private "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) \ No newline at end of file diff --git a/source/server/dnode/main.c b/source/server/main.c similarity index 100% rename from source/server/dnode/main.c rename to source/server/main.c From 7a78d7223dc377c1e6cc40265c2783a12bee57ff Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Sep 2021 14:05:33 +0800 Subject: [PATCH 12/44] [TD-10430] add dnode.h for dnode module --- source/server/dnode/inc/dnodeInt.h | 66 ++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 source/server/dnode/inc/dnodeInt.h diff --git a/source/server/dnode/inc/dnodeInt.h b/source/server/dnode/inc/dnodeInt.h new file mode 100644 index 0000000000..b0e5a448e0 --- /dev/null +++ b/source/server/dnode/inc/dnodeInt.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DNODE_INT_H_ +#define _TD_DNODE_INT_H_ + +#ifdef __cplusplus +extern "C" { +#endif +#include "taoserror.h" +#include "taosmsg.h" +#include "tglobal.h" +#include "tlog.h" +#include "trpc.h" +#include "tstep.h" +#include "dnode.h" + +struct DnCfg; +struct DnCheck; +struct DnEps; +struct DnMnEps; +struct DnStatus; +struct DnTelem; +struct DnTrans; +struct DnMain; +struct Mnode; +struct Vnode; + +typedef struct Dnode { + struct SSteps * steps; + struct DnCfg * cfg; + struct DnCheck * check; + struct DnEps * eps; + struct DnMnEps * meps; + struct DnStatus *status; + struct DnTelem * telem; + struct DnTrans * trans; + struct DnMain * main; + struct Mnode * mnode; + struct Vnode * vnode; +} Dnode; + +#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", 255, __VA_ARGS__); }} +#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", 255, __VA_ARGS__); }} +#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", 255, __VA_ARGS__); }} +#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", 255, __VA_ARGS__); }} +#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }} +#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }} + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DNODE_INT_H_*/ \ No newline at end of file From 9c29039ae75b8b1a10ab5ae071ad19146fb5e679 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Sep 2021 14:06:29 +0800 Subject: [PATCH 13/44] [TD-10430] add dnode cfg for dnode module --- source/server/dnode/inc/dnodeCfg.h | 45 ++++++++ source/server/dnode/src/dnodeCfg.c | 178 +++++++++++++++++++++++++++++ 2 files changed, 223 insertions(+) create mode 100644 source/server/dnode/inc/dnodeCfg.h create mode 100644 source/server/dnode/src/dnodeCfg.c diff --git a/source/server/dnode/inc/dnodeCfg.h b/source/server/dnode/inc/dnodeCfg.h new file mode 100644 index 0000000000..ec9cc397e1 --- /dev/null +++ b/source/server/dnode/inc/dnodeCfg.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DNODE_CFG_H_ +#define _TD_DNODE_CFG_H_ + +#ifdef __cplusplus +extern "C" { +#endif +#include "dnodeInt.h" + +typedef struct DnCfg { + Dnode * dnode; + int32_t dnodeId; + int32_t dropped; + char clusterId[TSDB_CLUSTER_ID_LEN]; + char file[PATH_MAX + 20]; + pthread_mutex_t mutex; +} DnCfg; + +int32_t dnodeInitCfg(Dnode *dnode, DnCfg **cfg); +void dnodeCleanupCfg(Dnode *dnode, DnCfg **cfg); +void dnodeUpdateCfg(DnCfg *cfg, SDnodeCfg *data); +int32_t dnodeGetDnodeId(DnCfg *cfg); +void dnodeGetClusterId(DnCfg *cfg, char *clusterId); +void dnodeGetCfg(DnCfg *cfg, int32_t *dnodeId, char *clusterId); +void dnodeSetDropped(DnCfg *cfg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DNODE_CFG_H_*/ diff --git a/source/server/dnode/src/dnodeCfg.c b/source/server/dnode/src/dnodeCfg.c new file mode 100644 index 0000000000..f4e26fd721 --- /dev/null +++ b/source/server/dnode/src/dnodeCfg.c @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "cJSON.h" +#include "dnodeCfg.h" + +static int32_t dnodeReadCfg(DnCfg *cfg) { + int32_t len = 0; + int32_t maxLen = 200; + char * content = calloc(1, maxLen + 1); + cJSON * root = NULL; + FILE * fp = NULL; + + fp = fopen(cfg->file, "r"); + if (!fp) { + dDebug("file %s not exist", cfg->file); + goto PARSE_CFG_OVER; + } + + len = (int32_t)fread(content, 1, maxLen, fp); + if (len <= 0) { + dError("failed to read %s since content is null", cfg->file); + goto PARSE_CFG_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", cfg->file); + goto PARSE_CFG_OVER; + } + + cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId"); + if (!dnodeId || dnodeId->type != cJSON_Number) { + dError("failed to read %s since dnodeId not found", cfg->file); + goto PARSE_CFG_OVER; + } + cfg->dnodeId = (int32_t)dnodeId->valueint; + + cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); + if (!dropped || dropped->type != cJSON_Number) { + dError("failed to read %s since dropped not found", cfg->file); + goto PARSE_CFG_OVER; + } + cfg->dropped = (int32_t)dropped->valueint; + + cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId"); + if (!clusterId || clusterId->type != cJSON_String) { + dError("failed to read %s since clusterId not found", cfg->file); + goto PARSE_CFG_OVER; + } + tstrncpy(cfg->clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN); + + dInfo("successed to read %s", cfg->file); + +PARSE_CFG_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (fp != NULL) fclose(fp); + terrno = 0; + + return 0; +} + +static int32_t dnodeWriteCfg(DnCfg *cfg) { + FILE *fp = fopen(cfg->file, "w"); + if (!fp) { + dError("failed to write %s since %s", cfg->file, strerror(errno)); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 200; + char * content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", cfg->dnodeId); + len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", cfg->dropped); + len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%s\"\n", cfg->clusterId); + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); + taosFsync(fileno(fp)); + fclose(fp); + free(content); + terrno = 0; + + dInfo("successed to write %s", cfg->file); + return 0; +} + +int32_t dnodeInitCfg(Dnode *dnode, DnCfg **out) { + DnCfg* cfg = calloc(1, sizeof(DnCfg)); + if (cfg == NULL) return -1; + + cfg->dnode = dnode; + cfg->dnodeId = 0; + cfg->dropped = 0; + cfg->clusterId[0] = 0; + snprintf(cfg->file, sizeof(cfg->file), "%s/dnodeCfg.json", tsDnodeDir); + pthread_mutex_init(&cfg->mutex, NULL); + *out = cfg; + + int32_t ret = dnodeReadCfg(cfg); + if (ret == 0) { + dInfo("dnode cfg is initialized"); + } + + if (cfg->dropped) { + dInfo("dnode is dropped and start to exit"); + return -1; + } + + return ret; +} + +void dnodeCleanupCfg(Dnode *dnode, DnCfg **out) { + DnCfg* cfg = *out; + *out = NULL; + + pthread_mutex_destroy(&cfg->mutex); + free(cfg); +} + +void dnodeUpdateCfg(DnCfg *cfg, SDnodeCfg *data) { + if (cfg == NULL || cfg->dnodeId == 0) return; + + pthread_mutex_lock(&cfg->mutex); + + cfg->dnodeId = data->dnodeId; + tstrncpy(cfg->clusterId, data->clusterId, TSDB_CLUSTER_ID_LEN); + dInfo("dnodeId is set to %d, clusterId is set to %s", cfg->dnodeId, cfg->clusterId); + + dnodeWriteCfg(cfg); + pthread_mutex_unlock(&cfg->mutex); +} + +void dnodeSetDropped(DnCfg *cfg) { + pthread_mutex_lock(&cfg->mutex); + cfg->dropped = 1; + dnodeWriteCfg(cfg); + pthread_mutex_unlock(&cfg->mutex); +} + +int32_t dnodeGetDnodeId(DnCfg *cfg) { + int32_t dnodeId = 0; + pthread_mutex_lock(&cfg->mutex); + dnodeId = cfg->dnodeId; + pthread_mutex_unlock(&cfg->mutex); + return dnodeId; +} + +void dnodeGetClusterId(DnCfg *cfg, char *clusterId) { + pthread_mutex_lock(&cfg->mutex); + tstrncpy(clusterId, cfg->clusterId, TSDB_CLUSTER_ID_LEN); + pthread_mutex_unlock(&cfg->mutex); +} + +void dnodeGetCfg(DnCfg *cfg, int32_t *dnodeId, char *clusterId) { + pthread_mutex_lock(&cfg->mutex); + *dnodeId = cfg->dnodeId; + tstrncpy(clusterId, cfg->clusterId, TSDB_CLUSTER_ID_LEN); + pthread_mutex_unlock(&cfg->mutex); +} From c73c3977d33d199bf7f2098b574853024384c6ee Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Sep 2021 14:07:42 +0800 Subject: [PATCH 14/44] [TD-10430] add dnode check for dnode module --- source/server/dnode/inc/dnodeCheck.h | 35 +++++ source/server/dnode/src/dnodeCheck.c | 202 +++++++++++++++++++++++++++ 2 files changed, 237 insertions(+) create mode 100644 source/server/dnode/inc/dnodeCheck.h create mode 100644 source/server/dnode/src/dnodeCheck.c diff --git a/source/server/dnode/inc/dnodeCheck.h b/source/server/dnode/inc/dnodeCheck.h new file mode 100644 index 0000000000..b8d6d2ae56 --- /dev/null +++ b/source/server/dnode/inc/dnodeCheck.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DNODE_CHECK_H_ +#define _TD_DNODE_CHECK_H_ + +#ifdef __cplusplus +extern "C" { +#endif +#include "dnodeInt.h" + +typedef struct DnCheck { + Dnode *dnode; +} DnCheck; + +int32_t dnodeInitCheck(Dnode *dnode, DnCheck **check); +void dnodeCleanupCheck(Dnode *dnode, DnCheck **check); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DNODE_CHECK_H_*/ diff --git a/source/server/dnode/src/dnodeCheck.c b/source/server/dnode/src/dnodeCheck.c new file mode 100644 index 0000000000..6a7f18f63d --- /dev/null +++ b/source/server/dnode/src/dnodeCheck.c @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tglobal.h" +#include "dnodeCheck.h" + +#define MIN_AVAIL_MEMORY_MB 32 + +static int32_t dnodeBindTcpPort(uint16_t port) { +#if 0 + SOCKET serverSocket; + struct sockaddr_in server_addr; + + if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { + dError("failed to create tcp socket since %s", strerror(errno)); + return -1; + } + + bzero(&server_addr, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + server_addr.sin_addr.s_addr = htonl(INADDR_ANY); + + if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { + dError("failed to bind tcp port:%d since %s", port, strerror(errno)); + taosCloseSocket(serverSocket); + return -1; + } + + if (listen(serverSocket, 5) < 0) { + dError("failed to listen tcp port:%d since %s", port, strerror(errno)); + taosCloseSocket(serverSocket); + return -1; + } + + taosCloseSocket(serverSocket); +#endif + return 0; +} + +static int32_t dnodeBindUdpPort(int16_t port) { +#if 0 + SOCKET serverSocket; + struct sockaddr_in server_addr; + + if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { + dError("failed to create udp socket since %s", strerror(errno)); + return -1; + } + + bzero(&server_addr, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + server_addr.sin_addr.s_addr = htonl(INADDR_ANY); + + if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { + dError("failed to bind udp port:%d since %s", port, strerror(errno)); + taosCloseSocket(serverSocket); + return -1; + } + + taosCloseSocket(serverSocket); +#endif + return 0; +} + +static int32_t dnodeCheckNetwork() { + int32_t ret; + uint16_t startPort = tsServerPort; + + for (uint16_t port = startPort; port < startPort + 12; port++) { + ret = dnodeBindTcpPort(port); + if (0 != ret) { + dError("failed to bind tcp port:%d", port); + return -1; + } + ret = dnodeBindUdpPort(port); + if (0 != ret) { + dError("failed to bind udp port:%d", port); + return -1; + } + } + + return 0; +} + +static int32_t dnodeCheckMem() { + float memoryUsedMB; + float memoryAvailMB; + if (true != taosGetSysMemory(&memoryUsedMB)) { + dError("failed to get system memory since %s, errno:%u,", strerror(errno), errno); + return -1; + } + + memoryAvailMB = (float)tsTotalMemoryMB - memoryUsedMB; + + if (memoryAvailMB < MIN_AVAIL_MEMORY_MB) { + dError("available memory %fMB less than the threshold %dMB", memoryAvailMB, MIN_AVAIL_MEMORY_MB); + return -1; + } + + return 0; +} + +static int32_t dnodeCheckDisk() { + taosGetDisk(); + + if (tsAvailDataDirGB < tsMinimalDataDirGB) { + dError("dataDir disk size:%fGB less than threshold %fGB ", tsAvailDataDirGB, tsMinimalDataDirGB); + return -1; + } + + if (tsAvailLogDirGB < tsMinimalLogDirGB) { + dError("logDir disk size:%fGB less than threshold %fGB", tsAvailLogDirGB, tsMinimalLogDirGB); + return -1; + } + + if (tsAvailTmpDirectorySpace < tsReservedTmpDirectorySpace) { + dError("tmpDir disk size:%fGB less than threshold %fGB", tsAvailTmpDirectorySpace, tsReservedTmpDirectorySpace); + return -1; + } + + return 0; +} + +static int32_t dnodeCheckCpu() { return 0; } +static int32_t dnodeCheckOs() { return 0; } +static int32_t dnodeCheckAccess() { return 0; } +static int32_t dnodeCheckVersion() { return 0; } +static int32_t dnodeCheckDatafile() { return 0; } + +int32_t dnodeInitCheck(Dnode *dnode, DnCheck **out) { + DnCheck *check = calloc(1, sizeof(DnCheck)); + if (check == NULL) return -1; + + check->dnode = dnode; + *out = check; + + if (dnodeCheckNetwork() != 0) { + dError("failed to check network"); + return -1; + } + + if (dnodeCheckMem() != 0) { + dError("failed to check memory"); + return -1; + } + + if (dnodeCheckCpu() != 0) { + dError("failed to check cpu"); + return -1; + } + + if (dnodeCheckDisk() != 0) { + dError("failed to check disk"); + return -1; + } + + if (dnodeCheckOs() != 0) { + dError("failed to check os"); + return -1; + } + + if (dnodeCheckAccess() != 0) { + dError("failed to check access"); + return -1; + } + + if (dnodeCheckVersion() != 0) { + dError("failed to check version"); + return -1; + } + + if (dnodeCheckDatafile() != 0) { + dError("failed to check datafile"); + return -1; + } + + dInfo("dnode check is finished"); + return 0; +} + +void dnodeCleanupCheck(Dnode *dnode, DnCheck **out) { + DnCheck *check = *out; + *out = NULL; + + free(check); +} \ No newline at end of file From 3e995bd37126bd765173f5edce34e6d675d5a729 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Sep 2021 14:08:57 +0800 Subject: [PATCH 15/44] [TD-10430] add dnode eps for dnode module --- source/server/dnode/inc/dnodeEps.h | 45 +++++ source/server/dnode/src/dnodeEps.c | 286 +++++++++++++++++++++++++++++ 2 files changed, 331 insertions(+) create mode 100644 source/server/dnode/inc/dnodeEps.h create mode 100644 source/server/dnode/src/dnodeEps.c diff --git a/source/server/dnode/inc/dnodeEps.h b/source/server/dnode/inc/dnodeEps.h new file mode 100644 index 0000000000..fa7e99d113 --- /dev/null +++ b/source/server/dnode/inc/dnodeEps.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DNODE_EPS_H_ +#define _TD_DNODE_EPS_H_ + +#ifdef __cplusplus +extern "C" { +#endif +#include "hash.h" +#include "dnodeInt.h" + +typedef struct DnEps { + Dnode * dnode; + int32_t dnodeId; + int32_t dnodeNum; + SDnodeEp * dnodeList; + SHashObj * dnodeHash; + char file[PATH_MAX + 20]; + pthread_mutex_t mutex; +} DnEps; + +int32_t dnodeInitEps(Dnode *dnode, DnEps **eps); +void dnodeCleanupEps(Dnode *dnode, DnEps **eps); +void dnodeUpdateEps(DnEps *eps, SDnodeEps *data); +bool dnodeIsDnodeEpChanged(DnEps *eps, int32_t dnodeId, char *epstr); +void dnodeGetDnodeEp(Dnode *dnode, int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DNODE_EPS_H_*/ \ No newline at end of file diff --git a/source/server/dnode/src/dnodeEps.c b/source/server/dnode/src/dnodeEps.c new file mode 100644 index 0000000000..c178c01c9f --- /dev/null +++ b/source/server/dnode/src/dnodeEps.c @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "cJSON.h" +#include "tglobal.h" +#include "dnodeEps.h" +#include "dnodeCfg.h" + +static void dnodePrintEps(DnEps *eps) { + dDebug("print dnodeEp, dnodeNum:%d", eps->dnodeNum); + for (int32_t i = 0; i < eps->dnodeNum; i++) { + SDnodeEp *ep = &eps->dnodeList[i]; + dDebug("dnode:%d, dnodeFqdn:%s dnodePort:%u", ep->dnodeId, ep->dnodeFqdn, ep->dnodePort); + } +} + +static void dnodeResetEps(DnEps *eps, SDnodeEps *data) { + assert(data != NULL); + + if (data->dnodeNum > eps->dnodeNum) { + SDnodeEp *tmp = calloc(data->dnodeNum, sizeof(SDnodeEp)); + if (tmp == NULL) return; + + tfree(eps->dnodeList); + eps->dnodeList = tmp; + eps->dnodeNum = data->dnodeNum; + memcpy(eps->dnodeList, data->dnodeEps, eps->dnodeNum * sizeof(SDnodeEp)); + dnodePrintEps(eps); + + for (int32_t i = 0; i < eps->dnodeNum; ++i) { + SDnodeEp *ep = &eps->dnodeList[i]; + taosHashPut(eps->dnodeHash, &ep->dnodeId, sizeof(int32_t), ep, sizeof(SDnodeEp)); + } + } +} + +static int32_t dnodeReadEps(DnEps *eps) { + int32_t len = 0; + int32_t maxLen = 30000; + char * content = calloc(1, maxLen + 1); + cJSON * root = NULL; + FILE * fp = NULL; + + fp = fopen(eps->file, "r"); + if (!fp) { + dDebug("file %s not exist", eps->file); + goto PRASE_EPS_OVER; + } + + len = (int32_t)fread(content, 1, maxLen, fp); + if (len <= 0) { + dError("failed to read %s since content is null", eps->file); + goto PRASE_EPS_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", eps->file); + goto PRASE_EPS_OVER; + } + + cJSON *dnodeNum = cJSON_GetObjectItem(root, "dnodeNum"); + if (!dnodeNum || dnodeNum->type != cJSON_Number) { + dError("failed to read %s since dnodeNum not found", eps->file); + goto PRASE_EPS_OVER; + } + + cJSON *dnodeInfos = cJSON_GetObjectItem(root, "dnodeInfos"); + if (!dnodeInfos || dnodeInfos->type != cJSON_Array) { + dError("failed to read %s since dnodeInfos not found", eps->file); + goto PRASE_EPS_OVER; + } + + int32_t dnodeInfosSize = cJSON_GetArraySize(dnodeInfos); + if (dnodeInfosSize != dnodeNum->valueint) { + dError("failed to read %s since dnodeInfos size:%d not matched dnodeNum:%d", eps->file, dnodeInfosSize, + (int32_t)dnodeNum->valueint); + goto PRASE_EPS_OVER; + } + + eps->dnodeNum = dnodeInfosSize; + eps->dnodeList = calloc(dnodeInfosSize, sizeof(SDnodeEp)); + if (eps->dnodeList == NULL) { + dError("failed to calloc dnodeEpList since %s", strerror(errno)); + goto PRASE_EPS_OVER; + } + + for (int32_t i = 0; i < dnodeInfosSize; ++i) { + cJSON *dnodeInfo = cJSON_GetArrayItem(dnodeInfos, i); + if (dnodeInfo == NULL) break; + + SDnodeEp *ep = &eps->dnodeList[i]; + + cJSON *dnodeId = cJSON_GetObjectItem(dnodeInfo, "dnodeId"); + if (!dnodeId || dnodeId->type != cJSON_Number) { + dError("failed to read %s, dnodeId not found", eps->file); + goto PRASE_EPS_OVER; + } + ep->dnodeId = (int32_t)dnodeId->valueint; + + cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn"); + if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { + dError("failed to read %s, dnodeFqdn not found", eps->file); + goto PRASE_EPS_OVER; + } + tstrncpy(ep->dnodeFqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN); + + cJSON *dnodePort = cJSON_GetObjectItem(dnodeInfo, "dnodePort"); + if (!dnodePort || dnodePort->type != cJSON_Number) { + dError("failed to read %s, dnodePort not found", eps->file); + goto PRASE_EPS_OVER; + } + ep->dnodePort = (uint16_t)dnodePort->valueint; + } + + dInfo("succcessed to read file %s", eps->file); + dnodePrintEps(eps); + +PRASE_EPS_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (fp != NULL) fclose(fp); + + if (dnodeIsDnodeEpChanged(eps, eps->dnodeId, tsLocalEp)) { + dError("dnode:%d, localEp different from %s dnodeEps.json and need reconfigured", eps->dnodeId, tsLocalEp); + return -1; + } + + terrno = 0; + return 0; +} + +static int32_t dnodeWriteEps(DnEps *eps) { + FILE *fp = fopen(eps->file, "w"); + if (!fp) { + dError("failed to write %s since %s", eps->file, strerror(errno)); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 30000; + char * content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"dnodeNum\": %d,\n", eps->dnodeNum); + len += snprintf(content + len, maxLen - len, " \"dnodeInfos\": [{\n"); + for (int32_t i = 0; i < eps->dnodeNum; ++i) { + SDnodeEp *ep = &eps->dnodeList[i]; + len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", ep->dnodeId); + len += snprintf(content + len, maxLen - len, " \"dnodeFqdn\": \"%s\",\n", ep->dnodeFqdn); + len += snprintf(content + len, maxLen - len, " \"dnodePort\": %u\n", ep->dnodePort); + if (i < eps->dnodeNum - 1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }]\n"); + } + } + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); + taosFsync(fileno(fp)); + fclose(fp); + free(content); + terrno = 0; + + dInfo("successed to write %s", eps->file); + return 0; +} + +int32_t dnodeInitEps(Dnode *dnode, DnEps **out) { + DnEps *eps = calloc(1, sizeof(DnEps)); + if (eps == NULL) return -1; + + eps->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (eps->dnodeHash == NULL) return -1; + + eps->dnode = dnode; + eps->dnodeId = dnode->cfg->dnodeId; + eps->dnodeNum = 0; + snprintf(eps->file, sizeof(eps->file), "%s/dnodeEps.json", tsDnodeDir); + pthread_mutex_init(&eps->mutex, NULL); + *out = eps; + + int32_t ret = dnodeReadEps(eps); + if (ret == 0) { + dInfo("dnode eps is initialized"); + } + + return ret; +} + +void dnodeCleanupEps(Dnode *dnode, DnEps **out) { + DnEps *eps = *out; + *out = NULL; + + pthread_mutex_lock(&eps->mutex); + + if (eps->dnodeList != NULL) { + free(eps->dnodeList); + eps->dnodeList = NULL; + } + + if (eps->dnodeHash) { + taosHashCleanup(eps->dnodeHash); + eps->dnodeHash = NULL; + } + + eps->dnodeNum = 0; + pthread_mutex_unlock(&eps->mutex); + pthread_mutex_destroy(&eps->mutex); + free(eps); +} + +void dnodeUpdateEps(DnEps *eps, SDnodeEps *data) { + if (data == NULL || data->dnodeNum <= 0) return; + + data->dnodeNum = htonl(data->dnodeNum); + for (int32_t i = 0; i < data->dnodeNum; ++i) { + data->dnodeEps[i].dnodeId = htonl(data->dnodeEps[i].dnodeId); + data->dnodeEps[i].dnodePort = htons(data->dnodeEps[i].dnodePort); + } + + pthread_mutex_lock(&eps->mutex); + + if (data->dnodeNum != eps->dnodeNum) { + dnodeResetEps(eps, data); + dnodeWriteEps(eps); + } else { + int32_t size = data->dnodeNum * sizeof(SDnodeEp); + if (memcmp(eps->dnodeList, data->dnodeEps, size) != 0) { + dnodeResetEps(eps, data); + dnodeWriteEps(eps); + } + } + + pthread_mutex_unlock(&eps->mutex); +} + +bool dnodeIsDnodeEpChanged(DnEps *eps, int32_t dnodeId, char *epstr) { + bool changed = false; + + pthread_mutex_lock(&eps->mutex); + + SDnodeEp *ep = taosHashGet(eps->dnodeHash, &dnodeId, sizeof(int32_t)); + if (ep != NULL) { + char epSaved[TSDB_EP_LEN + 1]; + snprintf(epSaved, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort); + changed = strcmp(epstr, epSaved) != 0; + tstrncpy(epstr, epSaved, TSDB_EP_LEN); + } + + pthread_mutex_unlock(&eps->mutex); + + return changed; +} + +void dnodeGetDnodeEp(Dnode *dnode, int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port) { + assert(dnode != NULL); + + DnEps *eps = dnode->eps; + pthread_mutex_lock(&eps->mutex); + + SDnodeEp *ep = taosHashGet(eps->dnodeHash, &dnodeId, sizeof(int32_t)); + if (ep != NULL) { + if (port) *port = ep->dnodePort; + if (fqdn) tstrncpy(fqdn, ep->dnodeFqdn, TSDB_FQDN_LEN); + if (epstr) snprintf(epstr, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort); + } + + pthread_mutex_unlock(&eps->mutex); +} From daafc0feb7f71df4d96e7bbdd5562e5a8df5dc93 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Sep 2021 19:26:11 +0800 Subject: [PATCH 16/44] [TD-10430] interface of dnode/mnode/vnode module --- include/server/dnode/dnode.h | 34 ++++----- include/server/mnode/mnode.h | 143 ++++++++++++++++++++++++++++------- include/server/vnode/vnode.h | 106 ++++++++++++++++++++------ 3 files changed, 215 insertions(+), 68 deletions(-) diff --git a/include/server/dnode/dnode.h b/include/server/dnode/dnode.h index 711b6d92a5..08658f3542 100644 --- a/include/server/dnode/dnode.h +++ b/include/server/dnode/dnode.h @@ -21,53 +21,53 @@ extern "C" { #endif /** - * Initialize and start the dnode module + * Initialize and start the dnode module. * - * @return Instance of dnode module + * @return Instance of dnode module. */ struct Dnode *dnodeCreateInstance(); /** - * Stop and cleanup dnode module + * Stop and cleanup dnode module. * - * @param dnode Instance of dnode module + * @param dnode, instance of dnode module. */ void dnodeCleanupInstance(struct Dnode *dnode); /** * Send messages to other dnodes, such as create vnode message. * - * @param dnode The instance of Dnode module - * @param epSet The endpoint list of the dnodes. - * @param rpcMsg Message to be sent. + * @param dnode, the instance of Dnode module. + * @param epSet, the endpoint list of the dnodes. + * @param rpcMsg, message to be sent. */ void dnodeSendMsgToDnode(struct Dnode *dnode, struct SRpcEpSet *epSet, struct SRpcMsg *rpcMsg); /** * Send messages to mnode, such as config message. * - * @param dnode The instance of dnode module - * @param rpcMsg Message to be sent. + * @param dnode, the instance of dnode module. + * @param rpcMsg, message to be sent. */ void dnodeSendMsgToMnode(struct Dnode *dnode, struct SRpcMsg *rpcMsg); /** * Send redirect message to dnode or shell. * - * @param dnode The instance of dnode module - * @param rpcMsg Message to be sent. - * @param forShell Used to identify whether to send to shell or dnode. + * @param dnode, the instance of dnode module. + * @param rpcMsg, message to be sent. + * @param forShell, used to identify whether to send to shell or dnode. */ void dnodeSendRedirectMsg(struct Dnode *dnode, struct SRpcMsg *rpcMsg, bool forShell); /** * Get the corresponding endpoint information from dnodeId. * - * @param dnode The instance of Dnode module - * @param dnodeId The id ot dnode - * @param ep The endpoint of dnode - * @param fqdn The fqdn of dnode - * @param port The port of dnode + * @param dnode, the instance of dnode module. + * @param dnodeId, the id ot dnode. + * @param ep, the endpoint of dnode. + * @param fqdn, the fqdn of dnode. + * @param port, the port of dnode. */ void dnodeGetDnodeEp(struct Dnode *dnode, int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); diff --git a/include/server/mnode/mnode.h b/include/server/mnode/mnode.h index 0d2fd00ba1..d9135f00e6 100644 --- a/include/server/mnode/mnode.h +++ b/include/server/mnode/mnode.h @@ -20,49 +20,140 @@ extern "C" { #endif -struct SRpcMsg; +struct Dnode; + +typedef struct { + /** + * Send messages to other dnodes, such as create vnode message. + * + * @param dnode, the instance of dnode module. + * @param epSet, the endpoint list of the dnodes. + * @param rpcMsg, message to be sent. + */ + void (*SendMsgToDnode)(struct Dnode *dnode, struct SRpcEpSet *epSet, struct SRpcMsg *rpcMsg); + + /** + * Send messages to mnode, such as config message. + * + * @param dnode, the instance of dnode module. + * @param rpcMsg, message to be sent. + */ + void (*SendMsgToMnode)(struct Dnode *dnode, struct SRpcMsg *rpcMsg); + + /** + * Send redirect message to dnode or shell. + * + * @param dnode, the instance of dnode module. + * @param rpcMsg, message to be sent. + * @param forShell, used to identify whether to send to shell or dnode. + */ + void (*SendRedirectMsg)(struct Dnode *dnode, struct SRpcMsg *rpcMsg, bool forShell); + + /** + * Get the corresponding endpoint information from dnodeId. + * + * @param dnode, the instance of dDnode module. + * @param dnodeId, the id ot dnode. + * @param ep, the endpoint of dnode. + * @param fqdn, the fqdn of dnode. + * @param port, the port of dnode. + */ + void (*GetDnodeEp)(struct Dnode *dnode, int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); + +} SMnodeFp; + +typedef struct { + struct Dnode *dnode; + SMnodeFp fp; + char clusterId[TSDB_CLUSTER_ID_LEN]; + int32_t dnodeId; +} SMnodePara; /** - * Deploy Mnode instances in Dnode. - * + * Initialize and start mnode module. + * + * @param para, initialization parameters. + * @return Instance of mnode module. + */ +struct Mnode *mnodeCreateInstance(SMnodePara para); + +/** + * Stop and cleanup mnode module. + * + * @param mnode, instance of mnode module. + */ +void mnodeCleanupInstance(struct Mnode *vnode); + +/** + * Deploy mnode instances in dnode. + * + * @param mnode, instance of mnode module. + * @param minfos, server information used to deploy the mnode instance. * @return Error Code. */ -int32_t mnodeDeploy(); +int32_t mnodeDeploy(struct Mnode *mnode, struct SMInfos *minfos); /** - * Delete the Mnode instance deployed in Dnode. + * Delete the mnode instance deployed in dnode. + * + * @param mnode, instance of mnode module. */ -void mnodeUnDeploy(); - -/** - * Start Mnode service. - * - * @return Error Code. - */ -int32_t mnodeStart(); - -/** - * Stop Mnode service. - */ -int32_t mnodeStop(); +void mnodeUnDeploy(struct Mnode *mnode); /** * Interface for processing messages. - * - * @param pMsg Message to be processed. - * @return Error code + * + * @param mnode, instance of mnode module. + * @param rpcMsg, message to be processed. + * @return Error code. */ -int32_t mnodeProcessMsg(SRpcMsg *pMsg); +void mnodeProcessMsg(struct Mnode *mnode, SRpcMsg *rpcMsg); /** - * Whether the Mnode is in service. - * + * Whether the mnode is in service. + * + * @param mnode, instance of mnode module. * @return Server status. */ -bool mnodeIsServing(); +bool mnodeIsServing(struct Mnode *mnode); + +typedef struct { + int64_t numOfDnode; + int64_t numOfMnode; + int64_t numOfVgroup; + int64_t numOfDatabase; + int64_t numOfSuperTable; + int64_t numOfChildTable; + int64_t numOfColumn; + int64_t totalPoints; + int64_t totalStorage; + int64_t compStorage; +} SMnodeStat; + +/** + * Get the statistical information of Mnode. + * + * @param mnode, instance of mnode module. + * @param stat, statistical information. + * @return Error Code. + */ +int32_t mnodeGetStatistics(struct Mnode *mnode, SMnodeStat *stat); + +/** + * Get the statistical information of Mnode. + * + * @param mnode, instance of mnode module. + * @param user, username. + * @param spi, security parameter index. + * @param encrypt, encrypt algorithm. + * @param secret, key for authentication. + * @param ckey, ciphering key. + * @return Error Code. + */ +int32_t mnodeRetriveAuth(struct Mnode *mnode, char *user, char *spi, char *encrypt, char *secret, char *ckey); #ifdef __cplusplus } #endif -#endif /*_TD_MNODE_H_*/ \ No newline at end of file +#endif /*_TD_MNODE_H_*/ diff --git a/include/server/vnode/vnode.h b/include/server/vnode/vnode.h index 4a848add59..748b93c8eb 100644 --- a/include/server/vnode/vnode.h +++ b/include/server/vnode/vnode.h @@ -20,43 +20,99 @@ extern "C" { #endif -struct SRpcMsg; +struct Dnode; -/** - * Start Initialize Vnode module. - * - * @return Error Code. - */ -int32_t vnodeInit(); - -/** - * Cleanup Vnode module. - */ -void vnodeCleanup(); - typedef struct { - int64_t queryMsgCount; - int64_t writeMsgCount; -} SVnodeStat; - + /** + * Send messages to other dnodes, such as create vnode message. + * + * @param dnode, the instance of dnode module. + * @param epSet, the endpoint list of dnodes. + * @param rpcMsg, message to be sent. + */ + void (*SendMsgToDnode)(struct Dnode *dnode, struct SRpcEpSet *epSet, struct SRpcMsg *rpcMsg); + + /** + * Send messages to mnode, such as config message. + * + * @param dnode, the instance of dnode module. + * @param rpcMsg, message to be sent. + */ + void (*SendMsgToMnode)(struct Dnode *dnode, struct SRpcMsg *rpcMsg); + + /** + * Get the corresponding endpoint information from dnodeId. + * + * @param dnode, the instance of dnode module. + * @param dnodeId, the id ot dnode. + * @param ep, the endpoint of dnode. + * @param fqdn, the fqdn of dnode. + * @param port, the port of dnode. + */ + void (*GetDnodeEp)(struct Dnode *dnode, int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); + +} SVnodeFp; + +typedef struct { + struct Dnode *dnode; + SVnodeFp fp; +} SVnodePara; + /** - * Get the statistical information of Vnode + * Start initialize vnode module. * - * @param stat Statistical information. + * @param para, initialization parameters. + * @return Instance of vnode module. + */ +struct Vnode *vnodeCreateInstance(SVnodePara para); + +/** + * Cleanup vnode module. + * + * @param vnode, instance of vnode module. + */ +void vnodeCleanupInstance(struct Vnode *vnode); + +typedef struct { + int32_t unused; +} SVnodeStat; + +/** + * Get the statistical information of vnode. + * + * @param vnode, instance of vnode module. + * @param sta, statistical information. * @return Error Code. */ -int32_t vnodeGetStatistics(SVnodeStat *stat); +int32_t vnodeGetStatistics(struct Vnode *vnode, SVnodeStat *stat); -/** +/** * Interface for processing messages. * - * @param pMsg Message to be processed. - * @return Error code + * @param vnode, instance of vnode module. + * @param msg, message to be processed. */ -int32_t vnodeProcessMsg(SRpcMsg *pMsg); +void vnodeProcessMsg(struct Vnode *vnode, SRpcMsg *msg); + +/** + * Get the status of all vnodes. + * + * @param vnode, instance of vnode module. + * @param status, status msg. + */ +void vnodeGetStatus(struct Vnode *vnode, struct SStatusMsg *status); + +/** + * Set access permissions for all vnodes. + * + * @param vnode, instance of vnode module. + * @param access, access permissions of vnodes. + * @param numOfVnodes, the size of vnodes. + */ +void vnodeSetAccess(struct Vnode *vnode, struct SVgroupAccess *access, int32_t numOfVnodes); #ifdef __cplusplus } #endif -#endif /*_TD_VNODE_H_*/ \ No newline at end of file +#endif /*_TD_VNODE_H_*/ From 89005c3b14b349f68400b68b50fb8620f648f828 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Sep 2021 19:38:37 +0800 Subject: [PATCH 17/44] [TD-10430] add dnode trans for dnode module --- source/server/dnode/inc/dnodeTrans.h | 53 ++++ source/server/dnode/src/dnodeTrans.c | 428 +++++++++++++++++++++++++++ 2 files changed, 481 insertions(+) create mode 100644 source/server/dnode/inc/dnodeTrans.h create mode 100644 source/server/dnode/src/dnodeTrans.c diff --git a/source/server/dnode/inc/dnodeTrans.h b/source/server/dnode/inc/dnodeTrans.h new file mode 100644 index 0000000000..ad2894c6f2 --- /dev/null +++ b/source/server/dnode/inc/dnodeTrans.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DNODE_TRANS_H_ +#define _TD_DNODE_TRANS_H_ + +#ifdef __cplusplus +extern "C" { +#endif +#include "dnodeInt.h" + +typedef void (*RpcMsgCfp)(void *owner, SRpcMsg *pMsg, SRpcEpSet *pEpSet); +typedef void (*RpcMsgFp)(void *owner, SRpcMsg *pMsg); + +typedef struct DnMsgFp { + void * module; + RpcMsgFp fp; +} DnMsgFp; + +typedef struct DnTrans { + Dnode * dnode; + void * serverRpc; + void * clientRpc; + void * shellRpc; + int32_t queryReqNum; + int32_t submitReqNum; + DnMsgFp fpPeerMsg[TSDB_MSG_TYPE_MAX]; + DnMsgFp fpShellMsg[TSDB_MSG_TYPE_MAX]; +} DnTrans; + +int32_t dnodeInitTrans(Dnode *dnode, DnTrans **trans); +void dnodeCleanupTrans(DnTrans **trans); +void dnodeSendMsgToMnode(Dnode *dnode, SRpcMsg *rpcMsg); +void dnodeSendMsgToDnode(Dnode *dnode, SRpcEpSet *epSet, SRpcMsg *rpcMsg); +void dnodeSendMsgToDnodeRecv(Dnode *dnode, SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DNODE_TRANS_H_*/ diff --git a/source/server/dnode/src/dnodeTrans.c b/source/server/dnode/src/dnodeTrans.c new file mode 100644 index 0000000000..4d03338835 --- /dev/null +++ b/source/server/dnode/src/dnodeTrans.c @@ -0,0 +1,428 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/* this file is mainly responsible for the communication between DNODEs. Each + * dnode works as both server and client. Dnode may send status, grant, config + * messages to mnode, mnode may send create/alter/drop table/vnode messages + * to dnode. All theses messages are handled from here + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tglobal.h" +#include "dnodeMain.h" +#include "dnodeMnodeEps.h" +#include "dnodeStatus.h" +#include "dnodeTrans.h" +#include "vnode.h" +#include "mnode.h" + +static void dnodeProcessPeerReq(DnTrans *trans, SRpcMsg *pMsg, SRpcEpSet *pEpSet) { + Dnode * dnode = trans->dnode; + SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0}; + + if (pMsg->pCont == NULL) return; + if (pMsg->msgType == TSDB_MSG_TYPE_NETWORK_TEST) { + dnodeProcessStartupReq(dnode, pMsg); + return; + } + + if (dnode->main->runStatus != TD_RUN_STAT_RUNNING) { + rspMsg.code = TSDB_CODE_APP_NOT_READY; + rpcSendResponse(&rspMsg); + rpcFreeCont(pMsg->pCont); + dTrace("RPC %p, msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]); + return; + } + + if (pMsg->pCont == NULL) { + rspMsg.code = TSDB_CODE_DND_INVALID_MSG_LEN; + rpcSendResponse(&rspMsg); + return; + } + + DnMsgFp fp = trans->fpPeerMsg[pMsg->msgType]; + if (fp.fp != NULL) { + (*fp.fp)(fp.module, pMsg); + } else { + dDebug("RPC %p, peer req:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]); + rspMsg.code = TSDB_CODE_DND_MSG_NOT_PROCESSED; + rpcSendResponse(&rspMsg); + rpcFreeCont(pMsg->pCont); + } +} + +int32_t dnodeInitServer(DnTrans *trans) { + struct Dnode *dnode = trans->dnode; + struct Vnode *vnode = dnode->vnode; + struct Mnode *mnode = dnode->mnode; + + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_CREATE_TABLE] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_DROP_TABLE] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_ALTER_TABLE] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_DROP_STABLE] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; + + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_CREATE_VNODE] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_ALTER_VNODE] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_SYNC_VNODE] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_COMPACT_VNODE] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_DROP_VNODE] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_ALTER_STREAM] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; + + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = (DnMsgFp){.module = dnode, .fp = (RpcMsgFp)dnodeProcessConfigDnodeReq}; + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_CREATE_MNODE] = (DnMsgFp){.module = dnode, .fp = (RpcMsgFp)dnodeProcessCreateMnodeReq}; + + trans->fpPeerMsg[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_DM_AUTH] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_DM_GRANT] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_DM_STATUS] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = tsDnodeDnodePort; + rpcInit.label = "DND-S"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = (RpcMsgCfp)dnodeProcessPeerReq; + rpcInit.sessions = TSDB_MAX_VNODES << 4; + rpcInit.connType = TAOS_CONN_SERVER; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.owner = trans; + + trans->serverRpc = rpcOpen(&rpcInit); + if (trans->serverRpc == NULL) { + dError("failed to init peer rpc server"); + return -1; + } + + dInfo("dnode peer rpc server is initialized"); + return 0; +} + +void dnodeCleanupServer(DnTrans *trans) { + if (trans->serverRpc) { + rpcClose(trans->serverRpc); + trans->serverRpc = NULL; + dInfo("dnode peer server is closed"); + } +} + +static void dnodeProcessRspFromPeer(DnTrans *trans, SRpcMsg *pMsg, SRpcEpSet *pEpSet) { + Dnode *dnode = trans->dnode; + if (dnode->main->runStatus == TD_RUN_STAT_STOPPED) { + if (pMsg == NULL || pMsg->pCont == NULL) return; + dTrace("msg:%p is ignored since dnode is stopping", pMsg); + rpcFreeCont(pMsg->pCont); + return; + } + + if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pEpSet) { + dnodeUpdateMnodeFromPeer(dnode->meps, pEpSet); + } + + DnMsgFp fp = trans->fpPeerMsg[pMsg->msgType]; + if (fp.fp != NULL) { + (*fp.fp)(fp.module, pMsg); + } else { + dDebug("RPC %p, peer rsp:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]); + SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0}; + rspMsg.code = TSDB_CODE_DND_MSG_NOT_PROCESSED; + rpcSendResponse(&rspMsg); + rpcFreeCont(pMsg->pCont); + } + + rpcFreeCont(pMsg->pCont); +} + +int32_t dnodeInitClient(DnTrans *trans) { + struct Dnode *dnode = trans->dnode; + struct Mnode *mnode = dnode->mnode; + + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_DROP_TABLE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_DROP_STABLE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_SYNC_VNODE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_COMPACT_VNODE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_DROP_VNODE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_ALTER_STREAM_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_MD_CREATE_MNODE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + + trans->fpPeerMsg[TSDB_MSG_TYPE_DM_CONFIG_TABLE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_DM_CONFIG_VNODE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_DM_AUTH_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_DM_GRANT_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpPeerMsg[TSDB_MSG_TYPE_DM_STATUS_RSP] = (DnMsgFp){.module = dnode, .fp = (RpcMsgFp)dnodeProcessStatusRsp}; + + char secret[TSDB_KEY_LEN] = "secret"; + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.label = "DND-C"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = (RpcMsgCfp)dnodeProcessRspFromPeer; + rpcInit.sessions = TSDB_MAX_VNODES << 4; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.user = "t"; + rpcInit.ckey = "key"; + rpcInit.secret = secret; + rpcInit.owner = trans; + + trans->clientRpc = rpcOpen(&rpcInit); + if (trans->clientRpc == NULL) { + dError("failed to init peer rpc client"); + return -1; + } + + dInfo("dnode peer rpc client is initialized"); + return 0; +} + +void dnodeCleanupClient(DnTrans *trans) { + if (trans->clientRpc) { + rpcClose(trans->clientRpc); + trans->clientRpc = NULL; + dInfo("dnode peer rpc client is closed"); + } +} + +static void dnodeProcessMsgFromShell(DnTrans *trans, SRpcMsg *pMsg, SRpcEpSet *pEpSet) { + Dnode * dnode = trans->dnode; + SRpcMsg rpcMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0}; + + if (pMsg->pCont == NULL) return; + if (dnode->main->runStatus == TD_RUN_STAT_STOPPED) { + dError("RPC %p, shell msg:%s is ignored since dnode exiting", pMsg->handle, taosMsg[pMsg->msgType]); + rpcMsg.code = TSDB_CODE_DND_EXITING; + rpcSendResponse(&rpcMsg); + rpcFreeCont(pMsg->pCont); + return; + } else if (dnode->main->runStatus != TD_RUN_STAT_RUNNING) { + dError("RPC %p, shell msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]); + rpcMsg.code = TSDB_CODE_APP_NOT_READY; + rpcSendResponse(&rpcMsg); + rpcFreeCont(pMsg->pCont); + return; + } + + if (pMsg->msgType == TSDB_MSG_TYPE_QUERY) { + atomic_fetch_add_32(&trans->queryReqNum, 1); + } else if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) { + atomic_fetch_add_32(&trans->submitReqNum, 1); + } else {} + + DnMsgFp fp = trans->fpShellMsg[pMsg->msgType]; + if (fp.fp != NULL) { + (*fp.fp)(fp.module, pMsg); + } else { + dError("RPC %p, shell req:%s is not processed", pMsg->handle, taosMsg[pMsg->msgType]); + rpcMsg.code = TSDB_CODE_DND_MSG_NOT_PROCESSED; + rpcSendResponse(&rpcMsg); + rpcFreeCont(pMsg->pCont); + } +} + +static int32_t dnodeAuthNetTest(char *user, char *spi, char *encrypt, char *secret, char *ckey) { + if (strcmp(user, "nettestinternal") == 0) { + char pass[32] = {0}; + taosEncryptPass((uint8_t *)user, strlen(user), pass); + *spi = 0; + *encrypt = 0; + *ckey = 0; + memcpy(secret, pass, TSDB_KEY_LEN); + dTrace("nettest user is authorized"); + return 0; + } + + return -1; +} + +void dnodeSendMsgToDnode(Dnode *dnode, SRpcEpSet *epSet, SRpcMsg *rpcMsg) { + rpcSendRequest(dnode->trans->clientRpc, epSet, rpcMsg, NULL); +} + +void dnodeSendMsgToMnode(Dnode *dnode, SRpcMsg *rpcMsg) { + SRpcEpSet epSet = {0}; + dnodeGetEpSetForPeer(dnode->meps, &epSet); + dnodeSendMsgToDnode(dnode, &epSet, rpcMsg); +} + +void dnodeSendMsgToMnodeRecv(Dnode *dnode, SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) { + SRpcEpSet epSet = {0}; + dnodeGetEpSetForPeer(dnode->meps, &epSet); + rpcSendRecv(dnode->trans->clientRpc, &epSet, rpcMsg, rpcRsp); +} + +void dnodeSendMsgToDnodeRecv(Dnode *dnode, SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet) { + rpcSendRecv(dnode->trans->clientRpc, epSet, rpcMsg, rpcRsp); +} + +static int32_t dnodeRetrieveUserAuthInfo(void *owner, char *user, char *spi, char *encrypt, char *secret, char *ckey) { + DnTrans *trans = owner; + + if (dnodeAuthNetTest(user, spi, encrypt, secret, ckey) == 0) return 0; + + int32_t code = mnodeRetriveAuth(trans->dnode->mnode, user, spi, encrypt, secret, ckey); + if (code != TSDB_CODE_APP_NOT_READY) return code; + + SAuthMsg *pMsg = rpcMallocCont(sizeof(SAuthMsg)); + tstrncpy(pMsg->user, user, sizeof(pMsg->user)); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pMsg; + rpcMsg.contLen = sizeof(SAuthMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DM_AUTH; + + dDebug("user:%s, send auth msg to mnodes", user); + SRpcMsg rpcRsp = {0}; + dnodeSendMsgToMnodeRecv(trans->dnode, &rpcMsg, &rpcRsp); + + if (rpcRsp.code != 0) { + dError("user:%s, auth msg received from mnodes, error:%s", user, tstrerror(rpcRsp.code)); + } else { + SAuthRsp *pRsp = rpcRsp.pCont; + dDebug("user:%s, auth msg received from mnodes", user); + memcpy(secret, pRsp->secret, TSDB_KEY_LEN); + memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN); + *spi = pRsp->spi; + *encrypt = pRsp->encrypt; + } + + rpcFreeCont(rpcRsp.pCont); + return rpcRsp.code; +} + +int32_t dnodeInitShell(DnTrans *trans) { + struct Dnode *dnode = trans->dnode; + struct Vnode *vnode = dnode->vnode; + struct Mnode *mnode = dnode->mnode; + + trans->fpShellMsg[TSDB_MSG_TYPE_SUBMIT] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_QUERY] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_FETCH] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; + + // the following message shall be treated as mnode write + trans->fpShellMsg[TSDB_MSG_TYPE_CM_CREATE_ACCT] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_ALTER_ACCT] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_DROP_ACCT] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_CREATE_USER] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_ALTER_USER] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_DROP_USER] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_CREATE_DNODE] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_DROP_DNODE] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_CREATE_DB] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_CREATE_TP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_CREATE_FUNCTION] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_DROP_DB] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_SYNC_DB] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_DROP_TP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_DROP_FUNCTION] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_ALTER_DB] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_ALTER_TP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_CREATE_TABLE] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_DROP_TABLE] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_ALTER_TABLE] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_ALTER_STREAM] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_KILL_QUERY] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_KILL_STREAM] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_KILL_CONN] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_CONFIG_DNODE] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_COMPACT_VNODE] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + + // the following message shall be treated as mnode query + trans->fpShellMsg[TSDB_MSG_TYPE_CM_HEARTBEAT] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_CONNECT] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_USE_DB] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_TABLE_META] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_STABLE_VGROUP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_TABLES_META] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_SHOW] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_RETRIEVE] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->fpShellMsg[TSDB_MSG_TYPE_CM_RETRIEVE_FUNC] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + + trans->fpShellMsg[TSDB_MSG_TYPE_NETWORK_TEST] = (DnMsgFp){.module = dnode, .fp = (RpcMsgFp)dnodeProcessStartupReq}; + + int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); + if (numOfThreads < 1) { + numOfThreads = 1; + } + + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = tsDnodeShellPort; + rpcInit.label = "SHELL"; + rpcInit.numOfThreads = numOfThreads; + rpcInit.cfp = (RpcMsgCfp)dnodeProcessMsgFromShell; + rpcInit.sessions = tsMaxShellConns; + rpcInit.connType = TAOS_CONN_SERVER; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.afp = dnodeRetrieveUserAuthInfo; + rpcInit.owner = trans; + + trans->shellRpc = rpcOpen(&rpcInit); + if (trans->shellRpc == NULL) { + dError("failed to init shell rpc server"); + return -1; + } + + dInfo("dnode shell rpc server is initialized"); + return 0; +} + +void dnodeCleanupShell(DnTrans *trans) { + if (trans->shellRpc) { + rpcClose(trans->shellRpc); + trans->shellRpc = NULL; + } +} + +int32_t dnodeInitTrans(Dnode *dnode, DnTrans **out) { + DnTrans* trans = calloc(1, sizeof(DnTrans)); + if (trans == NULL) return -1; + + trans->dnode = dnode; + *out = trans; + + if (dnodeInitClient(trans) != 0) { + return -1; + } + + if (dnodeInitServer(trans) != 0) { + return -1; + } + + if (dnodeInitShell(trans) != 0) { + return -1; + } + + return 0; +} + +void dnodeCleanupTrans(DnTrans **out) { + DnTrans* trans = *out; + *out = NULL; + + dnodeCleanupShell(trans); + dnodeCleanupServer(trans); + dnodeCleanupClient(trans); + + free(trans); +} From 7dc1fb79c4a13bf43e25bbcb6e36b6d326a87045 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Sep 2021 19:38:52 +0800 Subject: [PATCH 18/44] [TD-10430] add dnode main for dnode module --- source/server/dnode/inc/dnodeStatus.h | 39 +++++++ source/server/dnode/src/dnodeStatus.c | 149 ++++++++++++++++++++++++++ 2 files changed, 188 insertions(+) create mode 100644 source/server/dnode/inc/dnodeStatus.h create mode 100644 source/server/dnode/src/dnodeStatus.c diff --git a/source/server/dnode/inc/dnodeStatus.h b/source/server/dnode/inc/dnodeStatus.h new file mode 100644 index 0000000000..ed7782fb36 --- /dev/null +++ b/source/server/dnode/inc/dnodeStatus.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DNODE_STATUS_H_ +#define _TD_DNODE_STATUS_H_ + +#ifdef __cplusplus +extern "C" { +#endif +#include "dnodeInt.h" + +typedef struct DnStatus { + Dnode * dnode; + void * dnodeTimer; + void * statusTimer; + uint32_t rebootTime; +} DnStatus; + +int32_t dnodeInitStatus(Dnode *dnode, DnStatus **status); +void dnodeCleanupStatus(DnStatus **status); +void dnodeProcessStatusRsp(Dnode *dnode, SRpcMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DNODE_STATUS_H_*/ \ No newline at end of file diff --git a/source/server/dnode/src/dnodeStatus.c b/source/server/dnode/src/dnodeStatus.c new file mode 100644 index 0000000000..4408cc8b87 --- /dev/null +++ b/source/server/dnode/src/dnodeStatus.c @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "ttimer.h" +#include "tglobal.h" +#include "dnodeCfg.h" +#include "dnodeEps.h" +#include "dnodeMnodeEps.h" +#include "dnodeStatus.h" +#include "dnodeMain.h" +#include "vnode.h" + +static void dnodeSendStatusMsg(void *handle, void *tmrId) { + DnStatus *status = handle; + if (status->dnodeTimer == NULL) { + dError("dnode timer is already released"); + return; + } + + if (status->statusTimer == NULL) { + taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, status, status->dnodeTimer, &status->statusTimer); + dError("failed to start status timer"); + return; + } + + int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad); + SStatusMsg *pStatus = rpcMallocCont(contLen); + if (pStatus == NULL) { + taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, status, status->dnodeTimer, &status->statusTimer); + dError("failed to malloc status message"); + return; + } + + Dnode *dnode = status->dnode; + dnodeGetCfg(dnode->cfg, &pStatus->dnodeId, pStatus->clusterId); + pStatus->dnodeId = htonl(dnodeGetDnodeId(dnode->cfg)); + pStatus->version = htonl(tsVersion); + pStatus->lastReboot = htonl(status->rebootTime); + pStatus->numOfCores = htons((uint16_t)tsNumOfCores); + pStatus->diskAvailable = tsAvailDataDirGB; + pStatus->alternativeRole = tsAlternativeRole; + tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN); + + // fill cluster cfg parameters + pStatus->clusterCfg.numOfMnodes = htonl(tsNumOfMnodes); + pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(tsMnodeEqualVnodeNum); + pStatus->clusterCfg.offlineThreshold = htonl(tsOfflineThreshold); + pStatus->clusterCfg.statusInterval = htonl(tsStatusInterval); + pStatus->clusterCfg.maxtablesPerVnode = htonl(tsMaxTablePerVnode); + pStatus->clusterCfg.maxVgroupsPerDb = htonl(tsMaxVgroupsPerDb); + tstrncpy(pStatus->clusterCfg.arbitrator, tsArbitrator, TSDB_EP_LEN); + tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, 64); + pStatus->clusterCfg.checkTime = 0; + char timestr[32] = "1970-01-01 00:00:00.00"; + (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN); + tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN); + + pStatus->clusterCfg.enableBalance = tsEnableBalance; + pStatus->clusterCfg.flowCtrl = tsEnableFlowCtrl; + pStatus->clusterCfg.slaveQuery = tsEnableSlaveQuery; + pStatus->clusterCfg.adjustMaster = tsEnableAdjustMaster; + + vnodeGetStatus(dnode->vnode, pStatus); + contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad); + pStatus->openVnodes = htons(pStatus->openVnodes); + + SRpcMsg rpcMsg = {.ahandle = status, .pCont = pStatus, .contLen = contLen, .msgType = TSDB_MSG_TYPE_DM_STATUS}; + + dnodeSendMsgToMnode(status->dnode, &rpcMsg); +} + +void dnodeProcessStatusRsp(Dnode *dnode, SRpcMsg *pMsg) { + DnStatus *status = pMsg->ahandle; + + if (pMsg->code != TSDB_CODE_SUCCESS) { + dError("status rsp is received, error:%s", tstrerror(pMsg->code)); + if (pMsg->code == TSDB_CODE_MND_DNODE_NOT_EXIST) { + char clusterId[TSDB_CLUSTER_ID_LEN]; + dnodeGetClusterId(dnode->cfg, clusterId); + if (clusterId[0] != '\0') { + dnodeSetDropped(dnode->cfg); + dError("exit zombie dropped dnode"); + exit(EXIT_FAILURE); + } + } + + taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, status, status->dnodeTimer, &status->statusTimer); + return; + } + + SStatusRsp *pStatusRsp = pMsg->pCont; + SMInfos * minfos = &pStatusRsp->mnodes; + dnodeUpdateMnodeFromStatus(dnode->meps, minfos); + + SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg; + pCfg->numOfVnodes = htonl(pCfg->numOfVnodes); + pCfg->moduleStatus = htonl(pCfg->moduleStatus); + pCfg->dnodeId = htonl(pCfg->dnodeId); + dnodeUpdateCfg(dnode->cfg, pCfg); + + vnodeSetAccess(dnode->vnode, pStatusRsp->vgAccess, pCfg->numOfVnodes); + + SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess)); + dnodeUpdateEps(dnode->eps, pEps); + + taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, status, status->dnodeTimer, &status->statusTimer); +} + +int32_t dnodeInitStatus(Dnode *dnode, DnStatus **out) { + DnStatus *status = calloc(1, sizeof(DnStatus)); + if (status == NULL) return -1; + + status->dnode = dnode; + status->statusTimer = NULL; + status->dnodeTimer = dnode->main->dnodeTimer; + status->rebootTime = taosGetTimestampSec(); + taosTmrReset(dnodeSendStatusMsg, 500, status, status->dnodeTimer, &status->statusTimer); + *out = status; + + dInfo("dnode status timer is initialized"); + return TSDB_CODE_SUCCESS; +} + +void dnodeCleanupStatus(DnStatus **out) { + DnStatus *status = *out; + *out = NULL; + + if (status->statusTimer != NULL) { + taosTmrStopA(&status->statusTimer); + status->statusTimer = NULL; + } + + free(status); +} From 446411210dc755f3e71c17cebfdc594cc41769fc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Sep 2021 19:39:04 +0800 Subject: [PATCH 19/44] [TD-10430] add dnode telemetry for dnode module --- source/server/dnode/inc/dnodeTelemetry.h | 45 ++++ source/server/dnode/src/dnodeTelemetry.c | 323 +++++++++++++++++++++++ 2 files changed, 368 insertions(+) create mode 100644 source/server/dnode/inc/dnodeTelemetry.h create mode 100644 source/server/dnode/src/dnodeTelemetry.c diff --git a/source/server/dnode/inc/dnodeTelemetry.h b/source/server/dnode/inc/dnodeTelemetry.h new file mode 100644 index 0000000000..e3e097d2f2 --- /dev/null +++ b/source/server/dnode/inc/dnodeTelemetry.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2020 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DNODE_TELEMETRY_H_ +#define _TD_DNODE_TELEMETRY_H_ + +#ifdef __cplusplus +extern "C" { +#endif +#include "dnodeInt.h" + +/* + * sem_timedwait is NOT implemented on MacOSX + * thus we use pthread_mutex_t/pthread_cond_t to simulate + */ +typedef struct DnTelem { + Dnode * dnode; + bool enable; + pthread_mutex_t lock; + pthread_cond_t cond; + volatile int32_t exit; + pthread_t thread; + char email[TSDB_FQDN_LEN]; +} DnTelem; + +int32_t dnodeInitTelemetry(Dnode *dnode, DnTelem **telem); +void dnodeCleanupTelemetry(DnTelem **telem); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DNODE_TELEMETRY_H_*/ diff --git a/source/server/dnode/src/dnodeTelemetry.c b/source/server/dnode/src/dnodeTelemetry.c new file mode 100644 index 0000000000..8c2d4a431e --- /dev/null +++ b/source/server/dnode/src/dnodeTelemetry.c @@ -0,0 +1,323 @@ +/* + * Copyright (c) 2020 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "osTime.h" +#include "tbuffer.h" +#include "tglobal.h" +#include "tsocket.h" +#include "dnodeCfg.h" +#include "dnodeTelemetry.h" +#include "mnode.h" + +#define TELEMETRY_SERVER "telemetry.taosdata.com" +#define TELEMETRY_PORT 80 +#define REPORT_INTERVAL 86400 + +static void dnodeBeginObject(SBufferWriter* bw) { tbufWriteChar(bw, '{'); } + +static void dnodeCloseObject(SBufferWriter* bw) { + size_t len = tbufTell(bw); + if (tbufGetData(bw, false)[len - 1] == ',') { + tbufWriteCharAt(bw, len - 1, '}'); + } else { + tbufWriteChar(bw, '}'); + } + tbufWriteChar(bw, ','); +} + +#if 0 +static void beginArray(SBufferWriter* bw) { + tbufWriteChar(bw, '['); +} + +static void closeArray(SBufferWriter* bw) { + size_t len = tbufTell(bw); + if (tbufGetData(bw, false)[len - 1] == ',') { + tbufWriteCharAt(bw, len - 1, ']'); + } else { + tbufWriteChar(bw, ']'); + } + tbufWriteChar(bw, ','); +} +#endif + +static void dnodeWriteString(SBufferWriter* bw, const char* str) { + tbufWriteChar(bw, '"'); + tbufWrite(bw, str, strlen(str)); + tbufWriteChar(bw, '"'); +} + +static void dnodeAddIntField(SBufferWriter* bw, const char* k, int64_t v) { + dnodeWriteString(bw, k); + tbufWriteChar(bw, ':'); + char buf[32]; + sprintf(buf, "%" PRId64, v); + tbufWrite(bw, buf, strlen(buf)); + tbufWriteChar(bw, ','); +} + +static void dnodeAddStringField(SBufferWriter* bw, const char* k, const char* v) { + dnodeWriteString(bw, k); + tbufWriteChar(bw, ':'); + dnodeWriteString(bw, v); + tbufWriteChar(bw, ','); +} + +static void dnodeAddCpuInfo(SBufferWriter* bw) { + char* line = NULL; + size_t size = 0; + int32_t done = 0; + + FILE* fp = fopen("/proc/cpuinfo", "r"); + if (fp == NULL) { + return; + } + + while (done != 3 && (size = tgetline(&line, &size, fp)) != -1) { + line[size - 1] = '\0'; + if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) { + const char* v = strchr(line, ':') + 2; + dnodeAddStringField(bw, "cpuModel", v); + done |= 1; + } else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) { + const char* v = strchr(line, ':') + 2; + dnodeWriteString(bw, "numOfCpu"); + tbufWriteChar(bw, ':'); + tbufWrite(bw, v, strlen(v)); + tbufWriteChar(bw, ','); + done |= 2; + } + } + + free(line); + fclose(fp); +} + +static void dnodeAddOsInfo(SBufferWriter* bw) { + char* line = NULL; + size_t size = 0; + + FILE* fp = fopen("/etc/os-release", "r"); + if (fp == NULL) { + return; + } + + while ((size = tgetline(&line, &size, fp)) != -1) { + line[size - 1] = '\0'; + if (strncmp(line, "PRETTY_NAME", 11) == 0) { + const char* p = strchr(line, '=') + 1; + if (*p == '"') { + p++; + line[size - 2] = 0; + } + dnodeAddStringField(bw, "os", p); + break; + } + } + + free(line); + fclose(fp); +} + +static void dnodeAddMemoryInfo(SBufferWriter* bw) { + char* line = NULL; + size_t size = 0; + + FILE* fp = fopen("/proc/meminfo", "r"); + if (fp == NULL) { + return; + } + + while ((size = tgetline(&line, &size, fp)) != -1) { + line[size - 1] = '\0'; + if (strncmp(line, "MemTotal", 8) == 0) { + const char* p = strchr(line, ':') + 1; + while (*p == ' ') p++; + dnodeAddStringField(bw, "memory", p); + break; + } + } + + free(line); + fclose(fp); +} + +static void dnodeAddVersionInfo(DnTelem* telem, SBufferWriter* bw) { + dnodeAddStringField(bw, "version", version); + dnodeAddStringField(bw, "buildInfo", buildinfo); + dnodeAddStringField(bw, "gitInfo", gitinfo); + dnodeAddStringField(bw, "email", telem->email); +} + +static void dnodeAddRuntimeInfo(DnTelem* telem, SBufferWriter* bw) { + SMnodeStat stat = {0}; + if (mnodeGetStatistics(telem->dnode->mnode, &stat) != 0) { + return; + } + + dnodeAddIntField(bw, "numOfDnode", stat.numOfDnode); + dnodeAddIntField(bw, "numOfMnode", stat.numOfMnode); + dnodeAddIntField(bw, "numOfVgroup", stat.numOfVgroup); + dnodeAddIntField(bw, "numOfDatabase", stat.numOfDatabase); + dnodeAddIntField(bw, "numOfSuperTable", stat.numOfSuperTable); + dnodeAddIntField(bw, "numOfChildTable", stat.numOfChildTable); + dnodeAddIntField(bw, "numOfColumn", stat.numOfColumn); + dnodeAddIntField(bw, "numOfPoint", stat.totalPoints); + dnodeAddIntField(bw, "totalStorage", stat.totalStorage); + dnodeAddIntField(bw, "compStorage", stat.compStorage); +} + +static void dnodeSendTelemetryReport(DnTelem* telem) { + char buf[128] = {0}; + uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); + if (ip == 0xffffffff) { + dTrace("failed to get IP address of " TELEMETRY_SERVER ", reason:%s", strerror(errno)); + return; + } + SOCKET fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); + if (fd < 0) { + dTrace("failed to create socket for telemetry, reason:%s", strerror(errno)); + return; + } + + SBufferWriter bw = tbufInitWriter(NULL, false); + dnodeBeginObject(&bw); + dnodeAddStringField(&bw, "instanceId", telem->dnode->cfg->clusterId); + dnodeAddIntField(&bw, "reportVersion", 1); + dnodeAddOsInfo(&bw); + dnodeAddCpuInfo(&bw); + dnodeAddMemoryInfo(&bw); + dnodeAddVersionInfo(telem, &bw); + dnodeAddRuntimeInfo(telem, &bw); + dnodeCloseObject(&bw); + + const char* header = + "POST /report HTTP/1.1\n" + "Host: " TELEMETRY_SERVER + "\n" + "Content-Type: application/json\n" + "Content-Length: "; + + taosWriteSocket(fd, header, (int32_t)strlen(header)); + int32_t contLen = (int32_t)(tbufTell(&bw) - 1); + sprintf(buf, "%d\n\n", contLen); + taosWriteSocket(fd, buf, (int32_t)strlen(buf)); + taosWriteSocket(fd, tbufGetData(&bw, false), contLen); + tbufCloseWriter(&bw); + + // read something to avoid nginx error 499 + if (taosReadSocket(fd, buf, 10) < 0) { + dTrace("failed to receive response since %s", strerror(errno)); + } + + taosCloseSocket(fd); +} + +static void* dnodeTelemThreadFp(void* param) { + DnTelem* telem = param; + + struct timespec end = {0}; + clock_gettime(CLOCK_REALTIME, &end); + end.tv_sec += 300; // wait 5 minutes before send first report + + setThreadName("dnode-telem"); + + while (!telem->exit) { + int32_t r = 0; + struct timespec ts = end; + pthread_mutex_lock(&telem->lock); + r = pthread_cond_timedwait(&telem->cond, &telem->lock, &ts); + pthread_mutex_unlock(&telem->lock); + if (r == 0) break; + if (r != ETIMEDOUT) continue; + + if (mnodeIsServing(telem->dnode->mnode)) { + dnodeSendTelemetryReport(telem); + } + end.tv_sec += REPORT_INTERVAL; + } + + return NULL; +} + +static void dnodeGetEmail(DnTelem* telem, char* filepath) { + int32_t fd = open(filepath, O_RDONLY); + if (fd < 0) { + return; + } + + if (taosRead(fd, (void*)telem->email, TSDB_FQDN_LEN) < 0) { + dError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno)); + } + + taosClose(fd); +} + +int32_t dnodeInitTelemetry(Dnode* dnode, DnTelem** out) { + DnTelem* telem = calloc(1, sizeof(DnTelem)); + if (telem == NULL) return -1; + + telem->dnode = dnode; + telem->enable = tsEnableTelemetryReporting; + *out = telem; + + if (!telem->enable) return 0; + + telem->exit = 0; + pthread_mutex_init(&telem->lock, NULL); + pthread_cond_init(&telem->cond, NULL); + telem->email[0] = 0; + + dnodeGetEmail(telem, "/usr/local/taos/email"); + + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); + + int32_t code = pthread_create(&telem->thread, &attr, dnodeTelemThreadFp, telem); + pthread_attr_destroy(&attr); + if (code != 0) { + dTrace("failed to create telemetry thread since :%s", strerror(code)); + } + + dInfo("dnode telemetry is initialized"); + return 0; +} + +void dnodeCleanupTelemetry(DnTelem** out) { + DnTelem* telem = *out; + *out = NULL; + + if (!telem->enable) { + free(telem); + return; + } + + if (taosCheckPthreadValid(telem->thread)) { + pthread_mutex_lock(&telem->lock); + telem->exit = 1; + pthread_cond_signal(&telem->cond); + pthread_mutex_unlock(&telem->lock); + + pthread_join(telem->thread, NULL); + } + + pthread_mutex_destroy(&telem->lock); + pthread_cond_destroy(&telem->cond); + + free(telem); +} From b3d7ec362823d5a6d8efe2383cb6fff46533f2d9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Sep 2021 19:39:17 +0800 Subject: [PATCH 20/44] [TD-10430] add mnode epset for dnode module --- include/server/mnode/mnode.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/include/server/mnode/mnode.h b/include/server/mnode/mnode.h index d9135f00e6..f6c067f784 100644 --- a/include/server/mnode/mnode.h +++ b/include/server/mnode/mnode.h @@ -100,15 +100,6 @@ int32_t mnodeDeploy(struct Mnode *mnode, struct SMInfos *minfos); */ void mnodeUnDeploy(struct Mnode *mnode); -/** - * Interface for processing messages. - * - * @param mnode, instance of mnode module. - * @param rpcMsg, message to be processed. - * @return Error code. - */ -void mnodeProcessMsg(struct Mnode *mnode, SRpcMsg *rpcMsg); - /** * Whether the mnode is in service. * @@ -152,6 +143,15 @@ int32_t mnodeGetStatistics(struct Mnode *mnode, SMnodeStat *stat); */ int32_t mnodeRetriveAuth(struct Mnode *mnode, char *user, char *spi, char *encrypt, char *secret, char *ckey); +/** + * Interface for processing messages. + * + * @param mnode, instance of mnode module. + * @param rpcMsg, message to be processed. + * @return Error code. + */ +void mnodeProcessMsg(struct Mnode *mnode, SRpcMsg *rpcMsg); + #ifdef __cplusplus } #endif From 413818a1042ac91e032e264f130dc50361aeefd9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Sep 2021 19:39:40 +0800 Subject: [PATCH 21/44] [TD-10430] add mnode epset for dnode module --- source/server/dnode/inc/dnodeMnodeEps.h | 44 ++++ source/server/dnode/src/dnodeMnodeEps.c | 317 ++++++++++++++++++++++++ 2 files changed, 361 insertions(+) create mode 100644 source/server/dnode/inc/dnodeMnodeEps.h create mode 100644 source/server/dnode/src/dnodeMnodeEps.c diff --git a/source/server/dnode/inc/dnodeMnodeEps.h b/source/server/dnode/inc/dnodeMnodeEps.h new file mode 100644 index 0000000000..950fc3783d --- /dev/null +++ b/source/server/dnode/inc/dnodeMnodeEps.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DNODE_MNODE_EP_H_ +#define _TD_DNODE_MNODE_EP_H_ + +#ifdef __cplusplus +extern "C" { +#endif +#include "dnodeInt.h" + +typedef struct DnMnEps { + Dnode * dnode; + SRpcEpSet mnodeEpSet; + SMInfos mnodeInfos; + char file[PATH_MAX + 20]; + pthread_mutex_t mutex; +} DnMnEps; + +int32_t dnodeInitMnodeEps(Dnode *dnode, DnMnEps **meps); +void dnodeCleanupMnodeEps(DnMnEps **meps); +void dnodeUpdateMnodeFromStatus(DnMnEps *meps, SMInfos *pMinfos); +void dnodeUpdateMnodeFromPeer(DnMnEps *meps, SRpcEpSet *pEpSet); +void dnodeGetEpSetForPeer(DnMnEps *meps, SRpcEpSet *epSet); +void dnodeGetEpSetForShell(DnMnEps *meps, SRpcEpSet *epSet); +void dnodeSendRedirectMsg(Dnode *dnode, SRpcMsg *rpcMsg, bool forShell); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DNODE_MNODE_EP_H_*/ diff --git a/source/server/dnode/src/dnodeMnodeEps.c b/source/server/dnode/src/dnodeMnodeEps.c new file mode 100644 index 0000000000..ba926b0094 --- /dev/null +++ b/source/server/dnode/src/dnodeMnodeEps.c @@ -0,0 +1,317 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "cJSON.h" +#include "tglobal.h" +#include "dnodeCfg.h" +#include "dnodeEps.h" +#include "dnodeMnodeEps.h" +#include "mnode.h" + +static void dnodePrintMnodeEps(DnMnEps *meps) { + SRpcEpSet *epset = &meps->mnodeEpSet; + dInfo("print mnode eps, num:%d inuse:%d", epset->numOfEps, epset->inUse); + for (int32_t i = 0; i < epset->numOfEps; i++) { + dInfo("ep index:%d, %s:%u", i, epset->fqdn[i], epset->port[i]); + } +} + +static void dnodeResetMnodeEps(DnMnEps *meps, SMInfos *mInfos) { + if (mInfos == NULL || mInfos->mnodeNum == 0) { + meps->mnodeEpSet.numOfEps = 1; + taosGetFqdnPortFromEp(tsFirst, meps->mnodeEpSet.fqdn[0], &meps->mnodeEpSet.port[0]); + + if (strcmp(tsSecond, tsFirst) != 0) { + meps->mnodeEpSet.numOfEps = 2; + taosGetFqdnPortFromEp(tsSecond, meps->mnodeEpSet.fqdn[1], &meps->mnodeEpSet.port[1]); + } + dnodePrintMnodeEps(meps); + return; + } + + int32_t size = sizeof(SMInfos); + memcpy(&meps->mnodeInfos, mInfos, size); + + meps->mnodeEpSet.inUse = meps->mnodeInfos.inUse; + meps->mnodeEpSet.numOfEps = meps->mnodeInfos.mnodeNum; + for (int32_t i = 0; i < meps->mnodeInfos.mnodeNum; i++) { + taosGetFqdnPortFromEp(meps->mnodeInfos.mnodeInfos[i].mnodeEp, meps->mnodeEpSet.fqdn[i], &meps->mnodeEpSet.port[i]); + } + + dnodePrintMnodeEps(meps); +} + +static int32_t dnodeWriteMnodeEps(DnMnEps *meps) { + FILE *fp = fopen(meps->file, "w"); + if (!fp) { + dError("failed to write %s since %s", meps->file, strerror(errno)); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 2000; + char * content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"inUse\": %d,\n", meps->mnodeInfos.inUse); + len += snprintf(content + len, maxLen - len, " \"nodeNum\": %d,\n", meps->mnodeInfos.mnodeNum); + len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); + for (int32_t i = 0; i < meps->mnodeInfos.mnodeNum; i++) { + len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", meps->mnodeInfos.mnodeInfos[i].mnodeId); + len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", meps->mnodeInfos.mnodeInfos[i].mnodeEp); + if (i < meps->mnodeInfos.mnodeNum - 1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }]\n"); + } + } + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); + taosFsync(fileno(fp)); + fclose(fp); + free(content); + terrno = 0; + + dInfo("successed to write %s", meps->file); + return 0; +} + +static int32_t dnodeReadMnodeEps(DnMnEps *meps, DnEps *deps) { + int32_t len = 0; + int32_t maxLen = 2000; + char * content = calloc(1, maxLen + 1); + cJSON * root = NULL; + FILE * fp = NULL; + SMInfos mInfos = {0}; + bool nodeChanged = false; + + fp = fopen(meps->file, "r"); + if (!fp) { + dDebug("file %s not exist", meps->file); + goto PARSE_MINFOS_OVER; + } + + len = (int32_t)fread(content, 1, maxLen, fp); + if (len <= 0) { + dError("failed to read %s since content is null", meps->file); + goto PARSE_MINFOS_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", meps->file); + goto PARSE_MINFOS_OVER; + } + + cJSON *inUse = cJSON_GetObjectItem(root, "inUse"); + if (!inUse || inUse->type != cJSON_Number) { + dError("failed to read mnodeEpSet.json since inUse not found"); + goto PARSE_MINFOS_OVER; + } + meps->mnodeInfos.inUse = (int8_t)inUse->valueint; + + cJSON *nodeNum = cJSON_GetObjectItem(root, "nodeNum"); + if (!nodeNum || nodeNum->type != cJSON_Number) { + dError("failed to read mnodeEpSet.json since nodeNum not found"); + goto PARSE_MINFOS_OVER; + } + mInfos.mnodeNum = (int8_t)nodeNum->valueint; + + cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); + if (!nodeInfos || nodeInfos->type != cJSON_Array) { + dError("failed to read mnodeEpSet.json since nodeInfos not found"); + goto PARSE_MINFOS_OVER; + } + + int32_t size = cJSON_GetArraySize(nodeInfos); + if (size != mInfos.mnodeNum) { + dError("failed to read mnodeEpSet.json since nodeInfos size not matched"); + goto PARSE_MINFOS_OVER; + } + + for (int32_t i = 0; i < size; ++i) { + cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i); + if (nodeInfo == NULL) continue; + + cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId"); + if (!nodeId || nodeId->type != cJSON_Number) { + dError("failed to read mnodeEpSet.json since nodeId not found"); + goto PARSE_MINFOS_OVER; + } + + cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); + if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { + dError("failed to read mnodeEpSet.json since nodeName not found"); + goto PARSE_MINFOS_OVER; + } + + SMInfo *mInfo = &mInfos.mnodeInfos[i]; + mInfo->mnodeId = (int32_t)nodeId->valueint; + tstrncpy(mInfo->mnodeEp, nodeEp->valuestring, TSDB_EP_LEN); + + bool changed = dnodeIsDnodeEpChanged(deps, mInfo->mnodeId, mInfo->mnodeEp); + if (changed) nodeChanged = changed; + } + + dInfo("successed to read file %s", meps->file); + +PARSE_MINFOS_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (fp != NULL) fclose(fp); + terrno = 0; + + for (int32_t i = 0; i < mInfos.mnodeNum; ++i) { + SMInfo *mInfo = &mInfos.mnodeInfos[i]; + dnodeGetDnodeEp(meps->dnode, mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL); + } + + dnodeResetMnodeEps(meps, &mInfos); + + if (nodeChanged) { + dnodeWriteMnodeEps(meps); + } + + return 0; +} + +void dnodeSendRedirectMsg(struct Dnode *dnode, SRpcMsg *rpcMsg, bool forShell) { + DnMnEps *meps = dnode->meps; + SRpcConnInfo connInfo = {0}; + rpcGetConnInfo(rpcMsg->handle, &connInfo); + + SRpcEpSet epSet = {0}; + if (forShell) { + dnodeGetEpSetForShell(meps, &epSet); + } else { + dnodeGetEpSetForPeer(meps, &epSet); + } + + dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType], + taosIpStr(connInfo.clientIp), connInfo.user, epSet.numOfEps, epSet.inUse); + + for (int32_t i = 0; i < epSet.numOfEps; ++i) { + dDebug("mnode index:%d %s:%d", i, epSet.fqdn[i], epSet.port[i]); + if (strcmp(epSet.fqdn[i], tsLocalFqdn) == 0) { + if ((epSet.port[i] == tsServerPort + TSDB_PORT_DNODEDNODE && !forShell) || + (epSet.port[i] == tsServerPort && forShell)) { + epSet.inUse = (i + 1) % epSet.numOfEps; + dDebug("mnode index:%d %s:%d set inUse to %d", i, epSet.fqdn[i], epSet.port[i], epSet.inUse); + } + } + + epSet.port[i] = htons(epSet.port[i]); + } + + rpcSendRedirectRsp(rpcMsg->handle, &epSet); +} + +int32_t dnodeInitMnodeEps(Dnode *dnode, DnMnEps **out) { + DnMnEps *meps = calloc(1, sizeof(DnMnEps)); + if (meps == NULL) return -1; + + meps->dnode = dnode; + snprintf(meps->file, sizeof(meps->file), "%s/mnodeEpSet.json", tsDnodeDir); + pthread_mutex_init(&meps->mutex, NULL); + *out = meps; + + dnodeResetMnodeEps(meps, NULL); + int32_t ret = dnodeReadMnodeEps(meps, dnode->eps); + if (ret == 0) { + dInfo("dnode mInfos is initialized"); + } + + return ret; +} + +void dnodeCleanupMnodeEps(DnMnEps **out) { + DnMnEps *meps = *out; + *out = NULL; + + if (meps != NULL) { + pthread_mutex_destroy(&meps->mutex); + free(meps); + } +} + +void dnodeUpdateMnodeFromStatus(DnMnEps *meps, SMInfos *mInfos) { + if (mInfos->mnodeNum <= 0 || mInfos->mnodeNum > TSDB_MAX_REPLICA) { + dError("invalid mInfos since num:%d invalid", mInfos->mnodeNum); + return; + } + + for (int32_t i = 0; i < mInfos->mnodeNum; ++i) { + SMInfo *minfo = &mInfos->mnodeInfos[i]; + minfo->mnodeId = htonl(minfo->mnodeId); + if (minfo->mnodeId <= 0 || strlen(minfo->mnodeEp) <= 5) { + dError("invalid mInfo:%d since id:%d and ep:%s invalid", i, minfo->mnodeId, minfo->mnodeEp); + return; + } + } + + pthread_mutex_lock(&meps->mutex); + if (mInfos->mnodeNum != meps->mnodeInfos.mnodeNum) { + dnodeResetMnodeEps(meps, mInfos); + dnodeWriteMnodeEps(meps); + } else { + int32_t size = sizeof(SMInfos); + if (memcmp(mInfos, &meps->mnodeInfos, size) != 0) { + dnodeResetMnodeEps(meps, mInfos); + dnodeWriteMnodeEps(meps); + } + } + pthread_mutex_unlock(&meps->mutex); +} + +void dnodeUpdateMnodeFromPeer(DnMnEps *meps, SRpcEpSet *ep) { + if (ep->numOfEps <= 0) { + dError("mInfos is changed, but content is invalid, discard it"); + return; + } + + pthread_mutex_lock(&meps->mutex); + + dInfo("mInfos is changed, numOfEps:%d inUse:%d", ep->numOfEps, ep->inUse); + for (int32_t i = 0; i < ep->numOfEps; ++i) { + ep->port[i] -= TSDB_PORT_DNODEDNODE; + dInfo("minfo:%d %s:%u", i, ep->fqdn[i], ep->port[i]); + } + meps->mnodeEpSet = *ep; + + pthread_mutex_unlock(&meps->mutex); +} + +void dnodeGetEpSetForPeer(DnMnEps *meps, SRpcEpSet *epSet) { + pthread_mutex_lock(&meps->mutex); + + *epSet = meps->mnodeEpSet; + for (int32_t i = 0; i < epSet->numOfEps; ++i) { + epSet->port[i] += TSDB_PORT_DNODEDNODE; + } + + pthread_mutex_unlock(&meps->mutex); +} + +void dnodeGetEpSetForShell(DnMnEps *meps, SRpcEpSet *epSet) { + pthread_mutex_lock(&meps->mutex); + + *epSet = meps->mnodeEpSet; + + pthread_mutex_unlock(&meps->mutex); +} From bbcb0f0b132e7b3157f6363cc76d6e8c93f19a69 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Sep 2021 19:39:57 +0800 Subject: [PATCH 22/44] [TD-10430] add dnode main for dnode module --- source/server/dnode/inc/dnodeMain.h | 51 ++++++ source/server/dnode/src/dnodeMain.c | 260 ++++++++++++++++++++++++++++ 2 files changed, 311 insertions(+) create mode 100644 source/server/dnode/inc/dnodeMain.h create mode 100644 source/server/dnode/src/dnodeMain.c diff --git a/source/server/dnode/inc/dnodeMain.h b/source/server/dnode/inc/dnodeMain.h new file mode 100644 index 0000000000..f9c15db1c1 --- /dev/null +++ b/source/server/dnode/inc/dnodeMain.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DNODE_MAIN_H_ +#define _TD_DNODE_MAIN_H_ + +#ifdef __cplusplus +extern "C" { +#endif +#include "dnodeInt.h" + +typedef enum { + TD_RUN_STAT_INIT, + TD_RUN_STAT_RUNNING, + TD_RUN_STAT_STOPPED +} RunStat; + +typedef struct DnMain { + Dnode * dnode; + RunStat runStatus; + void * dnodeTimer; + SStartupStep startup; +} DnMain; + +int32_t dnodeInitMain(Dnode *dnode, DnMain **main); +void dnodeCleanupMain(DnMain **main); +int32_t dnodeInitStorage(Dnode *dnode, void **unused); +void dnodeCleanupStorage(void **unused); +void dnodeReportStartup(Dnode *dnode, char *name, char *desc); +void dnodeReportStartupFinished(Dnode *dnode, char *name, char *desc); +void dnodeProcessStartupReq(Dnode *dnode, SRpcMsg *pMsg); +void dnodeProcessCreateMnodeReq(Dnode *dnode, SRpcMsg *pMsg); +void dnodeProcessConfigDnodeReq(Dnode *dnode, SRpcMsg *pMsg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DNODE_MAIN_H_*/ diff --git a/source/server/dnode/src/dnodeMain.c b/source/server/dnode/src/dnodeMain.c new file mode 100644 index 0000000000..a792cf6d5b --- /dev/null +++ b/source/server/dnode/src/dnodeMain.c @@ -0,0 +1,260 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tcache.h" +#include "tconfig.h" +#include "tfs.h" +#include "tnote.h" +#include "tscompression.h" +#include "ttimer.h" +#include "dnodeCfg.h" +#include "dnodeMain.h" +#include "mnode.h" + +static int32_t dnodeCreateDir(const char *dir) { + if (mkdir(dir, 0755) != 0 && errno != EEXIST) { + return -1; + } + + return 0; +} + +static void dnodeCheckDataDirOpenned(char *dir) { + char filepath[256] = {0}; + snprintf(filepath, sizeof(filepath), "%s/.running", dir); + + int32_t fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); + if (fd < 0) { + dError("failed to open lock file:%s, reason: %s, quit", filepath, strerror(errno)); + exit(0); + } + + int32_t ret = flock(fd, LOCK_EX | LOCK_NB); + if (ret != 0) { + dError("failed to lock file:%s ret:%d since %s, database may be running, quit", filepath, ret, strerror(errno)); + close(fd); + exit(0); + } +} + +int32_t dnodeInitMain(Dnode *dnode, DnMain **out) { + DnMain* main = calloc(1, sizeof(DnMain)); + if (main == NULL) return -1; + + main->dnode = dnode; + main->runStatus = TD_RUN_STAT_STOPPED; + main->dnodeTimer = taosTmrInit(100, 200, 60000, "DND-TMR"); + if (main->dnodeTimer == NULL) { + dError("failed to init dnode timer"); + return -1; + } + + *out = main; + + tscEmbedded = 1; + taosIgnSIGPIPE(); + taosBlockSIGPIPE(); + taosResolveCRC(); + taosInitGlobalCfg(); + taosReadGlobalLogCfg(); + taosSetCoreDump(); + + if (dnodeCreateDir(tsLogDir) < 0) { + printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno)); + return -1; + } + + char temp[TSDB_FILENAME_LEN]; + sprintf(temp, "%s/taosdlog", tsLogDir); + if (taosInitLog(temp, tsNumOfLogLines, 1) < 0) { + printf("failed to init log file\n"); + } + + if (!taosReadGlobalCfg()) { + taosPrintGlobalCfg(); + dError("TDengine read global config failed"); + return -1; + } + + dInfo("start to initialize TDengine"); + + taosInitNotes(); + + return taosCheckGlobalCfg(); +} + +void dnodeCleanupMain(DnMain **out) { + DnMain *main = *out; + *out = NULL; + + if (main->dnodeTimer != NULL) { + taosTmrCleanUp(main->dnodeTimer); + main->dnodeTimer = NULL; + } + + taos_cleanup(); + taosCloseLog(); + taosStopCacheRefreshWorker(); + + free(main); +} + +int32_t dnodeInitStorage(Dnode *dnode, void **m) { +#ifdef TD_TSZ + // compress module init + tsCompressInit(); +#endif + + // storage module init + if (tsDiskCfgNum == 1 && dnodeCreateDir(tsDataDir) < 0) { + dError("failed to create dir:%s since %s", tsDataDir, strerror(errno)); + return -1; + } + + if (tfsInit(tsDiskCfg, tsDiskCfgNum) < 0) { + dError("failed to init TFS since %s", tstrerror(terrno)); + return -1; + } + strncpy(tsDataDir, TFS_PRIMARY_PATH(), TSDB_FILENAME_LEN); + sprintf(tsMnodeDir, "%s/mnode", tsDataDir); + sprintf(tsVnodeDir, "%s/vnode", tsDataDir); + sprintf(tsDnodeDir, "%s/dnode", tsDataDir); + + if (dnodeCreateDir(tsMnodeDir) < 0) { + dError("failed to create dir:%s since %s", tsMnodeDir, strerror(errno)); + return -1; + } + + if (dnodeCreateDir(tsDnodeDir) < 0) { + dError("failed to create dir:%s since %s", tsDnodeDir, strerror(errno)); + return -1; + } + + if (tfsMkdir("vnode") < 0) { + dError("failed to create vnode dir since %s", tstrerror(terrno)); + return -1; + } + + if (tfsMkdir("vnode_bak") < 0) { + dError("failed to create vnode_bak dir since %s", tstrerror(terrno)); + return -1; + } + + TDIR *tdir = tfsOpendir("vnode_bak/.staging"); + bool stagingNotEmpty = tfsReaddir(tdir) != NULL; + tfsClosedir(tdir); + + if (stagingNotEmpty) { + dError("vnode_bak/.staging dir not empty, fix it first."); + return -1; + } + + if (tfsMkdir("vnode_bak/.staging") < 0) { + dError("failed to create vnode_bak/.staging dir since %s", tstrerror(terrno)); + return -1; + } + + dnodeCheckDataDirOpenned(tsDnodeDir); + + taosGetDisk(); + taosPrintDiskInfo(); + + dInfo("dnode storage is initialized at %s", tsDnodeDir); + return 0; +} + +void dnodeCleanupStorage(void **m) { + // storage destroy + tfsDestroy(); + + #ifdef TD_TSZ + // compress destroy + tsCompressExit(); + #endif +} + +void dnodeReportStartup(Dnode *dnode, char *name, char *desc) { + SStartupStep *startup = &dnode->main->startup; + tstrncpy(startup->name, name, strlen(startup->name)); + tstrncpy(startup->desc, desc, strlen(startup->desc)); + startup->finished = 0; +} + +void dnodeReportStartupFinished(Dnode *dnode, char *name, char *desc) { + SStartupStep *startup = &dnode->main->startup; + tstrncpy(startup->name, name, strlen(startup->name)); + tstrncpy(startup->desc, desc, strlen(startup->desc)); + startup->finished = 1; +} + +void dnodeProcessStartupReq(Dnode *dnode, SRpcMsg *pMsg) { + dInfo("startup msg is received, cont:%s", (char *)pMsg->pCont); + + SStartupStep *pStep = rpcMallocCont(sizeof(SStartupStep)); + memcpy(pStep, &dnode->main->startup, sizeof(SStartupStep)); + + dDebug("startup msg is sent, step:%s desc:%s finished:%d", pStep->name, pStep->desc, pStep->finished); + + SRpcMsg rpcRsp = {.handle = pMsg->handle, .pCont = pStep, .contLen = sizeof(SStartupStep)}; + rpcSendResponse(&rpcRsp); + rpcFreeCont(pMsg->pCont); +} + +static int32_t dnodeStartMnode(Dnode *dnode, SRpcMsg *pMsg) { + SCreateMnodeMsg *pCfg = pMsg->pCont; + pCfg->dnodeId = htonl(pCfg->dnodeId); + if (pCfg->dnodeId != dnodeGetDnodeId(dnode->cfg)) { + dDebug("dnode:%d, in create meps msg is not equal with saved dnodeId:%d", pCfg->dnodeId, + dnodeGetDnodeId(dnode->cfg)); + return TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED; + } + + if (strcmp(pCfg->dnodeEp, tsLocalEp) != 0) { + dDebug("dnodeEp:%s, in create meps msg is not equal with saved dnodeEp:%s", pCfg->dnodeEp, tsLocalEp); + return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED; + } + + dDebug("dnode:%d, create meps msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.mnodeNum); + for (int32_t i = 0; i < pCfg->mnodes.mnodeNum; ++i) { + pCfg->mnodes.mnodeInfos[i].mnodeId = htonl(pCfg->mnodes.mnodeInfos[i].mnodeId); + dDebug("meps index:%d, meps:%d:%s", i, pCfg->mnodes.mnodeInfos[i].mnodeId, pCfg->mnodes.mnodeInfos[i].mnodeEp); + } + + if (mnodeIsServing(dnode->mnode)) return 0; + + return mnodeDeploy(dnode->mnode, &pCfg->mnodes); +} + +void dnodeProcessCreateMnodeReq(Dnode *dnode, SRpcMsg *pMsg) { + int32_t code = dnodeStartMnode(dnode, pMsg); + + SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0, .code = code}; + + rpcSendResponse(&rspMsg); + rpcFreeCont(pMsg->pCont); +} + +void dnodeProcessConfigDnodeReq(Dnode *dnode, SRpcMsg *pMsg) { + SCfgDnodeMsg *pCfg = pMsg->pCont; + + int32_t code = taosCfgDynamicOptions(pCfg->config); + + SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0, .code = code}; + + rpcSendResponse(&rspMsg); + rpcFreeCont(pMsg->pCont); +} \ No newline at end of file From d6976c0f38bb5c560d6ca45373cb609e526e2e97 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Sep 2021 19:40:13 +0800 Subject: [PATCH 23/44] [TD-10430] add dnode int for dnode module --- source/server/dnode/src/dnodeInt.c | 221 +++++++++++++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 source/server/dnode/src/dnodeInt.c diff --git a/source/server/dnode/src/dnodeInt.c b/source/server/dnode/src/dnodeInt.c new file mode 100644 index 0000000000..d77d81719b --- /dev/null +++ b/source/server/dnode/src/dnodeInt.c @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "qScript.h" +#include "tfile.h" +#include "tstep.h" +#include "tsync.h" +#include "twal.h" +#include "dnodeCfg.h" +#include "dnodeCheck.h" +#include "dnodeEps.h" +#include "dnodeMain.h" +#include "dnodeMnodeEps.h" +#include "dnodeStatus.h" +#include "dnodeTelemetry.h" +#include "dnodeTrans.h" +#include "mnode.h" +#include "vnode.h" + +static int32_t dnodeInitTfsEnv(Dnode *dnode, void **unUsed) { return tfInit(); } +static void dnodeCleanupTfsEnv(void **unUsed) { tfCleanup(); } +static int32_t dnodeInitRpcEnv(Dnode *dnode, void **unUsed) { return rpcInit(); } +static void dnodeCleanupRpcEnv(void **unUsed) { rpcCleanup(); } +static int32_t dnodeInitScriptEnv(Dnode *dnode, void **unUsed) { return scriptEnvPoolInit(); } +static void dnodeCleanupScriptEnv(void **unUsed) { scriptEnvPoolCleanup(); } +static int32_t dnodeInitWalEnv(Dnode *dnode, void **unUsed) { return walInit(); } +static void dnodeCleanupWalEnv(void **unUsed) { walCleanUp(); } +static int32_t dnodeInitSyncEnv(Dnode *dnode, void **unUsed) { return syncInit(); } +static void dnodeCleanupSyncEnv(void **unUsed) { syncCleanUp(); } + +static int32_t dnodeInitVnodeModule(Dnode *dnode, struct Vnode** out) { + SVnodePara para; + para.fp.GetDnodeEp = dnodeGetDnodeEp; + para.fp.SendMsgToDnode = dnodeSendMsgToDnode; + para.fp.SendMsgToMnode = dnodeSendMsgToMnode; + para.dnode = dnode; + + struct Vnode *vnode = vnodeCreateInstance(para); + if (vnode == NULL) { + return -1; + } + + *out = vnode; + return 0; +} + +static void dnodeCleanupVnodeModule(Dnode *dnode, struct Vnode **out) { + struct Vnode *vnode = *out; + *out = NULL; + vnodeCleanupInstance(vnode); +} + +static int32_t dnodeInitMnodeModule(Dnode *dnode, struct Mnode **out) { + SMnodePara para; + para.fp.GetDnodeEp = dnodeGetDnodeEp; + para.fp.SendMsgToDnode = dnodeSendMsgToDnode; + para.fp.SendMsgToMnode = dnodeSendMsgToMnode; + para.fp.SendRedirectMsg = dnodeSendRedirectMsg; + para.dnode = dnode; + para.dnodeId = dnode->cfg->dnodeId; + strncpy(para.clusterId, dnode->cfg->clusterId, sizeof(para.clusterId)); + + struct Mnode *mnode = mnodeCreateInstance(para); + if (mnode == NULL) { + return -1; + } + + *out = mnode; + return 0; +} + +static void dnodeCleanupMnodeModule(Dnode *dnode, struct Mnode **out) { + struct Mnode *mnode = *out; + *out = NULL; + mnodeCleanupInstance(mnode); +} + +Dnode *dnodeCreateInstance() { + Dnode *dnode = calloc(1, sizeof(Dnode)); + if (dnode == NULL) { + return NULL; + } + + SSteps *steps = taosStepInit(24); + if (steps == NULL) { + return NULL; + } + + SStepObj step = {0}; + step.parent = dnode; + + step.name = "dnode-main"; + step.self = (void **)&dnode->main; + step.initFp = (FnInitObj)dnodeInitMain; + step.cleanupFp = (FnCleanupObj)dnodeCleanupMain; + step.reportFp = NULL; + taosStepAdd(steps, &step); + + step.name = "dnode-storage"; + step.self = NULL; + step.initFp = (FnInitObj)dnodeInitStorage; + step.cleanupFp = (FnCleanupObj)dnodeCleanupStorage; + step.reportFp = (FnReportProgress)dnodeReportStartup; + taosStepAdd(steps, &step); + + step.name = "dnode-tfs-env"; + step.self = NULL; + step.initFp = (FnInitObj)dnodeInitTfsEnv; + step.cleanupFp = (FnCleanupObj)dnodeCleanupTfsEnv; + taosStepAdd(steps, &step); + + step.name = "dnode-rpc-env"; + step.self = NULL; + step.initFp = (FnInitObj)dnodeInitRpcEnv; + step.cleanupFp = (FnCleanupObj)dnodeCleanupRpcEnv; + taosStepAdd(steps, &step); + + step.name = "dnode-check"; + step.self = (void **)&dnode->check; + step.initFp = (FnInitObj)dnodeInitCheck; + step.cleanupFp = (FnCleanupObj)dnodeCleanupCheck; + taosStepAdd(steps, &step); + + step.name = "dnode-cfg"; + step.self = (void **)&dnode->cfg; + step.initFp = (FnInitObj)dnodeInitCfg; + step.cleanupFp = (FnCleanupObj)dnodeCleanupCfg; + taosStepAdd(steps, &step); + + step.name = "dnode-deps"; + step.self = (void **)&dnode->eps; + step.initFp = (FnInitObj)dnodeInitEps; + step.cleanupFp = (FnCleanupObj)dnodeCleanupEps; + taosStepAdd(steps, &step); + + step.name = "dnode-meps"; + step.self = (void **)&dnode->meps; + step.initFp = (FnInitObj)dnodeInitMnodeEps; + step.cleanupFp = (FnCleanupObj)dnodeCleanupMnodeEps; + taosStepAdd(steps, &step); + + step.name = "dnode-wal"; + step.self = NULL; + step.initFp = (FnInitObj)dnodeInitWalEnv; + step.cleanupFp = (FnCleanupObj)dnodeCleanupWalEnv; + taosStepAdd(steps, &step); + + step.name = "dnode-sync"; + step.self = NULL; + step.initFp = (FnInitObj)dnodeInitSyncEnv; + step.cleanupFp = (FnCleanupObj)dnodeCleanupSyncEnv; + taosStepAdd(steps, &step); + + step.name = "dnode-vnode"; + step.self = (void **)&dnode->vnode; + step.initFp = (FnInitObj)dnodeInitVnodeModule; + step.cleanupFp = (FnCleanupObj)dnodeCleanupVnodeModule; + taosStepAdd(steps, &step); + + step.name = "dnode-mnode"; + step.self = (void **)&dnode->mnode; + step.initFp = (FnInitObj)dnodeInitMnodeModule; + step.cleanupFp = (FnCleanupObj)dnodeCleanupMnodeModule; + taosStepAdd(steps, &step); + + step.name = "dnode-trans"; + step.self = (void **)&dnode->trans; + step.initFp = (FnInitObj)dnodeInitTrans; + step.cleanupFp = (FnCleanupObj)dnodeCleanupTrans; + taosStepAdd(steps, &step); + + step.name = "dnode-status"; + step.self = (void **)&dnode->status; + step.initFp = (FnInitObj)dnodeInitStatus; + step.cleanupFp = (FnCleanupObj)dnodeCleanupStatus; + taosStepAdd(steps, &step); + + step.name = "dnode-telem"; + step.self = (void **)&dnode->telem; + step.initFp = (FnInitObj)dnodeInitTelemetry; + step.cleanupFp = (FnCleanupObj)dnodeCleanupTelemetry; + taosStepAdd(steps, &step); + + step.name = "dnode-script"; + step.self = NULL; + step.initFp = (FnInitObj)dnodeInitScriptEnv; + step.cleanupFp = (FnCleanupObj)dnodeCleanupScriptEnv; + taosStepAdd(steps, &step); + + dnode->steps = steps; + taosStepExec(dnode->steps); + + if (dnode->main) { + dnode->main->runStatus = TD_RUN_STAT_RUNNING; + dnodeReportStartupFinished(dnode, "TDengine", "initialized successfully"); + dInfo("TDengine is initialized successfully"); + } + + return dnode; +} + +void dnodeCleanupInstance(Dnode *dnode) { + if (dnode->main->runStatus != TD_RUN_STAT_STOPPED) { + dnode->main->runStatus = TD_RUN_STAT_STOPPED; + taosStepCleanup(dnode->steps); + } +} From 6bf279b60a7a60804564a4c25c597c316c1489c2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Sep 2021 19:40:40 +0800 Subject: [PATCH 24/44] [TD-10430] minor changes --- include/server/vnode/vnode.h | 16 ++++++++-------- source/server/dnode/CMakeLists.txt | 5 +++++ source/server/dnode/inc/dnodeCfg.h | 2 +- source/server/dnode/inc/dnodeCheck.h | 2 +- source/server/dnode/inc/dnodeEps.h | 2 +- source/server/dnode/src/dnodeCfg.c | 2 +- source/server/dnode/src/dnodeCheck.c | 2 +- source/server/dnode/src/dnodeEps.c | 2 +- 8 files changed, 19 insertions(+), 14 deletions(-) diff --git a/include/server/vnode/vnode.h b/include/server/vnode/vnode.h index 748b93c8eb..8aebd5aa9b 100644 --- a/include/server/vnode/vnode.h +++ b/include/server/vnode/vnode.h @@ -86,14 +86,6 @@ typedef struct { */ int32_t vnodeGetStatistics(struct Vnode *vnode, SVnodeStat *stat); -/** - * Interface for processing messages. - * - * @param vnode, instance of vnode module. - * @param msg, message to be processed. - */ -void vnodeProcessMsg(struct Vnode *vnode, SRpcMsg *msg); - /** * Get the status of all vnodes. * @@ -111,6 +103,14 @@ void vnodeGetStatus(struct Vnode *vnode, struct SStatusMsg *status); */ void vnodeSetAccess(struct Vnode *vnode, struct SVgroupAccess *access, int32_t numOfVnodes); +/** + * Interface for processing messages. + * + * @param vnode, instance of vnode module. + * @param msg, message to be processed. + */ +void vnodeProcessMsg(struct Vnode *vnode, SRpcMsg *msg); + #ifdef __cplusplus } #endif diff --git a/source/server/dnode/CMakeLists.txt b/source/server/dnode/CMakeLists.txt index db80bf727d..dbb4bd7c99 100644 --- a/source/server/dnode/CMakeLists.txt +++ b/source/server/dnode/CMakeLists.txt @@ -1,5 +1,10 @@ aux_source_directory(src DNODE_SRC) add_library(dnode ${DNODE_SRC}) +target_link_libraries( + dnode + PUBLIC os + PUBLIC cJson +) target_include_directories( dnode PUBLIC "${CMAKE_SOURCE_DIR}/include/server/dnode" diff --git a/source/server/dnode/inc/dnodeCfg.h b/source/server/dnode/inc/dnodeCfg.h index ec9cc397e1..2284486e51 100644 --- a/source/server/dnode/inc/dnodeCfg.h +++ b/source/server/dnode/inc/dnodeCfg.h @@ -31,7 +31,7 @@ typedef struct DnCfg { } DnCfg; int32_t dnodeInitCfg(Dnode *dnode, DnCfg **cfg); -void dnodeCleanupCfg(Dnode *dnode, DnCfg **cfg); +void dnodeCleanupCfg(DnCfg **cfg); void dnodeUpdateCfg(DnCfg *cfg, SDnodeCfg *data); int32_t dnodeGetDnodeId(DnCfg *cfg); void dnodeGetClusterId(DnCfg *cfg, char *clusterId); diff --git a/source/server/dnode/inc/dnodeCheck.h b/source/server/dnode/inc/dnodeCheck.h index b8d6d2ae56..84232fbc1d 100644 --- a/source/server/dnode/inc/dnodeCheck.h +++ b/source/server/dnode/inc/dnodeCheck.h @@ -26,7 +26,7 @@ typedef struct DnCheck { } DnCheck; int32_t dnodeInitCheck(Dnode *dnode, DnCheck **check); -void dnodeCleanupCheck(Dnode *dnode, DnCheck **check); +void dnodeCleanupCheck(DnCheck **check); #ifdef __cplusplus } diff --git a/source/server/dnode/inc/dnodeEps.h b/source/server/dnode/inc/dnodeEps.h index fa7e99d113..aec412ee59 100644 --- a/source/server/dnode/inc/dnodeEps.h +++ b/source/server/dnode/inc/dnodeEps.h @@ -33,7 +33,7 @@ typedef struct DnEps { } DnEps; int32_t dnodeInitEps(Dnode *dnode, DnEps **eps); -void dnodeCleanupEps(Dnode *dnode, DnEps **eps); +void dnodeCleanupEps(DnEps **eps); void dnodeUpdateEps(DnEps *eps, SDnodeEps *data); bool dnodeIsDnodeEpChanged(DnEps *eps, int32_t dnodeId, char *epstr); void dnodeGetDnodeEp(Dnode *dnode, int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port); diff --git a/source/server/dnode/src/dnodeCfg.c b/source/server/dnode/src/dnodeCfg.c index f4e26fd721..a004ad7290 100644 --- a/source/server/dnode/src/dnodeCfg.c +++ b/source/server/dnode/src/dnodeCfg.c @@ -128,7 +128,7 @@ int32_t dnodeInitCfg(Dnode *dnode, DnCfg **out) { return ret; } -void dnodeCleanupCfg(Dnode *dnode, DnCfg **out) { +void dnodeCleanupCfg(DnCfg **out) { DnCfg* cfg = *out; *out = NULL; diff --git a/source/server/dnode/src/dnodeCheck.c b/source/server/dnode/src/dnodeCheck.c index 6a7f18f63d..402ef0d333 100644 --- a/source/server/dnode/src/dnodeCheck.c +++ b/source/server/dnode/src/dnodeCheck.c @@ -194,7 +194,7 @@ int32_t dnodeInitCheck(Dnode *dnode, DnCheck **out) { return 0; } -void dnodeCleanupCheck(Dnode *dnode, DnCheck **out) { +void dnodeCleanupCheck(DnCheck **out) { DnCheck *check = *out; *out = NULL; diff --git a/source/server/dnode/src/dnodeEps.c b/source/server/dnode/src/dnodeEps.c index c178c01c9f..725e54bb91 100644 --- a/source/server/dnode/src/dnodeEps.c +++ b/source/server/dnode/src/dnodeEps.c @@ -204,7 +204,7 @@ int32_t dnodeInitEps(Dnode *dnode, DnEps **out) { return ret; } -void dnodeCleanupEps(Dnode *dnode, DnEps **out) { +void dnodeCleanupEps(DnEps **out) { DnEps *eps = *out; *out = NULL; From 1ff3135d2089d9eb1933e590a700c070445f80de Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 30 Sep 2021 10:22:59 +0800 Subject: [PATCH 25/44] [TD-10430] minor changes --- deps/CMakeLists.txt | 4 ++++ source/server/dnode/CMakeLists.txt | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index 0e08be44ac..13f25fa112 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -3,6 +3,10 @@ if(${BUILD_TEST}) endif(${BUILD_TEST}) add_subdirectory(cJson) +target_include_directories( + cJSON + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/cJson +) add_subdirectory(lz4/build/cmake) add_subdirectory(zlib) target_include_directories( diff --git a/source/server/dnode/CMakeLists.txt b/source/server/dnode/CMakeLists.txt index dbb4bd7c99..d93873282e 100644 --- a/source/server/dnode/CMakeLists.txt +++ b/source/server/dnode/CMakeLists.txt @@ -3,7 +3,7 @@ add_library(dnode ${DNODE_SRC}) target_link_libraries( dnode PUBLIC os - PUBLIC cJson + PUBLIC cJSON ) target_include_directories( dnode From 4ac2bb927c3e388a92269a4cd23451e685adaae8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 30 Sep 2021 10:33:08 +0800 Subject: [PATCH 26/44] [TD-10430] cjson include dir --- deps/CMakeLists.txt | 2 +- source/server/dnode/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index 13f25fa112..f251ae34ed 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -4,7 +4,7 @@ endif(${BUILD_TEST}) add_subdirectory(cJson) target_include_directories( - cJSON + cjson PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/cJson ) add_subdirectory(lz4/build/cmake) diff --git a/source/server/dnode/CMakeLists.txt b/source/server/dnode/CMakeLists.txt index d93873282e..aea4e9e144 100644 --- a/source/server/dnode/CMakeLists.txt +++ b/source/server/dnode/CMakeLists.txt @@ -3,7 +3,7 @@ add_library(dnode ${DNODE_SRC}) target_link_libraries( dnode PUBLIC os - PUBLIC cJSON + PUBLIC cjson ) target_include_directories( dnode From 4e306a79cdc88a6f30c251add13a68ffc4ed44c2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 30 Sep 2021 13:37:52 +0800 Subject: [PATCH 27/44] use https for deps download --- cmake/cjson_CMakeLists.txt.in | 2 +- cmake/gtest_CMakeLists.txt.in | 2 +- cmake/lz4_CMakeLists.txt.in | 2 +- cmake/zlib_CMakeLists.txt.in | 2 +- source/server/vnode/src/vnodeWrite.c | 2 ++ 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cmake/cjson_CMakeLists.txt.in b/cmake/cjson_CMakeLists.txt.in index 9995d1f8d8..6c285d1a7e 100644 --- a/cmake/cjson_CMakeLists.txt.in +++ b/cmake/cjson_CMakeLists.txt.in @@ -1,7 +1,7 @@ # cjson ExternalProject_Add(cjson - GIT_REPOSITORY git@github.com:taosdata-contrib/cJSON.git + GIT_REPOSITORY https://github.com/taosdata-contrib/cJSON.git GIT_TAG v1.7.15 SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/cJson" BINARY_DIR "" diff --git a/cmake/gtest_CMakeLists.txt.in b/cmake/gtest_CMakeLists.txt.in index 655870e0c9..e54806ae3b 100644 --- a/cmake/gtest_CMakeLists.txt.in +++ b/cmake/gtest_CMakeLists.txt.in @@ -1,7 +1,7 @@ # googletest ExternalProject_Add(googletest - GIT_REPOSITORY git@github.com:taosdata-contrib/googletest.git + GIT_REPOSITORY https://github.com/taosdata-contrib/googletest.git GIT_TAG release-1.11.0 SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/googletest" BINARY_DIR "" diff --git a/cmake/lz4_CMakeLists.txt.in b/cmake/lz4_CMakeLists.txt.in index 5dba4f2ab1..4b25d48e86 100644 --- a/cmake/lz4_CMakeLists.txt.in +++ b/cmake/lz4_CMakeLists.txt.in @@ -1,7 +1,7 @@ # lz4 ExternalProject_Add(lz4 - GIT_REPOSITORY git@github.com:taosdata-contrib/lz4.git + GIT_REPOSITORY https://github.com/taosdata-contrib/lz4.git GIT_TAG v1.9.3 SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/lz4" BINARY_DIR "" diff --git a/cmake/zlib_CMakeLists.txt.in b/cmake/zlib_CMakeLists.txt.in index c997fc7983..2176486519 100644 --- a/cmake/zlib_CMakeLists.txt.in +++ b/cmake/zlib_CMakeLists.txt.in @@ -1,7 +1,7 @@ # zlib ExternalProject_Add(zlib - GIT_REPOSITORY git@github.com:taosdata-contrib/zlib.git + GIT_REPOSITORY https://github.com/taosdata-contrib/zlib.git GIT_TAG v1.2.11 SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/zlib" BINARY_DIR "" diff --git a/source/server/vnode/src/vnodeWrite.c b/source/server/vnode/src/vnodeWrite.c index a3aff78d02..d4ef4ab337 100644 --- a/source/server/vnode/src/vnodeWrite.c +++ b/source/server/vnode/src/vnodeWrite.c @@ -18,6 +18,7 @@ int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp) { // TODO: Check inputs +#if 0 void *pMem = NULL; if ((pMem = amalloc(pVnode->allocator, REQ_SIZE(pReq))) == NULL) { // No more memory to allocate, schedule an async commit @@ -45,6 +46,7 @@ int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp) { if (tsdbInsert(pVnode->pTsdb, (SSubmitReq *)pMem) < 0) { // TODO: handler error } +#endif return 0; } From 0d36837ee5e76ddc14025412fb4dbf9ae035d152 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 30 Sep 2021 13:40:57 +0800 Subject: [PATCH 28/44] [TD-10430] compile error while add dnode module --- {src/inc => include/common}/taosdef.h | 2 + {src/inc => include/common}/taoserror.h | 0 {src/inc => include/common}/taosmsg.h | 35 ++- .../inc => include/common}/tdataformat.h | 86 ++++++- {src/common/inc => include/common}/tglobal.h | 4 - {src/inc => include/common}/ttype.h | 0 {src/inc => include/libs/transport}/trpc.h | 7 +- include/os/os.h | 19 +- include/os/osDef.h | 27 ++ {src/os/inc => include/os}/osDir.h | 0 {src/os/inc => include/os}/osFile.h | 0 {src/os/inc => include/os}/osMath.h | 6 +- {src/os/inc => include/os}/osSemaphore.h | 0 {src/os/inc => include/os}/osSocket.h | 2 + {src/os/inc => include/os}/osString.h | 0 include/os/osTime.h | 56 +++++ {src/util/inc => include/util}/hash.h | 0 {src/util/inc => include/util}/hashfunc.h | 0 {src/util/inc => include/util}/tbuffer.h | 0 {src/util/inc => include/util}/tcache.h | 0 {src/util/inc => include/util}/tconfig.h | 1 + {src/util/inc => include/util}/tcrc32c.h | 0 {src/util/inc => include/util}/tmd5.h | 0 {src/util/inc => include/util}/tnote.h | 0 .../util/inc => include/util}/tscompression.h | 0 {src/util/inc => include/util}/tsocket.h | 0 include/util/tstep.h | 51 ++++ {src/util/inc => include/util}/ttimer.h | 0 {src/util/inc => include/util}/tutil.h | 0 source/server/dnode/CMakeLists.txt | 5 + source/server/dnode/src/dnodeCheck.c | 6 +- source/server/dnode/src/dnodeInt.c | 16 +- source/server/dnode/src/dnodeMain.c | 18 +- source/server/dnode/src/dnodeTelemetry.c | 2 +- src/inc/dnode.h | 85 ------- src/inc/mnode.h | 83 ------ src/os/inc/osDef.h | 236 ------------------ src/os/inc/osTime.h | 111 -------- 38 files changed, 310 insertions(+), 548 deletions(-) rename {src/inc => include/common}/taosdef.h (99%) rename {src/inc => include/common}/taoserror.h (100%) rename {src/inc => include/common}/taosmsg.h (97%) rename {src/common/inc => include/common}/tdataformat.h (90%) rename {src/common/inc => include/common}/tglobal.h (98%) rename {src/inc => include/common}/ttype.h (100%) rename {src/inc => include/libs/transport}/trpc.h (92%) rename {src/os/inc => include/os}/osDir.h (100%) rename {src/os/inc => include/os}/osFile.h (100%) rename {src/os/inc => include/os}/osMath.h (95%) rename {src/os/inc => include/os}/osSemaphore.h (100%) rename {src/os/inc => include/os}/osSocket.h (99%) rename {src/os/inc => include/os}/osString.h (100%) create mode 100644 include/os/osTime.h rename {src/util/inc => include/util}/hash.h (100%) rename {src/util/inc => include/util}/hashfunc.h (100%) rename {src/util/inc => include/util}/tbuffer.h (100%) rename {src/util/inc => include/util}/tcache.h (100%) rename {src/util/inc => include/util}/tconfig.h (95%) rename {src/util/inc => include/util}/tcrc32c.h (100%) rename {src/util/inc => include/util}/tmd5.h (100%) rename {src/util/inc => include/util}/tnote.h (100%) rename {src/util/inc => include/util}/tscompression.h (100%) rename {src/util/inc => include/util}/tsocket.h (100%) create mode 100644 include/util/tstep.h rename {src/util/inc => include/util}/ttimer.h (100%) rename {src/util/inc => include/util}/tutil.h (100%) delete mode 100644 src/inc/dnode.h delete mode 100644 src/inc/mnode.h delete mode 100644 src/os/inc/osDef.h delete mode 100644 src/os/inc/osTime.h diff --git a/src/inc/taosdef.h b/include/common/taosdef.h similarity index 99% rename from src/inc/taosdef.h rename to include/common/taosdef.h index 89de733ba1..d80caad88d 100644 --- a/src/inc/taosdef.h +++ b/include/common/taosdef.h @@ -84,6 +84,8 @@ extern const int32_t TYPE_BYTES[15]; #define TSDB_DEFAULT_PASS "powerdb" #elif (_TD_TQ_ == true) #define TSDB_DEFAULT_PASS "tqueue" +#elif (_TD_PRO_ == true) +#define TSDB_DEFAULT_PASS "prodb" #else #define TSDB_DEFAULT_PASS "taosdata" #endif diff --git a/src/inc/taoserror.h b/include/common/taoserror.h similarity index 100% rename from src/inc/taoserror.h rename to include/common/taoserror.h diff --git a/src/inc/taosmsg.h b/include/common/taosmsg.h similarity index 97% rename from src/inc/taosmsg.h rename to include/common/taosmsg.h index bb93c52142..76743f10a4 100644 --- a/src/inc/taosmsg.h +++ b/include/common/taosmsg.h @@ -25,7 +25,6 @@ extern "C" { #include "taosdef.h" #include "taoserror.h" -#include "trpc.h" #include "tdataformat.h" // message type @@ -339,6 +338,13 @@ typedef struct { int32_t pid; } SConnectMsg; +typedef struct SEpSet { + int8_t inUse; + int8_t numOfEps; + uint16_t port[TSDB_MAX_REPLICA]; + char fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN]; +} SEpSet; + typedef struct { char acctId[TSDB_ACCT_ID_LEN]; char serverVersion[TSDB_VERSION_LEN]; @@ -348,7 +354,7 @@ typedef struct { int8_t reserved1; int8_t reserved2; int32_t connId; - SRpcEpSet epSet; + SEpSet epSet; } SConnectRsp; typedef struct { @@ -465,6 +471,21 @@ typedef struct { int32_t tsOrder; // ts comp block order } STsBufInfo; +typedef struct SInterval { + int32_t tz; // query client timezone + char intervalUnit; + char slidingUnit; + char offsetUnit; + int64_t interval; + int64_t sliding; + int64_t offset; +} SInterval; + +typedef struct SSessionWindow { + int64_t gap; // gap between two session window(in microseconds) + int32_t primaryColId; // primary timestamp column +} SSessionWindow; + typedef struct { SMsgHead head; char version[TSDB_VERSION_LEN]; @@ -492,7 +513,6 @@ typedef struct { SSessionWindow sw; // session window uint16_t tagCondLen; // tag length in current query uint16_t colCondLen; // column length in current query - uint32_t tbnameCondLen; // table name filter condition string length int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; int16_t orderType; // used in group by xx order by xxx @@ -502,7 +522,6 @@ typedef struct { int64_t offset; uint32_t queryType; // denote another query process int16_t numOfOutput; // final output columns numbers - int16_t tagNameRelType; // relation of tag criteria and tbname criteria int16_t fillType; // interpolate type uint64_t fillVal; // default value array list int32_t secondStageOutput; @@ -631,7 +650,7 @@ typedef struct { char reserved[64]; } SVnodeStatisticInfo; -typedef struct { +typedef struct SVgroupAccess { int32_t vgId; int8_t accessState; } SVgroupAccess; @@ -660,7 +679,7 @@ typedef struct { char mnodeEp[TSDB_EP_LEN]; } SMInfo; -typedef struct { +typedef struct SMInfos { int8_t inUse; int8_t mnodeNum; SMInfo mnodeInfos[TSDB_MAX_REPLICA]; @@ -686,7 +705,7 @@ typedef struct { int8_t reserved[4]; } SClusterCfg; -typedef struct { +typedef struct SStatusMsg { uint32_t version; int32_t dnodeId; char dnodeEp[TSDB_EP_LEN]; @@ -904,7 +923,7 @@ typedef struct { uint32_t onlineDnodes; uint32_t connId; int8_t killConnection; - SRpcEpSet epSet; + SEpSet epSet; } SHeartBeatRsp; typedef struct { diff --git a/src/common/inc/tdataformat.h b/include/common/tdataformat.h similarity index 90% rename from src/common/inc/tdataformat.h rename to include/common/tdataformat.h index cf2b90fd9e..ab7baea2d7 100644 --- a/src/common/inc/tdataformat.h +++ b/include/common/tdataformat.h @@ -51,6 +51,78 @@ extern "C" { memcpy(varDataVal(x), (str), (_size)); \ } while (0); +// ----------------- TSDB COLUMN DEFINITION +typedef struct { + int8_t type; // Column type + int16_t colId; // column ID + int16_t bytes; // column bytes (restore to int16_t in case of misuse) + uint16_t offset; // point offset in SDataRow after the header part. +} STColumn; + +#define colType(col) ((col)->type) +#define colColId(col) ((col)->colId) +#define colBytes(col) ((col)->bytes) +#define colOffset(col) ((col)->offset) + +#define colSetType(col, t) (colType(col) = (t)) +#define colSetColId(col, id) (colColId(col) = (id)) +#define colSetBytes(col, b) (colBytes(col) = (b)) +#define colSetOffset(col, o) (colOffset(col) = (o)) + +// ----------------- TSDB SCHEMA DEFINITION +typedef struct { + int version; // version + int numOfCols; // Number of columns appended + int tlen; // maximum length of a SDataRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + (bytes)) + uint16_t flen; // First part length in a SDataRow after the header part + uint16_t vlen; // pure value part length, excluded the overhead (bytes only) + STColumn columns[]; +} STSchema; + +#define schemaNCols(s) ((s)->numOfCols) +#define schemaVersion(s) ((s)->version) +#define schemaTLen(s) ((s)->tlen) +#define schemaFLen(s) ((s)->flen) +#define schemaVLen(s) ((s)->vlen) +#define schemaColAt(s, i) ((s)->columns + i) +#define tdFreeSchema(s) tfree((s)) + +STSchema *tdDupSchema(STSchema *pSchema); +int tdEncodeSchema(void **buf, STSchema *pSchema); +void * tdDecodeSchema(void *buf, STSchema **pRSchema); + +static FORCE_INLINE int comparColId(const void *key1, const void *key2) { + if (*(int16_t *)key1 > ((STColumn *)key2)->colId) { + return 1; + } else if (*(int16_t *)key1 < ((STColumn *)key2)->colId) { + return -1; + } else { + return 0; + } +} + +static FORCE_INLINE STColumn *tdGetColOfID(STSchema *pSchema, int16_t colId) { + void *ptr = bsearch(&colId, (void *)pSchema->columns, schemaNCols(pSchema), sizeof(STColumn), comparColId); + if (ptr == NULL) return NULL; + return (STColumn *)ptr; +} + +// ----------------- SCHEMA BUILDER DEFINITION +typedef struct { + int tCols; + int nCols; + int tlen; + uint16_t flen; + uint16_t vlen; + int version; + STColumn *columns; +} STSchemaBuilder; + +int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); +void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); +void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); +int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes); +STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); // ----------------- Semantic timestamp key definition typedef uint64_t TKEY; @@ -134,7 +206,7 @@ SDataRow tdDataRowDup(SDataRow row); // offset here not include dataRow header length static FORCE_INLINE int tdAppendDataColVal(SDataRow row, const void *value, bool isCopyVarData, int8_t type, int32_t offset) { - ASSERT(value != NULL); + assert(value != NULL); int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE; if (IS_VAR_DATA_TYPE(type)) { @@ -145,7 +217,7 @@ static FORCE_INLINE int tdAppendDataColVal(SDataRow row, const void *value, bool dataRowLen(row) += varDataTLen(value); } else { if (offset == 0) { - ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP); + assert(type == TSDB_DATA_TYPE_TIMESTAMP); TKEY tvalue = tdGetTKEY(*(TSKEY *)value); memcpy(POINTER_SHIFT(row, toffset), (const void *)(&tvalue), TYPE_BYTES[type]); } else { @@ -199,7 +271,7 @@ static FORCE_INLINE void tdSetColOfRowNullBySchema(SDataRow row, STSchema *pSche static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSchema, int dstIdx, SDataRow src, STSchema *pSrcSchema, int srcIdx) { int8_t type = pDstSchema->columns[dstIdx].type; - ASSERT(type == pSrcSchema->columns[srcIdx].type); + assert(type == pSrcSchema->columns[srcIdx].type); void *pData = tdGetPtrToCol(dst, pDstSchema, dstIdx); void *value = tdGetPtrToCol(src, pSrcSchema, srcIdx); @@ -285,7 +357,7 @@ static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) { } static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) { - ASSERT(rows > 0); + assert(rows > 0); if (IS_VAR_DATA_TYPE(pDataCol->type)) { return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1)); @@ -315,7 +387,7 @@ static FORCE_INLINE TKEY dataColsTKeyFirst(SDataCols *pCols) { } static FORCE_INLINE TSKEY dataColsKeyAtRow(SDataCols *pCols, int row) { - ASSERT(row < pCols->numOfRows); + assert(row < pCols->numOfRows); return dataColsKeyAt(pCols, row); } @@ -413,7 +485,7 @@ static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) { // offset here not include kvRow header length static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, bool isCopyValData, int16_t colId, int8_t type, int32_t offset) { - ASSERT(value != NULL); + assert(value != NULL); int32_t toffset = offset + TD_KV_ROW_HEAD_SIZE; SColIdx *pColIdx = (SColIdx *)POINTER_SHIFT(row, toffset); char * ptr = (char *)POINTER_SHIFT(row, kvRowLen(row)); @@ -428,7 +500,7 @@ static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, bool isC kvRowLen(row) += varDataTLen(value); } else { if (offset == 0) { - ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP); + assert(type == TSDB_DATA_TYPE_TIMESTAMP); TKEY tvalue = tdGetTKEY(*(TSKEY *)value); memcpy(ptr, (void *)(&tvalue), TYPE_BYTES[type]); } else { diff --git a/src/common/inc/tglobal.h b/include/common/tglobal.h similarity index 98% rename from src/common/inc/tglobal.h rename to include/common/tglobal.h index 604ce89432..a5f8005e9c 100644 --- a/src/common/inc/tglobal.h +++ b/include/common/tglobal.h @@ -36,12 +36,9 @@ extern int32_t tsStatusInterval; extern int32_t tsNumOfMnodes; extern int8_t tsEnableVnodeBak; extern int8_t tsEnableTelemetryReporting; -extern char tsEmail[]; extern char tsArbitrator[]; extern int8_t tsArbOnline; extern int64_t tsArbOnlineTimestamp; -extern int32_t tsDnodeId; -extern int64_t tsDnodeStartTime; // common extern int tsRpcTimer; @@ -152,7 +149,6 @@ extern int32_t tsMonitorInterval; extern int8_t tsEnableStream; // internal -extern int8_t tsCompactMnodeWal; extern int8_t tsPrintAuth; extern int8_t tscEmbedded; extern char configDir[]; diff --git a/src/inc/ttype.h b/include/common/ttype.h similarity index 100% rename from src/inc/ttype.h rename to include/common/ttype.h diff --git a/src/inc/trpc.h b/include/libs/transport/trpc.h similarity index 92% rename from src/inc/trpc.h rename to include/libs/transport/trpc.h index 0ce2e3da14..2a0efab213 100644 --- a/src/inc/trpc.h +++ b/include/libs/transport/trpc.h @@ -64,6 +64,9 @@ typedef struct SRpcInit { int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS int idleTime; // milliseconds, 0 means idle timer is disabled + // owner of the rpc client/server, + void *owner; // set by the app when rpc init + // the following is for client app ecurity only char *user; // user name char spi; // security parameter index @@ -72,10 +75,10 @@ typedef struct SRpcInit { char *ckey; // ciphering key // call back to process incoming msg, code shall be ignored by server app - void (*cfp)(SRpcMsg *, SRpcEpSet *); + void (*cfp)(void *owner, SRpcMsg *, SRpcEpSet *); // call back to retrieve the client auth info, for server app only - int (*afp)(char *tableId, char *spi, char *encrypt, char *secret, char *ckey); + int (*afp)(void *owner, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); } SRpcInit; int32_t rpcInit(); diff --git a/include/os/os.h b/include/os/os.h index 259aef8143..c89dc65211 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -21,17 +21,34 @@ extern "C" { #endif #include +#include +#include +#include #include #include #include #include #include -#include + +#include +#include +#include +#include +#include +#include +#include #include "osAtomic.h" #include "osDef.h" +#include "osDir.h" #include "osEndian.h" +#include "osFile.h" +#include "osMath.h" #include "osMemory.h" +#include "osSemaphore.h" +#include "osSocket.h" +#include "osString.h" +#include "osTime.h" #ifdef __cplusplus } diff --git a/include/os/osDef.h b/include/os/osDef.h index a68e50e3b5..7de8fd2f26 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -26,6 +26,33 @@ extern "C" { #define FORCE_INLINE #endif +#define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b))) +#define POINTER_DISTANCE(p1, p2) ((char *)(p1) - (char *)(p2)) + +#ifndef PATH_MAX + #define PATH_MAX 1024 +#endif + + +#if defined(_TD_LINUX_64) || defined(_TD_LINUX_32) || defined(_TD_MIPS_64) || defined(_TD_ARM_32) || defined(_TD_ARM_64) || defined(_TD_DARWIN_64) + #if defined(_TD_DARWIN_64) + // MacOS + #if !defined(_GNU_SOURCE) + #define setThreadName(name) do { pthread_setname_np((name)); } while (0) + #else + // pthread_setname_np not defined + #define setThreadName(name) + #endif + #else + // Linux, length of name must <= 16 (the last '\0' included) + #define setThreadName(name) do { prctl(PR_SET_NAME, (name)); } while (0) + #endif +#else + // Windows + #define setThreadName(name) +#endif + + #ifdef __cplusplus } #endif diff --git a/src/os/inc/osDir.h b/include/os/osDir.h similarity index 100% rename from src/os/inc/osDir.h rename to include/os/osDir.h diff --git a/src/os/inc/osFile.h b/include/os/osFile.h similarity index 100% rename from src/os/inc/osFile.h rename to include/os/osFile.h diff --git a/src/os/inc/osMath.h b/include/os/osMath.h similarity index 95% rename from src/os/inc/osMath.h rename to include/os/osMath.h index 65e8a1f6fe..440d3bc787 100644 --- a/src/os/inc/osMath.h +++ b/include/os/osMath.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_OS_MATH_H -#define TDENGINE_OS_MATH_H +#ifndef _TD_OS_MATH_H_ +#define _TD_OS_MATH_H_ #ifdef __cplusplus extern "C" { @@ -62,4 +62,4 @@ extern "C" { } #endif -#endif +#endif /*_TD_OS_MATH_H_*/ diff --git a/src/os/inc/osSemaphore.h b/include/os/osSemaphore.h similarity index 100% rename from src/os/inc/osSemaphore.h rename to include/os/osSemaphore.h diff --git a/src/os/inc/osSocket.h b/include/os/osSocket.h similarity index 99% rename from src/os/inc/osSocket.h rename to include/os/osSocket.h index a172c0bf34..29653f1269 100644 --- a/src/os/inc/osSocket.h +++ b/include/os/osSocket.h @@ -88,7 +88,9 @@ int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *op int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t* optlen); uint32_t taosInetAddr(char *ipAddr); +#if 0 const char *taosInetNtoa(struct in_addr ipInt); +#endif #if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) #define htobe64 htonll diff --git a/src/os/inc/osString.h b/include/os/osString.h similarity index 100% rename from src/os/inc/osString.h rename to include/os/osString.h diff --git a/include/os/osTime.h b/include/os/osTime.h new file mode 100644 index 0000000000..81163b29e7 --- /dev/null +++ b/include/os/osTime.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_TIME_H +#define TDENGINE_OS_TIME_H + +#ifdef __cplusplus +extern "C" { +#endif + +//@return timestamp in second +int32_t taosGetTimestampSec(); + +//@return timestamp in millisecond +static FORCE_INLINE int64_t taosGetTimestampMs() { + struct timeval systemTime; + gettimeofday(&systemTime, NULL); + return (int64_t)systemTime.tv_sec * 1000L + (int64_t)systemTime.tv_usec / 1000; +} + +//@return timestamp in microsecond +static FORCE_INLINE int64_t taosGetTimestampUs() { + struct timeval systemTime; + gettimeofday(&systemTime, NULL); + return (int64_t)systemTime.tv_sec * 1000000L + (int64_t)systemTime.tv_usec; +} + +//@return timestamp in nanosecond +static FORCE_INLINE int64_t taosGetTimestampNs() { + struct timespec systemTime = {0}; + clock_gettime(CLOCK_REALTIME, &systemTime); + return (int64_t)systemTime.tv_sec * 1000000000L + (int64_t)systemTime.tv_nsec; +} + +int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth); +void deltaToUtcInitOnce(); + +int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TTIME_H diff --git a/src/util/inc/hash.h b/include/util/hash.h similarity index 100% rename from src/util/inc/hash.h rename to include/util/hash.h diff --git a/src/util/inc/hashfunc.h b/include/util/hashfunc.h similarity index 100% rename from src/util/inc/hashfunc.h rename to include/util/hashfunc.h diff --git a/src/util/inc/tbuffer.h b/include/util/tbuffer.h similarity index 100% rename from src/util/inc/tbuffer.h rename to include/util/tbuffer.h diff --git a/src/util/inc/tcache.h b/include/util/tcache.h similarity index 100% rename from src/util/inc/tcache.h rename to include/util/tcache.h diff --git a/src/util/inc/tconfig.h b/include/util/tconfig.h similarity index 95% rename from src/util/inc/tconfig.h rename to include/util/tconfig.h index cf8977ce06..2c632d4a17 100644 --- a/src/util/inc/tconfig.h +++ b/include/util/tconfig.h @@ -89,6 +89,7 @@ void taosDumpGlobalCfg(); void taosInitConfigOption(SGlobalCfg cfg); SGlobalCfg * taosGetConfigOption(const char *option); +bool taosReadConfigOption(const char *option, char *value, char *value2, char *value3, int8_t cfgStatus, int8_t sourceType); #ifdef __cplusplus } diff --git a/src/util/inc/tcrc32c.h b/include/util/tcrc32c.h similarity index 100% rename from src/util/inc/tcrc32c.h rename to include/util/tcrc32c.h diff --git a/src/util/inc/tmd5.h b/include/util/tmd5.h similarity index 100% rename from src/util/inc/tmd5.h rename to include/util/tmd5.h diff --git a/src/util/inc/tnote.h b/include/util/tnote.h similarity index 100% rename from src/util/inc/tnote.h rename to include/util/tnote.h diff --git a/src/util/inc/tscompression.h b/include/util/tscompression.h similarity index 100% rename from src/util/inc/tscompression.h rename to include/util/tscompression.h diff --git a/src/util/inc/tsocket.h b/include/util/tsocket.h similarity index 100% rename from src/util/inc/tsocket.h rename to include/util/tsocket.h diff --git a/include/util/tstep.h b/include/util/tstep.h new file mode 100644 index 0000000000..90dd5dd0fc --- /dev/null +++ b/include/util/tstep.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_STEP_H_ +#define _TD_UTIL_STEP_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef int32_t (*FnInitObj)(void *parent, void **self); +typedef void (*FnCleanupObj)(void **self); +typedef void (*FnReportProgress)(void *parent, const char *name, const char *desc); + +typedef struct SStepObj { + const char * name; + void * parent; + void ** self; + FnInitObj initFp; + FnCleanupObj cleanupFp; + FnReportProgress reportFp; +} SStepObj; + +typedef struct SSteps { + int32_t cursize; + int32_t maxsize; + SStepObj *steps; +} SSteps; + +SSteps *taosStepInit(int32_t stepsize); +int32_t taosStepAdd(SSteps *steps, SStepObj *step); +int32_t taosStepExec(SSteps *steps); +void taosStepCleanup(SSteps *steps); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_STEP_H_*/ diff --git a/src/util/inc/ttimer.h b/include/util/ttimer.h similarity index 100% rename from src/util/inc/ttimer.h rename to include/util/ttimer.h diff --git a/src/util/inc/tutil.h b/include/util/tutil.h similarity index 100% rename from src/util/inc/tutil.h rename to include/util/tutil.h diff --git a/source/server/dnode/CMakeLists.txt b/source/server/dnode/CMakeLists.txt index aea4e9e144..cf7482eca6 100644 --- a/source/server/dnode/CMakeLists.txt +++ b/source/server/dnode/CMakeLists.txt @@ -4,6 +4,11 @@ target_link_libraries( dnode PUBLIC os PUBLIC cjson + PUBLIC util + PUBLIC common + PUBLIC transport + PUBLIC mnode + PUBLIC vnode ) target_include_directories( dnode diff --git a/source/server/dnode/src/dnodeCheck.c b/source/server/dnode/src/dnodeCheck.c index 402ef0d333..7c3b539d70 100644 --- a/source/server/dnode/src/dnodeCheck.c +++ b/source/server/dnode/src/dnodeCheck.c @@ -99,6 +99,7 @@ static int32_t dnodeCheckNetwork() { } static int32_t dnodeCheckMem() { +#if 0 float memoryUsedMB; float memoryAvailMB; if (true != taosGetSysMemory(&memoryUsedMB)) { @@ -112,11 +113,12 @@ static int32_t dnodeCheckMem() { dError("available memory %fMB less than the threshold %dMB", memoryAvailMB, MIN_AVAIL_MEMORY_MB); return -1; } - +#endif return 0; } static int32_t dnodeCheckDisk() { +#if 0 taosGetDisk(); if (tsAvailDataDirGB < tsMinimalDataDirGB) { @@ -133,7 +135,7 @@ static int32_t dnodeCheckDisk() { dError("tmpDir disk size:%fGB less than threshold %fGB", tsAvailTmpDirectorySpace, tsReservedTmpDirectorySpace); return -1; } - +#endif return 0; } diff --git a/source/server/dnode/src/dnodeInt.c b/source/server/dnode/src/dnodeInt.c index d77d81719b..2f805bb882 100644 --- a/source/server/dnode/src/dnodeInt.c +++ b/source/server/dnode/src/dnodeInt.c @@ -15,11 +15,13 @@ #define _DEFAULT_SOURCE #include "os.h" +#if 0 #include "qScript.h" #include "tfile.h" -#include "tstep.h" #include "tsync.h" #include "twal.h" +#endif +#include "tstep.h" #include "dnodeCfg.h" #include "dnodeCheck.h" #include "dnodeEps.h" @@ -31,16 +33,18 @@ #include "mnode.h" #include "vnode.h" -static int32_t dnodeInitTfsEnv(Dnode *dnode, void **unUsed) { return tfInit(); } -static void dnodeCleanupTfsEnv(void **unUsed) { tfCleanup(); } static int32_t dnodeInitRpcEnv(Dnode *dnode, void **unUsed) { return rpcInit(); } static void dnodeCleanupRpcEnv(void **unUsed) { rpcCleanup(); } +#if 0 +static int32_t dnodeInitTfsEnv(Dnode *dnode, void **unUsed) { return tfInit(); } +static void dnodeCleanupTfsEnv(void **unUsed) { tfCleanup(); } static int32_t dnodeInitScriptEnv(Dnode *dnode, void **unUsed) { return scriptEnvPoolInit(); } static void dnodeCleanupScriptEnv(void **unUsed) { scriptEnvPoolCleanup(); } static int32_t dnodeInitWalEnv(Dnode *dnode, void **unUsed) { return walInit(); } static void dnodeCleanupWalEnv(void **unUsed) { walCleanUp(); } static int32_t dnodeInitSyncEnv(Dnode *dnode, void **unUsed) { return syncInit(); } static void dnodeCleanupSyncEnv(void **unUsed) { syncCleanUp(); } +#endif static int32_t dnodeInitVnodeModule(Dnode *dnode, struct Vnode** out) { SVnodePara para; @@ -117,11 +121,13 @@ Dnode *dnodeCreateInstance() { step.reportFp = (FnReportProgress)dnodeReportStartup; taosStepAdd(steps, &step); +#if 0 step.name = "dnode-tfs-env"; step.self = NULL; step.initFp = (FnInitObj)dnodeInitTfsEnv; step.cleanupFp = (FnCleanupObj)dnodeCleanupTfsEnv; taosStepAdd(steps, &step); +#endif step.name = "dnode-rpc-env"; step.self = NULL; @@ -153,6 +159,7 @@ Dnode *dnodeCreateInstance() { step.cleanupFp = (FnCleanupObj)dnodeCleanupMnodeEps; taosStepAdd(steps, &step); +#if 0 step.name = "dnode-wal"; step.self = NULL; step.initFp = (FnInitObj)dnodeInitWalEnv; @@ -164,6 +171,7 @@ Dnode *dnodeCreateInstance() { step.initFp = (FnInitObj)dnodeInitSyncEnv; step.cleanupFp = (FnCleanupObj)dnodeCleanupSyncEnv; taosStepAdd(steps, &step); +#endif step.name = "dnode-vnode"; step.self = (void **)&dnode->vnode; @@ -195,11 +203,13 @@ Dnode *dnodeCreateInstance() { step.cleanupFp = (FnCleanupObj)dnodeCleanupTelemetry; taosStepAdd(steps, &step); +#if 0 step.name = "dnode-script"; step.self = NULL; step.initFp = (FnInitObj)dnodeInitScriptEnv; step.cleanupFp = (FnCleanupObj)dnodeCleanupScriptEnv; taosStepAdd(steps, &step); +#endif dnode->steps = steps; taosStepExec(dnode->steps); diff --git a/source/server/dnode/src/dnodeMain.c b/source/server/dnode/src/dnodeMain.c index a792cf6d5b..715c27a9e0 100644 --- a/source/server/dnode/src/dnodeMain.c +++ b/source/server/dnode/src/dnodeMain.c @@ -17,16 +17,18 @@ #include "os.h" #include "tcache.h" #include "tconfig.h" +#if 0 #include "tfs.h" -#include "tnote.h" +#endif #include "tscompression.h" +#include "tnote.h" #include "ttimer.h" #include "dnodeCfg.h" #include "dnodeMain.h" #include "mnode.h" static int32_t dnodeCreateDir(const char *dir) { - if (mkdir(dir, 0755) != 0 && errno != EEXIST) { + if (taosMkDir(dir, 0755) != 0 && errno != EEXIST) { return -1; } @@ -34,6 +36,7 @@ static int32_t dnodeCreateDir(const char *dir) { } static void dnodeCheckDataDirOpenned(char *dir) { +#if 0 char filepath[256] = {0}; snprintf(filepath, sizeof(filepath), "%s/.running", dir); @@ -49,6 +52,7 @@ static void dnodeCheckDataDirOpenned(char *dir) { close(fd); exit(0); } +#endif } int32_t dnodeInitMain(Dnode *dnode, DnMain **out) { @@ -71,7 +75,9 @@ int32_t dnodeInitMain(Dnode *dnode, DnMain **out) { taosResolveCRC(); taosInitGlobalCfg(); taosReadGlobalLogCfg(); +#if 0 taosSetCoreDump(); +#endif if (dnodeCreateDir(tsLogDir) < 0) { printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno)); @@ -125,11 +131,14 @@ int32_t dnodeInitStorage(Dnode *dnode, void **m) { return -1; } +#if 0 if (tfsInit(tsDiskCfg, tsDiskCfgNum) < 0) { dError("failed to init TFS since %s", tstrerror(terrno)); return -1; } + strncpy(tsDataDir, TFS_PRIMARY_PATH(), TSDB_FILENAME_LEN); +#endif sprintf(tsMnodeDir, "%s/mnode", tsDataDir); sprintf(tsVnodeDir, "%s/vnode", tsDataDir); sprintf(tsDnodeDir, "%s/dnode", tsDataDir); @@ -144,6 +153,7 @@ int32_t dnodeInitStorage(Dnode *dnode, void **m) { return -1; } +#if 0 if (tfsMkdir("vnode") < 0) { dError("failed to create vnode dir since %s", tstrerror(terrno)); return -1; @@ -154,6 +164,7 @@ int32_t dnodeInitStorage(Dnode *dnode, void **m) { return -1; } + TDIR *tdir = tfsOpendir("vnode_bak/.staging"); bool stagingNotEmpty = tfsReaddir(tdir) != NULL; tfsClosedir(tdir); @@ -172,12 +183,14 @@ int32_t dnodeInitStorage(Dnode *dnode, void **m) { taosGetDisk(); taosPrintDiskInfo(); +#endif dInfo("dnode storage is initialized at %s", tsDnodeDir); return 0; } void dnodeCleanupStorage(void **m) { +#if 0 // storage destroy tfsDestroy(); @@ -185,6 +198,7 @@ void dnodeCleanupStorage(void **m) { // compress destroy tsCompressExit(); #endif +#endif } void dnodeReportStartup(Dnode *dnode, char *name, char *desc) { diff --git a/source/server/dnode/src/dnodeTelemetry.c b/source/server/dnode/src/dnodeTelemetry.c index 8c2d4a431e..51f1e90cff 100644 --- a/source/server/dnode/src/dnodeTelemetry.c +++ b/source/server/dnode/src/dnodeTelemetry.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "osTime.h" +// #include "osTime.h" #include "tbuffer.h" #include "tglobal.h" #include "tsocket.h" diff --git a/src/inc/dnode.h b/src/inc/dnode.h deleted file mode 100644 index 5ecaf19f61..0000000000 --- a/src/inc/dnode.h +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_DNODE_H -#define TDENGINE_DNODE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "trpc.h" -#include "taosmsg.h" - -typedef struct { - int32_t queryReqNum; - int32_t submitReqNum; - int32_t httpReqNum; -} SStatisInfo; - -SStatisInfo dnodeGetStatisInfo(); - -bool dnodeIsFirstDeploy(); -bool dnodeIsMasterEp(char *ep); -void dnodeGetEpSetForPeer(SRpcEpSet *epSet); -void dnodeGetEpSetForShell(SRpcEpSet *epSet); -int32_t dnodeGetDnodeId(); -void dnodeGetClusterId(char *clusterId); - -void dnodeUpdateEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); -bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr); -int32_t dnodeStartMnode(SMInfos *pMinfos); - -void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)); -void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg); -void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp); -void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet); -void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid); - -void *dnodeAllocVWriteQueue(void *pVnode); -void dnodeFreeVWriteQueue(void *pWqueue); -void dnodeSendRpcVWriteRsp(void *pVnode, void *pWrite, int32_t code); -void *dnodeAllocVQueryQueue(void *pVnode); -void *dnodeAllocVFetchQueue(void *pVnode); -void dnodeFreeVQueryQueue(void *pQqueue); -void dnodeFreeVFetchQueue(void *pFqueue); - -int32_t dnodeAllocateMPeerQueue(); -void dnodeFreeMPeerQueue(); -int32_t dnodeAllocMReadQueue(); -void dnodeFreeMReadQueue(); -int32_t dnodeAllocMWritequeue(); -void dnodeFreeMWritequeue(); -void dnodeSendRpcMWriteRsp(void *pMsg, int32_t code); -void dnodeReprocessMWriteMsg(void *pMsg); -void dnodeDelayReprocessMWriteMsg(void *pMsg); - -void dnodeSendStatusMsgToMnode(); - -typedef struct { - char *name; - int32_t (*initFp)(); - void (*cleanupFp)(); -} SStep; - -int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize); -void dnodeStepCleanup(SStep *pSteps, int32_t stepSize); -void dnodeReportStep(char *name, char *desc, int8_t finished); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc/mnode.h b/src/inc/mnode.h deleted file mode 100644 index 203ac57469..0000000000 --- a/src/inc/mnode.h +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_MNODE_H -#define TDENGINE_MNODE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "trpc.h" - -struct SAcctObj; -struct SDnodeObj; -struct SUserObj; -struct SDbObj; -struct SVgObj; -struct STableObj; - -typedef struct { - int32_t len; - void * rsp; -} SMnodeRsp; - -typedef struct SMnodeMsg { - struct SAcctObj * pAcct; - struct SDnodeObj *pDnode; - struct SUserObj * pUser; - struct SDbObj * pDb; - struct SVgObj * pVgroup; - struct STableObj *pTable; - struct SSTableObj*pSTable; - struct SMnodeMsg *pBatchMasterMsg; - SMnodeRsp rpcRsp; - int16_t received; - int16_t successed; - int16_t expected; - int16_t retry; - int32_t incomingTs; - int32_t code; - void * pObj; - SRpcMsg rpcMsg; - char pCont[]; -} SMnodeMsg; - -void * mnodeCreateMsg(SRpcMsg *pRpcMsg); -int32_t mnodeInitMsg(SMnodeMsg *pMsg); -void mnodeCleanupMsg(SMnodeMsg *pMsg); -void mnodeDestroySubMsg(SMnodeMsg *pSubMsg); - -int32_t mnodeInitSystem(); -int32_t mnodeStartSystem(); -void mnodeCleanupSystem(); -void mnodeStopSystem(); -void sdbUpdateAsync(); -int32_t sdbUpdateSync(void *pMnodes); -bool mnodeIsRunning(); -int32_t mnodeProcessRead(SMnodeMsg *pMsg); -int32_t mnodeProcessWrite(SMnodeMsg *pMsg); -int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg); -void mnodeProcessPeerRsp(SRpcMsg *pMsg); -int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey); - -int32_t mnodeCompactWal(); -int32_t mnodeCompactComponents(); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/os/inc/osDef.h b/src/os/inc/osDef.h deleted file mode 100644 index 54a4f98254..0000000000 --- a/src/os/inc/osDef.h +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_OS_DEF_H -#define TDENGINE_OS_DEF_H - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined(_TD_DARWIN_64) - // specific - typedef int(*__compar_fn_t)(const void *, const void *); - - // for send function in tsocket.c - #if defined(MSG_NOSIGNAL) - #undef MSG_NOSIGNAL - #endif - - #define MSG_NOSIGNAL 0 - - #define SO_NO_CHECK 0x1234 - #define SOL_TCP 0x1234 - #define TCP_KEEPIDLE 0x1234 - - #ifndef PTHREAD_MUTEX_RECURSIVE_NP - #define PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE - #endif -#endif - -#if defined(_ALPINE) - typedef int(*__compar_fn_t)(const void *, const void *); - void error (int, int, const char *); - #ifndef PTHREAD_MUTEX_RECURSIVE_NP - #define PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE - #endif -#endif - -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - char *stpcpy (char *dest, const char *src); - char *stpncpy (char *dest, const char *src, size_t n); - - // specific - typedef int (*__compar_fn_t)(const void *, const void *); - #define ssize_t int - #define bzero(ptr, size) memset((ptr), 0, (size)) - #define strcasecmp _stricmp - #define strncasecmp _strnicmp - #define wcsncasecmp _wcsnicmp - #define strtok_r strtok_s - #define snprintf _snprintf - #define in_addr_t unsigned long - #define socklen_t int - - struct tm *localtime_r(const time_t *timep, struct tm *result); - char * strptime(const char *buf, const char *fmt, struct tm *tm); - char * strsep(char **stringp, const char *delim); - char * getpass(const char *prefix); - int flock(int fd, int option); - char * strndup(const char *s, size_t n); - int gettimeofday(struct timeval *ptv, void *pTimeZone); - - // for send function in tsocket.c - #define MSG_NOSIGNAL 0 - #define SO_NO_CHECK 0x1234 - #define SOL_TCP 0x1234 - - #ifndef TCP_KEEPCNT - #define TCP_KEEPCNT 0x1234 - #endif - - #ifndef TCP_KEEPIDLE - #define TCP_KEEPIDLE 0x1234 - #endif - - #ifndef TCP_KEEPINTVL - #define TCP_KEEPINTVL 0x1234 - #endif - - #define SHUT_RDWR SD_BOTH - #define SHUT_RD SD_RECEIVE - #define SHUT_WR SD_SEND - - #define LOCK_EX 1 - #define LOCK_NB 2 - #define LOCK_UN 3 - - #ifndef PATH_MAX - #define PATH_MAX 256 - #endif - - typedef struct { - int we_wordc; - char *we_wordv[1]; - int we_offs; - char wordPos[1025]; - } wordexp_t; - int wordexp(char *words, wordexp_t *pwordexp, int flags); - void wordfree(wordexp_t *pwordexp); - - #define openlog(a, b, c) - #define closelog() - #define LOG_ERR 0 - #define LOG_INFO 1 - void syslog(int unused, const char *format, ...); -#endif - -#ifndef WINDOWS - #ifndef O_BINARY - #define O_BINARY 0 - #endif -#endif - -#define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b))) -#define POINTER_DISTANCE(p1, p2) ((char *)(p1) - (char *)(p2)) - -#ifndef NDEBUG -#define ASSERT(x) assert(x) -#else -#define ASSERT(x) -#endif - -#ifndef UNUSED -#define UNUSED(x) ((void)(x)) -#endif - -#ifdef UNUSED_FUNC -#undefine UNUSED_FUNC -#endif - -#ifdef UNUSED_PARAM -#undef UNUSED_PARAM -#endif - -#if defined(__GNUC__) -#define UNUSED_PARAM(x) _UNUSED##x __attribute__((unused)) -#define UNUSED_FUNC __attribute__((unused)) -#else -#define UNUSED_PARAM(x) x -#define UNUSED_FUNC -#endif - -#ifdef tListLen -#undefine tListLen -#endif -#define tListLen(x) (sizeof(x) / sizeof((x)[0])) - -#if defined(__GNUC__) -#define FORCE_INLINE inline __attribute__((always_inline)) -#else -#define FORCE_INLINE -#endif - -#define DEFAULT_UNICODE_ENCODEC "UCS-4LE" - -#define DEFAULT_COMP(x, y) \ - do { \ - if ((x) == (y)) { \ - return 0; \ - } else { \ - return (x) < (y) ? -1 : 1; \ - } \ - } while (0) - -#define DEFAULT_DOUBLE_COMP(x, y) \ - do { \ - if (isnan(x) && isnan(y)) { return 0; } \ - if (isnan(x)) { return -1; } \ - if (isnan(y)) { return 1; } \ - if ((x) == (y)) { \ - return 0; \ - } else { \ - return (x) < (y) ? -1 : 1; \ - } \ - } while (0) - -#define DEFAULT_FLOAT_COMP(x, y) DEFAULT_DOUBLE_COMP(x, y) - -#define ALIGN_NUM(n, align) (((n) + ((align)-1)) & (~((align)-1))) - -// align to 8bytes -#define ALIGN8(n) ALIGN_NUM(n, 8) - -#undef threadlocal -#ifdef _ISOC11_SOURCE - #define threadlocal _Thread_local -#elif defined(__APPLE__) - #define threadlocal __thread -#elif defined(__GNUC__) && !defined(threadlocal) - #define threadlocal __thread -#else - #define threadlocal __declspec( thread ) -#endif - -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - #define PRIzu "ld" -#else - #define PRIzu "zu" -#endif - - -#if defined(_TD_LINUX_64) || defined(_TD_LINUX_32) || defined(_TD_MIPS_64) || defined(_TD_ARM_32) || defined(_TD_ARM_64) || defined(_TD_DARWIN_64) - #if defined(_TD_DARWIN_64) - // MacOS - #if !defined(_GNU_SOURCE) - #define setThreadName(name) do { pthread_setname_np((name)); } while (0) - #else - // pthread_setname_np not defined - #define setThreadName(name) - #endif - #else - // Linux, length of name must <= 16 (the last '\0' included) - #define setThreadName(name) do { prctl(PR_SET_NAME, (name)); } while (0) - #endif -#else - // Windows - #define setThreadName(name) -#endif - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/os/inc/osTime.h b/src/os/inc/osTime.h deleted file mode 100644 index dcb0e4c9b6..0000000000 --- a/src/os/inc/osTime.h +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_OS_TIME_H -#define TDENGINE_OS_TIME_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" -#include "taosdef.h" - -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - #ifdef _TD_GO_DLL_ - #define MILLISECOND_PER_SECOND (1000LL) - #else - #define MILLISECOND_PER_SECOND (1000i64) - #endif -#else - #define MILLISECOND_PER_SECOND ((int64_t)1000L) -#endif - -#define MILLISECOND_PER_MINUTE (MILLISECOND_PER_SECOND * 60) -#define MILLISECOND_PER_HOUR (MILLISECOND_PER_MINUTE * 60) -#define MILLISECOND_PER_DAY (MILLISECOND_PER_HOUR * 24) -#define MILLISECOND_PER_WEEK (MILLISECOND_PER_DAY * 7) - -//@return timestamp in second -int32_t taosGetTimestampSec(); - -//@return timestamp in millisecond -static FORCE_INLINE int64_t taosGetTimestampMs() { - struct timeval systemTime; - gettimeofday(&systemTime, NULL); - return (int64_t)systemTime.tv_sec * 1000L + (int64_t)systemTime.tv_usec / 1000; -} - -//@return timestamp in microsecond -static FORCE_INLINE int64_t taosGetTimestampUs() { - struct timeval systemTime; - gettimeofday(&systemTime, NULL); - return (int64_t)systemTime.tv_sec * 1000000L + (int64_t)systemTime.tv_usec; -} - -//@return timestamp in nanosecond -static FORCE_INLINE int64_t taosGetTimestampNs() { - struct timespec systemTime = {0}; - clock_gettime(CLOCK_REALTIME, &systemTime); - return (int64_t)systemTime.tv_sec * 1000000000L + (int64_t)systemTime.tv_nsec; -} - -/* - * @return timestamp decided by global conf variable, tsTimePrecision - * if precision == TSDB_TIME_PRECISION_MICRO, it returns timestamp in microsecond. - * precision == TSDB_TIME_PRECISION_MILLI, it returns timestamp in millisecond. - */ -static FORCE_INLINE int64_t taosGetTimestamp(int32_t precision) { - if (precision == TSDB_TIME_PRECISION_MICRO) { - return taosGetTimestampUs(); - } else if (precision == TSDB_TIME_PRECISION_NANO) { - return taosGetTimestampNs(); - }else { - return taosGetTimestampMs(); - } -} - - -typedef struct SInterval { - int32_t tz; // query client timezone - char intervalUnit; - char slidingUnit; - char offsetUnit; - int64_t interval; - int64_t sliding; - int64_t offset; -} SInterval; - -typedef struct SSessionWindow { - int64_t gap; // gap between two session window(in microseconds) - int32_t primaryColId; // primary timestamp column -} SSessionWindow; - -int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); -int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision); -int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision); - -int32_t parseAbsoluteDuration(char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision); -int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision); - -int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth); -void deltaToUtcInitOnce(); - -int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision); -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_TTIME_H From 676819f31f52b4d4c61010d3306e8465ccfe5bb7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 30 Sep 2021 13:43:41 +0800 Subject: [PATCH 29/44] [TD-10430] add transport for compile errors --- .../libs/transport}/inc/rpcCache.h | 0 .../libs/transport}/inc/rpcHead.h | 0 .../libs/transport}/inc/rpcLog.h | 0 .../libs/transport}/inc/rpcTcp.h | 0 .../libs/transport}/inc/rpcUdp.h | 0 .../libs/transport}/src/rpcCache.c | 0 .../libs/transport}/src/rpcMain.c | 14 +- .../libs/transport}/src/rpcTcp.c | 0 .../libs/transport}/src/rpcUdp.c | 0 src/rpc/CMakeLists.txt | 10 - src/rpc/test/CMakeLists.txt | 32 --- src/rpc/test/rclient.c | 199 ------------------ src/rpc/test/rsclient.c | 196 ----------------- src/rpc/test/rserver.c | 192 ----------------- 14 files changed, 8 insertions(+), 635 deletions(-) rename {src/rpc => source/libs/transport}/inc/rpcCache.h (100%) rename {src/rpc => source/libs/transport}/inc/rpcHead.h (100%) rename {src/rpc => source/libs/transport}/inc/rpcLog.h (100%) rename {src/rpc => source/libs/transport}/inc/rpcTcp.h (100%) rename {src/rpc => source/libs/transport}/inc/rpcUdp.h (100%) rename {src/rpc => source/libs/transport}/src/rpcCache.c (100%) rename {src/rpc => source/libs/transport}/src/rpcMain.c (99%) rename {src/rpc => source/libs/transport}/src/rpcTcp.c (100%) rename {src/rpc => source/libs/transport}/src/rpcUdp.c (100%) delete mode 100644 src/rpc/CMakeLists.txt delete mode 100644 src/rpc/test/CMakeLists.txt delete mode 100644 src/rpc/test/rclient.c delete mode 100644 src/rpc/test/rsclient.c delete mode 100644 src/rpc/test/rserver.c diff --git a/src/rpc/inc/rpcCache.h b/source/libs/transport/inc/rpcCache.h similarity index 100% rename from src/rpc/inc/rpcCache.h rename to source/libs/transport/inc/rpcCache.h diff --git a/src/rpc/inc/rpcHead.h b/source/libs/transport/inc/rpcHead.h similarity index 100% rename from src/rpc/inc/rpcHead.h rename to source/libs/transport/inc/rpcHead.h diff --git a/src/rpc/inc/rpcLog.h b/source/libs/transport/inc/rpcLog.h similarity index 100% rename from src/rpc/inc/rpcLog.h rename to source/libs/transport/inc/rpcLog.h diff --git a/src/rpc/inc/rpcTcp.h b/source/libs/transport/inc/rpcTcp.h similarity index 100% rename from src/rpc/inc/rpcTcp.h rename to source/libs/transport/inc/rpcTcp.h diff --git a/src/rpc/inc/rpcUdp.h b/source/libs/transport/inc/rpcUdp.h similarity index 100% rename from src/rpc/inc/rpcUdp.h rename to source/libs/transport/inc/rpcUdp.h diff --git a/src/rpc/src/rpcCache.c b/source/libs/transport/src/rpcCache.c similarity index 100% rename from src/rpc/src/rpcCache.c rename to source/libs/transport/src/rpcCache.c diff --git a/src/rpc/src/rpcMain.c b/source/libs/transport/src/rpcMain.c similarity index 99% rename from src/rpc/src/rpcMain.c rename to source/libs/transport/src/rpcMain.c index 9ea5fd5392..9bf2773dde 100644 --- a/src/rpc/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -55,8 +55,9 @@ typedef struct { char secret[TSDB_KEY_LEN]; // secret for the link char ckey[TSDB_KEY_LEN]; // ciphering key - void (*cfp)(SRpcMsg *, SRpcEpSet *); - int (*afp)(char *user, char *spi, char *encrypt, char *secret, char *ckey); + void *owner; + void (*cfp)(void * owner, SRpcMsg *, SRpcEpSet *); + int (*afp)(void * owner, char *user, char *spi, char *encrypt, char *secret, char *ckey); int32_t refCount; void *idPool; // handle to ID pool @@ -259,6 +260,7 @@ void *rpcOpen(const SRpcInit *pInit) { if (pInit->secret) memcpy(pRpc->secret, pInit->secret, sizeof(pRpc->secret)); if (pInit->ckey) tstrncpy(pRpc->ckey, pInit->ckey, sizeof(pRpc->ckey)); pRpc->spi = pInit->spi; + pRpc->owner = pInit->owner; pRpc->cfp = pInit->cfp; pRpc->afp = pInit->afp; pRpc->refCount = 1; @@ -741,7 +743,7 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { if (pConn->user[0] == 0) { terrno = TSDB_CODE_RPC_AUTH_REQUIRED; } else { - terrno = (*pRpc->afp)(pConn->user, &pConn->spi, &pConn->encrypt, pConn->secret, pConn->ckey); + terrno = (*pRpc->afp)(pRpc->owner, pConn->user, &pConn->spi, &pConn->encrypt, pConn->secret, pConn->ckey); } if (terrno != 0) { @@ -1021,7 +1023,7 @@ static void doRpcReportBrokenLinkToServer(void *param, void *id) { SRpcMsg *pRpcMsg = (SRpcMsg *)(param); SRpcConn *pConn = (SRpcConn *)(pRpcMsg->handle); SRpcInfo *pRpc = pConn->pRpc; - (*(pRpc->cfp))(pRpcMsg, NULL); + (*(pRpc->cfp))(pRpc->owner, pRpcMsg, NULL); free(pRpcMsg); } static void rpcReportBrokenLinkToServer(SRpcConn *pConn) { @@ -1136,7 +1138,7 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) { if (pContext->epSet.inUse != pContext->oldInUse || pContext->redirect) pEpSet = &pContext->epSet; - (*pRpc->cfp)(pMsg, pEpSet); + (*pRpc->cfp)(pRpc->owner, pMsg, pEpSet); } // free the request message @@ -1160,7 +1162,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte rpcAddRef(pRpc); // add the refCount for requests // notify the server app - (*(pRpc->cfp))(&rpcMsg, NULL); + (*(pRpc->cfp))(pRpc->owner, &rpcMsg, NULL); } else { // it's a response rpcMsg.handle = pContext; diff --git a/src/rpc/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c similarity index 100% rename from src/rpc/src/rpcTcp.c rename to source/libs/transport/src/rpcTcp.c diff --git a/src/rpc/src/rpcUdp.c b/source/libs/transport/src/rpcUdp.c similarity index 100% rename from src/rpc/src/rpcUdp.c rename to source/libs/transport/src/rpcUdp.c diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt deleted file mode 100644 index 14b77356ba..0000000000 --- a/src/rpc/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(inc) -AUX_SOURCE_DIRECTORY(src SRC) - -ADD_LIBRARY(trpc ${SRC}) -TARGET_LINK_LIBRARIES(trpc tutil lz4 common) - -ADD_SUBDIRECTORY(test) diff --git a/src/rpc/test/CMakeLists.txt b/src/rpc/test/CMakeLists.txt deleted file mode 100644 index a32ac9943d..0000000000 --- a/src/rpc/test/CMakeLists.txt +++ /dev/null @@ -1,32 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc) - -IF (TD_LINUX) - LIST(APPEND CLIENT_SRC ./rclient.c) - ADD_EXECUTABLE(rclient ${CLIENT_SRC}) - TARGET_LINK_LIBRARIES(rclient trpc) - - LIST(APPEND SCLIENT_SRC ./rsclient.c) - ADD_EXECUTABLE(rsclient ${SCLIENT_SRC}) - TARGET_LINK_LIBRARIES(rsclient trpc) - - LIST(APPEND SERVER_SRC ./rserver.c) - ADD_EXECUTABLE(rserver ${SERVER_SRC}) - TARGET_LINK_LIBRARIES(rserver trpc) -ENDIF () - -IF (TD_DARWIN) - LIST(APPEND CLIENT_SRC ./rclient.c) - ADD_EXECUTABLE(rclient ${CLIENT_SRC}) - TARGET_LINK_LIBRARIES(rclient trpc) - - LIST(APPEND SCLIENT_SRC ./rsclient.c) - ADD_EXECUTABLE(rsclient ${SCLIENT_SRC}) - TARGET_LINK_LIBRARIES(rsclient trpc) - - LIST(APPEND SERVER_SRC ./rserver.c) - ADD_EXECUTABLE(rserver ${SERVER_SRC}) - TARGET_LINK_LIBRARIES(rserver trpc) -ENDIF () diff --git a/src/rpc/test/rclient.c b/src/rpc/test/rclient.c deleted file mode 100644 index 2f4433f1bb..0000000000 --- a/src/rpc/test/rclient.c +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tutil.h" -#include "tglobal.h" -#include "rpcLog.h" -#include "trpc.h" -#include "taoserror.h" - -typedef struct { - int index; - SRpcEpSet epSet; - int num; - int numOfReqs; - int msgSize; - tsem_t rspSem; - tsem_t *pOverSem; - pthread_t thread; - void *pRpc; -} SInfo; - -static void processResponse(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - SInfo *pInfo = (SInfo *)pMsg->ahandle; - tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, pMsg->code); - - if (pEpSet) pInfo->epSet = *pEpSet; - - rpcFreeCont(pMsg->pCont); - tsem_post(&pInfo->rspSem); -} - -static int tcount = 0; - -static void *sendRequest(void *param) { - SInfo *pInfo = (SInfo *)param; - SRpcMsg rpcMsg = {0}; - - tDebug("thread:%d, start to send request", pInfo->index); - - while ( pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) { - pInfo->num++; - rpcMsg.pCont = rpcMallocCont(pInfo->msgSize); - rpcMsg.contLen = pInfo->msgSize; - rpcMsg.ahandle = pInfo; - rpcMsg.msgType = 1; - tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); - rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL); - if ( pInfo->num % 20000 == 0 ) - tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); - tsem_wait(&pInfo->rspSem); - } - - tDebug("thread:%d, it is over", pInfo->index); - tcount++; - - return NULL; -} - -int main(int argc, char *argv[]) { - SRpcInit rpcInit; - SRpcEpSet epSet; - int msgSize = 128; - int numOfReqs = 0; - int appThreads = 1; - char serverIp[40] = "127.0.0.1"; - char secret[TSDB_KEY_LEN] = "mypassword"; - struct timeval systemTime; - int64_t startTime, endTime; - pthread_attr_t thattr; - - // server info - epSet.numOfEps = 1; - epSet.inUse = 0; - epSet.port[0] = 7000; - epSet.port[1] = 7000; - strcpy(epSet.fqdn[0], serverIp); - strcpy(epSet.fqdn[1], "192.168.0.1"); - - // client info - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 0; - rpcInit.label = "APP"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = processResponse; - rpcInit.sessions = 100; - rpcInit.idleTime = tsShellActivityTimer*1000; - rpcInit.user = "michael"; - rpcInit.secret = secret; - rpcInit.ckey = "key"; - rpcInit.spi = 1; - rpcInit.connType = TAOS_CONN_CLIENT; - - for (int i=1; iindex = i; - pInfo->epSet = epSet; - pInfo->numOfReqs = numOfReqs; - pInfo->msgSize = msgSize; - tsem_init(&pInfo->rspSem, 0, 0); - pInfo->pRpc = pRpc; - pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo); - pInfo++; - } - - do { - usleep(1); - } while ( tcount < appThreads); - - gettimeofday(&systemTime, NULL); - endTime = systemTime.tv_sec*1000000 + systemTime.tv_usec; - float usedTime = (endTime - startTime)/1000.0f; // mseconds - - tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs*appThreads); - tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0*numOfReqs*appThreads/usedTime, msgSize); - - int ch = getchar(); - UNUSED(ch); - - taosCloseLog(); - - return 0; -} - - diff --git a/src/rpc/test/rsclient.c b/src/rpc/test/rsclient.c deleted file mode 100644 index 65170d4abb..0000000000 --- a/src/rpc/test/rsclient.c +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - - -#include "os.h" -#include "tutil.h" -#include "tglobal.h" -#include "rpcLog.h" -#include "trpc.h" -#include "taoserror.h" - -typedef struct { - int index; - SRpcEpSet epSet; - int num; - int numOfReqs; - int msgSize; - tsem_t rspSem; - tsem_t *pOverSem; - pthread_t thread; - void *pRpc; -} SInfo; - - -static int tcount = 0; -static int terror = 0; - -static void *sendRequest(void *param) { - SInfo *pInfo = (SInfo *)param; - SRpcMsg rpcMsg, rspMsg; - - tDebug("thread:%d, start to send request", pInfo->index); - - while ( pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) { - pInfo->num++; - rpcMsg.pCont = rpcMallocCont(pInfo->msgSize); - rpcMsg.contLen = pInfo->msgSize; - rpcMsg.handle = pInfo; - rpcMsg.msgType = 1; - tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); - - rpcSendRecv(pInfo->pRpc, &pInfo->epSet, &rpcMsg, &rspMsg); - - // handle response - if (rspMsg.code != 0) terror++; - - tDebug("thread:%d, rspLen:%d code:%d", pInfo->index, rspMsg.contLen, rspMsg.code); - - rpcFreeCont(rspMsg.pCont); - - if ( pInfo->num % 20000 == 0 ) - tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); - } - - tDebug("thread:%d, it is over", pInfo->index); - tcount++; - - return NULL; -} - -int main(int argc, char *argv[]) { - SRpcInit rpcInit; - SRpcEpSet epSet; - int msgSize = 128; - int numOfReqs = 0; - int appThreads = 1; - char serverIp[40] = "127.0.0.1"; - char secret[TSDB_KEY_LEN] = "mypassword"; - struct timeval systemTime; - int64_t startTime, endTime; - pthread_attr_t thattr; - - // server info - epSet.numOfEps = 1; - epSet.inUse = 0; - epSet.port[0] = 7000; - epSet.port[1] = 7000; - strcpy(epSet.fqdn[0], serverIp); - strcpy(epSet.fqdn[1], "192.168.0.1"); - - // client info - memset(&rpcInit, 0, sizeof(rpcInit)); - //rpcInit.localIp = "0.0.0.0"; - rpcInit.localPort = 0; - rpcInit.label = "APP"; - rpcInit.numOfThreads = 1; - rpcInit.sessions = 100; - rpcInit.idleTime = tsShellActivityTimer*1000; - rpcInit.user = "michael"; - rpcInit.secret = secret; - rpcInit.ckey = "key"; - rpcInit.spi = 1; - rpcInit.connType = TAOS_CONN_CLIENT; - - for (int i=1; iindex = i; - pInfo->epSet = epSet; - pInfo->numOfReqs = numOfReqs; - pInfo->msgSize = msgSize; - tsem_init(&pInfo->rspSem, 0, 0); - pInfo->pRpc = pRpc; - pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo); - pInfo++; - } - - do { - usleep(1); - } while ( tcount < appThreads); - - gettimeofday(&systemTime, NULL); - endTime = systemTime.tv_sec*1000000 + systemTime.tv_usec; - float usedTime = (endTime - startTime)/1000.0; // mseconds - - tInfo("it takes %.3f mseconds to send %d requests to server, error num:%d", usedTime, numOfReqs*appThreads, terror); - tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0*numOfReqs*appThreads/usedTime, msgSize); - - taosCloseLog(); - - return 0; -} - - diff --git a/src/rpc/test/rserver.c b/src/rpc/test/rserver.c deleted file mode 100644 index 64960db044..0000000000 --- a/src/rpc/test/rserver.c +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -//#define _DEFAULT_SOURCE -#include "os.h" -#include "tglobal.h" -#include "rpcLog.h" -#include "trpc.h" -#include "tqueue.h" - -int msgSize = 128; -int commit = 0; -int dataFd = -1; -void *qhandle = NULL; -void *qset = NULL; - -void processShellMsg() { - static int num = 0; - taos_qall qall; - SRpcMsg *pRpcMsg, rpcMsg; - int type; - void *pvnode; - - qall = taosAllocateQall(); - - while (1) { - int numOfMsgs = taosReadAllQitemsFromQset(qset, qall, &pvnode); - tDebug("%d shell msgs are received", numOfMsgs); - if (numOfMsgs <= 0) break; - - for (int i=0; i=0) { - if ( write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) <0 ) { - tInfo("failed to write data file, reason:%s", strerror(errno)); - } - } - } - - if (commit >=2) { - num += numOfMsgs; - if ( taosFsync(dataFd) < 0 ) { - tInfo("failed to flush data to file, reason:%s", strerror(errno)); - } - - if (num % 10000 == 0) { - tInfo("%d request have been written into disk", num); - } - } - - taosResetQitems(qall); - for (int i=0; ipCont); - - memset(&rpcMsg, 0, sizeof(rpcMsg)); - rpcMsg.pCont = rpcMallocCont(msgSize); - rpcMsg.contLen = msgSize; - rpcMsg.handle = pRpcMsg->handle; - rpcMsg.code = 0; - rpcSendResponse(&rpcMsg); - - taosFreeQitem(pRpcMsg); - } - - } - - taosFreeQall(qall); - -} - -int retrieveAuthInfo(char *meterId, char *spi, char *encrypt, char *secret, char *ckey) { - // app shall retrieve the auth info based on meterID from DB or a data file - // demo code here only for simple demo - int ret = 0; - - if (strcmp(meterId, "michael") == 0) { - *spi = 1; - *encrypt = 0; - strcpy(secret, "mypassword"); - strcpy(ckey, "key"); - } else if (strcmp(meterId, "jeff") == 0) { - *spi = 0; - *encrypt = 0; - } else { - ret = -1; // user not there - } - - return ret; -} - -void processRequestMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - SRpcMsg *pTemp; - - pTemp = taosAllocateQitem(sizeof(SRpcMsg)); - memcpy(pTemp, pMsg, sizeof(SRpcMsg)); - - tDebug("request is received, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); - taosWriteQitem(qhandle, TAOS_QTYPE_RPC, pTemp); -} - -int main(int argc, char *argv[]) { - SRpcInit rpcInit; - char dataName[20] = "server.data"; - - taosBlockSIGPIPE(); - - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 7000; - rpcInit.label = "SER"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = processRequestMsg; - rpcInit.sessions = 1000; - rpcInit.idleTime = tsShellActivityTimer*1500; - rpcInit.afp = retrieveAuthInfo; - - for (int i=1; i= 0) { - close(dataFd); - remove(dataName); - } - - return 0; -} From a0aa00b51de0ae17f38b0a838474ea68019ff2bc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 30 Sep 2021 13:44:10 +0800 Subject: [PATCH 30/44] remove old dnode files --- src/dnode/CMakeLists.txt | 6 - src/dnode/CMakeLists.txt.in | 63 ------ src/dnode/inc/dnodeCfg.h | 36 ---- src/dnode/inc/dnodeCheck.h | 31 --- src/dnode/inc/dnodeEps.h | 34 ---- src/dnode/inc/dnodeInt.h | 51 ----- src/dnode/inc/dnodeMInfos.h | 39 ---- src/dnode/inc/dnodeMPeer.h | 34 ---- src/dnode/inc/dnodeMRead.h | 34 ---- src/dnode/inc/dnodeMWrite.h | 34 ---- src/dnode/inc/dnodeMain.h | 31 --- src/dnode/inc/dnodeModule.h | 32 ---- src/dnode/inc/dnodePeer.h | 33 ---- src/dnode/inc/dnodeShell.h | 31 --- src/dnode/inc/dnodeStep.h | 33 ---- src/dnode/inc/dnodeTelemetry.h | 31 --- src/dnode/inc/dnodeVMgmt.h | 32 ---- src/dnode/inc/dnodeVRead.h | 36 ---- src/dnode/inc/dnodeVWrite.h | 35 ---- src/dnode/inc/dnodeVnodes.h | 34 ---- src/dnode/source/dnode.c | 20 -- src/dnode/src/dnodeCfg.c | 190 ------------------- src/dnode/src/dnodeCheck.c | 265 -------------------------- src/dnode/src/dnodeEps.c | 287 ---------------------------- src/dnode/src/dnodeMInfos.c | 326 ------------------------------- src/dnode/src/dnodeMPeer.c | 168 ---------------- src/dnode/src/dnodeMRead.c | 173 ----------------- src/dnode/src/dnodeMWrite.c | 227 ---------------------- src/dnode/src/dnodeMain.c | 337 --------------------------------- src/dnode/src/dnodeModule.c | 174 ----------------- src/dnode/src/dnodePeer.c | 195 ------------------- src/dnode/src/dnodeShell.c | 249 ------------------------ src/dnode/src/dnodeStep.c | 74 -------- src/dnode/src/dnodeSystem.c | 188 ------------------ src/dnode/src/dnodeTelemetry.c | 315 ------------------------------ src/dnode/src/dnodeVMgmt.c | 236 ----------------------- src/dnode/src/dnodeVRead.c | 154 --------------- src/dnode/src/dnodeVWrite.c | 238 ----------------------- src/dnode/src/dnodeVnodes.c | 303 ----------------------------- 39 files changed, 4809 deletions(-) delete mode 100644 src/dnode/CMakeLists.txt delete mode 100644 src/dnode/CMakeLists.txt.in delete mode 100644 src/dnode/inc/dnodeCfg.h delete mode 100644 src/dnode/inc/dnodeCheck.h delete mode 100644 src/dnode/inc/dnodeEps.h delete mode 100644 src/dnode/inc/dnodeInt.h delete mode 100644 src/dnode/inc/dnodeMInfos.h delete mode 100644 src/dnode/inc/dnodeMPeer.h delete mode 100644 src/dnode/inc/dnodeMRead.h delete mode 100644 src/dnode/inc/dnodeMWrite.h delete mode 100644 src/dnode/inc/dnodeMain.h delete mode 100644 src/dnode/inc/dnodeModule.h delete mode 100644 src/dnode/inc/dnodePeer.h delete mode 100644 src/dnode/inc/dnodeShell.h delete mode 100644 src/dnode/inc/dnodeStep.h delete mode 100644 src/dnode/inc/dnodeTelemetry.h delete mode 100644 src/dnode/inc/dnodeVMgmt.h delete mode 100644 src/dnode/inc/dnodeVRead.h delete mode 100644 src/dnode/inc/dnodeVWrite.h delete mode 100644 src/dnode/inc/dnodeVnodes.h delete mode 100644 src/dnode/source/dnode.c delete mode 100644 src/dnode/src/dnodeCfg.c delete mode 100644 src/dnode/src/dnodeCheck.c delete mode 100644 src/dnode/src/dnodeEps.c delete mode 100644 src/dnode/src/dnodeMInfos.c delete mode 100644 src/dnode/src/dnodeMPeer.c delete mode 100644 src/dnode/src/dnodeMRead.c delete mode 100644 src/dnode/src/dnodeMWrite.c delete mode 100644 src/dnode/src/dnodeMain.c delete mode 100644 src/dnode/src/dnodeModule.c delete mode 100644 src/dnode/src/dnodePeer.c delete mode 100644 src/dnode/src/dnodeShell.c delete mode 100644 src/dnode/src/dnodeStep.c delete mode 100644 src/dnode/src/dnodeSystem.c delete mode 100644 src/dnode/src/dnodeTelemetry.c delete mode 100644 src/dnode/src/dnodeVMgmt.c delete mode 100644 src/dnode/src/dnodeVRead.c delete mode 100644 src/dnode/src/dnodeVWrite.c delete mode 100644 src/dnode/src/dnodeVnodes.c diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt deleted file mode 100644 index 1141d2e452..0000000000 --- a/src/dnode/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -aux_source_directory(source DNODE_SRC) -add_executable(taosd ${DNODE_SRC}) -target_link_libraries( - taosd - PRIVATE os -) \ No newline at end of file diff --git a/src/dnode/CMakeLists.txt.in b/src/dnode/CMakeLists.txt.in deleted file mode 100644 index 47186130ea..0000000000 --- a/src/dnode/CMakeLists.txt.in +++ /dev/null @@ -1,63 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/sync/inc) -INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) - -INCLUDE_DIRECTORIES(inc) -AUX_SOURCE_DIRECTORY(src SRC) - -IF (TD_LINUX_64 AND JEMALLOC_ENABLED) - ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc) - SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc") -ELSE () - SET(LINK_JEMALLOC "") -ENDIF () - -ADD_EXECUTABLE(taosd ${SRC}) -TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lua lz4 balance sync ${LINK_JEMALLOC}) - -IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(taosd taos_static) -ELSE () - TARGET_LINK_LIBRARIES(taosd taos) -ENDIF () - -IF (TD_ACCOUNT) - TARGET_LINK_LIBRARIES(taosd account) -ENDIF () - -IF (TD_GRANT) - TARGET_LINK_LIBRARIES(taosd grant) -ENDIF () - -IF (TD_MQTT) - TARGET_LINK_LIBRARIES(taosd mqtt) -ENDIF () - -IF (TD_TOPIC) - TARGET_LINK_LIBRARIES(taosd topic) -ENDIF () - -IF (TD_MODULE AND TD_LINUX) - TARGET_LINK_LIBRARIES(taosd module dl) -ENDIF () - -SET(PREPARE_ENV_CMD "prepare_env_cmd") -SET(PREPARE_ENV_TARGET "prepare_env_target") -ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD} - POST_BUILD - COMMAND echo "make test directory" - DEPENDS taosd - COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/cfg/ - COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/log/ - COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/data/ - COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - COMMAND ${CMAKE_COMMAND} -E echo monitor 0 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - COMMENT "prepare taosd environment") -ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD}) diff --git a/src/dnode/inc/dnodeCfg.h b/src/dnode/inc/dnodeCfg.h deleted file mode 100644 index 99733e46ef..0000000000 --- a/src/dnode/inc/dnodeCfg.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_DNODE_CFG_H -#define TDENGINE_DNODE_CFG_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "dnodeInt.h" - -int32_t dnodeInitCfg(); -void dnodeCleanupCfg(); -void dnodeUpdateCfg(SDnodeCfg *cfg); -int32_t dnodeGetDnodeId(); -void dnodeGetClusterId(char *clusterId); -void dnodeGetCfg(int32_t *dnodeId, char *clusterId); -void dnodeSetDropped(); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/dnode/inc/dnodeCheck.h b/src/dnode/inc/dnodeCheck.h deleted file mode 100644 index c94b9e9319..0000000000 --- a/src/dnode/inc/dnodeCheck.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_DNODE_CHECK_H -#define TDENGINE_DNODE_CHECK_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "dnodeInt.h" - -int32_t dnodeInitCheck(); -void dnodeCleanupCheck(); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/dnode/inc/dnodeEps.h b/src/dnode/inc/dnodeEps.h deleted file mode 100644 index a5840997b0..0000000000 --- a/src/dnode/inc/dnodeEps.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_DNODE_EP_H -#define TDENGINE_DNODE_EP_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "dnodeInt.h" - -int32_t dnodeInitEps(); -void dnodeCleanupEps(); -void dnodeUpdateEps(SDnodeEps *eps); -void dnodeUpdateEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port); -bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/dnode/inc/dnodeInt.h b/src/dnode/inc/dnodeInt.h deleted file mode 100644 index 1327cd4433..0000000000 --- a/src/dnode/inc/dnodeInt.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_DNODE_LOG_H -#define TDENGINE_DNODE_LOG_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "taoserror.h" -#include "taosmsg.h" -#include "tlog.h" -#include "trpc.h" -#include "tglobal.h" -#include "dnode.h" -#include "vnode.h" - -extern int32_t dDebugFlag; - -#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", 255, __VA_ARGS__); }} -#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", 255, __VA_ARGS__); }} -#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", 255, __VA_ARGS__); }} -#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", 255, __VA_ARGS__); }} -#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }} -#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }} - -typedef enum { - TSDB_RUN_STATUS_INITIALIZE, - TSDB_RUN_STATUS_RUNING, - TSDB_RUN_STATUS_STOPPED -} SRunStatus; - -SRunStatus dnodeGetRunStatus(); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/dnode/inc/dnodeMInfos.h b/src/dnode/inc/dnodeMInfos.h deleted file mode 100644 index f05e2b6f7b..0000000000 --- a/src/dnode/inc/dnodeMInfos.h +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_DNODE_MINFOS_H -#define TDENGINE_DNODE_MINFOS_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "dnodeInt.h" - -int32_t dnodeInitMInfos(); -void dnodeCleanupMInfos(); -void dnodeUpdateMInfos(SMInfos *pMinfos); -void dnodeUpdateEpSetForPeer(SRpcEpSet *pEpSet); -void dnodeGetMInfos(SMInfos *pMinfos); -bool dnodeIsMasterEp(char *ep); - -void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell); -void dnodeGetEpSetForPeer(SRpcEpSet *epSet); -void dnodeGetEpSetForShell(SRpcEpSet *epSet); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/dnode/inc/dnodeMPeer.h b/src/dnode/inc/dnodeMPeer.h deleted file mode 100644 index b7e566d7e4..0000000000 --- a/src/dnode/inc/dnodeMPeer.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_DNODE_MPEER_H -#define TDENGINE_DNODE_MPEER_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "dnodeInt.h" - -int32_t dnodeInitMPeer(); -void dnodeCleanupMPeer(); -int32_t dnodeAllocateMPeerQueue(); -void dnodeFreeMPeerQueue(); -void dnodeDispatchToMPeerQueue(SRpcMsg *pMsg); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/dnode/inc/dnodeMRead.h b/src/dnode/inc/dnodeMRead.h deleted file mode 100644 index 279098d30e..0000000000 --- a/src/dnode/inc/dnodeMRead.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_DNODE_MREAD_H -#define TDENGINE_DNODE_MREAD_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "dnodeInt.h" - -int32_t dnodeInitMRead(); -void dnodeCleanupMRead(); -int32_t dnodeAllocMReadQueue(); -void dnodeFreeMReadQueue(); -void dnodeDispatchToMReadQueue(SRpcMsg *rpcMsg); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/dnode/inc/dnodeMWrite.h b/src/dnode/inc/dnodeMWrite.h deleted file mode 100644 index 8d4fcce3be..0000000000 --- a/src/dnode/inc/dnodeMWrite.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_DNODE_MWRITE_H -#define TDENGINE_DNODE_MWRITE_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "dnodeInt.h" - -int32_t dnodeInitMWrite(); -void dnodeCleanupMWrite(); -int32_t dnodeAllocMWritequeue(); -void dnodeFreeMWritequeue(); -void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/dnode/inc/dnodeMain.h b/src/dnode/inc/dnodeMain.h deleted file mode 100644 index ca79d53afd..0000000000 --- a/src/dnode/inc/dnodeMain.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_DNODE_MAIN_H -#define TDENGINE_DNODE_MAIN_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "dnodeInt.h" - -int32_t dnodeInitSystem(); -void dnodeCleanUpSystem(); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/dnode/inc/dnodeModule.h b/src/dnode/inc/dnodeModule.h deleted file mode 100644 index 863ea433c4..0000000000 --- a/src/dnode/inc/dnodeModule.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_DNODE_MODULE_H -#define TDENGINE_DNODE_MODULE_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "dnodeInt.h" - -int32_t dnodeInitModules(); -void dnodeCleanupModules(); -int32_t dnodeStartMnode(SMInfos *pMinfos); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/dnode/inc/dnodePeer.h b/src/dnode/inc/dnodePeer.h deleted file mode 100644 index 6d337ef6dc..0000000000 --- a/src/dnode/inc/dnodePeer.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_DNODE_PEER_H -#define TDENGINE_DNODE_PEER_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "dnodeInt.h" - -int32_t dnodeInitServer(); -void dnodeCleanupServer(); -int32_t dnodeInitClient(); -void dnodeCleanupClient(); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/dnode/inc/dnodeShell.h b/src/dnode/inc/dnodeShell.h deleted file mode 100644 index 3fa66d6a3b..0000000000 --- a/src/dnode/inc/dnodeShell.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_DNODE_SHELL_H -#define TDENGINE_DNODE_SHELL_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "dnodeInt.h" - -int32_t dnodeInitShell(); -void dnodeCleanupShell(); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/dnode/inc/dnodeStep.h b/src/dnode/inc/dnodeStep.h deleted file mode 100644 index e181e19c46..0000000000 --- a/src/dnode/inc/dnodeStep.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_DNODE_STEP_H -#define TDENGINE_DNODE_STEP_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "dnodeInt.h" - -int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize); -void dnodeStepCleanup(SStep *pSteps, int32_t stepSize); -void dnodeReportStep(char *name, char *desc, int8_t finished); -void dnodeSendStartupStep(SRpcMsg *pMsg); - -#ifdef __cplusplus -} -#endif - -#endif \ No newline at end of file diff --git a/src/dnode/inc/dnodeTelemetry.h b/src/dnode/inc/dnodeTelemetry.h deleted file mode 100644 index e4fd5a0376..0000000000 --- a/src/dnode/inc/dnodeTelemetry.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2020 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_DNODE_TELEMETRY_H -#define TDENGINE_DNODE_TELEMETRY_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "dnodeInt.h" - -int32_t dnodeInitTelemetry(); -void dnodeCleanupTelemetry(); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/dnode/inc/dnodeVMgmt.h b/src/dnode/inc/dnodeVMgmt.h deleted file mode 100644 index 821196defc..0000000000 --- a/src/dnode/inc/dnodeVMgmt.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_DNODE_VMGMT_H -#define TDENGINE_DNODE_VMGMT_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "dnodeInt.h" - -int32_t dnodeInitVMgmt(); -void dnodeCleanupVMgmt(); -void dnodeDispatchToVMgmtQueue(SRpcMsg *rpcMsg); - -#ifdef __cplusplus -} -#endif - -#endif \ No newline at end of file diff --git a/src/dnode/inc/dnodeVRead.h b/src/dnode/inc/dnodeVRead.h deleted file mode 100644 index 9c88886f88..0000000000 --- a/src/dnode/inc/dnodeVRead.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_DNODE_VREAD_H -#define TDENGINE_DNODE_VREAD_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "dnodeInt.h" - -int32_t dnodeInitVRead(); -void dnodeCleanupVRead(); -void dnodeDispatchToVReadQueue(SRpcMsg *pMsg); -void * dnodeAllocVQueryQueue(void *pVnode); -void * dnodeAllocVFetchQueue(void *pVnode); -void dnodeFreeVQueryQueue(void *pQqueue); -void dnodeFreeVFetchQueue(void *pFqueue); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/dnode/inc/dnodeVWrite.h b/src/dnode/inc/dnodeVWrite.h deleted file mode 100644 index 2ddff210f8..0000000000 --- a/src/dnode/inc/dnodeVWrite.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_DNODE_VWRITE_H -#define TDENGINE_DNODE_VWRITE_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "dnodeInt.h" - -int32_t dnodeInitVWrite(); -void dnodeCleanupVWrite(); -void dnodeDispatchToVWriteQueue(SRpcMsg *pMsg); -void * dnodeAllocVWriteQueue(void *pVnode); -void dnodeFreeVWriteQueue(void *pWqueue); -void dnodeSendRpcVWriteRsp(void *pVnode, void *pWrite, int32_t code); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/dnode/inc/dnodeVnodes.h b/src/dnode/inc/dnodeVnodes.h deleted file mode 100644 index e60dd290ce..0000000000 --- a/src/dnode/inc/dnodeVnodes.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_DNODE_VNODES_H -#define TDENGINE_DNODE_VNODES_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "dnodeInt.h" - -int32_t dnodeInitVnodes(); -void dnodeCleanupVnodes(); -int32_t dnodeInitStatusTimer(); -void dnodeCleanupStatusTimer(); -void dnodeSendStatusMsgToMnode(); - -#ifdef __cplusplus -} -#endif - -#endif \ No newline at end of file diff --git a/src/dnode/source/dnode.c b/src/dnode/source/dnode.c deleted file mode 100644 index 75449ecd1a..0000000000 --- a/src/dnode/source/dnode.c +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "os.h" - -int main(int argc, char const *argv[]) { - printf("Hello world!\n"); - return 0; -} diff --git a/src/dnode/src/dnodeCfg.c b/src/dnode/src/dnodeCfg.c deleted file mode 100644 index 4269c77bf3..0000000000 --- a/src/dnode/src/dnodeCfg.c +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "cJSON.h" -#include "dnodeCfg.h" -#include "tglobal.h" - -static SDnodeCfg tsCfg = {0}; -static pthread_mutex_t tsCfgMutex; -static int32_t tsDnodeDropped; - -static int32_t dnodeReadCfg(); -static int32_t dnodeWriteCfg(); -static void dnodeResetCfg(SDnodeCfg *cfg); -static void dnodePrintCfg(SDnodeCfg *cfg); - -int32_t dnodeInitCfg() { - pthread_mutex_init(&tsCfgMutex, NULL); - dnodeResetCfg(NULL); - int32_t ret = dnodeReadCfg(); - if (ret == 0) { - dInfo("dnode cfg is initialized"); - } - if (tsDnodeDropped) { - dInfo("dnode is dropped, exiting"); - return -1; - } - return ret; -} - -void dnodeCleanupCfg() { pthread_mutex_destroy(&tsCfgMutex); } - -void dnodeUpdateCfg(SDnodeCfg *cfg) { - if (tsCfg.dnodeId != 0) return; - dnodeResetCfg(cfg); -} - -void dnodeSetDropped() { - pthread_mutex_lock(&tsCfgMutex); - tsDnodeDropped = 1; - - dnodeWriteCfg(); - pthread_mutex_unlock(&tsCfgMutex); -} - -int32_t dnodeGetDnodeId() { - int32_t dnodeId = 0; - pthread_mutex_lock(&tsCfgMutex); - dnodeId = tsCfg.dnodeId; - pthread_mutex_unlock(&tsCfgMutex); - return dnodeId; -} - -void dnodeGetClusterId(char *clusterId) { - pthread_mutex_lock(&tsCfgMutex); - tstrncpy(clusterId, tsCfg.clusterId, TSDB_CLUSTER_ID_LEN); - pthread_mutex_unlock(&tsCfgMutex); -} - -void dnodeGetCfg(int32_t *dnodeId, char *clusterId) { - pthread_mutex_lock(&tsCfgMutex); - *dnodeId = tsCfg.dnodeId; - tstrncpy(clusterId, tsCfg.clusterId, TSDB_CLUSTER_ID_LEN); - pthread_mutex_unlock(&tsCfgMutex); -} - -static void dnodeResetCfg(SDnodeCfg *cfg) { - if (cfg == NULL) return; - if (cfg->dnodeId == 0) return; - - pthread_mutex_lock(&tsCfgMutex); - tsCfg.dnodeId = cfg->dnodeId; - tsDnodeId = cfg->dnodeId; - tstrncpy(tsCfg.clusterId, cfg->clusterId, TSDB_CLUSTER_ID_LEN); - dnodePrintCfg(cfg); - dnodeWriteCfg(); - pthread_mutex_unlock(&tsCfgMutex); -} - -static void dnodePrintCfg(SDnodeCfg *cfg) { - dInfo("dnodeId is set to %d, clusterId is set to %s", cfg->dnodeId, cfg->clusterId); -} - -static int32_t dnodeReadCfg() { - int32_t len = 0; - int32_t maxLen = 200; - char * content = calloc(1, maxLen + 1); - cJSON * root = NULL; - FILE * fp = NULL; - SDnodeCfg cfg = {0}; - - char file[TSDB_FILENAME_LEN + 20] = {0}; - sprintf(file, "%s/dnodeCfg.json", tsDnodeDir); - - fp = fopen(file, "r"); - if (!fp) { - dDebug("failed to read %s, file not exist", file); - goto PARSE_CFG_OVER; - } - - len = (int32_t)fread(content, 1, maxLen, fp); - if (len <= 0) { - dError("failed to read %s, content is null", file); - goto PARSE_CFG_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s, invalid json format", file); - goto PARSE_CFG_OVER; - } - - cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId"); - if (!dnodeId || dnodeId->type != cJSON_Number) { - dError("failed to read %s, dnodeId not found", file); - goto PARSE_CFG_OVER; - } - cfg.dnodeId = (int32_t)dnodeId->valueint; - - cJSON *dnodeDropped = cJSON_GetObjectItem(root, "dnodeDropped"); - if (!dnodeDropped || dnodeDropped->type != cJSON_Number) { - dError("failed to read %s, dnodeDropped not found", file); - //goto PARSE_CFG_OVER; - } else { - tsDnodeDropped = (int32_t)dnodeDropped->valueint; - } - - cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId"); - if (!clusterId || clusterId->type != cJSON_String) { - dError("failed to read %s, clusterId not found", file); - goto PARSE_CFG_OVER; - } - tstrncpy(cfg.clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN); - - dInfo("read file %s successed", file); - -PARSE_CFG_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); - terrno = 0; - - dnodeResetCfg(&cfg); - return 0; -} - -static int32_t dnodeWriteCfg() { - char file[TSDB_FILENAME_LEN + 20] = {0}; - sprintf(file, "%s/dnodeCfg.json", tsDnodeDir); - - FILE *fp = fopen(file, "w"); - if (!fp) { - dError("failed to write %s, reason:%s", file, strerror(errno)); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 200; - char * content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", tsCfg.dnodeId); - len += snprintf(content + len, maxLen - len, " \"dnodeDropped\": %d,\n", tsDnodeDropped); - len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%s\"\n", tsCfg.clusterId); - len += snprintf(content + len, maxLen - len, "}\n"); - - fwrite(content, 1, len, fp); - taosFsync(fileno(fp)); - fclose(fp); - free(content); - terrno = 0; - - dInfo("successed to write %s", file); - return 0; -} diff --git a/src/dnode/src/dnodeCheck.c b/src/dnode/src/dnodeCheck.c deleted file mode 100644 index 87baff3067..0000000000 --- a/src/dnode/src/dnodeCheck.c +++ /dev/null @@ -1,265 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "dnodeCheck.h" - -typedef struct { - bool enable; - char * name; - int32_t (*initFp)(); - int32_t (*startFp)(); - void (*cleanUpFp)(); - void (*stopFp)(); -} SCheckItem; - -static SCheckItem tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}}; -int64_t tsMinFreeMemSizeForStart = 0; - -static int32_t bindTcpPort(uint16_t port) { - SOCKET serverSocket; - struct sockaddr_in server_addr; - - if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { - dError("socket() fail: %s", strerror(errno)); - return -1; - } - - bzero(&server_addr, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - server_addr.sin_addr.s_addr = htonl(INADDR_ANY); - - if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { - dError("port:%d tcp bind() fail: %s", port, strerror(errno)); - taosCloseSocket(serverSocket); - return -1; - } - - if (listen(serverSocket, 5) < 0) { - dError("port:%d listen() fail: %s", port, strerror(errno)); - taosCloseSocket(serverSocket); - return -1; - } - - taosCloseSocket(serverSocket); - return 0; -} - -static int32_t bindUdpPort(int16_t port) { - SOCKET serverSocket; - struct sockaddr_in server_addr; - - if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { - dError("socket() fail: %s", strerror(errno)); - return -1; - } - - bzero(&server_addr, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - server_addr.sin_addr.s_addr = htonl(INADDR_ANY); - - if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { - dError("port:%d udp bind() fail: %s", port, strerror(errno)); - taosCloseSocket(serverSocket); - return -1; - } - - taosCloseSocket(serverSocket); - return 0; -} - -static int32_t dnodeCheckNetwork() { - int32_t ret; - uint16_t startPort = tsServerPort; - - for (uint16_t port = startPort; port < startPort + 12; port++) { - ret = bindTcpPort(port); - if (0 != ret) { - dError("failed to tcp bind port %d, quit", port); - return -1; - } - ret = bindUdpPort(port); - if (0 != ret) { - dError("failed to udp bind port %d, quit", port); - return -1; - } - } - - return 0; -} - -static int32_t dnodeCheckMem() { - float memoryUsedMB; - float memoryAvailMB; - if (true != taosGetSysMemory(&memoryUsedMB)) { - dError("failed to get system mem infomation, errno:%u, reason:%s", errno, strerror(errno)); - return -1; - } - - memoryAvailMB = (float)tsTotalMemoryMB - memoryUsedMB; - - if (memoryAvailMB < tsMinFreeMemSizeForStart) { - dError("free mem %f too little, quit", memoryAvailMB); - return -1; - } - - return 0; -} - -static int32_t dnodeCheckCpu() { - // TODO: - return 0; -} - -static int32_t dnodeCheckDisk() { - taosGetDisk(); - - if (tsAvailDataDirGB < tsMinimalDataDirGB) { - dError("free disk size: %f GB, too little, quit", tsAvailDataDirGB); - return -1; - } - - if (tsAvailLogDirGB < tsMinimalLogDirGB) { - dError("free disk size: %f GB, too little, quit", tsAvailLogDirGB); - return -1; - } - - if (tsAvailTmpDirectorySpace < tsReservedTmpDirectorySpace) { - dError("free disk size: %f GB, too little, quit", tsAvailTmpDirectorySpace); - return -1; - } - - return 0; -} - -static int32_t dnodeCheckOs() { - // TODO: - - return 0; -} -static int32_t dnodeCheckAccess() { - // TODO: - - return 0; -} - -static int32_t dnodeCheckVersion() { - // TODO: - - return 0; -} - -static int32_t dnodeCheckDatafile() { - // TODO: - - return 0; -} - -static void dnodeAllocCheckItem() { - tsCheckItem[TSDB_CHECK_ITEM_NETWORK].enable = false; - tsCheckItem[TSDB_CHECK_ITEM_NETWORK].name = "network"; - tsCheckItem[TSDB_CHECK_ITEM_NETWORK].initFp = NULL; - tsCheckItem[TSDB_CHECK_ITEM_NETWORK].cleanUpFp = NULL; - tsCheckItem[TSDB_CHECK_ITEM_NETWORK].startFp = dnodeCheckNetwork; - tsCheckItem[TSDB_CHECK_ITEM_NETWORK].stopFp = NULL; - - tsCheckItem[TSDB_CHECK_ITEM_MEM].enable = true; - tsCheckItem[TSDB_CHECK_ITEM_MEM].name = "mem"; - tsCheckItem[TSDB_CHECK_ITEM_MEM].initFp = NULL; - tsCheckItem[TSDB_CHECK_ITEM_MEM].cleanUpFp = NULL; - tsCheckItem[TSDB_CHECK_ITEM_MEM].startFp = dnodeCheckMem; - tsCheckItem[TSDB_CHECK_ITEM_MEM].stopFp = NULL; - - tsCheckItem[TSDB_CHECK_ITEM_CPU].enable = true; - tsCheckItem[TSDB_CHECK_ITEM_CPU].name = "cpu"; - tsCheckItem[TSDB_CHECK_ITEM_CPU].initFp = NULL; - tsCheckItem[TSDB_CHECK_ITEM_CPU].cleanUpFp = NULL; - tsCheckItem[TSDB_CHECK_ITEM_CPU].startFp = dnodeCheckCpu; - tsCheckItem[TSDB_CHECK_ITEM_CPU].stopFp = NULL; - - tsCheckItem[TSDB_CHECK_ITEM_DISK].enable = true; - tsCheckItem[TSDB_CHECK_ITEM_DISK].name = "disk"; - tsCheckItem[TSDB_CHECK_ITEM_DISK].initFp = NULL; - tsCheckItem[TSDB_CHECK_ITEM_DISK].cleanUpFp = NULL; - tsCheckItem[TSDB_CHECK_ITEM_DISK].startFp = dnodeCheckDisk; - tsCheckItem[TSDB_CHECK_ITEM_DISK].stopFp = NULL; - - tsCheckItem[TSDB_CHECK_ITEM_OS].enable = true; - tsCheckItem[TSDB_CHECK_ITEM_OS].name = "os"; - tsCheckItem[TSDB_CHECK_ITEM_OS].initFp = NULL; - tsCheckItem[TSDB_CHECK_ITEM_OS].cleanUpFp = NULL; - tsCheckItem[TSDB_CHECK_ITEM_OS].startFp = dnodeCheckOs; - tsCheckItem[TSDB_CHECK_ITEM_OS].stopFp = NULL; - - tsCheckItem[TSDB_CHECK_ITEM_ACCESS].enable = true; - tsCheckItem[TSDB_CHECK_ITEM_ACCESS].name = "access"; - tsCheckItem[TSDB_CHECK_ITEM_ACCESS].initFp = NULL; - tsCheckItem[TSDB_CHECK_ITEM_ACCESS].cleanUpFp = NULL; - tsCheckItem[TSDB_CHECK_ITEM_ACCESS].startFp = dnodeCheckAccess; - tsCheckItem[TSDB_CHECK_ITEM_ACCESS].stopFp = NULL; - - tsCheckItem[TSDB_CHECK_ITEM_VERSION].enable = true; - tsCheckItem[TSDB_CHECK_ITEM_VERSION].name = "version"; - tsCheckItem[TSDB_CHECK_ITEM_VERSION].initFp = NULL; - tsCheckItem[TSDB_CHECK_ITEM_VERSION].cleanUpFp = NULL; - tsCheckItem[TSDB_CHECK_ITEM_VERSION].startFp = dnodeCheckVersion; - tsCheckItem[TSDB_CHECK_ITEM_VERSION].stopFp = NULL; - - tsCheckItem[TSDB_CHECK_ITEM_DATAFILE].enable = true; - tsCheckItem[TSDB_CHECK_ITEM_DATAFILE].name = "datafile"; - tsCheckItem[TSDB_CHECK_ITEM_DATAFILE].initFp = NULL; - tsCheckItem[TSDB_CHECK_ITEM_DATAFILE].cleanUpFp = NULL; - tsCheckItem[TSDB_CHECK_ITEM_DATAFILE].startFp = dnodeCheckDatafile; - tsCheckItem[TSDB_CHECK_ITEM_DATAFILE].stopFp = NULL; -} - -void dnodeCleanupCheck() { - for (ECheckItemType index = 0; index < TSDB_CHECK_ITEM_MAX; ++index) { - if (tsCheckItem[index].enable && tsCheckItem[index].stopFp) { - (*tsCheckItem[index].stopFp)(); - } - if (tsCheckItem[index].cleanUpFp) { - (*tsCheckItem[index].cleanUpFp)(); - } - } -} - -int32_t dnodeInitCheck() { - dnodeAllocCheckItem(); - - for (ECheckItemType index = 0; index < TSDB_CHECK_ITEM_MAX; ++index) { - if (tsCheckItem[index].initFp) { - if ((*tsCheckItem[index].initFp)() != 0) { - dError("failed to init check item:%s", tsCheckItem[index].name); - return -1; - } - } - } - - for (ECheckItemType index = 0; index < TSDB_CHECK_ITEM_MAX; ++index) { - if (tsCheckItem[index].enable && tsCheckItem[index].startFp) { - if ((*tsCheckItem[index].startFp)() != 0) { - dError("failed to check item:%s", tsCheckItem[index].name); - exit(-1); - } - } - } - - dInfo("dnode check is initialized"); - return 0; -} diff --git a/src/dnode/src/dnodeEps.c b/src/dnode/src/dnodeEps.c deleted file mode 100644 index 9b15353647..0000000000 --- a/src/dnode/src/dnodeEps.c +++ /dev/null @@ -1,287 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "cJSON.h" -#include "hash.h" -#include "dnodeEps.h" - -static SDnodeEps *tsEps = NULL; -static SHashObj * tsEpsHash = NULL; -static pthread_mutex_t tsEpsMutex; - -static int32_t dnodeReadEps(); -static int32_t dnodeWriteEps(); -static void dnodeResetEps(SDnodeEps *eps); -static void dnodePrintEps(SDnodeEps *eps); - -int32_t dnodeInitEps() { - pthread_mutex_init(&tsEpsMutex, NULL); - tsEpsHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - dnodeResetEps(NULL); - int32_t ret = dnodeReadEps(); - if (ret == 0) { - dInfo("dnode eps is initialized"); - } - return ret; -} - -void dnodeCleanupEps() { - pthread_mutex_lock(&tsEpsMutex); - if (tsEps) { - free(tsEps); - tsEps = NULL; - } - if (tsEpsHash) { - taosHashCleanup(tsEpsHash); - tsEpsHash = NULL; - } - pthread_mutex_unlock(&tsEpsMutex); - pthread_mutex_destroy(&tsEpsMutex); -} - -void dnodeUpdateEps(SDnodeEps *eps) { - if (eps == NULL) return; - - eps->dnodeNum = htonl(eps->dnodeNum); - for (int32_t i = 0; i < eps->dnodeNum; ++i) { - eps->dnodeEps[i].dnodeId = htonl(eps->dnodeEps[i].dnodeId); - eps->dnodeEps[i].dnodePort = htons(eps->dnodeEps[i].dnodePort); - } - - pthread_mutex_lock(&tsEpsMutex); - if (eps->dnodeNum != tsEps->dnodeNum) { - dnodeResetEps(eps); - dnodeWriteEps(); - } else { - int32_t size = sizeof(SDnodeEps) + eps->dnodeNum * sizeof(SDnodeEp); - if (memcmp(eps, tsEps, size) != 0) { - dnodeResetEps(eps); - dnodeWriteEps(); - } - } - pthread_mutex_unlock(&tsEpsMutex); -} - -bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr) { - bool changed = false; - pthread_mutex_lock(&tsEpsMutex); - SDnodeEp *ep = taosHashGet(tsEpsHash, &dnodeId, sizeof(int32_t)); - if (ep != NULL) { - char epSaved[TSDB_EP_LEN + 1]; - snprintf(epSaved, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort); - changed = strcmp(epstr, epSaved) != 0; - tstrncpy(epstr, epSaved, TSDB_EP_LEN); - } - pthread_mutex_unlock(&tsEpsMutex); - return changed; -} - -void dnodeUpdateEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port) { - pthread_mutex_lock(&tsEpsMutex); - SDnodeEp *ep = taosHashGet(tsEpsHash, &dnodeId, sizeof(int32_t)); - if (ep != NULL) { - if (port) *port = ep->dnodePort; - if (fqdn) tstrncpy(fqdn, ep->dnodeFqdn, TSDB_FQDN_LEN); - if (epstr) snprintf(epstr, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort); - } - pthread_mutex_unlock(&tsEpsMutex); -} - -static void dnodeResetEps(SDnodeEps *eps) { - if (eps == NULL) { - int32_t size = sizeof(SDnodeEps) + sizeof(SDnodeEp); - if (tsEps == NULL) { - tsEps = calloc(1, size); - } else { - tsEps->dnodeNum = 0; - } - } else { - assert(tsEps); - - int32_t size = sizeof(SDnodeEps) + sizeof(SDnodeEp) * eps->dnodeNum; - if (eps->dnodeNum > tsEps->dnodeNum) { - tsEps = realloc(tsEps, size); - } - memcpy(tsEps, eps, size); - dnodePrintEps(eps); - } - - for (int32_t i = 0; i < tsEps->dnodeNum; ++i) { - SDnodeEp *ep = &tsEps->dnodeEps[i]; - taosHashPut(tsEpsHash, &ep->dnodeId, sizeof(int32_t), ep, sizeof(SDnodeEp)); - } -} - -static void dnodePrintEps(SDnodeEps *eps) { - dDebug("print dnodeEp, dnodeNum:%d", eps->dnodeNum); - for (int32_t i = 0; i < eps->dnodeNum; i++) { - SDnodeEp *ep = &eps->dnodeEps[i]; - dDebug("dnode:%d, dnodeFqdn:%s dnodePort:%u", ep->dnodeId, ep->dnodeFqdn, ep->dnodePort); - } -} - -static int32_t dnodeReadEps() { - int32_t ret = -1; - int32_t len = 0; - int32_t maxLen = 30000; - char * content = calloc(1, maxLen + 1); - cJSON * root = NULL; - FILE * fp = NULL; - SDnodeEps *eps = NULL; - - char file[TSDB_FILENAME_LEN + 20] = {0}; - sprintf(file, "%s/dnodeEps.json", tsDnodeDir); - - fp = fopen(file, "r"); - if (!fp) { - dDebug("failed to read %s, file not exist", file); - goto PRASE_EPS_OVER; - } - - len = (int32_t)fread(content, 1, maxLen, fp); - if (len <= 0) { - dError("failed to read %s, content is null", file); - goto PRASE_EPS_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s, invalid json format", file); - goto PRASE_EPS_OVER; - } - - cJSON *dnodeNum = cJSON_GetObjectItem(root, "dnodeNum"); - if (!dnodeNum || dnodeNum->type != cJSON_Number) { - dError("failed to read %s, dnodeNum not found", file); - goto PRASE_EPS_OVER; - } - - cJSON *dnodeInfos = cJSON_GetObjectItem(root, "dnodeInfos"); - if (!dnodeInfos || dnodeInfos->type != cJSON_Array) { - dError("failed to read %s, dnodeInfos not found", file); - goto PRASE_EPS_OVER; - } - - int32_t dnodeInfosSize = cJSON_GetArraySize(dnodeInfos); - if (dnodeInfosSize != dnodeNum->valueint) { - dError("failed to read %s, dnodeInfos size:%d not matched dnodeNum:%d", file, dnodeInfosSize, - (int32_t)dnodeNum->valueint); - goto PRASE_EPS_OVER; - } - - int32_t epsSize = sizeof(SDnodeEps) + dnodeInfosSize * sizeof(SDnodeEp); - eps = calloc(1, epsSize); - eps->dnodeNum = dnodeInfosSize; - - for (int32_t i = 0; i < dnodeInfosSize; ++i) { - cJSON *dnodeInfo = cJSON_GetArrayItem(dnodeInfos, i); - if (dnodeInfo == NULL) break; - - SDnodeEp *ep = &eps->dnodeEps[i]; - - cJSON *dnodeId = cJSON_GetObjectItem(dnodeInfo, "dnodeId"); - if (!dnodeId || dnodeId->type != cJSON_Number) { - dError("failed to read %s, dnodeId not found", file); - goto PRASE_EPS_OVER; - } - ep->dnodeId = (int32_t)dnodeId->valueint; - - cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn"); - if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { - dError("failed to read %s, dnodeFqdn not found", file); - goto PRASE_EPS_OVER; - } - strncpy(ep->dnodeFqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN); - - cJSON *dnodePort = cJSON_GetObjectItem(dnodeInfo, "dnodePort"); - if (!dnodePort || dnodePort->type != cJSON_Number) { - dError("failed to read %s, dnodePort not found", file); - goto PRASE_EPS_OVER; - } - ep->dnodePort = (uint16_t)dnodePort->valueint; - } - - ret = 0; - - dInfo("read file %s successed", file); - dnodePrintEps(eps); - -PRASE_EPS_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); - if (ret != 0) { - if (eps) free(eps); - eps = NULL; - } - - dnodeResetEps(eps); - if (eps) free(eps); - -#if 0 - dnodeUpdateEp(dnodeGetDnodeId(), tsLocalEp, tsLocalFqdn, &tsServerPort); -#else - if (dnodeCheckEpChanged(dnodeGetDnodeId(), tsLocalEp)) { - dError("dnode:%d, localEp is different from %s in dnodeEps.json and need reconfigured", dnodeGetDnodeId(), tsLocalEp); - return -1; - } -#endif - - terrno = 0; - return 0; -} - -static int32_t dnodeWriteEps() { - char file[TSDB_FILENAME_LEN + 20] = {0}; - sprintf(file, "%s/dnodeEps.json", tsDnodeDir); - - FILE *fp = fopen(file, "w"); - if (!fp) { - dError("failed to write %s, reason:%s", file, strerror(errno)); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 30000; - char * content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"dnodeNum\": %d,\n", tsEps->dnodeNum); - len += snprintf(content + len, maxLen - len, " \"dnodeInfos\": [{\n"); - for (int32_t i = 0; i < tsEps->dnodeNum; ++i) { - SDnodeEp *ep = &tsEps->dnodeEps[i]; - len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", ep->dnodeId); - len += snprintf(content + len, maxLen - len, " \"dnodeFqdn\": \"%s\",\n", ep->dnodeFqdn); - len += snprintf(content + len, maxLen - len, " \"dnodePort\": %u\n", ep->dnodePort); - if (i < tsEps->dnodeNum - 1) { - len += snprintf(content + len, maxLen - len, " },{\n"); - } else { - len += snprintf(content + len, maxLen - len, " }]\n"); - } - } - len += snprintf(content + len, maxLen - len, "}\n"); - - fwrite(content, 1, len, fp); - taosFsync(fileno(fp)); - fclose(fp); - free(content); - terrno = 0; - - dInfo("successed to write %s", file); - return 0; -} diff --git a/src/dnode/src/dnodeMInfos.c b/src/dnode/src/dnodeMInfos.c deleted file mode 100644 index 611c30b843..0000000000 --- a/src/dnode/src/dnodeMInfos.c +++ /dev/null @@ -1,326 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "cJSON.h" -#include "mnode.h" -#include "dnodeMInfos.h" - -static SMInfos tsMInfos; -static SRpcEpSet tsMEpSet; -static pthread_mutex_t tsMInfosMutex; - -static void dnodeResetMInfos(SMInfos *minfos); -static void dnodePrintMInfos(SMInfos *minfos); -static int32_t dnodeReadMInfos(); -static int32_t dnodeWriteMInfos(); - -int32_t dnodeInitMInfos() { - pthread_mutex_init(&tsMInfosMutex, NULL); - dnodeResetMInfos(NULL); - int32_t ret = dnodeReadMInfos(); - if (ret == 0) { - dInfo("dnode minfos is initialized"); - } - - return ret; -} - -void dnodeCleanupMInfos() { pthread_mutex_destroy(&tsMInfosMutex); } - -void dnodeUpdateMInfos(SMInfos *pMinfos) { - if (pMinfos->mnodeNum <= 0 || pMinfos->mnodeNum > 3) { - dError("invalid mnode infos, mnodeNum:%d", pMinfos->mnodeNum); - return; - } - - for (int32_t i = 0; i < pMinfos->mnodeNum; ++i) { - SMInfo *minfo = &pMinfos->mnodeInfos[i]; - minfo->mnodeId = htonl(minfo->mnodeId); - if (minfo->mnodeId <= 0 || strlen(minfo->mnodeEp) <= 5) { - dError("invalid mnode info:%d, mnodeId:%d mnodeEp:%s", i, minfo->mnodeId, minfo->mnodeEp); - return; - } - } - - pthread_mutex_lock(&tsMInfosMutex); - if (pMinfos->mnodeNum != tsMInfos.mnodeNum) { - dnodeResetMInfos(pMinfos); - dnodeWriteMInfos(); - sdbUpdateAsync(); - } else { - int32_t size = sizeof(SMInfos); - if (memcmp(pMinfos, &tsMInfos, size) != 0) { - dnodeResetMInfos(pMinfos); - dnodeWriteMInfos(); - sdbUpdateAsync(); - } - } - pthread_mutex_unlock(&tsMInfosMutex); -} - -void dnodeUpdateEpSetForPeer(SRpcEpSet *ep) { - if (ep->numOfEps <= 0) { - dError("minfos is changed, but content is invalid, discard it"); - return; - } - - pthread_mutex_lock(&tsMInfosMutex); - dInfo("minfos is changed, numOfEps:%d inUse:%d", ep->numOfEps, ep->inUse); - for (int32_t i = 0; i < ep->numOfEps; ++i) { - ep->port[i] -= TSDB_PORT_DNODEDNODE; - dInfo("minfo:%d %s:%u", i, ep->fqdn[i], ep->port[i]); - } - tsMEpSet = *ep; - pthread_mutex_unlock(&tsMInfosMutex); -} - -bool dnodeIsMasterEp(char *ep) { - pthread_mutex_lock(&tsMInfosMutex); - bool isMaster = strcmp(ep, tsMInfos.mnodeInfos[tsMEpSet.inUse].mnodeEp) == 0; - pthread_mutex_unlock(&tsMInfosMutex); - - return isMaster; -} - -void dnodeGetMInfos(SMInfos *pMinfos) { - pthread_mutex_lock(&tsMInfosMutex); - memcpy(pMinfos, &tsMInfos, sizeof(SMInfos)); - for (int32_t i = 0; i < tsMInfos.mnodeNum; ++i) { - pMinfos->mnodeInfos[i].mnodeId = htonl(tsMInfos.mnodeInfos[i].mnodeId); - } - pthread_mutex_unlock(&tsMInfosMutex); -} - -void dnodeGetEpSetForPeer(SRpcEpSet *epSet) { - pthread_mutex_lock(&tsMInfosMutex); - *epSet = tsMEpSet; - for (int32_t i = 0; i < epSet->numOfEps; ++i) { - epSet->port[i] += TSDB_PORT_DNODEDNODE; - } - pthread_mutex_unlock(&tsMInfosMutex); -} - -void dnodeGetEpSetForShell(SRpcEpSet *epSet) { - pthread_mutex_lock(&tsMInfosMutex); - *epSet = tsMEpSet; - pthread_mutex_unlock(&tsMInfosMutex); -} - -static void dnodePrintMInfos(SMInfos *pMinfos) { - dInfo("print minfos, mnodeNum:%d inUse:%d", pMinfos->mnodeNum, pMinfos->inUse); - for (int32_t i = 0; i < pMinfos->mnodeNum; i++) { - dInfo("mnode index:%d, %s", pMinfos->mnodeInfos[i].mnodeId, pMinfos->mnodeInfos[i].mnodeEp); - } -} - -static void dnodeResetMInfos(SMInfos *pMinfos) { - if (pMinfos == NULL) { - tsMEpSet.numOfEps = 1; - taosGetFqdnPortFromEp(tsFirst, tsMEpSet.fqdn[0], &tsMEpSet.port[0]); - - if (strcmp(tsSecond, tsFirst) != 0) { - tsMEpSet.numOfEps = 2; - taosGetFqdnPortFromEp(tsSecond, tsMEpSet.fqdn[1], &tsMEpSet.port[1]); - } - return; - } - - if (pMinfos->mnodeNum == 0) return; - - int32_t size = sizeof(SMInfos); - memcpy(&tsMInfos, pMinfos, size); - - tsMEpSet.inUse = tsMInfos.inUse; - tsMEpSet.numOfEps = tsMInfos.mnodeNum; - for (int32_t i = 0; i < tsMInfos.mnodeNum; i++) { - taosGetFqdnPortFromEp(tsMInfos.mnodeInfos[i].mnodeEp, tsMEpSet.fqdn[i], &tsMEpSet.port[i]); - } - - dnodePrintMInfos(pMinfos); -} - -static int32_t dnodeReadMInfos() { - int32_t len = 0; - int32_t maxLen = 2000; - char * content = calloc(1, maxLen + 1); - cJSON * root = NULL; - FILE * fp = NULL; - SMInfos minfos = {0}; - bool nodeChanged = false; - - char file[TSDB_FILENAME_LEN + 20] = {0}; - sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir); - - fp = fopen(file, "r"); - if (!fp) { - dDebug("failed to read %s, file not exist", file); - goto PARSE_MINFOS_OVER; - } - - len = (int32_t)fread(content, 1, maxLen, fp); - if (len <= 0) { - dError("failed to read %s, content is null", file); - goto PARSE_MINFOS_OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s, invalid json format", file); - goto PARSE_MINFOS_OVER; - } - - cJSON *inUse = cJSON_GetObjectItem(root, "inUse"); - if (!inUse || inUse->type != cJSON_Number) { - dError("failed to read mnodeEpSet.json, inUse not found"); - goto PARSE_MINFOS_OVER; - } - tsMInfos.inUse = (int8_t)inUse->valueint; - - cJSON *nodeNum = cJSON_GetObjectItem(root, "nodeNum"); - if (!nodeNum || nodeNum->type != cJSON_Number) { - dError("failed to read mnodeEpSet.json, nodeNum not found"); - goto PARSE_MINFOS_OVER; - } - minfos.mnodeNum = (int8_t)nodeNum->valueint; - - cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); - if (!nodeInfos || nodeInfos->type != cJSON_Array) { - dError("failed to read mnodeEpSet.json, nodeInfos not found"); - goto PARSE_MINFOS_OVER; - } - - int32_t size = cJSON_GetArraySize(nodeInfos); - if (size != minfos.mnodeNum) { - dError("failed to read mnodeEpSet.json, nodeInfos size not matched"); - goto PARSE_MINFOS_OVER; - } - - for (int32_t i = 0; i < size; ++i) { - cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i); - if (nodeInfo == NULL) continue; - - cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId"); - if (!nodeId || nodeId->type != cJSON_Number) { - dError("failed to read mnodeEpSet.json, nodeId not found"); - goto PARSE_MINFOS_OVER; - } - - cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); - if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { - dError("failed to read mnodeEpSet.json, nodeName not found"); - goto PARSE_MINFOS_OVER; - } - - SMInfo *pMinfo = &minfos.mnodeInfos[i]; - pMinfo->mnodeId = (int32_t)nodeId->valueint; - tstrncpy(pMinfo->mnodeEp, nodeEp->valuestring, TSDB_EP_LEN); - - bool changed = dnodeCheckEpChanged(pMinfo->mnodeId, pMinfo->mnodeEp); - if (changed) nodeChanged = changed; - } - - dInfo("read file %s successed", file); - dnodePrintMInfos(&minfos); - -PARSE_MINFOS_OVER: - if (content != NULL) free(content); - if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); - terrno = 0; - - for (int32_t i = 0; i < minfos.mnodeNum; ++i) { - SMInfo *mInfo = &minfos.mnodeInfos[i]; - dnodeUpdateEp(mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL); - } - dnodeResetMInfos(&minfos); - - if (nodeChanged) { - dnodeWriteMInfos(); - } - - return 0; -} - -static int32_t dnodeWriteMInfos() { - char file[TSDB_FILENAME_LEN + 20] = {0}; - sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir); - - FILE *fp = fopen(file, "w"); - if (!fp) { - dError("failed to write %s, reason:%s", file, strerror(errno)); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 2000; - char * content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"inUse\": %d,\n", tsMInfos.inUse); - len += snprintf(content + len, maxLen - len, " \"nodeNum\": %d,\n", tsMInfos.mnodeNum); - len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); - for (int32_t i = 0; i < tsMInfos.mnodeNum; i++) { - len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsMInfos.mnodeInfos[i].mnodeId); - len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", tsMInfos.mnodeInfos[i].mnodeEp); - if (i < tsMInfos.mnodeNum - 1) { - len += snprintf(content + len, maxLen - len, " },{\n"); - } else { - len += snprintf(content + len, maxLen - len, " }]\n"); - } - } - len += snprintf(content + len, maxLen - len, "}\n"); - - fwrite(content, 1, len, fp); - taosFsync(fileno(fp)); - fclose(fp); - free(content); - terrno = 0; - - dInfo("successed to write %s", file); - return 0; -} - -void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) { - SRpcConnInfo connInfo = {0}; - rpcGetConnInfo(rpcMsg->handle, &connInfo); - - SRpcEpSet epSet = {0}; - if (forShell) { - dnodeGetEpSetForShell(&epSet); - } else { - dnodeGetEpSetForPeer(&epSet); - } - - dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType], - taosIpStr(connInfo.clientIp), connInfo.user, epSet.numOfEps, epSet.inUse); - - for (int32_t i = 0; i < epSet.numOfEps; ++i) { - dDebug("mnode index:%d %s:%d", i, epSet.fqdn[i], epSet.port[i]); - if (strcmp(epSet.fqdn[i], tsLocalFqdn) == 0) { - if ((epSet.port[i] == tsServerPort + TSDB_PORT_DNODEDNODE && !forShell) || - (epSet.port[i] == tsServerPort && forShell)) { - epSet.inUse = (i + 1) % epSet.numOfEps; - dDebug("mnode index:%d %s:%d set inUse to %d", i, epSet.fqdn[i], epSet.port[i], epSet.inUse); - } - } - - epSet.port[i] = htons(epSet.port[i]); - } - - rpcSendRedirectRsp(rpcMsg->handle, &epSet); -} \ No newline at end of file diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c deleted file mode 100644 index 8aa28d1618..0000000000 --- a/src/dnode/src/dnodeMPeer.c +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tqueue.h" -#include "twal.h" -#include "mnode.h" -#include "dnodeVMgmt.h" -#include "dnodeMInfos.h" -#include "dnodeMWrite.h" - -typedef struct { - pthread_t thread; - int32_t workerId; -} SMPeerWorker; - -typedef struct { - int32_t curNum; - int32_t maxNum; - SMPeerWorker *worker; -} SMPeerWorkerPool; - -static SMPeerWorkerPool tsMPeerWP; -static taos_qset tsMPeerQset; -static taos_queue tsMPeerQueue; - -static void *dnodeProcessMPeerQueue(void *param); - -int32_t dnodeInitMPeer() { - tsMPeerQset = taosOpenQset(); - - tsMPeerWP.maxNum = 1; - tsMPeerWP.curNum = 0; - tsMPeerWP.worker = (SMPeerWorker *)calloc(sizeof(SMPeerWorker), tsMPeerWP.maxNum); - - if (tsMPeerWP.worker == NULL) return -1; - for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) { - SMPeerWorker *pWorker = tsMPeerWP.worker + i; - pWorker->workerId = i; - dDebug("dnode mpeer worker:%d is created", i); - } - - dDebug("dnode mpeer is initialized, workers:%d qset:%p", tsMPeerWP.maxNum, tsMPeerQset); - return 0; -} - -void dnodeCleanupMPeer() { - for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) { - SMPeerWorker *pWorker = tsMPeerWP.worker + i; - if (taosCheckPthreadValid(pWorker->thread)) { - taosQsetThreadResume(tsMPeerQset); - } - dDebug("dnode mpeer worker:%d is closed", i); - } - - for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) { - SMPeerWorker *pWorker = tsMPeerWP.worker + i; - dDebug("dnode mpeer worker:%d start to join", i); - if (taosCheckPthreadValid(pWorker->thread)) { - pthread_join(pWorker->thread, NULL); - } - dDebug("dnode mpeer worker:%d join success", i); - } - - dDebug("dnode mpeer is closed, qset:%p", tsMPeerQset); - - taosCloseQset(tsMPeerQset); - tsMPeerQset = NULL; - tfree(tsMPeerWP.worker); -} - -int32_t dnodeAllocateMPeerQueue() { - tsMPeerQueue = taosOpenQueue(); - if (tsMPeerQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY; - - taosAddIntoQset(tsMPeerQset, tsMPeerQueue, NULL); - - for (int32_t i = tsMPeerWP.curNum; i < tsMPeerWP.maxNum; ++i) { - SMPeerWorker *pWorker = tsMPeerWP.worker + i; - pWorker->workerId = i; - - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - - if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMPeerQueue, pWorker) != 0) { - dError("failed to create thread to process mpeer queue, reason:%s", strerror(errno)); - } - - pthread_attr_destroy(&thAttr); - - tsMPeerWP.curNum = i + 1; - dDebug("dnode mpeer worker:%d is launched, total:%d", pWorker->workerId, tsMPeerWP.maxNum); - } - - dDebug("dnode mpeer queue:%p is allocated", tsMPeerQueue); - return TSDB_CODE_SUCCESS; -} - -void dnodeFreeMPeerQueue() { - dDebug("dnode mpeer queue:%p is freed", tsMPeerQueue); - taosCloseQueue(tsMPeerQueue); - tsMPeerQueue = NULL; -} - -void dnodeDispatchToMPeerQueue(SRpcMsg *pMsg) { - if (!mnodeIsRunning() || tsMPeerQueue == NULL) { - dnodeSendRedirectMsg(pMsg, false); - } else { - SMnodeMsg *pPeer = mnodeCreateMsg(pMsg); - taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer); - } - - rpcFreeCont(pMsg->pCont); -} - -static void dnodeFreeMPeerMsg(SMnodeMsg *pPeer) { - mnodeCleanupMsg(pPeer); - taosFreeQitem(pPeer); -} - -static void dnodeSendRpcMPeerRsp(SMnodeMsg *pPeer, int32_t code) { - if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return; - - SRpcMsg rpcRsp = { - .handle = pPeer->rpcMsg.handle, - .pCont = pPeer->rpcRsp.rsp, - .contLen = pPeer->rpcRsp.len, - .code = code, - }; - - rpcSendResponse(&rpcRsp); - dnodeFreeMPeerMsg(pPeer); -} - -static void *dnodeProcessMPeerQueue(void *param) { - SMnodeMsg *pPeerMsg; - int32_t type; - void * unUsed; - - setThreadName("dnodeMPeerQ"); - - while (1) { - if (taosReadQitemFromQset(tsMPeerQset, &type, (void **)&pPeerMsg, &unUsed) == 0) { - dDebug("qset:%p, mnode peer got no message from qset, exiting", tsMPeerQset); - break; - } - - dTrace("msg:%s will be processed in mpeer queue", taosMsg[pPeerMsg->rpcMsg.msgType]); - int32_t code = mnodeProcessPeerReq(pPeerMsg); - dnodeSendRpcMPeerRsp(pPeerMsg, code); - } - - return NULL; -} diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c deleted file mode 100644 index 184a6b743a..0000000000 --- a/src/dnode/src/dnodeMRead.c +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tqueue.h" -#include "mnode.h" -#include "dnodeMInfos.h" -#include "dnodeMRead.h" - -typedef struct { - pthread_t thread; - int32_t workerId; -} SMReadWorker; - -typedef struct { - int32_t curNum; - int32_t maxNum; - SMReadWorker *worker; -} SMReadWorkerPool; - -static SMReadWorkerPool tsMReadWP; -static taos_qset tsMReadQset; -static taos_queue tsMReadQueue; - -static void *dnodeProcessMReadQueue(void *param); - -int32_t dnodeInitMRead() { - tsMReadQset = taosOpenQset(); - - tsMReadWP.maxNum = (int32_t)(tsNumOfCores * tsNumOfThreadsPerCore / 2); - tsMReadWP.maxNum = MAX(2, tsMReadWP.maxNum); - tsMReadWP.maxNum = MIN(4, tsMReadWP.maxNum); - tsMReadWP.curNum = 0; - tsMReadWP.worker = (SMReadWorker *)calloc(sizeof(SMReadWorker), tsMReadWP.maxNum); - - if (tsMReadWP.worker == NULL) return -1; - for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) { - SMReadWorker *pWorker = tsMReadWP.worker + i; - pWorker->workerId = i; - dDebug("dnode mread worker:%d is created", i); - } - - dDebug("dnode mread is initialized, workers:%d qset:%p", tsMReadWP.maxNum, tsMReadQset); - return 0; -} - -void dnodeCleanupMRead() { - for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) { - SMReadWorker *pWorker = tsMReadWP.worker + i; - if (taosCheckPthreadValid(pWorker->thread)) { - taosQsetThreadResume(tsMReadQset); - } - dDebug("dnode mread worker:%d is closed", i); - } - - for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) { - SMReadWorker *pWorker = tsMReadWP.worker + i; - dDebug("dnode mread worker:%d start to join", i); - if (taosCheckPthreadValid(pWorker->thread)) { - pthread_join(pWorker->thread, NULL); - } - dDebug("dnode mread worker:%d start to join", i); - } - - dDebug("dnode mread is closed, qset:%p", tsMReadQset); - - taosCloseQset(tsMReadQset); - tsMReadQset = NULL; - free(tsMReadWP.worker); -} - -int32_t dnodeAllocMReadQueue() { - tsMReadQueue = taosOpenQueue(); - if (tsMReadQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY; - - taosAddIntoQset(tsMReadQset, tsMReadQueue, NULL); - - for (int32_t i = tsMReadWP.curNum; i < tsMReadWP.maxNum; ++i) { - SMReadWorker *pWorker = tsMReadWP.worker + i; - pWorker->workerId = i; - - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - - if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMReadQueue, pWorker) != 0) { - dError("failed to create thread to process mread queue, reason:%s", strerror(errno)); - } - - pthread_attr_destroy(&thAttr); - tsMReadWP.curNum = i + 1; - dDebug("dnode mread worker:%d is launched, total:%d", pWorker->workerId, tsMReadWP.maxNum); - } - - dDebug("dnode mread queue:%p is allocated", tsMReadQueue); - return TSDB_CODE_SUCCESS; -} - -void dnodeFreeMReadQueue() { - dDebug("dnode mread queue:%p is freed", tsMReadQueue); - taosCloseQueue(tsMReadQueue); - tsMReadQueue = NULL; -} - -void dnodeDispatchToMReadQueue(SRpcMsg *pMsg) { - if (!mnodeIsRunning() || tsMReadQueue == NULL) { - dnodeSendRedirectMsg(pMsg, true); - } else { - SMnodeMsg *pRead = mnodeCreateMsg(pMsg); - taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead); - } - - rpcFreeCont(pMsg->pCont); -} - -static void dnodeFreeMReadMsg(SMnodeMsg *pRead) { - mnodeCleanupMsg(pRead); - taosFreeQitem(pRead); -} - -static void dnodeSendRpcMReadRsp(SMnodeMsg *pRead, int32_t code) { - if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return; - if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) { - // may be a auto create req, should put into write queue - dnodeReprocessMWriteMsg(pRead); - return; - } - - SRpcMsg rpcRsp = { - .handle = pRead->rpcMsg.handle, - .pCont = pRead->rpcRsp.rsp, - .contLen = pRead->rpcRsp.len, - .code = code, - }; - - rpcSendResponse(&rpcRsp); - dnodeFreeMReadMsg(pRead); -} - -static void *dnodeProcessMReadQueue(void *param) { - SMnodeMsg *pRead; - int32_t type; - void * unUsed; - - setThreadName("dnodeMReadQ"); - - while (1) { - if (taosReadQitemFromQset(tsMReadQset, &type, (void **)&pRead, &unUsed) == 0) { - dDebug("qset:%p, mnode read got no message from qset, exiting", tsMReadQset); - break; - } - - dTrace("msg:%p, app:%p type:%s will be processed in mread queue", pRead->rpcMsg.ahandle, pRead, - taosMsg[pRead->rpcMsg.msgType]); - int32_t code = mnodeProcessRead(pRead); - dnodeSendRpcMReadRsp(pRead, code); - } - - return NULL; -} diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c deleted file mode 100644 index 904ddc21d0..0000000000 --- a/src/dnode/src/dnodeMWrite.c +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "ttimer.h" -#include "tqueue.h" -#include "mnode.h" -#include "dnodeMInfos.h" -#include "dnodeMWrite.h" - -typedef struct { - pthread_t thread; - int32_t workerId; -} SMWriteWorker; - -typedef struct { - int32_t curNum; - int32_t maxNum; - SMWriteWorker *worker; -} SMWriteWorkerPool; - -static SMWriteWorkerPool tsMWriteWP; -static taos_qset tsMWriteQset; -static taos_queue tsMWriteQueue; -extern void * tsDnodeTmr; - -static void *dnodeProcessMWriteQueue(void *param); - -int32_t dnodeInitMWrite() { - tsMWriteQset = taosOpenQset(); - - tsMWriteWP.maxNum = 1; - tsMWriteWP.curNum = 0; - tsMWriteWP.worker = (SMWriteWorker *)calloc(sizeof(SMWriteWorker), tsMWriteWP.maxNum); - - if (tsMWriteWP.worker == NULL) return -1; - for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) { - SMWriteWorker *pWorker = tsMWriteWP.worker + i; - pWorker->workerId = i; - dDebug("dnode mwrite worker:%d is created", i); - } - - dDebug("dnode mwrite is initialized, workers:%d qset:%p", tsMWriteWP.maxNum, tsMWriteQset); - return 0; -} - -void dnodeCleanupMWrite() { - for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) { - SMWriteWorker *pWorker = tsMWriteWP.worker + i; - if (taosCheckPthreadValid(pWorker->thread)) { - taosQsetThreadResume(tsMWriteQset); - } - dDebug("dnode mwrite worker:%d is closed", i); - } - - for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) { - SMWriteWorker *pWorker = tsMWriteWP.worker + i; - dDebug("dnode mwrite worker:%d start to join", i); - if (taosCheckPthreadValid(pWorker->thread)) { - pthread_join(pWorker->thread, NULL); - } - dDebug("dnode mwrite worker:%d join success", i); - } - - dDebug("dnode mwrite is closed, qset:%p", tsMWriteQset); - - taosCloseQset(tsMWriteQset); - tsMWriteQset = NULL; - tfree(tsMWriteWP.worker); -} - -int32_t dnodeAllocMWritequeue() { - tsMWriteQueue = taosOpenQueue(); - if (tsMWriteQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY; - - taosAddIntoQset(tsMWriteQset, tsMWriteQueue, NULL); - - for (int32_t i = tsMWriteWP.curNum; i < tsMWriteWP.maxNum; ++i) { - SMWriteWorker *pWorker = tsMWriteWP.worker + i; - pWorker->workerId = i; - - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - - if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMWriteQueue, pWorker) != 0) { - dError("failed to create thread to process mwrite queue, reason:%s", strerror(errno)); - } - - pthread_attr_destroy(&thAttr); - tsMWriteWP.curNum = i + 1; - dDebug("dnode mwrite worker:%d is launched, total:%d", pWorker->workerId, tsMWriteWP.maxNum); - } - - dDebug("dnode mwrite queue:%p is allocated", tsMWriteQueue); - return TSDB_CODE_SUCCESS; -} - -void dnodeFreeMWritequeue() { - dDebug("dnode mwrite queue:%p is freed", tsMWriteQueue); - taosCloseQueue(tsMWriteQueue); - tsMWriteQueue = NULL; -} - -void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) { - if (!mnodeIsRunning() || tsMWriteQueue == NULL) { - dnodeSendRedirectMsg(pMsg, true); - } else { - SMnodeMsg *pWrite = mnodeCreateMsg(pMsg); - dTrace("msg:%p, app:%p type:%s is put into mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle, - taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); - taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); - } - - rpcFreeCont(pMsg->pCont); -} - -static void dnodeFreeMWriteMsg(SMnodeMsg *pWrite) { - dTrace("msg:%p, app:%p type:%s is freed from mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle, - taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); - - mnodeCleanupMsg(pWrite); - taosFreeQitem(pWrite); -} - -void dnodeSendRpcMWriteRsp(void *pMsg, int32_t code) { - SMnodeMsg *pWrite = pMsg; - if (pWrite == NULL) return; - if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return; - if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) { - dnodeReprocessMWriteMsg(pWrite); - return; - } - - dTrace("msg:%p, app:%p type:%s master:%p will be responsed", pWrite, pWrite->rpcMsg.ahandle, - taosMsg[pWrite->rpcMsg.msgType], pWrite->pBatchMasterMsg); - if (pWrite->pBatchMasterMsg && pWrite != pWrite->pBatchMasterMsg) { - dError("msg:%p, app:%p type:%s master:%p sub message should not response!", pWrite, pWrite->rpcMsg.ahandle, - taosMsg[pWrite->rpcMsg.msgType], pWrite->pBatchMasterMsg); - return; - } - - SRpcMsg rpcRsp = { - .handle = pWrite->rpcMsg.handle, - .pCont = pWrite->rpcRsp.rsp, - .contLen = pWrite->rpcRsp.len, - .code = code, - }; - - rpcSendResponse(&rpcRsp); - dnodeFreeMWriteMsg(pWrite); -} - -static void *dnodeProcessMWriteQueue(void *param) { - SMnodeMsg *pWrite; - int32_t type; - void * unUsed; - - setThreadName("dnodeMWriteQ"); - - while (1) { - if (taosReadQitemFromQset(tsMWriteQset, &type, (void **)&pWrite, &unUsed) == 0) { - dDebug("qset:%p, mnode write got no message from qset, exiting", tsMWriteQset); - break; - } - - dTrace("msg:%p, app:%p type:%s will be processed in mwrite queue", pWrite, pWrite->rpcMsg.ahandle, - taosMsg[pWrite->rpcMsg.msgType]); - - int32_t code = mnodeProcessWrite(pWrite); - dnodeSendRpcMWriteRsp(pWrite, code); - } - - return NULL; -} - -void dnodeReprocessMWriteMsg(void *pMsg) { - SMnodeMsg *pWrite = pMsg; - - if (!mnodeIsRunning() || tsMWriteQueue == NULL) { - dDebug("msg:%p, app:%p type:%s is redirected for mnode not running, retry times:%d", pWrite, pWrite->rpcMsg.ahandle, - taosMsg[pWrite->rpcMsg.msgType], pWrite->retry); - - if (pWrite->pBatchMasterMsg) { - ++pWrite->pBatchMasterMsg->received; - if (pWrite->pBatchMasterMsg->successed + pWrite->pBatchMasterMsg->received - >= pWrite->pBatchMasterMsg->expected) { - dnodeSendRedirectMsg(&pWrite->pBatchMasterMsg->rpcMsg, true); - dnodeFreeMWriteMsg(pWrite->pBatchMasterMsg); - } - - mnodeDestroySubMsg(pWrite); - - return; - } - dnodeSendRedirectMsg(&pWrite->rpcMsg, true); - dnodeFreeMWriteMsg(pWrite); - } else { - dDebug("msg:%p, app:%p type:%s is reput into mwrite queue:%p, retry times:%d", pWrite, pWrite->rpcMsg.ahandle, - taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue, pWrite->retry); - - taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); - } -} - -static void dnodeDoDelayReprocessMWriteMsg(void *param, void *tmrId) { - dnodeReprocessMWriteMsg(param); -} - -void dnodeDelayReprocessMWriteMsg(void *pMsg) { - SMnodeMsg *mnodeMsg = pMsg; - void *unUsed = NULL; - taosTmrReset(dnodeDoDelayReprocessMWriteMsg, 300, mnodeMsg, tsDnodeTmr, &unUsed); -} diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c deleted file mode 100644 index c6f6e976f6..0000000000 --- a/src/dnode/src/dnodeMain.c +++ /dev/null @@ -1,337 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taos.h" -#include "tnote.h" -#include "ttimer.h" -#include "tconfig.h" -#include "tfile.h" -#include "twal.h" -#include "tfs.h" -#include "tsync.h" -#include "dnodeStep.h" -#include "dnodePeer.h" -#include "dnodeModule.h" -#include "dnodeEps.h" -#include "dnodeMInfos.h" -#include "dnodeCfg.h" -#include "dnodeCheck.h" -#include "dnodeVRead.h" -#include "dnodeVWrite.h" -#include "dnodeVMgmt.h" -#include "dnodeVnodes.h" -#include "dnodeMRead.h" -#include "dnodeMWrite.h" -#include "dnodeMPeer.h" -#include "dnodeShell.h" -#include "dnodeTelemetry.h" -#include "module.h" -#include "mnode.h" -#include "qScript.h" -#include "tcache.h" -#include "tscompression.h" - -#if !defined(_MODULE) || !defined(_TD_LINUX) -int32_t moduleStart() { return 0; } -void moduleStop() {} -#endif - - -void *tsDnodeTmr = NULL; -static SRunStatus tsRunStatus = TSDB_RUN_STATUS_STOPPED; - -static int32_t dnodeInitStorage(); -static void dnodeCleanupStorage(); -static void dnodeSetRunStatus(SRunStatus status); -static void dnodeCheckDataDirOpenned(char *dir); -static int dnodeCreateDir(const char *dir); - -static SStep tsDnodeSteps[] = { - {"dnode-tfile", tfInit, tfCleanup}, - {"dnode-rpc", rpcInit, rpcCleanup}, - {"dnode-globalcfg", taosCheckGlobalCfg, NULL}, - {"dnode-storage", dnodeInitStorage, dnodeCleanupStorage}, - {"dnode-cfg", dnodeInitCfg, dnodeCleanupCfg}, - {"dnode-eps", dnodeInitEps, dnodeCleanupEps}, - {"dnode-minfos", dnodeInitMInfos, dnodeCleanupMInfos}, - {"dnode-wal", walInit, walCleanUp}, - {"dnode-sync", syncInit, syncCleanUp}, - {"dnode-check", dnodeInitCheck, dnodeCleanupCheck}, // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!! - {"dnode-vread", dnodeInitVRead, dnodeCleanupVRead}, - {"dnode-vwrite", dnodeInitVWrite, dnodeCleanupVWrite}, - {"dnode-vmgmt", dnodeInitVMgmt, dnodeCleanupVMgmt}, - {"dnode-mread", dnodeInitMRead, NULL}, - {"dnode-mwrite", dnodeInitMWrite, NULL}, - {"dnode-mpeer", dnodeInitMPeer, NULL}, - {"dnode-client", dnodeInitClient, dnodeCleanupClient}, - {"dnode-server", dnodeInitServer, dnodeCleanupServer}, - {"dnode-vnodes", dnodeInitVnodes, dnodeCleanupVnodes}, - {"dnode-modules", dnodeInitModules, dnodeCleanupModules}, - {"dnode-mread", NULL, dnodeCleanupMRead}, - {"dnode-mwrite", NULL, dnodeCleanupMWrite}, - {"dnode-mpeer", NULL, dnodeCleanupMPeer}, - {"dnode-shell", dnodeInitShell, dnodeCleanupShell}, - {"dnode-statustmr", dnodeInitStatusTimer,dnodeCleanupStatusTimer}, - {"dnode-telemetry", dnodeInitTelemetry, dnodeCleanupTelemetry}, - {"dnode-script", scriptEnvPoolInit, scriptEnvPoolCleanup}, -}; - -static SStep tsDnodeCompactSteps[] = { - {"dnode-tfile", tfInit, tfCleanup}, - {"dnode-globalcfg", taosCheckGlobalCfg, NULL}, - {"dnode-storage", dnodeInitStorage, dnodeCleanupStorage}, - {"dnode-cfg", dnodeInitCfg, dnodeCleanupCfg}, - {"dnode-eps", dnodeInitEps, dnodeCleanupEps}, - {"dnode-minfos", dnodeInitMInfos, dnodeCleanupMInfos}, - {"dnode-wal", walInit, walCleanUp}, - {"dnode-sync", syncInit, syncCleanUp}, - {"dnode-vread", dnodeInitVRead, dnodeCleanupVRead}, - {"dnode-vwrite", dnodeInitVWrite, dnodeCleanupVWrite}, - {"dnode-vmgmt", dnodeInitVMgmt, dnodeCleanupVMgmt}, - {"dnode-mread", dnodeInitMRead, NULL}, - {"dnode-mwrite", dnodeInitMWrite, NULL}, - {"dnode-mpeer", dnodeInitMPeer, NULL}, - {"dnode-vnodes", dnodeInitVnodes, dnodeCleanupVnodes}, - {"dnode-modules", dnodeInitModules, dnodeCleanupModules}, - {"dnode-mread", NULL, dnodeCleanupMRead}, - {"dnode-mwrite", NULL, dnodeCleanupMWrite}, - {"dnode-mpeer", NULL, dnodeCleanupMPeer}, -}; - -static int dnodeCreateDir(const char *dir) { - if (mkdir(dir, 0755) != 0 && errno != EEXIST) { - return -1; - } - - return 0; -} - -static void dnodeCleanupComponents() { - if (!tsCompactMnodeWal) { - int32_t stepSize = sizeof(tsDnodeSteps) / sizeof(SStep); - dnodeStepCleanup(tsDnodeSteps, stepSize); - } else { - int32_t stepSize = sizeof(tsDnodeCompactSteps) / sizeof(SStep); - dnodeStepCleanup(tsDnodeCompactSteps, stepSize); - } -} - -static int32_t dnodeInitComponents() { - if (!tsCompactMnodeWal) { - int32_t stepSize = sizeof(tsDnodeSteps) / sizeof(SStep); - return dnodeStepInit(tsDnodeSteps, stepSize); - } else { - int32_t stepSize = sizeof(tsDnodeCompactSteps) / sizeof(SStep); - return dnodeStepInit(tsDnodeCompactSteps, stepSize); - } -} - -static int32_t dnodeInitTmr() { - tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM"); - if (tsDnodeTmr == NULL) { - dError("failed to init dnode timer"); - return -1; - } - - return 0; -} - -static void dnodeCleanupTmr() { - if (tsDnodeTmr != NULL) { - taosTmrCleanUp(tsDnodeTmr); - tsDnodeTmr = NULL; - } -} - -int32_t dnodeInitSystem() { - dnodeSetRunStatus(TSDB_RUN_STATUS_INITIALIZE); - tscEmbedded = 1; - taosIgnSIGPIPE(); - taosBlockSIGPIPE(); - taosResolveCRC(); - taosInitGlobalCfg(); - taosReadGlobalLogCfg(); - taosSetCoreDump(); - dnodeInitTmr(); - - if (dnodeCreateDir(tsLogDir) < 0) { - printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno)); - return -1; - } - - char temp[TSDB_FILENAME_LEN]; - sprintf(temp, "%s/taosdlog", tsLogDir); - if (taosInitLog(temp, tsNumOfLogLines, 1) < 0) { - printf("failed to init log file\n"); - } - - if (!taosReadGlobalCfg()) { - taosPrintGlobalCfg(); - dError("TDengine read global config failed"); - return -1; - } - - dInfo("start to initialize TDengine"); - - taosInitNotes(); - - if (dnodeInitComponents() != 0) { - return -1; - } - - dnodeSetRunStatus(TSDB_RUN_STATUS_RUNING); - moduleStart(); - - tsDnodeStartTime = taosGetTimestampMs(); - dnodeReportStep("TDengine", "initialized successfully", 1); - dInfo("TDengine is initialized successfully"); - - return 0; -} - -void dnodeCleanUpSystem() { - if (dnodeGetRunStatus() != TSDB_RUN_STATUS_STOPPED) { - moduleStop(); - dnodeSetRunStatus(TSDB_RUN_STATUS_STOPPED); - dnodeCleanupTmr(); - dnodeCleanupComponents(); - taos_cleanup(); - taosCloseLog(); - taosStopCacheRefreshWorker(); - } -} - -SRunStatus dnodeGetRunStatus() { - return tsRunStatus; -} - -static void dnodeSetRunStatus(SRunStatus status) { - tsRunStatus = status; -} - -static void dnodeCheckDataDirOpenned(char *dir) { - char filepath[256] = {0}; - sprintf(filepath, "%s/.running", dir); - - int fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); - if (fd < 0) { - dError("failed to open lock file:%s, reason: %s, quit", filepath, strerror(errno)); - exit(0); - } - int32_t ret = flock(fd, LOCK_EX | LOCK_NB); - if (ret != 0) { - dError("failed to lock file:%s ret:%d[%s], database may be running, quit", filepath, ret, strerror(errno)); - close(fd); - exit(0); - } -} - -static int32_t dnodeInitStorage() { -#ifdef TD_TSZ - // compress module init - tsCompressInit(); -#endif - - // storage module init - if (tsDiskCfgNum == 1 && dnodeCreateDir(tsDataDir) < 0) { - dError("failed to create dir: %s, reason: %s", tsDataDir, strerror(errno)); - return -1; - } - - if (tfsInit(tsDiskCfg, tsDiskCfgNum) < 0) { - dError("failed to init TFS since %s", tstrerror(terrno)); - return -1; - } - strncpy(tsDataDir, TFS_PRIMARY_PATH(), TSDB_FILENAME_LEN); - sprintf(tsMnodeDir, "%s/mnode", tsDataDir); - sprintf(tsVnodeDir, "%s/vnode", tsDataDir); - sprintf(tsDnodeDir, "%s/dnode", tsDataDir); - // sprintf(tsVnodeBakDir, "%s/vnode_bak", tsDataDir); - - if (tsCompactMnodeWal == 1) { - sprintf(tsMnodeTmpDir, "%s/mnode_tmp", tsDataDir); - if (taosDirExist(tsMnodeTmpDir)) { - dError("mnode_tmp dir already exist in %s,quit compact job", tsMnodeTmpDir); - return -1; - } - if (dnodeCreateDir(tsMnodeTmpDir) < 0) { - dError("failed to create dir: %s, reason: %s", tsMnodeTmpDir, strerror(errno)); - return -1; - } - - sprintf(tsMnodeBakDir, "%s/mnode_bak", tsDataDir); - if (taosDirExist(tsMnodeBakDir)) { - dError("mnode_bak dir already exist in %s,quit compact job", tsMnodeBakDir); - return -1; - } - } - //TODO(dengyihao): no need to init here - if (dnodeCreateDir(tsMnodeDir) < 0) { - dError("failed to create dir: %s, reason: %s", tsMnodeDir, strerror(errno)); - return -1; - } - - if (dnodeCreateDir(tsDnodeDir) < 0) { - dError("failed to create dir: %s, reason: %s", tsDnodeDir, strerror(errno)); - return -1; - } - - if (tfsMkdir("vnode") < 0) { - dError("failed to create vnode dir since %s", tstrerror(terrno)); - return -1; - } - - if (tfsMkdir("vnode_bak") < 0) { - dError("failed to create vnode_bak dir since %s", tstrerror(terrno)); - return -1; - } - - TDIR *tdir = tfsOpendir("vnode_bak/.staging"); - bool stagingNotEmpty = tfsReaddir(tdir) != NULL; - tfsClosedir(tdir); - - if (stagingNotEmpty) { - dError("vnode_bak/.staging dir not empty, fix it first."); - return -1; - } - - if (tfsMkdir("vnode_bak/.staging") < 0) { - dError("failed to create vnode_bak/.staging dir since %s", tstrerror(terrno)); - return -1; - } - - dnodeCheckDataDirOpenned(tsDnodeDir); - - taosGetDisk(); - taosPrintDiskInfo(); - dInfo("dnode storage is initialized at %s", tsDnodeDir); - return 0; -} - -static void dnodeCleanupStorage() { - // storage destroy - tfsDestroy(); - - #ifdef TD_TSZ - // compress destroy - tsCompressExit(); - #endif -} - -bool dnodeIsFirstDeploy() { - return strcmp(tsFirst, tsLocalEp) == 0; -} diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c deleted file mode 100644 index a661585b3b..0000000000 --- a/src/dnode/src/dnodeModule.c +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "mnode.h" -#include "http.h" -#include "tmqtt.h" -#include "monitor.h" -#include "dnodeModule.h" - -typedef struct { - bool enable; - char * name; - int32_t (*initFp)(); - int32_t (*startFp)(); - void (*cleanUpFp)(); - void (*stopFp)(); -} SModule; - -static SModule tsModule[TSDB_MOD_MAX] = {{0}}; -static uint32_t tsModuleStatus = 0; - -static void dnodeSetModuleStatus(int32_t module) { - tsModuleStatus |= (1 << module); -} - -static void dnodeUnSetModuleStatus(int32_t module) { - tsModuleStatus &= ~(1 << module); -} - -static void dnodeAllocModules() { - tsModule[TSDB_MOD_MNODE].enable = false; - tsModule[TSDB_MOD_MNODE].name = "mnode"; - tsModule[TSDB_MOD_MNODE].initFp = mnodeInitSystem; - tsModule[TSDB_MOD_MNODE].cleanUpFp = mnodeCleanupSystem; - tsModule[TSDB_MOD_MNODE].startFp = mnodeStartSystem; - tsModule[TSDB_MOD_MNODE].stopFp = mnodeStopSystem; - - tsModule[TSDB_MOD_HTTP].enable = (tsEnableHttpModule == 1); - tsModule[TSDB_MOD_HTTP].name = "http"; - tsModule[TSDB_MOD_HTTP].initFp = httpInitSystem; - tsModule[TSDB_MOD_HTTP].cleanUpFp = httpCleanUpSystem; - tsModule[TSDB_MOD_HTTP].startFp = httpStartSystem; - tsModule[TSDB_MOD_HTTP].stopFp = httpStopSystem; - if (tsEnableHttpModule) { - dnodeSetModuleStatus(TSDB_MOD_HTTP); - } - -#ifdef _MQTT - tsModule[TSDB_MOD_MQTT].enable = (tsEnableMqttModule == 1); - tsModule[TSDB_MOD_MQTT].name = "mqtt"; - tsModule[TSDB_MOD_MQTT].initFp = mqttInitSystem; - tsModule[TSDB_MOD_MQTT].cleanUpFp = mqttCleanUpSystem; - tsModule[TSDB_MOD_MQTT].startFp = mqttStartSystem; - tsModule[TSDB_MOD_MQTT].stopFp = mqttStopSystem; - if (tsEnableMqttModule) { - dnodeSetModuleStatus(TSDB_MOD_MQTT); - } -#endif - - tsModule[TSDB_MOD_MONITOR].enable = (tsEnableMonitorModule == 1); - tsModule[TSDB_MOD_MONITOR].name = "monitor"; - tsModule[TSDB_MOD_MONITOR].initFp = monInitSystem; - tsModule[TSDB_MOD_MONITOR].cleanUpFp = monCleanupSystem; - tsModule[TSDB_MOD_MONITOR].startFp = monStartSystem; - tsModule[TSDB_MOD_MONITOR].stopFp = monStopSystem; - if (tsEnableMonitorModule) { - dnodeSetModuleStatus(TSDB_MOD_MONITOR); - } -} - -void dnodeCleanupModules() { - for (EModuleType module = 1; module < TSDB_MOD_MAX; ++module) { - if (tsModule[module].enable && tsModule[module].stopFp) { - (*tsModule[module].stopFp)(); - } - if (tsModule[module].cleanUpFp) { - (*tsModule[module].cleanUpFp)(); - } - } - - if (tsModule[TSDB_MOD_MNODE].cleanUpFp) { - (*tsModule[TSDB_MOD_MNODE].cleanUpFp)(); - } -} - -static int32_t dnodeStartModules() { - for (EModuleType module = 1; module < TSDB_MOD_MAX; ++module) { - if (tsModule[module].enable && tsModule[module].startFp) { - int32_t code = (*tsModule[module].startFp)(); - if (code != 0) { - dError("failed to start module:%s, code:%d", tsModule[module].name, code); - return code; - } - } - } - - return 0; -} - -int32_t dnodeInitModules() { - dnodeAllocModules(); - - for (EModuleType module = 0; module < TSDB_MOD_MAX; ++module) { - if (tsModule[module].initFp) { - if ((*tsModule[module].initFp)() != 0) { - dError("failed to init module:%s", tsModule[module].name); - return -1; - } - } - } - - dInfo("dnode modules is initialized"); - return dnodeStartModules(); -} - -int32_t dnodeProcessModuleStatus(uint32_t moduleStatus) { - int32_t code = 0; - - for (int32_t module = TSDB_MOD_MNODE; module < TSDB_MOD_HTTP; ++module) { - bool enableModule = moduleStatus & (1 << module); - if (!tsModule[module].enable && enableModule) { - dInfo("module status:%u is set, start %s module", moduleStatus, tsModule[module].name); - tsModule[module].enable = true; - dnodeSetModuleStatus(module); - code = (*tsModule[module].startFp)(); - } - - if (tsModule[module].enable && !enableModule) { - dInfo("module status:%u is set, stop %s module", moduleStatus, tsModule[module].name); - tsModule[module].enable = false; - dnodeUnSetModuleStatus(module); - (*tsModule[module].stopFp)(); - } - } - - return code; -} - -int32_t dnodeStartMnode(SMInfos *pMinfos) { - if (tsModuleStatus & (1 << TSDB_MOD_MNODE)) { - dDebug("mnode module is already started, module status:%d", tsModuleStatus); - return 0; - } - - uint32_t moduleStatus = tsModuleStatus | (1 << TSDB_MOD_MNODE); - dInfo("start mnode module, module status:%d, new status:%d", tsModuleStatus, moduleStatus); - - int32_t code = dnodeProcessModuleStatus(moduleStatus); - if (code == 0) { - code = sdbUpdateSync(pMinfos); - } - - if (code != 0) { - dError("failed to start mnode module since %s", tstrerror(code)); - moduleStatus = tsModuleStatus & ~(1 << TSDB_MOD_MNODE); - dnodeProcessModuleStatus(moduleStatus); - } - - return code; -} diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c deleted file mode 100644 index 08269c0bf6..0000000000 --- a/src/dnode/src/dnodePeer.c +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -/* this file is mainly responsible for the communication between DNODEs. Each - * dnode works as both server and client. Dnode may send status, grant, config - * messages to mnode, mnode may send create/alter/drop table/vnode messages - * to dnode. All theses messages are handled from here - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "mnode.h" -#include "dnodeVMgmt.h" -#include "dnodeVWrite.h" -#include "dnodeMPeer.h" -#include "dnodeMInfos.h" -#include "dnodeStep.h" - -static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); -static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *); -static void (*dnodeProcessRspMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg); -static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet); -static void *tsServerRpc = NULL; -static void *tsClientRpc = NULL; - -int32_t dnodeInitServer() { - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = dnodeDispatchToVWriteQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE] = dnodeDispatchToVWriteQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE] = dnodeDispatchToVWriteQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = dnodeDispatchToVWriteQueue; - - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToVMgmtQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeDispatchToVMgmtQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE] = dnodeDispatchToVMgmtQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeDispatchToVMgmtQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeDispatchToVMgmtQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToVMgmtQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeDispatchToVMgmtQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE] = dnodeDispatchToVMgmtQueue; - - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMPeerQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMPeerQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH] = dnodeDispatchToMPeerQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT] = dnodeDispatchToMPeerQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS] = dnodeDispatchToMPeerQueue; - - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = tsDnodeDnodePort; - rpcInit.label = "DND-S"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = dnodeProcessReqMsgFromDnode; - rpcInit.sessions = TSDB_MAX_VNODES << 4; - rpcInit.connType = TAOS_CONN_SERVER; - rpcInit.idleTime = tsShellActivityTimer * 1000; - - tsServerRpc = rpcOpen(&rpcInit); - if (tsServerRpc == NULL) { - dError("failed to init inter-dnodes RPC server"); - return -1; - } - - dInfo("dnode inter-dnodes RPC server is initialized"); - return 0; -} - -void dnodeCleanupServer() { - if (tsServerRpc) { - rpcClose(tsServerRpc); - tsServerRpc = NULL; - dInfo("inter-dnodes RPC server is closed"); - } -} - -static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - SRpcMsg rspMsg = { - .handle = pMsg->handle, - .pCont = NULL, - .contLen = 0 - }; - - if (pMsg->pCont == NULL) return; - if (pMsg->msgType == TSDB_MSG_TYPE_NETWORK_TEST) { - dnodeSendStartupStep(pMsg); - return; - } - - if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) { - rspMsg.code = TSDB_CODE_APP_NOT_READY; - rpcSendResponse(&rspMsg); - rpcFreeCont(pMsg->pCont); - dTrace("RPC %p, msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]); - return; - } - - if (pMsg->pCont == NULL) { - rspMsg.code = TSDB_CODE_DND_INVALID_MSG_LEN; - rpcSendResponse(&rspMsg); - return; - } - - if (dnodeProcessReqMsgFp[pMsg->msgType]) { - (*dnodeProcessReqMsgFp[pMsg->msgType])(pMsg); - } else { - dDebug("RPC %p, message:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]); - rspMsg.code = TSDB_CODE_DND_MSG_NOT_PROCESSED; - rpcSendResponse(&rspMsg); - rpcFreeCont(pMsg->pCont); - } -} - -int32_t dnodeInitClient() { - char secret[TSDB_KEY_LEN] = "secret"; - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.label = "DND-C"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = dnodeProcessRspFromDnode; - rpcInit.sessions = TSDB_MAX_VNODES << 4; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.user = "t"; - rpcInit.ckey = "key"; - rpcInit.secret = secret; - - tsClientRpc = rpcOpen(&rpcInit); - if (tsClientRpc == NULL) { - dError("failed to init mnode rpc client"); - return -1; - } - - dInfo("dnode inter-dnodes rpc client is initialized"); - return 0; -} - -void dnodeCleanupClient() { - if (tsClientRpc) { - rpcClose(tsClientRpc); - tsClientRpc = NULL; - dInfo("dnode inter-dnodes rpc client is closed"); - } -} - -static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - if (dnodeGetRunStatus() == TSDB_RUN_STATUS_STOPPED) { - if (pMsg == NULL || pMsg->pCont == NULL) return; - dTrace("msg:%p is ignored since dnode is stopping", pMsg); - rpcFreeCont(pMsg->pCont); - return; - } - - if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pEpSet) { - dnodeUpdateEpSetForPeer(pEpSet); - } - - if (dnodeProcessRspMsgFp[pMsg->msgType]) { - (*dnodeProcessRspMsgFp[pMsg->msgType])(pMsg); - } else { - mnodeProcessPeerRsp(pMsg); - } - - rpcFreeCont(pMsg->pCont); -} - -void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) { - dnodeProcessRspMsgFp[msgType] = fp; -} - -void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) { - rpcSendRequest(tsClientRpc, epSet, rpcMsg, NULL); -} - -void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) { - SRpcEpSet epSet = {0}; - dnodeGetEpSetForPeer(&epSet); - - assert(tsClientRpc != 0); - rpcSendRecv(tsClientRpc, &epSet, rpcMsg, rpcRsp); -} - -void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet) { - rpcSendRecv(tsClientRpc, epSet, rpcMsg, rpcRsp); -} diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c deleted file mode 100644 index 5606681f0f..0000000000 --- a/src/dnode/src/dnodeShell.c +++ /dev/null @@ -1,249 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "http.h" -#include "mnode.h" -#include "dnodeVRead.h" -#include "dnodeVWrite.h" -#include "dnodeMRead.h" -#include "dnodeMWrite.h" -#include "dnodeShell.h" -#include "dnodeStep.h" - -static void (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); -static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *); -static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey); -static void * tsShellRpc = NULL; -static int32_t tsQueryReqNum = 0; -static int32_t tsSubmitReqNum = 0; - -int32_t dnodeInitShell() { - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT] = dnodeDispatchToVWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY] = dnodeDispatchToVReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH] = dnodeDispatchToVReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVWriteQueue; - - // the following message shall be treated as mnode write - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TP] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_FUNCTION] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SYNC_DB] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TP] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_FUNCTION] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TP] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]= dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_COMPACT_VNODE]= dnodeDispatchToMWriteQueue; - - // the following message shall be treated as mnode query - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT] = dnodeDispatchToMReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT] = dnodeDispatchToMReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB] = dnodeDispatchToMReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = dnodeDispatchToMReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = dnodeDispatchToMReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = dnodeDispatchToMReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE_FUNC] = dnodeDispatchToMReadQueue; - - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dnodeSendStartupStep; - - int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); - if (numOfThreads < 1) { - numOfThreads = 1; - } - - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = tsDnodeShellPort; - rpcInit.label = "SHELL"; - rpcInit.numOfThreads = numOfThreads; - rpcInit.cfp = dnodeProcessMsgFromShell; - rpcInit.sessions = tsMaxShellConns; - rpcInit.connType = TAOS_CONN_SERVER; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.afp = dnodeRetrieveUserAuthInfo; - - tsShellRpc = rpcOpen(&rpcInit); - if (tsShellRpc == NULL) { - dError("failed to init shell rpc server"); - return -1; - } - - dInfo("dnode shell rpc server is initialized"); - return 0; -} - -void dnodeCleanupShell() { - if (tsShellRpc) { - rpcClose(tsShellRpc); - tsShellRpc = NULL; - } -} - -static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - SRpcMsg rpcMsg = { - .handle = pMsg->handle, - .pCont = NULL, - .contLen = 0 - }; - - if (pMsg->pCont == NULL) return; - - SRunStatus dnodeStatus = dnodeGetRunStatus(); - if (dnodeStatus == TSDB_RUN_STATUS_STOPPED) { - dError("RPC %p, shell msg:%s is ignored since dnode exiting", pMsg->handle, taosMsg[pMsg->msgType]); - rpcMsg.code = TSDB_CODE_DND_EXITING; - rpcSendResponse(&rpcMsg); - rpcFreeCont(pMsg->pCont); - return; - } else if (dnodeStatus != TSDB_RUN_STATUS_RUNING) { - dError("RPC %p, shell msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]); - rpcMsg.code = TSDB_CODE_APP_NOT_READY; - rpcSendResponse(&rpcMsg); - rpcFreeCont(pMsg->pCont); - return; - } - - if (pMsg->msgType == TSDB_MSG_TYPE_QUERY) { - atomic_fetch_add_32(&tsQueryReqNum, 1); - } else if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) { - atomic_fetch_add_32(&tsSubmitReqNum, 1); - } else {} - - if ( dnodeProcessShellMsgFp[pMsg->msgType] ) { - (*dnodeProcessShellMsgFp[pMsg->msgType])(pMsg); - } else { - dError("RPC %p, shell msg:%s is not processed", pMsg->handle, taosMsg[pMsg->msgType]); - rpcMsg.code = TSDB_CODE_DND_MSG_NOT_PROCESSED; - rpcSendResponse(&rpcMsg); - rpcFreeCont(pMsg->pCont); - return; - } -} - -static int32_t dnodeAuthNettestUser(char *user, char *spi, char *encrypt, char *secret, char *ckey) { - if (strcmp(user, "nettestinternal") == 0) { - char pass[32] = {0}; - taosEncryptPass((uint8_t *)user, strlen(user), pass); - *spi = 0; - *encrypt = 0; - *ckey = 0; - memcpy(secret, pass, TSDB_KEY_LEN); - dTrace("nettest user is authorized"); - return 0; - } - - return -1; -} - -static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) { - if (dnodeAuthNettestUser(user, spi, encrypt, secret, ckey) == 0) return 0; - int code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey); - if (code != TSDB_CODE_APP_NOT_READY) return code; - - SAuthMsg *pMsg = rpcMallocCont(sizeof(SAuthMsg)); - tstrncpy(pMsg->user, user, sizeof(pMsg->user)); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pMsg; - rpcMsg.contLen = sizeof(SAuthMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_DM_AUTH; - - dDebug("user:%s, send auth msg to mnodes", user); - SRpcMsg rpcRsp = {0}; - dnodeSendMsgToMnodeRecv(&rpcMsg, &rpcRsp); - - if (rpcRsp.code != 0) { - dError("user:%s, auth msg received from mnodes, error:%s", user, tstrerror(rpcRsp.code)); - } else { - SAuthRsp *pRsp = rpcRsp.pCont; - dDebug("user:%s, auth msg received from mnodes", user); - memcpy(secret, pRsp->secret, TSDB_KEY_LEN); - memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN); - *spi = pRsp->spi; - *encrypt = pRsp->encrypt; - } - - rpcFreeCont(rpcRsp.pCont); - return rpcRsp.code; -} - -void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid) { - dDebug("vgId:%d, tid:%d send config table msg to mnode", vgId, tid); - - int32_t contLen = sizeof(SConfigTableMsg); - SConfigTableMsg *pMsg = rpcMallocCont(contLen); - - pMsg->dnodeId = htonl(dnodeGetDnodeId()); - pMsg->vgId = htonl(vgId); - pMsg->tid = htonl(tid); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pMsg; - rpcMsg.contLen = contLen; - rpcMsg.msgType = TSDB_MSG_TYPE_DM_CONFIG_TABLE; - - SRpcMsg rpcRsp = {0}; - dnodeSendMsgToMnodeRecv(&rpcMsg, &rpcRsp); - terrno = rpcRsp.code; - - if (rpcRsp.code != 0) { - rpcFreeCont(rpcRsp.pCont); - dError("vgId:%d, tid:%d failed to config table from mnode", vgId, tid); - return NULL; - } else { - dInfo("vgId:%d, tid:%d config table msg is received", vgId, tid); - - // delete this after debug finished - SMDCreateTableMsg *pTable = rpcRsp.pCont; - int16_t numOfColumns = htons(pTable->numOfColumns); - int16_t numOfTags = htons(pTable->numOfTags); - int32_t tableId = htonl(pTable->tid); - uint64_t uid = htobe64(pTable->uid); - dInfo("table:%s, numOfColumns:%d numOfTags:%d tid:%d uid:%" PRIu64, pTable->tableFname, numOfColumns, numOfTags, tableId, uid); - - return rpcRsp.pCont; - } -} - -SStatisInfo dnodeGetStatisInfo() { - SStatisInfo info = {0}; - if (dnodeGetRunStatus() == TSDB_RUN_STATUS_RUNING) { - info.httpReqNum = httpGetReqCount(); - info.queryReqNum = atomic_exchange_32(&tsQueryReqNum, 0); - info.submitReqNum = atomic_exchange_32(&tsSubmitReqNum, 0); - } - - return info; -} diff --git a/src/dnode/src/dnodeStep.c b/src/dnode/src/dnodeStep.c deleted file mode 100644 index eda6fb227d..0000000000 --- a/src/dnode/src/dnodeStep.c +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "dnodeStep.h" - -static SStartupStep tsStartupStep; - -void dnodeReportStep(char *name, char *desc, int8_t finished) { - tstrncpy(tsStartupStep.name, name, sizeof(tsStartupStep.name)); - tstrncpy(tsStartupStep.desc, desc, sizeof(tsStartupStep.desc)); - tsStartupStep.finished = finished; -} - -void dnodeSendStartupStep(SRpcMsg *pMsg) { - dInfo("nettest msg is received, cont:%s", (char *)pMsg->pCont); - - SStartupStep *pStep = rpcMallocCont(sizeof(SStartupStep)); - memcpy(pStep, &tsStartupStep, sizeof(SStartupStep)); - - dDebug("startup msg is sent, step:%s desc:%s finished:%d", pStep->name, pStep->desc, pStep->finished); - - SRpcMsg rpcRsp = {.handle = pMsg->handle, .pCont = pStep, .contLen = sizeof(SStartupStep)}; - rpcSendResponse(&rpcRsp); - rpcFreeCont(pMsg->pCont); -} - -void taosStepCleanupImp(SStep *pSteps, int32_t stepId) { - for (int32_t step = stepId; step >= 0; step--) { - SStep *pStep = pSteps + step; - dDebug("step:%s will cleanup", pStep->name); - if (pStep->cleanupFp != NULL) { - (*pStep->cleanupFp)(); - } - } -} - -int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize) { - for (int32_t step = 0; step < stepSize; step++) { - SStep *pStep = pSteps + step; - if (pStep->initFp == NULL) continue; - - dnodeReportStep(pStep->name, "Start initialization", 0); - - int32_t code = (*pStep->initFp)(); - if (code != 0) { - dDebug("step:%s will cleanup", pStep->name); - taosStepCleanupImp(pSteps, step); - return code; - } - dInfo("step:%s is initialized", pStep->name); - - dnodeReportStep(pStep->name, "Initialization complete", 0); - } - - return 0; -} - -void dnodeStepCleanup(SStep *pSteps, int32_t stepSize) { - taosStepCleanupImp(pSteps, stepSize - 1); -} diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c deleted file mode 100644 index 2f77788025..0000000000 --- a/src/dnode/src/dnodeSystem.c +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tgrant.h" -#include "tconfig.h" -#include "dnodeMain.h" - -static tsem_t exitSem; -static void siguser1Handler(int32_t signum, void *sigInfo, void *context); -static void siguser2Handler(int32_t signum, void *sigInfo, void *context); -static void sigintHandler(int32_t signum, void *sigInfo, void *context); - -int32_t main(int32_t argc, char *argv[]) { - int dump_config = 0; - - // Set global configuration file - for (int32_t i = 1; i < argc; ++i) { - if (strcmp(argv[i], "-c") == 0) { - if (i < argc - 1) { - if (strlen(argv[++i]) >= TSDB_FILENAME_LEN) { - printf("config file path overflow"); - exit(EXIT_FAILURE); - } - tstrncpy(configDir, argv[i], TSDB_FILENAME_LEN); - } else { - printf("'-c' requires a parameter, default:%s\n", configDir); - exit(EXIT_FAILURE); - } - } else if (strcmp(argv[i], "-C") == 0) { - dump_config = 1; - } else if (strcmp(argv[i], "--force-keep-file") == 0) { - tsdbForceKeepFile = true; - } else if (strcmp(argv[i], "--compact-mnode-wal") == 0) { - tsCompactMnodeWal = 1; - } else if (strcmp(argv[i], "-V") == 0) { -#ifdef _ACCT - char *versionStr = "enterprise"; -#else - char *versionStr = "community"; -#endif - printf("%s version: %s compatible_version: %s\n", versionStr, version, compatible_version); - printf("gitinfo: %s\n", gitinfo); - printf("gitinfoI: %s\n", gitinfoOfInternal); - printf("buildinfo: %s\n", buildinfo); - exit(EXIT_SUCCESS); - } else if (strcmp(argv[i], "-k") == 0) { - grantParseParameter(); - exit(EXIT_SUCCESS); - } else if (strcmp(argv[i], "-A") == 0) { - tsPrintAuth = 1; - } -#ifdef TAOS_MEM_CHECK - else if (strcmp(argv[i], "--alloc-random-fail") == 0) { - if ((i < argc - 1) && (argv[i + 1][0] != '-')) { - taosSetAllocMode(TAOS_ALLOC_MODE_RANDOM_FAIL, argv[++i], true); - } else { - taosSetAllocMode(TAOS_ALLOC_MODE_RANDOM_FAIL, NULL, true); - } - } else if (strcmp(argv[i], "--detect-mem-leak") == 0) { - if ((i < argc - 1) && (argv[i + 1][0] != '-')) { - taosSetAllocMode(TAOS_ALLOC_MODE_DETECT_LEAK, argv[++i], true); - } else { - taosSetAllocMode(TAOS_ALLOC_MODE_DETECT_LEAK, NULL, true); - } - } -#endif -#ifdef TAOS_RANDOM_FILE_FAIL - else if (strcmp(argv[i], "--random-file-fail-output") == 0) { - if ((i < argc - 1) && (argv[i + 1][0] != '-')) { - taosSetRandomFileFailOutput(argv[++i]); - } else { - taosSetRandomFileFailOutput(NULL); - } - } else if (strcmp(argv[i], "--random-file-fail-factor") == 0) { - if ((i + 1) < argc) { - int factor = atoi(argv[i + 1]); - printf("The factor of random failure is %d\n", factor); - taosSetRandomFileFailFactor(factor); - } else { - printf("Please specify a number for random failure factor!"); - exit(EXIT_FAILURE); - } - } -#endif - } - - if (0 != dump_config) { - tscEmbedded = 1; - taosInitGlobalCfg(); - taosReadGlobalLogCfg(); - - if (!taosReadGlobalCfg()) { - printf("TDengine read global config failed"); - exit(EXIT_FAILURE); - } - - taosDumpGlobalCfg(); - exit(EXIT_SUCCESS); - } - - if (tsem_init(&exitSem, 0, 0) != 0) { - printf("failed to create exit semphore\n"); - exit(EXIT_FAILURE); - } - - /* Set termination handler. */ - taosSetSignal(SIGUSR1, siguser1Handler); - taosSetSignal(SIGUSR2, siguser2Handler); - taosSetSignal(SIGTERM, sigintHandler); - taosSetSignal(SIGHUP, sigintHandler); - taosSetSignal(SIGINT, sigintHandler); - taosSetSignal(SIGABRT, sigintHandler); - taosSetSignal(SIGBREAK, sigintHandler); - - // Open /var/log/syslog file to record information. - openlog("TDengine:", LOG_PID | LOG_CONS | LOG_NDELAY, LOG_LOCAL1); - syslog(LOG_INFO, "Starting TDengine service..."); - - // Initialize the system - if (dnodeInitSystem() < 0) { - syslog(LOG_ERR, "Error initialize TDengine system"); - dInfo("Failed to start TDengine, please check the log at:%s", tsLogDir); - closelog(); - exit(EXIT_FAILURE); - } - - syslog(LOG_INFO, "Started TDengine service successfully."); - - if (tsem_wait(&exitSem) != 0) { - syslog(LOG_ERR, "failed to wait exit semphore: %s", strerror(errno)); - } - - dnodeCleanUpSystem(); - // close the syslog - syslog(LOG_INFO, "Shut down TDengine service successfully"); - dInfo("TDengine is shut down!"); - closelog(); - -#ifdef WINDOWS - tsem_post(&exitSem); -#endif - return EXIT_SUCCESS; -} - -static void siguser1Handler(int32_t signum, void *sigInfo, void *context) { taosCfgDynamicOptions("debugFlag 143"); } - -static void siguser2Handler(int32_t signum, void *sigInfo, void *context) { taosCfgDynamicOptions("resetlog"); } - -static void sigintHandler(int32_t signum, void *sigInfo, void *context) { - // protect the application from receive another signal - taosIgnSignal(SIGUSR1); - taosIgnSignal(SIGUSR2); - taosIgnSignal(SIGTERM); - taosIgnSignal(SIGHUP); - taosIgnSignal(SIGINT); - taosIgnSignal(SIGABRT); - taosIgnSignal(SIGBREAK); - - // clean the system. - dInfo("shut down signal is %d", signum); - -#ifndef WINDOWS - dInfo("sender PID:%d cmdline:%s", ((siginfo_t *)sigInfo)->si_pid, taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid)); -#endif - - syslog(LOG_INFO, "Shut down signal is %d", signum); - syslog(LOG_INFO, "Shutting down TDengine service..."); - - // inform main thread to exit - tsem_post(&exitSem); -#ifdef WINDOWS - tsem_wait(&exitSem); -#endif -} diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c deleted file mode 100644 index 22a6dc5b19..0000000000 --- a/src/dnode/src/dnodeTelemetry.c +++ /dev/null @@ -1,315 +0,0 @@ -/* - * Copyright (c) 2020 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "osTime.h" -#include "tsocket.h" -#include "tbuffer.h" -#include "mnode.h" -#include "mnodeDef.h" -#include "mnodeDb.h" -#include "mnodeDnode.h" -#include "mnodeCluster.h" -#include "mnodeDnode.h" -#include "mnodeVgroup.h" -#include "mnodeMnode.h" -#include "mnodeTable.h" -#include "mnodeSdb.h" -#include "mnodeAcct.h" -#include "dnodeTelemetry.h" - -// sem_timedwait is NOT implemented on MacOSX -// thus, we use pthread_mutex_t/pthread_cond_t to simulate -static pthread_mutex_t tsExitLock = PTHREAD_MUTEX_INITIALIZER; -static pthread_cond_t tsExitCond = PTHREAD_COND_INITIALIZER; -static volatile int tsExit = 0; -static pthread_t tsTelemetryThread; - -#define TELEMETRY_SERVER "telemetry.taosdata.com" -#define TELEMETRY_PORT 80 -#define REPORT_INTERVAL 86400 - -static void beginObject(SBufferWriter* bw) { - tbufWriteChar(bw, '{'); -} - -static void closeObject(SBufferWriter* bw) { - size_t len = tbufTell(bw); - if (tbufGetData(bw, false)[len - 1] == ',') { - tbufWriteCharAt(bw, len - 1, '}'); - } else { - tbufWriteChar(bw, '}'); - } - tbufWriteChar(bw, ','); -} - -#if 0 -static void beginArray(SBufferWriter* bw) { - tbufWriteChar(bw, '['); -} - -static void closeArray(SBufferWriter* bw) { - size_t len = tbufTell(bw); - if (tbufGetData(bw, false)[len - 1] == ',') { - tbufWriteCharAt(bw, len - 1, ']'); - } else { - tbufWriteChar(bw, ']'); - } - tbufWriteChar(bw, ','); -} -#endif - -static void writeString(SBufferWriter* bw, const char* str) { - tbufWriteChar(bw, '"'); - tbufWrite(bw, str, strlen(str)); - tbufWriteChar(bw, '"'); -} - -static void addIntField(SBufferWriter* bw, const char* k, int64_t v) { - writeString(bw, k); - tbufWriteChar(bw, ':'); - char buf[32]; - sprintf(buf, "%" PRId64, v); - tbufWrite(bw, buf, strlen(buf)); - tbufWriteChar(bw, ','); -} - -static void addStringField(SBufferWriter* bw, const char* k, const char* v) { - writeString(bw, k); - tbufWriteChar(bw, ':'); - writeString(bw, v); - tbufWriteChar(bw, ','); -} - -static void addCpuInfo(SBufferWriter* bw) { - char * line = NULL; - size_t size = 0; - int32_t done = 0; - - FILE* fp = fopen("/proc/cpuinfo", "r"); - if (fp == NULL) { - return; - } - - while (done != 3 && (size = tgetline(&line, &size, fp)) != -1) { - line[size - 1] = '\0'; - if (((done&1) == 0) && strncmp(line, "model name", 10) == 0) { - const char* v = strchr(line, ':') + 2; - addStringField(bw, "cpuModel", v); - done |= 1; - } else if (((done&2)==0) && strncmp(line, "cpu cores", 9) == 0) { - const char* v = strchr(line, ':') + 2; - writeString(bw, "numOfCpu"); - tbufWriteChar(bw, ':'); - tbufWrite(bw, v, strlen(v)); - tbufWriteChar(bw, ','); - done |= 2; - } - } - - free(line); - fclose(fp); -} - -static void addOsInfo(SBufferWriter* bw) { - char * line = NULL; - size_t size = 0; - - FILE* fp = fopen("/etc/os-release", "r"); - if (fp == NULL) { - return; - } - - while ((size = tgetline(&line, &size, fp)) != -1) { - line[size - 1] = '\0'; - if (strncmp(line, "PRETTY_NAME", 11) == 0) { - const char* p = strchr(line, '=') + 1; - if (*p == '"') { - p++; - line[size - 2] = 0; - } - addStringField(bw, "os", p); - break; - } - } - - free(line); - fclose(fp); -} - -static void addMemoryInfo(SBufferWriter* bw) { - char * line = NULL; - size_t size = 0; - - FILE* fp = fopen("/proc/meminfo", "r"); - if (fp == NULL) { - return; - } - - while ((size = tgetline(&line, &size, fp)) != -1) { - line[size - 1] = '\0'; - if (strncmp(line, "MemTotal", 8) == 0) { - const char* p = strchr(line, ':') + 1; - while (*p == ' ') p++; - addStringField(bw, "memory", p); - break; - } - } - - free(line); - fclose(fp); -} - -static void addVersionInfo(SBufferWriter* bw) { - addStringField(bw, "version", version); - addStringField(bw, "buildInfo", buildinfo); - addStringField(bw, "gitInfo", gitinfo); - addStringField(bw, "email", tsEmail); -} - -static void addRuntimeInfo(SBufferWriter* bw) { - addIntField(bw, "numOfDnode", mnodeGetDnodesNum()); - addIntField(bw, "numOfMnode", mnodeGetMnodesNum()); - addIntField(bw, "numOfVgroup", mnodeGetVgroupNum()); - addIntField(bw, "numOfDatabase", mnodeGetDbNum()); - addIntField(bw, "numOfSuperTable", mnodeGetSuperTableNum()); - addIntField(bw, "numOfChildTable", mnodeGetChildTableNum()); - - SAcctInfo info; - mnodeGetStatOfAllAcct(&info); - addIntField(bw, "numOfColumn", info.numOfTimeSeries); - addIntField(bw, "numOfPoint", info.totalPoints); - addIntField(bw, "totalStorage", info.totalStorage); - addIntField(bw, "compStorage", info.compStorage); - // addStringField(bw, "installTime", "2020-08-01T00:00:00Z"); -} - -static void sendTelemetryReport() { - char buf[128]; - uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); - if (ip == 0xffffffff) { - dTrace("failed to get IP address of " TELEMETRY_SERVER ", reason:%s", strerror(errno)); - return; - } - SOCKET fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); - if (fd < 0) { - dTrace("failed to create socket for telemetry, reason:%s", strerror(errno)); - return; - } - - SBufferWriter bw = tbufInitWriter(NULL, false); - beginObject(&bw); - addStringField(&bw, "instanceId", mnodeGetClusterId()); - addIntField(&bw, "reportVersion", 1); - addOsInfo(&bw); - addCpuInfo(&bw); - addMemoryInfo(&bw); - addVersionInfo(&bw); - addRuntimeInfo(&bw); - closeObject(&bw); - - const char* header = "POST /report HTTP/1.1\n" - "Host: " TELEMETRY_SERVER "\n" - "Content-Type: application/json\n" - "Content-Length: "; - - taosWriteSocket(fd, header, (int32_t)strlen(header)); - int32_t contLen = (int32_t)(tbufTell(&bw) - 1); - sprintf(buf, "%d\n\n", contLen); - taosWriteSocket(fd, buf, (int32_t)strlen(buf)); - taosWriteSocket(fd, tbufGetData(&bw, false), contLen); - tbufCloseWriter(&bw); - - // read something to avoid nginx error 499 - if (taosReadSocket(fd, buf, 10) < 0) { - dTrace("failed to receive response, reason:%s", strerror(errno)); - } - taosCloseSocket(fd); -} - -static void* telemetryThread(void* param) { - struct timespec end = {0}; - clock_gettime(CLOCK_REALTIME, &end); - end.tv_sec += 300; // wait 5 minutes before send first report - - setThreadName("telemetry"); - - while (!tsExit) { - int r = 0; - struct timespec ts = end; - pthread_mutex_lock(&tsExitLock); - r = pthread_cond_timedwait(&tsExitCond, &tsExitLock, &ts); - pthread_mutex_unlock(&tsExitLock); - if (r==0) break; - if (r!=ETIMEDOUT) continue; - - if (sdbIsMaster()) { - sendTelemetryReport(); - } - end.tv_sec += REPORT_INTERVAL; - } - - return NULL; -} - -static void dnodeGetEmail(char* filepath) { - int32_t fd = open(filepath, O_RDONLY); - if (fd < 0) { - return; - } - - if (taosRead(fd, (void *)tsEmail, TSDB_FQDN_LEN) < 0) { - dError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno)); - } - - taosClose(fd); -} - -int32_t dnodeInitTelemetry() { - if (!tsEnableTelemetryReporting) { - return 0; - } - - dnodeGetEmail("/usr/local/taos/email"); - - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - - int32_t code = pthread_create(&tsTelemetryThread, &attr, telemetryThread, NULL); - pthread_attr_destroy(&attr); - if (code != 0) { - dTrace("failed to create telemetry thread, reason:%s", strerror(code)); - } - - dInfo("dnode telemetry is initialized"); - return 0; -} - -void dnodeCleanupTelemetry() { - if (!tsEnableTelemetryReporting) { - return; - } - - if (taosCheckPthreadValid(tsTelemetryThread)) { - pthread_mutex_lock(&tsExitLock); - tsExit = 1; - pthread_cond_signal(&tsExitCond); - pthread_mutex_unlock(&tsExitLock); - - pthread_join(tsTelemetryThread, NULL); - } -} diff --git a/src/dnode/src/dnodeVMgmt.c b/src/dnode/src/dnodeVMgmt.c deleted file mode 100644 index c1bfb1460b..0000000000 --- a/src/dnode/src/dnodeVMgmt.c +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tqueue.h" -#include "tworker.h" -#include "dnodeVMgmt.h" - -typedef struct { - SRpcMsg rpcMsg; - char pCont[]; -} SMgmtMsg; - -static SWorkerPool tsVMgmtWP; -static taos_queue tsVMgmtQueue = NULL; - -static void * dnodeProcessMgmtQueue(void *param); -static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *pMsg); -static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg); -static int32_t dnodeProcessSyncVnodeMsg(SRpcMsg *pMsg); -static int32_t dnodeProcessCompactVnodeMsg(SRpcMsg *pMsg); -static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *pMsg); -static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg); -static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg); -static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg); -static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg); - -int32_t dnodeInitVMgmt() { - dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeProcessCreateVnodeMsg; - dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeProcessAlterVnodeMsg; - dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE] = dnodeProcessSyncVnodeMsg; - dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE]= dnodeProcessCompactVnodeMsg; - dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeProcessDropVnodeMsg; - dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeProcessAlterStreamMsg; - dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeMsg; - dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeProcessCreateMnodeMsg; - - int32_t code = vnodeInitMgmt(); - if (code != TSDB_CODE_SUCCESS) return -1; - - tsVMgmtWP.name = "vmgmt"; - tsVMgmtWP.workerFp = dnodeProcessMgmtQueue; - tsVMgmtWP.min = 1; - tsVMgmtWP.max = 1; - if (tWorkerInit(&tsVMgmtWP) != 0) return -1; - - tsVMgmtQueue = tWorkerAllocQueue(&tsVMgmtWP, NULL); - - dInfo("dnode vmgmt is initialized"); - return TSDB_CODE_SUCCESS; -} - -void dnodeCleanupVMgmt() { - tWorkerFreeQueue(&tsVMgmtWP, tsVMgmtQueue); - tWorkerCleanup(&tsVMgmtWP); - - tsVMgmtQueue = NULL; - vnodeCleanupMgmt(); -} - -static int32_t dnodeWriteToMgmtQueue(SRpcMsg *pMsg) { - int32_t size = sizeof(SMgmtMsg) + pMsg->contLen; - SMgmtMsg *pMgmt = taosAllocateQitem(size); - if (pMgmt == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY; - - pMgmt->rpcMsg = *pMsg; - pMgmt->rpcMsg.pCont = pMgmt->pCont; - memcpy(pMgmt->pCont, pMsg->pCont, pMsg->contLen); - taosWriteQitem(tsVMgmtQueue, TAOS_QTYPE_RPC, pMgmt); - - return TSDB_CODE_SUCCESS; -} - -void dnodeDispatchToVMgmtQueue(SRpcMsg *pMsg) { - int32_t code = dnodeWriteToMgmtQueue(pMsg); - if (code != TSDB_CODE_SUCCESS) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = code}; - rpcSendResponse(&rsp); - } - - rpcFreeCont(pMsg->pCont); -} - -static void *dnodeProcessMgmtQueue(void *wparam) { - SWorker * pWorker = wparam; - SWorkerPool *pPool = pWorker->pPool; - SMgmtMsg * pMgmt; - SRpcMsg * pMsg; - SRpcMsg rsp = {0}; - int32_t qtype; - void * handle; - - setThreadName("dnodeMgmtQ"); - - while (1) { - if (taosReadQitemFromQset(pPool->qset, &qtype, (void **)&pMgmt, &handle) == 0) { - dDebug("qdnode mgmt got no message from qset:%p, , exit", pPool->qset); - break; - } - - pMsg = &pMgmt->rpcMsg; - dTrace("msg:%p, ahandle:%p type:%s will be processed", pMgmt, pMsg->ahandle, taosMsg[pMsg->msgType]); - if (dnodeProcessMgmtMsgFp[pMsg->msgType]) { - rsp.code = (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg); - } else { - rsp.code = TSDB_CODE_DND_MSG_NOT_PROCESSED; - } - - dTrace("msg:%p, is processed, code:0x%x", pMgmt, rsp.code); - if (rsp.code != TSDB_CODE_DND_ACTION_IN_PROGRESS) { - rsp.handle = pMsg->handle; - rsp.pCont = NULL; - rpcSendResponse(&rsp); - } - - taosFreeQitem(pMsg); - } - - return NULL; -} - -static SCreateVnodeMsg* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) { - SCreateVnodeMsg *pCreate = rpcMsg->pCont; - pCreate->cfg.vgId = htonl(pCreate->cfg.vgId); - pCreate->cfg.dbCfgVersion = htonl(pCreate->cfg.dbCfgVersion); - pCreate->cfg.vgCfgVersion = htonl(pCreate->cfg.vgCfgVersion); - pCreate->cfg.maxTables = htonl(pCreate->cfg.maxTables); - pCreate->cfg.cacheBlockSize = htonl(pCreate->cfg.cacheBlockSize); - pCreate->cfg.totalBlocks = htonl(pCreate->cfg.totalBlocks); - pCreate->cfg.daysPerFile = htonl(pCreate->cfg.daysPerFile); - pCreate->cfg.daysToKeep1 = htonl(pCreate->cfg.daysToKeep1); - pCreate->cfg.daysToKeep2 = htonl(pCreate->cfg.daysToKeep2); - pCreate->cfg.daysToKeep = htonl(pCreate->cfg.daysToKeep); - pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock); - pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock); - pCreate->cfg.fsyncPeriod = htonl(pCreate->cfg.fsyncPeriod); - pCreate->cfg.commitTime = htonl(pCreate->cfg.commitTime); - - for (int32_t j = 0; j < pCreate->cfg.vgReplica; ++j) { - pCreate->nodes[j].nodeId = htonl(pCreate->nodes[j].nodeId); - } - - return pCreate; -} - -static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) { - SCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg); - void *pVnode = vnodeAcquire(pCreate->cfg.vgId); - if (pVnode != NULL) { - dDebug("vgId:%d, already exist, return success", pCreate->cfg.vgId); - vnodeRelease(pVnode); - return TSDB_CODE_SUCCESS; - } else { - dDebug("vgId:%d, create vnode msg is received", pCreate->cfg.vgId); - return vnodeCreate(pCreate); - } -} - -static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) { - SAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg); - - void *pVnode = vnodeAcquireNotClose(pAlter->cfg.vgId); - if (pVnode != NULL) { - dDebug("vgId:%d, alter vnode msg is received", pAlter->cfg.vgId); - int32_t code = vnodeAlter(pVnode, pAlter); - vnodeRelease(pVnode); - return code; - } else { - dInfo("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId); - return TSDB_CODE_VND_INVALID_VGROUP_ID; - } -} - -static int32_t dnodeProcessSyncVnodeMsg(SRpcMsg *rpcMsg) { - SSyncVnodeMsg *pSyncVnode = rpcMsg->pCont; - pSyncVnode->vgId = htonl(pSyncVnode->vgId); - - return vnodeSync(pSyncVnode->vgId); -} - -static int32_t dnodeProcessCompactVnodeMsg(SRpcMsg *rpcMsg) { - SCompactVnodeMsg *pCompactVnode = rpcMsg->pCont; - pCompactVnode->vgId = htonl(pCompactVnode->vgId); - return vnodeCompact(pCompactVnode->vgId); -} - -static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) { - SDropVnodeMsg *pDrop = rpcMsg->pCont; - pDrop->vgId = htonl(pDrop->vgId); - - return vnodeDrop(pDrop->vgId); -} - -static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) { - return 0; -} - -static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) { - SCfgDnodeMsg *pCfg = pMsg->pCont; - return taosCfgDynamicOptions(pCfg->config); -} - -static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) { - SCreateMnodeMsg *pCfg = pMsg->pCont; - pCfg->dnodeId = htonl(pCfg->dnodeId); - if (pCfg->dnodeId != dnodeGetDnodeId()) { - dDebug("dnode:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId()); - return TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED; - } - - if (strcmp(pCfg->dnodeEp, tsLocalEp) != 0) { - dDebug("dnodeEp:%s, in create mnode msg is not equal with saved dnodeEp:%s", pCfg->dnodeEp, tsLocalEp); - return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED; - } - - dDebug("dnode:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.mnodeNum); - for (int i = 0; i < pCfg->mnodes.mnodeNum; ++i) { - pCfg->mnodes.mnodeInfos[i].mnodeId = htonl(pCfg->mnodes.mnodeInfos[i].mnodeId); - dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.mnodeInfos[i].mnodeId, pCfg->mnodes.mnodeInfos[i].mnodeEp); - } - - return dnodeStartMnode(&pCfg->mnodes); -} diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c deleted file mode 100644 index c404ab1a55..0000000000 --- a/src/dnode/src/dnodeVRead.c +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tqueue.h" -#include "tworker.h" -#include "dnodeVRead.h" - -static void *dnodeProcessReadQueue(void *pWorker); - -// module global variable -static SWorkerPool tsVQueryWP; -static SWorkerPool tsVFetchWP; - -int32_t dnodeInitVRead() { - const int32_t maxFetchThreads = 4; - - // calculate the available query thread - float threadsForQuery = MAX(tsNumOfCores * tsRatioOfQueryCores, 1); - - tsVQueryWP.name = "vquery"; - tsVQueryWP.workerFp = dnodeProcessReadQueue; - tsVQueryWP.min = (int32_t) threadsForQuery; - tsVQueryWP.max = tsVQueryWP.min; - if (tWorkerInit(&tsVQueryWP) != 0) return -1; - - tsVFetchWP.name = "vfetch"; - tsVFetchWP.workerFp = dnodeProcessReadQueue; - tsVFetchWP.min = MIN(maxFetchThreads, tsNumOfCores); - tsVFetchWP.max = tsVFetchWP.min; - if (tWorkerInit(&tsVFetchWP) != 0) return -1; - - return 0; -} - -void dnodeCleanupVRead() { - tWorkerCleanup(&tsVFetchWP); - tWorkerCleanup(&tsVQueryWP); -} - -void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) { - int32_t queuedMsgNum = 0; - int32_t leftLen = pMsg->contLen; - int32_t code = TSDB_CODE_VND_INVALID_VGROUP_ID; - char * pCont = pMsg->pCont; - - while (leftLen > 0) { - SMsgHead *pHead = (SMsgHead *)pCont; - pHead->vgId = htonl(pHead->vgId); - pHead->contLen = htonl(pHead->contLen); - - assert(pHead->contLen > 0); - void *pVnode = vnodeAcquireNotClose(pHead->vgId); - if (pVnode != NULL) { - code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg); - if (code == TSDB_CODE_SUCCESS) queuedMsgNum++; - vnodeRelease(pVnode); - } - - leftLen -= pHead->contLen; - pCont -= pHead->contLen; - } - - if (queuedMsgNum == 0) { - SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = code}; - rpcSendResponse(&rpcRsp); - } - - rpcFreeCont(pMsg->pCont); -} - -void *dnodeAllocVQueryQueue(void *pVnode) { - return tWorkerAllocQueue(&tsVQueryWP, pVnode); -} - -void *dnodeAllocVFetchQueue(void *pVnode) { - return tWorkerAllocQueue(&tsVFetchWP, pVnode); -} - -void dnodeFreeVQueryQueue(void *pQqueue) { - tWorkerFreeQueue(&tsVQueryWP, pQqueue); -} - -void dnodeFreeVFetchQueue(void *pFqueue) { - tWorkerFreeQueue(&tsVFetchWP, pFqueue); -} - -void dnodeSendRpcVReadRsp(void *pVnode, SVReadMsg *pRead, int32_t code) { - SRpcMsg rpcRsp = { - .handle = pRead->rpcHandle, - .pCont = pRead->rspRet.rsp, - .contLen = pRead->rspRet.len, - .code = code, - }; - - rpcSendResponse(&rpcRsp); -} - -void dnodeDispatchNonRspMsg(void *pVnode, SVReadMsg *pRead, int32_t code) { -} - -static void *dnodeProcessReadQueue(void *wparam) { - SWorker * pWorker = wparam; - SWorkerPool *pPool = pWorker->pPool; - SVReadMsg * pRead; - int32_t qtype; - void * pVnode; - - char* threadname = strcmp(pPool->name, "vquery") == 0? "dnodeQueryQ":"dnodeFetchQ"; - - char name[16] = {0}; - snprintf(name, tListLen(name), "%s", threadname); - setThreadName(name); - - while (1) { - if (taosReadQitemFromQset(pPool->qset, &qtype, (void **)&pRead, &pVnode) == 0) { - dDebug("dnode vquery got no message from qset:%p, exiting", pPool->qset); - break; - } - - dTrace("msg:%p, app:%p type:%s will be processed in vquery queue, qtype:%d", pRead, pRead->rpcAhandle, - taosMsg[pRead->msgType], qtype); - - int32_t code = vnodeProcessRead(pVnode, pRead); - - if (qtype == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) { - dnodeSendRpcVReadRsp(pVnode, pRead, code); - } else { - if (code == TSDB_CODE_QRY_HAS_RSP) { - dnodeSendRpcVReadRsp(pVnode, pRead, pRead->code); - } else { // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client - assert(pRead->rpcHandle == NULL || (pRead->rpcHandle != NULL && pRead->msgType == 5)); - dnodeDispatchNonRspMsg(pVnode, pRead, code); - } - } - - vnodeFreeFromRQueue(pVnode, pRead); - } - - return NULL; -} diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c deleted file mode 100644 index ed2a6e2109..0000000000 --- a/src/dnode/src/dnodeVWrite.c +++ /dev/null @@ -1,238 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tqueue.h" -#include "dnodeVWrite.h" - -typedef struct { - taos_qall qall; - taos_qset qset; // queue set - int32_t workerId; // worker ID - pthread_t thread; // thread -} SVWriteWorker; - -typedef struct { - int32_t max; // max number of workers - int32_t nextId; // from 0 to max-1, cyclic - SVWriteWorker * worker; - pthread_mutex_t mutex; -} SVWriteWorkerPool; - -static SVWriteWorkerPool tsVWriteWP; -static void *dnodeProcessVWriteQueue(void *pWorker); - -int32_t dnodeInitVWrite() { - tsVWriteWP.max = tsNumOfCores; - tsVWriteWP.worker = tcalloc(sizeof(SVWriteWorker), tsVWriteWP.max); - if (tsVWriteWP.worker == NULL) return -1; - pthread_mutex_init(&tsVWriteWP.mutex, NULL); - - for (int32_t i = 0; i < tsVWriteWP.max; ++i) { - tsVWriteWP.worker[i].workerId = i; - } - - dInfo("dnode vwrite is initialized, max worker %d", tsVWriteWP.max); - return 0; -} - -void dnodeCleanupVWrite() { - for (int32_t i = 0; i < tsVWriteWP.max; ++i) { - SVWriteWorker *pWorker = tsVWriteWP.worker + i; - if (taosCheckPthreadValid(pWorker->thread)) { - if (pWorker->qset) taosQsetThreadResume(pWorker->qset); - } - } - - for (int32_t i = 0; i < tsVWriteWP.max; ++i) { - SVWriteWorker *pWorker = tsVWriteWP.worker + i; - if (taosCheckPthreadValid(pWorker->thread)) { - pthread_join(pWorker->thread, NULL); - taosFreeQall(pWorker->qall); - taosCloseQset(pWorker->qset); - } - } - - pthread_mutex_destroy(&tsVWriteWP.mutex); - tfree(tsVWriteWP.worker); - dInfo("dnode vwrite is closed"); -} - -void dnodeDispatchToVWriteQueue(SRpcMsg *pRpcMsg) { - int32_t code; - char *pCont = pRpcMsg->pCont; - - if (pRpcMsg->msgType == TSDB_MSG_TYPE_SUBMIT) { - SMsgDesc *pDesc = (SMsgDesc *)pCont; - pDesc->numOfVnodes = htonl(pDesc->numOfVnodes); - pCont += sizeof(SMsgDesc); - } - - SMsgHead *pMsg = (SMsgHead *)pCont; - pMsg->vgId = htonl(pMsg->vgId); - pMsg->contLen = htonl(pMsg->contLen); - - void *pVnode = vnodeAcquireNotClose(pMsg->vgId); - if (pVnode == NULL) { - code = TSDB_CODE_VND_INVALID_VGROUP_ID; - } else { - SWalHead *pHead = (SWalHead *)(pCont - sizeof(SWalHead)); - pHead->msgType = pRpcMsg->msgType; - pHead->version = 0; - pHead->len = pMsg->contLen; - code = vnodeWriteToWQueue(pVnode, pHead, TAOS_QTYPE_RPC, pRpcMsg); - } - - if (code != TSDB_CODE_SUCCESS) { - SRpcMsg rpcRsp = {.handle = pRpcMsg->handle, .code = code}; - rpcSendResponse(&rpcRsp); - } - - vnodeRelease(pVnode); - rpcFreeCont(pRpcMsg->pCont); -} - -void *dnodeAllocVWriteQueue(void *pVnode) { - pthread_mutex_lock(&tsVWriteWP.mutex); - SVWriteWorker *pWorker = tsVWriteWP.worker + tsVWriteWP.nextId; - taos_queue *queue = taosOpenQueue(); - if (queue == NULL) { - pthread_mutex_unlock(&tsVWriteWP.mutex); - return NULL; - } - - if (pWorker->qset == NULL) { - pWorker->qset = taosOpenQset(); - if (pWorker->qset == NULL) { - taosCloseQueue(queue); - pthread_mutex_unlock(&tsVWriteWP.mutex); - return NULL; - } - - taosAddIntoQset(pWorker->qset, queue, pVnode); - pWorker->qall = taosAllocateQall(); - if (pWorker->qall == NULL) { - taosCloseQset(pWorker->qset); - taosCloseQueue(queue); - pthread_mutex_unlock(&tsVWriteWP.mutex); - return NULL; - } - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - - if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessVWriteQueue, pWorker) != 0) { - dError("failed to create thread to process vwrite queue since %s", strerror(errno)); - taosFreeQall(pWorker->qall); - taosCloseQset(pWorker->qset); - taosCloseQueue(queue); - queue = NULL; - } else { - dDebug("dnode vwrite worker:%d is launched", pWorker->workerId); - tsVWriteWP.nextId = (tsVWriteWP.nextId + 1) % tsVWriteWP.max; - } - - pthread_attr_destroy(&thAttr); - } else { - taosAddIntoQset(pWorker->qset, queue, pVnode); - tsVWriteWP.nextId = (tsVWriteWP.nextId + 1) % tsVWriteWP.max; - } - - pthread_mutex_unlock(&tsVWriteWP.mutex); - dDebug("pVnode:%p, dnode vwrite queue:%p is allocated", pVnode, queue); - - return queue; -} - -void dnodeFreeVWriteQueue(void *pWqueue) { - taosCloseQueue(pWqueue); -} - -void dnodeSendRpcVWriteRsp(void *pVnode, void *wparam, int32_t code) { - if (wparam == NULL) return; - SVWriteMsg *pWrite = wparam; - - if (code < 0) pWrite->code = code; - int32_t count = atomic_add_fetch_32(&pWrite->processedCount, 1); - - if (count <= 1) return; - - SRpcMsg rpcRsp = { - .handle = pWrite->rpcMsg.handle, - .pCont = pWrite->rspRet.rsp, - .contLen = pWrite->rspRet.len, - .code = pWrite->code, - }; - - rpcSendResponse(&rpcRsp); - vnodeFreeFromWQueue(pVnode, pWrite); -} - -static void *dnodeProcessVWriteQueue(void *wparam) { - SVWriteWorker *pWorker = wparam; - SVWriteMsg * pWrite; - void * pVnode; - int32_t numOfMsgs; - int32_t qtype; - - taosBlockSIGPIPE(); - dDebug("dnode vwrite worker:%d is running", pWorker->workerId); - - setThreadName("dnodeWriteQ"); - - while (1) { - numOfMsgs = taosReadAllQitemsFromQset(pWorker->qset, pWorker->qall, &pVnode); - if (numOfMsgs == 0) { - dDebug("qset:%p, dnode vwrite got no message from qset, exiting", pWorker->qset); - break; - } - - bool forceFsync = false; - for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite); - dTrace("msg:%p, app:%p type:%s will be processed in vwrite queue, qtype:%s hver:%" PRIu64, pWrite, - pWrite->rpcMsg.ahandle, taosMsg[pWrite->walHead.msgType], qtypeStr[qtype], pWrite->walHead.version); - - pWrite->code = vnodeProcessWrite(pVnode, &pWrite->walHead, qtype, pWrite); - if (pWrite->code <= 0) atomic_add_fetch_32(&pWrite->processedCount, 1); - if (pWrite->code > 0) pWrite->code = 0; - if (pWrite->code == 0 && pWrite->walHead.msgType != TSDB_MSG_TYPE_SUBMIT) forceFsync = true; - - dTrace("msg:%p is processed in vwrite queue, code:0x%x", pWrite, pWrite->code); - } - - walFsync(vnodeGetWal(pVnode), forceFsync); - - // browse all items, and process them one by one - taosResetQitems(pWorker->qall); - for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite); - if (qtype == TAOS_QTYPE_RPC) { - dnodeSendRpcVWriteRsp(pVnode, pWrite, pWrite->code); - } else { - if (qtype == TAOS_QTYPE_FWD) { - vnodeConfirmForward(pVnode, pWrite->walHead.version, pWrite->code, pWrite->walHead.msgType != TSDB_MSG_TYPE_SUBMIT); - } - if (pWrite->rspRet.rsp) { - rpcFreeCont(pWrite->rspRet.rsp); - } - vnodeFreeFromWQueue(pVnode, pWrite); - } - } - } - - return NULL; -} diff --git a/src/dnode/src/dnodeVnodes.c b/src/dnode/src/dnodeVnodes.c deleted file mode 100644 index a5b0e9fe30..0000000000 --- a/src/dnode/src/dnodeVnodes.c +++ /dev/null @@ -1,303 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "ttimer.h" -#include "dnodeEps.h" -#include "dnodeCfg.h" -#include "dnodeMInfos.h" -#include "dnodeVnodes.h" - -typedef struct { - pthread_t thread; - int32_t threadIndex; - int32_t failed; - int32_t opened; - int32_t vnodeNum; - int32_t * vnodeList; -} SOpenVnodeThread; - -extern void * tsDnodeTmr; -static void * tsStatusTimer = NULL; -static uint32_t tsRebootTime = 0; -static int32_t tsOpenVnodes = 0; -static int32_t tsTotalVnodes = 0; - -static void dnodeSendStatusMsg(void *handle, void *tmrId); -static void dnodeProcessStatusRsp(SRpcMsg *pMsg); - -int32_t dnodeInitStatusTimer() { - dnodeAddClientRspHandle(TSDB_MSG_TYPE_DM_STATUS_RSP, dnodeProcessStatusRsp); - - tsRebootTime = taosGetTimestampSec(); - taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer); - - dInfo("dnode status timer is initialized"); - return TSDB_CODE_SUCCESS; -} - -void dnodeCleanupStatusTimer() { - if (tsStatusTimer != NULL) { - taosTmrStopA(&tsStatusTimer); - tsStatusTimer = NULL; - } -} - -static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { - DIR *dir = opendir(tsVnodeDir); - if (dir == NULL) return TSDB_CODE_DND_NO_WRITE_ACCESS; - - *numOfVnodes = 0; - struct dirent *de = NULL; - while ((de = readdir(dir)) != NULL) { - if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; - if (de->d_type & DT_DIR) { - if (strncmp("vnode", de->d_name, 5) != 0) continue; - int32_t vnode = atoi(de->d_name + 5); - if (vnode == 0) continue; - - (*numOfVnodes)++; - - if (*numOfVnodes >= TSDB_MAX_VNODES) { - dError("vgId:%d, too many vnode directory in disk, exist:%d max:%d", vnode, *numOfVnodes, TSDB_MAX_VNODES); - closedir(dir); - return TSDB_CODE_DND_TOO_MANY_VNODES; - } else { - vnodeList[*numOfVnodes - 1] = vnode; - } - } - } - closedir(dir); - - return TSDB_CODE_SUCCESS; -} - -static void *dnodeOpenVnode(void *param) { - SOpenVnodeThread *pThread = param; - char stepDesc[TSDB_STEP_DESC_LEN] = {0}; - - dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); - setThreadName("dnodeOpenVnode"); - - for (int32_t v = 0; v < pThread->vnodeNum; ++v) { - int32_t vgId = pThread->vnodeList[v]; - snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", vgId, tsOpenVnodes, tsTotalVnodes); - dnodeReportStep("open-vnodes", stepDesc, 0); - - if (vnodeOpen(vgId) < 0) { - dError("vgId:%d, failed to open vnode by thread:%d", vgId, pThread->threadIndex); - pThread->failed++; - } else { - dDebug("vgId:%d, is opened by thread:%d", vgId, pThread->threadIndex); - pThread->opened++; - } - - atomic_add_fetch_32(&tsOpenVnodes, 1); - } - - dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, - pThread->failed); - return NULL; -} - -int32_t dnodeInitVnodes() { - int32_t vnodeList[TSDB_MAX_VNODES] = {0}; - int32_t numOfVnodes = 0; - int32_t status = dnodeGetVnodeList(vnodeList, &numOfVnodes); - tsTotalVnodes = numOfVnodes; - - if (status != TSDB_CODE_SUCCESS) { - dInfo("get dnode list failed"); - return status; - } - - int32_t threadNum = tsNumOfCores; - int32_t vnodesPerThread = numOfVnodes / threadNum + 1; - SOpenVnodeThread *threads = calloc(threadNum, sizeof(SOpenVnodeThread)); - for (int32_t t = 0; t < threadNum; ++t) { - threads[t].threadIndex = t; - threads[t].vnodeList = calloc(vnodesPerThread, sizeof(int32_t)); - } - - for (int32_t v = 0; v < numOfVnodes; ++v) { - int32_t t = v % threadNum; - SOpenVnodeThread *pThread = &threads[t]; - pThread->vnodeList[pThread->vnodeNum++] = vnodeList[v]; - } - - dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes); - - for (int32_t t = 0; t < threadNum; ++t) { - SOpenVnodeThread *pThread = &threads[t]; - if (pThread->vnodeNum == 0) continue; - - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnode, pThread) != 0) { - dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno)); - } - - pthread_attr_destroy(&thAttr); - } - - int32_t openVnodes = 0; - int32_t failedVnodes = 0; - for (int32_t t = 0; t < threadNum; ++t) { - SOpenVnodeThread *pThread = &threads[t]; - if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { - pthread_join(pThread->thread, NULL); - } - openVnodes += pThread->opened; - failedVnodes += pThread->failed; - free(pThread->vnodeList); - } - - free(threads); - dInfo("there are total vnodes:%d, opened:%d", numOfVnodes, openVnodes); - - if (failedVnodes != 0) { - dError("there are total vnodes:%d, failed:%d", numOfVnodes, failedVnodes); - return -1; - } - - return TSDB_CODE_SUCCESS; -} - -void dnodeCleanupVnodes() { - int32_t vnodeList[TSDB_MAX_VNODES]= {0}; - int32_t numOfVnodes = 0; - int32_t status; - - status = vnodeGetVnodeList(vnodeList, &numOfVnodes); - - if (status != TSDB_CODE_SUCCESS) { - dInfo("get dnode list failed"); - return; - } - - for (int32_t i = 0; i < numOfVnodes; ++i) { - vnodeClose(vnodeList[i]); - } - - dInfo("total vnodes:%d are all closed", numOfVnodes); -} - -static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { - if (pMsg->code != TSDB_CODE_SUCCESS) { - dError("status rsp is received, error:%s", tstrerror(pMsg->code)); - if (pMsg->code == TSDB_CODE_MND_DNODE_NOT_EXIST) { - char clusterId[TSDB_CLUSTER_ID_LEN]; - dnodeGetClusterId(clusterId); - if (clusterId[0] != '\0') { - dnodeSetDropped(); - dError("exit zombie dropped dnode"); - exit(EXIT_FAILURE); - } - } - - taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); - return; - } - - SStatusRsp *pStatusRsp = pMsg->pCont; - SMInfos *minfos = &pStatusRsp->mnodes; - dnodeUpdateMInfos(minfos); - - SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg; - pCfg->numOfVnodes = htonl(pCfg->numOfVnodes); - pCfg->moduleStatus = htonl(pCfg->moduleStatus); - pCfg->dnodeId = htonl(pCfg->dnodeId); - dnodeUpdateCfg(pCfg); - - vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes); - - SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess)); - dnodeUpdateEps(pEps); - - taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); -} - -static void dnodeSendStatusMsg(void *handle, void *tmrId) { - if (tsDnodeTmr == NULL) { - dError("dnode timer is already released"); - return; - } - - if (tsStatusTimer == NULL) { - taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); - dError("failed to start status timer"); - return; - } - - int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad); - SStatusMsg *pStatus = rpcMallocCont(contLen); - if (pStatus == NULL) { - taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); - dError("failed to malloc status message"); - return; - } - - dnodeGetCfg(&pStatus->dnodeId, pStatus->clusterId); - pStatus->dnodeId = htonl(dnodeGetDnodeId()); - pStatus->version = htonl(tsVersion); - pStatus->lastReboot = htonl(tsRebootTime); - pStatus->numOfCores = htons((uint16_t) tsNumOfCores); - pStatus->diskAvailable = tsAvailDataDirGB; - pStatus->alternativeRole = tsAlternativeRole; - tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN); - - // fill cluster cfg parameters - pStatus->clusterCfg.numOfMnodes = htonl(tsNumOfMnodes); - pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(tsMnodeEqualVnodeNum); - pStatus->clusterCfg.offlineThreshold = htonl(tsOfflineThreshold); - pStatus->clusterCfg.statusInterval = htonl(tsStatusInterval); - pStatus->clusterCfg.maxtablesPerVnode = htonl(tsMaxTablePerVnode); - pStatus->clusterCfg.maxVgroupsPerDb = htonl(tsMaxVgroupsPerDb); - tstrncpy(pStatus->clusterCfg.arbitrator, tsArbitrator, TSDB_EP_LEN); - tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, 64); - pStatus->clusterCfg.checkTime = 0; - char timestr[32] = "1970-01-01 00:00:00.00"; - (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); - tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN); - tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN); - - pStatus->clusterCfg.enableBalance = tsEnableBalance; - pStatus->clusterCfg.flowCtrl = tsEnableFlowCtrl; - pStatus->clusterCfg.slaveQuery = tsEnableSlaveQuery; - pStatus->clusterCfg.adjustMaster = tsEnableAdjustMaster; - - vnodeBuildStatusMsg(pStatus); - contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad); - pStatus->openVnodes = htons(pStatus->openVnodes); - - SRpcMsg rpcMsg = { - .pCont = pStatus, - .contLen = contLen, - .msgType = TSDB_MSG_TYPE_DM_STATUS - }; - - SRpcEpSet epSet; - dnodeGetEpSetForPeer(&epSet); - dnodeSendMsgToDnode(&epSet, &rpcMsg); -} - -void dnodeSendStatusMsgToMnode() { - if (tsDnodeTmr != NULL && tsStatusTimer != NULL) { - dInfo("force send status msg to mnode"); - taosTmrReset(dnodeSendStatusMsg, 3, NULL, tsDnodeTmr, &tsStatusTimer); - } -} From 7af18fc96c41cf059bafd68aba7555c44a0e8a37 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 30 Sep 2021 16:41:02 +0800 Subject: [PATCH 31/44] add some os files --- deps/CMakeLists.txt | 6 + include/common/taosdef.h | 9 - include/os/os.h | 25 +- include/os/osDef.h | 15 +- include/os/osFile.h | 69 +- include/os/osRand.h | 31 + include/os/osSemaphore.h | 2 + include/os/osSleep.h | 29 + include/os/osSocket.h | 59 +- include/os/osThread.h | 30 + include/os/osTime.h | 6 +- include/util/tidpool.h | 43 + include/util/tmempool.h | 36 + include/util/tref.h | 77 ++ include/util/tutil.h | 10 +- source/libs/transport/CMakeLists.txt | 8 + source/libs/transport/src/rpcTcp.c | 30 +- {src/os/src/detail => source/os/src}/osFile.c | 191 +++- .../src/detail => source/os/src}/osSocket.c | 36 +- source/os/src/osTime.c | 37 + source/server/dnode/src/dnodeCfg.c | 2 +- source/server/dnode/src/dnodeEps.c | 2 +- source/server/dnode/src/dnodeMnodeEps.c | 2 +- source/server/dnode/src/dnodeTelemetry.c | 7 +- source/server/dnode/src/dnodeTrans.c | 2 +- source/util/CMakeLists.txt | 2 +- source/util/inc/tulog.h | 42 + source/util/src/hash.c | 916 ++++++++++++++++++ {src => source}/util/src/tlog.c | 90 +- 29 files changed, 1596 insertions(+), 218 deletions(-) create mode 100644 include/os/osRand.h create mode 100644 include/os/osSleep.h create mode 100644 include/os/osThread.h create mode 100644 include/util/tidpool.h create mode 100644 include/util/tmempool.h create mode 100644 include/util/tref.h rename {src/os/src/detail => source/os/src}/osFile.c (68%) rename {src/os/src/detail => source/os/src}/osSocket.c (79%) create mode 100644 source/os/src/osTime.c create mode 100644 source/util/inc/tulog.h create mode 100644 source/util/src/hash.c rename {src => source}/util/src/tlog.c (89%) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index 137ad31d82..410e0c564e 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -8,7 +8,13 @@ target_include_directories( # see https://stackoverflow.com/questions/25676277/cmake-target-include-directories-prints-an-error-when-i-try-to-add-the-source PUBLIC $ ) + add_subdirectory(lz4/build/cmake) +target_include_directories( + lz4_static + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib +) + add_subdirectory(zlib) target_include_directories( zlib diff --git a/include/common/taosdef.h b/include/common/taosdef.h index d80caad88d..0bcd693f5e 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -41,15 +41,6 @@ extern "C" { // Bytes for each type. extern const int32_t TYPE_BYTES[15]; -// TODO: replace and remove code below -#define CHAR_BYTES sizeof(char) -#define SHORT_BYTES sizeof(int16_t) -#define INT_BYTES sizeof(int32_t) -#define LONG_BYTES sizeof(int64_t) -#define FLOAT_BYTES sizeof(float) -#define DOUBLE_BYTES sizeof(double) -#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) - #define TSDB_KEYSIZE sizeof(TSKEY) #if LINUX diff --git a/include/os/os.h b/include/os/os.h index c89dc65211..935923195b 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -21,22 +21,29 @@ extern "C" { #endif #include +#include #include +#include #include -#include +#include +#include +#include +#include #include #include #include #include #include +#include -#include -#include -#include -#include -#include -#include -#include +// #include +// #include +// #include +// // #include +// +// #include +// #include +// #include #include "osAtomic.h" #include "osDef.h" @@ -48,7 +55,9 @@ extern "C" { #include "osSemaphore.h" #include "osSocket.h" #include "osString.h" +#include "osSleep.h" #include "osTime.h" +#include "osThread.h" #ifdef __cplusplus } diff --git a/include/os/osDef.h b/include/os/osDef.h index 7de8fd2f26..053bf42ece 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -29,11 +29,6 @@ extern "C" { #define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b))) #define POINTER_DISTANCE(p1, p2) ((char *)(p1) - (char *)(p2)) -#ifndef PATH_MAX - #define PATH_MAX 1024 -#endif - - #if defined(_TD_LINUX_64) || defined(_TD_LINUX_32) || defined(_TD_MIPS_64) || defined(_TD_ARM_32) || defined(_TD_ARM_64) || defined(_TD_DARWIN_64) #if defined(_TD_DARWIN_64) // MacOS @@ -53,6 +48,16 @@ extern "C" { #endif + +// TODO: replace and remove code below +#define CHAR_BYTES sizeof(char) +#define SHORT_BYTES sizeof(int16_t) +#define INT_BYTES sizeof(int32_t) +#define LONG_BYTES sizeof(int64_t) +#define FLOAT_BYTES sizeof(float) +#define DOUBLE_BYTES sizeof(double) +#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) + #ifdef __cplusplus } #endif diff --git a/include/os/osFile.h b/include/os/osFile.h index 86c08c0a3a..66c3ae9cbd 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_OS_FILE_H -#define TDENGINE_OS_FILE_H +#ifndef _TD_OS_FILE_H_ +#define _TD_OS_FILE_H_ #ifdef __cplusplus extern "C" { @@ -22,13 +22,6 @@ extern "C" { #include "osSocket.h" -#define FD_VALID(x) ((x) > STDERR_FILENO) -#define FD_INITIALIZER ((int32_t)-1) - -#ifndef PATH_MAX - #define PATH_MAX 256 -#endif - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) typedef int32_t FileFd; typedef SOCKET SocketFd; @@ -37,40 +30,44 @@ typedef int32_t FileFd; typedef int32_t SocketFd; #endif -int64_t taosRead(FileFd fd, void *buf, int64_t count); -int64_t taosWrite(FileFd fd, void *buf, int64_t count); +#define FD_INITIALIZER ((int32_t)-1) -int64_t taosLSeek(FileFd fd, int64_t offset, int32_t whence); -int32_t taosFtruncate(FileFd fd, int64_t length); -int32_t taosFsync(FileFd fd); +#ifndef PATH_MAX +#define PATH_MAX 256 +#endif -int32_t taosRename(char* oldName, char *newName); -int64_t taosCopy(char *from, char *to); +int32_t taosLockFile(FileFd fd); +int32_t taosUnLockFile(FileFd fd); + +int32_t taosUmaskFile(FileFd fd); + +int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime); +int32_t taosFStatFile(FileFd fd, int64_t *size, int32_t *mtime); + +FileFd taosOpenFileWrite(const char *path); +FileFd taosOpenFileCreateWrite(const char *path); +FileFd taosOpenFileTruncCreateWrite(const char *path); +FileFd taosOpenFileRead(const char *path); + +int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence); +int32_t taosFtruncateFile(FileFd fd, int64_t length); +int32_t taosFsyncFile(FileFd fd); + +int64_t taosReadFile(FileFd fd, void *buf, int64_t count); +int64_t taosWriteFile(FileFd fd, void *buf, int64_t count); + +void taosCloseFile(FileFd fd); + +int32_t taosRenameFile(char *oldName, char *newName); +int64_t taosCopyFile(char *from, char *to); + +void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size); int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size); -void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath); -void taosClose(FileFd fd); - -#ifdef TAOS_RANDOM_FILE_FAIL - void taosSetRandomFileFailFactor(int32_t factor); - void taosSetRandomFileFailOutput(const char *path); - #ifdef TAOS_RANDOM_FILE_FAIL_TEST - int64_t taosReadFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line); - int64_t taosWriteFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line); - int64_t taosLSeekRandomFail(int32_t fd, int64_t offset, int32_t whence, const char *file, uint32_t line); - #undef taosRead - #undef taosWrite - #undef taosLSeek - #define taosRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__) - #define taosWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__) - #define taosLSeek(fd, offset, whence) taosLSeekRandomFail(fd, offset, whence, __FILE__, __LINE__) - #endif -#endif - #ifdef __cplusplus } #endif -#endif +#endif /*_TD_OS_FILE_H_*/ diff --git a/include/os/osRand.h b/include/os/osRand.h new file mode 100644 index 0000000000..e08768c2cc --- /dev/null +++ b/include/os/osRand.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_RAND_H +#define TDENGINE_OS_RAND_H + +#ifdef __cplusplus +extern "C" { +#endif + +uint32_t taosRand(void); +void taosRandStr(char* str, int32_t size); +uint32_t taosSafeRand(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/os/osSemaphore.h b/include/os/osSemaphore.h index 10d14700e0..d945e721c2 100644 --- a/include/os/osSemaphore.h +++ b/include/os/osSemaphore.h @@ -20,6 +20,8 @@ extern "C" { #endif +#include + #if defined (_TD_DARWIN_64) typedef struct tsem_s *tsem_t; int tsem_init(tsem_t *sem, int pshared, unsigned int value); diff --git a/include/os/osSleep.h b/include/os/osSleep.h new file mode 100644 index 0000000000..e42da8d5a6 --- /dev/null +++ b/include/os/osSleep.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_SLEEP_H +#define TDENGINE_OS_SLEEP_H + +#ifdef __cplusplus +extern "C" { +#endif + +void taosMsleep(int32_t ms); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 29653f1269..0b183b9ece 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -20,6 +20,19 @@ extern "C" { #endif +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + #include "winsock2.h" + #include + #include + #include +#else + #include + #include + #include + #include + #include +#endif + #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #define taosSend(sockfd, buf, len, flags) send((SOCKET)sockfd, buf, len, flags) #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) sendto((SOCKET)sockfd, buf, len, flags, dest_addr, addrlen) @@ -35,50 +48,20 @@ extern "C" { #define taosCloseSocketNoCheck(x) close(x) #define taosCloseSocket(x) \ { \ - if (FD_VALID(x)) { \ + if ((x) > -1) { \ close(x); \ x = FD_INITIALIZER; \ } \ } #endif -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - #define TAOS_EPOLL_WAIT_TIME 100 - typedef SOCKET eventfd_t; - #define eventfd(a, b) -1 - typedef SOCKET EpollFd; - #define EpollClose(pollFd) epoll_close(pollFd) - #ifndef EPOLLWAKEUP - #define EPOLLWAKEUP (1u << 29) - #endif -#elif defined(_TD_DARWIN_64) - #define TAOS_EPOLL_WAIT_TIME 500 - typedef int32_t SOCKET; - typedef SOCKET EpollFd; - #define EpollClose(pollFd) epoll_close(pollFd) -#else - #define TAOS_EPOLL_WAIT_TIME 500 - typedef int32_t SOCKET; - typedef SOCKET EpollFd; - #define EpollClose(pollFd) taosCloseSocket(pollFd) -#endif +#define TAOS_EPOLL_WAIT_TIME 500 +typedef int32_t SOCKET; +typedef SOCKET EpollFd; +#define EpollClose(pollFd) taosCloseSocket(pollFd) -#ifdef TAOS_RANDOM_NETWORK_FAIL - #ifdef TAOS_RANDOM_NETWORK_FAIL_TEST - int64_t taosSendRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags); - int64_t taosSendToRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags, const struct sockaddr *dest_addr, socklen_t addrlen); - int64_t taosReadSocketRandomFail(int32_t fd, void *buf, size_t count); - int64_t taosWriteSocketRandomFail(int32_t fd, const void *buf, size_t count); - #undef taosSend - #undef taosSendto - #undef taosReadSocket - #undef taosWriteSocket - #define taosSend(sockfd, buf, len, flags) taosSendRandomFail(sockfd, buf, len, flags) - #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) taosSendToRandomFail(sockfd, buf, len, flags, dest_addr, addrlen) - #define taosReadSocket(fd, buf, len) taosReadSocketRandomFail(fd, buf, len) - #define taosWriteSocket(fd, buf, len) taosWriteSocketRandomFail(fd, buf, len) - #endif -#endif +void taosShutDownSocketRD(SOCKET fd); +void taosShutDownSocketWR(SOCKET fd); int32_t taosSetNonblocking(SOCKET sock, int32_t on); void taosIgnSIGPIPE(); @@ -88,9 +71,7 @@ int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *op int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t* optlen); uint32_t taosInetAddr(char *ipAddr); -#if 0 const char *taosInetNtoa(struct in_addr ipInt); -#endif #if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) #define htobe64 htonll diff --git a/include/os/osThread.h b/include/os/osThread.h new file mode 100644 index 0000000000..ecb085cd06 --- /dev/null +++ b/include/os/osThread.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_OS_THREAD_H_ +#define _TD_OS_THREAD_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_OS_THREAD_H_*/ diff --git a/include/os/osTime.h b/include/os/osTime.h index 81163b29e7..c5ee26e6ef 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -20,20 +20,22 @@ extern "C" { #endif +int32_t taosGetTimeOfDay(struct timeval *tv); + //@return timestamp in second int32_t taosGetTimestampSec(); //@return timestamp in millisecond static FORCE_INLINE int64_t taosGetTimestampMs() { struct timeval systemTime; - gettimeofday(&systemTime, NULL); + taosGetTimeOfDay(&systemTime); return (int64_t)systemTime.tv_sec * 1000L + (int64_t)systemTime.tv_usec / 1000; } //@return timestamp in microsecond static FORCE_INLINE int64_t taosGetTimestampUs() { struct timeval systemTime; - gettimeofday(&systemTime, NULL); + taosGetTimeOfDay(&systemTime); return (int64_t)systemTime.tv_sec * 1000000L + (int64_t)systemTime.tv_usec; } diff --git a/include/util/tidpool.h b/include/util/tidpool.h new file mode 100644 index 0000000000..e4439439ce --- /dev/null +++ b/include/util/tidpool.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TIDPOOL_H +#define TDENGINE_TIDPOOL_H + +#ifdef __cplusplus +extern "C" { +#endif + +void *taosInitIdPool(int maxId); + +int taosUpdateIdPool(void *handle, int maxId); + +int taosIdPoolMaxSize(void *handle); + +int taosAllocateId(void *handle); + +void taosFreeId(void *handle, int id); + +void taosIdPoolCleanUp(void *handle); + +int taosIdPoolNumOfUsed(void *handle); + +bool taosIdPoolMarkStatus(void *handle, int id); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/util/tmempool.h b/include/util/tmempool.h new file mode 100644 index 0000000000..f2c6a0ef00 --- /dev/null +++ b/include/util/tmempool.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_TMEMPOOL_H +#define TDENGINE_TMEMPOOL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define mpool_h void * + +mpool_h taosMemPoolInit(int maxNum, int blockSize); + +char *taosMemPoolMalloc(mpool_h handle); + +void taosMemPoolFree(mpool_h handle, char *p); + +void taosMemPoolCleanUp(mpool_h handle); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/util/tref.h b/include/util/tref.h new file mode 100644 index 0000000000..085c10c551 --- /dev/null +++ b/include/util/tref.h @@ -0,0 +1,77 @@ + +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TREF_H +#define TDENGINE_TREF_H + +#ifdef __cplusplus +extern "C" { +#endif + +// open a reference set, max is the mod used by hash, fp is the pointer to free resource function +// return rsetId which will be used by other APIs. On error, -1 is returned, and terrno is set appropriately +int taosOpenRef(int max, void (*fp)(void *)); + +// close the reference set, refId is the return value by taosOpenRef +// return 0 if success. On error, -1 is returned, and terrno is set appropriately +int taosCloseRef(int refId); + +// add ref, p is the pointer to resource or pointer ID +// return Reference ID(rid) allocated. On error, -1 is returned, and terrno is set appropriately +int64_t taosAddRef(int refId, void *p); + +// remove ref, rid is the reference ID returned by taosAddRef +// return 0 if success. On error, -1 is returned, and terrno is set appropriately +int taosRemoveRef(int rsetId, int64_t rid); + +// acquire ref, rid is the reference ID returned by taosAddRef +// return the resource p. On error, NULL is returned, and terrno is set appropriately +void *taosAcquireRef(int rsetId, int64_t rid); + +// release ref, rid is the reference ID returned by taosAddRef +// return 0 if success. On error, -1 is returned, and terrno is set appropriately +int taosReleaseRef(int rsetId, int64_t rid); + +// return the first reference if rid is 0, otherwise return the next after current reference. +// if return value is NULL, it means list is over(if terrno is set, it means error happens) +void *taosIterateRef(int rsetId, int64_t rid); + +// return the number of references in system +int taosListRef(); + +#define RID_VALID(x) ((x) > 0) + +/* sample code to iterate the refs + +void demoIterateRefs(int rsetId) { + + void *p = taosIterateRef(refId, 0); + while (p) { + // process P + + // get the rid from p + + p = taosIterateRef(rsetId, rid); + } +} + +*/ + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TREF_H diff --git a/include/util/tutil.h b/include/util/tutil.h index 6bcfb5de29..16e5af1ade 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -23,7 +23,6 @@ extern "C" { #include "os.h" #include "tmd5.h" #include "tcrc32c.h" -#include "taosdef.h" int32_t strdequote(char *src); int32_t strRmquote(char *z, int32_t len); @@ -46,14 +45,19 @@ int taosCheckVersion(char *input_client_version, char *input_server_version, in char * taosIpStr(uint32_t ipInt); uint32_t ip2uint(const char *const ip_addr); -static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) { +static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target, int32_t keylen) { MD5_CTX context; MD5Init(&context); MD5Update(&context, inBuf, (unsigned int)inLen); MD5Final(&context); - memcpy(target, context.digest, TSDB_KEY_LEN); + memcpy(target, context.digest, keylen); } +#ifdef tListLen +#undefine tListLen +#endif +#define tListLen(x) (sizeof(x) / sizeof((x)[0])) + #ifdef __cplusplus } #endif diff --git a/source/libs/transport/CMakeLists.txt b/source/libs/transport/CMakeLists.txt index c01996cb8c..98a380dc8f 100644 --- a/source/libs/transport/CMakeLists.txt +++ b/source/libs/transport/CMakeLists.txt @@ -4,4 +4,12 @@ target_include_directories( transport PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/transport" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + transport + PUBLIC lz4_static + PUBLIC os + PUBLIC util + PUBLIC common ) \ No newline at end of file diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index 2549518249..5eb7e39ff2 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -22,6 +22,9 @@ #include "rpcHead.h" #include "rpcTcp.h" + +#include + typedef struct SFdObj { void *signature; SOCKET fd; // TCP socket FD @@ -195,16 +198,7 @@ void taosStopTcpServer(void *handle) { pServerObj->stop = 1; if (pServerObj->fd >= 0) { -#ifdef WINDOWS - closesocket(pServerObj->fd); -#elif defined(__APPLE__) - if (pServerObj->fd!=-1) { - close(pServerObj->fd); - pServerObj->fd = -1; - } -#else - shutdown(pServerObj->fd, SHUT_RD); -#endif + taosShutDownSocketRD(pServerObj->fd); } if (taosCheckPthreadValid(pServerObj->thread)) { if (taosComparePthread(pServerObj->thread, pthread_self())) { @@ -267,8 +261,8 @@ static void *taosAcceptTcpConnection(void *arg) { int32_t ret = taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to)); if (ret != 0) { taosCloseSocket(connFd); - tError("%s failed to set recv timeout fd(%s)for connection from:%s:%hu", pServerObj->label, strerror(errno), - taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port)); + tError("%s failed to set recv timeout fd(%s)for connection from:%hu", pServerObj->label, strerror(errno), + htons(caddr.sin_port)); continue; } @@ -280,12 +274,12 @@ static void *taosAcceptTcpConnection(void *arg) { if (pFdObj) { pFdObj->ip = caddr.sin_addr.s_addr; pFdObj->port = htons(caddr.sin_port); - tDebug("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label, - taosInetNtoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds); + tDebug("%s new TCP connection from %hu, fd:%d FD:%p numOfFds:%d", pServerObj->label, + pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds); } else { taosCloseSocket(connFd); - tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno), - taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port)); + tError("%s failed to malloc FdObj(%s) for connection from:%hu", pServerObj->label, strerror(errno), + htons(caddr.sin_port)); } // pick up next thread for next connection @@ -436,7 +430,7 @@ void taosCloseTcpConnection(void *chandle) { // pFdObj->thandle = NULL; pFdObj->closedByApp = 1; - shutdown(pFdObj->fd, SHUT_WR); + taosShutDownSocketWR(pFdObj->fd); } int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) { @@ -456,7 +450,7 @@ static void taosReportBrokenLink(SFdObj *pFdObj) { // notify the upper layer, so it will clean the associated context if (pFdObj->closedByApp == 0) { - shutdown(pFdObj->fd, SHUT_WR); + taosShutDownSocketWR(pFdObj->fd); SRecvInfo recvInfo; recvInfo.msg = NULL; diff --git a/src/os/src/detail/osFile.c b/source/os/src/osFile.c similarity index 68% rename from src/os/src/detail/osFile.c rename to source/os/src/osFile.c index cc12968c72..5acbfd6fa7 100644 --- a/src/os/src/detail/osFile.c +++ b/source/os/src/osFile.c @@ -13,24 +13,29 @@ * along with this program. If not, see . */ -#define _DEFAULT_SOURCE #include "os.h" -#include "tglobal.h" -#include "tulog.h" -void taosClose(FileFd fd) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#else +#include +#include +#include +#include +#include +#endif + +void taosCloseFile(FileFd fd) { close(fd); fd = FD_INITIALIZER; } +void taosGetTmpfilePath(const char * inputTmpDir, const char *fileNamePrefix, char *dstPath) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - -void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { const char *tdengineTmpFileNamePrefix = "tdengine-"; char tmpPath[PATH_MAX]; - int32_t len = (int32_t)strlen(tsTempDir); - memcpy(tmpPath, tsTempDir, len); + int32_t len = (int32_t)strlen(inputTmpDir); + memcpy(tmpPath, inputTmpDir, len); if (tmpPath[len - 1] != '/' && tmpPath[len - 1] != '\\') { tmpPath[len++] = '\\'; @@ -46,16 +51,14 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { char rand[8] = {0}; taosRandStr(rand, tListLen(rand) - 1); snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); -} #else -void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { const char *tdengineTmpFileNamePrefix = "tdengine-"; - char tmpPath[PATH_MAX]; - int32_t len = strlen(tsTempDir); - memcpy(tmpPath, tsTempDir, len); + char tmpPath[PATH_MAX]; + int32_t len = strlen(inputTmpDir); + memcpy(tmpPath, inputTmpDir, len); static uint64_t seqId = 0; if (tmpPath[len - 1] != '/') { @@ -73,11 +76,11 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { sprintf(rand, "%" PRIu64, atomic_add_fetch_64(&seqId, 1)); snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); -} #endif +} -int64_t taosRead(FileFd fd, void *buf, int64_t count) { +int64_t taosReadFile(FileFd fd, void *buf, int64_t count) { int64_t leftbytes = count; int64_t readbytes; char * tbuf = (char *)buf; @@ -101,7 +104,7 @@ int64_t taosRead(FileFd fd, void *buf, int64_t count) { return count; } -int64_t taosWrite(FileFd fd, void *buf, int64_t n) { +int64_t taosWriteFile(FileFd fd, void *buf, int64_t n) { int64_t nleft = n; int64_t nwritten = 0; char * tbuf = (char *)buf; @@ -121,42 +124,46 @@ int64_t taosWrite(FileFd fd, void *buf, int64_t n) { return n; } -int64_t taosLSeek(FileFd fd, int64_t offset, int32_t whence) { return (int64_t)lseek(fd, (long)offset, whence); } +int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence) { return (int64_t)lseek(fd, (long)offset, whence); } -int64_t taosCopy(char *from, char *to) { +int64_t taosCopyFile(char *from, char *to) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else char buffer[4096]; int fidto = -1, fidfrom = -1; int64_t size = 0; int64_t bytes; - fidfrom = open(from, O_RDONLY | O_BINARY); + fidfrom = open(from, O_RDONLY); if (fidfrom < 0) goto _err; - fidto = open(to, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0755); + fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755); if (fidto < 0) goto _err; while (true) { - bytes = taosRead(fidfrom, buffer, sizeof(buffer)); + bytes = taosReadFile(fidfrom, buffer, sizeof(buffer)); if (bytes < 0) goto _err; if (bytes == 0) break; size += bytes; - if (taosWrite(fidto, (void *)buffer, bytes) < bytes) goto _err; + if (taosWriteFile(fidto, (void *)buffer, bytes) < bytes) goto _err; if (bytes < sizeof(buffer)) break; } - taosFsync(fidto); + taosFsyncFile(fidto); - taosClose(fidfrom); - taosClose(fidto); + taosCloseFile(fidfrom); + taosCloseFile(fidto); return size; _err: - if (fidfrom >= 0) taosClose(fidfrom); - if (fidto >= 0) taosClose(fidto); + if (fidfrom >= 0) taosCloseFile(fidfrom); + if (fidto >= 0) taosCloseFile(fidto); remove(to); return -1; +#endif } #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) @@ -306,9 +313,8 @@ int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size #endif +int32_t taosFtruncateFile(FileFd fd, int64_t l_size) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - -int32_t taosFtruncate(int32_t fd, int64_t l_size) { if (fd < 0) { errno = EBADF; uError("%s\n", "fd arg was negative"); @@ -357,9 +363,13 @@ int32_t taosFtruncate(int32_t fd, int64_t l_size) { } return 0; +#else + return ftruncate(fd, l_size); +#endif } -int32_t taosFsync(FileFd fd) { +int32_t taosFsyncFile(FileFd fd) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) if (fd < 0) { errno = EBADF; uError("%s\n", "fd arg was negative"); @@ -369,33 +379,126 @@ int32_t taosFsync(FileFd fd) { HANDLE h = (HANDLE)_get_osfhandle(fd); return FlushFileBuffers(h); +#else + return fsync(fd); +#endif } -int32_t taosRename(char *oldName, char *newName) { +int32_t taosRenameFile(char *oldName, char *newName) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); if (code < 0) { - uError("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); - } else { - uTrace("successfully to rename file %s to %s", oldName, newName); + printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); } return code; -} - #else - -int32_t taosFtruncate(FileFd fd, int64_t length) { return ftruncate(fd, length); } -int32_t taosFsync(FileFd fd) { return fsync(fd); } - -int32_t taosRename(char *oldName, char *newName) { int32_t code = rename(oldName, newName); if (code < 0) { - uError("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); - } else { - uTrace("successfully to rename file %s to %s", oldName, newName); + printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); } return code; +#endif } + +int32_t taosLockFile(int32_t fd) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + return (int32_t)flock(fd, LOCK_EX | LOCK_NB); #endif +} + +int32_t taosUnLockFile(int32_t fd) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + return (int32_t)flock(fd, LOCK_UN | LOCK_NB); +#endif +} + +int32_t taosUmaskFile(int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + return umask(val); +#endif +} + +int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + struct stat fileStat; + int32_t code = stat(path, &fileStat); + if (code < 0) { + return code; + } + + if (size != NULL) { + *size = fileStat.st_size; + } + + if (mtime != NULL) { + *mtime = fileStat.st_mtime; + } + + return 0; +#endif +} + +int32_t taosFStatFile(int32_t fd, int64_t *size, int32_t *mtime) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + struct stat fileStat; + int32_t code = fstat(fd, &fileStat); + if (code < 0) { + return code; + } + + if (size != NULL) { + *size = fileStat.st_size; + } + + if (mtime != NULL) { + *mtime = fileStat.st_mtime; + } + + return 0; +#endif +} + +int32_t taosOpenFileWrite(const char *path) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + return open(path, O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); +#endif +} + +FileFd taosOpenFileRead(const char *path) { + #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + return open(path, O_RDONLY); +#endif +} + +int32_t taosOpenFileCreateWrite(const char *path) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + return open(path, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); +#endif +} + +int32_t taosOpenFileTruncCreateWrite(const char *path) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + return open(path, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); +#endif +} \ No newline at end of file diff --git a/src/os/src/detail/osSocket.c b/source/os/src/osSocket.c similarity index 79% rename from src/os/src/detail/osSocket.c rename to source/os/src/osSocket.c index 7ce9d6eb06..7964482db6 100644 --- a/src/os/src/detail/osSocket.c +++ b/source/os/src/osSocket.c @@ -15,7 +15,26 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "tulog.h" + +void taosShutDownSocketRD(SOCKET fd) { +#ifdef WINDOWS + closesocket(fd); +#elif __APPLE__ + close(fd); +#else + shutdown(fd, SHUT_RD); +#endif +} + +void taosShutDownSocketWR(SOCKET fd) { +#ifdef WINDOWS + closesocket(fd); +#elif __APPLE__ + close(fd); +#else + shutdown(fd, SHUT_WR); +#endif +} #if !(defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) @@ -87,4 +106,19 @@ const char *taosInetNtoa(struct in_addr ipInt) { return inet_ntoa(ipInt); } +#else + +const char *taosInetNtoa(struct in_addr ipInt) { + // not thread safe, only for debug usage while print log + static char tmpDstStr[16]; + return inet_ntop(AF_INET, &ipInt, tmpDstStr, INET6_ADDRSTRLEN); +} + #endif + + +#if defined(_TD_GO_DLL_) + +uint64_t htonll(uint64_t val) { return (((uint64_t)htonl(val)) << 32) + htonl(val >> 32); } + +#endif \ No newline at end of file diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c new file mode 100644 index 0000000000..27d046d4c9 --- /dev/null +++ b/source/os/src/osTime.c @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#include +#else +#endif + +FORCE_INLINE int32_t taosGetTimeOfDay(struct timeval *tv) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + time_t t; + t = time(NULL); + SYSTEMTIME st; + GetLocalTime(&st); + + tv->tv_sec = (long)t; + tv->tv_usec = st.wMilliseconds * 1000; + + return 0; +#else + return gettimeofday(tv, NULL); +#endif +} \ No newline at end of file diff --git a/source/server/dnode/src/dnodeCfg.c b/source/server/dnode/src/dnodeCfg.c index a004ad7290..2162e3f791 100644 --- a/source/server/dnode/src/dnodeCfg.c +++ b/source/server/dnode/src/dnodeCfg.c @@ -94,7 +94,7 @@ static int32_t dnodeWriteCfg(DnCfg *cfg) { len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); - taosFsync(fileno(fp)); + taosFsyncFile(fileno(fp)); fclose(fp); free(content); terrno = 0; diff --git a/source/server/dnode/src/dnodeEps.c b/source/server/dnode/src/dnodeEps.c index 725e54bb91..8595b1b339 100644 --- a/source/server/dnode/src/dnodeEps.c +++ b/source/server/dnode/src/dnodeEps.c @@ -173,7 +173,7 @@ static int32_t dnodeWriteEps(DnEps *eps) { len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); - taosFsync(fileno(fp)); + taosFsyncFile(fileno(fp)); fclose(fp); free(content); terrno = 0; diff --git a/source/server/dnode/src/dnodeMnodeEps.c b/source/server/dnode/src/dnodeMnodeEps.c index ba926b0094..9858f9d21f 100644 --- a/source/server/dnode/src/dnodeMnodeEps.c +++ b/source/server/dnode/src/dnodeMnodeEps.c @@ -82,7 +82,7 @@ static int32_t dnodeWriteMnodeEps(DnMnEps *meps) { len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); - taosFsync(fileno(fp)); + taosFsyncFile(fileno(fp)); fclose(fp); free(content); terrno = 0; diff --git a/source/server/dnode/src/dnodeTelemetry.c b/source/server/dnode/src/dnodeTelemetry.c index 51f1e90cff..466470c8c7 100644 --- a/source/server/dnode/src/dnodeTelemetry.c +++ b/source/server/dnode/src/dnodeTelemetry.c @@ -15,7 +15,6 @@ #define _DEFAULT_SOURCE #include "os.h" -// #include "osTime.h" #include "tbuffer.h" #include "tglobal.h" #include "tsocket.h" @@ -255,16 +254,16 @@ static void* dnodeTelemThreadFp(void* param) { } static void dnodeGetEmail(DnTelem* telem, char* filepath) { - int32_t fd = open(filepath, O_RDONLY); + int32_t fd = taosOpenFileRead(filepath); if (fd < 0) { return; } - if (taosRead(fd, (void*)telem->email, TSDB_FQDN_LEN) < 0) { + if (taosReadFile(fd, (void*)telem->email, TSDB_FQDN_LEN) < 0) { dError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno)); } - taosClose(fd); + taosCloseFile(fd); } int32_t dnodeInitTelemetry(Dnode* dnode, DnTelem** out) { diff --git a/source/server/dnode/src/dnodeTrans.c b/source/server/dnode/src/dnodeTrans.c index 4d03338835..d9d2c9ad47 100644 --- a/source/server/dnode/src/dnodeTrans.c +++ b/source/server/dnode/src/dnodeTrans.c @@ -242,7 +242,7 @@ static void dnodeProcessMsgFromShell(DnTrans *trans, SRpcMsg *pMsg, SRpcEpSet *p static int32_t dnodeAuthNetTest(char *user, char *spi, char *encrypt, char *secret, char *ckey) { if (strcmp(user, "nettestinternal") == 0) { char pass[32] = {0}; - taosEncryptPass((uint8_t *)user, strlen(user), pass); + taosEncryptPass((uint8_t *)user, strlen(user), pass, TSDB_KEY_LEN); *spi = 0; *encrypt = 0; *ckey = 0; diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index c53be0c59b..cec2554b52 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -3,7 +3,7 @@ add_library(util ${UTIL_SRC}) target_include_directories( util PUBLIC "${CMAKE_SOURCE_DIR}/include/util" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( util diff --git a/source/util/inc/tulog.h b/source/util/inc/tulog.h new file mode 100644 index 0000000000..566da40a10 --- /dev/null +++ b/source/util/inc/tulog.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_COMMON_ULOG_H +#define TDENGINE_COMMON_ULOG_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tlog.h" + +extern int32_t uDebugFlag; +extern int8_t tscEmbedded; + +#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }} +#define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }} + +#define pError(...) { taosPrintLog("APP ERROR ", 255, __VA_ARGS__); } +#define pPrint(...) { taosPrintLog("APP ", 255, __VA_ARGS__); } + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/source/util/src/hash.c b/source/util/src/hash.c new file mode 100644 index 0000000000..21e835a6bf --- /dev/null +++ b/source/util/src/hash.c @@ -0,0 +1,916 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "hash.h" +#include "tulog.h" +// #include "taosdef.h" + +#define EXT_SIZE 1024 + +#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) + +#define DO_FREE_HASH_NODE(_n) \ + do { \ + tfree(_n); \ + } while (0) + +#define FREE_HASH_NODE(_h, _n) \ + do { \ + if ((_h)->freeFp) { \ + (_h)->freeFp(GET_HASH_NODE_DATA(_n)); \ + } \ + \ + DO_FREE_HASH_NODE(_n); \ + } while (0); + +static FORCE_INLINE void __wr_lock(void *lock, int32_t type) { + if (type == HASH_NO_LOCK) { + return; + } + taosWLockLatch(lock); +} + +static FORCE_INLINE void __rd_lock(void *lock, int32_t type) { + if (type == HASH_NO_LOCK) { + return; + } + + taosRLockLatch(lock); +} + +static FORCE_INLINE void __rd_unlock(void *lock, int32_t type) { + if (type == HASH_NO_LOCK) { + return; + } + + taosRUnLockLatch(lock); +} + +static FORCE_INLINE void __wr_unlock(void *lock, int32_t type) { + if (type == HASH_NO_LOCK) { + return; + } + + taosWUnLockLatch(lock); +} + +static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { + int32_t len = MIN(length, HASH_MAX_CAPACITY); + + int32_t i = 4; + while (i < len) i = (i << 1u); + return i; +} + +static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { + SHashNode *pNode = pe->next; + while (pNode) { + if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { + assert(pNode->hashVal == hashVal); + break; + } + + pNode = pNode->next; + } + + return pNode; +} + +/** + * Resize the hash list if the threshold is reached + * + * @param pHashObj + */ +static void taosHashTableResize(SHashObj *pHashObj); + +/** + * @param key key of object for hash, usually a null-terminated string + * @param keyLen length of key + * @param pData actually data. Requires a consecutive memory block, no pointer is allowed in pData. + * Pointer copy causes memory access error. + * @param dsize size of data + * @return SHashNode + */ +static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal); + +/** + * Update the hash node + * + * @param pNode hash node + * @param key key for generate hash value + * @param keyLen key length + * @param pData actual data + * @param dsize size of actual data + * @return hash node + */ +static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { + assert(pNode->keyLen == pNewNode->keyLen); + + pNode->count--; + if (prev != NULL) { + prev->next = pNewNode; + } else { + pe->next = pNewNode; + } + + if (pNode->count <= 0) { + pNewNode->next = pNode->next; + DO_FREE_HASH_NODE(pNode); + } else { + pNewNode->next = pNode; + pe->num++; + atomic_add_fetch_64(&pHashObj->size, 1); + } + + return pNewNode; +} + +/** + * insert the hash node at the front of the linked list + * + * @param pHashObj + * @param pNode + */ +static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode); + +/** + * Check whether the hash table is empty or not. + * + * @param pHashObj the hash table object + * @return if the hash table is empty or not + */ +static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj); + +/** + * Get the next element in hash table for iterator + * @param pIter + * @return + */ + +SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) { + assert(fn != NULL); + if (capacity == 0) { + capacity = 4; + } + + SHashObj *pHashObj = (SHashObj *)calloc(1, sizeof(SHashObj)); + if (pHashObj == NULL) { + uError("failed to allocate memory, reason:%s", strerror(errno)); + return NULL; + } + + // the max slots is not defined by user + pHashObj->capacity = taosHashCapacity((int32_t)capacity); + assert((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); + pHashObj->equalFp = memcmp; + pHashObj->hashFp = fn; + pHashObj->type = type; + pHashObj->enableUpdate = update; + + pHashObj->hashList = (SHashEntry **)calloc(pHashObj->capacity, sizeof(void *)); + if (pHashObj->hashList == NULL) { + free(pHashObj); + uError("failed to allocate memory, reason:%s", strerror(errno)); + return NULL; + } else { + pHashObj->pMemBlock = taosArrayInit(8, sizeof(void *)); + + void *p = calloc(pHashObj->capacity, sizeof(SHashEntry)); + for (int32_t i = 0; i < pHashObj->capacity; ++i) { + pHashObj->hashList[i] = (void *)((char *)p + i * sizeof(SHashEntry)); + } + + taosArrayPush(pHashObj->pMemBlock, &p); + } + + return pHashObj; +} + +void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp) { + if (pHashObj != NULL && fp != NULL) { + pHashObj->equalFp = fp; + } +} + +int32_t taosHashGetSize(const SHashObj *pHashObj) { + if (!pHashObj) { + return 0; + } + return (int32_t)atomic_load_64(&pHashObj->size); +} + +static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { + return taosHashGetSize(pHashObj) == 0; +} + +int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { + uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); + SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal); + if (pNewNode == NULL) { + return -1; + } + + // need the resize process, write lock applied + if (HASH_NEED_RESIZE(pHashObj)) { + __wr_lock(&pHashObj->lock, pHashObj->type); + taosHashTableResize(pHashObj); + __wr_unlock(&pHashObj->lock, pHashObj->type); + } + + __rd_lock(&pHashObj->lock, pHashObj->type); + + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + SHashEntry *pe = pHashObj->hashList[slot]; + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWLockLatch(&pe->latch); + } + + SHashNode *pNode = pe->next; + if (pe->num > 0) { + assert(pNode != NULL); + } else { + assert(pNode == NULL); + } + + SHashNode* prev = NULL; + while (pNode) { + if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { + assert(pNode->hashVal == hashVal); + break; + } + + prev = pNode; + pNode = pNode->next; + } + + if (pNode == NULL) { + // no data in hash table with the specified key, add it into hash table + pushfrontNodeInEntryList(pe, pNewNode); + + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); + } + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + + // enable resize + __rd_unlock(&pHashObj->lock, pHashObj->type); + atomic_add_fetch_64(&pHashObj->size, 1); + + return 0; + } else { + // not support the update operation, return error + if (pHashObj->enableUpdate) { + doUpdateHashNode(pHashObj, pe, prev, pNode, pNewNode); + } else { + DO_FREE_HASH_NODE(pNewNode); + } + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + + // enable resize + __rd_unlock(&pHashObj->lock, pHashObj->type); + + return pHashObj->enableUpdate ? 0 : -1; + } +} + +void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { + return taosHashGetClone(pHashObj, key, keyLen, NULL, NULL); +} +//TODO(yihaoDeng), merge with taosHashGetClone +void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void** d, size_t *sz) { + if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { + return NULL; + } + + uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); + + // only add the read lock to disable the resize process + __rd_lock(&pHashObj->lock, pHashObj->type); + + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + SHashEntry *pe = pHashObj->hashList[slot]; + + // no data, return directly + if (atomic_load_32(&pe->num) == 0) { + __rd_unlock(&pHashObj->lock, pHashObj->type); + return NULL; + } + + char *data = NULL; + + // lock entry + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosRLockLatch(&pe->latch); + } + + if (pe->num > 0) { + assert(pe->next != NULL); + } else { + assert(pe->next == NULL); + } + + SHashNode *pNode = doSearchInEntryList(pHashObj, pe, key, keyLen, hashVal); + if (pNode != NULL) { + if (fp != NULL) { + fp(GET_HASH_NODE_DATA(pNode)); + } + + if (*d == NULL) { + *sz = pNode->dataLen + EXT_SIZE; + *d = calloc(1, *sz); + } else if (*sz < pNode->dataLen){ + *sz = pNode->dataLen + EXT_SIZE; + *d = realloc(*d, *sz); + } + memcpy((char *)(*d), GET_HASH_NODE_DATA(pNode), pNode->dataLen); + // just make runtime happy + if ((*sz) - pNode->dataLen > 0) { + memset((char *)(*d) + pNode->dataLen, 0, (*sz) - pNode->dataLen); + } + + data = GET_HASH_NODE_DATA(pNode); + } + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosRUnLockLatch(&pe->latch); + } + + __rd_unlock(&pHashObj->lock, pHashObj->type); + return data; +} + +void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d) { + if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { + return NULL; + } + + uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); + + // only add the read lock to disable the resize process + __rd_lock(&pHashObj->lock, pHashObj->type); + + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + SHashEntry *pe = pHashObj->hashList[slot]; + + // no data, return directly + if (atomic_load_32(&pe->num) == 0) { + __rd_unlock(&pHashObj->lock, pHashObj->type); + return NULL; + } + + char *data = NULL; + + // lock entry + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosRLockLatch(&pe->latch); + } + + if (pe->num > 0) { + assert(pe->next != NULL); + } else { + assert(pe->next == NULL); + } + + SHashNode *pNode = doSearchInEntryList(pHashObj, pe, key, keyLen, hashVal); + if (pNode != NULL) { + if (fp != NULL) { + fp(GET_HASH_NODE_DATA(pNode)); + } + + if (d != NULL) { + memcpy(d, GET_HASH_NODE_DATA(pNode), pNode->dataLen); + } + + data = GET_HASH_NODE_DATA(pNode); + } + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosRUnLockLatch(&pe->latch); + } + + __rd_unlock(&pHashObj->lock, pHashObj->type); + return data; +} + +int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { + return taosHashRemoveWithData(pHashObj, key, keyLen, NULL, 0); +} + +int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t dsize) { + if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { + return -1; + } + + uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); + + // disable the resize process + __rd_lock(&pHashObj->lock, pHashObj->type); + + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + SHashEntry *pe = pHashObj->hashList[slot]; + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWLockLatch(&pe->latch); + } + + // double check after locked + if (pe->num == 0) { + assert(pe->next == NULL); + taosWUnLockLatch(&pe->latch); + + __rd_unlock(&pHashObj->lock, pHashObj->type); + return -1; + } + + int code = -1; + SHashNode *pNode = pe->next; + SHashNode *prevNode = NULL; + + while (pNode) { + if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) + break; + + prevNode = pNode; + pNode = pNode->next; + } + + if (pNode) { + code = 0; // it is found + + pNode->count--; + pNode->removed = 1; + if (pNode->count <= 0) { + if (prevNode) { + prevNode->next = pNode->next; + } else { + pe->next = pNode->next; + } + + if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize); + + pe->num--; + atomic_sub_fetch_64(&pHashObj->size, 1); + FREE_HASH_NODE(pHashObj, pNode); + } + } + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + + __rd_unlock(&pHashObj->lock, pHashObj->type); + + return code; +} + +int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param) { + if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { + return 0; + } + + // disable the resize process + __rd_lock(&pHashObj->lock, pHashObj->type); + + int32_t numOfEntries = (int32_t)pHashObj->capacity; + for (int32_t i = 0; i < numOfEntries; ++i) { + SHashEntry *pEntry = pHashObj->hashList[i]; + if (pEntry->num == 0) { + continue; + } + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWLockLatch(&pEntry->latch); + } + + // todo remove the first node + SHashNode *pNode = NULL; + while((pNode = pEntry->next) != NULL) { + if (fp && (!fp(param, GET_HASH_NODE_DATA(pNode)))) { + pEntry->num -= 1; + atomic_sub_fetch_64(&pHashObj->size, 1); + + pEntry->next = pNode->next; + + if (pEntry->num == 0) { + assert(pEntry->next == NULL); + } else { + assert(pEntry->next != NULL); + } + + FREE_HASH_NODE(pHashObj, pNode); + } else { + break; + } + } + + // handle the following node + if (pNode != NULL) { + assert(pNode == pEntry->next); + SHashNode *pNext = NULL; + + while ((pNext = pNode->next) != NULL) { + // not qualified, remove it + if (fp && (!fp(param, GET_HASH_NODE_DATA(pNext)))) { + pNode->next = pNext->next; + pEntry->num -= 1; + atomic_sub_fetch_64(&pHashObj->size, 1); + + if (pEntry->num == 0) { + assert(pEntry->next == NULL); + } else { + assert(pEntry->next != NULL); + } + + FREE_HASH_NODE(pHashObj, pNext); + } else { + pNode = pNext; + } + } + } + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pEntry->latch); + } + } + + __rd_unlock(&pHashObj->lock, pHashObj->type); + return 0; +} + +void taosHashClear(SHashObj *pHashObj) { + if (pHashObj == NULL) { + return; + } + + SHashNode *pNode, *pNext; + + __wr_lock(&pHashObj->lock, pHashObj->type); + + for (int32_t i = 0; i < pHashObj->capacity; ++i) { + SHashEntry *pEntry = pHashObj->hashList[i]; + if (pEntry->num == 0) { + assert(pEntry->next == 0); + continue; + } + + pNode = pEntry->next; + assert(pNode != NULL); + + while (pNode) { + pNext = pNode->next; + FREE_HASH_NODE(pHashObj, pNode); + + pNode = pNext; + } + + pEntry->num = 0; + pEntry->next = NULL; + } + + pHashObj->size = 0; + __wr_unlock(&pHashObj->lock, pHashObj->type); +} + +void taosHashCleanup(SHashObj *pHashObj) { + if (pHashObj == NULL) { + return; + } + + taosHashClear(pHashObj); + tfree(pHashObj->hashList); + + // destroy mem block + size_t memBlock = taosArrayGetSize(pHashObj->pMemBlock); + for (int32_t i = 0; i < memBlock; ++i) { + void *p = taosArrayGetP(pHashObj->pMemBlock, i); + tfree(p); + } + + taosArrayDestroy(pHashObj->pMemBlock); + + memset(pHashObj, 0, sizeof(SHashObj)); + free(pHashObj); +} + +// for profile only +int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) { + if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { + return 0; + } + + int32_t num = 0; + + for (int32_t i = 0; i < pHashObj->size; ++i) { + SHashEntry *pEntry = pHashObj->hashList[i]; + if (num < pEntry->num) { + num = pEntry->num; + } + } + + return num; +} + +void taosHashTableResize(SHashObj *pHashObj) { + if (!HASH_NEED_RESIZE(pHashObj)) { + return; + } + + // double the original capacity + SHashNode *pNode = NULL; + SHashNode *pNext = NULL; + + int32_t newSize = (int32_t)(pHashObj->capacity << 1u); + if (newSize > HASH_MAX_CAPACITY) { + // uDebug("current capacity:%d, maximum capacity:%d, no resize applied due to limitation is reached", + // pHashObj->capacity, HASH_MAX_CAPACITY); + return; + } + + int64_t st = taosGetTimestampUs(); + void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newSize); + if (pNewEntryList == NULL) { // todo handle error + // uDebug("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity); + return; + } + + pHashObj->hashList = pNewEntryList; + + size_t inc = newSize - pHashObj->capacity; + void * p = calloc(inc, sizeof(SHashEntry)); + + for (int32_t i = 0; i < inc; ++i) { + pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry)); + } + + taosArrayPush(pHashObj->pMemBlock, &p); + + pHashObj->capacity = newSize; + for (int32_t i = 0; i < pHashObj->capacity; ++i) { + SHashEntry *pe = pHashObj->hashList[i]; + + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); + } + + if (pe->num == 0) { + assert(pe->next == NULL); + continue; + } + + while ((pNode = pe->next) != NULL) { + int32_t j = HASH_INDEX(pNode->hashVal, pHashObj->capacity); + if (j != i) { + pe->num -= 1; + pe->next = pNode->next; + + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); + } + + SHashEntry *pNewEntry = pHashObj->hashList[j]; + pushfrontNodeInEntryList(pNewEntry, pNode); + } else { + break; + } + } + + if (pNode != NULL) { + while ((pNext = pNode->next) != NULL) { + int32_t j = HASH_INDEX(pNext->hashVal, pHashObj->capacity); + if (j != i) { + pe->num -= 1; + + pNode->next = pNext->next; + pNext->next = NULL; + + // added into new slot + SHashEntry *pNewEntry = pHashObj->hashList[j]; + + if (pNewEntry->num == 0) { + assert(pNewEntry->next == NULL); + } else { + assert(pNewEntry->next != NULL); + } + + pushfrontNodeInEntryList(pNewEntry, pNext); + } else { + pNode = pNext; + } + } + + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); + } + + } + + } + + int64_t et = taosGetTimestampUs(); + + uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity, + ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); +} + +SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { + SHashNode *pNewNode = malloc(sizeof(SHashNode) + keyLen + dsize); + + if (pNewNode == NULL) { + uError("failed to allocate memory, reason:%s", strerror(errno)); + return NULL; + } + + pNewNode->keyLen = (uint32_t)keyLen; + pNewNode->hashVal = hashVal; + pNewNode->dataLen = (uint32_t) dsize; + pNewNode->count = 1; + pNewNode->removed = 0; + pNewNode->next = NULL; + + memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize); + memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen); + + return pNewNode; +} + +void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode) { + assert(pNode != NULL && pEntry != NULL); + + pNode->next = pEntry->next; + pEntry->next = pNode; + + pEntry->num += 1; +} + +size_t taosHashGetMemSize(const SHashObj *pHashObj) { + if (pHashObj == NULL) { + return 0; + } + + return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj); +} + +FORCE_INLINE void *taosHashGetDataKey(SHashObj *pHashObj, void *data) { + SHashNode * node = GET_HASH_PNODE(data); + return GET_HASH_NODE_KEY(node); +} + +FORCE_INLINE uint32_t taosHashGetDataKeyLen(SHashObj *pHashObj, void *data) { + SHashNode * node = GET_HASH_PNODE(data); + return node->keyLen; +} + + +// release the pNode, return next pNode, and lock the current entry +static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { + + SHashNode *pOld = (SHashNode *)GET_HASH_PNODE(p); + SHashNode *prevNode = NULL; + + *slot = HASH_INDEX(pOld->hashVal, pHashObj->capacity); + SHashEntry *pe = pHashObj->hashList[*slot]; + + // lock entry + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWLockLatch(&pe->latch); + } + + SHashNode *pNode = pe->next; + + while (pNode) { + if (pNode == pOld) + break; + + prevNode = pNode; + pNode = pNode->next; + } + + if (pNode) { + pNode = pNode->next; + while (pNode) { + if (pNode->removed == 0) break; + pNode = pNode->next; + } + + pOld->count--; + if (pOld->count <=0) { + if (prevNode) { + prevNode->next = pOld->next; + } else { + pe->next = pOld->next; + } + + pe->num--; + atomic_sub_fetch_64(&pHashObj->size, 1); + FREE_HASH_NODE(pHashObj, pOld); + } + } else { + uError("pNode:%p data:%p is not there!!!", pNode, p); + } + + return pNode; +} + +void *taosHashIterate(SHashObj *pHashObj, void *p) { + if (pHashObj == NULL) return NULL; + + int slot = 0; + char *data = NULL; + + // only add the read lock to disable the resize process + __rd_lock(&pHashObj->lock, pHashObj->type); + + SHashNode *pNode = NULL; + if (p) { + pNode = taosHashReleaseNode(pHashObj, p, &slot); + if (pNode == NULL) { + SHashEntry *pe = pHashObj->hashList[slot]; + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + + slot = slot + 1; + } + } + + if (pNode == NULL) { + for (; slot < pHashObj->capacity; ++slot) { + SHashEntry *pe = pHashObj->hashList[slot]; + + // lock entry + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWLockLatch(&pe->latch); + } + + pNode = pe->next; + while (pNode) { + if (pNode->removed == 0) break; + pNode = pNode->next; + } + + if (pNode) break; + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + } + } + + if (pNode) { + SHashEntry *pe = pHashObj->hashList[slot]; + pNode->count++; + data = GET_HASH_NODE_DATA(pNode); + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + } + + __rd_unlock(&pHashObj->lock, pHashObj->type); + return data; + +} + +void taosHashCancelIterate(SHashObj *pHashObj, void *p) { + if (pHashObj == NULL || p == NULL) return; + + // only add the read lock to disable the resize process + __rd_lock(&pHashObj->lock, pHashObj->type); + + int slot; + taosHashReleaseNode(pHashObj, p, &slot); + + SHashEntry *pe = pHashObj->hashList[slot]; + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + + __rd_unlock(&pHashObj->lock, pHashObj->type); +} diff --git a/src/util/src/tlog.c b/source/util/src/tlog.c similarity index 89% rename from src/util/src/tlog.c rename to source/util/src/tlog.c index 1ce3eadf58..1906ea1629 100644 --- a/src/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -19,7 +19,7 @@ #include "tlog.h" #include "tnote.h" #include "tutil.h" - + #define MAX_LOGLINE_SIZE (1000) #define MAX_LOGLINE_BUFFER_SIZE (MAX_LOGLINE_SIZE + 10) #define MAX_LOGLINE_CONTENT_SIZE (MAX_LOGLINE_SIZE - 100) @@ -85,6 +85,8 @@ int64_t dbgWSize = 0; char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/power"; #elif (_TD_TQ_ == true) char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/tq"; +#elif (_TD_PRO_ == true) +char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/ProDB"; #else char tsLogDir[PATH_MAX] = "/var/log/taos"; #endif @@ -134,11 +136,11 @@ void taosCloseLog() { // taosCloseLog(); } -static bool taosLockFile(int32_t fd) { +static bool taosLockLogFile(int32_t fd) { if (fd < 0) return false; if (tsLogObj.fileNum > 1) { - int32_t ret = flock(fd, LOCK_EX | LOCK_NB); + int32_t ret = taosUnLockFile(fd); if (ret == 0) { return true; } @@ -147,11 +149,11 @@ static bool taosLockFile(int32_t fd) { return false; } -static void taosUnLockFile(int32_t fd) { +static void taosUnLockLogFile(int32_t fd) { if (fd < 0) return; if (tsLogObj.fileNum > 1) { - flock(fd, LOCK_UN | LOCK_NB); + taosUnLockFile(fd); } } @@ -183,9 +185,9 @@ static void *taosThreadToOpenNewFile(void *param) { char name[LOG_FILE_NAME_LEN + 20]; sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag); - umask(0); + taosUmaskFile(0); - int32_t fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); + int32_t fd = taosOpenFileTruncCreateWrite(name); if (fd < 0) { tsLogObj.openInProgress = 0; tsLogObj.lines = tsLogObj.maxLines - 1000; @@ -193,8 +195,8 @@ static void *taosThreadToOpenNewFile(void *param) { return NULL; } - taosLockFile(fd); - (void)lseek(fd, 0, SEEK_SET); + taosLockLogFile(fd); + (void)taosLSeekFile(fd, 0, SEEK_SET); int32_t oldFd = tsLogObj.logHandle->fd; tsLogObj.logHandle->fd = fd; @@ -246,7 +248,7 @@ void taosResetLog() { } static bool taosCheckFileIsOpen(char *logFileName) { - int32_t fd = open(logFileName, O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); + int32_t fd = taosOpenFileWrite(logFileName); if (fd < 0) { if (errno == ENOENT) { return false; @@ -256,12 +258,12 @@ static bool taosCheckFileIsOpen(char *logFileName) { } } - if (taosLockFile(fd)) { - taosUnLockFile(fd); - taosClose(fd); + if (taosLockLogFile(fd)) { + taosUnLockLogFile(fd); + taosCloseFile(fd); return false; } else { - taosClose(fd); + taosCloseFile(fd); return true; } } @@ -298,9 +300,9 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { maxFileNum = 1; #endif - char name[LOG_FILE_NAME_LEN + 50] = "\0"; - struct stat logstat0, logstat1; - int32_t size; + char name[LOG_FILE_NAME_LEN + 50] = "\0"; + int32_t logstat0_mtime, logstat1_mtime; + int32_t size; tsLogObj.maxLines = maxLines; tsLogObj.fileNum = maxFileNum; @@ -310,14 +312,14 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { strcpy(name, fn); strcat(name, ".0"); } - bool log0Exist = stat(name, &logstat0) >= 0; + bool log0Exist = taosStatFile(name, NULL, &logstat0_mtime) >= 0; if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) { strcpy(name, fn); strcat(name, ".1"); } - bool log1Exist = stat(name, &logstat1) >= 0; - + bool log1Exist = taosStatFile(name, NULL, &logstat1_mtime) >= 0; + // if none of the log files exist, open 0, if both exists, open the old one if (!log0Exist && !log1Exist) { tsLogObj.flag = 0; @@ -326,39 +328,39 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { } else if (!log0Exist) { tsLogObj.flag = 1; } else { - tsLogObj.flag = (logstat0.st_mtime > logstat1.st_mtime) ? 0 : 1; + tsLogObj.flag = (logstat0_mtime > logstat1_mtime) ? 0 : 1; } char fileName[LOG_FILE_NAME_LEN + 50] = "\0"; sprintf(fileName, "%s.%d", tsLogObj.logName, tsLogObj.flag); pthread_mutex_init(&tsLogObj.logMutex, NULL); - umask(0); - tsLogObj.logHandle->fd = open(fileName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + taosUmaskFile(0); + tsLogObj.logHandle->fd = taosOpenFileCreateWrite(fileName); if (tsLogObj.logHandle->fd < 0) { printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } - taosLockFile(tsLogObj.logHandle->fd); + taosLockLogFile(tsLogObj.logHandle->fd); // only an estimate for number of lines - struct stat filestat; - if (fstat(tsLogObj.logHandle->fd, &filestat) < 0) { + int64_t filesize = 0; + if (taosFStatFile(tsLogObj.logHandle->fd, &filesize, NULL) < 0) { printf("\nfailed to fstat log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } - size = (int32_t)filestat.st_size; + size = (int32_t)filesize; tsLogObj.lines = size / 60; - lseek(tsLogObj.logHandle->fd, 0, SEEK_END); + taosLSeekFile(tsLogObj.logHandle->fd, 0, SEEK_END); sprintf(name, "==================================================\n"); - taosWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); sprintf(name, " new log file \n"); - taosWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); sprintf(name, "==================================================\n"); - taosWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); return 0; } @@ -377,7 +379,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { struct timeval timeSecs; time_t curTime; - gettimeofday(&timeSecs, NULL); + taosGetTimeOfDay(&timeSecs); curTime = timeSecs.tv_sec; ptm = localtime_r(&curTime, &Tm); @@ -408,7 +410,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { if (tsAsyncLog) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { - taosWrite(tsLogObj.logHandle->fd, buffer, len); + taosWriteFile(tsLogObj.logHandle->fd, buffer, len); } if (tsLogObj.maxLines > 0) { @@ -419,7 +421,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { } if (dflag & DEBUG_SCREEN) - taosWrite(1, buffer, (uint32_t)len); + taosWriteFile(1, buffer, (uint32_t)len); if (dflag == 255) nInfo(buffer, len); } @@ -439,7 +441,7 @@ void taosDumpData(unsigned char *msg, int32_t len) { pos += 3; if (c >= 16) { temp[pos++] = '\n'; - taosWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos); + taosWriteFile(tsLogObj.logHandle->fd, temp, (uint32_t)pos); c = 0; pos = 0; } @@ -447,7 +449,7 @@ void taosDumpData(unsigned char *msg, int32_t len) { temp[pos++] = '\n'; - taosWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos); + taosWriteFile(tsLogObj.logHandle->fd, temp, (uint32_t)pos); } void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) { @@ -464,7 +466,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . struct timeval timeSecs; time_t curTime; - gettimeofday(&timeSecs, NULL); + taosGetTimeOfDay(&timeSecs); curTime = timeSecs.tv_sec; ptm = localtime_r(&curTime, &Tm); @@ -485,7 +487,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . if (tsAsyncLog) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { - taosWrite(tsLogObj.logHandle->fd, buffer, len); + taosWriteFile(tsLogObj.logHandle->fd, buffer, len); } if (tsLogObj.maxLines > 0) { @@ -495,7 +497,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . } } - if (dflag & DEBUG_SCREEN) taosWrite(1, buffer, (uint32_t)len); + if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len); } #if 0 @@ -506,8 +508,8 @@ void taosCloseLog() { static void taosCloseLogByFd(int32_t fd) { if (fd >= 0) { - taosUnLockFile(fd); - taosClose(fd); + taosUnLockLogFile(fd); + taosCloseFile(fd); } } @@ -644,12 +646,12 @@ static void taosWriteLog(SLogBuff *tLogBuff) { } if (start < end) { - taosWrite(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); + taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); } else { int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start; - taosWrite(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, tsize); + taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, tsize); - taosWrite(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff), end); + taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff), end); } dbgWN++; From e889008732b3ac5a7db00c7c165723115a41d2cd Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 30 Sep 2021 17:10:36 +0800 Subject: [PATCH 32/44] include file for os module --- include/os/os.h | 16 +++++----------- include/os/osAtomic.h | 6 +++--- include/os/osDir.h | 6 +++--- include/os/osRand.h | 6 +++--- include/os/osSemaphore.h | 6 +++--- include/os/osSignal.h | 30 ++++++++++++++++++++++++++++++ include/os/osSocket.h | 1 + include/os/osString.h | 6 +++--- include/os/osTime.h | 6 +++--- include/os/osTimer.h | 30 ++++++++++++++++++++++++++++++ source/libs/transport/src/rpcTcp.c | 3 --- 11 files changed, 84 insertions(+), 32 deletions(-) create mode 100644 include/os/osSignal.h create mode 100644 include/os/osTimer.h diff --git a/include/os/os.h b/include/os/os.h index 935923195b..d450ff620f 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -36,15 +36,6 @@ extern "C" { #include #include -// #include -// #include -// #include -// // #include -// -// #include -// #include -// #include - #include "osAtomic.h" #include "osDef.h" #include "osDir.h" @@ -52,12 +43,15 @@ extern "C" { #include "osFile.h" #include "osMath.h" #include "osMemory.h" +#include "osRand.h" #include "osSemaphore.h" +#include "osSignal.h" +#include "osSleep.h" #include "osSocket.h" #include "osString.h" -#include "osSleep.h" -#include "osTime.h" #include "osThread.h" +#include "osTime.h" +#include "osTimer.h" #ifdef __cplusplus } diff --git a/include/os/osAtomic.h b/include/os/osAtomic.h index 7affa444ee..bd3d372fc0 100644 --- a/include/os/osAtomic.h +++ b/include/os/osAtomic.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_OS_ATOMIC_H -#define TDENGINE_OS_ATOMIC_H +#ifndef _TD_OS_ATOMIC_H_ +#define _TD_OS_ATOMIC_H_ #ifdef __cplusplus extern "C" { @@ -355,4 +355,4 @@ extern "C" { } #endif -#endif +#endif /*_TD_OS_ATOMIC_H_*/ diff --git a/include/os/osDir.h b/include/os/osDir.h index 7afe126451..dfd814d70b 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_OS_DIR_H -#define TDENGINE_OS_DIR_H +#ifndef _TD_OS_DIR_H_ +#define _TD_OS_DIR_H_ #ifdef __cplusplus extern "C" { @@ -31,4 +31,4 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName); } #endif -#endif +#endif /*_TD_OS_DIR_H_*/ diff --git a/include/os/osRand.h b/include/os/osRand.h index e08768c2cc..422ea92a71 100644 --- a/include/os/osRand.h +++ b/include/os/osRand.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_OS_RAND_H -#define TDENGINE_OS_RAND_H +#ifndef _TD_OS_RAND_H_ +#define _TD_OS_RAND_H_ #ifdef __cplusplus extern "C" { @@ -28,4 +28,4 @@ uint32_t taosSafeRand(void); } #endif -#endif +#endif /*_TD_OS_RAND_H_*/ diff --git a/include/os/osSemaphore.h b/include/os/osSemaphore.h index d945e721c2..86a9f10819 100644 --- a/include/os/osSemaphore.h +++ b/include/os/osSemaphore.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_OS_SEMAPHORE_H -#define TDENGINE_OS_SEMAPHORE_H +#ifndef _TD_OS_SEMPHONE_H_ +#define _TD_OS_SEMPHONE_H_ #ifdef __cplusplus extern "C" { @@ -63,4 +63,4 @@ int32_t taosGetCurrentAPPName(char* name, int32_t* len); } #endif -#endif +#endif /*_TD_OS_SEMPHONE_H_*/ diff --git a/include/os/osSignal.h b/include/os/osSignal.h new file mode 100644 index 0000000000..ce133e62b7 --- /dev/null +++ b/include/os/osSignal.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_OS_SIGNAL_H_ +#define _TD_OS_SIGNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t taosInitTimer(void (*callback)(int32_t), int32_t ms); +void taosUninitTimer(); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_OS_SIGNAL_H_*/ diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 0b183b9ece..719c693765 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -30,6 +30,7 @@ extern "C" { #include #include #include + #include #include #endif diff --git a/include/os/osString.h b/include/os/osString.h index 33f858485c..3adee83c84 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_OS_STRING_H -#define TDENGINE_OS_STRING_H +#ifndef _TD_OS_STRING_H_ +#define _TD_OS_STRING_H_ #ifdef __cplusplus extern "C" { @@ -51,4 +51,4 @@ char * taosCharsetReplace(char *charsetstr); } #endif -#endif +#endif /*_TD_OS_STRING_H_*/ diff --git a/include/os/osTime.h b/include/os/osTime.h index c5ee26e6ef..d076d50276 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_OS_TIME_H -#define TDENGINE_OS_TIME_H +#ifndef _TD_OS_TIME_H_ +#define _TD_OS_TIME_H_ #ifdef __cplusplus extern "C" { @@ -55,4 +55,4 @@ int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrec } #endif -#endif // TDENGINE_TTIME_H +#endif /*_TD_OS_TIME_H_*/ diff --git a/include/os/osTimer.h b/include/os/osTimer.h new file mode 100644 index 0000000000..623d2da20a --- /dev/null +++ b/include/os/osTimer.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_OS_TIMER_H_ +#define _TD_OS_TIMER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t taosInitTimer(void (*callback)(int32_t), int32_t ms); +void taosUninitTimer(); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_OS_TIMER_H_*/ diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index 5eb7e39ff2..1f657a5b8d 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -22,9 +22,6 @@ #include "rpcHead.h" #include "rpcTcp.h" - -#include - typedef struct SFdObj { void *signature; SOCKET fd; // TCP socket FD From c0a171b144fc8b784e570f73f4a9a32c9d419f96 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 30 Sep 2021 21:21:51 +0800 Subject: [PATCH 33/44] [TD-10430] adjust os module --- CMakeLists.txt | 2 + include/client/taos.h | 32 +- include/common/taosdef.h | 381 +----- .../inc => include/common}/tarithoperator.h | 0 {src/common/inc => include/common}/tcmdtype.h | 0 {src/common/inc => include/common}/texpr.h | 0 include/common/tglobal.h | 45 +- {src/common/inc => include/common}/tlocale.h | 0 {src/common/inc => include/common}/tname.h | 0 include/common/ttime.h | 58 + .../common/inc => include/common}/ttimezone.h | 0 include/common/ttype.h | 37 - {src/common/inc => include/common}/tvariant.h | 0 include/os/os.h | 10 + include/os/osDef.h | 155 ++- include/os/osDir.h | 12 +- src/os/inc/osTimer.h => include/os/osEnv.h | 14 +- {src/os/inc => include/os}/osLz4.h | 6 +- include/os/osSignal.h | 30 +- include/os/osSleep.h | 6 +- include/os/osSocket.h | 6 +- include/os/osString.h | 2 +- include/os/osSysinfo.h | 73 ++ {src/os/inc => include/os}/osSystem.h | 12 +- include/os/osTime.h | 20 +- include/os/osTimer.h | 2 + {src/util/inc => include/util}/exception.h | 11 +- include/util/hash.h | 6 +- include/util/hashfunc.h | 6 +- include/util/talgo.h | 6 +- include/util/tarray.h | 8 +- include/util/tbuffer.h | 9 +- include/util/tcache.h | 6 +- {src/util/inc => include/util}/tchecksum.h | 0 include/util/tcoding.h | 6 +- {src/util/inc => include/util}/tcompare.h | 0 include/util/tconfig.h | 7 +- include/util/tcrc32c.h | 8 +- include/util/tdef.h | 451 +++++++ {src/util/inc => include/util}/tfile.h | 0 {src/util/inc => include/util}/tfunctional.h | 0 include/util/tlist.h | 6 +- include/util/tlockfree.h | 8 +- include/util/tlog.h | 31 +- {src/util/inc => include/util}/tlosertree.h | 0 {src/util/inc => include/util}/tnettest.h | 0 {src/util/inc => include/util}/tqueue.h | 0 {src/util/inc => include/util}/tsched.h | 0 include/util/tscompression.h | 2 +- {src/util/inc => include/util}/tskiplist.h | 0 {src/util/inc => include/util}/tstrbuild.h | 0 {src/util/inc => include/util}/tthread.h | 0 {src/util/inc => include/util}/ttoken.h | 0 {source/util/inc => include/util}/tulog.h | 0 include/util/tutil.h | 5 - {src/util/inc => include/util}/tworker.h | 0 source/common/CMakeLists.txt | 5 + {src => source}/common/src/sqlcmdstr.c | 0 {src => source}/common/src/tarithoperator.c | 0 {src => source}/common/src/tdataformat.c | 0 {src/util => source/common}/src/terror.c | 0 {src => source}/common/src/texpr.c | 0 {src => source}/common/src/tglobal.c | 54 +- {src => source}/common/src/tlocale.c | 0 {src => source}/common/src/tmessage.c | 0 {src => source}/common/src/tname.c | 0 .../osTime.c => source/common/src/ttime.c | 1 + {src => source}/common/src/ttimezone.c | 0 {src => source}/common/src/ttypes.c | 0 {src => source}/common/src/tvariant.c | 0 source/libs/transport/src/rpcCache.c | 1 + source/os/src/os.c | 14 - .../wAtomic.c => source/os/src/osAtomic.c | 47 +- {src/os/src/detail => source/os/src}/osDir.c | 123 +- source/os/src/osEnv.c | 66 + source/os/src/osFile.c | 1 - .../windows/wLz4.c => source/os/src/osLz4.c | 15 +- {src/os/src/detail => source/os/src}/osRand.c | 4 + .../os/src/osSemaphore.c | 224 +++- .../wSignal.c => source/os/src/osSignal.c | 36 +- .../os/src/detail => source/os/src}/osSleep.c | 0 source/os/src/osSocket.c | 139 +- source/os/src/osString.c | 382 ++++++ .../wStrptime.c => source/os/src/osStrptime.c | 7 +- source/os/src/osSysinfo.c | 1130 +++++++++++++++++ source/os/src/osSystem.c | 143 +++ source/os/src/osTime.c | 42 +- source/os/src/osTimer.c | 225 ++++ source/server/dnode/src/dnodeMain.c | 14 +- source/server/dnode/src/dnodeStatus.c | 1 + source/util/CMakeLists.txt | 4 +- {src => source}/util/src/exception.c | 0 source/util/src/hash.c | 1 - source/util/src/talgo.c | 1 - {src => source}/util/src/tbase64.c | 0 {src => source}/util/src/tbuffer.c | 0 {src => source}/util/src/tcache.c | 0 {src => source}/util/src/tcompare.c | 2 +- {src => source}/util/src/tcompression.c | 3 - {src => source}/util/src/tconfig.c | 80 +- {src => source}/util/src/tcrc32c.c | 7 +- {src => source}/util/src/tdes.c | 0 {src => source}/util/src/tfile.c | 0 {src => source}/util/src/tfunctional.c | 0 {src => source}/util/src/thashutil.c | 0 {src => source}/util/src/tidpool.c | 0 source/util/src/tlog.c | 97 +- {src => source}/util/src/tlosertree.c | 0 {src => source}/util/src/tmd5.c | 0 {src => source}/util/src/tmempool.c | 0 {src => source}/util/src/tnettest.c | 0 {src => source}/util/src/tnote.c | 0 {src => source}/util/src/tqueue.c | 0 {src => source}/util/src/tref.c | 0 {src => source}/util/src/tsched.c | 0 {src => source}/util/src/tskiplist.c | 0 {src => source}/util/src/tsocket.c | 0 {src => source}/util/src/tstrbuild.c | 0 {src => source}/util/src/tthread.c | 0 {src => source}/util/src/ttimer.c | 0 {src => source}/util/src/ttokenizer.c | 0 {src => source}/util/src/tutil.c | 0 {src => source}/util/src/tworker.c | 0 {src => source}/util/src/version.c.in | 0 .../tests => source/util/test}/CMakeLists.txt | 0 .../tests => source/util/test}/arrayTest.cpp | 0 .../tests => source/util/test}/cacheTest.cpp | 0 .../util/test}/codingTests.cpp | 0 .../tests => source/util/test}/hashTest.cpp | 0 .../util/test}/skiplistTest.cpp | 0 .../tests => source/util/test}/stringTest.cpp | 0 .../util/test}/taosbsearchTest.cpp | 0 .../tests => source/util/test}/trefTest.c | 0 src/client/src/tscSub.c | 2 +- src/common/CMakeLists.txt | 11 - src/common/inc/tulog.h | 42 - src/common/source/common.c | 14 - src/os/CMakeLists.txt | 7 - src/os/inc/osArm32.h | 0 src/os/inc/osArm64.h | 0 src/os/inc/osEok.h | 93 -- src/os/inc/osInc.h | 120 -- src/os/inc/osLinux32.h | 0 src/os/inc/osLinux64.h | 0 src/os/inc/osMemory.h | 82 -- src/os/inc/osMips64.h | 87 -- src/os/inc/osRand.h | 31 - src/os/inc/osSignal.h | 58 - src/os/inc/osSleep.h | 29 - src/os/inc/osSysinfo.h | 54 - src/os/source/os.c | 14 - src/os/src/darwin/CMakeLists.txt | 5 - src/os/src/darwin/darwinSystem.c | 56 - src/os/src/darwin/dwEnv.c | 43 - src/os/src/darwin/dwEok.c | 891 ------------- src/os/src/darwin/dwSocket.c | 29 - src/os/src/darwin/dwSysInfo.c | 273 ---- src/os/src/darwin/dwTimer.c | 124 -- src/os/src/detail/CMakeLists.txt | 21 - src/os/src/detail/osAtomic.c | 55 - src/os/src/detail/osFail.c | 141 -- src/os/src/detail/osMemory.c | 521 -------- src/os/src/detail/osSemaphore.c | 72 -- src/os/src/detail/osSignal.c | 46 - src/os/src/detail/osString.c | 130 -- src/os/src/detail/osSysinfo.c | 670 ---------- src/os/src/detail/osTimer.c | 108 -- src/os/src/linux/CMakeLists.txt | 7 - src/os/src/linux/linuxEnv.c | 69 - src/os/src/linux/osSystem.c | 78 -- src/os/src/windows/CMakeLists.txt | 7 - src/os/src/windows/wEnv.c | 69 - src/os/src/windows/wGetline.c | 125 -- src/os/src/windows/wSemaphore.c | 59 - src/os/src/windows/wSocket.c | 99 -- src/os/src/windows/wString.c | 152 --- src/os/src/windows/wSysLog.c | 19 - src/os/src/windows/wSysinfo.c | 293 ----- src/os/src/windows/wSystem.c | 46 - src/os/src/windows/wTime.c | 34 - src/os/src/windows/wTimer.c | 45 - src/os/src/windows/wWordexp.c | 40 - src/os/tests/CMakeLists.txt | 21 - src/os/tests/test.cpp | 34 - src/query/src/qExtbuffer.c | 2 +- src/tfs/src/tfs.c | 2 +- src/util/CMakeLists.txt | 11 - src/util/CMakeLists.txt.in | 44 - src/util/inc/tidpool.h | 43 - src/util/inc/tlog.h | 56 - src/util/inc/tmempool.h | 36 - src/util/inc/tref.h | 77 -- src/util/src/hash.c | 916 ------------- src/wal/src/walMgmt.c | 2 +- 194 files changed, 3462 insertions(+), 7101 deletions(-) rename {src/common/inc => include/common}/tarithoperator.h (100%) rename {src/common/inc => include/common}/tcmdtype.h (100%) rename {src/common/inc => include/common}/texpr.h (100%) rename {src/common/inc => include/common}/tlocale.h (100%) rename {src/common/inc => include/common}/tname.h (100%) create mode 100644 include/common/ttime.h rename {src/common/inc => include/common}/ttimezone.h (100%) rename {src/common/inc => include/common}/tvariant.h (100%) rename src/os/inc/osTimer.h => include/os/osEnv.h (76%) rename {src/os/inc => include/os}/osLz4.h (95%) create mode 100644 include/os/osSysinfo.h rename {src/os/inc => include/os}/osSystem.h (79%) rename {src/util/inc => include/util}/exception.h (96%) rename {src/util/inc => include/util}/tchecksum.h (100%) rename {src/util/inc => include/util}/tcompare.h (100%) create mode 100644 include/util/tdef.h rename {src/util/inc => include/util}/tfile.h (100%) rename {src/util/inc => include/util}/tfunctional.h (100%) rename {src/util/inc => include/util}/tlosertree.h (100%) rename {src/util/inc => include/util}/tnettest.h (100%) rename {src/util/inc => include/util}/tqueue.h (100%) rename {src/util/inc => include/util}/tsched.h (100%) rename {src/util/inc => include/util}/tskiplist.h (100%) rename {src/util/inc => include/util}/tstrbuild.h (100%) rename {src/util/inc => include/util}/tthread.h (100%) rename {src/util/inc => include/util}/ttoken.h (100%) rename {source/util/inc => include/util}/tulog.h (100%) rename {src/util/inc => include/util}/tworker.h (100%) rename {src => source}/common/src/sqlcmdstr.c (100%) rename {src => source}/common/src/tarithoperator.c (100%) rename {src => source}/common/src/tdataformat.c (100%) rename {src/util => source/common}/src/terror.c (100%) rename {src => source}/common/src/texpr.c (100%) rename {src => source}/common/src/tglobal.c (97%) rename {src => source}/common/src/tlocale.c (100%) rename {src => source}/common/src/tmessage.c (100%) rename {src => source}/common/src/tname.c (100%) rename src/os/src/detail/osTime.c => source/common/src/ttime.c (99%) rename {src => source}/common/src/ttimezone.c (100%) rename {src => source}/common/src/ttypes.c (100%) rename {src => source}/common/src/tvariant.c (100%) delete mode 100644 source/os/src/os.c rename src/os/src/windows/wAtomic.c => source/os/src/osAtomic.c (81%) rename {src/os/src/detail => source/os/src}/osDir.c (52%) create mode 100644 source/os/src/osEnv.c rename src/os/src/windows/wLz4.c => source/os/src/osLz4.c (94%) rename {src/os/src/detail => source/os/src}/osRand.c (93%) rename src/os/src/darwin/dwSemaphore.c => source/os/src/osSemaphore.c (67%) rename src/os/src/windows/wSignal.c => source/os/src/osSignal.c (64%) rename {src/os/src/detail => source/os/src}/osSleep.c (100%) create mode 100644 source/os/src/osString.c rename src/os/src/windows/wStrptime.c => source/os/src/osStrptime.c (99%) create mode 100644 source/os/src/osSysinfo.c create mode 100644 source/os/src/osSystem.c create mode 100644 source/os/src/osTimer.c rename {src => source}/util/src/exception.c (100%) rename {src => source}/util/src/tbase64.c (100%) rename {src => source}/util/src/tbuffer.c (100%) rename {src => source}/util/src/tcache.c (100%) rename {src => source}/util/src/tcompare.c (99%) rename {src => source}/util/src/tcompression.c (99%) rename {src => source}/util/src/tconfig.c (87%) rename {src => source}/util/src/tcrc32c.c (99%) rename {src => source}/util/src/tdes.c (100%) rename {src => source}/util/src/tfile.c (100%) rename {src => source}/util/src/tfunctional.c (100%) rename {src => source}/util/src/thashutil.c (100%) rename {src => source}/util/src/tidpool.c (100%) rename {src => source}/util/src/tlosertree.c (100%) rename {src => source}/util/src/tmd5.c (100%) rename {src => source}/util/src/tmempool.c (100%) rename {src => source}/util/src/tnettest.c (100%) rename {src => source}/util/src/tnote.c (100%) rename {src => source}/util/src/tqueue.c (100%) rename {src => source}/util/src/tref.c (100%) rename {src => source}/util/src/tsched.c (100%) rename {src => source}/util/src/tskiplist.c (100%) rename {src => source}/util/src/tsocket.c (100%) rename {src => source}/util/src/tstrbuild.c (100%) rename {src => source}/util/src/tthread.c (100%) rename {src => source}/util/src/ttimer.c (100%) rename {src => source}/util/src/ttokenizer.c (100%) rename {src => source}/util/src/tutil.c (100%) rename {src => source}/util/src/tworker.c (100%) rename {src => source}/util/src/version.c.in (100%) rename {src/util/tests => source/util/test}/CMakeLists.txt (100%) rename {src/util/tests => source/util/test}/arrayTest.cpp (100%) rename {src/util/tests => source/util/test}/cacheTest.cpp (100%) rename {src/util/tests => source/util/test}/codingTests.cpp (100%) rename {src/util/tests => source/util/test}/hashTest.cpp (100%) rename {src/util/tests => source/util/test}/skiplistTest.cpp (100%) rename {src/util/tests => source/util/test}/stringTest.cpp (100%) rename {src/util/tests => source/util/test}/taosbsearchTest.cpp (100%) rename {src/util/tests => source/util/test}/trefTest.c (100%) delete mode 100644 src/common/CMakeLists.txt delete mode 100644 src/common/inc/tulog.h delete mode 100644 src/common/source/common.c delete mode 100644 src/os/CMakeLists.txt delete mode 100644 src/os/inc/osArm32.h delete mode 100644 src/os/inc/osArm64.h delete mode 100644 src/os/inc/osEok.h delete mode 100644 src/os/inc/osInc.h delete mode 100644 src/os/inc/osLinux32.h delete mode 100644 src/os/inc/osLinux64.h delete mode 100644 src/os/inc/osMemory.h delete mode 100644 src/os/inc/osMips64.h delete mode 100644 src/os/inc/osRand.h delete mode 100644 src/os/inc/osSignal.h delete mode 100644 src/os/inc/osSleep.h delete mode 100644 src/os/inc/osSysinfo.h delete mode 100644 src/os/source/os.c delete mode 100644 src/os/src/darwin/CMakeLists.txt delete mode 100644 src/os/src/darwin/darwinSystem.c delete mode 100644 src/os/src/darwin/dwEnv.c delete mode 100644 src/os/src/darwin/dwEok.c delete mode 100644 src/os/src/darwin/dwSocket.c delete mode 100644 src/os/src/darwin/dwSysInfo.c delete mode 100644 src/os/src/darwin/dwTimer.c delete mode 100644 src/os/src/detail/CMakeLists.txt delete mode 100644 src/os/src/detail/osAtomic.c delete mode 100644 src/os/src/detail/osFail.c delete mode 100644 src/os/src/detail/osMemory.c delete mode 100644 src/os/src/detail/osSemaphore.c delete mode 100644 src/os/src/detail/osSignal.c delete mode 100644 src/os/src/detail/osString.c delete mode 100644 src/os/src/detail/osSysinfo.c delete mode 100644 src/os/src/detail/osTimer.c delete mode 100644 src/os/src/linux/CMakeLists.txt delete mode 100644 src/os/src/linux/linuxEnv.c delete mode 100644 src/os/src/linux/osSystem.c delete mode 100644 src/os/src/windows/CMakeLists.txt delete mode 100644 src/os/src/windows/wEnv.c delete mode 100644 src/os/src/windows/wGetline.c delete mode 100644 src/os/src/windows/wSemaphore.c delete mode 100644 src/os/src/windows/wSocket.c delete mode 100644 src/os/src/windows/wString.c delete mode 100644 src/os/src/windows/wSysLog.c delete mode 100644 src/os/src/windows/wSysinfo.c delete mode 100644 src/os/src/windows/wSystem.c delete mode 100644 src/os/src/windows/wTime.c delete mode 100644 src/os/src/windows/wTimer.c delete mode 100644 src/os/src/windows/wWordexp.c delete mode 100644 src/os/tests/CMakeLists.txt delete mode 100644 src/os/tests/test.cpp delete mode 100644 src/util/CMakeLists.txt delete mode 100644 src/util/CMakeLists.txt.in delete mode 100644 src/util/inc/tidpool.h delete mode 100644 src/util/inc/tlog.h delete mode 100644 src/util/inc/tmempool.h delete mode 100644 src/util/inc/tref.h delete mode 100644 src/util/src/hash.c diff --git a/CMakeLists.txt b/CMakeLists.txt index e16c93c1f1..95e1cad2fe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,6 +42,8 @@ execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . execute_process(COMMAND "${CMAKE_COMMAND}" --build . WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/deps/deps-download") +SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma") + # deps add_subdirectory(deps) diff --git a/include/client/taos.h b/include/client/taos.h index edb1552b81..635ca3c774 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -31,21 +31,23 @@ typedef void TAOS_SUB; typedef void **TAOS_ROW; // Data type definition -#define TSDB_DATA_TYPE_NULL 0 // 1 bytes -#define TSDB_DATA_TYPE_BOOL 1 // 1 bytes -#define TSDB_DATA_TYPE_TINYINT 2 // 1 byte -#define TSDB_DATA_TYPE_SMALLINT 3 // 2 bytes -#define TSDB_DATA_TYPE_INT 4 // 4 bytes -#define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes -#define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes -#define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes -#define TSDB_DATA_TYPE_BINARY 8 // string -#define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes -#define TSDB_DATA_TYPE_NCHAR 10 // unicode string -#define TSDB_DATA_TYPE_UTINYINT 11 // 1 byte -#define TSDB_DATA_TYPE_USMALLINT 12 // 2 bytes -#define TSDB_DATA_TYPE_UINT 13 // 4 bytes -#define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes +#ifndef TSDB_DATA_TYPE_NULL + #define TSDB_DATA_TYPE_NULL 0 // 1 bytes + #define TSDB_DATA_TYPE_BOOL 1 // 1 bytes + #define TSDB_DATA_TYPE_TINYINT 2 // 1 byte + #define TSDB_DATA_TYPE_SMALLINT 3 // 2 bytes + #define TSDB_DATA_TYPE_INT 4 // 4 bytes + #define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes + #define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes + #define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes + #define TSDB_DATA_TYPE_BINARY 8 // string + #define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes + #define TSDB_DATA_TYPE_NCHAR 10 // unicode string + #define TSDB_DATA_TYPE_UTINYINT 11 // 1 byte + #define TSDB_DATA_TYPE_USMALLINT 12 // 2 bytes + #define TSDB_DATA_TYPE_UINT 13 // 4 bytes + #define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes +#endif typedef enum { TSDB_OPTION_LOCALE, diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 0bcd693f5e..dd7dfd9518 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -13,389 +13,20 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TAOS_DEF_H -#define TDENGINE_TAOS_DEF_H +#ifndef _TD_COMMON_TAOS_DEF_H +#define _TD_COMMON_TAOS_DEF_H #ifdef __cplusplus extern "C" { #endif -#include -#include +#include "tdef.h" #include "taos.h" -#define TSDB__packed - -#ifdef TSKEY32 -#define TSKEY int32_t; -#else -#define TSKEY int64_t -#endif - #define TSWINDOW_INITIALIZER ((STimeWindow) {INT64_MIN, INT64_MAX}) #define TSWINDOW_DESC_INITIALIZER ((STimeWindow) {INT64_MAX, INT64_MIN}) #define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX)) -#define TSKEY_INITIAL_VAL INT64_MIN - -// Bytes for each type. -extern const int32_t TYPE_BYTES[15]; - -#define TSDB_KEYSIZE sizeof(TSKEY) - -#if LINUX -#define TSDB_NCHAR_SIZE sizeof(wchar_t) -#else -#define TSDB_NCHAR_SIZE sizeof(int32_t) -#endif - -// NULL definition -#define TSDB_DATA_BOOL_NULL 0x02 -#define TSDB_DATA_TINYINT_NULL 0x80 -#define TSDB_DATA_SMALLINT_NULL 0x8000 -#define TSDB_DATA_INT_NULL 0x80000000L -#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L -#define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL - -#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN -#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN -#define TSDB_DATA_NCHAR_NULL 0xFFFFFFFF -#define TSDB_DATA_BINARY_NULL 0xFF - -#define TSDB_DATA_UTINYINT_NULL 0xFF -#define TSDB_DATA_USMALLINT_NULL 0xFFFF -#define TSDB_DATA_UINT_NULL 0xFFFFFFFF -#define TSDB_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL - -#define TSDB_DATA_NULL_STR "NULL" -#define TSDB_DATA_NULL_STR_L "null" - -#define TSDB_DEFAULT_USER "root" -#ifdef _TD_POWER_ -#define TSDB_DEFAULT_PASS "powerdb" -#elif (_TD_TQ_ == true) -#define TSDB_DEFAULT_PASS "tqueue" -#elif (_TD_PRO_ == true) -#define TSDB_DEFAULT_PASS "prodb" -#else -#define TSDB_DEFAULT_PASS "taosdata" -#endif - -#define SHELL_MAX_PASSWORD_LEN 20 - -#define TSDB_TRUE 1 -#define TSDB_FALSE 0 -#define TSDB_OK 0 -#define TSDB_ERR -1 - -#define TS_PATH_DELIMITER "." - -#define TSDB_TIME_PRECISION_MILLI 0 -#define TSDB_TIME_PRECISION_MICRO 1 -#define TSDB_TIME_PRECISION_NANO 2 - -#define TSDB_TIME_PRECISION_MILLI_STR "ms" -#define TSDB_TIME_PRECISION_MICRO_STR "us" -#define TSDB_TIME_PRECISION_NANO_STR "ns" - -#define TSDB_TICK_PER_SECOND(precision) ((int64_t)((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))) - -#define T_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) -#define T_APPEND_MEMBER(dst, ptr, type, member) \ -do {\ - memcpy((void *)(dst), (void *)(&((ptr)->member)), T_MEMBER_SIZE(type, member));\ - dst = (void *)((char *)(dst) + T_MEMBER_SIZE(type, member));\ -} while(0) -#define T_READ_MEMBER(src, type, target) \ -do { \ - (target) = *(type *)(src); \ - (src) = (void *)((char *)src + sizeof(type));\ -} while(0) - -#define GET_INT8_VAL(x) (*(int8_t *)(x)) -#define GET_INT16_VAL(x) (*(int16_t *)(x)) -#define GET_INT32_VAL(x) (*(int32_t *)(x)) -#define GET_INT64_VAL(x) (*(int64_t *)(x)) -#define GET_UINT8_VAL(x) (*(uint8_t*) (x)) -#define GET_UINT16_VAL(x) (*(uint16_t *)(x)) -#define GET_UINT32_VAL(x) (*(uint32_t *)(x)) -#define GET_UINT64_VAL(x) (*(uint64_t *)(x)) - -#ifdef _TD_ARM_32 - float taos_align_get_float(const char* pBuf); - double taos_align_get_double(const char* pBuf); - - #define GET_FLOAT_VAL(x) taos_align_get_float(x) - #define GET_DOUBLE_VAL(x) taos_align_get_double(x) - #define SET_FLOAT_VAL(x, y) { float z = (float)(y); (*(int32_t*) x = *(int32_t*)(&z)); } - #define SET_DOUBLE_VAL(x, y) { double z = (double)(y); (*(int64_t*) x = *(int64_t*)(&z)); } - #define SET_FLOAT_PTR(x, y) { (*(int32_t*) x = *(int32_t*)y); } - #define SET_DOUBLE_PTR(x, y) { (*(int64_t*) x = *(int64_t*)y); } -#else - #define GET_FLOAT_VAL(x) (*(float *)(x)) - #define GET_DOUBLE_VAL(x) (*(double *)(x)) - #define SET_FLOAT_VAL(x, y) { (*(float *)(x)) = (float)(y); } - #define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y); } - #define SET_FLOAT_PTR(x, y) { (*(float *)(x)) = (*(float *)(y)); } - #define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); } -#endif - -// TODO: check if below is necessary -#define TSDB_RELATION_INVALID 0 -#define TSDB_RELATION_LESS 1 -#define TSDB_RELATION_GREATER 2 -#define TSDB_RELATION_EQUAL 3 -#define TSDB_RELATION_LESS_EQUAL 4 -#define TSDB_RELATION_GREATER_EQUAL 5 -#define TSDB_RELATION_NOT_EQUAL 6 -#define TSDB_RELATION_LIKE 7 -#define TSDB_RELATION_ISNULL 8 -#define TSDB_RELATION_NOTNULL 9 -#define TSDB_RELATION_IN 10 - -#define TSDB_RELATION_AND 11 -#define TSDB_RELATION_OR 12 -#define TSDB_RELATION_NOT 13 - -#define TSDB_RELATION_MATCH 14 -#define TSDB_RELATION_NMATCH 15 - -#define TSDB_BINARY_OP_ADD 30 -#define TSDB_BINARY_OP_SUBTRACT 31 -#define TSDB_BINARY_OP_MULTIPLY 32 -#define TSDB_BINARY_OP_DIVIDE 33 -#define TSDB_BINARY_OP_REMAINDER 34 - - -#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN)) -#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER)) - -#define TS_PATH_DELIMITER_LEN 1 - -#define TSDB_UNI_LEN 24 -#define TSDB_USER_LEN TSDB_UNI_LEN - -// ACCOUNT is a 32 bit positive integer -// this is the length of its string representation, including the terminator zero -#define TSDB_ACCT_ID_LEN 11 - -#define TSDB_MAX_COLUMNS 4096 -#define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns - -#define TSDB_NODE_NAME_LEN 64 -#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string -#define TSDB_DB_NAME_LEN 33 -#define TSDB_FUNC_NAME_LEN 65 -#define TSDB_FUNC_CODE_LEN (65535 - 512) -#define TSDB_FUNC_BUF_SIZE 512 -#define TSDB_TYPE_STR_MAX_LEN 32 -#define TSDB_TABLE_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN) -#define TSDB_COL_NAME_LEN 65 -#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 -#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE -#define TSDB_MAX_SQL_SHOW_LEN 512 -#define TSDB_MAX_ALLOWED_SQL_LEN (1*1024*1024u) // sql length should be less than 1mb - -#define TSDB_APPNAME_LEN TSDB_UNI_LEN - - /** - * In some scenarios uint16_t (0~65535) is used to store the row len. - * - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header. - * - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus - * the final value is 65531-(4096-1)*4 = 49151. - */ -#define TSDB_MAX_BYTES_PER_ROW 49151 -#define TSDB_MAX_TAGS_LEN 16384 -#define TSDB_MAX_TAGS 128 -#define TSDB_MAX_TAG_CONDITIONS 1024 - -#define TSDB_AUTH_LEN 16 -#define TSDB_KEY_LEN 16 -#define TSDB_VERSION_LEN 12 -#define TSDB_LOCALE_LEN 64 -#define TSDB_TIMEZONE_LEN 96 -#define TSDB_LABEL_LEN 8 - -#define TSDB_CLUSTER_ID_LEN 40 -#define TSDB_FQDN_LEN 128 -#define TSDB_EP_LEN (TSDB_FQDN_LEN+6) -#define TSDB_IPv4ADDR_LEN 16 -#define TSDB_FILENAME_LEN 128 -#define TSDB_SHOW_SQL_LEN 512 -#define TSDB_SHOW_SUBQUERY_LEN 1000 -#define TSDB_SLOW_QUERY_SQL_LEN 512 - -#define TSDB_STEP_NAME_LEN 32 -#define TSDB_STEP_DESC_LEN 128 - -#define TSDB_MQTT_HOSTNAME_LEN 64 -#define TSDB_MQTT_PORT_LEN 8 -#define TSDB_MQTT_USER_LEN 24 -#define TSDB_MQTT_PASS_LEN 24 -#define TSDB_MQTT_TOPIC_LEN 64 -#define TSDB_MQTT_CLIENT_ID_LEN 32 - -#define TSDB_DB_TYPE_DEFAULT 0 -#define TSDB_DB_TYPE_TOPIC 1 - -#define TSDB_DEFAULT_PKT_SIZE 65480 //same as RPC_MAX_UDP_SIZE - -#define TSDB_PAYLOAD_SIZE TSDB_DEFAULT_PKT_SIZE -#define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value -#define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth -#define TSDB_CQ_SQL_SIZE 1024 -#define TSDB_MIN_VNODES 64 -#define TSDB_MAX_VNODES 2048 -#define TSDB_MIN_VNODES_PER_DB 2 -#define TSDB_MAX_VNODES_PER_DB 64 - -#define TSDB_DNODE_ROLE_ANY 0 -#define TSDB_DNODE_ROLE_MGMT 1 -#define TSDB_DNODE_ROLE_VNODE 2 - -#define TSDB_MAX_REPLICA 5 - -#define TSDB_TBNAME_COLUMN_INDEX (-1) -#define TSDB_UD_COLUMN_INDEX (-1000) -#define TSDB_RES_COL_ID (-5000) - -#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta - -#define TSDB_MIN_CACHE_BLOCK_SIZE 1 -#define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode -#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 - -#define TSDB_MIN_TOTAL_BLOCKS 3 -#define TSDB_MAX_TOTAL_BLOCKS 10000 -#define TSDB_DEFAULT_TOTAL_BLOCKS 6 - -#define TSDB_MIN_TABLES 4 -#define TSDB_MAX_TABLES 10000000 -#define TSDB_DEFAULT_TABLES 1000000 -#define TSDB_TABLES_STEP 1000 -#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default - -#define TSDB_MIN_DAYS_PER_FILE 1 -#define TSDB_MAX_DAYS_PER_FILE 3650 -#define TSDB_DEFAULT_DAYS_PER_FILE 10 - -#define TSDB_MIN_KEEP 1 // data in db to be reserved. -#define TSDB_MAX_KEEP 365000 // data in db to be reserved. -#define TSDB_DEFAULT_KEEP 3650 // ten years - -#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 -#define TSDB_MIN_MIN_ROW_FBLOCK 10 -#define TSDB_MAX_MIN_ROW_FBLOCK 1000 - -#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 -#define TSDB_MIN_MAX_ROW_FBLOCK 200 -#define TSDB_MAX_MAX_ROW_FBLOCK 10000 - -#define TSDB_MIN_COMMIT_TIME 30 -#define TSDB_MAX_COMMIT_TIME 40960 -#define TSDB_DEFAULT_COMMIT_TIME 3600 - -#define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI -#define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO -#define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI - -#define TSDB_MIN_COMP_LEVEL 0 -#define TSDB_MAX_COMP_LEVEL 2 -#define TSDB_DEFAULT_COMP_LEVEL 2 - -#define TSDB_MIN_WAL_LEVEL 0 -#define TSDB_MAX_WAL_LEVEL 2 -#define TSDB_DEFAULT_WAL_LEVEL 1 - -#define TSDB_MIN_DB_UPDATE 0 -#define TSDB_MAX_DB_UPDATE 2 -#define TSDB_DEFAULT_DB_UPDATE_OPTION 0 - -#define TSDB_MIN_DB_CACHE_LAST_ROW 0 -#define TSDB_MAX_DB_CACHE_LAST_ROW 3 -#define TSDB_DEFAULT_CACHE_LAST_ROW 0 - -#define TSDB_MIN_FSYNC_PERIOD 0 -#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond -#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second - -#define TSDB_MIN_DB_REPLICA_OPTION 1 -#define TSDB_MAX_DB_REPLICA_OPTION 3 -#define TSDB_DEFAULT_DB_REPLICA_OPTION 1 - -#define TSDB_MIN_DB_PARTITON_OPTION 0 -#define TSDB_MAX_DB_PARTITON_OPTION 1000 -#define TSDB_DEFAULT_DB_PARTITON_OPTION 4 - -#define TSDB_MIN_DB_QUORUM_OPTION 1 -#define TSDB_MAX_DB_QUORUM_OPTION 2 -#define TSDB_DEFAULT_DB_QUORUM_OPTION 1 - -#define TSDB_MAX_JOIN_TABLE_NUM 10 -#define TSDB_MAX_UNION_CLAUSE 5 - -#define TSDB_MAX_FIELD_LEN 16384 -#define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 -#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 -#define PRIMARYKEY_TIMESTAMP_COL_INDEX 0 - -#define TSDB_MAX_RPC_THREADS 5 - -#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type -#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode - -#define TSDB_UDF_TYPE_SCALAR 1 -#define TSDB_UDF_TYPE_AGGREGATE 2 - - -/* - * 1. ordinary sub query for select * from super_table - * 2. all sqlobj generated by createSubqueryObj with this flag - */ -#define TSDB_QUERY_TYPE_SUBQUERY 0x02u -#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table - -#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side -#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table -#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query -#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query -#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80u // join sub query at the second stage - -#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u -#define TSDB_QUERY_TYPE_INSERT 0x100u // insert type -#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u -#define TSDB_QUERY_TYPE_FILE_INSERT 0x400u // insert data from file -#define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type -#define TSDB_QUERY_TYPE_NEST_SUBQUERY 0x1000u // nested sub query - -#define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) -#define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) -#define TSDB_QUERY_CLEAR_TYPE(x, _type) ((x) &= (~_type)) -#define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE) - -#define TSDB_ORDER_ASC 1 -#define TSDB_ORDER_DESC 2 - -#define TSDB_DEFAULT_CLUSTER_HASH_SIZE 1 -#define TSDB_DEFAULT_MNODES_HASH_SIZE 5 -#define TSDB_DEFAULT_DNODES_HASH_SIZE 10 -#define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10 -#define TSDB_DEFAULT_USERS_HASH_SIZE 20 -#define TSDB_DEFAULT_DBS_HASH_SIZE 100 -#define TSDB_DEFAULT_VGROUPS_HASH_SIZE 100 -#define TSDB_DEFAULT_STABLES_HASH_SIZE 100 -#define TSDB_DEFAULT_CTABLES_HASH_SIZE 20000 - -#define TSDB_PORT_DNODESHELL 0 -#define TSDB_PORT_DNODEDNODE 5 -#define TSDB_PORT_SYNC 10 -#define TSDB_PORT_HTTP 11 -#define TSDB_PORT_ARBITRATOR 12 - -#define TSDB_MAX_WAL_SIZE (1024*1024*3) - -#define TSDB_ARB_DUMMY_TIME 4765104000000 // 2121-01-01 00:00:00.000, :P - typedef enum { TAOS_QTYPE_RPC = 0, TAOS_QTYPE_FWD = 1, @@ -404,10 +35,6 @@ typedef enum { TAOS_QTYPE_QUERY = 4 } EQType; -#define TSDB_MAX_TIERS 3 -#define TSDB_MAX_DISKS_PER_TIER 16 -#define TSDB_MAX_DISKS (TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER) - typedef enum { TSDB_SUPER_TABLE = 0, // super table TSDB_CHILD_TABLE = 1, // table created from super table @@ -449,4 +76,4 @@ extern char *qtypeStr[]; } #endif -#endif +#endif /*_TD_COMMON_TAOS_DEF_H*/ diff --git a/src/common/inc/tarithoperator.h b/include/common/tarithoperator.h similarity index 100% rename from src/common/inc/tarithoperator.h rename to include/common/tarithoperator.h diff --git a/src/common/inc/tcmdtype.h b/include/common/tcmdtype.h similarity index 100% rename from src/common/inc/tcmdtype.h rename to include/common/tcmdtype.h diff --git a/src/common/inc/texpr.h b/include/common/texpr.h similarity index 100% rename from src/common/inc/texpr.h rename to include/common/texpr.h diff --git a/include/common/tglobal.h b/include/common/tglobal.h index a5f8005e9c..867d4ae2dd 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -13,10 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_COMMON_GLOBAL_H -#define TDENGINE_COMMON_GLOBAL_H - -#include "taosdef.h" +#ifndef _TD_COMMON_GLOBAL_H +#define _TD_COMMON_GLOBAL_H #ifdef __cplusplus extern "C" { @@ -52,9 +50,6 @@ extern float tsNumOfThreadsPerCore; extern int32_t tsNumOfCommitThreads; extern float tsRatioOfQueryCores; extern int8_t tsDaylight; -extern char tsTimezone[]; -extern char tsLocale[]; -extern char tsCharset[]; // default encode string extern int8_t tsEnableCoreFile; extern int32_t tsCompressMsgSize; extern int32_t tsCompressColData; @@ -151,24 +146,14 @@ extern int8_t tsEnableStream; // internal extern int8_t tsPrintAuth; extern int8_t tscEmbedded; -extern char configDir[]; extern char tsVnodeDir[]; -extern char tsDnodeDir[]; extern char tsMnodeDir[]; extern char tsMnodeBakDir[]; extern char tsMnodeTmpDir[]; -extern char tsDataDir[]; -extern char tsLogDir[]; -extern char tsScriptDir[]; extern int64_t tsTickPerDay[3]; extern int32_t tsTopicBianryLen; // system info -extern char tsOsName[]; -extern int64_t tsPageSize; -extern int64_t tsOpenMax; -extern int64_t tsStreamMax; -extern int32_t tsNumOfCores; extern float tsTotalLogDirGB; extern float tsTotalTmpDirGB; extern float tsTotalDataDirGB; @@ -179,7 +164,6 @@ extern float tsUsedDataDirGB; extern float tsMinimalLogDirGB; extern float tsReservedTmpDirectorySpace; extern float tsMinimalDataDirGB; -extern int32_t tsTotalMemoryMB; extern uint32_t tsVersion; // build info @@ -189,28 +173,6 @@ extern char gitinfo[]; extern char gitinfoOfInternal[]; extern char buildinfo[]; -// log -extern int8_t tsAsyncLog; -extern int32_t tsNumOfLogLines; -extern int32_t tsLogKeepDays; -extern int32_t dDebugFlag; -extern int32_t vDebugFlag; -extern int32_t mDebugFlag; -extern uint32_t cDebugFlag; -extern int32_t jniDebugFlag; -extern int32_t tmrDebugFlag; -extern int32_t sdbDebugFlag; -extern int32_t httpDebugFlag; -extern int32_t mqttDebugFlag; -extern int32_t monDebugFlag; -extern int32_t uDebugFlag; -extern int32_t rpcDebugFlag; -extern int32_t odbcDebugFlag; -extern uint32_t qDebugFlag; -extern int32_t wDebugFlag; -extern int32_t cqDebugFlag; -extern int32_t debugFlag; - #ifdef TD_TSZ // lossy extern char lossyColumns[]; @@ -224,7 +186,7 @@ extern char Compressor[]; extern int8_t tsDeadLockKillQuery; typedef struct { - char dir[TSDB_FILENAME_LEN]; + char dir[128]; int level; int primary; } SDiskCfg; @@ -235,7 +197,6 @@ extern SDiskCfg tsDiskCfg[]; void taosInitGlobalCfg(); int32_t taosCheckGlobalCfg(); -void taosSetAllDebugFlag(); bool taosCfgDynamicOptions(char *msg); int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); diff --git a/src/common/inc/tlocale.h b/include/common/tlocale.h similarity index 100% rename from src/common/inc/tlocale.h rename to include/common/tlocale.h diff --git a/src/common/inc/tname.h b/include/common/tname.h similarity index 100% rename from src/common/inc/tname.h rename to include/common/tname.h diff --git a/include/common/ttime.h b/include/common/ttime.h new file mode 100644 index 0000000000..0d9b89b6f9 --- /dev/null +++ b/include/common/ttime.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_COMMON_TIME_H_ +#define _TD_COMMON_TIME_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "taosdef.h" +#include "taosmsg.h" + +/* + * @return timestamp decided by global conf variable, tsTimePrecision + * if precision == TSDB_TIME_PRECISION_MICRO, it returns timestamp in microsecond. + * precision == TSDB_TIME_PRECISION_MILLI, it returns timestamp in millisecond. + */ +static FORCE_INLINE int64_t taosGetTimestamp(int32_t precision) { + if (precision == TSDB_TIME_PRECISION_MICRO) { + return taosGetTimestampUs(); + } else if (precision == TSDB_TIME_PRECISION_NANO) { + return taosGetTimestampNs(); + }else { + return taosGetTimestampMs(); + } +} + +int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); +int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision); +int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision); + +int32_t parseAbsoluteDuration(char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision); +int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision); + +int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth); +void deltaToUtcInitOnce(); + +int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision); + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_COMMON_TIME_H_*/ diff --git a/src/common/inc/ttimezone.h b/include/common/ttimezone.h similarity index 100% rename from src/common/inc/ttimezone.h rename to include/common/ttimezone.h diff --git a/include/common/ttype.h b/include/common/ttype.h index 3f1606e957..42296f80ba 100644 --- a/include/common/ttype.h +++ b/include/common/ttype.h @@ -9,43 +9,6 @@ extern "C" { #include #include "taosdef.h" -// ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR -typedef int32_t VarDataOffsetT; -typedef int16_t VarDataLenT; // maxVarDataLen: 32767 -typedef uint16_t TDRowLenT; // not including overhead: 0 ~ 65535 -typedef uint32_t TDRowTLenT; // total length, including overhead - -typedef struct tstr { - VarDataLenT len; - char data[]; -} tstr; - -#pragma pack(push, 1) -typedef struct { - VarDataLenT len; - uint8_t data; -} SBinaryNullT; - -typedef struct { - VarDataLenT len; - uint32_t data; -} SNCharNullT; -#pragma pack(pop) - -#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) - -#define varDataLen(v) ((VarDataLenT *)(v))[0] -#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) -#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE)) -#define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v)) -#define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE)) -#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len)) -#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) - -#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0])) -#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v)) - - // this data type is internally used only in 'in' query to hold the values #define TSDB_DATA_TYPE_POINTER_ARRAY (1000) #define TSDB_DATA_TYPE_VALUE_ARRAY (1001) diff --git a/src/common/inc/tvariant.h b/include/common/tvariant.h similarity index 100% rename from src/common/inc/tvariant.h rename to include/common/tvariant.h diff --git a/include/os/os.h b/include/os/os.h index d450ff620f..447df9d65c 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -27,20 +27,26 @@ extern "C" { #include #include #include +#include #include #include #include +#include #include #include #include #include #include +#include +#include #include "osAtomic.h" #include "osDef.h" #include "osDir.h" #include "osEndian.h" +#include "osEnv.h" #include "osFile.h" +#include "osLz4.h" #include "osMath.h" #include "osMemory.h" #include "osRand.h" @@ -49,10 +55,14 @@ extern "C" { #include "osSleep.h" #include "osSocket.h" #include "osString.h" +#include "osSysinfo.h" +#include "osSystem.h" #include "osThread.h" #include "osTime.h" #include "osTimer.h" +void osInit(); + #ifdef __cplusplus } #endif diff --git a/include/os/osDef.h b/include/os/osDef.h index 053bf42ece..8db9c6584f 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -20,14 +20,154 @@ extern "C" { #endif +#if defined(_TD_DARWIN_64) + // specific + typedef int(*__compar_fn_t)(const void *, const void *); + + // for send function in tsocket.c + #if defined(MSG_NOSIGNAL) + #undef MSG_NOSIGNAL + #endif + + #define MSG_NOSIGNAL 0 + + #define SO_NO_CHECK 0x1234 + #define SOL_TCP 0x1234 + #define TCP_KEEPIDLE 0x1234 + + #ifndef PTHREAD_MUTEX_RECURSIVE_NP + #define PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE + #endif +#endif + +#if defined(_ALPINE) + typedef int(*__compar_fn_t)(const void *, const void *); + void error (int, int, const char *); + #ifndef PTHREAD_MUTEX_RECURSIVE_NP + #define PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE + #endif +#endif + + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + char *stpcpy (char *dest, const char *src); + char *stpncpy (char *dest, const char *src, size_t n); + + // specific + typedef int (*__compar_fn_t)(const void *, const void *); + #define ssize_t int + #define bzero(ptr, size) memset((ptr), 0, (size)) + #define strcasecmp _stricmp + #define strncasecmp _strnicmp + #define wcsncasecmp _wcsnicmp + #define strtok_r strtok_s + #define snprintf _snprintf + #define in_addr_t unsigned long + #define socklen_t int + + struct tm *localtime_r(const time_t *timep, struct tm *result); + char * strptime(const char *buf, const char *fmt, struct tm *tm); + char * strsep(char **stringp, const char *delim); + char * getpass(const char *prefix); + char * strndup(const char *s, size_t n); + +#endif + +#define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b))) +#define POINTER_DISTANCE(p1, p2) ((char *)(p1) - (char *)(p2)) + +#ifndef NDEBUG +#define ASSERT(x) assert(x) +#else +#define ASSERT(x) +#endif + +#ifndef UNUSED +#define UNUSED(x) ((void)(x)) +#endif + +#ifdef UNUSED_FUNC +#undefine UNUSED_FUNC +#endif + +#ifdef UNUSED_PARAM +#undef UNUSED_PARAM +#endif + +#if defined(__GNUC__) +#define UNUSED_PARAM(x) _UNUSED##x __attribute__((unused)) +#define UNUSED_FUNC __attribute__((unused)) +#else +#define UNUSED_PARAM(x) x +#define UNUSED_FUNC +#endif + +// TODO: replace and remove code below +#define CHAR_BYTES sizeof(char) +#define SHORT_BYTES sizeof(int16_t) +#define INT_BYTES sizeof(int32_t) +#define LONG_BYTES sizeof(int64_t) +#define FLOAT_BYTES sizeof(float) +#define DOUBLE_BYTES sizeof(double) +#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) + +#ifdef tListLen +#undefine tListLen +#endif +#define tListLen(x) (sizeof(x) / sizeof((x)[0])) + #if defined(__GNUC__) #define FORCE_INLINE inline __attribute__((always_inline)) #else #define FORCE_INLINE #endif -#define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b))) -#define POINTER_DISTANCE(p1, p2) ((char *)(p1) - (char *)(p2)) +#define DEFAULT_UNICODE_ENCODEC "UCS-4LE" + +#define DEFAULT_COMP(x, y) \ + do { \ + if ((x) == (y)) { \ + return 0; \ + } else { \ + return (x) < (y) ? -1 : 1; \ + } \ + } while (0) + +#define DEFAULT_DOUBLE_COMP(x, y) \ + do { \ + if (isnan(x) && isnan(y)) { return 0; } \ + if (isnan(x)) { return -1; } \ + if (isnan(y)) { return 1; } \ + if ((x) == (y)) { \ + return 0; \ + } else { \ + return (x) < (y) ? -1 : 1; \ + } \ + } while (0) + +#define DEFAULT_FLOAT_COMP(x, y) DEFAULT_DOUBLE_COMP(x, y) + +#define ALIGN_NUM(n, align) (((n) + ((align)-1)) & (~((align)-1))) + +// align to 8bytes +#define ALIGN8(n) ALIGN_NUM(n, 8) + +#undef threadlocal +#ifdef _ISOC11_SOURCE + #define threadlocal _Thread_local +#elif defined(__APPLE__) + #define threadlocal __thread +#elif defined(__GNUC__) && !defined(threadlocal) + #define threadlocal __thread +#else + #define threadlocal __declspec( thread ) +#endif + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + #define PRIzu "ld" +#else + #define PRIzu "zu" +#endif #if defined(_TD_LINUX_64) || defined(_TD_LINUX_32) || defined(_TD_MIPS_64) || defined(_TD_ARM_32) || defined(_TD_ARM_64) || defined(_TD_DARWIN_64) #if defined(_TD_DARWIN_64) @@ -47,17 +187,6 @@ extern "C" { #define setThreadName(name) #endif - - -// TODO: replace and remove code below -#define CHAR_BYTES sizeof(char) -#define SHORT_BYTES sizeof(int16_t) -#define INT_BYTES sizeof(int32_t) -#define LONG_BYTES sizeof(int64_t) -#define FLOAT_BYTES sizeof(float) -#define DOUBLE_BYTES sizeof(double) -#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) - #ifdef __cplusplus } #endif diff --git a/include/os/osDir.h b/include/os/osDir.h index dfd814d70b..74286edbe8 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -20,12 +20,12 @@ extern "C" { #endif -void taosRemoveDir(char *rootDir); -bool taosDirExist(const char* dirname); -int32_t taosMkDir(const char *pathname, mode_t mode); -void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays); -int32_t taosRename(char *oldName, char *newName); -int32_t taosCompressFile(char *srcFileName, char *destFileName); +void taosRemoveDir(char *dirname); +bool taosDirExist(char *dirname); +bool taosMkDir(char *dirname, mode_t mode); +void taosRemoveOldFiles(char *dirname, int32_t keepDays); +bool taosExpandDir(char *dirname, char *outname, int32_t maxlen); +bool taosRealPath(char *dirname, int32_t maxlen); #ifdef __cplusplus } diff --git a/src/os/inc/osTimer.h b/include/os/osEnv.h similarity index 76% rename from src/os/inc/osTimer.h rename to include/os/osEnv.h index 72da19cd89..b857670214 100644 --- a/src/os/inc/osTimer.h +++ b/include/os/osEnv.h @@ -13,18 +13,22 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_OS_TIMER_H -#define TDENGINE_OS_TIMER_H +#ifndef _TD_OS_ENV_H_ +#define _TD_OS_ENV_H_ #ifdef __cplusplus extern "C" { #endif -int taosInitTimer(void (*callback)(int), int ms); -void taosUninitTimer(); +extern char tsOsName[]; +extern char tsDnodeDir[]; +extern char tsDataDir[]; +extern char tsLogDir[]; +extern char tsScriptDir[]; +extern char configDir[]; #ifdef __cplusplus } #endif -#endif +#endif /*_TD_OS_ENV_H_*/ \ No newline at end of file diff --git a/src/os/inc/osLz4.h b/include/os/osLz4.h similarity index 95% rename from src/os/inc/osLz4.h rename to include/os/osLz4.h index 8363861e03..09aaa553ac 100644 --- a/src/os/inc/osLz4.h +++ b/include/os/osLz4.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_OS_LZ4_H -#define TDENGINE_OS_LZ4_H +#ifndef _TD_OS_LZ4_H_ +#define _TD_OS_LZ4_H_ #ifdef __cplusplus extern "C" { @@ -46,4 +46,4 @@ extern "C" { } #endif -#endif +#endif /*_TD_OS_LZ4_H_*/ diff --git a/include/os/osSignal.h b/include/os/osSignal.h index ce133e62b7..ab4b6fa242 100644 --- a/include/os/osSignal.h +++ b/include/os/osSignal.h @@ -20,8 +20,34 @@ extern "C" { #endif -int32_t taosInitTimer(void (*callback)(int32_t), int32_t ms); -void taosUninitTimer(); +#ifndef SIGALRM + #define SIGALRM 1234 +#endif + +#ifndef SIGHUP + #define SIGHUP 1230 +#endif + +#ifndef SIGCHLD + #define SIGCHLD 1234 +#endif + +#ifndef SIGUSR1 + #define SIGUSR1 1234 +#endif + +#ifndef SIGUSR2 + #define SIGUSR2 1234 +#endif + +#ifndef SIGBREAK + #define SIGBREAK 1234 +#endif + +typedef void (*FSignalHandler)(int32_t signum, void *sigInfo, void *context); +void taosSetSignal(int32_t signum, FSignalHandler sigfp); +void taosIgnSignal(int32_t signum); +void taosDflSignal(int32_t signum); #ifdef __cplusplus } diff --git a/include/os/osSleep.h b/include/os/osSleep.h index e42da8d5a6..686bdd292e 100644 --- a/include/os/osSleep.h +++ b/include/os/osSleep.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_OS_SLEEP_H -#define TDENGINE_OS_SLEEP_H +#ifndef _TD_OS_SLEEP_H_ +#define _TD_OS_SLEEP_H_ #ifdef __cplusplus extern "C" { @@ -26,4 +26,4 @@ void taosMsleep(int32_t ms); } #endif -#endif \ No newline at end of file +#endif /*_TD_OS_SLEEP_H_*/ \ No newline at end of file diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 719c693765..11a6486ff6 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_OS_SOCKET_H -#define TDENGINE_OS_SOCKET_H +#ifndef _TD_OS_SOCKET_H_ +#define _TD_OS_SOCKET_H_ #ifdef __cplusplus extern "C" { @@ -89,4 +89,4 @@ const char *taosInetNtoa(struct in_addr ipInt); } #endif -#endif +#endif /*_TD_OS_SOCKET_H_*/ diff --git a/include/os/osString.h b/include/os/osString.h index 3adee83c84..358324075f 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -43,7 +43,7 @@ int64_t taosStr2int64(char *str); // USE_LIBICONV int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); bool taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len); -int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes); +int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize); bool taosValidateEncodec(const char *encodec); char * taosCharsetReplace(char *charsetstr); diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h new file mode 100644 index 0000000000..62b52827d6 --- /dev/null +++ b/include/os/osSysinfo.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_OS_SYSINFO_H_ +#define _TD_OS_SYSINFO_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#define TSDB_LOCALE_LEN 64 +#define TSDB_TIMEZONE_LEN 96 + +extern int64_t tsPageSize; +extern int64_t tsOpenMax; +extern int64_t tsStreamMax; +extern int32_t tsNumOfCores; +extern int32_t tsTotalMemoryMB; +extern char tsTimezone[]; +extern char tsLocale[]; +extern char tsCharset[]; // default encode string + +typedef struct { + int64_t tsize; + int64_t used; + int64_t avail; +} SysDiskSize; + +int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize); +int32_t taosGetCpuCores(); +void taosGetSystemInfo(); +bool taosReadProcIO(int64_t *rchars, int64_t *wchars); +bool taosGetProcIO(float *readKB, float *writeKB); +bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes); +bool taosGetBandSpeed(float *bandSpeedKb); +void taosGetDisk(); +bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage); +bool taosGetProcMemory(float *memoryUsedMB); +bool taosGetSysMemory(float *memoryUsedMB); +void taosPrintOsInfo(); +int taosSystem(const char *cmd); +void taosKillSystem(); +bool taosGetSystemUid(char *uid, int32_t uidlen); +char * taosGetCmdlineByPID(int pid); +void taosSetCoreDump(bool enable); + +typedef struct { + const char *sysname; + const char *nodename; + const char *release; + const char *version; + const char *machine; +} SysNameInfo; + +SysNameInfo taosGetSysNameInfo(); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/inc/osSystem.h b/include/os/osSystem.h similarity index 79% rename from src/os/inc/osSystem.h rename to include/os/osSystem.h index 4b79250740..8554c4ad12 100644 --- a/src/os/inc/osSystem.h +++ b/include/os/osSystem.h @@ -13,21 +13,21 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_OS_SYSTEM_H -#define TDENGINE_OS_SYSTEM_H +#ifndef _TD_OS_SYSTEM_H_ +#define _TD_OS_SYSTEM_H_ #ifdef __cplusplus extern "C" { #endif -void* taosLoadDll(const char *filename); +void* taosLoadDll(const char* filename); void* taosLoadSym(void* handle, char* name); -void taosCloseDll(void *handle); +void taosCloseDll(void* handle); -int taosSetConsoleEcho(bool on); +int32_t taosSetConsoleEcho(bool on); #ifdef __cplusplus } #endif -#endif +#endif /*_TD_OS_SYSTEM_H_*/ diff --git a/include/os/osTime.h b/include/os/osTime.h index d076d50276..62ecd5477f 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -20,6 +20,21 @@ extern "C" { #endif +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + #ifdef _TD_GO_DLL_ + #define MILLISECOND_PER_SECOND (1000LL) + #else + #define MILLISECOND_PER_SECOND (1000i64) + #endif +#else + #define MILLISECOND_PER_SECOND ((int64_t)1000L) +#endif + +#define MILLISECOND_PER_MINUTE (MILLISECOND_PER_SECOND * 60) +#define MILLISECOND_PER_HOUR (MILLISECOND_PER_MINUTE * 60) +#define MILLISECOND_PER_DAY (MILLISECOND_PER_HOUR * 24) +#define MILLISECOND_PER_WEEK (MILLISECOND_PER_DAY * 7) + int32_t taosGetTimeOfDay(struct timeval *tv); //@return timestamp in second @@ -46,11 +61,6 @@ static FORCE_INLINE int64_t taosGetTimestampNs() { return (int64_t)systemTime.tv_sec * 1000000000L + (int64_t)systemTime.tv_nsec; } -int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth); -void deltaToUtcInitOnce(); - -int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision); - #ifdef __cplusplus } #endif diff --git a/include/os/osTimer.h b/include/os/osTimer.h index 623d2da20a..2afe018c43 100644 --- a/include/os/osTimer.h +++ b/include/os/osTimer.h @@ -20,6 +20,8 @@ extern "C" { #endif +#define MSECONDS_PER_TICK 5 + int32_t taosInitTimer(void (*callback)(int32_t), int32_t ms); void taosUninitTimer(); diff --git a/src/util/inc/exception.h b/include/util/exception.h similarity index 96% rename from src/util/inc/exception.h rename to include/util/exception.h index 38f94bfe28..8012e34315 100644 --- a/src/util/inc/exception.h +++ b/include/util/exception.h @@ -13,13 +13,10 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_EXCEPTION_H -#define TDENGINE_EXCEPTION_H +#ifndef _TD_UTIL_EXCEPTION_H +#define _TD_UTIL_EXCEPTION_H -#include -#include -#include -#include +#include "os.h" #ifdef __cplusplus extern "C" { @@ -125,4 +122,4 @@ void exceptionThrow( int32_t code ); } #endif -#endif +#endif /*_TD_UTIL_EXCEPTION_H*/ diff --git a/include/util/hash.h b/include/util/hash.h index 7a93745dc8..5c723be106 100644 --- a/include/util/hash.h +++ b/include/util/hash.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_HASH_H -#define TDENGINE_HASH_H +#ifndef _TD_UTIL_HASH_H +#define _TD_UTIL_HASH_H #ifdef __cplusplus extern "C" { @@ -178,4 +178,4 @@ uint32_t taosHashGetDataKeyLen(SHashObj *pHashObj, void *data); } #endif -#endif // TDENGINE_HASH_H +#endif /*_TD_UTIL_HASH_H*/ diff --git a/include/util/hashfunc.h b/include/util/hashfunc.h index a9563d0394..04c2d1fcf0 100644 --- a/include/util/hashfunc.h +++ b/include/util/hashfunc.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_HASHUTIL_H -#define TDENGINE_HASHUTIL_H +#ifndef _TD_UTIL_HASH_FUNC_H +#define _TD_UTIL_HASH_FUNC_H #include "os.h" @@ -48,4 +48,4 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type); _equal_fn_t taosGetDefaultEqualFunction(int32_t type); -#endif //TDENGINE_HASHUTIL_H +#endif /*_TD_UTIL_HASH_FUNC_H*/ diff --git a/include/util/talgo.h b/include/util/talgo.h index 4aa5430605..575f98b486 100644 --- a/include/util/talgo.h +++ b/include/util/talgo.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TALGO_H -#define TDENGINE_TALGO_H +#ifndef _TD_UTIL_TALGO_H +#define _TD_UTIL_TALGO_H #ifdef __cplusplus extern "C" { @@ -95,4 +95,4 @@ void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, #ifdef __cplusplus } #endif -#endif // TDENGINE_TALGO_H +#endif /*_TD_UTIL_TALGO_H*/ diff --git a/include/util/tarray.h b/include/util/tarray.h index 2da74eac82..5807c980e0 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TAOSARRAY_H -#define TDENGINE_TAOSARRAY_H +#ifndef _TD_UTIL_ARRAY_H +#define _TD_UTIL_ARRAY_H #ifdef __cplusplus extern "C" { @@ -228,6 +228,4 @@ void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void *par } #endif - - -#endif // TDENGINE_TAOSARRAY_H +#endif /*_TD_UTIL_ARRAY_H*/ diff --git a/include/util/tbuffer.h b/include/util/tbuffer.h index b19984b4be..d1d403e996 100644 --- a/include/util/tbuffer.h +++ b/include/util/tbuffer.h @@ -13,11 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TBUFFER_H -#define TDENGINE_TBUFFER_H - -#include -#include +#ifndef _TD_UTIL_BUFFER_H +#define _TD_UTIL_BUFFER_H #ifdef __cplusplus extern "C" { @@ -175,4 +172,4 @@ void tbufWriteDoubleAt( SBufferWriter* buf, size_t pos, double data ); } #endif -#endif +#endif /*_TD_UTIL_BUFFER_H*/ diff --git a/include/util/tcache.h b/include/util/tcache.h index 40069d7d27..f0bf78cd92 100644 --- a/include/util/tcache.h +++ b/include/util/tcache.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TCACHE_H -#define TDENGINE_TCACHE_H +#ifndef _TD_UTIL_CACHE_H +#define _TD_UTIL_CACHE_H #ifdef __cplusplus extern "C" { @@ -188,4 +188,4 @@ void taosStopCacheRefreshWorker(); } #endif -#endif // TDENGINE_TCACHE_H +#endif /*_TD_UTIL_CACHE_H*/ diff --git a/src/util/inc/tchecksum.h b/include/util/tchecksum.h similarity index 100% rename from src/util/inc/tchecksum.h rename to include/util/tchecksum.h diff --git a/include/util/tcoding.h b/include/util/tcoding.h index 3ccf51c2dd..ccc3c26124 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -12,8 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef _TD_CODING_H_ -#define _TD_CODING_H_ +#ifndef _TD_UTIL_CODING_H +#define _TD_UTIL_CODING_H #ifdef __cplusplus extern "C" { @@ -361,4 +361,4 @@ static FORCE_INLINE void *taosDecodeString(void *buf, char **value) { } #endif -#endif \ No newline at end of file +#endif /*_TD_UTIL_CODING_H*/ \ No newline at end of file diff --git a/src/util/inc/tcompare.h b/include/util/tcompare.h similarity index 100% rename from src/util/inc/tcompare.h rename to include/util/tcompare.h diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 2c632d4a17..68fa4128ae 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_CFG_H -#define TDENGINE_CFG_H +#ifndef _TD_UTIL_CONFIG_H +#define _TD_UTIL_CONFIG_H #ifdef __cplusplus extern "C" { @@ -89,10 +89,9 @@ void taosDumpGlobalCfg(); void taosInitConfigOption(SGlobalCfg cfg); SGlobalCfg * taosGetConfigOption(const char *option); -bool taosReadConfigOption(const char *option, char *value, char *value2, char *value3, int8_t cfgStatus, int8_t sourceType); #ifdef __cplusplus } #endif -#endif // TDENGINE_TGLOBALCFG_H +#endif /*_TD_UTIL_CONFIG_H*/ diff --git a/include/util/tcrc32c.h b/include/util/tcrc32c.h index 78e52e226f..200103b173 100644 --- a/include/util/tcrc32c.h +++ b/include/util/tcrc32c.h @@ -18,10 +18,8 @@ 3. This notice may not be removed or altered from any source distribution. */ -#ifndef TDENGINE_TCRC32C_H -#define TDENGINE_TCRC32C_H - -#include +#ifndef _TD_UTIL_CRC32_H +#define _TD_UTIL_CRC32_H #ifdef __cplusplus extern "C" { @@ -41,4 +39,4 @@ void taosResolveCRC(); } #endif -#endif // TDENGINE_TCRC32C_H +#endif /*_TD_UTIL_CRC32_H*/ diff --git a/include/util/tdef.h b/include/util/tdef.h new file mode 100644 index 0000000000..b09f39fc32 --- /dev/null +++ b/include/util/tdef.h @@ -0,0 +1,451 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_DEF_H +#define _TD_UTIL_DEF_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define TSDB__packed + +#ifdef TSKEY32 +#define TSKEY int32_t; +#else +#define TSKEY int64_t +#endif + +#define TSKEY_INITIAL_VAL INT64_MIN + +// Bytes for each type. +extern const int32_t TYPE_BYTES[15]; + +#define TSDB_KEYSIZE sizeof(TSKEY) + +#if LINUX +#define TSDB_NCHAR_SIZE sizeof(wchar_t) +#else +#define TSDB_NCHAR_SIZE sizeof(int32_t) +#endif + +// NULL definition +#define TSDB_DATA_BOOL_NULL 0x02 +#define TSDB_DATA_TINYINT_NULL 0x80 +#define TSDB_DATA_SMALLINT_NULL 0x8000 +#define TSDB_DATA_INT_NULL 0x80000000L +#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L +#define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL + +#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN +#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN +#define TSDB_DATA_NCHAR_NULL 0xFFFFFFFF +#define TSDB_DATA_BINARY_NULL 0xFF + +#define TSDB_DATA_UTINYINT_NULL 0xFF +#define TSDB_DATA_USMALLINT_NULL 0xFFFF +#define TSDB_DATA_UINT_NULL 0xFFFFFFFF +#define TSDB_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL + +#define TSDB_DATA_NULL_STR "NULL" +#define TSDB_DATA_NULL_STR_L "null" + +#define TSDB_DEFAULT_USER "root" +#ifdef _TD_POWER_ +#define TSDB_DEFAULT_PASS "powerdb" +#elif (_TD_TQ_ == true) +#define TSDB_DEFAULT_PASS "tqueue" +#elif (_TD_PRO_ == true) +#define TSDB_DEFAULT_PASS "prodb" +#else +#define TSDB_DEFAULT_PASS "taosdata" +#endif + +#define SHELL_MAX_PASSWORD_LEN 20 + +#define TSDB_TRUE 1 +#define TSDB_FALSE 0 +#define TSDB_OK 0 +#define TSDB_ERR -1 + +#define TS_PATH_DELIMITER "." + +#define TSDB_TIME_PRECISION_MILLI 0 +#define TSDB_TIME_PRECISION_MICRO 1 +#define TSDB_TIME_PRECISION_NANO 2 + +#define TSDB_TIME_PRECISION_MILLI_STR "ms" +#define TSDB_TIME_PRECISION_MICRO_STR "us" +#define TSDB_TIME_PRECISION_NANO_STR "ns" + +#define TSDB_TICK_PER_SECOND(precision) ((int64_t)((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))) + +#define T_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) +#define T_APPEND_MEMBER(dst, ptr, type, member) \ +do {\ + memcpy((void *)(dst), (void *)(&((ptr)->member)), T_MEMBER_SIZE(type, member));\ + dst = (void *)((char *)(dst) + T_MEMBER_SIZE(type, member));\ +} while(0) +#define T_READ_MEMBER(src, type, target) \ +do { \ + (target) = *(type *)(src); \ + (src) = (void *)((char *)src + sizeof(type));\ +} while(0) + +#define GET_INT8_VAL(x) (*(int8_t *)(x)) +#define GET_INT16_VAL(x) (*(int16_t *)(x)) +#define GET_INT32_VAL(x) (*(int32_t *)(x)) +#define GET_INT64_VAL(x) (*(int64_t *)(x)) +#define GET_UINT8_VAL(x) (*(uint8_t*) (x)) +#define GET_UINT16_VAL(x) (*(uint16_t *)(x)) +#define GET_UINT32_VAL(x) (*(uint32_t *)(x)) +#define GET_UINT64_VAL(x) (*(uint64_t *)(x)) + +#ifdef _TD_ARM_32 + float taos_align_get_float(const char* pBuf); + double taos_align_get_double(const char* pBuf); + + #define GET_FLOAT_VAL(x) taos_align_get_float(x) + #define GET_DOUBLE_VAL(x) taos_align_get_double(x) + #define SET_FLOAT_VAL(x, y) { float z = (float)(y); (*(int32_t*) x = *(int32_t*)(&z)); } + #define SET_DOUBLE_VAL(x, y) { double z = (double)(y); (*(int64_t*) x = *(int64_t*)(&z)); } + #define SET_FLOAT_PTR(x, y) { (*(int32_t*) x = *(int32_t*)y); } + #define SET_DOUBLE_PTR(x, y) { (*(int64_t*) x = *(int64_t*)y); } +#else + #define GET_FLOAT_VAL(x) (*(float *)(x)) + #define GET_DOUBLE_VAL(x) (*(double *)(x)) + #define SET_FLOAT_VAL(x, y) { (*(float *)(x)) = (float)(y); } + #define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y); } + #define SET_FLOAT_PTR(x, y) { (*(float *)(x)) = (*(float *)(y)); } + #define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); } +#endif + +// TODO: check if below is necessary +#define TSDB_RELATION_INVALID 0 +#define TSDB_RELATION_LESS 1 +#define TSDB_RELATION_GREATER 2 +#define TSDB_RELATION_EQUAL 3 +#define TSDB_RELATION_LESS_EQUAL 4 +#define TSDB_RELATION_GREATER_EQUAL 5 +#define TSDB_RELATION_NOT_EQUAL 6 +#define TSDB_RELATION_LIKE 7 +#define TSDB_RELATION_ISNULL 8 +#define TSDB_RELATION_NOTNULL 9 +#define TSDB_RELATION_IN 10 + +#define TSDB_RELATION_AND 11 +#define TSDB_RELATION_OR 12 +#define TSDB_RELATION_NOT 13 + +#define TSDB_RELATION_MATCH 14 +#define TSDB_RELATION_NMATCH 15 + +#define TSDB_BINARY_OP_ADD 30 +#define TSDB_BINARY_OP_SUBTRACT 31 +#define TSDB_BINARY_OP_MULTIPLY 32 +#define TSDB_BINARY_OP_DIVIDE 33 +#define TSDB_BINARY_OP_REMAINDER 34 + + +#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN)) +#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER)) + +#define TS_PATH_DELIMITER_LEN 1 + +#define TSDB_UNI_LEN 24 +#define TSDB_USER_LEN TSDB_UNI_LEN + +// ACCOUNT is a 32 bit positive integer +// this is the length of its string representation, including the terminator zero +#define TSDB_ACCT_ID_LEN 11 + +#define TSDB_MAX_COLUMNS 4096 +#define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns + +#define TSDB_NODE_NAME_LEN 64 +#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string +#define TSDB_DB_NAME_LEN 33 +#define TSDB_FUNC_NAME_LEN 65 +#define TSDB_FUNC_CODE_LEN (65535 - 512) +#define TSDB_FUNC_BUF_SIZE 512 +#define TSDB_TYPE_STR_MAX_LEN 32 +#define TSDB_TABLE_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN) +#define TSDB_COL_NAME_LEN 65 +#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 +#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE +#define TSDB_MAX_SQL_SHOW_LEN 512 +#define TSDB_MAX_ALLOWED_SQL_LEN (1*1024*1024u) // sql length should be less than 1mb + +#define TSDB_APPNAME_LEN TSDB_UNI_LEN + + /** + * In some scenarios uint16_t (0~65535) is used to store the row len. + * - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header. + * - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus + * the final value is 65531-(4096-1)*4 = 49151. + */ +#define TSDB_MAX_BYTES_PER_ROW 49151 +#define TSDB_MAX_TAGS_LEN 16384 +#define TSDB_MAX_TAGS 128 +#define TSDB_MAX_TAG_CONDITIONS 1024 + +#define TSDB_AUTH_LEN 16 +#define TSDB_KEY_LEN 16 +#define TSDB_VERSION_LEN 12 +#define TSDB_LABEL_LEN 8 + +#define TSDB_CLUSTER_ID_LEN 40 +#define TSDB_FQDN_LEN 128 +#define TSDB_EP_LEN (TSDB_FQDN_LEN+6) +#define TSDB_IPv4ADDR_LEN 16 +#define TSDB_FILENAME_LEN 128 +#define TSDB_SHOW_SQL_LEN 512 +#define TSDB_SHOW_SUBQUERY_LEN 1000 +#define TSDB_SLOW_QUERY_SQL_LEN 512 + +#define TSDB_STEP_NAME_LEN 32 +#define TSDB_STEP_DESC_LEN 128 + +#define TSDB_MQTT_HOSTNAME_LEN 64 +#define TSDB_MQTT_PORT_LEN 8 +#define TSDB_MQTT_USER_LEN 24 +#define TSDB_MQTT_PASS_LEN 24 +#define TSDB_MQTT_TOPIC_LEN 64 +#define TSDB_MQTT_CLIENT_ID_LEN 32 + +#define TSDB_DB_TYPE_DEFAULT 0 +#define TSDB_DB_TYPE_TOPIC 1 + +#define TSDB_DEFAULT_PKT_SIZE 65480 //same as RPC_MAX_UDP_SIZE + +#define TSDB_PAYLOAD_SIZE TSDB_DEFAULT_PKT_SIZE +#define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value +#define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth +#define TSDB_CQ_SQL_SIZE 1024 +#define TSDB_MIN_VNODES 64 +#define TSDB_MAX_VNODES 2048 +#define TSDB_MIN_VNODES_PER_DB 2 +#define TSDB_MAX_VNODES_PER_DB 64 + +#define TSDB_DNODE_ROLE_ANY 0 +#define TSDB_DNODE_ROLE_MGMT 1 +#define TSDB_DNODE_ROLE_VNODE 2 + +#define TSDB_MAX_REPLICA 5 + +#define TSDB_TBNAME_COLUMN_INDEX (-1) +#define TSDB_UD_COLUMN_INDEX (-1000) +#define TSDB_RES_COL_ID (-5000) + +#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta + +#define TSDB_MIN_CACHE_BLOCK_SIZE 1 +#define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode +#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 + +#define TSDB_MIN_TOTAL_BLOCKS 3 +#define TSDB_MAX_TOTAL_BLOCKS 10000 +#define TSDB_DEFAULT_TOTAL_BLOCKS 6 + +#define TSDB_MIN_TABLES 4 +#define TSDB_MAX_TABLES 10000000 +#define TSDB_DEFAULT_TABLES 1000000 +#define TSDB_TABLES_STEP 1000 +#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default + +#define TSDB_MIN_DAYS_PER_FILE 1 +#define TSDB_MAX_DAYS_PER_FILE 3650 +#define TSDB_DEFAULT_DAYS_PER_FILE 10 + +#define TSDB_MIN_KEEP 1 // data in db to be reserved. +#define TSDB_MAX_KEEP 365000 // data in db to be reserved. +#define TSDB_DEFAULT_KEEP 3650 // ten years + +#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 +#define TSDB_MIN_MIN_ROW_FBLOCK 10 +#define TSDB_MAX_MIN_ROW_FBLOCK 1000 + +#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 +#define TSDB_MIN_MAX_ROW_FBLOCK 200 +#define TSDB_MAX_MAX_ROW_FBLOCK 10000 + +#define TSDB_MIN_COMMIT_TIME 30 +#define TSDB_MAX_COMMIT_TIME 40960 +#define TSDB_DEFAULT_COMMIT_TIME 3600 + +#define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI +#define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO +#define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI + +#define TSDB_MIN_COMP_LEVEL 0 +#define TSDB_MAX_COMP_LEVEL 2 +#define TSDB_DEFAULT_COMP_LEVEL 2 + +#define TSDB_MIN_WAL_LEVEL 0 +#define TSDB_MAX_WAL_LEVEL 2 +#define TSDB_DEFAULT_WAL_LEVEL 1 + +#define TSDB_MIN_DB_UPDATE 0 +#define TSDB_MAX_DB_UPDATE 2 +#define TSDB_DEFAULT_DB_UPDATE_OPTION 0 + +#define TSDB_MIN_DB_CACHE_LAST_ROW 0 +#define TSDB_MAX_DB_CACHE_LAST_ROW 3 +#define TSDB_DEFAULT_CACHE_LAST_ROW 0 + +#define TSDB_MIN_FSYNC_PERIOD 0 +#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond +#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second + +#define TSDB_MIN_DB_REPLICA_OPTION 1 +#define TSDB_MAX_DB_REPLICA_OPTION 3 +#define TSDB_DEFAULT_DB_REPLICA_OPTION 1 + +#define TSDB_MIN_DB_PARTITON_OPTION 0 +#define TSDB_MAX_DB_PARTITON_OPTION 1000 +#define TSDB_DEFAULT_DB_PARTITON_OPTION 4 + +#define TSDB_MIN_DB_QUORUM_OPTION 1 +#define TSDB_MAX_DB_QUORUM_OPTION 2 +#define TSDB_DEFAULT_DB_QUORUM_OPTION 1 + +#define TSDB_MAX_JOIN_TABLE_NUM 10 +#define TSDB_MAX_UNION_CLAUSE 5 + +#define TSDB_MAX_FIELD_LEN 16384 +#define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 +#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 +#define PRIMARYKEY_TIMESTAMP_COL_INDEX 0 + +#define TSDB_MAX_RPC_THREADS 5 + +#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type +#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode + +#define TSDB_UDF_TYPE_SCALAR 1 +#define TSDB_UDF_TYPE_AGGREGATE 2 + + +/* + * 1. ordinary sub query for select * from super_table + * 2. all sqlobj generated by createSubqueryObj with this flag + */ +#define TSDB_QUERY_TYPE_SUBQUERY 0x02u +#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table + +#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side +#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table +#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query +#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query +#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80u // join sub query at the second stage + +#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u +#define TSDB_QUERY_TYPE_INSERT 0x100u // insert type +#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u +#define TSDB_QUERY_TYPE_FILE_INSERT 0x400u // insert data from file +#define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type +#define TSDB_QUERY_TYPE_NEST_SUBQUERY 0x1000u // nested sub query + +#define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) +#define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) +#define TSDB_QUERY_CLEAR_TYPE(x, _type) ((x) &= (~_type)) +#define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE) + +#define TSDB_ORDER_ASC 1 +#define TSDB_ORDER_DESC 2 + +#define TSDB_DEFAULT_CLUSTER_HASH_SIZE 1 +#define TSDB_DEFAULT_MNODES_HASH_SIZE 5 +#define TSDB_DEFAULT_DNODES_HASH_SIZE 10 +#define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10 +#define TSDB_DEFAULT_USERS_HASH_SIZE 20 +#define TSDB_DEFAULT_DBS_HASH_SIZE 100 +#define TSDB_DEFAULT_VGROUPS_HASH_SIZE 100 +#define TSDB_DEFAULT_STABLES_HASH_SIZE 100 +#define TSDB_DEFAULT_CTABLES_HASH_SIZE 20000 + +#define TSDB_PORT_DNODESHELL 0 +#define TSDB_PORT_DNODEDNODE 5 +#define TSDB_PORT_SYNC 10 +#define TSDB_PORT_HTTP 11 +#define TSDB_PORT_ARBITRATOR 12 + +#define TSDB_MAX_WAL_SIZE (1024*1024*3) + +#define TSDB_ARB_DUMMY_TIME 4765104000000 // 2121-01-01 00:00:00.000, :P + +#define TSDB_MAX_TIERS 3 +#define TSDB_MAX_DISKS_PER_TIER 16 +#define TSDB_MAX_DISKS (TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER) + +#define TSDB_DATA_TYPE_NULL 0 // 1 bytes +#define TSDB_DATA_TYPE_BOOL 1 // 1 bytes +#define TSDB_DATA_TYPE_TINYINT 2 // 1 byte +#define TSDB_DATA_TYPE_SMALLINT 3 // 2 bytes +#define TSDB_DATA_TYPE_INT 4 // 4 bytes +#define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes +#define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes +#define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes +#define TSDB_DATA_TYPE_BINARY 8 // string +#define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes +#define TSDB_DATA_TYPE_NCHAR 10 // unicode string +#define TSDB_DATA_TYPE_UTINYINT 11 // 1 byte +#define TSDB_DATA_TYPE_USMALLINT 12 // 2 bytes +#define TSDB_DATA_TYPE_UINT 13 // 4 bytes +#define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes + +// ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR + +typedef int32_t VarDataOffsetT; +typedef int16_t VarDataLenT; // maxVarDataLen: 32767 +typedef uint16_t TDRowLenT; // not including overhead: 0 ~ 65535 +typedef uint32_t TDRowTLenT; // total length, including overhead + +typedef struct tstr { + VarDataLenT len; + char data[]; +} tstr; + +#pragma pack(push, 1) +typedef struct { + VarDataLenT len; + uint8_t data; +} SBinaryNullT; + +typedef struct { + VarDataLenT len; + uint32_t data; +} SNCharNullT; +#pragma pack(pop) + +#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) + +#define varDataLen(v) ((VarDataLenT *)(v))[0] +#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) +#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE)) +#define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v)) +#define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE)) +#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len)) +#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) + +#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0])) +#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v)) + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/util/inc/tfile.h b/include/util/tfile.h similarity index 100% rename from src/util/inc/tfile.h rename to include/util/tfile.h diff --git a/src/util/inc/tfunctional.h b/include/util/tfunctional.h similarity index 100% rename from src/util/inc/tfunctional.h rename to include/util/tfunctional.h diff --git a/include/util/tlist.h b/include/util/tlist.h index 7581904540..8246205e9b 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -12,8 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef _TD_LIST_ -#define _TD_LIST_ +#ifndef _TD_UTIL_LIST_H +#define _TD_UTIL_LIST_H #ifdef __cplusplus extern "C" { @@ -69,4 +69,4 @@ SListNode *tdListNext(SListIter *pIter); } #endif -#endif \ No newline at end of file +#endif /*_TD_UTIL_LIST_H*/ \ No newline at end of file diff --git a/include/util/tlockfree.h b/include/util/tlockfree.h index e960b601ca..82c9607ff4 100644 --- a/include/util/tlockfree.h +++ b/include/util/tlockfree.h @@ -12,8 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef __TD_LOCK_FREE_H__ -#define __TD_LOCK_FREE_H__ +#ifndef _TD_UTIL_LOCK_FREE_H +#define _TD_UTIL_LOCK_FREE_H #include "os.h" @@ -21,7 +21,6 @@ extern "C" { #endif - // reference counting typedef void (*_ref_fn_t)(const void* pObj); @@ -106,9 +105,8 @@ void taosRUnLockLatch(SRWLatch *pLatch); break; \ } - #ifdef __cplusplus } #endif -#endif +#endif /*_TD_UTIL_LOCK_FREE_H*/ diff --git a/include/util/tlog.h b/include/util/tlog.h index 50bc45e7dd..a99085923e 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -13,13 +13,38 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TLOG_H -#define TDENGINE_TLOG_H +#ifndef _TD_UTIL_LOG_H +#define _TD_UTIL_LOG_H #ifdef __cplusplus extern "C" { #endif + +// log +extern int8_t tsAsyncLog; +extern int32_t tsNumOfLogLines; +extern int32_t tsLogKeepDays; +extern int32_t dDebugFlag; +extern int32_t vDebugFlag; +extern int32_t mDebugFlag; +extern int32_t cDebugFlag; +extern int32_t jniDebugFlag; +extern int32_t tmrDebugFlag; +extern int32_t sdbDebugFlag; +extern int32_t httpDebugFlag; +extern int32_t mqttDebugFlag; +extern int32_t monDebugFlag; +extern int32_t uDebugFlag; +extern int32_t rpcDebugFlag; +extern int32_t odbcDebugFlag; +extern int32_t qDebugFlag; +extern int32_t wDebugFlag; +extern int32_t sDebugFlag; +extern int32_t tsdbDebugFlag; +extern int32_t cqDebugFlag; +extern int32_t debugFlag; + #define DEBUG_FATAL 1U #define DEBUG_ERROR DEBUG_FATAL #define DEBUG_WARN 2U @@ -53,4 +78,4 @@ void taosDumpData(unsigned char *msg, int32_t len); } #endif -#endif +#endif /*_TD_UTIL_LOG_H*/ diff --git a/src/util/inc/tlosertree.h b/include/util/tlosertree.h similarity index 100% rename from src/util/inc/tlosertree.h rename to include/util/tlosertree.h diff --git a/src/util/inc/tnettest.h b/include/util/tnettest.h similarity index 100% rename from src/util/inc/tnettest.h rename to include/util/tnettest.h diff --git a/src/util/inc/tqueue.h b/include/util/tqueue.h similarity index 100% rename from src/util/inc/tqueue.h rename to include/util/tqueue.h diff --git a/src/util/inc/tsched.h b/include/util/tsched.h similarity index 100% rename from src/util/inc/tsched.h rename to include/util/tsched.h diff --git a/include/util/tscompression.h b/include/util/tscompression.h index 3f2160bbf2..6f722533ac 100644 --- a/include/util/tscompression.h +++ b/include/util/tscompression.h @@ -20,7 +20,7 @@ extern "C" { #endif -#include "taosdef.h" +#include "tdef.h" #include "tutil.h" #define COMP_OVERFLOW_BYTES 2 diff --git a/src/util/inc/tskiplist.h b/include/util/tskiplist.h similarity index 100% rename from src/util/inc/tskiplist.h rename to include/util/tskiplist.h diff --git a/src/util/inc/tstrbuild.h b/include/util/tstrbuild.h similarity index 100% rename from src/util/inc/tstrbuild.h rename to include/util/tstrbuild.h diff --git a/src/util/inc/tthread.h b/include/util/tthread.h similarity index 100% rename from src/util/inc/tthread.h rename to include/util/tthread.h diff --git a/src/util/inc/ttoken.h b/include/util/ttoken.h similarity index 100% rename from src/util/inc/ttoken.h rename to include/util/ttoken.h diff --git a/source/util/inc/tulog.h b/include/util/tulog.h similarity index 100% rename from source/util/inc/tulog.h rename to include/util/tulog.h diff --git a/include/util/tutil.h b/include/util/tutil.h index 16e5af1ade..ba48fcdc04 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -53,11 +53,6 @@ static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *tar memcpy(target, context.digest, keylen); } -#ifdef tListLen -#undefine tListLen -#endif -#define tListLen(x) (sizeof(x) / sizeof((x)[0])) - #ifdef __cplusplus } #endif diff --git a/src/util/inc/tworker.h b/include/util/tworker.h similarity index 100% rename from src/util/inc/tworker.h rename to include/util/tworker.h diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index 1273a61c02..4a8dffb433 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -4,4 +4,9 @@ target_include_directories( common PUBLIC "${CMAKE_SOURCE_DIR}/include/common" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +target_link_libraries( + common + PUBLIC os + PUBLIC util ) \ No newline at end of file diff --git a/src/common/src/sqlcmdstr.c b/source/common/src/sqlcmdstr.c similarity index 100% rename from src/common/src/sqlcmdstr.c rename to source/common/src/sqlcmdstr.c diff --git a/src/common/src/tarithoperator.c b/source/common/src/tarithoperator.c similarity index 100% rename from src/common/src/tarithoperator.c rename to source/common/src/tarithoperator.c diff --git a/src/common/src/tdataformat.c b/source/common/src/tdataformat.c similarity index 100% rename from src/common/src/tdataformat.c rename to source/common/src/tdataformat.c diff --git a/src/util/src/terror.c b/source/common/src/terror.c similarity index 100% rename from src/util/src/terror.c rename to source/common/src/terror.c diff --git a/src/common/src/texpr.c b/source/common/src/texpr.c similarity index 100% rename from src/common/src/texpr.c rename to source/common/src/texpr.c diff --git a/src/common/src/tglobal.c b/source/common/src/tglobal.c similarity index 97% rename from src/common/src/tglobal.c rename to source/common/src/tglobal.c index 339fa35bb3..fbf8926fb4 100644 --- a/src/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -18,14 +18,14 @@ #include "taosdef.h" #include "taoserror.h" #include "tulog.h" +#include "tlog.h" #include "tconfig.h" #include "tglobal.h" -#include "monitor.h" #include "tsocket.h" -#include "tutil.h" -#include "tlocale.h" -#include "ttimezone.h" #include "tcompare.h" +#include "tutil.h" +#include "ttimezone.h" +#include "tlocale.h" // cluster char tsFirst[TSDB_EP_LEN] = {0}; @@ -59,9 +59,6 @@ float tsNumOfThreadsPerCore = 1.0f; int32_t tsNumOfCommitThreads = 4; float tsRatioOfQueryCores = 1.0f; int8_t tsDaylight = 0; -char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; -char tsLocale[TSDB_LOCALE_LEN] = {0}; -char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string int8_t tsEnableCoreFile = 0; int32_t tsMaxBinaryDisplayWidth = 30; @@ -201,15 +198,11 @@ int8_t tsEnableStream = 1; int8_t tsCompactMnodeWal = 0; int8_t tsPrintAuth = 0; int8_t tscEmbedded = 0; -char configDir[PATH_MAX] = {0}; char tsVnodeDir[PATH_MAX] = {0}; char tsDnodeDir[PATH_MAX] = {0}; char tsMnodeDir[PATH_MAX] = {0}; char tsMnodeTmpDir[PATH_MAX] = {0}; char tsMnodeBakDir[PATH_MAX] = {0}; -char tsDataDir[PATH_MAX] = {0}; -char tsScriptDir[PATH_MAX] = {0}; -char tsTempDir[PATH_MAX] = "/tmp/"; int32_t tsDiskCfgNum = 0; int32_t tsTopicBianryLen = 16000; @@ -229,11 +222,6 @@ SDiskCfg tsDiskCfg[TSDB_MAX_DISKS]; int64_t tsTickPerDay[] = {86400000L, 86400000000L, 86400000000000L}; // system info -char tsOsName[10] = "Linux"; -int64_t tsPageSize; -int64_t tsOpenMax; -int64_t tsStreamMax; -int32_t tsNumOfCores = 1; float tsTotalTmpDirGB = 0; float tsTotalDataDirGB = 0; float tsAvailTmpDirectorySpace = 0; @@ -244,28 +232,6 @@ float tsMinimalDataDirGB = 2.0f; int32_t tsTotalMemoryMB = 0; uint32_t tsVersion = 0; -// log -int32_t tsNumOfLogLines = 10000000; -int32_t mDebugFlag = 131; -int32_t sdbDebugFlag = 131; -int32_t dDebugFlag = 135; -int32_t vDebugFlag = 135; -uint32_t cDebugFlag = 131; -int32_t jniDebugFlag = 131; -int32_t odbcDebugFlag = 131; -int32_t httpDebugFlag = 131; -int32_t mqttDebugFlag = 131; -int32_t monDebugFlag = 131; -uint32_t qDebugFlag = 131; -int32_t rpcDebugFlag = 131; -int32_t uDebugFlag = 131; -int32_t debugFlag = 0; -int32_t sDebugFlag = 135; -int32_t wDebugFlag = 135; -int32_t tsdbDebugFlag = 131; -int32_t cqDebugFlag = 131; -int32_t fsDebugFlag = 135; - #ifdef TD_TSZ // // lossy compress 6 @@ -420,6 +386,7 @@ static void taosCheckDataDirCfg() { } static int32_t taosCheckTmpDir(void) { +#if 0 if (strlen(tsTempDir) <= 0){ uError("tempDir is not set"); return -1; @@ -432,10 +399,11 @@ static int32_t taosCheckTmpDir(void) { } closedir(dir); - +#endif return 0; } + static void doInitGlobalConfig(void) { osInit(); srand(taosSafeRand()); @@ -1096,7 +1064,7 @@ static void doInitGlobalConfig(void) { cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 0; - cfg.ptrLength = tListLen(tsTimezone); + cfg.ptrLength = TSDB_TIMEZONE_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); @@ -1106,7 +1074,7 @@ static void doInitGlobalConfig(void) { cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 0; - cfg.ptrLength = tListLen(tsLocale); + cfg.ptrLength = TSDB_LOCALE_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); @@ -1116,7 +1084,7 @@ static void doInitGlobalConfig(void) { cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 0; - cfg.ptrLength = tListLen(tsCharset); + cfg.ptrLength = TSDB_LOCALE_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); @@ -1612,7 +1580,7 @@ static void doInitGlobalConfig(void) { cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 0; - cfg.ptrLength = tListLen(tsTempDir); + cfg.ptrLength = PATH_MAX; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); diff --git a/src/common/src/tlocale.c b/source/common/src/tlocale.c similarity index 100% rename from src/common/src/tlocale.c rename to source/common/src/tlocale.c diff --git a/src/common/src/tmessage.c b/source/common/src/tmessage.c similarity index 100% rename from src/common/src/tmessage.c rename to source/common/src/tmessage.c diff --git a/src/common/src/tname.c b/source/common/src/tname.c similarity index 100% rename from src/common/src/tname.c rename to source/common/src/tname.c diff --git a/src/os/src/detail/osTime.c b/source/common/src/ttime.c similarity index 99% rename from src/os/src/detail/osTime.c rename to source/common/src/ttime.c index 5b2a01edc3..0f15dcc13b 100644 --- a/src/os/src/detail/osTime.c +++ b/source/common/src/ttime.c @@ -25,6 +25,7 @@ #include "os.h" #include "taosdef.h" +#include "ttime.h" #include "tutil.h" /* diff --git a/src/common/src/ttimezone.c b/source/common/src/ttimezone.c similarity index 100% rename from src/common/src/ttimezone.c rename to source/common/src/ttimezone.c diff --git a/src/common/src/ttypes.c b/source/common/src/ttypes.c similarity index 100% rename from src/common/src/ttypes.c rename to source/common/src/ttypes.c diff --git a/src/common/src/tvariant.c b/source/common/src/tvariant.c similarity index 100% rename from src/common/src/tvariant.c rename to source/common/src/tvariant.c diff --git a/source/libs/transport/src/rpcCache.c b/source/libs/transport/src/rpcCache.c index 60a12c26b7..7de7aa341b 100644 --- a/source/libs/transport/src/rpcCache.c +++ b/source/libs/transport/src/rpcCache.c @@ -14,6 +14,7 @@ */ #include "os.h" +#include "taosdef.h" #include "tglobal.h" #include "tmempool.h" #include "ttimer.h" diff --git a/source/os/src/os.c b/source/os/src/os.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/os/src/os.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/src/os/src/windows/wAtomic.c b/source/os/src/osAtomic.c similarity index 81% rename from src/os/src/windows/wAtomic.c rename to source/os/src/osAtomic.c index b645893030..6429cdc9d3 100644 --- a/src/os/src/windows/wAtomic.c +++ b/source/os/src/osAtomic.c @@ -13,13 +13,9 @@ * along with this program. If not, see . */ -#define _DEFAULT_SOURCE #include "os.h" -#include "taosdef.h" -#include "tglobal.h" -#include "ttimer.h" -#include "tulog.h" -#include "tutil.h" + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) // add char interlocked_add_fetch_8(char volatile* ptr, char val) { @@ -162,3 +158,42 @@ __int64 interlocked_fetch_xor_64(__int64 volatile* ptr, __int64 val) { #endif } +#endif + +#ifdef _TD_NINGSI_60 +void* atomic_exchange_ptr_impl(void** ptr, void* val ) { + void *old; + do { + old = *ptr; + } while( !__sync_bool_compare_and_swap(ptr, old, val) ); + return old; +} +int8_t atomic_exchange_8_impl(int8_t* ptr, int8_t val ) { + int8_t old; + do { + old = *ptr; + } while( !__sync_bool_compare_and_swap(ptr, old, val) ); + return old; +} +int16_t atomic_exchange_16_impl(int16_t* ptr, int16_t val ) { + int16_t old; + do { + old = *ptr; + } while( !__sync_bool_compare_and_swap(ptr, old, val) ); + return old; +} +int32_t atomic_exchange_32_impl(int32_t* ptr, int32_t val ) { + int32_t old; + do { + old = *ptr; + } while( !__sync_bool_compare_and_swap(ptr, old, val) ); + return old; +} +int64_t atomic_exchange_64_impl(int64_t* ptr, int64_t val ) { + int64_t old; + do { + old = *ptr; + } while( !__sync_bool_compare_and_swap(ptr, old, val) ); + return old; +} +#endif diff --git a/src/os/src/detail/osDir.c b/source/os/src/osDir.c similarity index 52% rename from src/os/src/detail/osDir.c rename to source/os/src/osDir.c index c467c64872..9bca1e6c2e 100644 --- a/src/os/src/detail/osDir.c +++ b/source/os/src/osDir.c @@ -13,16 +13,29 @@ * along with this program. If not, see . */ -#define _DEFAULT_SOURCE #include "os.h" -#include "tglobal.h" -#include "tulog.h" -#include "zlib.h" +#include "osString.h" -#define COMPRESS_STEP_SIZE 163840 +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +/* + * windows implementation + */ -void taosRemoveDir(char *rootDir) { - DIR *dir = opendir(rootDir); +// todo + +#else +/* + * linux implementation + */ + +#include +#include +#include +#include +#include + +void taosRemoveDir(char *dirname) { + DIR *dir = opendir(dirname); if (dir == NULL) return; struct dirent *de = NULL; @@ -30,33 +43,34 @@ void taosRemoveDir(char *rootDir) { if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; char filename[1024]; - snprintf(filename, 1023, "%s/%s", rootDir, de->d_name); + snprintf(filename, sizeof(filename), "%s/%s", dirname, de->d_name); if (de->d_type & DT_DIR) { taosRemoveDir(filename); } else { (void)remove(filename); - uInfo("file:%s is removed", filename); + printf("file:%s is removed", filename); } } closedir(dir); - rmdir(rootDir); + rmdir(dirname); - uInfo("dir:%s is removed", rootDir); + printf("dir:%s is removed", dirname); } -bool taosDirExist(const char* dirname) { - return access(dirname, F_OK) == 0; +bool taosDirExist(char *dirname) { return access(dirname, F_OK) == 0; } + +bool taosMkDir(char *dirname, mode_t mode) { + int32_t code = mkdir(dirname, 0755); + if (code < 0 && errno == EEXIST) { + return true; + } + + return code == 0; } -int taosMkDir(const char *path, mode_t mode) { - int code = mkdir(path, 0755); - if (code < 0 && errno == EEXIST) code = 0; - return code; -} - -void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) { - DIR *dir = opendir(rootDir); +void taosRemoveOldFiles(char *dirname, int32_t keepDays) { + DIR *dir = opendir(dirname); if (dir == NULL) return; int64_t sec = taosGetTimestampSec(); @@ -66,7 +80,7 @@ void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) { if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; char filename[1024]; - snprintf(filename, 1023, "%s/%s", rootDir, de->d_name); + snprintf(filename, sizeof(filename), "%s/%s", dirname, de->d_name); if (de->d_type & DT_DIR) { continue; } else { @@ -76,7 +90,7 @@ void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) { } int64_t fileSec = 0; - for (int i = len - 1; i >= 0; i--) { + for (int32_t i = len - 1; i >= 0; i--) { if (filename[i] == '.') { fileSec = atoll(filename + i + 1); break; @@ -87,56 +101,41 @@ void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) { int32_t days = (int32_t)(ABS(sec - fileSec) / 86400 + 1); if (days > keepDays) { (void)remove(filename); - uInfo("file:%s is removed, days:%d keepDays:%d", filename, days, keepDays); + printf("file:%s is removed, days:%d keepDays:%d", filename, days, keepDays); } else { - uTrace("file:%s won't be removed, days:%d keepDays:%d", filename, days, keepDays); + printf("file:%s won't be removed, days:%d keepDays:%d", filename, days, keepDays); } } } closedir(dir); - rmdir(rootDir); + rmdir(dirname); } -int32_t taosCompressFile(char *srcFileName, char *destFileName) { - int32_t ret = 0; - int32_t len = 0; - char * data = malloc(COMPRESS_STEP_SIZE); - FILE * srcFp = NULL; - gzFile dstFp = NULL; - - srcFp = fopen(srcFileName, "r"); - if (srcFp == NULL) { - ret = -1; - goto cmp_end; +bool taosExpandDir(char *dirname, char *outname, int32_t maxlen) { + wordexp_t full_path; + if (0 != wordexp(dirname, &full_path, 0)) { + printf("failed to expand path:%s since %s", dirname, strerror(errno)); + wordfree(&full_path); + return false; } - int32_t fd = open(destFileName, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO); - if (fd < 0) { - ret = -2; - goto cmp_end; + if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) { + strncpy(outname, full_path.we_wordv[0], maxlen); } - dstFp = gzdopen(fd, "wb6f"); - if (dstFp == NULL) { - ret = -3; - close(fd); - goto cmp_end; - } + wordfree(&full_path); - while (!feof(srcFp)) { - len = (int32_t)fread(data, 1, COMPRESS_STEP_SIZE, srcFp); - (void)gzwrite(dstFp, data, len); - } - -cmp_end: - if (srcFp) { - fclose(srcFp); - } - if (dstFp) { - gzclose(dstFp); - } - free(data); - - return ret; + return true; } + +bool taosRealPath(char *dirname, int32_t maxlen) { + char tmp[PATH_MAX] = {0}; + if (realpath(dirname, tmp) != NULL) { + strncpy(dirname, tmp, maxlen); + } + + return true; +} + +#endif \ No newline at end of file diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c new file mode 100644 index 0000000000..7bc5e0a7fc --- /dev/null +++ b/source/os/src/osEnv.c @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + +#define _DEFAULT_SOURCE +#include "os.h" + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + +char tsOsName[10] = "Windows"; +char configDir[PATH_MAX] = "C:/TDengine/cfg"; +char tsDataDir[PATH_MAX] = "C:/TDengine/data"; +char tsLogDir[PATH_MAX] = "C:/TDengine/log"; +char tsScriptDir[PATH_MAX] = "C:/TDengine/script"; +char tsTempDir[PATH_MAX] = "C:\\Windows\\Temp"; + +extern taosWinSocketInit(); + +void osInit() { + taosWinSocketInit(); + + const char *tmpDir = getenv("tmp"); + if (tmpDir == NULL) { + tmpDir = getenv("temp"); + } + + if (tmpDir != NULL) { + strcpy(tsTempDir, tmpDir); + } +} + +#elif defined(_TD_DARWIN_64) + +char tsOsName[10] = "Darwin"; +char configDir[PATH_MAX] = "/usr/local/etc/taos"; +char tsDataDir[PATH_MAX] = "/usr/local/var/lib/taos"; +char tsLogDir[PATH_MAX] = "/usr/local/var/log/taos"; +char tsScriptDir[PATH_MAX] = "/usr/local/etc/taos"; +char tsTempDir[PATH_MAX] = "/tmp/taosd"; + +void osInit() {} + +#else + +char tsOsName[10] = "Linux"; +char configDir[PATH_MAX] = "/etc/taos"; +char tsDataDir[PATH_MAX] = "/var/lib/taos"; +char tsLogDir[PATH_MAX] = "/var/log/taos"; +char tsScriptDir[PATH_MAX] = "/etc/taos"; +char tsTempDir[PATH_MAX] = "/tmp/"; + +void osInit() {} + +#endif diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 5acbfd6fa7..3726830635 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -402,7 +402,6 @@ int32_t taosRenameFile(char *oldName, char *newName) { #endif } - int32_t taosLockFile(int32_t fd) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) return 0; diff --git a/src/os/src/windows/wLz4.c b/source/os/src/osLz4.c similarity index 94% rename from src/os/src/windows/wLz4.c rename to source/os/src/osLz4.c index 631a22e572..29d219f37b 100644 --- a/src/os/src/windows/wLz4.c +++ b/source/os/src/osLz4.c @@ -15,11 +15,12 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taosdef.h" -#include "tglobal.h" -#include "ttimer.h" -#include "tulog.h" -#include "tutil.h" + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + +/* + * windows implementation + */ unsigned char _MyBitScanForward64(unsigned long *ret, uint64_t x) { unsigned long x0 = (unsigned long)x, top, bottom; @@ -67,4 +68,6 @@ int32_t BUILDIN_CTZ(uint32_t val) { unsigned long r = 0; _BitScanForward(&r, val); return (int)(r >> 3); -} \ No newline at end of file +} + +#endif diff --git a/src/os/src/detail/osRand.c b/source/os/src/osRand.c similarity index 93% rename from src/os/src/detail/osRand.c rename to source/os/src/osRand.c index 0dda908bb3..247097f9ce 100644 --- a/src/os/src/detail/osRand.c +++ b/source/os/src/osRand.c @@ -15,6 +15,10 @@ #define _DEFAULT_SOURCE #include "os.h" +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#else +#include +#endif uint32_t taosRand(void) { return rand(); } diff --git a/src/os/src/darwin/dwSemaphore.c b/source/os/src/osSemaphore.c similarity index 67% rename from src/os/src/darwin/dwSemaphore.c rename to source/os/src/osSemaphore.c index 25cb28cff1..c9533c8b8a 100644 --- a/src/os/src/darwin/dwSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -13,13 +13,66 @@ * along with this program. If not, see . */ -// fail-fast or let-it-crash philosophy -// https://en.wikipedia.org/wiki/Fail-fast -// https://stackoverflow.com/questions/4393197/erlangs-let-it-crash-philosophy-applicable-elsewhere -// experimentally, we follow log-and-crash here - #define _DEFAULT_SOURCE #include "os.h" + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + +/* + * windows implementation + */ + +#include + +bool taosCheckPthreadValid(pthread_t thread) { return thread.p != NULL; } + +void taosResetPthread(pthread_t* thread) { thread->p = 0; } + +int64_t taosGetPthreadId(pthread_t thread) { +#ifdef PTW32_VERSION + return pthread_getw32threadid_np(thread); +#else + return (int64_t)thread; +#endif +} + +int64_t taosGetSelfPthreadId() { return GetCurrentThreadId(); } + +bool taosComparePthread(pthread_t first, pthread_t second) { return first.p == second.p; } + +int32_t taosGetPId() { return GetCurrentProcessId(); } + +int32_t taosGetCurrentAPPName(char* name, int32_t* len) { + char filepath[1024] = {0}; + + GetModuleFileName(NULL, filepath, MAX_PATH); + char* sub = strrchr(filepath, '.'); + if (sub != NULL) { + *sub = '\0'; + } + strcpy(name, filepath); + + if (len != NULL) { + *len = (int32_t)strlen(filepath); + } + + return 0; +} + +int32_t tsem_wait(tsem_t* sem) { + int ret = 0; + do { + ret = sem_wait(sem); + } while (ret != 0 && errno == EINTR); + return ret; +} + +#elif defined(_TD_DARWIN_64) + +/* + * darwin implementation + */ + #include // #define SEM_USE_PTHREAD @@ -27,19 +80,19 @@ #define SEM_USE_SEM #ifdef SEM_USE_SEM -#include #include +#include #include #include #include -static pthread_t sem_thread; -static pthread_once_t sem_once; -static task_t sem_port; -static volatile int sem_inited = 0; -static semaphore_t sem_exit; +static pthread_t sem_thread; +static pthread_once_t sem_once; +static task_t sem_port; +static volatile int sem_inited = 0; +static semaphore_t sem_exit; -static void* sem_thread_routine(void *arg) { +static void *sem_thread_routine(void *arg) { (void)arg; setThreadName("sem_thrd"); @@ -62,7 +115,7 @@ static void once_init(void) { fprintf(stderr, "==%s[%d]%s()==failed to create thread\n", basename(__FILE__), __LINE__, __func__); return; } - while (sem_inited==0) { + while (sem_inited == 0) { ; } } @@ -70,19 +123,19 @@ static void once_init(void) { struct tsem_s { #ifdef SEM_USE_PTHREAD - pthread_mutex_t lock; - pthread_cond_t cond; - volatile int64_t val; + pthread_mutex_t lock; + pthread_cond_t cond; + volatile int64_t val; #elif defined(SEM_USE_POSIX) - size_t id; - sem_t *sem; + size_t id; + sem_t * sem; #elif defined(SEM_USE_SEM) - semaphore_t sem; -#else // SEM_USE_PTHREAD - dispatch_semaphore_t sem; -#endif // SEM_USE_PTHREAD + semaphore_t sem; +#else // SEM_USE_PTHREAD + dispatch_semaphore_t sem; +#endif // SEM_USE_PTHREAD - volatile unsigned int valid:1; + volatile unsigned int valid : 1; }; int tsem_init(tsem_t *sem, int pshared, unsigned int value) { @@ -91,7 +144,7 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", basename(__FILE__), __LINE__, __func__, sem); abort(); } - struct tsem_s *p = (struct tsem_s*)calloc(1, sizeof(*p)); + struct tsem_s *p = (struct tsem_s *)calloc(1, sizeof(*p)); if (!p) { fprintf(stderr, "==%s[%d]%s():[%p]==out of memory\n", basename(__FILE__), __LINE__, __func__, sem); abort(); @@ -116,24 +169,25 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { static size_t tick = 0; do { size_t id = atomic_add_fetch_64(&tick, 1); - if (id==SEM_VALUE_MAX) { + if (id == SEM_VALUE_MAX) { atomic_store_64(&tick, 0); id = 0; } - char name[NAME_MAX-4]; + char name[NAME_MAX - 4]; snprintf(name, sizeof(name), "/t%ld", id); - p->sem = sem_open(name, O_CREAT|O_EXCL, pshared, value); - p->id = id; - if (p->sem!=SEM_FAILED) break; + p->sem = sem_open(name, O_CREAT | O_EXCL, pshared, value); + p->id = id; + if (p->sem != SEM_FAILED) break; int e = errno; - if (e==EEXIST) continue; - if (e==EINTR) continue; - fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", basename(__FILE__), __LINE__, __func__, sem, e, strerror(e)); + if (e == EEXIST) continue; + if (e == EINTR) continue; + fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", basename(__FILE__), __LINE__, __func__, sem, e, + strerror(e)); abort(); - } while (p->sem==SEM_FAILED); + } while (p->sem == SEM_FAILED); #elif defined(SEM_USE_SEM) pthread_once(&sem_once, once_init); - if (sem_inited!=1) { + if (sem_inited != 1) { fprintf(stderr, "==%s[%d]%s():[%p]==internal resource init failed\n", basename(__FILE__), __LINE__, __func__, sem); errno = ENOMEM; return -1; @@ -144,13 +198,13 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { // we fail-fast here, because we have less-doc about semaphore_create for the moment abort(); } -#else // SEM_USE_PTHREAD +#else // SEM_USE_PTHREAD p->sem = dispatch_semaphore_create(value); if (p->sem == NULL) { fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", basename(__FILE__), __LINE__, __func__, sem); abort(); } -#endif // SEM_USE_PTHREAD +#endif // SEM_USE_PTHREAD p->valid = 1; @@ -190,9 +244,9 @@ int tsem_wait(tsem_t *sem) { return sem_wait(p->sem); #elif defined(SEM_USE_SEM) return semaphore_wait(p->sem); -#else // SEM_USE_PTHREAD +#else // SEM_USE_PTHREAD return dispatch_semaphore_wait(p->sem, DISPATCH_TIME_FOREVER); -#endif // SEM_USE_PTHREAD +#endif // SEM_USE_PTHREAD } int tsem_post(tsem_t *sem) { @@ -226,9 +280,9 @@ int tsem_post(tsem_t *sem) { return sem_post(p->sem); #elif defined(SEM_USE_SEM) return semaphore_signal(p->sem); -#else // SEM_USE_PTHREAD +#else // SEM_USE_PTHREAD return dispatch_semaphore_signal(p->sem); -#endif // SEM_USE_PTHREAD +#endif // SEM_USE_PTHREAD } int tsem_destroy(tsem_t *sem) { @@ -263,18 +317,19 @@ int tsem_destroy(tsem_t *sem) { abort(); } #elif defined(SEM_USE_POSIX) - char name[NAME_MAX-4]; + char name[NAME_MAX - 4]; snprintf(name, sizeof(name), "/t%ld", p->id); int r = sem_unlink(name); if (r) { int e = errno; - fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", basename(__FILE__), __LINE__, __func__, sem, e, strerror(e)); + fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", basename(__FILE__), __LINE__, __func__, sem, e, + strerror(e)); abort(); } #elif defined(SEM_USE_SEM) semaphore_destroy(sem_port, p->sem); -#else // SEM_USE_PTHREAD -#endif // SEM_USE_PTHREAD +#else // SEM_USE_PTHREAD +#endif // SEM_USE_PTHREAD p->valid = 0; free(p); @@ -285,36 +340,28 @@ int tsem_destroy(tsem_t *sem) { bool taosCheckPthreadValid(pthread_t thread) { uint64_t id = 0; - int r = pthread_threadid_np(thread, &id); + int r = pthread_threadid_np(thread, &id); return r ? false : true; } int64_t taosGetSelfPthreadId() { uint64_t id; pthread_threadid_np(0, &id); - return (int64_t) id; + return (int64_t)id; } -int64_t taosGetPthreadId(pthread_t thread) { - return (int64_t)thread; -} +int64_t taosGetPthreadId(pthread_t thread) { return (int64_t)thread; } -void taosResetPthread(pthread_t* thread) { - *thread = NULL; -} +void taosResetPthread(pthread_t *thread) { *thread = NULL; } -bool taosComparePthread(pthread_t first, pthread_t second) { - return pthread_equal(first, second) ? true : false; -} +bool taosComparePthread(pthread_t first, pthread_t second) { return pthread_equal(first, second) ? true : false; } -int32_t taosGetPId() { - return (int32_t)getpid(); -} +int32_t taosGetPId() { return (int32_t)getpid(); } -int32_t taosGetCurrentAPPName(char* name, int32_t* len) { - char buf[PATH_MAX+1]; +int32_t taosGetCurrentAPPName(char *name, int32_t *len) { + char buf[PATH_MAX + 1]; buf[0] = '\0'; - proc_name(getpid(), buf, sizeof(buf)-1); + proc_name(getpid(), buf, sizeof(buf) - 1); buf[PATH_MAX] = '\0'; size_t n = strlen(buf); if (len) *len = n; @@ -322,4 +369,59 @@ int32_t taosGetCurrentAPPName(char* name, int32_t* len) { return 0; } +#else +/* + * linux implementation + */ + +#include + +bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; } + +int64_t taosGetSelfPthreadId() { + static __thread int id = 0; + if (id != 0) return id; + id = syscall(SYS_gettid); + return id; +} + +int64_t taosGetPthreadId(pthread_t thread) { return (int64_t)thread; } +void taosResetPthread(pthread_t* thread) { *thread = 0; } +bool taosComparePthread(pthread_t first, pthread_t second) { return first == second; } +int32_t taosGetPId() { return getpid(); } + +int32_t taosGetCurrentAPPName(char* name, int32_t* len) { + const char* self = "/proc/self/exe"; + char path[PATH_MAX] = {0}; + + if (readlink(self, path, PATH_MAX) <= 0) { + return -1; + } + + path[PATH_MAX - 1] = 0; + char* end = strrchr(path, '/'); + if (end == NULL) { + return -1; + } + + ++end; + + strcpy(name, end); + + if (len != NULL) { + *len = strlen(name); + } + + return 0; +} + +int32_t tsem_wait(tsem_t* sem) { + int ret = 0; + do { + ret = sem_wait(sem); + } while (ret != 0 && errno == EINTR); + return ret; +} + +#endif diff --git a/src/os/src/windows/wSignal.c b/source/os/src/osSignal.c similarity index 64% rename from src/os/src/windows/wSignal.c rename to source/os/src/osSignal.c index 9de6b5f343..12721a17f5 100644 --- a/src/os/src/windows/wSignal.c +++ b/source/os/src/osSignal.c @@ -15,7 +15,13 @@ #define _DEFAULT_SOURCE #include "os.h" -#include + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + +/* + * windows implementation + */ + #include typedef void (*FWinSignalHandler)(int32_t signum); @@ -27,7 +33,7 @@ void taosSetSignal(int32_t signum, FSignalHandler sigfp) { if (signum == SIGHUP) { SetConsoleCtrlHandler((PHANDLER_ROUTINE)sigfp, TRUE); } else { - signal(signum, (FWinSignalHandler)sigfp); + signal(signum, (FWinSignalHandler)sigfp); } } @@ -40,3 +46,29 @@ void taosDflSignal(int32_t signum) { if (signum == SIGUSR1 || signum == SIGHUP) return; signal(signum, SIG_DFL); } + +#else + +/* + * linux and darwin implementation + */ + +typedef void (*FLinuxSignalHandler)(int32_t signum, siginfo_t *sigInfo, void *context); + +void taosSetSignal(int32_t signum, FSignalHandler sigfp) { + struct sigaction act; + memset(&act, 0, sizeof(act)); +#if 1 + act.sa_flags = SA_SIGINFO; + act.sa_sigaction = (FLinuxSignalHandler)sigfp; +#else + act.sa_handler = sigfp; +#endif + sigaction(signum, &act, NULL); +} + +void taosIgnSignal(int32_t signum) { signal(signum, SIG_IGN); } + +void taosDflSignal(int32_t signum) { signal(signum, SIG_DFL); } + +#endif diff --git a/src/os/src/detail/osSleep.c b/source/os/src/osSleep.c similarity index 100% rename from src/os/src/detail/osSleep.c rename to source/os/src/osSleep.c diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 7964482db6..d2f3c120db 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -16,6 +16,19 @@ #define _DEFAULT_SOURCE #include "os.h" +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#else +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + void taosShutDownSocketRD(SOCKET fd) { #ifdef WINDOWS closesocket(fd); @@ -41,7 +54,7 @@ void taosShutDownSocketWR(SOCKET fd) { int32_t taosSetNonblocking(SOCKET sock, int32_t on) { int32_t flags = 0; if ((flags = fcntl(sock, F_GETFL, 0)) < 0) { - uError("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno)); + printf("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno)); return 1; } @@ -51,16 +64,14 @@ int32_t taosSetNonblocking(SOCKET sock, int32_t on) { flags &= ~O_NONBLOCK; if ((flags = fcntl(sock, F_SETFL, flags)) < 0) { - uError("fcntl(F_SETFL) error: %d (%s)\n", errno, strerror(errno)); + printf("fcntl(F_SETFL) error: %d (%s)\n", errno, strerror(errno)); return 1; } return 0; } -void taosIgnSIGPIPE() { - signal(SIGPIPE, SIG_IGN); -} +void taosIgnSIGPIPE() { signal(SIGPIPE, SIG_IGN); } void taosBlockSIGPIPE() { sigset_t signal_mask; @@ -68,7 +79,7 @@ void taosBlockSIGPIPE() { sigaddset(&signal_mask, SIGPIPE); int32_t rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL); if (rc != 0) { - uError("failed to block SIGPIPE"); + printf("failed to block SIGPIPE"); } } @@ -78,7 +89,7 @@ void taosSetMaskSIGPIPE() { sigaddset(&signal_mask, SIGPIPE); int32_t rc = pthread_sigmask(SIG_SETMASK, &signal_mask, NULL); if (rc != 0) { - uError("failed to setmask SIGPIPE"); + printf("failed to setmask SIGPIPE"); } } @@ -90,35 +101,127 @@ int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *op return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); } -int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t* optlen) { +int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen) { return getsockopt(socketfd, level, optname, optval, (socklen_t *)optlen); -} +} #endif -#if !( (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) && defined(_MSC_VER) ) +#if !((defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) && defined(_MSC_VER)) + +uint32_t taosInetAddr(char *ipAddr) { return inet_addr(ipAddr); } + +const char *taosInetNtoa(struct in_addr ipInt) { return inet_ntoa(ipInt); } + +#endif + +#if defined(_TD_DARWIN_64) + +/* + * darwin implementation + */ + +int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen) { + if (level == SOL_SOCKET && optname == SO_SNDBUF) { + return 0; + } + + if (level == SOL_SOCKET && optname == SO_RCVBUF) { + return 0; + } + + return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); +} +#endif + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + +/* + * windows implementation + */ + +#include +#include +#include +#include +#include +#include +#include + +void taosWinSocketInit() { + static char flag = 0; + if (flag == 0) { + WORD wVersionRequested; + WSADATA wsaData; + wVersionRequested = MAKEWORD(1, 1); + if (WSAStartup(wVersionRequested, &wsaData) == 0) { + flag = 1; + } + } +} + +int32_t taosSetNonblocking(SOCKET sock, int32_t on) { + u_long mode; + if (on) { + mode = 1; + ioctlsocket(sock, FIONBIO, &mode); + } else { + mode = 0; + ioctlsocket(sock, FIONBIO, &mode); + } + return 0; +} + +void taosIgnSIGPIPE() {} +void taosBlockSIGPIPE() {} +void taosSetMaskSIGPIPE() {} + +int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { + if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { + return 0; + } + + if (level == SOL_TCP && optname == TCP_KEEPIDLE) { + return 0; + } + + if (level == SOL_TCP && optname == TCP_KEEPINTVL) { + return 0; + } + + if (level == SOL_TCP && optname == TCP_KEEPCNT) { + return 0; + } + + return setsockopt(socketfd, level, optname, optval, optlen); +} + +#ifdef _MSC_VER +//#if _MSC_VER >= 1900 uint32_t taosInetAddr(char *ipAddr) { - return inet_addr(ipAddr); + uint32_t value; + int32_t ret = inet_pton(AF_INET, ipAddr, &value); + if (ret <= 0) { + return INADDR_NONE; + } else { + return value; + } } -const char *taosInetNtoa(struct in_addr ipInt) { - return inet_ntoa(ipInt); -} - -#else - const char *taosInetNtoa(struct in_addr ipInt) { // not thread safe, only for debug usage while print log static char tmpDstStr[16]; return inet_ntop(AF_INET, &ipInt, tmpDstStr, INET6_ADDRSTRLEN); } +//#endif #endif - #if defined(_TD_GO_DLL_) uint64_t htonll(uint64_t val) { return (((uint64_t)htonl(val)) << 32) + htonl(val >> 32); } +#endif + #endif \ No newline at end of file diff --git a/source/os/src/osString.c b/source/os/src/osString.c new file mode 100644 index 0000000000..10606a3d7b --- /dev/null +++ b/source/os/src/osString.c @@ -0,0 +1,382 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include +#include + +typedef struct CharsetPair { + char *oldCharset; + char *newCharset; +} CharsetPair; + +char *taosCharsetReplace(char *charsetstr) { + CharsetPair charsetRep[] = { + { "utf8", "UTF-8" }, { "936", "CP936" }, + }; + + for (int32_t i = 0; i < tListLen(charsetRep); ++i) { + if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { + return strdup(charsetRep[i].newCharset); + } + } + + return strdup(charsetstr); +} + +int64_t taosStr2int64(char *str) { + char *endptr = NULL; + return strtoll(str, &endptr, 10); +} + +#ifdef USE_LIBICONV +#include "iconv.h" + +int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { + iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); + size_t ucs4_input_len = ucs4_max_len; + size_t outLen = ucs4_max_len; + if (iconv(cd, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) { + iconv_close(cd); + return -1; + } + + iconv_close(cd); + return (int32_t)(ucs4_max_len - outLen); +} + +bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) { + memset(ucs4, 0, ucs4_max_len); + iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); + size_t ucs4_input_len = mbsLength; + size_t outLeft = ucs4_max_len; + if (iconv(cd, &mbs, &ucs4_input_len, &ucs4, &outLeft) == -1) { + iconv_close(cd); + return false; + } + + iconv_close(cd); + if (len != NULL) { + *len = (int32_t)(ucs4_max_len - outLeft); + if (*len < 0) { + return false; + } + } + + return true; +} + +bool taosValidateEncodec(const char *encodec) { + iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC); + if (cd == (iconv_t)(-1)) { + return false; + } + + iconv_close(cd); + return true; +} + +#else + +int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { + mbstate_t state = {0}; + int32_t len = (int32_t)wcsnrtombs(NULL, (const wchar_t **)&ucs4, ucs4_max_len / 4, 0, &state); + if (len < 0) { + return -1; + } + + memset(&state, 0, sizeof(state)); + len = wcsnrtombs(mbs, (const wchar_t **)&ucs4, ucs4_max_len / 4, (size_t)len, &state); + if (len < 0) { + return -1; + } + + return len; +} + +bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) { + memset(ucs4, 0, ucs4_max_len); + mbstate_t state = {0}; + int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state); + *len = retlen; + + return retlen >= 0; +} + +bool taosValidateEncodec(const char *encodec) { + return true; +} + +#endif + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + +/* + * windows implementation + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#if STDC_HEADERS +#include +#else +char *malloc(), *realloc(); +#endif + +/* Always add at least this many bytes when extending the buffer. */ +#define MIN_CHUNK 64 + +/* Read up to (and including) a TERMINATOR from STREAM into *LINEPTR ++ OFFSET (and null-terminate it). *LINEPTR is a pointer returned from +malloc (or NULL), pointing to *N characters of space. It is realloc'd +as necessary. Return the number of characters read (not including the +null terminator), or -1 on error or EOF. On a -1 return, the caller +should check feof(), if not then errno has been set to indicate +the error. */ + +int32_t getstr(char **lineptr, size_t *n, FILE *stream, char terminator, int32_t offset) { + int32_t nchars_avail; /* Allocated but unused chars in *LINEPTR. */ + char * read_pos; /* Where we're reading into *LINEPTR. */ + int32_t ret; + + if (!lineptr || !n || !stream) { + errno = EINVAL; + return -1; + } + + if (!*lineptr) { + *n = MIN_CHUNK; + *lineptr = malloc(*n); + if (!*lineptr) { + errno = ENOMEM; + return -1; + } + } + + nchars_avail = (int32_t)(*n - offset); + read_pos = *lineptr + offset; + + for (;;) { + int32_t save_errno; + register int32_t c = getc(stream); + + save_errno = errno; + + /* We always want at least one char left in the buffer, since we + always (unless we get an error while reading the first char) + NUL-terminate the line buffer. */ + + assert((*lineptr + *n) == (read_pos + nchars_avail)); + if (nchars_avail < 2) { + if (*n > MIN_CHUNK) + *n *= 2; + else + *n += MIN_CHUNK; + + nchars_avail = (int32_t)(*n + *lineptr - read_pos); + char* lineptr1 = realloc(*lineptr, *n); + if (!lineptr1) { + errno = ENOMEM; + return -1; + } + *lineptr = lineptr1; + + read_pos = *n - nchars_avail + *lineptr; + assert((*lineptr + *n) == (read_pos + nchars_avail)); + } + + if (ferror(stream)) { + /* Might like to return partial line, but there is no + place for us to store errno. And we don't want to just + lose errno. */ + errno = save_errno; + return -1; + } + + if (c == EOF) { + /* Return partial line, if any. */ + if (read_pos == *lineptr) + return -1; + else + break; + } + + *read_pos++ = c; + nchars_avail--; + + if (c == terminator) /* Return the line. */ + break; + } + + /* Done - NUL terminate and return the number of chars read. */ + *read_pos = '\0'; + + ret = (int32_t)(read_pos - (*lineptr + offset)); + return ret; +} + +int32_t tgetline(char **lineptr, size_t *n, FILE *stream) { return getstr(lineptr, n, stream, '\n', 0); } + + +/* + * Get next token from string *stringp, where tokens are possibly-empty + * strings separated by characters from delim. + * + * Writes NULs into the string at *stringp to end tokens. + * delim need not remain constant from call to call. + * On return, *stringp points past the last NUL written (if there might + * be further tokens), or is NULL (if there are definitely no moretokens). + * + * If *stringp is NULL, strsep returns NULL. + */ +char *strsep(char **stringp, const char *delim) { + char * s; + const char *spanp; + int32_t c, sc; + char *tok; + if ((s = *stringp) == NULL) + return (NULL); + for (tok = s;;) { + c = *s++; + spanp = delim; + do { + if ((sc = *spanp++) == c) { + if (c == 0) + s = NULL; + else + s[-1] = 0; + *stringp = s; + return (tok); + } + } while (sc != 0); + } + /* NOTREACHED */ +} + +char *getpass(const char *prefix) { + static char passwd[TSDB_KEY_LEN] = {0}; + memset(passwd, 0, TSDB_KEY_LEN); + printf("%s", prefix); + + int32_t index = 0; + char ch; + while (index < TSDB_KEY_LEN) { + ch = getch(); + if (ch == '\n' || ch == '\r') { + break; + } else { + passwd[index++] = ch; + } + } + + return passwd; +} + +int32_t twcslen(const wchar_t *wcs) { + int32_t *wstr = (int32_t *)wcs; + if (NULL == wstr) { + return 0; + } + + int32_t n = 0; + while (1) { + if (0 == *wstr++) { + break; + } + n++; + } + + return n; +} + +int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes) { + for (int32_t i = 0; i < bytes; ++i) { + int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i * 4); + int32_t f2 = *(int32_t *)((char *)f2_ucs4 + i * 4); + + if ((f1 == 0 && f2 != 0) || (f1 != 0 && f2 == 0)) { + return f1 - f2; + } else if (f1 == 0 && f2 == 0) { + return 0; + } + + if (f1 != f2) { + return f1 - f2; + } + } + + return 0; + +#if 0 + int32_t ucs4_max_len = bytes + 4; + char *f1_mbs = calloc(bytes, 1); + char *f2_mbs = calloc(bytes, 1); + if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) { + return -1; + } + if (taosUcs4ToMbs(f2_ucs4, ucs4_max_len, f2_mbs) < 0) { + return -1; + } + int32_t ret = strcmp(f1_mbs, f2_mbs); + free(f1_mbs); + free(f2_mbs); + return ret; +#endif +} + +/* Copy memory to memory until the specified number of bytes +has been copied, return pointer to following byte. +Overlap is NOT handled correctly. */ +void *mempcpy(void *dest, const void *src, size_t len) { + return (char*)memcpy(dest, src, len) + len; +} + +/* Copy SRC to DEST, returning the address of the terminating '\0' in DEST. */ +char *stpcpy (char *dest, const char *src) { + size_t len = strlen (src); + return (char*)memcpy(dest, src, len + 1) + len; +} + +/* Copy no more than N characters of SRC to DEST, returning the address of + the terminating '\0' in DEST, if any, or else DEST + N. */ +char *stpncpy (char *dest, const char *src, size_t n) { + size_t size = strnlen (src, n); + memcpy (dest, src, size); + dest += size; + if (size == n) + return dest; + return memset (dest, '\0', n - size); +} + +#else + +/* + * linux and darwin implementation + */ + +int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize) { + return wcsncmp((wchar_t *)f1_ucs4, (wchar_t *)f2_ucs4, bytes / ncharSize); +} + +#endif \ No newline at end of file diff --git a/src/os/src/windows/wStrptime.c b/source/os/src/osStrptime.c similarity index 99% rename from src/os/src/windows/wStrptime.c rename to source/os/src/osStrptime.c index 2401818274..e8c873bc49 100644 --- a/src/os/src/windows/wStrptime.c +++ b/source/os/src/osStrptime.c @@ -37,6 +37,9 @@ */ // //#include "lukemftp.h" + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + #include #include #include @@ -395,4 +398,6 @@ conv_num(const char **buf, int *dest, int llim, int ulim) *dest = result; return (1); -} \ No newline at end of file +} + +#endif \ No newline at end of file diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c new file mode 100644 index 0000000000..54af2ab4ad --- /dev/null +++ b/source/os/src/osSysinfo.c @@ -0,0 +1,1130 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +int32_t tsTotalMemoryMB = 0; +int64_t tsPageSize = 0; +int64_t tsOpenMax = 0; +int64_t tsStreamMax = 0; +int32_t tsNumOfCores = 1; +char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; +char tsLocale[TSDB_LOCALE_LEN] = {0}; +char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + +/* + * windows implementation + */ + +#if (_WIN64) +#include +#include +#include +#include +#include +#include +#pragma comment(lib, "Mswsock.lib ") +#endif + +#include + +#pragma warning(push) +#pragma warning(disable : 4091) +#include +#pragma warning(pop) + +static int32_t taosGetTotalMemory() { + MEMORYSTATUSEX memsStat; + memsStat.dwLength = sizeof(memsStat); + if (!GlobalMemoryStatusEx(&memsStat)) { + return 0; + } + + float nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f); + return (int32_t)nMemTotal; +} + +bool taosGetSysMemory(float *memoryUsedMB) { + MEMORYSTATUSEX memsStat; + memsStat.dwLength = sizeof(memsStat); + if (!GlobalMemoryStatusEx(&memsStat)) { + return false; + } + + float nMemFree = memsStat.ullAvailPhys / (1024.0f * 1024.0f); + float nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f); + + *memoryUsedMB = nMemTotal - nMemFree; + return true; +} + +bool taosGetProcMemory(float *memoryUsedMB) { + unsigned bytes_used = 0; + +#if defined(_WIN64) && defined(_MSC_VER) + PROCESS_MEMORY_COUNTERS pmc; + HANDLE cur_proc = GetCurrentProcess(); + + if (GetProcessMemoryInfo(cur_proc, &pmc, sizeof(pmc))) { + bytes_used = (unsigned)(pmc.WorkingSetSize + pmc.PagefileUsage); + } +#endif + + *memoryUsedMB = (float)bytes_used / 1024 / 1024; + return true; +} + +static void taosGetSystemTimezone() { + // get and set default timezone + char *tz = getenv("TZ"); + if (tz == NULL || strlen(tz) == 0) { + strcpy(tsTimezone, "not configured"); + } else { + strcpy(tsTimezone, tz); + } +} + +static void taosGetSystemLocale() { + // get and set default locale + char *locale = setlocale(LC_CTYPE, "chs"); + if (locale != NULL) { + tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); + } + + strcpy(tsCharset, "cp936"); +} + +int32_t taosGetCpuCores() { + SYSTEM_INFO info; + GetSystemInfo(&info); + return (int32_t)info.dwNumberOfProcessors; +} + +bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { + *sysCpuUsage = 0; + *procCpuUsage = 0; + return true; +} + +int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) { + unsigned _int64 i64FreeBytesToCaller; + unsigned _int64 i64TotalBytes; + unsigned _int64 i64FreeBytes; + + BOOL fResult = GetDiskFreeSpaceExA(dataDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes, + (PULARGE_INTEGER)&i64FreeBytes); + if (fResult) { + diskSize->tsize = (int64_t)(i64TotalBytes); + diskSize->avail = (int64_t)(i64FreeBytesToCaller); + diskSize->used = (int64_t)(i64TotalBytes - i64FreeBytes); + return 0; + } else { + printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } +} + +bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { + if (bytes) *bytes = 0; + if (rbytes) *rbytes = 0; + if (tbytes) *tbytes = 0; + return true; +} + +bool taosGetBandSpeed(float *bandSpeedKb) { + *bandSpeedKb = 0; + return true; +} + +bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) { + IO_COUNTERS io_counter; + if (GetProcessIoCounters(GetCurrentProcess(), &io_counter)) { + if (readbyte) *readbyte = io_counter.ReadTransferCount; + if (writebyte) *writebyte = io_counter.WriteTransferCount; + return true; + } + return false; +} + +bool taosGetProcIO(float *readKB, float *writeKB) { + static int64_t lastReadbyte = -1; + static int64_t lastWritebyte = -1; + + int64_t curReadbyte = 0; + int64_t curWritebyte = 0; + + if (!taosReadProcIO(&curReadbyte, &curWritebyte)) { + return false; + } + + if (lastReadbyte == -1 || lastWritebyte == -1) { + lastReadbyte = curReadbyte; + lastWritebyte = curWritebyte; + return false; + } + + *readKB = (float)((double)(curReadbyte - lastReadbyte) / 1024); + *writeKB = (float)((double)(curWritebyte - lastWritebyte) / 1024); + if (*readKB < 0) *readKB = 0; + if (*writeKB < 0) *writeKB = 0; + + lastReadbyte = curReadbyte; + lastWritebyte = curWritebyte; + + return true; +} + +void taosGetSystemInfo() { + tsNumOfCores = taosGetCpuCores(); + tsTotalMemoryMB = taosGetTotalMemory(); + + float tmp1, tmp2; + // taosGetDisk(); + taosGetBandSpeed(&tmp1); + taosGetCpuUsage(&tmp1, &tmp2); + taosGetProcIO(&tmp1, &tmp2); + + taosGetSystemTimezone(); + taosGetSystemLocale(); +} + +void taosKillSystem() { + printf("function taosKillSystem, exit!"); + exit(0); +} + +int taosSystem(const char *cmd) { + printf("taosSystem not support"); + return -1; +} + +LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { + typedef BOOL(WINAPI * FxMiniDumpWriteDump)(IN HANDLE hProcess, IN DWORD ProcessId, IN HANDLE hFile, + IN MINIDUMP_TYPE DumpType, + IN CONST PMINIDUMP_EXCEPTION_INFORMATION ExceptionParam, + IN CONST PMINIDUMP_USER_STREAM_INFORMATION UserStreamParam, + IN CONST PMINIDUMP_CALLBACK_INFORMATION CallbackParam); + + HMODULE dll = LoadLibrary("dbghelp.dll"); + if (dll == NULL) return EXCEPTION_CONTINUE_SEARCH; + FxMiniDumpWriteDump mdwd = (FxMiniDumpWriteDump)(GetProcAddress(dll, "MiniDumpWriteDump")); + if (mdwd == NULL) { + FreeLibrary(dll); + return EXCEPTION_CONTINUE_SEARCH; + } + + TCHAR path[MAX_PATH]; + DWORD len = GetModuleFileName(NULL, path, _countof(path)); + path[len - 3] = 'd'; + path[len - 2] = 'm'; + path[len - 1] = 'p'; + + HANDLE file = CreateFile(path, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + + MINIDUMP_EXCEPTION_INFORMATION mei; + mei.ThreadId = GetCurrentThreadId(); + mei.ExceptionPointers = ep; + mei.ClientPointers = FALSE; + + (*mdwd)(GetCurrentProcess(), GetCurrentProcessId(), file, MiniDumpWithHandleData, &mei, NULL, NULL); + + CloseHandle(file); + FreeLibrary(dll); + + return EXCEPTION_CONTINUE_SEARCH; +} + +void taosSetCoreDump() { SetUnhandledExceptionFilter(&FlCrashDump); } + +bool taosGetSystemUid(char *uid) { + GUID guid; + CoCreateGuid(&guid); + + sprintf(uid, "%08X-%04X-%04X-%02X%02X-%02X%02X%02X%02X%02X%02X", guid.Data1, guid.Data2, guid.Data3, guid.Data4[0], + guid.Data4[1], guid.Data4[2], guid.Data4[3], guid.Data4[4], guid.Data4[5], guid.Data4[6], guid.Data4[7]); + + return true; +} + +char *taosGetCmdlineByPID(int pid) { return ""; } + +#elif defined(_TD_DARWIN_64) + +/* + * darwin implementation + */ + +#include +#include + +static void taosGetSystemTimezone() { + /* load time zone string from /etc/localtime */ + char buf[4096]; + char *tz = NULL; + { + int n = readlink("/etc/localtime", buf, sizeof(buf)); + if (n < 0) { + printf("read /etc/localtime error, reason:%s", strerror(errno)); + return; + } + buf[n] = '\0'; + for (int i = n - 1; i >= 0; --i) { + if (buf[i] == '/') { + if (tz) { + tz = buf + i + 1; + break; + } + tz = buf + i + 1; + } + } + if (!tz || 0 == strchr(tz, '/')) { + printf("parsing /etc/localtime failed"); + return; + } + + setenv("TZ", tz, 1); + tzset(); + } + + /* + * NOTE: do not remove it. + * Enforce set the correct daylight saving time(DST) flag according + * to current time + */ + time_t tx1 = time(NULL); + struct tm tm1; + localtime_r(&tx1, &tm1); + + /* + * format example: + * + * Asia/Shanghai (CST, +0800) + * Europe/London (BST, +0100) + */ + snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], + -timezone / 3600); + + // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; + printf("timezone not configured, set to system default:%s", tsTimezone); +} + +/* + * originally from src/os/src/detail/osSysinfo.c + * POSIX format locale string: + * (Language Strings)_(Country/Region Strings).(code_page) + * + * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, + * + * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. + * + * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page + * for libiconv that is employed to convert string in this system. This program will automatically use + * UTF-8 instead as the charset. + * + * In case of windows client, the locale string is not valid POSIX format, user needs to set the + * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is + * CP936, CP437 for English charset. + * + */ +static void taosGetSystemLocale() { // get and set default locale + char sep = '.'; + char *locale = NULL; + + locale = setlocale(LC_CTYPE, ""); + if (locale == NULL) { + printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); + strcpy(tsLocale, "en_US.UTF-8"); + } else { + tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); + printf("locale not configured, set to system default:%s", tsLocale); + } + + /* if user does not specify the charset, extract it from locale */ + char *str = strrchr(tsLocale, sep); + if (str != NULL) { + str++; + + char *revisedCharset = taosCharsetReplace(str); + tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN); + + free(revisedCharset); + printf("charset not configured, set to system default:%s", tsCharset); + } else { + strcpy(tsCharset, "UTF-8"); + printf("can't get locale and charset from system, set it to UTF-8"); + } +} + +void taosKillSystem() { + printf("function taosKillSystem, exit!"); + exit(0); +} + +int32_t taosGetCpuCores() { return sysconf(_SC_NPROCESSORS_ONLN); } + +void taosGetSystemInfo() { + // taosGetProcInfos(); + + tsNumOfCores = sysconf(_SC_NPROCESSORS_ONLN); + long physical_pages = sysconf(_SC_PHYS_PAGES); + long page_size = sysconf(_SC_PAGESIZE); + tsTotalMemoryMB = physical_pages * page_size / (1024 * 1024); + tsPageSize = page_size; + + taosGetSystemTimezone(); + taosGetSystemLocale(); +} + +bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { + if (rchars) *rchars = 0; + if (wchars) *wchars = 0; + return true; +} + +bool taosGetProcIO(float *readKB, float *writeKB) { + *readKB = 0; + *writeKB = 0; + return true; +} + +bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { + if (bytes) *bytes = 0; + if (rbytes) *rbytes = 0; + if (tbytes) *tbytes = 0; + return true; +} + +bool taosGetBandSpeed(float *bandSpeedKb) { + *bandSpeedKb = 0; + return true; +} + +bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { + *sysCpuUsage = 0; + *procCpuUsage = 0; + return true; +} + +bool taosGetProcMemory(float *memoryUsedMB) { + *memoryUsedMB = 0; + return true; +} + +bool taosGetSysMemory(float *memoryUsedMB) { + *memoryUsedMB = 0; + return true; +} + +int taosSystem(const char *cmd) { + printf("un support funtion"); + return -1; +} + +void taosSetCoreDump() {} + +int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) { + struct statvfs info; + if (statvfs(dataDir, &info)) { + printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } else { + diskSize->tsize = info.f_blocks * info.f_frsize; + diskSize->avail = info.f_bavail * info.f_frsize; + diskSize->used = (info.f_blocks - info.f_bfree) * info.f_frsize; + return 0; + } +} + +bool taosGetSystemUid(char *uid) { + uuid_t uuid = {0}; + uuid_generate(uuid); + // it's caller's responsibility to make enough space for `uid`, that's 36-char + 1-null + uuid_unparse_lower(uuid, uid); + return true; +} + +char *taosGetCmdlineByPID(int pid) { + static char cmdline[1024]; + errno = 0; + + if (proc_pidpath(pid, cmdline, sizeof(cmdline)) <= 0) { + fprintf(stderr, "PID is %d, %s", pid, strerror(errno)); + return strerror(errno); + } + + return cmdline; +} + +#else + +/* + * linux implementation + */ + +#include +#include +#include +#include +#include +#include +#include + +#define PROCESS_ITEM 12 + +typedef struct { + uint64_t user; + uint64_t nice; + uint64_t system; + uint64_t idle; +} SysCpuInfo; + +typedef struct { + uint64_t utime; // user time + uint64_t stime; // kernel time + uint64_t cutime; // all user time + uint64_t cstime; // all dead time +} ProcCpuInfo; + +static pid_t tsProcId; +static char tsSysNetFile[] = "/proc/net/dev"; +static char tsSysCpuFile[] = "/proc/stat"; +static char tsProcCpuFile[25] = {0}; +static char tsProcMemFile[25] = {0}; +static char tsProcIOFile[25] = {0}; +static float tsPageSizeKB = 0; + +static void taosGetProcInfos() { + tsPageSize = sysconf(_SC_PAGESIZE); + tsOpenMax = sysconf(_SC_OPEN_MAX); + tsStreamMax = sysconf(_SC_STREAM_MAX); + + tsProcId = (pid_t)syscall(SYS_gettid); + tsPageSizeKB = (float)(sysconf(_SC_PAGESIZE)) / 1024; + + snprintf(tsProcMemFile, 25, "/proc/%d/status", tsProcId); + snprintf(tsProcCpuFile, 25, "/proc/%d/stat", tsProcId); + snprintf(tsProcIOFile, 25, "/proc/%d/io", tsProcId); +} + +static int32_t taosGetTotalMemory() { return (int32_t)((float)sysconf(_SC_PHYS_PAGES) * tsPageSizeKB / 1024); } + +bool taosGetSysMemory(float *memoryUsedMB) { + float memoryAvailMB = (float)sysconf(_SC_AVPHYS_PAGES) * tsPageSizeKB / 1024; + *memoryUsedMB = (float)tsTotalMemoryMB - memoryAvailMB; + return true; +} + +bool taosGetProcMemory(float *memoryUsedMB) { + FILE *fp = fopen(tsProcMemFile, "r"); + if (fp == NULL) { + printf("open file:%s failed", tsProcMemFile); + return false; + } + + ssize_t _bytes = 0; + size_t len; + char * line = NULL; + while (!feof(fp)) { + tfree(line); + len = 0; + _bytes = getline(&line, &len, fp); + if ((_bytes < 0) || (line == NULL)) { + break; + } + if (strstr(line, "VmRSS:") != NULL) { + break; + } + } + + if (line == NULL) { + printf("read file:%s failed", tsProcMemFile); + fclose(fp); + return false; + } + + int64_t memKB = 0; + char tmp[10]; + sscanf(line, "%s %" PRId64, tmp, &memKB); + *memoryUsedMB = (float)((double)memKB / 1024); + + tfree(line); + fclose(fp); + return true; +} + +static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { + FILE *fp = fopen(tsSysCpuFile, "r"); + if (fp == NULL) { + printf("open file:%s failed", tsSysCpuFile); + return false; + } + + size_t len; + char * line = NULL; + ssize_t _bytes = getline(&line, &len, fp); + if ((_bytes < 0) || (line == NULL)) { + printf("read file:%s failed", tsSysCpuFile); + fclose(fp); + return false; + } + + char cpu[10] = {0}; + sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, + &cpuInfo->idle); + + tfree(line); + fclose(fp); + return true; +} + +static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { + FILE *fp = fopen(tsProcCpuFile, "r"); + if (fp == NULL) { + printf("open file:%s failed", tsProcCpuFile); + return false; + } + + size_t len = 0; + char * line = NULL; + ssize_t _bytes = getline(&line, &len, fp); + if ((_bytes < 0) || (line == NULL)) { + printf("read file:%s failed", tsProcCpuFile); + fclose(fp); + return false; + } + + for (int i = 0, blank = 0; line[i] != 0; ++i) { + if (line[i] == ' ') blank++; + if (blank == PROCESS_ITEM) { + sscanf(line + i + 1, "%" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, &cpuInfo->utime, &cpuInfo->stime, + &cpuInfo->cutime, &cpuInfo->cstime); + break; + } + } + + tfree(line); + fclose(fp); + return true; +} + +static void taosGetSystemTimezone() { + /* + * NOTE: do not remove it. + * Enforce set the correct daylight saving time(DST) flag according + * to current time + */ + time_t tx1 = time(NULL); + struct tm tm1; + localtime_r(&tx1, &tm1); + + /* load time zone string from /etc/timezone */ + FILE *f = fopen("/etc/timezone", "r"); + char buf[68] = {0}; + if (f != NULL) { + int len = fread(buf, 64, 1, f); + if (len < 64 && ferror(f)) { + fclose(f); + printf("read /etc/timezone error, reason:%s", strerror(errno)); + return; + } + + fclose(f); + + buf[sizeof(buf) - 1] = 0; + char *lineEnd = strstr(buf, "\n"); + if (lineEnd != NULL) { + *lineEnd = 0; + } + + // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables + if (strlen(buf) > 0) { + setenv("TZ", buf, 1); + } + } + // get and set default timezone + tzset(); + + /* + * get CURRENT time zone. + * system current time zone is affected by daylight saving time(DST) + * + * e.g., the local time zone of London in DST is GMT+01:00, + * otherwise is GMT+00:00 + */ + int32_t tz = (-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR; + tz += daylight; + + /* + * format example: + * + * Asia/Shanghai (CST, +0800) + * Europe/London (BST, +0100) + */ + snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + + // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; + printf("timezone not configured, set to system default:%s", tsTimezone); +} + +/* + * POSIX format locale string: + * (Language Strings)_(Country/Region Strings).(code_page) + * + * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, + * + * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. + * + * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page + * for libiconv that is employed to convert string in this system. This program will automatically use + * UTF-8 instead as the charset. + * + * In case of windows client, the locale string is not valid POSIX format, user needs to set the + * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is + * CP936, CP437 for English charset. + * + */ +static void taosGetSystemLocale() { // get and set default locale + char sep = '.'; + char *locale = NULL; + + locale = setlocale(LC_CTYPE, ""); + if (locale == NULL) { + printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); + strcpy(tsLocale, "en_US.UTF-8"); + } else { + tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); + printf("locale not configured, set to system default:%s", tsLocale); + } + + /* if user does not specify the charset, extract it from locale */ + char *str = strrchr(tsLocale, sep); + if (str != NULL) { + str++; + + char *revisedCharset = taosCharsetReplace(str); + tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN); + + free(revisedCharset); + printf("charset not configured, set to system default:%s", tsCharset); + } else { + strcpy(tsCharset, "UTF-8"); + printf("can't get locale and charset from system, set it to UTF-8"); + } +} + +int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); } + +bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { + static uint64_t lastSysUsed = 0; + static uint64_t lastSysTotal = 0; + static uint64_t lastProcTotal = 0; + + SysCpuInfo sysCpu; + ProcCpuInfo procCpu; + if (!taosGetSysCpuInfo(&sysCpu)) { + return false; + } + if (!taosGetProcCpuInfo(&procCpu)) { + return false; + } + + uint64_t curSysUsed = sysCpu.user + sysCpu.nice + sysCpu.system; + uint64_t curSysTotal = curSysUsed + sysCpu.idle; + uint64_t curProcTotal = procCpu.utime + procCpu.stime + procCpu.cutime + procCpu.cstime; + + if (lastSysUsed == 0 || lastSysTotal == 0 || lastProcTotal == 0) { + lastSysUsed = curSysUsed > 1 ? curSysUsed : 1; + lastSysTotal = curSysTotal > 1 ? curSysTotal : 1; + lastProcTotal = curProcTotal > 1 ? curProcTotal : 1; + return false; + } + + if (curSysTotal == lastSysTotal) { + return false; + } + + *sysCpuUsage = (float)((double)(curSysUsed - lastSysUsed) / (double)(curSysTotal - lastSysTotal) * 100); + *procCpuUsage = (float)((double)(curProcTotal - lastProcTotal) / (double)(curSysTotal - lastSysTotal) * 100); + + lastSysUsed = curSysUsed; + lastSysTotal = curSysTotal; + lastProcTotal = curProcTotal; + + return true; +} + +int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) { + struct statvfs info; + if (statvfs(dataDir, &info)) { + printf("failed to get disk size, dataDir:%s errno:%s", dataDir, strerror(errno)); + return -1; + } else { + diskSize->tsize = info.f_blocks * info.f_frsize; + diskSize->avail = info.f_bavail * info.f_frsize; + diskSize->used = (info.f_blocks - info.f_bfree) * info.f_frsize; + return 0; + } +} + +bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { + *bytes = 0; + FILE *fp = fopen(tsSysNetFile, "r"); + if (fp == NULL) { + printf("open file:%s failed", tsSysNetFile); + return false; + } + + ssize_t _bytes = 0; + size_t len = 2048; + char * line = calloc(1, len); + + while (!feof(fp)) { + memset(line, 0, len); + + int64_t o_rbytes = 0; + int64_t rpackts = 0; + int64_t o_tbytes = 0; + int64_t tpackets = 0; + int64_t nouse1 = 0; + int64_t nouse2 = 0; + int64_t nouse3 = 0; + int64_t nouse4 = 0; + int64_t nouse5 = 0; + int64_t nouse6 = 0; + char nouse0[200] = {0}; + + _bytes = getline(&line, &len, fp); + if (_bytes < 0) { + break; + } + + line[len - 1] = 0; + + if (strstr(line, "lo:") != NULL) { + continue; + } + + sscanf(line, + "%s %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 + " %" PRId64, + nouse0, &o_rbytes, &rpackts, &nouse1, &nouse2, &nouse3, &nouse4, &nouse5, &nouse6, &o_tbytes, &tpackets); + if (rbytes) *rbytes = o_rbytes; + if (tbytes) *tbytes = o_tbytes; + *bytes += (o_rbytes + o_tbytes); + } + + tfree(line); + fclose(fp); + + return true; +} + +bool taosGetBandSpeed(float *bandSpeedKb) { + static int64_t lastBytes = 0; + static time_t lastTime = 0; + int64_t curBytes = 0; + time_t curTime = time(NULL); + + if (!taosGetCardInfo(&curBytes, NULL, NULL)) { + return false; + } + + if (lastTime == 0 || lastBytes == 0) { + lastTime = curTime; + lastBytes = curBytes; + *bandSpeedKb = 0; + return true; + } + + if (lastTime >= curTime || lastBytes > curBytes) { + lastTime = curTime; + lastBytes = curBytes; + *bandSpeedKb = 0; + return true; + } + + double totalBytes = (double)(curBytes - lastBytes) / 1024 * 8; // Kb + *bandSpeedKb = (float)(totalBytes / (double)(curTime - lastTime)); + + // printf("bandwidth lastBytes:%ld, lastTime:%ld, curBytes:%ld, curTime:%ld, + // speed:%f", lastBytes, lastTime, curBytes, curTime, *bandSpeed); + + lastTime = curTime; + lastBytes = curBytes; + + return true; +} + +bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { + FILE *fp = fopen(tsProcIOFile, "r"); + if (fp == NULL) { + printf("open file:%s failed", tsProcIOFile); + return false; + } + + ssize_t _bytes = 0; + size_t len; + char * line = NULL; + char tmp[10]; + int readIndex = 0; + + while (!feof(fp)) { + tfree(line); + len = 0; + _bytes = getline(&line, &len, fp); + if ((_bytes < 0) || (line == NULL)) { + break; + } + if (strstr(line, "rchar:") != NULL) { + sscanf(line, "%s %" PRId64, tmp, rchars); + readIndex++; + } else if (strstr(line, "wchar:") != NULL) { + sscanf(line, "%s %" PRId64, tmp, wchars); + readIndex++; + } else { + } + + if (readIndex >= 2) break; + } + + tfree(line); + fclose(fp); + + if (readIndex < 2) { + printf("read file:%s failed", tsProcIOFile); + return false; + } + + return true; +} + +bool taosGetProcIO(float *readKB, float *writeKB) { + static int64_t lastReadbyte = -1; + static int64_t lastWritebyte = -1; + + int64_t curReadbyte = 0; + int64_t curWritebyte = 0; + + if (!taosReadProcIO(&curReadbyte, &curWritebyte)) { + return false; + } + + if (lastReadbyte == -1 || lastWritebyte == -1) { + lastReadbyte = curReadbyte; + lastWritebyte = curWritebyte; + return false; + } + + *readKB = (float)((double)(curReadbyte - lastReadbyte) / 1024); + *writeKB = (float)((double)(curWritebyte - lastWritebyte) / 1024); + if (*readKB < 0) *readKB = 0; + if (*writeKB < 0) *writeKB = 0; + + lastReadbyte = curReadbyte; + lastWritebyte = curWritebyte; + + return true; +} + +void taosGetSystemInfo() { + taosGetProcInfos(); + + tsNumOfCores = taosGetCpuCores(); + tsTotalMemoryMB = taosGetTotalMemory(); + + float tmp1, tmp2; + taosGetSysMemory(&tmp1); + taosGetProcMemory(&tmp2); + // taosGetDisk(); + taosGetBandSpeed(&tmp1); + taosGetCpuUsage(&tmp1, &tmp2); + taosGetProcIO(&tmp1, &tmp2); + + taosGetSystemTimezone(); + taosGetSystemLocale(); +} + +void taosKillSystem() { + // SIGINT + printf("taosd will shut down soon"); + kill(tsProcId, 2); +} + +int taosSystem(const char *cmd) { + FILE *fp; + int res; + char buf[1024]; + if (cmd == NULL) { + printf("taosSystem cmd is NULL!"); + return -1; + } + + if ((fp = popen(cmd, "r")) == NULL) { + printf("popen cmd:%s error: %s", cmd, strerror(errno)); + return -1; + } else { + while (fgets(buf, sizeof(buf), fp)) { + printf("popen result:%s", buf); + } + + if ((res = pclose(fp)) == -1) { + printf("close popen file pointer fp error!"); + } else { + printf("popen res is :%d", res); + } + + return res; + } +} + +void taosSetCoreDump(bool enable) { + if (!enable) return; + + // 1. set ulimit -c unlimited + struct rlimit rlim; + struct rlimit rlim_new; + if (getrlimit(RLIMIT_CORE, &rlim) == 0) { +#ifndef _ALPINE + printf("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); +#else + printf("the old unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max); +#endif + rlim_new.rlim_cur = RLIM_INFINITY; + rlim_new.rlim_max = RLIM_INFINITY; + if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) { + printf("set unlimited fail, error: %s", strerror(errno)); + rlim_new.rlim_cur = rlim.rlim_max; + rlim_new.rlim_max = rlim.rlim_max; + (void)setrlimit(RLIMIT_CORE, &rlim_new); + } + } + + if (getrlimit(RLIMIT_CORE, &rlim) == 0) { +#ifndef _ALPINE + printf("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); +#else + printf("the new unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max); +#endif + } + +#ifndef _TD_ARM_ + // 2. set the path for saving core file + struct __sysctl_args args; + + int old_usespid = 0; + size_t old_len = 0; + int new_usespid = 1; + size_t new_len = sizeof(new_usespid); + + int name[] = {CTL_KERN, KERN_CORE_USES_PID}; + + memset(&args, 0, sizeof(struct __sysctl_args)); + args.name = name; + args.nlen = sizeof(name) / sizeof(name[0]); + args.oldval = &old_usespid; + args.oldlenp = &old_len; + args.newval = &new_usespid; + args.newlen = new_len; + + old_len = sizeof(old_usespid); + + if (syscall(SYS__sysctl, &args) == -1) { + printf("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno)); + } + + printf("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); + + old_usespid = 0; + old_len = 0; + memset(&args, 0, sizeof(struct __sysctl_args)); + args.name = name; + args.nlen = sizeof(name) / sizeof(name[0]); + args.oldval = &old_usespid; + args.oldlenp = &old_len; + + old_len = sizeof(old_usespid); + + if (syscall(SYS__sysctl, &args) == -1) { + printf("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno)); + } + + printf("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); +#endif +} + +bool taosGetSystemUid(char *uid, int32_t uidlen) { + int fd; + int len = 0; + + fd = open("/proc/sys/kernel/random/uuid", 0); + if (fd < 0) { + return false; + } else { + len = read(fd, uid, uidlen); + close(fd); + } + + if (len >= 36) { + uid[36] = 0; + return true; + } + return false; +} + +char *taosGetCmdlineByPID(int pid) { + static char cmdline[1024]; + sprintf(cmdline, "/proc/%d/cmdline", pid); + + int fd = open(cmdline, O_RDONLY); + if (fd >= 0) { + int n = read(fd, cmdline, sizeof(cmdline) - 1); + if (n < 0) n = 0; + + if (n > 0 && cmdline[n - 1] == '\n') --n; + + cmdline[n] = 0; + + close(fd); + } else { + cmdline[0] = 0; + } + + return cmdline; +} + +#endif + +#if !(defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) + +SysNameInfo taosGetSysNameInfo() { + SysNameInfo info = {0}; + + struct utsname buf; + if (!uname(&buf)) { + info.sysname = buf.sysname; + info.sysname == buf.nodename; + info.sysname = buf.release; + info.sysname = buf.version; + info.sysname = buf.machine; + } + + return info; +} +#endif \ No newline at end of file diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c new file mode 100644 index 0000000000..1a57e88c58 --- /dev/null +++ b/source/os/src/osSystem.c @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + +/* + * windows implementation + */ + +void* taosLoadDll(const char* filename) { return NULL; } +void* taosLoadSym(void* handle, char* name) { return NULL; } +void taosCloseDll(void* handle) {} + +int taosSetConsoleEcho(bool on) { + HANDLE hStdin = GetStdHandle(STD_INPUT_HANDLE); + DWORD mode = 0; + GetConsoleMode(hStdin, &mode); + if (on) { + mode |= ENABLE_ECHO_INPUT; + } else { + mode &= ~ENABLE_ECHO_INPUT; + } + SetConsoleMode(hStdin, mode); + + return 0; +} + +#elif defined(_TD_DARWIN_64) + +/* + * darwin implementation + */ + +void* taosLoadDll(const char* filename) { return NULL; } +void* taosLoadSym(void* handle, char* name) { return NULL; } +void taosCloseDll(void* handle) {} + +int taosSetConsoleEcho(bool on) { +#define ECHOFLAGS (ECHO | ECHOE | ECHOK | ECHONL) + int err; + struct termios term; + + if (tcgetattr(STDIN_FILENO, &term) == -1) { + perror("Cannot get the attribution of the terminal"); + return -1; + } + + if (on) + term.c_lflag |= ECHOFLAGS; + else + term.c_lflag &= ~ECHOFLAGS; + + err = tcsetattr(STDIN_FILENO, TCSAFLUSH, &term); + if (err == -1 && err == EINTR) { + perror("Cannot set the attribution of the terminal"); + return -1; + } + + return 0; +} + +#else + +/* + * linux implementation + */ + +#include +#include +#include + +void* taosLoadDll(const char* filename) { + void* handle = dlopen(filename, RTLD_LAZY); + if (!handle) { + printf("load dll:%s failed, error:%s", filename, dlerror()); + return NULL; + } + + printf("dll %s loaded", filename); + + return handle; +} + +void* taosLoadSym(void* handle, char* name) { + void* sym = dlsym(handle, name); + char* error = NULL; + + if ((error = dlerror()) != NULL) { + printf("load sym:%s failed, error:%s", name, dlerror()); + return NULL; + } + + printf("sym %s loaded", name); + + return sym; +} + +void taosCloseDll(void* handle) { + if (handle) { + dlclose(handle); + } +} + +int taosSetConsoleEcho(bool on) { +#define ECHOFLAGS (ECHO | ECHOE | ECHOK | ECHONL) + int err; + struct termios term; + + if (tcgetattr(STDIN_FILENO, &term) == -1) { + perror("Cannot get the attribution of the terminal"); + return -1; + } + + if (on) + term.c_lflag |= ECHOFLAGS; + else + term.c_lflag &= ~ECHOFLAGS; + + err = tcsetattr(STDIN_FILENO, TCSAFLUSH, &term); + if (err == -1 || err == EINTR) { + printf("Cannot set the attribution of the terminal"); + return -1; + } + + return 0; +} + +#endif diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 27d046d4c9..2deac7e216 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -13,16 +13,28 @@ * along with this program. If not, see . */ +#define _BSD_SOURCE + +#ifdef DARWIN +#define _XOPEN_SOURCE +#else +#define _XOPEN_SOURCE 500 +#endif + +#define _DEFAULT_SOURCE + #include "os.h" #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) -#include -#else -#endif +/* + * windows implementation + */ -FORCE_INLINE int32_t taosGetTimeOfDay(struct timeval *tv) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - time_t t; +#include +#include + +int taosGetTimeOfDay(struct timeval *tv, struct timezone *tz) { + time_t t; t = time(NULL); SYSTEMTIME st; GetLocalTime(&st); @@ -31,7 +43,23 @@ FORCE_INLINE int32_t taosGetTimeOfDay(struct timeval *tv) { tv->tv_usec = st.wMilliseconds * 1000; return 0; +} + +struct tm *localtime_r(const time_t *timep, struct tm *result) { + localtime_s(result, timep); + return result; +} + #else + +/* + * linux and darwin implementation + */ + +#include + +FORCE_INLINE int32_t taosGetTimeOfDay(struct timeval *tv) { return gettimeofday(tv, NULL); +} + #endif -} \ No newline at end of file diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c new file mode 100644 index 0000000000..d0114044f3 --- /dev/null +++ b/source/os/src/osTimer.c @@ -0,0 +1,225 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + +/* + * windows implementation + */ + + +#include +#include +#include +#include + +#pragma warning( disable : 4244 ) + +typedef void (*win_timer_f)(int signo); + +void WINAPI taosWinOnTimer(UINT wTimerID, UINT msg, DWORD_PTR dwUser, DWORD_PTR dwl, DWORD_PTR dw2) { + win_timer_f callback = *((win_timer_f *)&dwUser); + if (callback != NULL) { + callback(0); + } +} + +static MMRESULT timerId; +int taosInitTimer(win_timer_f callback, int ms) { + DWORD_PTR param = *((int64_t *) & callback); + + timerId = timeSetEvent(ms, 1, (LPTIMECALLBACK)taosWinOnTimer, param, TIME_PERIODIC); + if (timerId == 0) { + return -1; + } + return 0; +} + +void taosUninitTimer() { + timeKillEvent(timerId); +} + +#elif defined(_TD_DARWIN_64) + +/* + * darwin implementation + */ + +#include +#include + +static void (*timer_callback)(int); +static int timer_ms = 0; +static pthread_t timer_thread; +static int timer_kq = -1; +static volatile int timer_stop = 0; + +static void* timer_routine(void *arg) { + (void)arg; + setThreadName("timer"); + + int r = 0; + struct timespec to = {0}; + to.tv_sec = timer_ms / 1000; + to.tv_nsec = (timer_ms % 1000) * 1000000; + while (!timer_stop) { + struct kevent64_s kev[10] = {0}; + r = kevent64(timer_kq, NULL, 0, kev, sizeof(kev)/sizeof(kev[0]), 0, &to); + if (r!=0) { + fprintf(stderr, "==%s[%d]%s()==kevent64 failed\n", basename(__FILE__), __LINE__, __func__); + abort(); + } + timer_callback(SIGALRM); // just mock + } + + return NULL; +} + +int taosInitTimer(void (*callback)(int), int ms) { + int r = 0; + timer_ms = ms; + timer_callback = callback; + + timer_kq = kqueue(); + if (timer_kq==-1) { + fprintf(stderr, "==%s[%d]%s()==failed to create timer kq\n", basename(__FILE__), __LINE__, __func__); + // since no caller of this func checks the return value for the moment + abort(); + } + + r = pthread_create(&timer_thread, NULL, timer_routine, NULL); + if (r) { + fprintf(stderr, "==%s[%d]%s()==failed to create timer thread\n", basename(__FILE__), __LINE__, __func__); + // since no caller of this func checks the return value for the moment + abort(); + } + return 0; +} + +void taosUninitTimer() { + int r = 0; + timer_stop = 1; + r = pthread_join(timer_thread, NULL); + if (r) { + fprintf(stderr, "==%s[%d]%s()==failed to join timer thread\n", basename(__FILE__), __LINE__, __func__); + // since no caller of this func checks the return value for the moment + abort(); + } + close(timer_kq); + timer_kq = -1; +} + +void taos_block_sigalrm(void) { + // we don't know if there's any specific API for SIGALRM to deliver to specific thread + // this implementation relies on kqueue rather than SIGALRM +} + +#else + +/* + * linux implementation + */ + +#include + +static void taosDeleteTimer(void *tharg) { + timer_t *pTimer = tharg; + timer_delete(*pTimer); +} + +static pthread_t timerThread; +static timer_t timerId; +static volatile bool stopTimer = false; +static void *taosProcessAlarmSignal(void *tharg) { + // Block the signal + sigset_t sigset; + sigemptyset(&sigset); + sigaddset(&sigset, SIGALRM); + sigprocmask(SIG_BLOCK, &sigset, NULL); + void (*callback)(int) = tharg; + + struct sigevent sevent = {{0}}; + + setThreadName("tmr"); + + #ifdef _ALPINE + sevent.sigev_notify = SIGEV_THREAD; + sevent.sigev_value.sival_int = syscall(__NR_gettid); + #else + sevent.sigev_notify = SIGEV_THREAD_ID; + sevent._sigev_un._tid = syscall(__NR_gettid); + #endif + + sevent.sigev_signo = SIGALRM; + + if (timer_create(CLOCK_REALTIME, &sevent, &timerId) == -1) { + printf("Failed to create timer"); + } + + pthread_cleanup_push(taosDeleteTimer, &timerId); + + struct itimerspec ts; + ts.it_value.tv_sec = 0; + ts.it_value.tv_nsec = 1000000 * MSECONDS_PER_TICK; + ts.it_interval.tv_sec = 0; + ts.it_interval.tv_nsec = 1000000 * MSECONDS_PER_TICK; + + if (timer_settime(timerId, 0, &ts, NULL)) { + printf("Failed to init timer"); + return NULL; + } + + int signo; + while (!stopTimer) { + if (sigwait(&sigset, &signo)) { + printf("Failed to wait signal: number %d", signo); + continue; + } + /* printf("Signal handling: number %d ......\n", signo); */ + + callback(0); + } + + pthread_cleanup_pop(1); + + return NULL; +} + +int taosInitTimer(void (*callback)(int), int ms) { + pthread_attr_t tattr; + pthread_attr_init(&tattr); + int code = pthread_create(&timerThread, &tattr, taosProcessAlarmSignal, callback); + pthread_attr_destroy(&tattr); + if (code != 0) { + printf("failed to create timer thread"); + return -1; + } else { + printf("timer thread:0x%08" PRIx64 " is created", taosGetPthreadId(timerThread)); + } + + return 0; +} + +void taosUninitTimer() { + stopTimer = true; + + printf("join timer thread:0x%08" PRIx64, taosGetPthreadId(timerThread)); + pthread_join(timerThread, NULL); +} + +#endif diff --git a/source/server/dnode/src/dnodeMain.c b/source/server/dnode/src/dnodeMain.c index 715c27a9e0..668b844f6c 100644 --- a/source/server/dnode/src/dnodeMain.c +++ b/source/server/dnode/src/dnodeMain.c @@ -28,7 +28,7 @@ #include "mnode.h" static int32_t dnodeCreateDir(const char *dir) { - if (taosMkDir(dir, 0755) != 0 && errno != EEXIST) { + if (!taosMkDir(dir, 0755) && errno != EEXIST) { return -1; } @@ -55,6 +55,14 @@ static void dnodeCheckDataDirOpenned(char *dir) { #endif } +void dnodePrintDiskInfo() { + dInfo("=================================="); + dInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); + dInfo(" os usedDisk: %f(GB)", tsUsedDataDirGB); + dInfo(" os availDisk: %f(GB)", tsAvailDataDirGB); + dInfo("=================================="); +} + int32_t dnodeInitMain(Dnode *dnode, DnMain **out) { DnMain* main = calloc(1, sizeof(DnMain)); if (main == NULL) return -1; @@ -76,7 +84,7 @@ int32_t dnodeInitMain(Dnode *dnode, DnMain **out) { taosInitGlobalCfg(); taosReadGlobalLogCfg(); #if 0 - taosSetCoreDump(); + taosSetCoreDump(tsEnableCoreFile); #endif if (dnodeCreateDir(tsLogDir) < 0) { @@ -182,7 +190,7 @@ int32_t dnodeInitStorage(Dnode *dnode, void **m) { dnodeCheckDataDirOpenned(tsDnodeDir); taosGetDisk(); - taosPrintDiskInfo(); + dnodePrintDiskInfo(); #endif dInfo("dnode storage is initialized at %s", tsDnodeDir); diff --git a/source/server/dnode/src/dnodeStatus.c b/source/server/dnode/src/dnodeStatus.c index 4408cc8b87..58effb5b44 100644 --- a/source/server/dnode/src/dnodeStatus.c +++ b/source/server/dnode/src/dnodeStatus.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "ttime.h" #include "ttimer.h" #include "tglobal.h" #include "dnodeCfg.h" diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index cec2554b52..df7efb3380 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -8,4 +8,6 @@ target_include_directories( target_link_libraries( util PRIVATE os -) \ No newline at end of file + PUBLIC zlib + PUBLIC lz4_static +) diff --git a/src/util/src/exception.c b/source/util/src/exception.c similarity index 100% rename from src/util/src/exception.c rename to source/util/src/exception.c diff --git a/source/util/src/hash.c b/source/util/src/hash.c index 21e835a6bf..2d48a7c3bf 100644 --- a/source/util/src/hash.c +++ b/source/util/src/hash.c @@ -16,7 +16,6 @@ #include "os.h" #include "hash.h" #include "tulog.h" -// #include "taosdef.h" #define EXT_SIZE 1024 diff --git a/source/util/src/talgo.c b/source/util/src/talgo.c index b9aabd7c4a..f8520b0742 100644 --- a/source/util/src/talgo.c +++ b/source/util/src/talgo.c @@ -14,7 +14,6 @@ */ #include "os.h" - #include "talgo.h" #define doswap(__left, __right, __size, __buf) do {\ diff --git a/src/util/src/tbase64.c b/source/util/src/tbase64.c similarity index 100% rename from src/util/src/tbase64.c rename to source/util/src/tbase64.c diff --git a/src/util/src/tbuffer.c b/source/util/src/tbuffer.c similarity index 100% rename from src/util/src/tbuffer.c rename to source/util/src/tbuffer.c diff --git a/src/util/src/tcache.c b/source/util/src/tcache.c similarity index 100% rename from src/util/src/tcache.c rename to source/util/src/tcache.c diff --git a/src/util/src/tcompare.c b/source/util/src/tcompare.c similarity index 99% rename from src/util/src/tcompare.c rename to source/util/src/tcompare.c index 482dd8a6a1..e31fbaaff1 100644 --- a/src/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -22,7 +22,7 @@ #include "hash.h" #include "regex.h" #include "os.h" -#include "ttype.h" +#include "tdef.h" int32_t setCompareBytes1(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; diff --git a/src/util/src/tcompression.c b/source/util/src/tcompression.c similarity index 99% rename from src/util/src/tcompression.c rename to source/util/src/tcompression.c index 48bba75926..990008a840 100644 --- a/src/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -52,11 +52,8 @@ #ifdef TD_TSZ #include "td_sz.h" #endif -#include "taosdef.h" #include "tscompression.h" #include "tulog.h" -#include "tglobal.h" - static const int TEST_NUMBER = 1; #define is_bigendian() ((*(char *)&TEST_NUMBER) == 0) diff --git a/src/util/src/tconfig.c b/source/util/src/tconfig.c similarity index 87% rename from src/util/src/tconfig.c rename to source/util/src/tconfig.c index 5a3dc3f9bc..e7c0758b61 100644 --- a/src/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -15,10 +15,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taosdef.h" -#include "taoserror.h" #include "tconfig.h" -#include "tglobal.h" #include "tulog.h" #include "tsocket.h" #include "tutil.h" @@ -78,7 +75,6 @@ static void taosReadDoubleConfig(SGlobalCfg *cfg, char *input_value) { } } - static void taosReadInt32Config(SGlobalCfg *cfg, char *input_value) { int32_t value = atoi(input_value); int32_t *option = (int32_t *)cfg->ptr; @@ -156,27 +152,10 @@ static bool taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) { return false; } else { if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - wordexp_t full_path; - if (0 != wordexp(input_value, &full_path, 0)) { - printf("\nconfig dir: %s wordexp fail! reason:%s\n", input_value, strerror(errno)); - wordfree(&full_path); - return false; - } + taosExpandDir(input_value, option, cfg->ptrLength); + taosRealPath(option, cfg->ptrLength); - if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) { - strcpy(option, full_path.we_wordv[0]); - } - - wordfree(&full_path); - - char tmp[PATH_MAX] = {0}; - if (realpath(option, tmp) != NULL) { - strcpy(option, tmp); - } - - int code = taosMkDir(option, 0755); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); + if (!taosMkDir(option, 0755)) { uError("config option:%s, input value:%s, directory not exist, create fail:%s", cfg->option, input_value, strerror(errno)); return false; @@ -234,9 +213,9 @@ static void taosReadLogOption(char *option, char *value) { switch (cfg->valType) { case TAOS_CFG_VTYPE_INT32: taosReadInt32Config(cfg, value); - if (strcasecmp(cfg->option, "debugFlag") == 0) { - taosSetAllDebugFlag(); - } + // if (strcasecmp(cfg->option, "debugFlag") == 0) { + // taosSetAllDebugFlag(); + // } break; case TAOS_CFG_VTYPE_DIRECTORY: taosReadDirectoryConfig(cfg, value); @@ -249,7 +228,6 @@ static void taosReadLogOption(char *option, char *value) { } SGlobalCfg *taosGetConfigOption(const char *option) { - taosInitGlobalCfg(); for (int i = 0; i < tsGlobalConfigNum; ++i) { SGlobalCfg *cfg = tsGlobalConfig + i; if (strcasecmp(cfg->option, option) != 0) continue; @@ -294,7 +272,7 @@ static void taosReadConfigOption(const char *option, char *value, char *value2, break; case TAOS_CFG_VTYPE_DATA_DIRCTORY: if (taosReadDirectoryConfig(cfg, value)) { - taosReadDataDirCfg(value, value2, value3); + // taosReadDataDirCfg(value, value2, value3); } break; default: @@ -315,34 +293,7 @@ void taosReadGlobalLogCfg() { int olen, vlen; char fileName[PATH_MAX] = {0}; - wordexp_t full_path; - if ( 0 != wordexp(configDir, &full_path, 0)) { - printf("\nconfig file: %s wordexp fail! reason:%s\n", configDir, strerror(errno)); - wordfree(&full_path); - return; - } - - if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) { - if (strlen(full_path.we_wordv[0]) >= TSDB_FILENAME_LEN) { - printf("\nconfig file: %s path overflow max len %d, all variables are set to default\n", full_path.we_wordv[0], TSDB_FILENAME_LEN - 1); - wordfree(&full_path); - return; - } - strcpy(configDir, full_path.we_wordv[0]); - } else { - #ifdef _TD_POWER_ - printf("configDir:%s not there, use default value: /etc/power", configDir); - strcpy(configDir, "/etc/power"); - #elif (_TD_TQ_ == true) - printf("configDir:%s not there, use default value: /etc/tq", configDir); - strcpy(configDir, "/etc/tq"); - #else - printf("configDir:%s not there, use default value: /etc/taos", configDir); - strcpy(configDir, "/etc/taos"); - #endif - } - wordfree(&full_path); - + taosExpandDir(configDir, configDir, PATH_MAX); taosReadLogOption("logDir", tsLogDir); sprintf(fileName, "%s/taos.cfg", configDir); @@ -391,13 +342,8 @@ bool taosReadGlobalCfg() { char fileName[PATH_MAX] = {0}; sprintf(fileName, "%s/taos.cfg", configDir); - FILE* fp = fopen(fileName, "r"); + FILE *fp = fopen(fileName, "r"); if (fp == NULL) { - struct stat s; - if (stat(configDir, &s) != 0 || (!S_ISREG(s.st_mode) && !S_ISLNK(s.st_mode))) { - //return true to follow behavior before file support - return true; - } fp = fopen(configDir, "r"); if (fp == NULL) { return false; @@ -444,9 +390,9 @@ bool taosReadGlobalCfg() { tfree(line); - if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { - taosSetAllDebugFlag(); - } + // if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { + // taosSetAllDebugFlag(); + // } return true; } @@ -498,7 +444,7 @@ void taosPrintGlobalCfg() { } taosPrintOsInfo(); - taosPrintDataDirCfg(); + // taosPrintDataDirCfg(); uInfo("=================================="); } diff --git a/src/util/src/tcrc32c.c b/source/util/src/tcrc32c.c similarity index 99% rename from src/util/src/tcrc32c.c rename to source/util/src/tcrc32c.c index d2b63eb9ee..73bba4480e 100644 --- a/src/util/src/tcrc32c.c +++ b/source/util/src/tcrc32c.c @@ -18,14 +18,15 @@ 3. This notice may not be removed or altered from any source distribution. */ +#include "os.h" +#include "tcrc32c.h" +#include "tdef.h" + #if !defined(_TD_ARM_) && !defined(_TD_MIPS_) #include #endif -#include -#include -#include "tcrc32c.h" #define POLY 0x82f63b78 #define LONG_SHIFT 8192 diff --git a/src/util/src/tdes.c b/source/util/src/tdes.c similarity index 100% rename from src/util/src/tdes.c rename to source/util/src/tdes.c diff --git a/src/util/src/tfile.c b/source/util/src/tfile.c similarity index 100% rename from src/util/src/tfile.c rename to source/util/src/tfile.c diff --git a/src/util/src/tfunctional.c b/source/util/src/tfunctional.c similarity index 100% rename from src/util/src/tfunctional.c rename to source/util/src/tfunctional.c diff --git a/src/util/src/thashutil.c b/source/util/src/thashutil.c similarity index 100% rename from src/util/src/thashutil.c rename to source/util/src/thashutil.c diff --git a/src/util/src/tidpool.c b/source/util/src/tidpool.c similarity index 100% rename from src/util/src/tidpool.c rename to source/util/src/tidpool.c diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 1906ea1629..63dd1ef606 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -19,6 +19,7 @@ #include "tlog.h" #include "tnote.h" #include "tutil.h" +#include "zlib.h" #define MAX_LOGLINE_SIZE (1000) #define MAX_LOGLINE_BUFFER_SIZE (MAX_LOGLINE_SIZE + 10) @@ -75,22 +76,34 @@ float tsMinimalLogDirGB = 1.0f; int64_t asyncLogLostLines = 0; int32_t writeInterval = DEFAULT_LOG_INTERVAL; +// log +int32_t tsNumOfLogLines = 10000000; +int32_t mDebugFlag = 131; +int32_t sdbDebugFlag = 131; +int32_t dDebugFlag = 135; +int32_t vDebugFlag = 135; +int32_t cDebugFlag = 131; +int32_t jniDebugFlag = 131; +int32_t odbcDebugFlag = 131; +int32_t httpDebugFlag = 131; +int32_t mqttDebugFlag = 131; +int32_t monDebugFlag = 131; +int32_t qDebugFlag = 131; +int32_t rpcDebugFlag = 131; +int32_t uDebugFlag = 131; +int32_t debugFlag = 0; +int32_t sDebugFlag = 135; +int32_t wDebugFlag = 135; +int32_t tsdbDebugFlag = 131; +int32_t cqDebugFlag = 131; +int32_t fsDebugFlag = 135; + int64_t dbgEmptyW = 0; int64_t dbgWN = 0; int64_t dbgSmallWN = 0; int64_t dbgBigWN = 0; int64_t dbgWSize = 0; -#ifdef _TD_POWER_ -char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/power"; -#elif (_TD_TQ_ == true) -char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/tq"; -#elif (_TD_PRO_ == true) -char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/ProDB"; -#else -char tsLogDir[PATH_MAX] = "/var/log/taos"; -#endif - static SLogObj tsLogObj = { .fileNum = 1 }; static void * taosAsyncOutputLog(void *param); static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen); @@ -98,6 +111,7 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize); static void taosCloseLogByFd(int32_t oldFd); static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum); extern void taosPrintGlobalCfg(); +static int32_t taosCompressFile(char *srcFileName, char *destFileName); static int32_t taosStartLog() { pthread_attr_t threadAttr; @@ -164,7 +178,7 @@ static void taosKeepOldLog(char *oldName) { char fileName[LOG_FILE_NAME_LEN + 20]; snprintf(fileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64, tsLogObj.logName, fileSec); - taosRename(oldName, fileName); + taosRenameFile(oldName, fileName); if (tsLogKeepDays < 0) { char compressFileName[LOG_FILE_NAME_LEN + 20]; snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec); @@ -173,7 +187,7 @@ static void taosKeepOldLog(char *oldName) { } } - taosRemoveOldLogFiles(tsLogDir, ABS(tsLogKeepDays)); + taosRemoveOldFiles(tsLogDir, ABS(tsLogKeepDays)); } static void *taosThreadToOpenNewFile(void *param) { @@ -702,3 +716,62 @@ static void *taosAsyncOutputLog(void *param) { return NULL; } + +int32_t taosCompressFile(char *srcFileName, char *destFileName) { + int32_t compressSize = 163840; + int32_t ret = 0; + int32_t len = 0; + char * data = malloc(compressSize); + FILE * srcFp = NULL; + gzFile dstFp = NULL; + + srcFp = fopen(srcFileName, "r"); + if (srcFp == NULL) { + ret = -1; + goto cmp_end; + } + + int32_t fd = taosOpenFileTruncCreateWrite(destFileName); + if (fd < 0) { + ret = -2; + goto cmp_end; + } + + dstFp = gzdopen(fd, "wb6f"); + if (dstFp == NULL) { + ret = -3; + close(fd); + goto cmp_end; + } + + while (!feof(srcFp)) { + len = (int32_t)fread(data, 1, compressSize, srcFp); + (void)gzwrite(dstFp, data, len); + } + +cmp_end: + if (srcFp) { + fclose(srcFp); + } + if (dstFp) { + gzclose(dstFp); + } + free(data); + + return ret; +} + +void taosPrintOsInfo() { + SysNameInfo info = taosGetSysNameInfo(); + + uInfo(" os pageSize: %" PRId64 "(KB)", tsPageSize); + uInfo(" os openMax: %" PRId64, tsOpenMax); + uInfo(" os streamMax: %" PRId64, tsStreamMax); + uInfo(" os numOfCores: %d", tsNumOfCores); + uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); + uInfo(" os sysname: %s", info.sysname); + uInfo(" os nodename: %s", info.nodename); + uInfo(" os release: %s", info.release); + uInfo(" os version: %s", info.version); + uInfo(" os machine: %s", info.machine); +} diff --git a/src/util/src/tlosertree.c b/source/util/src/tlosertree.c similarity index 100% rename from src/util/src/tlosertree.c rename to source/util/src/tlosertree.c diff --git a/src/util/src/tmd5.c b/source/util/src/tmd5.c similarity index 100% rename from src/util/src/tmd5.c rename to source/util/src/tmd5.c diff --git a/src/util/src/tmempool.c b/source/util/src/tmempool.c similarity index 100% rename from src/util/src/tmempool.c rename to source/util/src/tmempool.c diff --git a/src/util/src/tnettest.c b/source/util/src/tnettest.c similarity index 100% rename from src/util/src/tnettest.c rename to source/util/src/tnettest.c diff --git a/src/util/src/tnote.c b/source/util/src/tnote.c similarity index 100% rename from src/util/src/tnote.c rename to source/util/src/tnote.c diff --git a/src/util/src/tqueue.c b/source/util/src/tqueue.c similarity index 100% rename from src/util/src/tqueue.c rename to source/util/src/tqueue.c diff --git a/src/util/src/tref.c b/source/util/src/tref.c similarity index 100% rename from src/util/src/tref.c rename to source/util/src/tref.c diff --git a/src/util/src/tsched.c b/source/util/src/tsched.c similarity index 100% rename from src/util/src/tsched.c rename to source/util/src/tsched.c diff --git a/src/util/src/tskiplist.c b/source/util/src/tskiplist.c similarity index 100% rename from src/util/src/tskiplist.c rename to source/util/src/tskiplist.c diff --git a/src/util/src/tsocket.c b/source/util/src/tsocket.c similarity index 100% rename from src/util/src/tsocket.c rename to source/util/src/tsocket.c diff --git a/src/util/src/tstrbuild.c b/source/util/src/tstrbuild.c similarity index 100% rename from src/util/src/tstrbuild.c rename to source/util/src/tstrbuild.c diff --git a/src/util/src/tthread.c b/source/util/src/tthread.c similarity index 100% rename from src/util/src/tthread.c rename to source/util/src/tthread.c diff --git a/src/util/src/ttimer.c b/source/util/src/ttimer.c similarity index 100% rename from src/util/src/ttimer.c rename to source/util/src/ttimer.c diff --git a/src/util/src/ttokenizer.c b/source/util/src/ttokenizer.c similarity index 100% rename from src/util/src/ttokenizer.c rename to source/util/src/ttokenizer.c diff --git a/src/util/src/tutil.c b/source/util/src/tutil.c similarity index 100% rename from src/util/src/tutil.c rename to source/util/src/tutil.c diff --git a/src/util/src/tworker.c b/source/util/src/tworker.c similarity index 100% rename from src/util/src/tworker.c rename to source/util/src/tworker.c diff --git a/src/util/src/version.c.in b/source/util/src/version.c.in similarity index 100% rename from src/util/src/version.c.in rename to source/util/src/version.c.in diff --git a/src/util/tests/CMakeLists.txt b/source/util/test/CMakeLists.txt similarity index 100% rename from src/util/tests/CMakeLists.txt rename to source/util/test/CMakeLists.txt diff --git a/src/util/tests/arrayTest.cpp b/source/util/test/arrayTest.cpp similarity index 100% rename from src/util/tests/arrayTest.cpp rename to source/util/test/arrayTest.cpp diff --git a/src/util/tests/cacheTest.cpp b/source/util/test/cacheTest.cpp similarity index 100% rename from src/util/tests/cacheTest.cpp rename to source/util/test/cacheTest.cpp diff --git a/src/util/tests/codingTests.cpp b/source/util/test/codingTests.cpp similarity index 100% rename from src/util/tests/codingTests.cpp rename to source/util/test/codingTests.cpp diff --git a/src/util/tests/hashTest.cpp b/source/util/test/hashTest.cpp similarity index 100% rename from src/util/tests/hashTest.cpp rename to source/util/test/hashTest.cpp diff --git a/src/util/tests/skiplistTest.cpp b/source/util/test/skiplistTest.cpp similarity index 100% rename from src/util/tests/skiplistTest.cpp rename to source/util/test/skiplistTest.cpp diff --git a/src/util/tests/stringTest.cpp b/source/util/test/stringTest.cpp similarity index 100% rename from src/util/tests/stringTest.cpp rename to source/util/test/stringTest.cpp diff --git a/src/util/tests/taosbsearchTest.cpp b/source/util/test/taosbsearchTest.cpp similarity index 100% rename from src/util/tests/taosbsearchTest.cpp rename to source/util/test/taosbsearchTest.cpp diff --git a/src/util/tests/trefTest.c b/source/util/test/trefTest.c similarity index 100% rename from src/util/tests/trefTest.c rename to source/util/test/trefTest.c diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 52ba424fa5..2d2ce38299 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -371,7 +371,7 @@ void tscSaveSubscriptionProgress(void* sub) { char path[256]; sprintf(path, "%s/subscribe", tsDataDir); - if (taosMkDir(path, 0777) != 0) { + if (!taosMkDir(path, 0777)) { tscError("failed to create subscribe dir: %s", path); } diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt deleted file mode 100644 index 57d4890f49..0000000000 --- a/src/common/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -aux_source_directory(source COMMON_SRC) -add_library(common ${COMMON_SRC}) -target_include_directories( - common - PUBLIC "${CMAKE_SOURCE_DIR}/include/common" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" -) -target_include_directories( - common - PRIVATE util -) \ No newline at end of file diff --git a/src/common/inc/tulog.h b/src/common/inc/tulog.h deleted file mode 100644 index 566da40a10..0000000000 --- a/src/common/inc/tulog.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_COMMON_ULOG_H -#define TDENGINE_COMMON_ULOG_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "tlog.h" - -extern int32_t uDebugFlag; -extern int8_t tscEmbedded; - -#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }} -#define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }} - -#define pError(...) { taosPrintLog("APP ERROR ", 255, __VA_ARGS__); } -#define pPrint(...) { taosPrintLog("APP ", 255, __VA_ARGS__); } - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/common/source/common.c b/src/common/source/common.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/src/common/source/common.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/src/os/CMakeLists.txt b/src/os/CMakeLists.txt deleted file mode 100644 index 4d3bf4d7dc..0000000000 --- a/src/os/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -aux_source_directory(source OS_SRC) -add_library(os ${OS_SRC}) -target_include_directories( - os - PUBLIC "${CMAKE_SOURCE_DIR}/include/os" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" -) \ No newline at end of file diff --git a/src/os/inc/osArm32.h b/src/os/inc/osArm32.h deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/os/inc/osArm64.h b/src/os/inc/osArm64.h deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/os/inc/osEok.h b/src/os/inc/osEok.h deleted file mode 100644 index 3ca476f840..0000000000 --- a/src/os/inc/osEok.h +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_OS_EOK_H -#define TDENGINE_OS_EOK_H - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef __APPLE__ - -enum EPOLL_EVENTS - { - EPOLLIN = 0x001, -#define EPOLLIN EPOLLIN - EPOLLPRI = 0x002, -#define EPOLLPRI EPOLLPRI - EPOLLOUT = 0x004, -#define EPOLLOUT EPOLLOUT - EPOLLRDNORM = 0x040, -#define EPOLLRDNORM EPOLLRDNORM - EPOLLRDBAND = 0x080, -#define EPOLLRDBAND EPOLLRDBAND - EPOLLWRNORM = 0x100, -#define EPOLLWRNORM EPOLLWRNORM - EPOLLWRBAND = 0x200, -#define EPOLLWRBAND EPOLLWRBAND - EPOLLMSG = 0x400, -#define EPOLLMSG EPOLLMSG - EPOLLERR = 0x008, -#define EPOLLERR EPOLLERR - EPOLLHUP = 0x010, -#define EPOLLHUP EPOLLHUP - EPOLLRDHUP = 0x2000, -#define EPOLLRDHUP EPOLLRDHUP - EPOLLEXCLUSIVE = 1u << 28, -#define EPOLLEXCLUSIVE EPOLLEXCLUSIVE - EPOLLWAKEUP = 1u << 29, -#define EPOLLWAKEUP EPOLLWAKEUP - EPOLLONESHOT = 1u << 30, -#define EPOLLONESHOT EPOLLONESHOT - EPOLLET = 1u << 31 -#define EPOLLET EPOLLET - }; - -/* Valid opcodes ( "op" parameter ) to issue to epoll_ctl(). */ -#define EPOLL_CTL_ADD 1 /* Add a file descriptor to the interface. */ -#define EPOLL_CTL_DEL 2 /* Remove a file descriptor from the interface. */ -#define EPOLL_CTL_MOD 3 /* Change file descriptor epoll_event structure. */ - - -typedef union epoll_data -{ - void *ptr; - int fd; - uint32_t u32; - uint64_t u64; -} epoll_data_t; - -struct epoll_event -{ - uint32_t events; /* Epoll events */ - epoll_data_t data; /* User data variable */ -}; - -int epoll_create(int size); -int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event); -int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout); -int epoll_close(int epfd); - -#endif // __APPLE__ - -#ifdef __cplusplus -} -#endif - -#endif // _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ - diff --git a/src/os/inc/osInc.h b/src/os/inc/osInc.h deleted file mode 100644 index 9b78110833..0000000000 --- a/src/os/inc/osInc.h +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_OS_INC_H -#define TDENGINE_OS_INC_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if defined(_TD_LINUX_64) || defined(_TD_LINUX_32) || defined(_TD_MIPS_64) || defined(_TD_ARM_32) || defined(_TD_ARM_64) || defined(_TD_DARWIN_64) - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - - #if defined(_TD_DARWIN_64) - #include - #include "osEok.h" - #else - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - - #if !(defined(_ALPINE)) - #include - #endif - #endif -#endif - -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - #include - #include - #include "winsock2.h" - #include - #include - #include - #include - #include - #include "msvcProcess.h" - #include "msvcDirect.h" - #include "msvcFcntl.h" - #include "msvcLibgen.h" - #include "msvcStdio.h" - #include "msvcUnistd.h" - #include "msvcLibgen.h" - #include "sys/msvcStat.h" - #include "sys/msvcTypes.h" -#endif - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/os/inc/osLinux32.h b/src/os/inc/osLinux32.h deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/os/inc/osLinux64.h b/src/os/inc/osLinux64.h deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/os/inc/osMemory.h b/src/os/inc/osMemory.h deleted file mode 100644 index 793992c197..0000000000 --- a/src/os/inc/osMemory.h +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_OS_MEMORY_H -#define TDENGINE_OS_MEMORY_H - -#include "osString.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef TD_JEMALLOC_ENABLED -#include -#endif - -typedef enum { - TAOS_ALLOC_MODE_DEFAULT = 0, - TAOS_ALLOC_MODE_RANDOM_FAIL = 1, - TAOS_ALLOC_MODE_DETECT_LEAK = 2 -} ETaosMemoryAllocMode; - -void taosSetAllocMode(int mode, const char *path, bool autoDump); -void taosDumpMemoryLeak(); - -// used in tsdb module -void * taosTMalloc(size_t size); -void * taosTCalloc(size_t nmemb, size_t size); -void * taosTRealloc(void *ptr, size_t size); -void * taosTZfree(void *ptr); -size_t taosTSizeof(void *ptr); -void taosTMemset(void *ptr, int c); - -// used in other module -#define tmalloc(size) malloc(size) -#define tcalloc(num, size) calloc(num, size) -#define trealloc(ptr, size) realloc(ptr, size) -#define tfree(x) \ - do { \ - if (x) { \ - free((void *)(x)); \ - x = 0; \ - } \ - } while (0) - -#ifdef TAOS_MEM_CHECK - #ifdef TAOS_MEM_CHECK_TEST - void * taosMallocMem(size_t size, const char *file, uint32_t line); - void * taosCallocMem(size_t num, size_t size, const char *file, uint32_t line); - void * taosReallocMem(void *ptr, size_t size, const char *file, uint32_t line); - void taosFreeMem(void *ptr, const char *file, uint32_t line); - char * taosStrdupMem(const char *str, const char *file, uint32_t line); - char * taosStrndupMem(const char *str, size_t size, const char *file, uint32_t line); - ssize_t taosGetlineMem(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line); - #undef tmalloc - #undef tcalloc - #undef trealloc - #undef tfree - #define tmalloc(size) taosMallocMem(size, __FILE__, __LINE__) - #define tcalloc(num, size) taosCallocMem(num, size, __FILE__, __LINE__) - #define trealloc(ptr, size) taosReallocMem(ptr, size, __FILE__, __LINE__) - #define tfree(ptr) taosFreeMem(ptr, __FILE__, __LINE__) - #endif -#endif - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/os/inc/osMips64.h b/src/os/inc/osMips64.h deleted file mode 100644 index ed7b08a311..0000000000 --- a/src/os/inc/osMips64.h +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_OS_MIPS64_H -#define TDENGINE_OS_MIPS64_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/os/inc/osRand.h b/src/os/inc/osRand.h deleted file mode 100644 index e08768c2cc..0000000000 --- a/src/os/inc/osRand.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_OS_RAND_H -#define TDENGINE_OS_RAND_H - -#ifdef __cplusplus -extern "C" { -#endif - -uint32_t taosRand(void); -void taosRandStr(char* str, int32_t size); -uint32_t taosSafeRand(void); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/os/inc/osSignal.h b/src/os/inc/osSignal.h deleted file mode 100644 index eca1d3b3f6..0000000000 --- a/src/os/inc/osSignal.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_OS_SIGNAL_H -#define TDENGINE_OS_SIGNAL_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -#ifndef SIGALRM - #define SIGALRM 1234 -#endif - -#ifndef SIGHUP - #define SIGHUP 1230 -#endif - -#ifndef SIGCHLD - #define SIGCHLD 1234 -#endif - -#ifndef SIGUSR1 - #define SIGUSR1 1234 -#endif - -#ifndef SIGUSR2 - #define SIGUSR2 1234 -#endif - -#ifndef SIGBREAK - #define SIGBREAK 1234 -#endif - -typedef void (*FSignalHandler)(int32_t signum, void *sigInfo, void *context); -void taosSetSignal(int32_t signum, FSignalHandler sigfp); -void taosIgnSignal(int32_t signum); -void taosDflSignal(int32_t signum); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_OS_SIGNAL_H diff --git a/src/os/inc/osSleep.h b/src/os/inc/osSleep.h deleted file mode 100644 index e42da8d5a6..0000000000 --- a/src/os/inc/osSleep.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_OS_SLEEP_H -#define TDENGINE_OS_SLEEP_H - -#ifdef __cplusplus -extern "C" { -#endif - -void taosMsleep(int32_t ms); - -#ifdef __cplusplus -} -#endif - -#endif \ No newline at end of file diff --git a/src/os/inc/osSysinfo.h b/src/os/inc/osSysinfo.h deleted file mode 100644 index 0320ab0f7f..0000000000 --- a/src/os/inc/osSysinfo.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_OS_SYSINFO_H -#define TDENGINE_OS_SYSINFO_H - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - int64_t tsize; - int64_t used; - int64_t avail; -} SysDiskSize; - -int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize); - -int32_t taosGetCpuCores(); -void taosGetSystemInfo(); -bool taosReadProcIO(int64_t* rchars, int64_t* wchars); -bool taosGetProcIO(float *readKB, float *writeKB); -bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes); -bool taosGetBandSpeed(float *bandSpeedKb); -void taosGetDisk(); -bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) ; -bool taosGetProcMemory(float *memoryUsedMB) ; -bool taosGetSysMemory(float *memoryUsedMB); -void taosPrintOsInfo(); -void taosPrintDiskInfo(); -int taosSystem(const char * cmd) ; -void taosKillSystem(); -bool taosGetSystemUid(char *uid); -char *taosGetCmdlineByPID(int pid); - -void taosSetCoreDump(); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/os/source/os.c b/src/os/source/os.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/src/os/source/os.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/src/os/src/darwin/CMakeLists.txt b/src/os/src/darwin/CMakeLists.txt deleted file mode 100644 index ed75cac03d..0000000000 --- a/src/os/src/darwin/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -AUX_SOURCE_DIRECTORY(. SRC) -ADD_LIBRARY(osdarwin ${SRC}) diff --git a/src/os/src/darwin/darwinSystem.c b/src/os/src/darwin/darwinSystem.c deleted file mode 100644 index f152e36d7b..0000000000 --- a/src/os/src/darwin/darwinSystem.c +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tglobal.h" -#include "tulog.h" - -void* taosLoadDll(const char *filename) { - return NULL; -} - -void* taosLoadSym(void* handle, char* name) { - return NULL; -} - -void taosCloseDll(void *handle) { -} - -int taosSetConsoleEcho(bool on) -{ -#define ECHOFLAGS (ECHO | ECHOE | ECHOK | ECHONL) - int err; - struct termios term; - - if (tcgetattr(STDIN_FILENO, &term) == -1) { - perror("Cannot get the attribution of the terminal"); - return -1; - } - - if (on) - term.c_lflag|=ECHOFLAGS; - else - term.c_lflag &=~ECHOFLAGS; - - err = tcsetattr(STDIN_FILENO,TCSAFLUSH,&term); - if (err == -1 && err == EINTR) { - perror("Cannot set the attribution of the terminal"); - return -1; - } - - return 0; -} - diff --git a/src/os/src/darwin/dwEnv.c b/src/os/src/darwin/dwEnv.c deleted file mode 100644 index 8ac97eacfb..0000000000 --- a/src/os/src/darwin/dwEnv.c +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tglobal.h" - -static const char* expand_like_shell(const char *path) { - static __thread char buf[TSDB_FILENAME_LEN]; - buf[0] = '\0'; - wordexp_t we; - if (wordexp(path, &we, 0)) return "/tmp/taosd"; - if (sizeof(buf)<=snprintf(buf, sizeof(buf), "%s", we.we_wordv[0])) return "/tmp/taosd"; - wordfree(&we); - return buf; -} - -void osInit() { - if (configDir[0] == 0) { - strcpy(configDir, expand_like_shell("/usr/local/etc/taos")); - } - strcpy(tsDataDir, expand_like_shell("/usr/local/var/lib/taos")); - strcpy(tsLogDir, expand_like_shell("/usr/local/var/log/taos")); - strcpy(tsScriptDir, expand_like_shell("/usr/local/etc/taos")); - - strcpy(tsVnodeDir, ""); - strcpy(tsDnodeDir, ""); - strcpy(tsMnodeDir, ""); - strcpy(tsOsName, "Darwin"); -} - diff --git a/src/os/src/darwin/dwEok.c b/src/os/src/darwin/dwEok.c deleted file mode 100644 index 22d9260763..0000000000 --- a/src/os/src/darwin/dwEok.c +++ /dev/null @@ -1,891 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -// fail-fast or let-it-crash philosophy -// https://en.wikipedia.org/wiki/Fail-fast -// https://stackoverflow.com/questions/4393197/erlangs-let-it-crash-philosophy-applicable-elsewhere -// experimentally, we follow log-and-crash here - -#define _DEFAULT_SOURCE -#include "os.h" -#include "osEok.h" - -#include - -// #define BALANCE_CHECK_WHEN_CLOSE - -#ifdef ENABLE_LOG -#define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) -#define E(fmt, ...) do { \ - fprintf(stderr, "%s[%d]%s(): %d[%s]: " fmt "\n", \ - basename(__FILE__), __LINE__, __func__, \ - errno, strerror(errno), \ - ##__VA_ARGS__); \ -} while (0) -#else // !ENABLE_LOG -#define D(fmt, ...) (void)fmt -#define E(fmt, ...) (void)fmt -#endif // ENABLE_LOG - -#define A(statement, fmt, ...) do { \ - if (statement) break; \ - fprintf(stderr, "%s[%d]%s(): assert [%s] failed: %d[%s]: " fmt "\n", \ - basename(__FILE__), __LINE__, __func__, \ - #statement, errno, strerror(errno), \ - ##__VA_ARGS__); \ - abort(); \ -} while (0) - -static int eok_dummy = 0; - -typedef struct ep_over_kq_s ep_over_kq_t; -typedef struct eok_event_s eok_event_t; - -struct ep_over_kq_s { - int kq; - - // !!! - // idx in the eoks list - // used as pseudo-file-desciptor - // must be 'closed' with epoll_close - int idx; - - ep_over_kq_t *next; - - int sv[2]; // 0 for read, 1 for write - - // all registered 'epoll events, key by fd' - int evs_count; - eok_event_t *evs_head; - eok_event_t *evs_tail; - eok_event_t *evs_free; - - // all kev changes list pending to be processed by kevent64 - // key by tuple (ident,filter), ident === fd in this case - struct kevent64_s *kchanges; - int nchanges; - int ichanges; - - // kev eventslist for kevent64 to store active events - // they remain alive among kevent64 calls - struct kevent64_s *kevslist; - int nevslist; - - pthread_mutex_t lock; - - volatile unsigned int lock_valid:1; - volatile unsigned int waiting:1; - volatile unsigned int changed:1; - volatile unsigned int wakenup:1; - volatile unsigned int stopping:1; -}; - -struct eok_event_s { - int fd; - struct epoll_event epev; - volatile unsigned int changed; // 0:registered;1:add;2:mod;3:del - - eok_event_t *next; - eok_event_t *prev; -}; - -typedef struct eoks_s eoks_t; -struct eoks_s { - pthread_mutex_t lock; - ep_over_kq_t **eoks; // note: this memory leaks when process terminates - int neoks; // we can add an extra api to let user clean - ep_over_kq_t *eoks_free_list; // currently, we just keep it simple stupid -}; - -static eoks_t eoks = { - .lock = PTHREAD_MUTEX_INITIALIZER, - .eoks = NULL, - .neoks = 0, - .eoks_free_list = NULL, -}; - -#ifdef ENABLE_LOG -static const char* op_str(int op) { - switch (op) { - case EPOLL_CTL_ADD: return "EPOLL_CTL_ADD"; - case EPOLL_CTL_MOD: return "EPOLL_CTL_MOD"; - case EPOLL_CTL_DEL: return "EPOLL_CTL_DEL"; - default: return "UNKNOWN"; - } -} - -static __thread char buf_slots[10][1024] = {0}; -static __thread int buf_slots_linelen = sizeof(buf_slots[0])/sizeof(buf_slots[0][0]); -static __thread int buf_slots_count = sizeof(buf_slots)/(sizeof(buf_slots[0])/sizeof(buf_slots[0][0])); - -static const char* events_str(uint32_t events, int slots) { - A(slots>=0 && slots on linux - // EPOLLIN = 0x001, - // #define EPOLLIN EPOLLIN - // EPOLLPRI = 0x002, - // #define EPOLLPRI EPOLLPRI - // EPOLLOUT = 0x004, - // #define EPOLLOUT EPOLLOUT - // EPOLLRDNORM = 0x040, - // #define EPOLLRDNORM EPOLLRDNORM - // EPOLLRDBAND = 0x080, - // #define EPOLLRDBAND EPOLLRDBAND - // EPOLLWRNORM = 0x100, - // #define EPOLLWRNORM EPOLLWRNORM - // EPOLLWRBAND = 0x200, - // #define EPOLLWRBAND EPOLLWRBAND - // EPOLLMSG = 0x400, - // #define EPOLLMSG EPOLLMSG - // EPOLLERR = 0x008, - // #define EPOLLERR EPOLLERR - // EPOLLHUP = 0x010, - // #define EPOLLHUP EPOLLHUP - // EPOLLRDHUP = 0x2000, - // #define EPOLLRDHUP EPOLLRDHUP - // EPOLLEXCLUSIVE = 1u << 28, - // #define EPOLLEXCLUSIVE EPOLLEXCLUSIVE - // EPOLLWAKEUP = 1u << 29, - // #define EPOLLWAKEUP EPOLLWAKEUP - // EPOLLONESHOT = 1u << 30, - // #define EPOLLONESHOT EPOLLONESHOT - // EPOLLET = 1u << 31 - // #define EPOLLET EPOLLET -#define CHK_EV(ev) \ - if (len>0 && (events & (ev))==(ev)) { \ - n = snprintf(p, len, "%s%s", p!=buf ? "|" : "", #ev); \ - p += n; \ - len -= n; \ - } - CHK_EV(EPOLLIN); - CHK_EV(EPOLLPRI); - CHK_EV(EPOLLOUT); - CHK_EV(EPOLLRDNORM); - CHK_EV(EPOLLRDBAND); - CHK_EV(EPOLLWRNORM); - CHK_EV(EPOLLWRBAND); - CHK_EV(EPOLLMSG); - CHK_EV(EPOLLERR); - CHK_EV(EPOLLHUP); - CHK_EV(EPOLLRDHUP); - CHK_EV(EPOLLEXCLUSIVE); - CHK_EV(EPOLLWAKEUP); - CHK_EV(EPOLLONESHOT); - CHK_EV(EPOLLET); -#undef CHK_EV - return buf; -} - -static const char* kev_flags_str(uint16_t flags, int slots) { - A(slots>=0 && slots - // #define EV_ADD 0x0001 /* add event to kq (implies enable) */ - // #define EV_DELETE 0x0002 /* delete event from kq */ - // #define EV_ENABLE 0x0004 /* enable event */ - // #define EV_DISABLE 0x0008 /* disable event (not reported) */ - // /* flags */ - // #define EV_ONESHOT 0x0010 /* only report one occurrence */ - // #define EV_CLEAR 0x0020 /* clear event state after reporting */ - // #define EV_RECEIPT 0x0040 /* force immediate event output */ - // /* ... with or without EV_ERROR */ - // /* ... use KEVENT_FLAG_ERROR_EVENTS */ - // /* on syscalls supporting flags */ - // #define EV_DISPATCH 0x0080 /* disable event after reporting */ - // #define EV_UDATA_SPECIFIC 0x0100 /* unique kevent per udata value */ - // #define EV_DISPATCH2 (EV_DISPATCH | EV_UDATA_SPECIFIC) - // /* ... in combination with EV_DELETE */ - // /* will defer delete until udata-specific */ - // /* event enabled. EINPROGRESS will be */ - // /* returned to indicate the deferral */ - // #define EV_VANISHED 0x0200 /* report that source has vanished */ - // /* ... only valid with EV_DISPATCH2 */ - // #define EV_SYSFLAGS 0xF000 /* reserved by system */ - // #define EV_FLAG0 0x1000 /* filter-specific flag */ - // #define EV_FLAG1 0x2000 /* filter-specific flag */ - // /* returned values */ - // #define EV_EOF 0x8000 /* EOF detected */ - // #define EV_ERROR 0x4000 /* error, data contains errno */ -#define CHK_EV(ev) \ - if (len>0 && (flags & (ev))==(ev)) { \ - n = snprintf(p, len, "%s%s", p!=buf ? "|" : "", #ev); \ - p += n; \ - len -= n; \ - } - CHK_EV(EV_ADD); - CHK_EV(EV_DELETE); - CHK_EV(EV_ENABLE); - CHK_EV(EV_DISABLE); - CHK_EV(EV_ONESHOT); - CHK_EV(EV_CLEAR); - CHK_EV(EV_RECEIPT); - CHK_EV(EV_DISPATCH); - CHK_EV(EV_UDATA_SPECIFIC); - CHK_EV(EV_DISPATCH2); - CHK_EV(EV_VANISHED); - CHK_EV(EV_SYSFLAGS); - CHK_EV(EV_FLAG0); - CHK_EV(EV_FLAG1); - CHK_EV(EV_EOF); - CHK_EV(EV_ERROR); -#undef CHK_EV - return buf; -} -#endif // ENABLE_LOG - -static ep_over_kq_t* eoks_alloc(void); -static void eoks_free(ep_over_kq_t *eok); -static ep_over_kq_t* eoks_find(int epfd); - -static eok_event_t* eok_find_ev(ep_over_kq_t *eok, int fd); -static eok_event_t* eok_calloc_ev(ep_over_kq_t *eok); -static void eok_free_ev(ep_over_kq_t *eok, eok_event_t *ev); -static void eok_wakeup(ep_over_kq_t *eok); - -static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev, struct kevent64_s *krev, struct kevent64_s *kwev); - -static struct kevent64_s* eok_alloc_eventslist(ep_over_kq_t *eok, int maxevents); - -int epoll_create(int size) { - (void)size; - int e = 0; - ep_over_kq_t *eok = eoks_alloc(); - if (!eok) return -1; - - A(eok->kq==-1, "internal logic error"); - A(eok->lock_valid, "internal logic error"); - A(eok->idx>=0 && eok->idxnext==NULL, "internal logic error"); - A(eok->sv[0]==-1, "internal logic error"); - A(eok->sv[1]==-1, "internal logic error"); - - eok->kq = kqueue(); - if (eok->kq==-1) { - e = errno; - eoks_free(eok); - errno = e; - return -1; - } - - if (socketpair(AF_LOCAL, SOCK_STREAM, 0, eok->sv)) { - e = errno; - eoks_free(eok); - errno = e; - return -1; - } - - struct epoll_event ev = {0}; - ev.events = EPOLLIN; - ev.data.ptr = &eok_dummy; - D("epoll_create epfd:[%d] and sv0[%d]", eok->idx, eok->sv[0]); - if (epoll_ctl(eok->idx, EPOLL_CTL_ADD, eok->sv[0], &ev)) { - e = errno; - epoll_close(eok->idx); - errno = e; - return -1; - } - - return eok->idx; -} - -int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { - D("epoll_ctling epfd:[%d], op:[%s], fd:[%d], events:[%04x:%s]", - epfd, op_str(op), fd, - event ? event->events : 0, - event ? events_str(event->events, 0) : "NULL"); - int e = 0; - if (epfd<0 || epfd>=eoks.neoks) { - errno = EBADF; - return -1; - } - if (fd==-1) { - errno = EBADF; - return -1; - } - if (event && !(event->events & (EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP | EPOLLOUT))) { - e = ENOTSUP; - return -1; - } - - ep_over_kq_t *eok = eoks_find(epfd); - if (!eok) { - errno = EBADF; - return -1; - } - - A(0==pthread_mutex_lock(&eok->lock), ""); - do { - eok_event_t* oev = eok_find_ev(eok, fd); - if (op==EPOLL_CTL_ADD && oev) { - e = EEXIST; - break; - } - if (op!=EPOLL_CTL_ADD && !oev) { - e = ENOENT; - break; - } - if (op!=EPOLL_CTL_DEL && !event) { - e = EINVAL; - break; - } - - // prepare krev/kwev - struct kevent64_s krev = {0}; - struct kevent64_s kwev = {0}; - krev.ident = -1; - kwev.ident = -1; - uint16_t flags = 0; - // prepare internal eok event - eok_event_t ev = {0}; - ev.fd = fd; - if (event) ev.epev = *event; - struct epoll_event *pev = event; - switch (op) { - case EPOLL_CTL_ADD: { - flags = EV_ADD; - ev.changed = 1; - } break; - case EPOLL_CTL_MOD: { - flags = EV_ADD; - ev.changed = 2; - } break; - case EPOLL_CTL_DEL: { - // event is ignored - // pev points to registered epoll_event - pev = &oev->epev; - flags = EV_DELETE; - ev.changed = 3; - } break; - default: { - e = ENOTSUP; - } break; - } - - if (e) break; - - // udata will be delayed to be set - if (pev->events & (EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP)) { - flags |= EV_EOF; - EV_SET64(&krev, ev.fd, EVFILT_READ, flags, 0, 0, -1, 0, 0); - } - if (pev->events & EPOLLOUT) { - EV_SET64(&kwev, ev.fd, EVFILT_WRITE, flags, 0, 0, -1, 0, 0); - } - - // refresh registered evlist and changelist in a transaction way - if (eok_chgs_refresh(eok, oev, &ev, &krev, &kwev)) { - e = errno; - A(e, "internal logic error"); - break; - } - eok->changed = 1; - eok_wakeup(eok); - } while (0); - A(0==pthread_mutex_unlock(&eok->lock), ""); - - if (e) { - errno = e; - return -1; - } - - return 0; -} - -static struct timespec do_timespec_diff(struct timespec *from, struct timespec *to); - -int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) { - int e = 0; - if (!events) { - errno = EINVAL; - E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); - return -1; - } - if (maxevents<=0) { - errno = EINVAL; - E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); - return -1; - } - - struct timespec abstime = {0}; - A(TIME_UTC==timespec_get(&abstime, TIME_UTC), "internal logic error"); - - if (timeout!=-1) { - if (timeout<0) timeout = 0; - int64_t t = abstime.tv_nsec + timeout * 1000000; - abstime.tv_sec += t / 1000000000; - abstime.tv_nsec = t % 1000000000; - } - - int r = 0; - - ep_over_kq_t *eok = eoks_find(epfd); - if (!eok) { - errno = EBADF; - E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); - errno = EBADF; - return -1; - } - - int cnts = 0; - A(0==pthread_mutex_lock(&eok->lock), ""); - do { - cnts = 0; - A(eok->waiting==0, "internal logic error"); - struct kevent64_s *eventslist = eok_alloc_eventslist(eok, maxevents); - if (!eventslist) { - e = ENOMEM; - E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); - break; - } - memset(eventslist, 0, maxevents * sizeof(*eventslist)); - - struct timespec now = {0}; - A(TIME_UTC==timespec_get(&now, TIME_UTC), "internal logic error"); - struct timespec to = do_timespec_diff(&now, &abstime); - struct timespec *pto = &to; - if (timeout==-1) { - pto = NULL; - } - - // taking the changelist - struct kevent64_s *kchanges = eok->kchanges; - int nchanges = eok->nchanges; - int ichanges = eok->ichanges; - // let outside world to add changes - eok->kchanges = NULL; - eok->nchanges = 0; - eok->ichanges = 0; - - eok->changed = 0; - eok->wakenup = 0; - eok->waiting = 1; - - A(0==pthread_mutex_unlock(&eok->lock), ""); - if (ichanges>0) { - D("kevent64 epfd[%d] changing [%d] changes and waiting...", eok->idx, ichanges); - } - errno = 0; - r = kevent64(eok->kq, kchanges, ichanges, eventslist, maxevents, 0, pto); - e = errno; - if (e) { - E("kevent64 epfd[%d] waiting done, with r[%d]", eok->idx, r); - } - A(0==pthread_mutex_lock(&eok->lock), ""); - - eok->waiting = 0; - - if (kchanges) { - if (eok->kchanges==NULL) { - // reuse - A(eok->nchanges==0 && eok->ichanges==0, "internal logic error"); - eok->kchanges = kchanges; - eok->nchanges = nchanges; - } else { - free(kchanges); - kchanges = NULL; - } - nchanges = 0; - ichanges = 0; - } - - if (r==0) { - A(timeout!=-1, "internal logic error"); - } - for (int i=0; iudata && eok->evs_head && eok->evs_tail, "internal logic error"); - eok_event_t *ev = (eok_event_t*)kev->udata; - A(kev->ident == ev->fd, "internal logic error"); - if (kev->flags & EV_ERROR) { - D("epfd[%d] error when processing change list for fd[%d], error[%s], kev_flags:[%04x:%s]", - epfd, ev->fd, strerror(kev->data), kev->flags, kev_flags_str(kev->flags, 0)); - } - switch (kev->filter) { - case EVFILT_READ: { - A((ev->epev.events & EPOLLIN), "internal logic errro"); - if (ev->epev.data.ptr==&eok_dummy) { - // it's coming from wakeup socket pair - char c = '\0'; - A(1==recv(kev->ident, &c, 1, 0), "internal logic error"); - A(0==memcmp(&c, "1", 1), "internal logic error"); - D("epfd[%d] wokenup", epfd); - continue; - } else { - if (ev->changed==3) { - D("epfd[%d] already requested to delete for fd[%d]", epfd, ev->fd); - // TODO: write a unit test for this case - // EV_DELETE? - continue; - } - // converting to epoll_event - // we shall collect all kevents for the uniq fd into one epoll_evnt - // but currently, taos never use EPOLLOUT - // just let it this way for the moment - struct epoll_event pev = {0}; - pev.data.ptr = ev->epev.data.ptr; - pev.events = EPOLLIN; - if (kev->flags & EV_EOF) { - // take all these as EOF for the moment - pev.events |= (EPOLLHUP | EPOLLERR | EPOLLRDHUP); - } - // rounded to what user care - pev.events = pev.events & ev->epev.events; - D("epfd[%d] events found for fd[%d]: [%04x:%s], which was registered: [%04x:%s], kev_flags: [%04x:%s]", - epfd, - ev->fd, pev.events, events_str(pev.events, 0), - ev->epev.events, events_str(ev->epev.events, 1), - kev->flags, kev_flags_str(kev->flags, 2)); - // now we get ev and store it - events[cnts++] = pev; - } - } break; - case EVFILT_WRITE: { - A(0, "not implemented yet"); - } break; - default: { - A(0, "internal logic error"); - } break; - } - } - if (r>=0) { - // we can safely rule out delete-requested-events from the regitered evlists - // if only changelist are correctly registered - eok_event_t *p = eok->evs_head; - while (p) { - eok_event_t *next = p->next; - if (p->changed==3) { - D("epfd[%d] removing registered event for fd[%d]: [%04x:%s]", epfd, p->fd, p->epev.events, events_str(p->epev.events, 0)); - eok_free_ev(eok, p); - } - p = next; - } - } - if (cnts==0) { - // if no user-cared-events is up - // we check to see if time is up - if (timeout!=-1) { - A(TIME_UTC==timespec_get(&now, TIME_UTC), "internal logic error"); - to = do_timespec_diff(&now, &abstime); - if (to.tv_sec==0 && to.tv_nsec==0) break; - } - // time is not up yet, continue loop - } - } while (cnts==0); - if (cnts>0) { - D("kevent64 epfd[%d] waiting done with [%d] events", epfd, cnts); - } - A(0==pthread_mutex_unlock(&eok->lock), ""); - - if (e) { - errno = e; - E("epfd[%d] epoll_wait failed", epfd); - return -1; - } - - // tell user how many events are valid - return cnts; -} - -static struct timespec do_timespec_diff(struct timespec *from, struct timespec *to) { - struct timespec delta; - delta.tv_sec = to->tv_sec - from->tv_sec; - delta.tv_nsec = to->tv_nsec - from->tv_nsec; - // norm and round up - while (delta.tv_nsec<0) { - delta.tv_sec -= 1; - delta.tv_nsec += 1000000000; - } - if (delta.tv_sec < 0) { - delta.tv_sec = 0; - delta.tv_nsec = 0; - } - return delta; -} - -int epoll_close(int epfd) { - D("epoll_closing epfd: [%d]", epfd); - ep_over_kq_t *eok = eoks_find(epfd); - if (!eok) { - errno = EBADF; - return -1; - } - - A(0==pthread_mutex_lock(&eok->lock), ""); - do { - // panic if it would be double-closed - A(eok->stopping==0, "internal logic error"); - eok->stopping = 1; - // panic if epoll_wait is pending - A(eok->waiting==0, "internal logic error"); - - if (eok->kq!=-1) { - close(eok->kq); - eok->kq = -1; - } - } while (0); - A(0==pthread_mutex_unlock(&eok->lock), ""); - eoks_free(eok); - - return 0; -} - -static struct kevent64_s* eok_alloc_eventslist(ep_over_kq_t *eok, int maxevents) { - if (maxevents<=eok->nevslist) return eok->kevslist; - struct kevent64_s *p = (struct kevent64_s*)realloc(eok->kevslist, sizeof(*p)*maxevents); - if (!p) return NULL; - eok->kevslist = p; - eok->nevslist = maxevents; - return p; -} - -static eok_event_t* eok_find_ev(ep_over_kq_t *eok, int fd) { - eok_event_t *p = eok->evs_head; - while (p) { - if (p->fd == fd) return p; - p = p->next; - } - errno = ENOENT; - return NULL; -} - -static eok_event_t* eok_calloc_ev(ep_over_kq_t *eok) { - eok_event_t *p = NULL; - if (eok->evs_free) { - p = eok->evs_free; - eok->evs_free = p->next; - p->next = NULL; - A(p->prev==NULL, "internal logic error"); - } else { - p = (eok_event_t*)calloc(1, sizeof(*p)); - if (!p) return NULL; - A(p->next==NULL, "internal logic error"); - A(p->prev==NULL, "internal logic error"); - } - // dirty link - p->prev = eok->evs_tail; - if (eok->evs_tail) eok->evs_tail->next = p; - else eok->evs_head = p; - eok->evs_tail = p; - - eok->evs_count += 1; - - return p; -} - -static void eok_free_ev(ep_over_kq_t *eok, eok_event_t *ev) { - if (ev->prev) ev->prev->next = ev->next; - else eok->evs_head = ev->next; - if (ev->next) ev->next->prev = ev->prev; - else eok->evs_tail = ev->prev; - ev->prev = NULL; - ev->next = eok->evs_free; - eok->evs_free = ev; - - eok->evs_count -= 1; -} - -static void eok_wakeup(ep_over_kq_t *eok) { - if (!eok->waiting) return; - if (eok->wakenup) return; - eok->wakenup = 1; - A(1==send(eok->sv[1], "1", 1, 0), ""); -} - -static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev, struct kevent64_s *krev, struct kevent64_s *kwev) { - if (!oev) oev = eok_calloc_ev(eok); - if (!oev) return -1; - int n = 0; - if (krev->ident==ev->fd) ++n; - if (kwev->ident==ev->fd) ++n; - A(n, "internal logic error"); - if (eok->ichanges+n>eok->nchanges) { - struct kevent64_s *p = (struct kevent64_s*)realloc(eok->kchanges, sizeof(*p) * (eok->nchanges + 10)); - if (!p) { - if (ev->changed==1) { - // roll back - A(oev, "internal logic error"); - eok_free_ev(eok, oev); - } - errno = ENOMEM; - return -1; - } - eok->kchanges = p; - eok->nchanges += 10; - } - - // copy to registered event slot - oev->fd = ev->fd; - if (ev->changed!=3) { - // if add/mod, copy epoll_event - oev->epev = ev->epev; - } - oev->changed = ev->changed; - - // copy to changes list - n = 0; - if (krev->ident==ev->fd) { - krev->udata = (uint64_t)oev; - eok->kchanges[eok->ichanges++] = *krev; - ++n; - } - if (kwev->ident==ev->fd) { - kwev->udata = (uint64_t)oev; - eok->kchanges[eok->ichanges++] = *kwev; - ++n; - } - D("epfd[%d]: add #changes[%d] for fd[%d], and now #changes/registers [%d/%d]", eok->idx, n, ev->fd, eok->ichanges, eok->evs_count); - return 0; -} - -static ep_over_kq_t* eoks_alloc(void) { - ep_over_kq_t *eok = NULL; - - A(0==pthread_mutex_lock(&eoks.lock), ""); - do { - if (eoks.eoks_free_list) { - eok = eoks.eoks_free_list; - eoks.eoks_free_list = eok->next; - A(eoks.eoks, "internal logic error"); - A(eok->idx>=0 && eok->idxidx)==NULL, "internal logic error"); - *(eoks.eoks + eok->idx) = eok; - eok->next = NULL; - eok->stopping = 0; - break; - } - eok = (ep_over_kq_t*)calloc(1, sizeof(*eok)); - if (!eok) break; - eok->idx = -1; - ep_over_kq_t **ar = (ep_over_kq_t**)realloc(eoks.eoks, sizeof(**ar) * (eoks.neoks+1)); - if (!ar) break; - eoks.eoks = ar; - *(eoks.eoks + eoks.neoks) = eok; - eok->idx = eoks.neoks; - eok->kq = -1; - eok->sv[0] = -1; - eok->sv[1] = -1; - eoks.neoks += 1; - } while (0); - A(0==pthread_mutex_unlock(&eoks.lock), ""); - - if (!eok) { - errno = ENOMEM; - return NULL; - } - if (eok->idx==-1) { - free(eok); - errno = ENOMEM; - return NULL; - } - if (eok->lock_valid) { - return eok; - } - - if (0==pthread_mutex_init(&eok->lock, NULL)) { - eok->lock_valid = 1; - return eok; - } - - eoks_free(eok); - errno = ENOMEM; - return NULL; -} - -static void eoks_free(ep_over_kq_t *eok) { - A(0==pthread_mutex_lock(&eoks.lock), ""); - do { - A(eok->idx>=0 && eok->idxnext==NULL, "internal logic error"); - - // leave eok->kchanges as is - A(eok->ichanges==0, "internal logic error"); - - A(eok->waiting==0, "internal logic error"); - eok_event_t *ev = eok->evs_head; - int sv_closed = 0; - while (ev) { - eok_event_t *next = ev->next; - if (ev->fd==eok->sv[0]) { - // fd is critical system resource - A(sv_closed==0, "internal logic error"); - close(eok->sv[0]); - eok->sv[0] = -1; - close(eok->sv[1]); - eok->sv[1] = -1; - eok_free_ev(eok, ev); - } else { - // user forget calling epoll_ctl(EPOLL_CTL_DEL) before calling epoll_close/close? - // calling close(ev->fd) here smells really bad -#ifndef BALANCE_CHECK_WHEN_CLOSE - // we just let it be and reclaim ev - eok_free_ev(eok, ev); -#else - // panic otherwise, if BALANCE_CHECK_WHEN_CLOSE is defined - A(eok->evs_head==NULL && eok->evs_tail==NULL && eok->evs_count==0, - "epfd[%d] fd[%d]: internal logic error: have you epoll_ctl(EPOLL_CTL_DEL) everything before calling epoll_close?", - eok->idx, ev->fd); -#endif - } - ev = next; - } - // if (eok->evs_count==1) { - // A(eok->evs_head && eok->evs_tail && eok->evs_head==eok->evs_tail, "internal logic error"); - // A(eok->evs_head->fd==eok->sv[0] && eok->sv[0]!=-1 && eok->sv[1]!=-1, "internal logic error"); - // // fd is critical system resource - // close(eok->sv[0]); - // eok->sv[0] = -1; - // close(eok->sv[1]); - // eok->sv[1] = -1; - // eok_free_ev(eok, eok->evs_head); - // } - A(eok->evs_head==NULL && eok->evs_tail==NULL && eok->evs_count==0, - "internal logic error: have you epoll_ctl(EPOLL_CTL_DEL) everything before calling epoll_close?"); - A(eok->sv[0]==-1 && eok->sv[1]==-1, "internal logic error"); - if (eok->kq!=-1) { - close(eok->kq); - eok->kq = -1; - } - eok->next = eoks.eoks_free_list; - eoks.eoks_free_list = eok; - *(eoks.eoks + eok->idx) = NULL; - } while (0); - A(0==pthread_mutex_unlock(&eoks.lock), ""); -} - -static ep_over_kq_t* eoks_find(int epfd) { - ep_over_kq_t *eok = NULL; - A(0==pthread_mutex_lock(&eoks.lock), ""); - do { - if (epfd<0 || epfd>=eoks.neoks) { - break; - } - A(eoks.eoks, "internal logic error"); - eok = *(eoks.eoks + epfd); - A(eok->next==NULL, "internal logic error"); - A(eok->lock_valid, "internal logic error"); - } while (0); - A(0==pthread_mutex_unlock(&eoks.lock), ""); - return eok; -} - diff --git a/src/os/src/darwin/dwSocket.c b/src/os/src/darwin/dwSocket.c deleted file mode 100644 index 69a4666d34..0000000000 --- a/src/os/src/darwin/dwSocket.c +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" - -int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen) { - if (level == SOL_SOCKET && optname == SO_SNDBUF) { - return 0; - } - - if (level == SOL_SOCKET && optname == SO_RCVBUF) { - return 0; - } - - return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); -} diff --git a/src/os/src/darwin/dwSysInfo.c b/src/os/src/darwin/dwSysInfo.c deleted file mode 100644 index 54c6fb1d32..0000000000 --- a/src/os/src/darwin/dwSysInfo.c +++ /dev/null @@ -1,273 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tconfig.h" -#include "tglobal.h" -#include "tulog.h" -#include "taoserror.h" -#include -#include - -static void taosGetSystemTimezone() { - SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); - if (cfg_timezone == NULL) return; - if (cfg_timezone->cfgStatus >= TAOS_CFG_CSTATUS_DEFAULT) { - return; - } - - /* load time zone string from /etc/localtime */ - char buf[4096]; - char *tz = NULL; { - int n = readlink("/etc/localtime", buf, sizeof(buf)); - if (n<0) { - uError("read /etc/localtime error, reason:%s", strerror(errno)); - return; - } - buf[n] = '\0'; - for(int i=n-1; i>=0; --i) { - if (buf[i]=='/') { - if (tz) { - tz = buf + i + 1; - break; - } - tz = buf + i + 1; - } - } - if (!tz || 0==strchr(tz, '/')) { - uError("parsing /etc/localtime failed"); - return; - } - - setenv("TZ", tz, 1); - tzset(); - } - - /* - * NOTE: do not remove it. - * Enforce set the correct daylight saving time(DST) flag according - * to current time - */ - time_t tx1 = time(NULL); - struct tm tm1; - localtime_r(&tx1, &tm1); - - /* - * format example: - * - * Asia/Shanghai (CST, +0800) - * Europe/London (BST, +0100) - */ - snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %+03ld00)", - tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], -timezone/3600); - - // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - uWarn("timezone not configured, set to system default:%s", tsTimezone); -} - -/* - * originally from src/os/src/detail/osSysinfo.c - * POSIX format locale string: - * (Language Strings)_(Country/Region Strings).(code_page) - * - * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, - * - * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. - * - * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page - * for libiconv that is employed to convert string in this system. This program will automatically use - * UTF-8 instead as the charset. - * - * In case of windows client, the locale string is not valid POSIX format, user needs to set the - * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is - * CP936, CP437 for English charset. - * - */ -static void taosGetSystemLocale() { // get and set default locale - char sep = '.'; - char *locale = NULL; - - SGlobalCfg *cfg_locale = taosGetConfigOption("locale"); - if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - locale = setlocale(LC_CTYPE, ""); - if (locale == NULL) { - uError("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); - strcpy(tsLocale, "en_US.UTF-8"); - } else { - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - uWarn("locale not configured, set to system default:%s", tsLocale); - } - } - - /* if user does not specify the charset, extract it from locale */ - SGlobalCfg *cfg_charset = taosGetConfigOption("charset"); - if (cfg_charset && cfg_charset->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - char *str = strrchr(tsLocale, sep); - if (str != NULL) { - str++; - - char *revisedCharset = taosCharsetReplace(str); - tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN); - - free(revisedCharset); - uWarn("charset not configured, set to system default:%s", tsCharset); - } else { - strcpy(tsCharset, "UTF-8"); - uWarn("can't get locale and charset from system, set it to UTF-8"); - } - } -} - -void taosPrintOsInfo() { - uInfo(" os pageSize: %" PRId64 "(KB)", tsPageSize / 1024); - // uInfo(" os openMax: %" PRId64, tsOpenMax); - // uInfo(" os streamMax: %" PRId64, tsStreamMax); - uInfo(" os numOfCores: %d", tsNumOfCores); - uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); - - struct utsname buf; - if (uname(&buf)) { - uInfo(" can't fetch os info"); - return; - } - uInfo(" os sysname: %s", buf.sysname); - uInfo(" os nodename: %s", buf.nodename); - uInfo(" os release: %s", buf.release); - uInfo(" os version: %s", buf.version); - uInfo(" os machine: %s", buf.machine); - uInfo("=================================="); -} - -void taosPrintDiskInfo() { - uInfo("=================================="); - uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); - uInfo(" os usedDisk: %f(GB)", tsUsedDataDirGB); - uInfo(" os availDisk: %f(GB)", tsAvailDataDirGB); - uInfo("=================================="); -} - -void taosKillSystem() { - uError("function taosKillSystem, exit!"); - exit(0); -} - -int32_t taosGetCpuCores() { - return sysconf(_SC_NPROCESSORS_ONLN); -} - -void taosGetSystemInfo() { - // taosGetProcInfos(); - - tsNumOfCores = sysconf(_SC_NPROCESSORS_ONLN); - long physical_pages = sysconf(_SC_PHYS_PAGES); - long page_size = sysconf(_SC_PAGESIZE); - tsTotalMemoryMB = physical_pages * page_size / (1024 * 1024); - tsPageSize = page_size; - - // float tmp1, tmp2; - // taosGetSysMemory(&tmp1); - // taosGetProcMemory(&tmp2); - // taosGetDisk(); - // taosGetBandSpeed(&tmp1); - // taosGetCpuUsage(&tmp1, &tmp2); - // taosGetProcIO(&tmp1, &tmp2); - - taosGetSystemTimezone(); - taosGetSystemLocale(); -} - -bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { - if (rchars) *rchars = 0; - if (wchars) *wchars = 0; - return true; -} - -bool taosGetProcIO(float *readKB, float *writeKB) { - *readKB = 0; - *writeKB = 0; - return true; -} - -bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { - if (bytes) *bytes = 0; - if (rbytes) *rbytes = 0; - if (tbytes) *tbytes = 0; - return true; -} - -bool taosGetBandSpeed(float *bandSpeedKb) { - *bandSpeedKb = 0; - return true; -} - -bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { - *sysCpuUsage = 0; - *procCpuUsage = 0; - return true; -} - -bool taosGetProcMemory(float *memoryUsedMB) { - *memoryUsedMB = 0; - return true; -} - -bool taosGetSysMemory(float *memoryUsedMB) { - *memoryUsedMB = 0; - return true; -} - -int taosSystem(const char *cmd) { - uError("un support funtion"); - return -1; -} - -void taosSetCoreDump() {} - -int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) { - struct statvfs info; - if (statvfs(dataDir, &info)) { - uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } else { - diskSize->tsize = info.f_blocks * info.f_frsize; - diskSize->avail = info.f_bavail * info.f_frsize; - diskSize->used = (info.f_blocks - info.f_bfree) * info.f_frsize; - return 0; - } -} - -char cmdline[1024]; - -char *taosGetCmdlineByPID(int pid) { - errno = 0; - - if (proc_pidpath(pid, cmdline, sizeof(cmdline)) <= 0) { - fprintf(stderr, "PID is %d, %s", pid, strerror(errno)); - return strerror(errno); - } - - return cmdline; -} - -bool taosGetSystemUid(char *uid) { - uuid_t uuid = {0}; - uuid_generate(uuid); - // it's caller's responsibility to make enough space for `uid`, that's 36-char + 1-null - uuid_unparse_lower(uuid, uid); - return true; -} - diff --git a/src/os/src/darwin/dwTimer.c b/src/os/src/darwin/dwTimer.c deleted file mode 100644 index d395a7f53f..0000000000 --- a/src/os/src/darwin/dwTimer.c +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -// fail-fast or let-it-crash philosophy -// https://en.wikipedia.org/wiki/Fail-fast -// https://stackoverflow.com/questions/4393197/erlangs-let-it-crash-philosophy-applicable-elsewhere -// experimentally, we follow log-and-crash here - -#define _DEFAULT_SOURCE -#include "os.h" - -#if 1 -#include - -static void (*timer_callback)(int); -static int timer_ms = 0; -static pthread_t timer_thread; -static int timer_kq = -1; -static volatile int timer_stop = 0; - -static void* timer_routine(void *arg) { - (void)arg; - setThreadName("timer"); - - int r = 0; - struct timespec to = {0}; - to.tv_sec = timer_ms / 1000; - to.tv_nsec = (timer_ms % 1000) * 1000000; - while (!timer_stop) { - struct kevent64_s kev[10] = {0}; - r = kevent64(timer_kq, NULL, 0, kev, sizeof(kev)/sizeof(kev[0]), 0, &to); - if (r!=0) { - fprintf(stderr, "==%s[%d]%s()==kevent64 failed\n", basename(__FILE__), __LINE__, __func__); - abort(); - } - timer_callback(SIGALRM); // just mock - } - - return NULL; -} - -int taosInitTimer(void (*callback)(int), int ms) { - int r = 0; - timer_ms = ms; - timer_callback = callback; - - timer_kq = kqueue(); - if (timer_kq==-1) { - fprintf(stderr, "==%s[%d]%s()==failed to create timer kq\n", basename(__FILE__), __LINE__, __func__); - // since no caller of this func checks the return value for the moment - abort(); - } - - r = pthread_create(&timer_thread, NULL, timer_routine, NULL); - if (r) { - fprintf(stderr, "==%s[%d]%s()==failed to create timer thread\n", basename(__FILE__), __LINE__, __func__); - // since no caller of this func checks the return value for the moment - abort(); - } - return 0; -} - -void taosUninitTimer() { - int r = 0; - timer_stop = 1; - r = pthread_join(timer_thread, NULL); - if (r) { - fprintf(stderr, "==%s[%d]%s()==failed to join timer thread\n", basename(__FILE__), __LINE__, __func__); - // since no caller of this func checks the return value for the moment - abort(); - } - close(timer_kq); - timer_kq = -1; -} - -void taos_block_sigalrm(void) { - // we don't know if there's any specific API for SIGALRM to deliver to specific thread - // this implementation relies on kqueue rather than SIGALRM -} -#else -int taosInitTimer(void (*callback)(int), int ms) { - signal(SIGALRM, callback); - - struct itimerval tv; - tv.it_interval.tv_sec = 0; /* my timer resolution */ - tv.it_interval.tv_usec = 1000 * ms; // resolution is in msecond - tv.it_value = tv.it_interval; - - setitimer(ITIMER_REAL, &tv, NULL); - - return 0; -} - -void taosUninitTimer() { - struct itimerval tv = { 0 }; - setitimer(ITIMER_REAL, &tv, NULL); -} - -void taos_block_sigalrm(void) { - // since SIGALRM has been used - // consideration: any better solution? - static __thread int already_set = 0; - if (!already_set) { - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGALRM); - pthread_sigmask(SIG_BLOCK, &set, NULL); - already_set = 1; - } -} -#endif - diff --git a/src/os/src/detail/CMakeLists.txt b/src/os/src/detail/CMakeLists.txt deleted file mode 100644 index ac68cf4cd8..0000000000 --- a/src/os/src/detail/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(.) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc) -AUX_SOURCE_DIRECTORY(. SRC) -SET_SOURCE_FILES_PROPERTIES(osSysinfo.c PROPERTIES COMPILE_FLAGS -w) -SET_SOURCE_FILES_PROPERTIES(osCoredump.c PROPERTIES COMPILE_FLAGS -w) - -ADD_LIBRARY(os ${SRC}) - -IF (TD_LINUX) - TARGET_LINK_LIBRARIES(os oslinux) - IF (TD_ARM_32 OR TD_LINUX_32) - TARGET_LINK_LIBRARIES(os atomic) - ENDIF () -ELSEIF (TD_DARWIN) - TARGET_LINK_LIBRARIES(os osdarwin) -ELSEIF (TD_WINDOWS) - TARGET_LINK_LIBRARIES(os oswindows) -ENDIF () diff --git a/src/os/src/detail/osAtomic.c b/src/os/src/detail/osAtomic.c deleted file mode 100644 index ecda573528..0000000000 --- a/src/os/src/detail/osAtomic.c +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" - -#ifdef _TD_NINGSI_60 -void* atomic_exchange_ptr_impl(void** ptr, void* val ) { - void *old; - do { - old = *ptr; - } while( !__sync_bool_compare_and_swap(ptr, old, val) ); - return old; -} -int8_t atomic_exchange_8_impl(int8_t* ptr, int8_t val ) { - int8_t old; - do { - old = *ptr; - } while( !__sync_bool_compare_and_swap(ptr, old, val) ); - return old; -} -int16_t atomic_exchange_16_impl(int16_t* ptr, int16_t val ) { - int16_t old; - do { - old = *ptr; - } while( !__sync_bool_compare_and_swap(ptr, old, val) ); - return old; -} -int32_t atomic_exchange_32_impl(int32_t* ptr, int32_t val ) { - int32_t old; - do { - old = *ptr; - } while( !__sync_bool_compare_and_swap(ptr, old, val) ); - return old; -} -int64_t atomic_exchange_64_impl(int64_t* ptr, int64_t val ) { - int64_t old; - do { - old = *ptr; - } while( !__sync_bool_compare_and_swap(ptr, old, val) ); - return old; -} -#endif diff --git a/src/os/src/detail/osFail.c b/src/os/src/detail/osFail.c deleted file mode 100644 index 6ddeefc521..0000000000 --- a/src/os/src/detail/osFail.c +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" - -#define RANDOM_NETWORK_FAIL_FACTOR 20 - -#ifdef TAOS_RANDOM_NETWORK_FAIL - -int64_t taosSendRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags) { - if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) { - errno = ECONNRESET; - return -1; - } - - return send(sockfd, buf, len, flags); -} - -int64_t taosSendToRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags, - const struct sockaddr *dest_addr, socklen_t addrlen) { - if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) { - errno = ECONNRESET; - return -1; - } - - return sendto(sockfd, buf, len, flags, dest_addr, addrlen); -} - -int64_t taosReadSocketRandomFail(int32_t fd, void *buf, size_t count) { - if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) { - errno = ECONNRESET; - return -1; - } - - return read(fd, buf, count); -} - -int64_t taosWriteSocketRandomFail(int32_t fd, const void *buf, size_t count) { - if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) { - errno = EINTR; - return -1; - } - - return write(fd, buf, count); -} - -#endif //TAOS_RANDOM_NETWORK_FAIL - -#ifdef TAOS_RANDOM_FILE_FAIL - -static int32_t random_file_fail_factor = 20; -static FILE *fpRandomFileFailOutput = NULL; - -void taosSetRandomFileFailFactor(int32_t factor) { - random_file_fail_factor = factor; -} - -static void close_random_file_fail_output() { - if (fpRandomFileFailOutput != NULL) { - if (fpRandomFileFailOutput != stdout) { - fclose(fpRandomFileFailOutput); - } - fpRandomFileFailOutput = NULL; - } -} - -static void random_file_fail_output_sig(int32_t sig) { - fprintf(fpRandomFileFailOutput, "signal %d received.\n", sig); - - struct sigaction act = {0}; - act.sa_handler = SIG_DFL; - sigaction(sig, &act, NULL); - - close_random_file_fail_output(); - exit(EXIT_FAILURE); -} - -void taosSetRandomFileFailOutput(const char *path) { - if (path == NULL) { - fpRandomFileFailOutput = stdout; - } else if ((fpRandomFileFailOutput = fopen(path, "w")) != NULL) { - atexit(close_random_file_fail_output); - } else { - printf("failed to open random file fail log file '%s', errno=%d\n", path, errno); - return; - } - - struct sigaction act = {0}; - act.sa_handler = random_file_fail_output_sig; - sigaction(SIGFPE, &act, NULL); - sigaction(SIGSEGV, &act, NULL); - sigaction(SIGILL, &act, NULL); -} - -int64_t taosReadFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line) { - if (random_file_fail_factor > 0) { - if (rand() % random_file_fail_factor == 0) { - errno = EIO; - return -1; - } - } - - return taosRead(fd, buf, count); -} - -int64_t taosWriteFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line) { - if (random_file_fail_factor > 0) { - if (rand() % random_file_fail_factor == 0) { - errno = EIO; - return -1; - } - } - - return taosWrite(fd, buf, count); -} - -int64_t taosLSeekRandomFail(int32_t fd, int64_t offset, int32_t whence, const char *file, uint32_t line) { - if (random_file_fail_factor > 0) { - if (rand() % random_file_fail_factor == 0) { - errno = EIO; - return -1; - } - } - - return taosLSeek(fd, offset, whence); -} - -#endif //TAOS_RANDOM_FILE_FAIL diff --git a/src/os/src/detail/osMemory.c b/src/os/src/detail/osMemory.c deleted file mode 100644 index 22954f1523..0000000000 --- a/src/os/src/detail/osMemory.c +++ /dev/null @@ -1,521 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tulog.h" - -#ifdef TAOS_MEM_CHECK - -static ETaosMemoryAllocMode allocMode = TAOS_ALLOC_MODE_DEFAULT; -static FILE* fpAllocLog = NULL; - -//////////////////////////////////////////////////////////////////////////////// -// memory allocator which fails randomly - -extern int32_t taosGetTimestampSec(); -static int32_t startTime = INT32_MAX; - -static bool taosRandomAllocFail(size_t size, const char* file, uint32_t line) { - if (taosGetTimestampSec() < startTime) { - return false; - } - - if (size < 100 * (size_t)1024) { - return false; - } - - if (rand() % 20 != 0) { - return false; - } - - if (fpAllocLog != NULL) { - fprintf(fpAllocLog, "%s:%d: memory allocation of %" PRIzu " bytes will fail.\n", file, line, size); - } - - return true; -} - -static void* taosRandmoMalloc(size_t size, const char* file, uint32_t line) { - return taosRandomAllocFail(size, file, line) ? NULL : malloc(size); -} - -static void* taosRandomCalloc(size_t num, size_t size, const char* file, uint32_t line) { - return taosRandomAllocFail(num * size, file, line) ? NULL : calloc(num, size); -} - -static void* taosRandomRealloc(void* ptr, size_t size, const char* file, uint32_t line) { - return taosRandomAllocFail(size, file, line) ? NULL : realloc(ptr, size); -} - -static char* taosRandomStrdup(const char* str, const char* file, uint32_t line) { - size_t len = strlen(str); - return taosRandomAllocFail(len + 1, file, line) ? NULL : tstrdup(str); -} - -static char* taosRandomStrndup(const char* str, size_t size, const char* file, uint32_t line) { - size_t len = strlen(str); - if (len > size) { - len = size; - } - return taosRandomAllocFail(len + 1, file, line) ? NULL : tstrndup(str, len); -} - -static ssize_t taosRandomGetline(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { - return taosRandomAllocFail(*n, file, line) ? -1 : tgetline(lineptr, n, stream); -} - -//////////////////////////////////////////////////////////////////////////////// -// memory allocator with leak detection - -#define MEMBLK_MAGIC 0x55AA - -typedef struct SMemBlock { - const char* file; - uint16_t line; - uint16_t magic; - uint32_t size; - struct SMemBlock* prev; - struct SMemBlock* next; - // TODO: need pading in 32bit platform - char data[0]; -} SMemBlock; - -static SMemBlock *blocks = NULL; -static uintptr_t lock = 0; - -static void taosAddMemBlock(SMemBlock* blk) { - blk->prev = NULL; - while (atomic_val_compare_exchange_ptr(&lock, 0, 1) != 0); - blk->next = blocks; - if (blocks != NULL) { - blocks->prev = blk; - } - blocks = blk; - atomic_store_ptr(&lock, 0); -} - -static void taosRemoveMemBlock(SMemBlock* blk) { - while (atomic_val_compare_exchange_ptr(&lock, 0, 1) != 0); - - if (blocks == blk) { - blocks = blk->next; - } - if (blk->prev != NULL) { - blk->prev->next = blk->next; - } - if (blk->next != NULL) { - blk->next->prev = blk->prev; - } - - atomic_store_ptr(&lock, 0); - - blk->prev = NULL; - blk->next = NULL; -} - -static void taosFreeDetectLeak(void* ptr, const char* file, uint32_t line) { - if (ptr == NULL) { - return; - } - - SMemBlock* blk = (SMemBlock*)(((char*)ptr) - sizeof(SMemBlock)); - if (blk->magic != MEMBLK_MAGIC) { - if (fpAllocLog != NULL) { - fprintf(fpAllocLog, "%s:%d: memory is allocated by default allocator.\n", file, line); - } - free(ptr); - return; - } - - taosRemoveMemBlock(blk); - free(blk); -} - -static void* taosMallocDetectLeak(size_t size, const char* file, uint32_t line) { - if (size == 0) { - return NULL; - } - - SMemBlock *blk = (SMemBlock*)malloc(size + sizeof(SMemBlock)); - if (blk == NULL) { - return NULL; - } - - if (line > UINT16_MAX && fpAllocLog != NULL) { - fprintf(fpAllocLog, "%s:%d: line number too large.\n", file, line); - } - - if (size > UINT32_MAX && fpAllocLog != NULL) { - fprintf(fpAllocLog, "%s:%d: size too large: %" PRIzu ".\n", file, line, size); - } - - blk->file = file; - blk->line = (uint16_t)line; - blk->magic = MEMBLK_MAGIC; - blk->size = size; - taosAddMemBlock(blk); - - return blk->data; -} - -static void* taosCallocDetectLeak(size_t num, size_t size, const char* file, uint32_t line) { - size *= num; - void* p = taosMallocDetectLeak(size, file, line); - if (p != NULL) { - memset(p, 0, size); - } - return p; -} - -static void* taosReallocDetectLeak(void* ptr, size_t size, const char* file, uint32_t line) { - if (size == 0) { - taosFreeDetectLeak(ptr, file, line); - return NULL; - } - - if (ptr == NULL) { - return taosMallocDetectLeak(size, file, line); - } - - SMemBlock* blk = (SMemBlock *)((char*)ptr) - sizeof(SMemBlock); - if (blk->magic != MEMBLK_MAGIC) { - if (fpAllocLog != NULL) { - fprintf(fpAllocLog, "%s:%d: memory is allocated by default allocator.\n", file, line); - } - return realloc(ptr, size); - } - - taosRemoveMemBlock(blk); - - void* p = realloc(blk, size + sizeof(SMemBlock)); - if (p == NULL) { - taosAddMemBlock(blk); - return NULL; - } - - if (size > UINT32_MAX && fpAllocLog != NULL) { - fprintf(fpAllocLog, "%s:%d: size too large: %" PRIzu ".\n", file, line, size); - } - - blk = (SMemBlock*)p; - blk->size = size; - taosAddMemBlock(blk); - return blk->data; -} - -static char* taosStrdupDetectLeak(const char* str, const char* file, uint32_t line) { - size_t len = strlen(str); - char *p = taosMallocDetectLeak(len + 1, file, line); - if (p != NULL) { - memcpy(p, str, len); - p[len] = 0; - } - return p; -} - -static char* taosStrndupDetectLeak(const char* str, size_t size, const char* file, uint32_t line) { - size_t len = strlen(str); - if (len > size) { - len = size; - } - char *p = taosMallocDetectLeak(len + 1, file, line); - if (p != NULL) { - memcpy(p, str, len); - p[len] = 0; - } - return p; -} - -static ssize_t taosGetlineDetectLeak(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { - char* buf = NULL; - size_t bufSize = 0; - ssize_t size = tgetline(&buf, &bufSize, stream); - if (size != -1) { - if (*n < size + 1) { - void* p = taosReallocDetectLeak(*lineptr, size + 1, file, line); - if (p == NULL) { - free(buf); - return -1; - } - *lineptr = (char*)p; - *n = size + 1; - } - memcpy(*lineptr, buf, size + 1); - } - - free(buf); - return size; -} - -static void taosDumpMemoryLeakImp() { - const char* hex = "0123456789ABCDEF"; - const char* fmt = ":%d: addr=%p, size=%d, content(first 16 bytes)="; - size_t numOfBlk = 0, totalSize = 0; - - if (fpAllocLog == NULL) { - return; - } - - fputs("memory blocks allocated but not freed before exit:\n", fpAllocLog); - - while (atomic_val_compare_exchange_ptr(&lock, 0, 1) != 0); - - for (SMemBlock* blk = blocks; blk != NULL; blk = blk->next) { - ++numOfBlk; - totalSize += blk->size; - - fputs(blk->file, fpAllocLog); - fprintf(fpAllocLog, fmt, blk->line, blk->data, blk->size); - - char sep = '\''; - size_t size = blk->size > 16 ? 16 : blk->size; - for (size_t i = 0; i < size; ++i) { - uint8_t c = (uint8_t)(blk->data[i]); - fputc(sep, fpAllocLog); - sep = ' '; - fputc(hex[c >> 4], fpAllocLog); - fputc(hex[c & 0x0f], fpAllocLog); - } - - fputs("'\n", fpAllocLog); - } - - atomic_store_ptr(&lock, 0); - - fprintf(fpAllocLog, "\nnumber of blocks: %" PRIzu ", total bytes: %" PRIzu "\n", numOfBlk, totalSize); - fflush(fpAllocLog); -} - -static void taosDumpMemoryLeakOnSig(int sig) { - fprintf(fpAllocLog, "signal %d received.\n", sig); - - // restore default signal handler - struct sigaction act = {0}; - act.sa_handler = SIG_DFL; - sigaction(sig, &act, NULL); - - taosDumpMemoryLeakImp(); -} - -//////////////////////////////////////////////////////////////////////////////// -// interface functions - -void* taosMallocMem(size_t size, const char* file, uint32_t line) { - switch (allocMode) { - case TAOS_ALLOC_MODE_DEFAULT: - return malloc(size); - - case TAOS_ALLOC_MODE_RANDOM_FAIL: - return taosRandmoMalloc(size, file, line); - - case TAOS_ALLOC_MODE_DETECT_LEAK: - return taosMallocDetectLeak(size, file, line); - } - return malloc(size); -} - -void* taosCallocMem(size_t num, size_t size, const char* file, uint32_t line) { - switch (allocMode) { - case TAOS_ALLOC_MODE_DEFAULT: - return calloc(num, size); - - case TAOS_ALLOC_MODE_RANDOM_FAIL: - return taosRandomCalloc(num, size, file, line); - - case TAOS_ALLOC_MODE_DETECT_LEAK: - return taosCallocDetectLeak(num, size, file, line); - } - return calloc(num, size); -} - -void* taosReallocMem(void* ptr, size_t size, const char* file, uint32_t line) { - switch (allocMode) { - case TAOS_ALLOC_MODE_DEFAULT: - return realloc(ptr, size); - - case TAOS_ALLOC_MODE_RANDOM_FAIL: - return taosRandomRealloc(ptr, size, file, line); - - case TAOS_ALLOC_MODE_DETECT_LEAK: - return taosReallocDetectLeak(ptr, size, file, line); - } - return realloc(ptr, size); -} - -void taosFreeMem(void* ptr, const char* file, uint32_t line) { - switch (allocMode) { - case TAOS_ALLOC_MODE_DEFAULT: - return free(ptr); - - case TAOS_ALLOC_MODE_RANDOM_FAIL: - return free(ptr); - - case TAOS_ALLOC_MODE_DETECT_LEAK: - return taosFreeDetectLeak(ptr, file, line); - } - return free(ptr); -} - -char* taosStrdupMem(const char* str, const char* file, uint32_t line) { - switch (allocMode) { - case TAOS_ALLOC_MODE_DEFAULT: - return tstrdup(str); - - case TAOS_ALLOC_MODE_RANDOM_FAIL: - return taosRandomStrdup(str, file, line); - - case TAOS_ALLOC_MODE_DETECT_LEAK: - return taosStrdupDetectLeak(str, file, line); - } - return tstrdup(str); -} - -char* taosStrndupMem(const char* str, size_t size, const char* file, uint32_t line) { - switch (allocMode) { - case TAOS_ALLOC_MODE_DEFAULT: - return tstrndup(str, size); - - case TAOS_ALLOC_MODE_RANDOM_FAIL: - return taosRandomStrndup(str, size, file, line); - - case TAOS_ALLOC_MODE_DETECT_LEAK: - return taosStrndupDetectLeak(str, size, file, line); - } - return tstrndup(str, size); -} - -ssize_t taosGetlineMem(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { - switch (allocMode) { - case TAOS_ALLOC_MODE_DEFAULT: - return tgetline(lineptr, n, stream); - - case TAOS_ALLOC_MODE_RANDOM_FAIL: - return taosRandomGetline(lineptr, n, stream, file, line); - - case TAOS_ALLOC_MODE_DETECT_LEAK: - return taosGetlineDetectLeak(lineptr, n, stream, file, line); - } - return tgetline(lineptr, n, stream); -} - -static void taosCloseAllocLog() { - if (fpAllocLog != NULL) { - if (fpAllocLog != stdout) { - fclose(fpAllocLog); - } - fpAllocLog = NULL; - } -} - -void taosSetAllocMode(int mode, const char* path, bool autoDump) { - assert(mode >= TAOS_ALLOC_MODE_DEFAULT); - assert(mode <= TAOS_ALLOC_MODE_DETECT_LEAK); - - if (fpAllocLog != NULL || allocMode != TAOS_ALLOC_MODE_DEFAULT) { - printf("memory allocation mode can only be set once.\n"); - return; - } - - if (path == NULL || path[0] == 0) { - fpAllocLog = stdout; - } else if ((fpAllocLog = fopen(path, "w")) != NULL) { - atexit(taosCloseAllocLog); - } else { - printf("failed to open memory allocation log file '%s', errno=%d\n", path, errno); - return; - } - - allocMode = mode; - - if (mode == TAOS_ALLOC_MODE_RANDOM_FAIL) { - startTime = taosGetTimestampSec() + 10; - return; - } - - if (autoDump && mode == TAOS_ALLOC_MODE_DETECT_LEAK) { - atexit(taosDumpMemoryLeakImp); - - struct sigaction act = {0}; - act.sa_handler = taosDumpMemoryLeakOnSig; - sigaction(SIGFPE, &act, NULL); - sigaction(SIGSEGV, &act, NULL); - sigaction(SIGILL, &act, NULL); - } -} - -void taosDumpMemoryLeak() { - taosDumpMemoryLeakImp(); - taosCloseAllocLog(); -} - -#else // 'TAOS_MEM_CHECK' not defined - -void taosSetAllocMode(int mode, const char* path, bool autoDump) { - // do nothing -} - -void taosDumpMemoryLeak() { - // do nothing -} - -#endif // TAOS_MEM_CHECK - -void *taosTMalloc(size_t size) { - if (size <= 0) return NULL; - - void *ret = malloc(size + sizeof(size_t)); - if (ret == NULL) return NULL; - - *(size_t *)ret = size; - - return (void *)((char *)ret + sizeof(size_t)); -} - -void *taosTCalloc(size_t nmemb, size_t size) { - size_t tsize = nmemb * size; - void * ret = taosTMalloc(tsize); - if (ret == NULL) return NULL; - - taosTMemset(ret, 0); - return ret; -} - -size_t taosTSizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; } - -void taosTMemset(void *ptr, int c) { memset(ptr, c, taosTSizeof(ptr)); } - -void * taosTRealloc(void *ptr, size_t size) { - if (ptr == NULL) return taosTMalloc(size); - - if (size <= taosTSizeof(ptr)) return ptr; - - void * tptr = (void *)((char *)ptr - sizeof(size_t)); - size_t tsize = size + sizeof(size_t); - void* tptr1 = realloc(tptr, tsize); - if (tptr1 == NULL) return NULL; - tptr = tptr1; - - *(size_t *)tptr = size; - - return (void *)((char *)tptr + sizeof(size_t)); -} - -void* taosTZfree(void* ptr) { - if (ptr) { - free((void*)((char*)ptr - sizeof(size_t))); - } - return NULL; -} diff --git a/src/os/src/detail/osSemaphore.c b/src/os/src/detail/osSemaphore.c deleted file mode 100644 index 06907d5258..0000000000 --- a/src/os/src/detail/osSemaphore.c +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" - -#if !defined (_TD_DARWIN_64) - -int32_t tsem_wait(tsem_t* sem) { - int ret = 0; - do { - ret = sem_wait(sem); - } while (ret != 0 && errno == EINTR); - return ret; -} - -#endif - -#if !(defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) || defined (_TD_DARWIN_64)) - -bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; } - -int64_t taosGetSelfPthreadId() { - static __thread int id = 0; - if (id != 0) return id; - id = syscall(SYS_gettid); - return id; -} - -int64_t taosGetPthreadId(pthread_t thread) { return (int64_t)thread; } -void taosResetPthread(pthread_t *thread) { *thread = 0; } -bool taosComparePthread(pthread_t first, pthread_t second) { return first == second; } -int32_t taosGetPId() { return getpid(); } - -int32_t taosGetCurrentAPPName(char *name, int32_t* len) { - const char* self = "/proc/self/exe"; - char path[PATH_MAX] = {0}; - - if (readlink(self, path, PATH_MAX) <= 0) { - return -1; - } - - path[PATH_MAX - 1] = 0; - char* end = strrchr(path, '/'); - if (end == NULL) { - return -1; - } - - ++end; - - strcpy(name, end); - - if (len != NULL) { - *len = strlen(name); - } - - return 0; -} - -#endif diff --git a/src/os/src/detail/osSignal.c b/src/os/src/detail/osSignal.c deleted file mode 100644 index 33cc39e112..0000000000 --- a/src/os/src/detail/osSignal.c +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taosdef.h" -#include "tconfig.h" -#include "tglobal.h" -#include "tulog.h" - -#if !(defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) - -typedef void (*FLinuxSignalHandler)(int32_t signum, siginfo_t *sigInfo, void *context); - -void taosSetSignal(int32_t signum, FSignalHandler sigfp) { - struct sigaction act; memset(&act, 0, sizeof(act)); -#if 1 - act.sa_flags = SA_SIGINFO; - act.sa_sigaction = (FLinuxSignalHandler)sigfp; -#else - act.sa_handler = sigfp; -#endif - sigaction(signum, &act, NULL); -} - -void taosIgnSignal(int32_t signum) { - signal(signum, SIG_IGN); -} - -void taosDflSignal(int32_t signum) { - signal(signum, SIG_DFL); -} - -#endif diff --git a/src/os/src/detail/osString.c b/src/os/src/detail/osString.c deleted file mode 100644 index fdc70b667e..0000000000 --- a/src/os/src/detail/osString.c +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tglobal.h" - -int64_t taosStr2int64(char *str) { - char *endptr = NULL; - return strtoll(str, &endptr, 10); -} - -#if !(defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) - -int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes) { - return wcsncmp((wchar_t *)f1_ucs4, (wchar_t *)f2_ucs4, bytes / TSDB_NCHAR_SIZE); -} - -#endif - -#ifdef USE_LIBICONV -#include "iconv.h" - -int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { - iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); - size_t ucs4_input_len = ucs4_max_len; - size_t outLen = ucs4_max_len; - if (iconv(cd, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) { - iconv_close(cd); - return -1; - } - - iconv_close(cd); - return (int32_t)(ucs4_max_len - outLen); -} - -bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) { - memset(ucs4, 0, ucs4_max_len); - iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); - size_t ucs4_input_len = mbsLength; - size_t outLeft = ucs4_max_len; - if (iconv(cd, &mbs, &ucs4_input_len, &ucs4, &outLeft) == -1) { - iconv_close(cd); - return false; - } - - iconv_close(cd); - if (len != NULL) { - *len = (int32_t)(ucs4_max_len - outLeft); - if (*len < 0) { - return false; - } - } - - return true; -} - -bool taosValidateEncodec(const char *encodec) { - iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC); - if (cd == (iconv_t)(-1)) { - return false; - } - - iconv_close(cd); - return true; -} - -#else - -int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { - mbstate_t state = {0}; - int32_t len = (int32_t)wcsnrtombs(NULL, (const wchar_t **)&ucs4, ucs4_max_len / 4, 0, &state); - if (len < 0) { - return -1; - } - - memset(&state, 0, sizeof(state)); - len = wcsnrtombs(mbs, (const wchar_t **)&ucs4, ucs4_max_len / 4, (size_t)len, &state); - if (len < 0) { - return -1; - } - - return len; -} - -bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) { - memset(ucs4, 0, ucs4_max_len); - mbstate_t state = {0}; - int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state); - *len = retlen; - - return retlen >= 0; -} - -bool taosValidateEncodec(const char *encodec) { - return true; -} - -#endif - -typedef struct CharsetPair { - char *oldCharset; - char *newCharset; -} CharsetPair; - -char *taosCharsetReplace(char *charsetstr) { - CharsetPair charsetRep[] = { - { "utf8", "UTF-8" }, { "936", "CP936" }, - }; - - for (int32_t i = 0; i < tListLen(charsetRep); ++i) { - if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { - return strdup(charsetRep[i].newCharset); - } - } - - return strdup(charsetstr); -} diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c deleted file mode 100644 index 8094358853..0000000000 --- a/src/os/src/detail/osSysinfo.c +++ /dev/null @@ -1,670 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tconfig.h" -#include "tglobal.h" -#include "tulog.h" -#include "taoserror.h" - -#if !(defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) || defined (_TD_DARWIN_64)) - -#define PROCESS_ITEM 12 - -typedef struct { - uint64_t user; - uint64_t nice; - uint64_t system; - uint64_t idle; -} SysCpuInfo; - -typedef struct { - uint64_t utime; // user time - uint64_t stime; // kernel time - uint64_t cutime; // all user time - uint64_t cstime; // all dead time -} ProcCpuInfo; - -static pid_t tsProcId; -static char tsSysNetFile[] = "/proc/net/dev"; -static char tsSysCpuFile[] = "/proc/stat"; -static char tsProcCpuFile[25] = {0}; -static char tsProcMemFile[25] = {0}; -static char tsProcIOFile[25] = {0}; -static float tsPageSizeKB = 0; - -static void taosGetProcInfos() { - tsPageSize = sysconf(_SC_PAGESIZE); - tsOpenMax = sysconf(_SC_OPEN_MAX); - tsStreamMax = sysconf(_SC_STREAM_MAX); - - tsProcId = (pid_t)syscall(SYS_gettid); - tsPageSizeKB = (float)(sysconf(_SC_PAGESIZE)) / 1024; - - snprintf(tsProcMemFile, 25, "/proc/%d/status", tsProcId); - snprintf(tsProcCpuFile, 25, "/proc/%d/stat", tsProcId); - snprintf(tsProcIOFile, 25, "/proc/%d/io", tsProcId); -} - -static int32_t taosGetTotalMemory() { return (int32_t)((float)sysconf(_SC_PHYS_PAGES) * tsPageSizeKB / 1024); } - -bool taosGetSysMemory(float *memoryUsedMB) { - float memoryAvailMB = (float)sysconf(_SC_AVPHYS_PAGES) * tsPageSizeKB / 1024; - *memoryUsedMB = (float)tsTotalMemoryMB - memoryAvailMB; - return true; -} - -bool taosGetProcMemory(float *memoryUsedMB) { - FILE *fp = fopen(tsProcMemFile, "r"); - if (fp == NULL) { - uError("open file:%s failed", tsProcMemFile); - return false; - } - - ssize_t _bytes = 0; - size_t len; - char * line = NULL; - while (!feof(fp)) { - tfree(line); - len = 0; - _bytes = getline(&line, &len, fp); - if ((_bytes < 0) || (line == NULL)) { - break; - } - if (strstr(line, "VmRSS:") != NULL) { - break; - } - } - - if (line == NULL) { - uError("read file:%s failed", tsProcMemFile); - fclose(fp); - return false; - } - - int64_t memKB = 0; - char tmp[10]; - sscanf(line, "%s %" PRId64, tmp, &memKB); - *memoryUsedMB = (float)((double)memKB / 1024); - - tfree(line); - fclose(fp); - return true; -} - -static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { - FILE *fp = fopen(tsSysCpuFile, "r"); - if (fp == NULL) { - uError("open file:%s failed", tsSysCpuFile); - return false; - } - - size_t len; - char * line = NULL; - ssize_t _bytes = getline(&line, &len, fp); - if ((_bytes < 0) || (line == NULL)) { - uError("read file:%s failed", tsSysCpuFile); - fclose(fp); - return false; - } - - char cpu[10] = {0}; - sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, - &cpuInfo->idle); - - tfree(line); - fclose(fp); - return true; -} - -static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { - FILE *fp = fopen(tsProcCpuFile, "r"); - if (fp == NULL) { - uError("open file:%s failed", tsProcCpuFile); - return false; - } - - size_t len = 0; - char * line = NULL; - ssize_t _bytes = getline(&line, &len, fp); - if ((_bytes < 0) || (line == NULL)) { - uError("read file:%s failed", tsProcCpuFile); - fclose(fp); - return false; - } - - for (int i = 0, blank = 0; line[i] != 0; ++i) { - if (line[i] == ' ') blank++; - if (blank == PROCESS_ITEM) { - sscanf(line + i + 1, "%" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, &cpuInfo->utime, &cpuInfo->stime, - &cpuInfo->cutime, &cpuInfo->cstime); - break; - } - } - - tfree(line); - fclose(fp); - return true; -} - -static void taosGetSystemTimezone() { - SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); - if (cfg_timezone == NULL) return; - if (cfg_timezone->cfgStatus >= TAOS_CFG_CSTATUS_DEFAULT) { - return; - } - - /* - * NOTE: do not remove it. - * Enforce set the correct daylight saving time(DST) flag according - * to current time - */ - time_t tx1 = time(NULL); - struct tm tm1; - localtime_r(&tx1, &tm1); - - /* load time zone string from /etc/timezone */ - FILE *f = fopen("/etc/timezone", "r"); - char buf[68] = {0}; - if (f != NULL) { - int len = fread(buf, 64, 1, f); - if (len < 64 && ferror(f)) { - fclose(f); - uError("read /etc/timezone error, reason:%s", strerror(errno)); - return; - } - - fclose(f); - - buf[sizeof(buf) - 1] = 0; - char *lineEnd = strstr(buf, "\n"); - if (lineEnd != NULL) { - *lineEnd = 0; - } - - // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables - if (strlen(buf) > 0) { - setenv("TZ", buf, 1); - } - } - // get and set default timezone - tzset(); - - /* - * get CURRENT time zone. - * system current time zone is affected by daylight saving time(DST) - * - * e.g., the local time zone of London in DST is GMT+01:00, - * otherwise is GMT+00:00 - */ - int32_t tz = (-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR; - tz += daylight; - - /* - * format example: - * - * Asia/Shanghai (CST, +0800) - * Europe/London (BST, +0100) - */ - snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); - - // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - uWarn("timezone not configured, set to system default:%s", tsTimezone); -} - -/* - * POSIX format locale string: - * (Language Strings)_(Country/Region Strings).(code_page) - * - * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, - * - * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. - * - * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page - * for libiconv that is employed to convert string in this system. This program will automatically use - * UTF-8 instead as the charset. - * - * In case of windows client, the locale string is not valid POSIX format, user needs to set the - * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is - * CP936, CP437 for English charset. - * - */ -static void taosGetSystemLocale() { // get and set default locale - char sep = '.'; - char *locale = NULL; - - SGlobalCfg *cfg_locale = taosGetConfigOption("locale"); - if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - locale = setlocale(LC_CTYPE, ""); - if (locale == NULL) { - uError("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); - strcpy(tsLocale, "en_US.UTF-8"); - } else { - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - uWarn("locale not configured, set to system default:%s", tsLocale); - } - } - - /* if user does not specify the charset, extract it from locale */ - SGlobalCfg *cfg_charset = taosGetConfigOption("charset"); - if (cfg_charset && cfg_charset->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - char *str = strrchr(tsLocale, sep); - if (str != NULL) { - str++; - - char *revisedCharset = taosCharsetReplace(str); - tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN); - - free(revisedCharset); - uWarn("charset not configured, set to system default:%s", tsCharset); - } else { - strcpy(tsCharset, "UTF-8"); - uWarn("can't get locale and charset from system, set it to UTF-8"); - } - } -} - -int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); } - -bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { - static uint64_t lastSysUsed = 0; - static uint64_t lastSysTotal = 0; - static uint64_t lastProcTotal = 0; - - SysCpuInfo sysCpu; - ProcCpuInfo procCpu; - if (!taosGetSysCpuInfo(&sysCpu)) { - return false; - } - if (!taosGetProcCpuInfo(&procCpu)) { - return false; - } - - uint64_t curSysUsed = sysCpu.user + sysCpu.nice + sysCpu.system; - uint64_t curSysTotal = curSysUsed + sysCpu.idle; - uint64_t curProcTotal = procCpu.utime + procCpu.stime + procCpu.cutime + procCpu.cstime; - - if (lastSysUsed == 0 || lastSysTotal == 0 || lastProcTotal == 0) { - lastSysUsed = curSysUsed > 1 ? curSysUsed : 1; - lastSysTotal = curSysTotal > 1 ? curSysTotal : 1; - lastProcTotal = curProcTotal > 1 ? curProcTotal : 1; - return false; - } - - if (curSysTotal == lastSysTotal) { - return false; - } - - *sysCpuUsage = (float)((double)(curSysUsed - lastSysUsed) / (double)(curSysTotal - lastSysTotal) * 100); - *procCpuUsage = (float)((double)(curProcTotal - lastProcTotal) / (double)(curSysTotal - lastSysTotal) * 100); - - lastSysUsed = curSysUsed; - lastSysTotal = curSysTotal; - lastProcTotal = curProcTotal; - - return true; -} - -int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) { - struct statvfs info; - if (statvfs(dataDir, &info)) { - uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } else { - diskSize->tsize = info.f_blocks * info.f_frsize; - diskSize->avail = info.f_bavail * info.f_frsize; - diskSize->used = (info.f_blocks - info.f_bfree) * info.f_frsize; - return 0; - } -} - -bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { - *bytes = 0; - FILE *fp = fopen(tsSysNetFile, "r"); - if (fp == NULL) { - uError("open file:%s failed", tsSysNetFile); - return false; - } - - ssize_t _bytes = 0; - size_t len = 2048; - char * line = calloc(1, len); - - while (!feof(fp)) { - memset(line, 0, len); - - int64_t o_rbytes = 0; - int64_t rpackts = 0; - int64_t o_tbytes = 0; - int64_t tpackets = 0; - int64_t nouse1 = 0; - int64_t nouse2 = 0; - int64_t nouse3 = 0; - int64_t nouse4 = 0; - int64_t nouse5 = 0; - int64_t nouse6 = 0; - char nouse0[200] = {0}; - - _bytes = getline(&line, &len, fp); - if (_bytes < 0) - { - break; - } - - line[len - 1] = 0; - - if (strstr(line, "lo:") != NULL) { - continue; - } - - sscanf(line, - "%s %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 - " %" PRId64, - nouse0, &o_rbytes, &rpackts, &nouse1, &nouse2, &nouse3, &nouse4, &nouse5, &nouse6, &o_tbytes, &tpackets); - if (rbytes) *rbytes = o_rbytes; - if (tbytes) *tbytes = o_tbytes; - *bytes += (o_rbytes + o_tbytes); - } - - tfree(line); - fclose(fp); - - return true; -} - -bool taosGetBandSpeed(float *bandSpeedKb) { - static int64_t lastBytes = 0; - static time_t lastTime = 0; - int64_t curBytes = 0; - time_t curTime = time(NULL); - - if (!taosGetCardInfo(&curBytes, NULL, NULL)) { - return false; - } - - if (lastTime == 0 || lastBytes == 0) { - lastTime = curTime; - lastBytes = curBytes; - *bandSpeedKb = 0; - return true; - } - - if (lastTime >= curTime || lastBytes > curBytes) { - lastTime = curTime; - lastBytes = curBytes; - *bandSpeedKb = 0; - return true; - } - - double totalBytes = (double)(curBytes - lastBytes) / 1024 * 8; // Kb - *bandSpeedKb = (float)(totalBytes / (double)(curTime - lastTime)); - - // uInfo("bandwidth lastBytes:%ld, lastTime:%ld, curBytes:%ld, curTime:%ld, - // speed:%f", lastBytes, lastTime, curBytes, curTime, *bandSpeed); - - lastTime = curTime; - lastBytes = curBytes; - - return true; -} - -bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { - FILE *fp = fopen(tsProcIOFile, "r"); - if (fp == NULL) { - uError("open file:%s failed", tsProcIOFile); - return false; - } - - ssize_t _bytes = 0; - size_t len; - char * line = NULL; - char tmp[10]; - int readIndex = 0; - - while (!feof(fp)) { - tfree(line); - len = 0; - _bytes = getline(&line, &len, fp); - if ((_bytes < 0) || (line == NULL)) { - break; - } - if (strstr(line, "rchar:") != NULL) { - sscanf(line, "%s %" PRId64, tmp, rchars); - readIndex++; - } else if (strstr(line, "wchar:") != NULL) { - sscanf(line, "%s %" PRId64, tmp, wchars); - readIndex++; - } else { - } - - if (readIndex >= 2) break; - } - - tfree(line); - fclose(fp); - - if (readIndex < 2) { - uError("read file:%s failed", tsProcIOFile); - return false; - } - - return true; -} - -bool taosGetProcIO(float *readKB, float *writeKB) { - static int64_t lastReadbyte = -1; - static int64_t lastWritebyte = -1; - - int64_t curReadbyte = 0; - int64_t curWritebyte = 0; - - if (!taosReadProcIO(&curReadbyte, &curWritebyte)) { - return false; - } - - if (lastReadbyte == -1 || lastWritebyte == -1) { - lastReadbyte = curReadbyte; - lastWritebyte = curWritebyte; - return false; - } - - *readKB = (float)((double)(curReadbyte - lastReadbyte) / 1024); - *writeKB = (float)((double)(curWritebyte - lastWritebyte) / 1024); - if (*readKB < 0) *readKB = 0; - if (*writeKB < 0) *writeKB = 0; - - lastReadbyte = curReadbyte; - lastWritebyte = curWritebyte; - - return true; -} - -void taosGetSystemInfo() { - taosGetProcInfos(); - - tsNumOfCores = taosGetCpuCores(); - tsTotalMemoryMB = taosGetTotalMemory(); - - float tmp1, tmp2; - taosGetSysMemory(&tmp1); - taosGetProcMemory(&tmp2); - // taosGetDisk(); - taosGetBandSpeed(&tmp1); - taosGetCpuUsage(&tmp1, &tmp2); - taosGetProcIO(&tmp1, &tmp2); - - taosGetSystemTimezone(); - taosGetSystemLocale(); -} - -void taosPrintOsInfo() { - uInfo(" os pageSize: %" PRId64 "(KB)", tsPageSize); - uInfo(" os openMax: %" PRId64, tsOpenMax); - uInfo(" os streamMax: %" PRId64, tsStreamMax); - uInfo(" os numOfCores: %d", tsNumOfCores); - uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); - - struct utsname buf; - if (uname(&buf)) { - uInfo(" can't fetch os info"); - return; - } - uInfo(" os sysname: %s", buf.sysname); - uInfo(" os nodename: %s", buf.nodename); - uInfo(" os release: %s", buf.release); - uInfo(" os version: %s", buf.version); - uInfo(" os machine: %s", buf.machine); -} - -void taosPrintDiskInfo() { - uInfo("=================================="); - uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); - uInfo(" os usedDisk: %f(GB)", tsUsedDataDirGB); - uInfo(" os availDisk: %f(GB)", tsAvailDataDirGB); - uInfo("=================================="); -} - -void taosKillSystem() { - // SIGINT - uInfo("taosd will shut down soon"); - kill(tsProcId, 2); -} - -int taosSystem(const char *cmd) { - FILE *fp; - int res; - char buf[1024]; - if (cmd == NULL) { - uError("taosSystem cmd is NULL!"); - return -1; - } - - if ((fp = popen(cmd, "r")) == NULL) { - uError("popen cmd:%s error: %s", cmd, strerror(errno)); - return -1; - } else { - while (fgets(buf, sizeof(buf), fp)) { - uDebug("popen result:%s", buf); - } - - if ((res = pclose(fp)) == -1) { - uError("close popen file pointer fp error!"); - } else { - uDebug("popen res is :%d", res); - } - - return res; - } -} - -void taosSetCoreDump() { - if (0 == tsEnableCoreFile) { - return; - } - - // 1. set ulimit -c unlimited - struct rlimit rlim; - struct rlimit rlim_new; - if (getrlimit(RLIMIT_CORE, &rlim) == 0) { -#ifndef _ALPINE - uInfo("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); -#else - uInfo("the old unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max); -#endif - rlim_new.rlim_cur = RLIM_INFINITY; - rlim_new.rlim_max = RLIM_INFINITY; - if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) { - uInfo("set unlimited fail, error: %s", strerror(errno)); - rlim_new.rlim_cur = rlim.rlim_max; - rlim_new.rlim_max = rlim.rlim_max; - (void)setrlimit(RLIMIT_CORE, &rlim_new); - } - } - - if (getrlimit(RLIMIT_CORE, &rlim) == 0) { -#ifndef _ALPINE - uInfo("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); -#else - uInfo("the new unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max); -#endif - } - -#ifndef _TD_ARM_ - // 2. set the path for saving core file - struct __sysctl_args args; - - int old_usespid = 0; - size_t old_len = 0; - int new_usespid = 1; - size_t new_len = sizeof(new_usespid); - - int name[] = {CTL_KERN, KERN_CORE_USES_PID}; - - memset(&args, 0, sizeof(struct __sysctl_args)); - args.name = name; - args.nlen = sizeof(name) / sizeof(name[0]); - args.oldval = &old_usespid; - args.oldlenp = &old_len; - args.newval = &new_usespid; - args.newlen = new_len; - - old_len = sizeof(old_usespid); - - if (syscall(SYS__sysctl, &args) == -1) { - uInfo("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno)); - } - - uInfo("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); - - old_usespid = 0; - old_len = 0; - memset(&args, 0, sizeof(struct __sysctl_args)); - args.name = name; - args.nlen = sizeof(name) / sizeof(name[0]); - args.oldval = &old_usespid; - args.oldlenp = &old_len; - - old_len = sizeof(old_usespid); - - if (syscall(SYS__sysctl, &args) == -1) { - uInfo("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno)); - } - - uInfo("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); -#endif -} - -bool taosGetSystemUid(char *uid) { - int fd; - int len = 0; - - fd = open("/proc/sys/kernel/random/uuid", 0); - if (fd < 0) { - return false; - } else { - len = read(fd, uid, TSDB_CLUSTER_ID_LEN); - close(fd); - } - - if (len >= 36) { - uid[36] = 0; - return true; - } - return false; -} - -#endif diff --git a/src/os/src/detail/osTimer.c b/src/os/src/detail/osTimer.c deleted file mode 100644 index 618df8a8ba..0000000000 --- a/src/os/src/detail/osTimer.c +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "ttimer.h" -#include "tulog.h" - -#if !(defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) || defined(_TD_DARWIN_64)) - -static void taosDeleteTimer(void *tharg) { - timer_t *pTimer = tharg; - timer_delete(*pTimer); -} - -static pthread_t timerThread; -static timer_t timerId; -static volatile bool stopTimer = false; -static void *taosProcessAlarmSignal(void *tharg) { - // Block the signal - sigset_t sigset; - sigemptyset(&sigset); - sigaddset(&sigset, SIGALRM); - sigprocmask(SIG_BLOCK, &sigset, NULL); - void (*callback)(int) = tharg; - - struct sigevent sevent = {{0}}; - - setThreadName("tmr"); - - #ifdef _ALPINE - sevent.sigev_notify = SIGEV_THREAD; - sevent.sigev_value.sival_int = syscall(__NR_gettid); - #else - sevent.sigev_notify = SIGEV_THREAD_ID; - sevent._sigev_un._tid = syscall(__NR_gettid); - #endif - - sevent.sigev_signo = SIGALRM; - - if (timer_create(CLOCK_REALTIME, &sevent, &timerId) == -1) { - uError("Failed to create timer"); - } - - pthread_cleanup_push(taosDeleteTimer, &timerId); - - struct itimerspec ts; - ts.it_value.tv_sec = 0; - ts.it_value.tv_nsec = 1000000 * MSECONDS_PER_TICK; - ts.it_interval.tv_sec = 0; - ts.it_interval.tv_nsec = 1000000 * MSECONDS_PER_TICK; - - if (timer_settime(timerId, 0, &ts, NULL)) { - uError("Failed to init timer"); - return NULL; - } - - int signo; - while (!stopTimer) { - if (sigwait(&sigset, &signo)) { - uError("Failed to wait signal: number %d", signo); - continue; - } - /* printf("Signal handling: number %d ......\n", signo); */ - - callback(0); - } - - pthread_cleanup_pop(1); - - return NULL; -} - -int taosInitTimer(void (*callback)(int), int ms) { - pthread_attr_t tattr; - pthread_attr_init(&tattr); - int code = pthread_create(&timerThread, &tattr, taosProcessAlarmSignal, callback); - pthread_attr_destroy(&tattr); - if (code != 0) { - uError("failed to create timer thread"); - return -1; - } else { - uDebug("timer thread:0x%08" PRIx64 " is created", taosGetPthreadId(timerThread)); - } - - return 0; -} - -void taosUninitTimer() { - stopTimer = true; - - uDebug("join timer thread:0x%08" PRIx64, taosGetPthreadId(timerThread)); - pthread_join(timerThread, NULL); -} - -#endif diff --git a/src/os/src/linux/CMakeLists.txt b/src/os/src/linux/CMakeLists.txt deleted file mode 100644 index f60c10b65a..0000000000 --- a/src/os/src/linux/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -AUX_SOURCE_DIRECTORY(. SRC) -ADD_LIBRARY(oslinux ${SRC}) - -TARGET_LINK_LIBRARIES(oslinux m rt z dl) \ No newline at end of file diff --git a/src/os/src/linux/linuxEnv.c b/src/os/src/linux/linuxEnv.c deleted file mode 100644 index 650a45aae4..0000000000 --- a/src/os/src/linux/linuxEnv.c +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tglobal.h" - -void osInit() { -#ifdef _TD_POWER_ - if (configDir[0] == 0) { - strcpy(configDir, "/etc/power"); - } - strcpy(tsDataDir, "/var/lib/power"); - strcpy(tsLogDir, "/var/log/power"); - strcpy(tsScriptDir, "/etc/power"); -#elif (_TD_TQ_ == true) - if (configDir[0] == 0) { - strcpy(configDir, "/etc/tq"); - } - strcpy(tsDataDir, "/var/lib/tq"); - strcpy(tsLogDir, "/var/log/tq"); - strcpy(tsScriptDir, "/etc/tq"); -#else - if (configDir[0] == 0) { - strcpy(configDir, "/etc/taos"); - } - strcpy(tsDataDir, "/var/lib/taos"); - strcpy(tsLogDir, "/var/log/taos"); - strcpy(tsScriptDir, "/etc/taos"); -#endif - - strcpy(tsVnodeDir, ""); - strcpy(tsDnodeDir, ""); - strcpy(tsMnodeDir, ""); - strcpy(tsOsName, "Linux"); -} - -char* taosGetCmdlineByPID(int pid) { - static char cmdline[1024]; - sprintf(cmdline, "/proc/%d/cmdline", pid); - - int fd = open(cmdline, O_RDONLY); - if (fd >= 0) { - int n = read(fd, cmdline, sizeof(cmdline) - 1); - if (n < 0) n = 0; - - if (n > 0 && cmdline[n - 1] == '\n') --n; - - cmdline[n] = 0; - - close(fd); - } else { - cmdline[0] = 0; - } - - return cmdline; -} diff --git a/src/os/src/linux/osSystem.c b/src/os/src/linux/osSystem.c deleted file mode 100644 index a82149dccb..0000000000 --- a/src/os/src/linux/osSystem.c +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tconfig.h" -#include "tglobal.h" -#include "tulog.h" - -void* taosLoadDll(const char *filename) { - void *handle = dlopen (filename, RTLD_LAZY); - if (!handle) { - uError("load dll:%s failed, error:%s", filename, dlerror()); - return NULL; - } - - uDebug("dll %s loaded", filename); - - return handle; -} - -void* taosLoadSym(void* handle, char* name) { - void* sym = dlsym(handle, name); - char* error = NULL; - - if ((error = dlerror()) != NULL) { - uWarn("load sym:%s failed, error:%s", name, dlerror()); - return NULL; - } - - uDebug("sym %s loaded", name) - - return sym; -} - -void taosCloseDll(void *handle) { - if (handle) { - dlclose(handle); - } -} - -int taosSetConsoleEcho(bool on) -{ -#define ECHOFLAGS (ECHO | ECHOE | ECHOK | ECHONL) - int err; - struct termios term; - - if (tcgetattr(STDIN_FILENO, &term) == -1) { - perror("Cannot get the attribution of the terminal"); - return -1; - } - - if (on) - term.c_lflag |= ECHOFLAGS; - else - term.c_lflag &= ~ECHOFLAGS; - - err = tcsetattr(STDIN_FILENO, TCSAFLUSH, &term); - if (err == -1 || err == EINTR) { - perror("Cannot set the attribution of the terminal"); - return -1; - } - - return 0; -} - diff --git a/src/os/src/windows/CMakeLists.txt b/src/os/src/windows/CMakeLists.txt deleted file mode 100644 index 83012d6e3e..0000000000 --- a/src/os/src/windows/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -AUX_SOURCE_DIRECTORY(. SRC) -ADD_LIBRARY(oswindows ${SRC}) - -TARGET_LINK_LIBRARIES(oswindows winmm IPHLPAPI ws2_32 MsvcLibXw) \ No newline at end of file diff --git a/src/os/src/windows/wEnv.c b/src/os/src/windows/wEnv.c deleted file mode 100644 index b35cb8f040..0000000000 --- a/src/os/src/windows/wEnv.c +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tglobal.h" -#include "tulog.h" - -extern void taosWinSocketInit(); - -void osInit() { - taosSetCoreDump(); -#ifdef _TD_POWER_ - if (configDir[0] == 0) { - strcpy(configDir, "C:/PowerDB/cfg"); - } - - strcpy(tsVnodeDir, "C:/PowerDB/data"); - strcpy(tsDataDir, "C:/PowerDB/data"); - strcpy(tsLogDir, "C:/PowerDB/log"); - strcpy(tsScriptDir, "C:/PowerDB/script"); -#elif (_TD_TQ_ == true) - if (configDir[0] == 0) { - strcpy(configDir, "C:/TQ/cfg"); - } - strcpy(tsVnodeDir, "C:/TQ/data"); - strcpy(tsDataDir, "C:/TQ/data"); - strcpy(tsLogDir, "C:/TQ/log"); - strcpy(tsScriptDir, "C:/TQ/script"); -#else - if (configDir[0] == 0) { - strcpy(configDir, "C:/TDengine/cfg"); - } - - strcpy(tsVnodeDir, "C:/TDengine/data"); - strcpy(tsDataDir, "C:/TDengine/data"); - strcpy(tsLogDir, "C:/TDengine/log"); - strcpy(tsScriptDir, "C:/TDengine/script"); -#endif - - strcpy(tsDnodeDir, ""); - strcpy(tsMnodeDir, ""); - strcpy(tsOsName, "Windows"); - - const char *tmpDir = getenv("tmp"); - if (tmpDir == NULL) { - tmpDir = getenv("temp"); - } - - if (tmpDir != NULL) { - strcpy(tsTempDir, tmpDir); - } else { - strcpy(tsTempDir, "C:\\Windows\\Temp"); - } - - taosWinSocketInit(); -} diff --git a/src/os/src/windows/wGetline.c b/src/os/src/windows/wGetline.c deleted file mode 100644 index aa45854884..0000000000 --- a/src/os/src/windows/wGetline.c +++ /dev/null @@ -1,125 +0,0 @@ -/* getline.c -- Replacement for GNU C library function getline - -Copyright (C) 1993 Free Software Foundation, Inc. - -This program is free software; you can redistribute it and/or -modify it under the terms of the GNU General Public License as -published by the Free Software Foundation; either version 2 of the -License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -General Public License for more details. */ - -/* Written by Jan Brittenson, bson@gnu.ai.mit.edu. */ - -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include -#include -#include - -#if STDC_HEADERS -#include -#else -char *malloc(), *realloc(); -#endif - -/* Always add at least this many bytes when extending the buffer. */ -#define MIN_CHUNK 64 - -/* Read up to (and including) a TERMINATOR from STREAM into *LINEPTR -+ OFFSET (and null-terminate it). *LINEPTR is a pointer returned from -malloc (or NULL), pointing to *N characters of space. It is realloc'd -as necessary. Return the number of characters read (not including the -null terminator), or -1 on error or EOF. On a -1 return, the caller -should check feof(), if not then errno has been set to indicate -the error. */ - -int32_t getstr(char **lineptr, size_t *n, FILE *stream, char terminator, int32_t offset) { - int32_t nchars_avail; /* Allocated but unused chars in *LINEPTR. */ - char * read_pos; /* Where we're reading into *LINEPTR. */ - int32_t ret; - - if (!lineptr || !n || !stream) { - errno = EINVAL; - return -1; - } - - if (!*lineptr) { - *n = MIN_CHUNK; - *lineptr = malloc(*n); - if (!*lineptr) { - errno = ENOMEM; - return -1; - } - } - - nchars_avail = (int32_t)(*n - offset); - read_pos = *lineptr + offset; - - for (;;) { - int32_t save_errno; - register int32_t c = getc(stream); - - save_errno = errno; - - /* We always want at least one char left in the buffer, since we - always (unless we get an error while reading the first char) - NUL-terminate the line buffer. */ - - assert((*lineptr + *n) == (read_pos + nchars_avail)); - if (nchars_avail < 2) { - if (*n > MIN_CHUNK) - *n *= 2; - else - *n += MIN_CHUNK; - - nchars_avail = (int32_t)(*n + *lineptr - read_pos); - char* lineptr1 = realloc(*lineptr, *n); - if (!lineptr1) { - errno = ENOMEM; - return -1; - } - *lineptr = lineptr1; - - read_pos = *n - nchars_avail + *lineptr; - assert((*lineptr + *n) == (read_pos + nchars_avail)); - } - - if (ferror(stream)) { - /* Might like to return partial line, but there is no - place for us to store errno. And we don't want to just - lose errno. */ - errno = save_errno; - return -1; - } - - if (c == EOF) { - /* Return partial line, if any. */ - if (read_pos == *lineptr) - return -1; - else - break; - } - - *read_pos++ = c; - nchars_avail--; - - if (c == terminator) /* Return the line. */ - break; - } - - /* Done - NUL terminate and return the number of chars read. */ - *read_pos = '\0'; - - ret = (int32_t)(read_pos - (*lineptr + offset)); - return ret; -} - -int32_t tgetline(char **lineptr, size_t *n, FILE *stream) { return getstr(lineptr, n, stream, '\n', 0); } \ No newline at end of file diff --git a/src/os/src/windows/wSemaphore.c b/src/os/src/windows/wSemaphore.c deleted file mode 100644 index 878ceba791..0000000000 --- a/src/os/src/windows/wSemaphore.c +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE - -#include "os.h" -#include "taosdef.h" -#include "tglobal.h" -#include "ttimer.h" -#include "tulog.h" -#include "tutil.h" -#include - -bool taosCheckPthreadValid(pthread_t thread) { return thread.p != NULL; } - -void taosResetPthread(pthread_t* thread) { thread->p = 0; } - -int64_t taosGetPthreadId(pthread_t thread) { -#ifdef PTW32_VERSION - return pthread_getw32threadid_np(thread); -#else - return (int64_t)thread; -#endif -} - -int64_t taosGetSelfPthreadId() { return GetCurrentThreadId(); } - -bool taosComparePthread(pthread_t first, pthread_t second) { return first.p == second.p; } - -int32_t taosGetPId() { return GetCurrentProcessId(); } - -int32_t taosGetCurrentAPPName(char* name, int32_t* len) { - char filepath[1024] = {0}; - - GetModuleFileName(NULL, filepath, MAX_PATH); - char* sub = strrchr(filepath, '.'); - if (sub != NULL) { - *sub = '\0'; - } - strcpy(name, filepath); - - if (len != NULL) { - *len = (int32_t)strlen(filepath); - } - - return 0; -} diff --git a/src/os/src/windows/wSocket.c b/src/os/src/windows/wSocket.c deleted file mode 100644 index ad0d87418d..0000000000 --- a/src/os/src/windows/wSocket.c +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include -#include -#include -#include -#include -#include -#include - -void taosWinSocketInit() { - static char flag = 0; - if (flag == 0) { - WORD wVersionRequested; - WSADATA wsaData; - wVersionRequested = MAKEWORD(1, 1); - if (WSAStartup(wVersionRequested, &wsaData) == 0) { - flag = 1; - } - } -} - -int32_t taosSetNonblocking(SOCKET sock, int32_t on) { - u_long mode; - if (on) { - mode = 1; - ioctlsocket(sock, FIONBIO, &mode); - } else { - mode = 0; - ioctlsocket(sock, FIONBIO, &mode); - } - return 0; -} - -void taosIgnSIGPIPE() {} -void taosBlockSIGPIPE() {} -void taosSetMaskSIGPIPE() {} - -int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { - if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { - return 0; - } - - if (level == SOL_TCP && optname == TCP_KEEPIDLE) { - return 0; - } - - if (level == SOL_TCP && optname == TCP_KEEPINTVL) { - return 0; - } - - if (level == SOL_TCP && optname == TCP_KEEPCNT) { - return 0; - } - - return setsockopt(socketfd, level, optname, optval, optlen); -} - -#ifdef _MSC_VER -//#if _MSC_VER >= 1900 - -uint32_t taosInetAddr(char *ipAddr) { - uint32_t value; - int32_t ret = inet_pton(AF_INET, ipAddr, &value); - if (ret <= 0) { - return INADDR_NONE; - } else { - return value; - } -} - -const char *taosInetNtoa(struct in_addr ipInt) { - // not thread safe, only for debug usage while print log - static char tmpDstStr[16]; - return inet_ntop(AF_INET, &ipInt, tmpDstStr, INET6_ADDRSTRLEN); -} - -//#endif -#endif - -#if defined(_TD_GO_DLL_) - -uint64_t htonll(uint64_t val) { return (((uint64_t)htonl(val)) << 32) + htonl(val >> 32); } - -#endif diff --git a/src/os/src/windows/wString.c b/src/os/src/windows/wString.c deleted file mode 100644 index 96a604ae6e..0000000000 --- a/src/os/src/windows/wString.c +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taosdef.h" -#include "tglobal.h" -#include "ttimer.h" -#include "tulog.h" -#include "tutil.h" - -/* - * Get next token from string *stringp, where tokens are possibly-empty - * strings separated by characters from delim. - * - * Writes NULs into the string at *stringp to end tokens. - * delim need not remain constant from call to call. - * On return, *stringp points past the last NUL written (if there might - * be further tokens), or is NULL (if there are definitely no moretokens). - * - * If *stringp is NULL, strsep returns NULL. - */ -char *strsep(char **stringp, const char *delim) { - char * s; - const char *spanp; - int32_t c, sc; - char *tok; - if ((s = *stringp) == NULL) - return (NULL); - for (tok = s;;) { - c = *s++; - spanp = delim; - do { - if ((sc = *spanp++) == c) { - if (c == 0) - s = NULL; - else - s[-1] = 0; - *stringp = s; - return (tok); - } - } while (sc != 0); - } - /* NOTREACHED */ -} - -char *getpass(const char *prefix) { - static char passwd[TSDB_KEY_LEN] = {0}; - memset(passwd, 0, TSDB_KEY_LEN); - printf("%s", prefix); - - int32_t index = 0; - char ch; - while (index < TSDB_KEY_LEN) { - ch = getch(); - if (ch == '\n' || ch == '\r') { - break; - } else { - passwd[index++] = ch; - } - } - - return passwd; -} - -int32_t twcslen(const wchar_t *wcs) { - int32_t *wstr = (int32_t *)wcs; - if (NULL == wstr) { - return 0; - } - - int32_t n = 0; - while (1) { - if (0 == *wstr++) { - break; - } - n++; - } - - return n; -} - -int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes) { - for (int32_t i = 0; i < bytes; ++i) { - int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i * 4); - int32_t f2 = *(int32_t *)((char *)f2_ucs4 + i * 4); - - if ((f1 == 0 && f2 != 0) || (f1 != 0 && f2 == 0)) { - return f1 - f2; - } else if (f1 == 0 && f2 == 0) { - return 0; - } - - if (f1 != f2) { - return f1 - f2; - } - } - - return 0; - -#if 0 - int32_t ucs4_max_len = bytes + 4; - char *f1_mbs = calloc(bytes, 1); - char *f2_mbs = calloc(bytes, 1); - if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) { - return -1; - } - if (taosUcs4ToMbs(f2_ucs4, ucs4_max_len, f2_mbs) < 0) { - return -1; - } - int32_t ret = strcmp(f1_mbs, f2_mbs); - free(f1_mbs); - free(f2_mbs); - return ret; -#endif -} - -/* Copy memory to memory until the specified number of bytes -has been copied, return pointer to following byte. -Overlap is NOT handled correctly. */ -void *mempcpy(void *dest, const void *src, size_t len) { - return (char*)memcpy(dest, src, len) + len; -} - -/* Copy SRC to DEST, returning the address of the terminating '\0' in DEST. */ -char *stpcpy (char *dest, const char *src) { - size_t len = strlen (src); - return (char*)memcpy(dest, src, len + 1) + len; -} - -/* Copy no more than N characters of SRC to DEST, returning the address of - the terminating '\0' in DEST, if any, or else DEST + N. */ -char *stpncpy (char *dest, const char *src, size_t n) { - size_t size = strnlen (src, n); - memcpy (dest, src, size); - dest += size; - if (size == n) - return dest; - return memset (dest, '\0', n - size); -} \ No newline at end of file diff --git a/src/os/src/windows/wSysLog.c b/src/os/src/windows/wSysLog.c deleted file mode 100644 index 866cacbaba..0000000000 --- a/src/os/src/windows/wSysLog.c +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" - -void syslog(int unused, const char *format, ...) {} \ No newline at end of file diff --git a/src/os/src/windows/wSysinfo.c b/src/os/src/windows/wSysinfo.c deleted file mode 100644 index 89101ee148..0000000000 --- a/src/os/src/windows/wSysinfo.c +++ /dev/null @@ -1,293 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taosdef.h" -#include "tconfig.h" -#include "tglobal.h" -#include "ttimer.h" -#include "tulog.h" -#include "tutil.h" -#include "taoserror.h" -#if (_WIN64) -#include -#include -#include -#include -#include -#include -#pragma comment(lib, "Mswsock.lib ") -#endif - -#include - -#pragma warning(push) -#pragma warning(disable : 4091) -#include -#pragma warning(pop) - -static int32_t taosGetTotalMemory() { - MEMORYSTATUSEX memsStat; - memsStat.dwLength = sizeof(memsStat); - if (!GlobalMemoryStatusEx(&memsStat)) { - return 0; - } - - float nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f); - return (int32_t)nMemTotal; -} - -bool taosGetSysMemory(float *memoryUsedMB) { - MEMORYSTATUSEX memsStat; - memsStat.dwLength = sizeof(memsStat); - if (!GlobalMemoryStatusEx(&memsStat)) { - return false; - } - - float nMemFree = memsStat.ullAvailPhys / (1024.0f * 1024.0f); - float nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f); - - *memoryUsedMB = nMemTotal - nMemFree; - return true; -} - -bool taosGetProcMemory(float *memoryUsedMB) { - unsigned bytes_used = 0; - -#if defined(_WIN64) && defined(_MSC_VER) - PROCESS_MEMORY_COUNTERS pmc; - HANDLE cur_proc = GetCurrentProcess(); - - if (GetProcessMemoryInfo(cur_proc, &pmc, sizeof(pmc))) { - bytes_used = (unsigned)(pmc.WorkingSetSize + pmc.PagefileUsage); - } -#endif - - *memoryUsedMB = (float)bytes_used / 1024 / 1024; - return true; -} - -static void taosGetSystemTimezone() { - // get and set default timezone - SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); - if (cfg_timezone && cfg_timezone->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - char *tz = getenv("TZ"); - if (tz == NULL || strlen(tz) == 0) { - strcpy(tsTimezone, "not configured"); - } else { - strcpy(tsTimezone, tz); - } - cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - uInfo("timezone not configured, use default"); - } -} - -static void taosGetSystemLocale() { - // get and set default locale - SGlobalCfg *cfg_locale = taosGetConfigOption("locale"); - if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - char *locale = setlocale(LC_CTYPE, "chs"); - if (locale != NULL) { - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - cfg_locale->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - uInfo("locale not configured, set to default:%s", tsLocale); - } - } - - SGlobalCfg *cfg_charset = taosGetConfigOption("charset"); - if (cfg_charset && cfg_charset->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - strcpy(tsCharset, "cp936"); - cfg_charset->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - uInfo("charset not configured, set to default:%s", tsCharset); - } -} - -int32_t taosGetCpuCores() { - SYSTEM_INFO info; - GetSystemInfo(&info); - return (int32_t)info.dwNumberOfProcessors; -} - -bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { - *sysCpuUsage = 0; - *procCpuUsage = 0; - return true; -} - -int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) { - unsigned _int64 i64FreeBytesToCaller; - unsigned _int64 i64TotalBytes; - unsigned _int64 i64FreeBytes; - - BOOL fResult = GetDiskFreeSpaceExA(dataDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes, - (PULARGE_INTEGER)&i64FreeBytes); - if (fResult) { - diskSize->tsize = (int64_t)(i64TotalBytes); - diskSize->avail = (int64_t)(i64FreeBytesToCaller); - diskSize->used = (int64_t)(i64TotalBytes - i64FreeBytes); - return 0; - } else { - uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } -} - -bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { - if (bytes) *bytes = 0; - if (rbytes) *rbytes = 0; - if (tbytes) *tbytes = 0; - return true; -} - -bool taosGetBandSpeed(float *bandSpeedKb) { - *bandSpeedKb = 0; - return true; -} - -bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) { - IO_COUNTERS io_counter; - if (GetProcessIoCounters(GetCurrentProcess(), &io_counter)) { - if (readbyte) *readbyte = io_counter.ReadTransferCount; - if (writebyte) *writebyte = io_counter.WriteTransferCount; - return true; - } - return false; -} - -bool taosGetProcIO(float *readKB, float *writeKB) { - static int64_t lastReadbyte = -1; - static int64_t lastWritebyte = -1; - - int64_t curReadbyte = 0; - int64_t curWritebyte = 0; - - if (!taosReadProcIO(&curReadbyte, &curWritebyte)) { - return false; - } - - if (lastReadbyte == -1 || lastWritebyte == -1) { - lastReadbyte = curReadbyte; - lastWritebyte = curWritebyte; - return false; - } - - *readKB = (float)((double)(curReadbyte - lastReadbyte) / 1024); - *writeKB = (float)((double)(curWritebyte - lastWritebyte) / 1024); - if (*readKB < 0) *readKB = 0; - if (*writeKB < 0) *writeKB = 0; - - lastReadbyte = curReadbyte; - lastWritebyte = curWritebyte; - - return true; -} - -void taosGetSystemInfo() { - tsNumOfCores = taosGetCpuCores(); - tsTotalMemoryMB = taosGetTotalMemory(); - - float tmp1, tmp2; - // taosGetDisk(); - taosGetBandSpeed(&tmp1); - taosGetCpuUsage(&tmp1, &tmp2); - taosGetProcIO(&tmp1, &tmp2); - - taosGetSystemTimezone(); - taosGetSystemLocale(); -} - -void taosPrintOsInfo() { - uInfo(" os numOfCores: %d", tsNumOfCores); - uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); - uInfo("=================================="); -} - -void taosPrintDiskInfo() { - uInfo("=================================="); - uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); - uInfo(" os usedDisk: %f(GB)", tsUsedDataDirGB); - uInfo(" os availDisk: %f(GB)", tsAvailDataDirGB); - uInfo("=================================="); -} - -void taosKillSystem() { - uError("function taosKillSystem, exit!"); - exit(0); -} - -int taosSystem(const char *cmd) { - uError("taosSystem not support"); - return -1; -} - -int flock(int fd, int option) { return 0; } - -LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { - typedef BOOL(WINAPI * FxMiniDumpWriteDump)(IN HANDLE hProcess, IN DWORD ProcessId, IN HANDLE hFile, - IN MINIDUMP_TYPE DumpType, - IN CONST PMINIDUMP_EXCEPTION_INFORMATION ExceptionParam, - IN CONST PMINIDUMP_USER_STREAM_INFORMATION UserStreamParam, - IN CONST PMINIDUMP_CALLBACK_INFORMATION CallbackParam); - - HMODULE dll = LoadLibrary("dbghelp.dll"); - if (dll == NULL) return EXCEPTION_CONTINUE_SEARCH; - FxMiniDumpWriteDump mdwd = (FxMiniDumpWriteDump)(GetProcAddress(dll, "MiniDumpWriteDump")); - if (mdwd == NULL) { - FreeLibrary(dll); - return EXCEPTION_CONTINUE_SEARCH; - } - - TCHAR path[MAX_PATH]; - DWORD len = GetModuleFileName(NULL, path, _countof(path)); - path[len - 3] = 'd'; - path[len - 2] = 'm'; - path[len - 1] = 'p'; - - HANDLE file = CreateFile(path, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); - - MINIDUMP_EXCEPTION_INFORMATION mei; - mei.ThreadId = GetCurrentThreadId(); - mei.ExceptionPointers = ep; - mei.ClientPointers = FALSE; - - (*mdwd)(GetCurrentProcess(), GetCurrentProcessId(), file, MiniDumpWithHandleData, &mei, NULL, NULL); - - CloseHandle(file); - FreeLibrary(dll); - - return EXCEPTION_CONTINUE_SEARCH; -} - -void taosSetCoreDump() { SetUnhandledExceptionFilter(&FlCrashDump); } - -bool taosGetSystemUid(char *uid) { - GUID guid; - CoCreateGuid(&guid); - - sprintf( - uid, - "%08X-%04X-%04X-%02X%02X-%02X%02X%02X%02X%02X%02X", - guid.Data1, guid.Data2, guid.Data3, - guid.Data4[0], guid.Data4[1], - guid.Data4[2], guid.Data4[3], - guid.Data4[4], guid.Data4[5], - guid.Data4[6], guid.Data4[7]); - - return true; -} - -char *taosGetCmdlineByPID(int pid) { return ""; } diff --git a/src/os/src/windows/wSystem.c b/src/os/src/windows/wSystem.c deleted file mode 100644 index 564005f79b..0000000000 --- a/src/os/src/windows/wSystem.c +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tglobal.h" -#include "tulog.h" - -void* taosLoadDll(const char *filename) { - return NULL; -} - -void* taosLoadSym(void* handle, char* name) { - return NULL; -} - -void taosCloseDll(void *handle) { -} - - -int taosSetConsoleEcho(bool on) -{ - HANDLE hStdin = GetStdHandle(STD_INPUT_HANDLE); - DWORD mode = 0; - GetConsoleMode(hStdin, &mode ); - if (on) { - mode |= ENABLE_ECHO_INPUT; - } else { - mode &= ~ENABLE_ECHO_INPUT; - } - SetConsoleMode(hStdin, mode); - - return 0; -} diff --git a/src/os/src/windows/wTime.c b/src/os/src/windows/wTime.c deleted file mode 100644 index 1484b13843..0000000000 --- a/src/os/src/windows/wTime.c +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include - -int gettimeofday(struct timeval *tv, struct timezone *tz) { - time_t t; - t = time(NULL); - SYSTEMTIME st; - GetLocalTime(&st); - - tv->tv_sec = (long)t; - tv->tv_usec = st.wMilliseconds * 1000; - - return 0; -} - -struct tm *localtime_r(const time_t *timep, struct tm *result) { - localtime_s(result, timep); - return result; -} \ No newline at end of file diff --git a/src/os/src/windows/wTimer.c b/src/os/src/windows/wTimer.c deleted file mode 100644 index c5984f915a..0000000000 --- a/src/os/src/windows/wTimer.c +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include -#include -#include - -#pragma warning( disable : 4244 ) - -typedef void (*win_timer_f)(int signo); - -void WINAPI taosWinOnTimer(UINT wTimerID, UINT msg, DWORD_PTR dwUser, DWORD_PTR dwl, DWORD_PTR dw2) { - win_timer_f callback = *((win_timer_f *)&dwUser); - if (callback != NULL) { - callback(0); - } -} - -static MMRESULT timerId; -int taosInitTimer(win_timer_f callback, int ms) { - DWORD_PTR param = *((int64_t *) & callback); - - timerId = timeSetEvent(ms, 1, (LPTIMECALLBACK)taosWinOnTimer, param, TIME_PERIODIC); - if (timerId == 0) { - return -1; - } - return 0; -} - -void taosUninitTimer() { - timeKillEvent(timerId); -} diff --git a/src/os/src/windows/wWordexp.c b/src/os/src/windows/wWordexp.c deleted file mode 100644 index febe22ac8f..0000000000 --- a/src/os/src/windows/wWordexp.c +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taosdef.h" -#include "tglobal.h" -#include "ttimer.h" -#include "tulog.h" -#include "tutil.h" - -int wordexp(char *words, wordexp_t *pwordexp, int flags) { - pwordexp->we_offs = 0; - pwordexp->we_wordc = 1; - pwordexp->we_wordv[0] = pwordexp->wordPos; - - memset(pwordexp->wordPos, 0, 1025); - if (_fullpath(pwordexp->wordPos, words, 1024) == NULL) { - pwordexp->we_wordv[0] = words; - uError("failed to parse relative path:%s to abs path", words); - return -1; - } - - uTrace("parse relative path:%s to abs path:%s", words, pwordexp->wordPos); - return 0; -} - -void wordfree(wordexp_t *pwordexp) {} diff --git a/src/os/tests/CMakeLists.txt b/src/os/tests/CMakeLists.txt deleted file mode 100644 index 3c47764189..0000000000 --- a/src/os/tests/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) -FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib /usr/lib64) -FIND_LIBRARY(LIB_GTEST_SHARED_DIR libgtest.so /usr/lib/ /usr/local/lib /usr/lib64) - -IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR)) - MESSAGE(STATUS "gTest library found, build unit test") - - # GoogleTest requires at least C++11 - SET(CMAKE_CXX_STANDARD 11) - - INCLUDE_DIRECTORIES(/usr/include /usr/local/include) - LINK_DIRECTORIES(/usr/lib /usr/local/lib) - - AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) - - ADD_EXECUTABLE(osTest ${SOURCE_LIST}) - TARGET_LINK_LIBRARIES(osTest taos os tutil common gtest pthread) -ENDIF() diff --git a/src/os/tests/test.cpp b/src/os/tests/test.cpp deleted file mode 100644 index 6b04540615..0000000000 --- a/src/os/tests/test.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include "os.h" -#include -#include -#include - -#include "taos.h" -#include "ttoken.h" -#include "tutil.h" - -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - -// test function in os module -TEST(testCase, parse_time) { - taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); - deltaToUtcInitOnce(); - - // window: 1500000001000, 1500002000000 - // pQueryAttr->interval: interval: 86400000, sliding:3600000 - int64_t key = 1500000001000; - SInterval interval = {0}; - interval.interval = 86400000; - interval.intervalUnit = 'd'; - interval.sliding = 3600000; - interval.slidingUnit = 'h'; - - int64_t s = taosTimeTruncate(key, &interval, TSDB_TIME_PRECISION_MILLI); - ASSERT_TRUE(s + interval.interval >= key); -} - - - diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c index 5994099a0d..2ff9874856 100644 --- a/src/query/src/qExtbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -398,7 +398,7 @@ int32_t columnValueAscendingComparator(char *f1, char *f2, int32_t type, int32_t if (len1 != len2) { return len1 > len2 ? 1 : -1; } else { - int32_t ret = tasoUcs4Compare(varDataVal(f1), varDataVal(f2), len1); + int32_t ret = tasoUcs4Compare(varDataVal(f1), varDataVal(f2), len1, TSDB_NCHAR_SIZE); if (ret == 0) { return 0; } diff --git a/src/tfs/src/tfs.c b/src/tfs/src/tfs.c index 43ccb324b2..ac7dbc7fd2 100644 --- a/src/tfs/src/tfs.c +++ b/src/tfs/src/tfs.c @@ -252,7 +252,7 @@ int tfsMkdirAt(const char *rname, int level, int id) { char aname[TMPNAME_LEN]; snprintf(aname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), rname); - if (taosMkDir(aname, 0755) != 0) { + if (!taosMkDir(aname, 0755)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt deleted file mode 100644 index 42d0bb6316..0000000000 --- a/src/util/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -aux_source_directory(source UTIL_SRC) -add_library(util ${UTIL_SRC}) -target_include_directories( - util - PUBLIC "${CMAKE_SOURCE_DIR}/include/util" - PRIVATE "${CMAKE_SOURCE_DIR}/include" -) -target_link_libraries( - util - PRIVATE os -) \ No newline at end of file diff --git a/src/util/CMakeLists.txt.in b/src/util/CMakeLists.txt.in deleted file mode 100644 index ef304d2fcb..0000000000 --- a/src/util/CMakeLists.txt.in +++ /dev/null @@ -1,44 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/sync/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/rmonotonic/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/TSZ/sz/include) - -AUX_SOURCE_DIRECTORY(src SRC) -ADD_LIBRARY(tutil ${SRC}) - -TARGET_LINK_LIBRARIES(tutil pthread os lz4 z rmonotonic ${VAR_TSZ} ) - - - -IF (TD_LINUX) - TARGET_LINK_LIBRARIES(tutil m rt) - ADD_SUBDIRECTORY(tests) - - FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) - IF (ICONV_INCLUDE_EXIST) - FIND_PATH(ICONV_LIBRARY_A_EXIST libiconv.a /usr/lib/ /usr/local/lib/ /lib64) - FIND_PATH(ICONV_LIBRARY_SO_EXIST libiconv.so /usr/lib/ /usr/local/lib/ /lib64) - IF (ICONV_LIBRARY_A_EXIST OR ICONV_LIBRARY_SO_EXIST) - MESSAGE(STATUS "Use the installed libiconv library") - TARGET_LINK_LIBRARIES(tutil iconv) - ELSE () - # libiconv library is already included in GLIBC, - MESSAGE(STATUS "Use the iconv functions in GLIBC") - ENDIF () - ELSE () - MESSAGE(STATUS "Failed to find iconv, use default encoding method") - ENDIF () - -ELSEIF (TD_WINDOWS) - TARGET_LINK_LIBRARIES(tutil iconv regex winmm IPHLPAPI ws2_32 wepoll) -ELSEIF(TD_DARWIN) - TARGET_LINK_LIBRARIES(tutil m) - TARGET_LINK_LIBRARIES(tutil iconv) -ENDIF() - -IF (TD_STORAGE) - TARGET_LINK_LIBRARIES(tutil storage) -ENDIF () diff --git a/src/util/inc/tidpool.h b/src/util/inc/tidpool.h deleted file mode 100644 index e4439439ce..0000000000 --- a/src/util/inc/tidpool.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TIDPOOL_H -#define TDENGINE_TIDPOOL_H - -#ifdef __cplusplus -extern "C" { -#endif - -void *taosInitIdPool(int maxId); - -int taosUpdateIdPool(void *handle, int maxId); - -int taosIdPoolMaxSize(void *handle); - -int taosAllocateId(void *handle); - -void taosFreeId(void *handle, int id); - -void taosIdPoolCleanUp(void *handle); - -int taosIdPoolNumOfUsed(void *handle); - -bool taosIdPoolMarkStatus(void *handle, int id); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/util/inc/tlog.h b/src/util/inc/tlog.h deleted file mode 100644 index 1f6a81d4b4..0000000000 --- a/src/util/inc/tlog.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TLOG_H -#define TDENGINE_TLOG_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define DEBUG_FATAL 1U -#define DEBUG_ERROR DEBUG_FATAL -#define DEBUG_WARN 2U -#define DEBUG_INFO DEBUG_WARN -#define DEBUG_DEBUG 4U -#define DEBUG_TRACE 8U -#define DEBUG_DUMP 16U - -#define DEBUG_SCREEN 64U -#define DEBUG_FILE 128U - -int32_t taosInitLog(char *logName, int32_t numOfLogLines, int32_t maxFiles); -void taosCloseLog(); -void taosResetLog(); - -void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) -#ifdef __GNUC__ - __attribute__((format(printf, 3, 4))) -#endif -; - -void taosPrintLongString(const char * flags, int32_t dflag, const char *format, ...) -#ifdef __GNUC__ - __attribute__((format(printf, 3, 4))) -#endif -; - -void taosDumpData(unsigned char *msg, int32_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/util/inc/tmempool.h b/src/util/inc/tmempool.h deleted file mode 100644 index f2c6a0ef00..0000000000 --- a/src/util/inc/tmempool.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef TDENGINE_TMEMPOOL_H -#define TDENGINE_TMEMPOOL_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define mpool_h void * - -mpool_h taosMemPoolInit(int maxNum, int blockSize); - -char *taosMemPoolMalloc(mpool_h handle); - -void taosMemPoolFree(mpool_h handle, char *p); - -void taosMemPoolCleanUp(mpool_h handle); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/util/inc/tref.h b/src/util/inc/tref.h deleted file mode 100644 index 085c10c551..0000000000 --- a/src/util/inc/tref.h +++ /dev/null @@ -1,77 +0,0 @@ - -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TREF_H -#define TDENGINE_TREF_H - -#ifdef __cplusplus -extern "C" { -#endif - -// open a reference set, max is the mod used by hash, fp is the pointer to free resource function -// return rsetId which will be used by other APIs. On error, -1 is returned, and terrno is set appropriately -int taosOpenRef(int max, void (*fp)(void *)); - -// close the reference set, refId is the return value by taosOpenRef -// return 0 if success. On error, -1 is returned, and terrno is set appropriately -int taosCloseRef(int refId); - -// add ref, p is the pointer to resource or pointer ID -// return Reference ID(rid) allocated. On error, -1 is returned, and terrno is set appropriately -int64_t taosAddRef(int refId, void *p); - -// remove ref, rid is the reference ID returned by taosAddRef -// return 0 if success. On error, -1 is returned, and terrno is set appropriately -int taosRemoveRef(int rsetId, int64_t rid); - -// acquire ref, rid is the reference ID returned by taosAddRef -// return the resource p. On error, NULL is returned, and terrno is set appropriately -void *taosAcquireRef(int rsetId, int64_t rid); - -// release ref, rid is the reference ID returned by taosAddRef -// return 0 if success. On error, -1 is returned, and terrno is set appropriately -int taosReleaseRef(int rsetId, int64_t rid); - -// return the first reference if rid is 0, otherwise return the next after current reference. -// if return value is NULL, it means list is over(if terrno is set, it means error happens) -void *taosIterateRef(int rsetId, int64_t rid); - -// return the number of references in system -int taosListRef(); - -#define RID_VALID(x) ((x) > 0) - -/* sample code to iterate the refs - -void demoIterateRefs(int rsetId) { - - void *p = taosIterateRef(refId, 0); - while (p) { - // process P - - // get the rid from p - - p = taosIterateRef(rsetId, rid); - } -} - -*/ - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_TREF_H diff --git a/src/util/src/hash.c b/src/util/src/hash.c deleted file mode 100644 index 6577a0a0f4..0000000000 --- a/src/util/src/hash.c +++ /dev/null @@ -1,916 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "hash.h" -#include "tulog.h" -#include "taosdef.h" - -#define EXT_SIZE 1024 - -#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) - -#define DO_FREE_HASH_NODE(_n) \ - do { \ - tfree(_n); \ - } while (0) - -#define FREE_HASH_NODE(_h, _n) \ - do { \ - if ((_h)->freeFp) { \ - (_h)->freeFp(GET_HASH_NODE_DATA(_n)); \ - } \ - \ - DO_FREE_HASH_NODE(_n); \ - } while (0); - -static FORCE_INLINE void __wr_lock(void *lock, int32_t type) { - if (type == HASH_NO_LOCK) { - return; - } - taosWLockLatch(lock); -} - -static FORCE_INLINE void __rd_lock(void *lock, int32_t type) { - if (type == HASH_NO_LOCK) { - return; - } - - taosRLockLatch(lock); -} - -static FORCE_INLINE void __rd_unlock(void *lock, int32_t type) { - if (type == HASH_NO_LOCK) { - return; - } - - taosRUnLockLatch(lock); -} - -static FORCE_INLINE void __wr_unlock(void *lock, int32_t type) { - if (type == HASH_NO_LOCK) { - return; - } - - taosWUnLockLatch(lock); -} - -static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { - int32_t len = MIN(length, HASH_MAX_CAPACITY); - - int32_t i = 4; - while (i < len) i = (i << 1u); - return i; -} - -static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { - SHashNode *pNode = pe->next; - while (pNode) { - if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { - assert(pNode->hashVal == hashVal); - break; - } - - pNode = pNode->next; - } - - return pNode; -} - -/** - * Resize the hash list if the threshold is reached - * - * @param pHashObj - */ -static void taosHashTableResize(SHashObj *pHashObj); - -/** - * @param key key of object for hash, usually a null-terminated string - * @param keyLen length of key - * @param pData actually data. Requires a consecutive memory block, no pointer is allowed in pData. - * Pointer copy causes memory access error. - * @param dsize size of data - * @return SHashNode - */ -static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal); - -/** - * Update the hash node - * - * @param pNode hash node - * @param key key for generate hash value - * @param keyLen key length - * @param pData actual data - * @param dsize size of actual data - * @return hash node - */ -static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { - assert(pNode->keyLen == pNewNode->keyLen); - - pNode->count--; - if (prev != NULL) { - prev->next = pNewNode; - } else { - pe->next = pNewNode; - } - - if (pNode->count <= 0) { - pNewNode->next = pNode->next; - DO_FREE_HASH_NODE(pNode); - } else { - pNewNode->next = pNode; - pe->num++; - atomic_add_fetch_64(&pHashObj->size, 1); - } - - return pNewNode; -} - -/** - * insert the hash node at the front of the linked list - * - * @param pHashObj - * @param pNode - */ -static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode); - -/** - * Check whether the hash table is empty or not. - * - * @param pHashObj the hash table object - * @return if the hash table is empty or not - */ -static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj); - -/** - * Get the next element in hash table for iterator - * @param pIter - * @return - */ - -SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) { - assert(fn != NULL); - if (capacity == 0) { - capacity = 4; - } - - SHashObj *pHashObj = (SHashObj *)calloc(1, sizeof(SHashObj)); - if (pHashObj == NULL) { - uError("failed to allocate memory, reason:%s", strerror(errno)); - return NULL; - } - - // the max slots is not defined by user - pHashObj->capacity = taosHashCapacity((int32_t)capacity); - assert((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); - pHashObj->equalFp = memcmp; - pHashObj->hashFp = fn; - pHashObj->type = type; - pHashObj->enableUpdate = update; - - pHashObj->hashList = (SHashEntry **)calloc(pHashObj->capacity, sizeof(void *)); - if (pHashObj->hashList == NULL) { - free(pHashObj); - uError("failed to allocate memory, reason:%s", strerror(errno)); - return NULL; - } else { - pHashObj->pMemBlock = taosArrayInit(8, sizeof(void *)); - - void *p = calloc(pHashObj->capacity, sizeof(SHashEntry)); - for (int32_t i = 0; i < pHashObj->capacity; ++i) { - pHashObj->hashList[i] = (void *)((char *)p + i * sizeof(SHashEntry)); - } - - taosArrayPush(pHashObj->pMemBlock, &p); - } - - return pHashObj; -} - -void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp) { - if (pHashObj != NULL && fp != NULL) { - pHashObj->equalFp = fp; - } -} - -int32_t taosHashGetSize(const SHashObj *pHashObj) { - if (!pHashObj) { - return 0; - } - return (int32_t)atomic_load_64(&pHashObj->size); -} - -static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { - return taosHashGetSize(pHashObj) == 0; -} - -int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { - uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); - SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal); - if (pNewNode == NULL) { - return -1; - } - - // need the resize process, write lock applied - if (HASH_NEED_RESIZE(pHashObj)) { - __wr_lock(&pHashObj->lock, pHashObj->type); - taosHashTableResize(pHashObj); - __wr_unlock(&pHashObj->lock, pHashObj->type); - } - - __rd_lock(&pHashObj->lock, pHashObj->type); - - int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); - SHashEntry *pe = pHashObj->hashList[slot]; - - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWLockLatch(&pe->latch); - } - - SHashNode *pNode = pe->next; - if (pe->num > 0) { - assert(pNode != NULL); - } else { - assert(pNode == NULL); - } - - SHashNode* prev = NULL; - while (pNode) { - if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { - assert(pNode->hashVal == hashVal); - break; - } - - prev = pNode; - pNode = pNode->next; - } - - if (pNode == NULL) { - // no data in hash table with the specified key, add it into hash table - pushfrontNodeInEntryList(pe, pNewNode); - - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); - } - - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } - - // enable resize - __rd_unlock(&pHashObj->lock, pHashObj->type); - atomic_add_fetch_64(&pHashObj->size, 1); - - return 0; - } else { - // not support the update operation, return error - if (pHashObj->enableUpdate) { - doUpdateHashNode(pHashObj, pe, prev, pNode, pNewNode); - } else { - DO_FREE_HASH_NODE(pNewNode); - } - - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } - - // enable resize - __rd_unlock(&pHashObj->lock, pHashObj->type); - - return pHashObj->enableUpdate ? 0 : -1; - } -} - -void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { - return taosHashGetClone(pHashObj, key, keyLen, NULL, NULL); -} -//TODO(yihaoDeng), merge with taosHashGetClone -void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void** d, size_t *sz) { - if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { - return NULL; - } - - uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); - - // only add the read lock to disable the resize process - __rd_lock(&pHashObj->lock, pHashObj->type); - - int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); - SHashEntry *pe = pHashObj->hashList[slot]; - - // no data, return directly - if (atomic_load_32(&pe->num) == 0) { - __rd_unlock(&pHashObj->lock, pHashObj->type); - return NULL; - } - - char *data = NULL; - - // lock entry - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosRLockLatch(&pe->latch); - } - - if (pe->num > 0) { - assert(pe->next != NULL); - } else { - assert(pe->next == NULL); - } - - SHashNode *pNode = doSearchInEntryList(pHashObj, pe, key, keyLen, hashVal); - if (pNode != NULL) { - if (fp != NULL) { - fp(GET_HASH_NODE_DATA(pNode)); - } - - if (*d == NULL) { - *sz = pNode->dataLen + EXT_SIZE; - *d = calloc(1, *sz); - } else if (*sz < pNode->dataLen){ - *sz = pNode->dataLen + EXT_SIZE; - *d = realloc(*d, *sz); - } - memcpy((char *)(*d), GET_HASH_NODE_DATA(pNode), pNode->dataLen); - // just make runtime happy - if ((*sz) - pNode->dataLen > 0) { - memset((char *)(*d) + pNode->dataLen, 0, (*sz) - pNode->dataLen); - } - - data = GET_HASH_NODE_DATA(pNode); - } - - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosRUnLockLatch(&pe->latch); - } - - __rd_unlock(&pHashObj->lock, pHashObj->type); - return data; -} - -void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d) { - if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { - return NULL; - } - - uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); - - // only add the read lock to disable the resize process - __rd_lock(&pHashObj->lock, pHashObj->type); - - int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); - SHashEntry *pe = pHashObj->hashList[slot]; - - // no data, return directly - if (atomic_load_32(&pe->num) == 0) { - __rd_unlock(&pHashObj->lock, pHashObj->type); - return NULL; - } - - char *data = NULL; - - // lock entry - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosRLockLatch(&pe->latch); - } - - if (pe->num > 0) { - assert(pe->next != NULL); - } else { - assert(pe->next == NULL); - } - - SHashNode *pNode = doSearchInEntryList(pHashObj, pe, key, keyLen, hashVal); - if (pNode != NULL) { - if (fp != NULL) { - fp(GET_HASH_NODE_DATA(pNode)); - } - - if (d != NULL) { - memcpy(d, GET_HASH_NODE_DATA(pNode), pNode->dataLen); - } - - data = GET_HASH_NODE_DATA(pNode); - } - - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosRUnLockLatch(&pe->latch); - } - - __rd_unlock(&pHashObj->lock, pHashObj->type); - return data; -} - -int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { - return taosHashRemoveWithData(pHashObj, key, keyLen, NULL, 0); -} - -int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t dsize) { - if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { - return -1; - } - - uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); - - // disable the resize process - __rd_lock(&pHashObj->lock, pHashObj->type); - - int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); - SHashEntry *pe = pHashObj->hashList[slot]; - - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWLockLatch(&pe->latch); - } - - // double check after locked - if (pe->num == 0) { - assert(pe->next == NULL); - taosWUnLockLatch(&pe->latch); - - __rd_unlock(&pHashObj->lock, pHashObj->type); - return -1; - } - - int code = -1; - SHashNode *pNode = pe->next; - SHashNode *prevNode = NULL; - - while (pNode) { - if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) - break; - - prevNode = pNode; - pNode = pNode->next; - } - - if (pNode) { - code = 0; // it is found - - pNode->count--; - pNode->removed = 1; - if (pNode->count <= 0) { - if (prevNode) { - prevNode->next = pNode->next; - } else { - pe->next = pNode->next; - } - - if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize); - - pe->num--; - atomic_sub_fetch_64(&pHashObj->size, 1); - FREE_HASH_NODE(pHashObj, pNode); - } - } - - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } - - __rd_unlock(&pHashObj->lock, pHashObj->type); - - return code; -} - -int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param) { - if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { - return 0; - } - - // disable the resize process - __rd_lock(&pHashObj->lock, pHashObj->type); - - int32_t numOfEntries = (int32_t)pHashObj->capacity; - for (int32_t i = 0; i < numOfEntries; ++i) { - SHashEntry *pEntry = pHashObj->hashList[i]; - if (pEntry->num == 0) { - continue; - } - - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWLockLatch(&pEntry->latch); - } - - // todo remove the first node - SHashNode *pNode = NULL; - while((pNode = pEntry->next) != NULL) { - if (fp && (!fp(param, GET_HASH_NODE_DATA(pNode)))) { - pEntry->num -= 1; - atomic_sub_fetch_64(&pHashObj->size, 1); - - pEntry->next = pNode->next; - - if (pEntry->num == 0) { - assert(pEntry->next == NULL); - } else { - assert(pEntry->next != NULL); - } - - FREE_HASH_NODE(pHashObj, pNode); - } else { - break; - } - } - - // handle the following node - if (pNode != NULL) { - assert(pNode == pEntry->next); - SHashNode *pNext = NULL; - - while ((pNext = pNode->next) != NULL) { - // not qualified, remove it - if (fp && (!fp(param, GET_HASH_NODE_DATA(pNext)))) { - pNode->next = pNext->next; - pEntry->num -= 1; - atomic_sub_fetch_64(&pHashObj->size, 1); - - if (pEntry->num == 0) { - assert(pEntry->next == NULL); - } else { - assert(pEntry->next != NULL); - } - - FREE_HASH_NODE(pHashObj, pNext); - } else { - pNode = pNext; - } - } - } - - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pEntry->latch); - } - } - - __rd_unlock(&pHashObj->lock, pHashObj->type); - return 0; -} - -void taosHashClear(SHashObj *pHashObj) { - if (pHashObj == NULL) { - return; - } - - SHashNode *pNode, *pNext; - - __wr_lock(&pHashObj->lock, pHashObj->type); - - for (int32_t i = 0; i < pHashObj->capacity; ++i) { - SHashEntry *pEntry = pHashObj->hashList[i]; - if (pEntry->num == 0) { - assert(pEntry->next == 0); - continue; - } - - pNode = pEntry->next; - assert(pNode != NULL); - - while (pNode) { - pNext = pNode->next; - FREE_HASH_NODE(pHashObj, pNode); - - pNode = pNext; - } - - pEntry->num = 0; - pEntry->next = NULL; - } - - pHashObj->size = 0; - __wr_unlock(&pHashObj->lock, pHashObj->type); -} - -void taosHashCleanup(SHashObj *pHashObj) { - if (pHashObj == NULL) { - return; - } - - taosHashClear(pHashObj); - tfree(pHashObj->hashList); - - // destroy mem block - size_t memBlock = taosArrayGetSize(pHashObj->pMemBlock); - for (int32_t i = 0; i < memBlock; ++i) { - void *p = taosArrayGetP(pHashObj->pMemBlock, i); - tfree(p); - } - - taosArrayDestroy(pHashObj->pMemBlock); - - memset(pHashObj, 0, sizeof(SHashObj)); - free(pHashObj); -} - -// for profile only -int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) { - if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { - return 0; - } - - int32_t num = 0; - - for (int32_t i = 0; i < pHashObj->size; ++i) { - SHashEntry *pEntry = pHashObj->hashList[i]; - if (num < pEntry->num) { - num = pEntry->num; - } - } - - return num; -} - -void taosHashTableResize(SHashObj *pHashObj) { - if (!HASH_NEED_RESIZE(pHashObj)) { - return; - } - - // double the original capacity - SHashNode *pNode = NULL; - SHashNode *pNext = NULL; - - int32_t newSize = (int32_t)(pHashObj->capacity << 1u); - if (newSize > HASH_MAX_CAPACITY) { - // uDebug("current capacity:%d, maximum capacity:%d, no resize applied due to limitation is reached", - // pHashObj->capacity, HASH_MAX_CAPACITY); - return; - } - - int64_t st = taosGetTimestampUs(); - void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newSize); - if (pNewEntryList == NULL) { // todo handle error - // uDebug("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity); - return; - } - - pHashObj->hashList = pNewEntryList; - - size_t inc = newSize - pHashObj->capacity; - void * p = calloc(inc, sizeof(SHashEntry)); - - for (int32_t i = 0; i < inc; ++i) { - pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry)); - } - - taosArrayPush(pHashObj->pMemBlock, &p); - - pHashObj->capacity = newSize; - for (int32_t i = 0; i < pHashObj->capacity; ++i) { - SHashEntry *pe = pHashObj->hashList[i]; - - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); - } - - if (pe->num == 0) { - assert(pe->next == NULL); - continue; - } - - while ((pNode = pe->next) != NULL) { - int32_t j = HASH_INDEX(pNode->hashVal, pHashObj->capacity); - if (j != i) { - pe->num -= 1; - pe->next = pNode->next; - - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); - } - - SHashEntry *pNewEntry = pHashObj->hashList[j]; - pushfrontNodeInEntryList(pNewEntry, pNode); - } else { - break; - } - } - - if (pNode != NULL) { - while ((pNext = pNode->next) != NULL) { - int32_t j = HASH_INDEX(pNext->hashVal, pHashObj->capacity); - if (j != i) { - pe->num -= 1; - - pNode->next = pNext->next; - pNext->next = NULL; - - // added into new slot - SHashEntry *pNewEntry = pHashObj->hashList[j]; - - if (pNewEntry->num == 0) { - assert(pNewEntry->next == NULL); - } else { - assert(pNewEntry->next != NULL); - } - - pushfrontNodeInEntryList(pNewEntry, pNext); - } else { - pNode = pNext; - } - } - - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); - } - - } - - } - - int64_t et = taosGetTimestampUs(); - - uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity, - ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); -} - -SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { - SHashNode *pNewNode = malloc(sizeof(SHashNode) + keyLen + dsize); - - if (pNewNode == NULL) { - uError("failed to allocate memory, reason:%s", strerror(errno)); - return NULL; - } - - pNewNode->keyLen = (uint32_t)keyLen; - pNewNode->hashVal = hashVal; - pNewNode->dataLen = (uint32_t) dsize; - pNewNode->count = 1; - pNewNode->removed = 0; - pNewNode->next = NULL; - - memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize); - memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen); - - return pNewNode; -} - -void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode) { - assert(pNode != NULL && pEntry != NULL); - - pNode->next = pEntry->next; - pEntry->next = pNode; - - pEntry->num += 1; -} - -size_t taosHashGetMemSize(const SHashObj *pHashObj) { - if (pHashObj == NULL) { - return 0; - } - - return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj); -} - -FORCE_INLINE void *taosHashGetDataKey(SHashObj *pHashObj, void *data) { - SHashNode * node = GET_HASH_PNODE(data); - return GET_HASH_NODE_KEY(node); -} - -FORCE_INLINE uint32_t taosHashGetDataKeyLen(SHashObj *pHashObj, void *data) { - SHashNode * node = GET_HASH_PNODE(data); - return node->keyLen; -} - - -// release the pNode, return next pNode, and lock the current entry -static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { - - SHashNode *pOld = (SHashNode *)GET_HASH_PNODE(p); - SHashNode *prevNode = NULL; - - *slot = HASH_INDEX(pOld->hashVal, pHashObj->capacity); - SHashEntry *pe = pHashObj->hashList[*slot]; - - // lock entry - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWLockLatch(&pe->latch); - } - - SHashNode *pNode = pe->next; - - while (pNode) { - if (pNode == pOld) - break; - - prevNode = pNode; - pNode = pNode->next; - } - - if (pNode) { - pNode = pNode->next; - while (pNode) { - if (pNode->removed == 0) break; - pNode = pNode->next; - } - - pOld->count--; - if (pOld->count <=0) { - if (prevNode) { - prevNode->next = pOld->next; - } else { - pe->next = pOld->next; - } - - pe->num--; - atomic_sub_fetch_64(&pHashObj->size, 1); - FREE_HASH_NODE(pHashObj, pOld); - } - } else { - uError("pNode:%p data:%p is not there!!!", pNode, p); - } - - return pNode; -} - -void *taosHashIterate(SHashObj *pHashObj, void *p) { - if (pHashObj == NULL) return NULL; - - int slot = 0; - char *data = NULL; - - // only add the read lock to disable the resize process - __rd_lock(&pHashObj->lock, pHashObj->type); - - SHashNode *pNode = NULL; - if (p) { - pNode = taosHashReleaseNode(pHashObj, p, &slot); - if (pNode == NULL) { - SHashEntry *pe = pHashObj->hashList[slot]; - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } - - slot = slot + 1; - } - } - - if (pNode == NULL) { - for (; slot < pHashObj->capacity; ++slot) { - SHashEntry *pe = pHashObj->hashList[slot]; - - // lock entry - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWLockLatch(&pe->latch); - } - - pNode = pe->next; - while (pNode) { - if (pNode->removed == 0) break; - pNode = pNode->next; - } - - if (pNode) break; - - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } - } - } - - if (pNode) { - SHashEntry *pe = pHashObj->hashList[slot]; - pNode->count++; - data = GET_HASH_NODE_DATA(pNode); - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } - } - - __rd_unlock(&pHashObj->lock, pHashObj->type); - return data; - -} - -void taosHashCancelIterate(SHashObj *pHashObj, void *p) { - if (pHashObj == NULL || p == NULL) return; - - // only add the read lock to disable the resize process - __rd_lock(&pHashObj->lock, pHashObj->type); - - int slot; - taosHashReleaseNode(pHashObj, p, &slot); - - SHashEntry *pe = pHashObj->hashList[slot]; - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } - - __rd_unlock(&pHashObj->lock, pHashObj->type); -} diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c index 05324d31ee..946d1aea26 100644 --- a/src/wal/src/walMgmt.c +++ b/src/wal/src/walMgmt.c @@ -139,7 +139,7 @@ void walClose(void *handle) { } static int32_t walInitObj(SWal *pWal) { - if (taosMkDir(pWal->path, 0755) != 0) { + if (!taosMkDir(pWal->path, 0755)) { wError("vgId:%d, path:%s, failed to create directory since %s", pWal->vgId, pWal->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } From 140ec7c8aaaff97686c5121070ab86623e8230ef Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 4 Oct 2021 12:09:50 +0800 Subject: [PATCH 34/44] [TD-10430] adjust util and common module --- include/{util => common}/ttoken.h | 0 {src/inc => include/common}/ttokendef.h | 0 include/os/osDir.h | 2 +- include/os/osFile.h | 4 +- include/os/osSocket.h | 17 + include/os/osTimer.h | 2 + include/util/tcoding.h | 2 +- include/util/tfile.h | 6 +- include/util/tskiplist.h | 2 +- include/util/tthread.h | 2 +- source/common/src/texpr.c | 4 +- source/{util => common}/src/ttokenizer.c | 0 source/common/src/tvariant.c | 1 + source/os/src/osDir.c | 2 +- source/os/src/osFile.c | 54 +- source/os/src/osSocket.c | 523 ++++++++++++++++- source/os/src/osTime.c | 1 + source/os/src/osTimer.c | 16 + source/server/dnode/src/dnodeMain.c | 16 +- source/util/src/tconfig.c | 2 +- source/util/src/tfile.c | 23 +- source/util/src/thashutil.c | 3 +- source/util/src/tlog.c | 4 +- source/util/src/tlosertree.c | 6 +- source/util/src/tmd5.c | 1 - source/util/src/tnote.c | 36 +- source/util/src/tsched.c | 2 +- source/util/src/tskiplist.c | 1 - source/util/src/tsocket.c | 539 ------------------ source/util/src/tthread.c | 3 +- source/util/src/ttimer.c | 14 +- source/util/src/tutil.c | 3 +- src/client/src/tscSub.c | 2 +- .../util => src/kit/shell/inc}/tnettest.h | 0 {source/util => src/kit/shell}/src/tnettest.c | 0 src/tfs/src/tfs.c | 2 +- src/wal/src/walMgmt.c | 2 +- src/wal/src/walWrite.c | 6 +- 38 files changed, 675 insertions(+), 628 deletions(-) rename include/{util => common}/ttoken.h (100%) rename {src/inc => include/common}/ttokendef.h (100%) rename source/{util => common}/src/ttokenizer.c (100%) delete mode 100644 source/util/src/tsocket.c rename {include/util => src/kit/shell/inc}/tnettest.h (100%) rename {source/util => src/kit/shell}/src/tnettest.c (100%) diff --git a/include/util/ttoken.h b/include/common/ttoken.h similarity index 100% rename from include/util/ttoken.h rename to include/common/ttoken.h diff --git a/src/inc/ttokendef.h b/include/common/ttokendef.h similarity index 100% rename from src/inc/ttokendef.h rename to include/common/ttokendef.h diff --git a/include/os/osDir.h b/include/os/osDir.h index 74286edbe8..32733753a8 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -22,7 +22,7 @@ extern "C" { void taosRemoveDir(char *dirname); bool taosDirExist(char *dirname); -bool taosMkDir(char *dirname, mode_t mode); +bool taosMkDir(char *dirname); void taosRemoveOldFiles(char *dirname, int32_t keepDays); bool taosExpandDir(char *dirname, char *outname, int32_t maxlen); bool taosRealPath(char *dirname, int32_t maxlen); diff --git a/include/os/osFile.h b/include/os/osFile.h index 66c3ae9cbd..83f90d4ba2 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -46,8 +46,10 @@ int32_t taosFStatFile(FileFd fd, int64_t *size, int32_t *mtime); FileFd taosOpenFileWrite(const char *path); FileFd taosOpenFileCreateWrite(const char *path); -FileFd taosOpenFileTruncCreateWrite(const char *path); +FileFd taosOpenFileCreateWriteTrunc(const char *path); +FileFd taosOpenFileCreateWriteAppend(const char *path); FileFd taosOpenFileRead(const char *path); +FileFd taosOpenFileReadWrite(const char *path); int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence); int32_t taosFtruncateFile(FileFd fd, int64_t length); diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 11a6486ff6..c503e667e6 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -85,6 +85,23 @@ const char *taosInetNtoa(struct in_addr ipInt); #define htobe64 htonll #endif +int32_t taosReadn(SOCKET sock, char *buffer, int32_t len); +int32_t taosWriteMsg(SOCKET fd, void *ptr, int32_t nbytes); +int32_t taosReadMsg(SOCKET fd, void *ptr, int32_t nbytes); +int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes); +int64_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len); +int32_t taosSetNonblocking(SOCKET sock, int32_t on); + +SOCKET taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); +SOCKET taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); +SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port); +int32_t taosKeepTcpAlive(SOCKET sockFd); + +int32_t taosGetFqdn(char *); +uint32_t taosGetIpv4FromFqdn(const char *); +void tinet_ntoa(char *ipstr, uint32_t ip); +uint32_t ip2uint(const char *const ip_addr); + #ifdef __cplusplus } #endif diff --git a/include/os/osTimer.h b/include/os/osTimer.h index 2afe018c43..4b5db895a2 100644 --- a/include/os/osTimer.h +++ b/include/os/osTimer.h @@ -24,6 +24,8 @@ extern "C" { int32_t taosInitTimer(void (*callback)(int32_t), int32_t ms); void taosUninitTimer(); +int64_t taosGetMonotonicMs(); +const char *taosMonotonicInit(); #ifdef __cplusplus } diff --git a/include/util/tcoding.h b/include/util/tcoding.h index ccc3c26124..a11768a1b0 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -19,7 +19,7 @@ extern "C" { #endif -#include "os.h " +#include "os.h" #define ENCODE_LIMIT (((uint8_t)1) << 7) #define ZIGZAGE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode diff --git a/include/util/tfile.h b/include/util/tfile.h index 066040170e..155c127eff 100644 --- a/include/util/tfile.h +++ b/include/util/tfile.h @@ -28,8 +28,10 @@ void tfCleanup(); // the same syntax as UNIX standard open/close/read/write // but FD is int64_t and will never be reused -int64_t tfOpen(const char *pathname, int32_t flags); -int64_t tfOpenM(const char *pathname, int32_t flags, mode_t mode); +int64_t tfOpenReadWrite(const char *pathname); +int64_t tfOpenCreateWrite(const char *pathname); +int64_t tfOpenCreateWriteAppend(const char *pathname); + int64_t tfClose(int64_t tfd); int64_t tfWrite(int64_t tfd, void *buf, int64_t count); int64_t tfRead(int64_t tfd, void *buf, int64_t count); diff --git a/include/util/tskiplist.h b/include/util/tskiplist.h index d9dc001ccd..9aa225078b 100644 --- a/include/util/tskiplist.h +++ b/include/util/tskiplist.h @@ -21,7 +21,7 @@ extern "C" { #endif #include "os.h" -#include "taosdef.h" +#include "tdef.h" #include "tarray.h" #include "tfunctional.h" diff --git a/include/util/tthread.h b/include/util/tthread.h index 7443ad706d..879c73ef89 100644 --- a/include/util/tthread.h +++ b/include/util/tthread.h @@ -21,7 +21,7 @@ extern "C" { #endif #include "os.h" -#include "taosdef.h" +#include "tdef.h" // create new thread pthread_t* taosCreateThread( void *(*__start_routine) (void *), void* param); diff --git a/source/common/src/texpr.c b/source/common/src/texpr.c index 6823de631f..01e1f71c2f 100644 --- a/source/common/src/texpr.c +++ b/source/common/src/texpr.c @@ -23,7 +23,9 @@ #include "tarray.h" #include "tbuffer.h" #include "tcompare.h" -#include "tsdb.h" +#include "tname.h" +#include "hash.h" +// #include "tsdb.h" #include "tskiplist.h" #include "texpr.h" #include "tarithoperator.h" diff --git a/source/util/src/ttokenizer.c b/source/common/src/ttokenizer.c similarity index 100% rename from source/util/src/ttokenizer.c rename to source/common/src/ttokenizer.c diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index ca3bb956a2..c12650ee8b 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -17,6 +17,7 @@ #include "hash.h" #include "taos.h" #include "taosdef.h" +#include "ttime.h" #include "ttoken.h" #include "ttokendef.h" #include "ttype.h" diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 9bca1e6c2e..0526dffe9e 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -60,7 +60,7 @@ void taosRemoveDir(char *dirname) { bool taosDirExist(char *dirname) { return access(dirname, F_OK) == 0; } -bool taosMkDir(char *dirname, mode_t mode) { +bool taosMkDir(char *dirname) { int32_t code = mkdir(dirname, 0755); if (code < 0 && errno == EEXIST) { return true; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 3726830635..b5d30d7c25 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -16,6 +16,22 @@ #include "os.h" #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#include + +#if defined(_MSDOS) +#define open _open +#endif + +#if defined(_WIN32) +extern int openA(const char *, int, ...); /* MsvcLibX ANSI version of open */ +extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */ +#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE) +#define open openU +#else /* _ANSI_SOURCE */ +#define open openA +#endif /* defined(_UTF8_SOURCE) */ +#endif /* defined(_WIN32) */ + #else #include #include @@ -478,14 +494,6 @@ int32_t taosOpenFileWrite(const char *path) { #endif } -FileFd taosOpenFileRead(const char *path) { - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_RDONLY); -#endif -} - int32_t taosOpenFileCreateWrite(const char *path) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) return 0; @@ -494,10 +502,36 @@ int32_t taosOpenFileCreateWrite(const char *path) { #endif } -int32_t taosOpenFileTruncCreateWrite(const char *path) { +int32_t taosOpenFileCreateWriteTrunc(const char *path) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) return 0; #else return open(path, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); #endif -} \ No newline at end of file +} + +int32_t taosOpenFileCreateWriteAppend(const char *path) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + return open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); +#endif +} + +FileFd taosOpenFileRead(const char *path) { + #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + return open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); +#endif +} + +FileFd taosOpenFileReadWrite(const char *path) { + #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + return open(path, O_RDWR, S_IRWXU | S_IRWXG | S_IRWXO); +#endif +} + + diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index d2f3c120db..0b62df4524 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -224,4 +224,525 @@ uint64_t htonll(uint64_t val) { return (((uint64_t)htonl(val)) << 32) + htonl(va #endif -#endif \ No newline at end of file +#endif + + + +#ifndef SIGPIPE + #define SIGPIPE EPIPE +#endif + +#define TCP_CONN_TIMEOUT 3000 // conn timeout + +int32_t taosGetFqdn(char *fqdn) { + char hostname[1024]; + hostname[1023] = '\0'; + if (gethostname(hostname, 1023) == -1) { + printf("failed to get hostname, reason:%s", strerror(errno)); + return -1; + } + + struct addrinfo hints = {0}; + struct addrinfo *result = NULL; +#ifdef __APPLE__ + // on macosx, hostname -f has the form of xxx.local + // which will block getaddrinfo for a few seconds if AI_CANONNAME is set + // thus, we choose AF_INET (ipv4 for the moment) to make getaddrinfo return + // immediately + hints.ai_family = AF_INET; +#else // __APPLE__ + hints.ai_flags = AI_CANONNAME; +#endif // __APPLE__ + int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); + if (!result) { + printf("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); + return -1; + } + +#ifdef __APPLE__ + // refer to comments above + strcpy(fqdn, hostname); +#else // __APPLE__ + strcpy(fqdn, result->ai_canonname); +#endif // __APPLE__ + freeaddrinfo(result); + return 0; +} + +uint32_t taosGetIpv4FromFqdn(const char *fqdn) { + struct addrinfo hints = {0}; + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + + struct addrinfo *result = NULL; + + int32_t ret = getaddrinfo(fqdn, NULL, &hints, &result); + if (result) { + struct sockaddr * sa = result->ai_addr; + struct sockaddr_in *si = (struct sockaddr_in *)sa; + struct in_addr ia = si->sin_addr; + uint32_t ip = ia.s_addr; + freeaddrinfo(result); + return ip; + } else { +#ifdef EAI_SYSTEM + if (ret == EAI_SYSTEM) { + printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, strerror(errno)); + } else { + printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret)); + } +#else + printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret)); +#endif + return 0xFFFFFFFF; + } +} + +// Function converting an IP address string to an uint32_t. +uint32_t ip2uint(const char *const ip_addr) { + char ip_addr_cpy[20]; + char ip[5]; + + tstrncpy(ip_addr_cpy, ip_addr, sizeof(ip_addr_cpy)); + + char *s_start, *s_end; + s_start = ip_addr_cpy; + s_end = ip_addr_cpy; + + int32_t k; + + for (k = 0; *s_start != '\0'; s_start = s_end) { + for (s_end = s_start; *s_end != '.' && *s_end != '\0'; s_end++) { + } + if (*s_end == '.') { + *s_end = '\0'; + s_end++; + } + ip[k++] = (char)atoi(s_start); + } + + ip[k] = '\0'; + + return *((uint32_t *)ip); +} + +int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) { + int32_t nleft, nwritten; + char * ptr = (char *)buf; + + nleft = nbytes; + + while (nleft > 0) { + nwritten = (int32_t)taosWriteSocket(fd, (char *)ptr, (size_t)nleft); + if (nwritten <= 0) { + if (errno == EINTR /* || errno == EAGAIN || errno == EWOULDBLOCK */) + continue; + else + return -1; + } else { + nleft -= nwritten; + ptr += nwritten; + } + + if (errno == SIGPIPE || errno == EPIPE) { + return -1; + } + } + + return (nbytes - nleft); +} + +int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) { + int32_t nleft, nread; + char * ptr = (char *)buf; + + nleft = nbytes; + + if (fd < 0) return -1; + + while (nleft > 0) { + nread = (int32_t)taosReadSocket(fd, ptr, (size_t)nleft); + if (nread == 0) { + break; + } else if (nread < 0) { + if (errno == EINTR/* || errno == EAGAIN || errno == EWOULDBLOCK*/) { + continue; + } else { + return -1; + } + } else { + nleft -= nread; + ptr += nread; + } + + if (errno == SIGPIPE || errno == EPIPE) { + return -1; + } + } + + return (nbytes - nleft); +} + +int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) { + taosSetNonblocking(fd, 1); + + int32_t nleft, nwritten, nready; + fd_set fset; + struct timeval tv; + + nleft = nbytes; + while (nleft > 0) { + tv.tv_sec = 30; + tv.tv_usec = 0; + FD_ZERO(&fset); + FD_SET(fd, &fset); + if ((nready = select((int32_t)(fd + 1), NULL, &fset, NULL, &tv)) == 0) { + errno = ETIMEDOUT; + printf("fd %d timeout, no enough space to write", fd); + break; + + } else if (nready < 0) { + if (errno == EINTR) continue; + + printf("select error, %d (%s)", errno, strerror(errno)); + return -1; + } + + nwritten = (int32_t)taosSend(fd, ptr, (size_t)nleft, MSG_NOSIGNAL); + if (nwritten <= 0) { + if (errno == EAGAIN || errno == EINTR) continue; + + printf("write error, %d (%s)", errno, strerror(errno)); + return -1; + } + + nleft -= nwritten; + ptr += nwritten; + } + + taosSetNonblocking(fd, 0); + + return (nbytes - nleft); +} + +int32_t taosReadn(SOCKET fd, char *ptr, int32_t nbytes) { + int32_t nread, nready, nleft = nbytes; + + fd_set fset; + struct timeval tv; + + while (nleft > 0) { + tv.tv_sec = 30; + tv.tv_usec = 0; + FD_ZERO(&fset); + FD_SET(fd, &fset); + if ((nready = select((int32_t)(fd + 1), NULL, &fset, NULL, &tv)) == 0) { + errno = ETIMEDOUT; + printf("fd %d timeout\n", fd); + break; + } else if (nready < 0) { + if (errno == EINTR) continue; + printf("select error, %d (%s)", errno, strerror(errno)); + return -1; + } + + if ((nread = (int32_t)taosReadSocket(fd, ptr, (size_t)nleft)) < 0) { + if (errno == EINTR) continue; + printf("read error, %d (%s)", errno, strerror(errno)); + return -1; + + } else if (nread == 0) { + printf("fd %d EOF", fd); + break; // EOF + } + + nleft -= nread; + ptr += nread; + } + + return (nbytes - nleft); +} + +SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { + struct sockaddr_in localAddr; + SOCKET sockFd; + int32_t bufSize = 1024000; + + printf("open udp socket:0x%x:%hu", ip, port); + + memset((char *)&localAddr, 0, sizeof(localAddr)); + localAddr.sin_family = AF_INET; + localAddr.sin_addr.s_addr = ip; + localAddr.sin_port = (uint16_t)htons(port); + + if ((sockFd = socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { + printf("failed to open udp socket: %d (%s)", errno, strerror(errno)); + taosCloseSocketNoCheck(sockFd); + return -1; + } + + if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + printf("failed to set the send buffer size for UDP socket\n"); + taosCloseSocket(sockFd); + return -1; + } + + if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + printf("failed to set the receive buffer size for UDP socket\n"); + taosCloseSocket(sockFd); + return -1; + } + + /* bind socket to local address */ + if (bind(sockFd, (struct sockaddr *)&localAddr, sizeof(localAddr)) < 0) { + printf("failed to bind udp socket: %d (%s), 0x%x:%hu", errno, strerror(errno), ip, port); + taosCloseSocket(sockFd); + return -1; + } + + return sockFd; +} + +SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { + SOCKET sockFd = 0; + int32_t ret; + struct sockaddr_in serverAddr, clientAddr; + int32_t bufSize = 1024 * 1024; + + sockFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); + + if (sockFd <= 2) { + printf("failed to open the socket: %d (%s)", errno, strerror(errno)); + taosCloseSocketNoCheck(sockFd); + return -1; + } + + /* set REUSEADDR option, so the portnumber can be re-used */ + int32_t reuse = 1; + if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); + taosCloseSocket(sockFd); + return -1; + } + + if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + printf("failed to set the send buffer size for TCP socket\n"); + taosCloseSocket(sockFd); + return -1; + } + + if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + printf("failed to set the receive buffer size for TCP socket\n"); + taosCloseSocket(sockFd); + return -1; + } + + if (clientIp != 0) { + memset((char *)&clientAddr, 0, sizeof(clientAddr)); + clientAddr.sin_family = AF_INET; + clientAddr.sin_addr.s_addr = clientIp; + clientAddr.sin_port = 0; + + /* bind socket to client address */ + if (bind(sockFd, (struct sockaddr *)&clientAddr, sizeof(clientAddr)) < 0) { + printf("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)", clientIp, destIp, destPort, + strerror(errno)); + taosCloseSocket(sockFd); + return -1; + } + } + + memset((char *)&serverAddr, 0, sizeof(serverAddr)); + serverAddr.sin_family = AF_INET; + serverAddr.sin_addr.s_addr = destIp; + serverAddr.sin_port = (uint16_t)htons((uint16_t)destPort); + +#ifdef _TD_LINUX + taosSetNonblocking(sockFd, 1); + ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); + if (ret == -1) { + if (errno == EHOSTUNREACH) { + printf("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); + taosCloseSocket(sockFd); + return -1; + } else if (errno == EINPROGRESS || errno == EAGAIN || errno == EWOULDBLOCK) { + struct pollfd wfd[1]; + + wfd[0].fd = sockFd; + wfd[0].events = POLLOUT; + + int res = poll(wfd, 1, TCP_CONN_TIMEOUT); + if (res == -1 || res == 0) { + printf("failed to connect socket, ip:0x%x, port:%hu(poll error/conn timeout)", destIp, destPort); + taosCloseSocket(sockFd); // + return -1; + } + int optVal = -1, optLen = sizeof(int); + if ((0 != taosGetSockOpt(sockFd, SOL_SOCKET, SO_ERROR, &optVal, &optLen)) || (optVal != 0)) { + printf("failed to connect socket, ip:0x%x, port:%hu(connect host error)", destIp, destPort); + taosCloseSocket(sockFd); // + return -1; + } + ret = 0; + } else { // Other error + printf("failed to connect socket, ip:0x%x, port:%hu(target host cannot be reached)", destIp, destPort); + taosCloseSocket(sockFd); // + return -1; + } + } + taosSetNonblocking(sockFd, 0); + +#else + ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); +#endif + + if (ret != 0) { + printf("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); + taosCloseSocket(sockFd); + sockFd = -1; + } else { + taosKeepTcpAlive(sockFd); + } + + return sockFd; +} + +int32_t taosKeepTcpAlive(SOCKET sockFd) { + int32_t alive = 1; + if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_KEEPALIVE, (void *)&alive, sizeof(alive)) < 0) { + printf("fd:%d setsockopt SO_KEEPALIVE failed: %d (%s)", sockFd, errno, strerror(errno)); + taosCloseSocket(sockFd); + return -1; + } + +#ifndef __APPLE__ + // all fails on macosx + int32_t probes = 3; + if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) { + printf("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno)); + taosCloseSocket(sockFd); + return -1; + } + + int32_t alivetime = 10; + if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPIDLE, (void *)&alivetime, sizeof(alivetime)) < 0) { + printf("fd:%d setsockopt SO_KEEPIDLE failed: %d (%s)", sockFd, errno, strerror(errno)); + taosCloseSocket(sockFd); + return -1; + } + + int32_t interval = 3; + if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPINTVL, (void *)&interval, sizeof(interval)) < 0) { + printf("fd:%d setsockopt SO_KEEPINTVL failed: %d (%s)", sockFd, errno, strerror(errno)); + taosCloseSocket(sockFd); + return -1; + } +#endif // __APPLE__ + + int32_t nodelay = 1; + if (taosSetSockOpt(sockFd, IPPROTO_TCP, TCP_NODELAY, (void *)&nodelay, sizeof(nodelay)) < 0) { + printf("fd:%d setsockopt TCP_NODELAY failed %d (%s)", sockFd, errno, strerror(errno)); + taosCloseSocket(sockFd); + return -1; + } + + struct linger linger = {0}; + linger.l_onoff = 1; + linger.l_linger = 3; + if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_LINGER, (void *)&linger, sizeof(linger)) < 0) { + printf("setsockopt SO_LINGER failed: %d (%s)", errno, strerror(errno)); + taosCloseSocket(sockFd); + return -1; + } + + return 0; +} + +SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { + struct sockaddr_in serverAdd; + SOCKET sockFd; + int32_t reuse; + + printf("open tcp server socket:0x%x:%hu", ip, port); + + bzero((char *)&serverAdd, sizeof(serverAdd)); + serverAdd.sin_family = AF_INET; + serverAdd.sin_addr.s_addr = ip; + serverAdd.sin_port = (uint16_t)htons(port); + + if ((sockFd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { + printf("failed to open TCP socket: %d (%s)", errno, strerror(errno)); + taosCloseSocketNoCheck(sockFd); + return -1; + } + + /* set REUSEADDR option, so the portnumber can be re-used */ + reuse = 1; + if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); + taosCloseSocket(sockFd); + return -1; + } + + /* bind socket to server address */ + if (bind(sockFd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) { + printf("bind tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); + taosCloseSocket(sockFd); + return -1; + } + + if (taosKeepTcpAlive(sockFd) < 0) { + printf("failed to set tcp server keep-alive option, 0x%x:%hu(%s)", ip, port, strerror(errno)); + taosCloseSocket(sockFd); + return -1; + } + + if (listen(sockFd, 1024) < 0) { + printf("listen tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); + taosCloseSocket(sockFd); + return -1; + } + + return sockFd; +} + +void tinet_ntoa(char *ipstr, uint32_t ip) { + sprintf(ipstr, "%d.%d.%d.%d", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, ip >> 24); +} + +#define COPY_SIZE 32768 +// sendfile shall be used + +int64_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len) { + int64_t leftLen; + int64_t readLen, writeLen; + char temp[COPY_SIZE]; + + leftLen = len; + + while (leftLen > 0) { + if (leftLen < COPY_SIZE) + readLen = leftLen; + else + readLen = COPY_SIZE; // 4K + + int64_t retLen = taosReadMsg(sfd, temp, (int32_t)readLen); + if (readLen != retLen) { + printf("read error, readLen:%" PRId64 " retLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", + readLen, retLen, len, leftLen, strerror(errno)); + return -1; + } + + writeLen = taosWriteMsg(dfd, temp, (int32_t)readLen); + + if (readLen != writeLen) { + printf("copy error, readLen:%" PRId64 " writeLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", + readLen, writeLen, len, leftLen, strerror(errno)); + return -1; + } + + leftLen -= readLen; + } + + return len; +} diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 2deac7e216..bf0585e86d 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -57,6 +57,7 @@ struct tm *localtime_r(const time_t *timep, struct tm *result) { */ #include +// #include "monotonic.h" FORCE_INLINE int32_t taosGetTimeOfDay(struct timeval *tv) { return gettimeofday(tv, NULL); diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c index d0114044f3..2f18d76db9 100644 --- a/source/os/src/osTimer.c +++ b/source/os/src/osTimer.c @@ -222,4 +222,20 @@ void taosUninitTimer() { pthread_join(timerThread, NULL); } +int64_t taosGetMonotonicMs() { +#if 0 + return getMonotonicUs() / 1000; +#else + return taosGetTimestampMs(); +#endif +} + +const char *taosMonotonicInit() { +#if 0 + return monotonicInit(); +#else + return NULL; +#endif +} + #endif diff --git a/source/server/dnode/src/dnodeMain.c b/source/server/dnode/src/dnodeMain.c index 668b844f6c..8836b633e4 100644 --- a/source/server/dnode/src/dnodeMain.c +++ b/source/server/dnode/src/dnodeMain.c @@ -27,14 +27,6 @@ #include "dnodeMain.h" #include "mnode.h" -static int32_t dnodeCreateDir(const char *dir) { - if (!taosMkDir(dir, 0755) && errno != EEXIST) { - return -1; - } - - return 0; -} - static void dnodeCheckDataDirOpenned(char *dir) { #if 0 char filepath[256] = {0}; @@ -87,7 +79,7 @@ int32_t dnodeInitMain(Dnode *dnode, DnMain **out) { taosSetCoreDump(tsEnableCoreFile); #endif - if (dnodeCreateDir(tsLogDir) < 0) { + if (!taosMkDir(tsLogDir)) { printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno)); return -1; } @@ -134,7 +126,7 @@ int32_t dnodeInitStorage(Dnode *dnode, void **m) { #endif // storage module init - if (tsDiskCfgNum == 1 && dnodeCreateDir(tsDataDir) < 0) { + if (tsDiskCfgNum == 1 && !taosMkDir(tsDataDir)) { dError("failed to create dir:%s since %s", tsDataDir, strerror(errno)); return -1; } @@ -151,12 +143,12 @@ int32_t dnodeInitStorage(Dnode *dnode, void **m) { sprintf(tsVnodeDir, "%s/vnode", tsDataDir); sprintf(tsDnodeDir, "%s/dnode", tsDataDir); - if (dnodeCreateDir(tsMnodeDir) < 0) { + if (!taosMkDir(tsMnodeDir)) { dError("failed to create dir:%s since %s", tsMnodeDir, strerror(errno)); return -1; } - if (dnodeCreateDir(tsDnodeDir) < 0) { + if (!taosMkDir(tsDnodeDir)) { dError("failed to create dir:%s since %s", tsDnodeDir, strerror(errno)); return -1; } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index e7c0758b61..b39e619ba2 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -155,7 +155,7 @@ static bool taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) { taosExpandDir(input_value, option, cfg->ptrLength); taosRealPath(option, cfg->ptrLength); - if (!taosMkDir(option, 0755)) { + if (!taosMkDir(option)) { uError("config option:%s, input value:%s, directory not exist, create fail:%s", cfg->option, input_value, strerror(errno)); return false; diff --git a/source/util/src/tfile.c b/source/util/src/tfile.c index 455c885e75..f4e1e5c115 100644 --- a/source/util/src/tfile.c +++ b/source/util/src/tfile.c @@ -53,13 +53,18 @@ static int64_t tfOpenImp(int32_t fd) { return rid; } -int64_t tfOpen(const char *pathname, int32_t flags) { - int32_t fd = open(pathname, flags | O_BINARY); +int64_t tfOpenReadWrite(const char *pathname, int32_t flags) { + int32_t fd = taosOpenFileReadWrite(pathname); return tfOpenImp(fd); } -int64_t tfOpenM(const char *pathname, int32_t flags, mode_t mode) { - int32_t fd = open(pathname, flags | O_BINARY, mode); +int64_t tfOpenCreateWrite(const char *pathname, int32_t flags, mode_t mode) { + int32_t fd = taosOpenFileCreateWrite(pathname); + return tfOpenImp(fd); +} + +int64_t tfOpenCreateWriteAppend(const char *pathname, int32_t flags, mode_t mode) { + int32_t fd = taosOpenFileCreateWriteAppend(pathname); return tfOpenImp(fd); } @@ -73,7 +78,7 @@ int64_t tfWrite(int64_t tfd, void *buf, int64_t count) { int32_t fd = (int32_t)(uintptr_t)p; - int64_t ret = taosWrite(fd, buf, count); + int64_t ret = taosWriteFile(fd, buf, count); if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); taosReleaseRef(tsFileRsetId, tfd); @@ -86,7 +91,7 @@ int64_t tfRead(int64_t tfd, void *buf, int64_t count) { int32_t fd = (int32_t)(uintptr_t)p; - int64_t ret = taosRead(fd, buf, count); + int64_t ret = taosReadFile(fd, buf, count); if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); taosReleaseRef(tsFileRsetId, tfd); @@ -98,7 +103,7 @@ int32_t tfFsync(int64_t tfd) { if (p == NULL) return -1; int32_t fd = (int32_t)(uintptr_t)p; - int32_t code = taosFsync(fd); + int32_t code = taosFsyncFile(fd); taosReleaseRef(tsFileRsetId, tfd); return code; @@ -117,7 +122,7 @@ int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) { if (p == NULL) return -1; int32_t fd = (int32_t)(uintptr_t)p; - int64_t ret = taosLSeek(fd, offset, whence); + int64_t ret = taosLSeekFile(fd, offset, whence); taosReleaseRef(tsFileRsetId, tfd); return ret; @@ -128,7 +133,7 @@ int32_t tfFtruncate(int64_t tfd, int64_t length) { if (p == NULL) return -1; int32_t fd = (int32_t)(uintptr_t)p; - int32_t code = taosFtruncate(fd, length); + int32_t code = taosFtruncateFile(fd, length); taosReleaseRef(tsFileRsetId, tfd); return code; diff --git a/source/util/src/thashutil.c b/source/util/src/thashutil.c index 4a0208a3d0..7f1262e265 100644 --- a/source/util/src/thashutil.c +++ b/source/util/src/thashutil.c @@ -15,8 +15,9 @@ #include "os.h" #include "hashfunc.h" -#include "tutil.h" #include "tcompare.h" +#include "tdef.h" +#include "tutil.h" #define ROTL32(x, r) ((x) << (r) | (x) >> (32u - (r))) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 63dd1ef606..0f81f59604 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -201,7 +201,7 @@ static void *taosThreadToOpenNewFile(void *param) { taosUmaskFile(0); - int32_t fd = taosOpenFileTruncCreateWrite(name); + int32_t fd = taosOpenFileCreateWriteTrunc(name); if (fd < 0) { tsLogObj.openInProgress = 0; tsLogObj.lines = tsLogObj.maxLines - 1000; @@ -731,7 +731,7 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { goto cmp_end; } - int32_t fd = taosOpenFileTruncCreateWrite(destFileName); + int32_t fd = taosOpenFileCreateWriteTrunc(destFileName); if (fd < 0) { ret = -2; goto cmp_end; diff --git a/source/util/src/tlosertree.c b/source/util/src/tlosertree.c index 0f104c4b63..7be2ca7461 100644 --- a/source/util/src/tlosertree.c +++ b/source/util/src/tlosertree.c @@ -15,7 +15,7 @@ #include "os.h" #include "tlosertree.h" -#include "taosmsg.h" +// #include "taosmsg.h" #include "tulog.h" // set initial value for loser tree @@ -46,7 +46,7 @@ uint32_t tLoserTreeCreate(SLoserTreeInfo** pTree, int32_t numOfEntries, void* pa *pTree = (SLoserTreeInfo*)calloc(1, sizeof(SLoserTreeInfo) + sizeof(SLoserTreeNode) * totalEntries); if ((*pTree) == NULL) { uError("allocate memory for loser-tree failed. reason:%s", strerror(errno)); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return -1; } (*pTree)->pNode = (SLoserTreeNode*)(((char*)(*pTree)) + sizeof(SLoserTreeInfo)); @@ -74,7 +74,7 @@ uint32_t tLoserTreeCreate(SLoserTreeInfo** pTree, int32_t numOfEntries, void* pa printf("initialize local reducer completed!\n"); #endif - return TSDB_CODE_SUCCESS; + return 0; } void tLoserTreeAdjust(SLoserTreeInfo* pTree, int32_t idx) { diff --git a/source/util/src/tmd5.c b/source/util/src/tmd5.c index a1fdcc6a05..9cc4b3b9d5 100644 --- a/source/util/src/tmd5.c +++ b/source/util/src/tmd5.c @@ -35,7 +35,6 @@ #include "os.h" #include "tmd5.h" -#include "taosdef.h" /* forward declaration */ static void Transform(uint32_t *buf, uint32_t *in); diff --git a/source/util/src/tnote.c b/source/util/src/tnote.c index b691abc5b9..e4a326ba8f 100644 --- a/source/util/src/tnote.c +++ b/source/util/src/tnote.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tutil.h" -#include "tglobal.h" +#include "tdef.h" #include "tnote.h" SNoteObj tsHttpNote; @@ -43,6 +43,7 @@ static void taosInitNote(int32_t numOfLines, int32_t maxNotes, SNoteObj *pNote, void taosInitNotes() { char name[TSDB_FILENAME_LEN * 2] = {0}; +#if 0 if (tsTscEnableRecordSql) { snprintf(name, TSDB_FILENAME_LEN * 2, "%s/tscsql-%d", tsLogDir, taosGetPId()); taosInitNote(tsNumOfLogLines, 1, &tsTscNote, name); @@ -57,13 +58,14 @@ void taosInitNotes() { snprintf(name, TSDB_FILENAME_LEN * 2, "%s/taosinfo", tsLogDir); taosInitNote(tsNumOfLogLines, 1, &tsInfoNote, name); } +#endif } static bool taosLockNote(int32_t fd, SNoteObj *pNote) { if (fd < 0) return false; if (pNote->fileNum > 1) { - int32_t ret = (int32_t)(flock(fd, LOCK_EX | LOCK_NB)); + int32_t ret = (int32_t)taosLockFile(fd); if (ret == 0) { return true; } @@ -76,7 +78,7 @@ static void taosUnLockNote(int32_t fd, SNoteObj *pNote) { if (fd < 0) return; if (pNote->fileNum > 1) { - flock(fd, LOCK_UN | LOCK_NB); + taosUnLockFile(fd); } } @@ -90,9 +92,9 @@ static void *taosThreadToOpenNewNote(void *param) { pNote->lines = 0; sprintf(name, "%s.%d", pNote->name, pNote->flag); - umask(0); + taosUmaskFile(0); - int32_t fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); + int32_t fd = taosOpenFileCreateWriteTrunc(name); if (fd < 0) { return NULL; } @@ -132,7 +134,7 @@ static int32_t taosOpenNewNote(SNoteObj *pNote) { } static bool taosCheckNoteIsOpen(char *noteName, SNoteObj *pNote) { - int32_t fd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + int32_t fd = taosOpenFileCreateWrite(noteName); if (fd < 0) { fprintf(stderr, "failed to open note:%s reason:%s\n", noteName, strerror(errno)); return true; @@ -174,7 +176,7 @@ static void taosGetNoteName(char *fn, SNoteObj *pNote) { static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxNoteNum, SNoteObj *pNote) { char name[NOTE_FILE_NAME_LEN * 2] = {0}; int32_t size; - struct stat logstat0, logstat1; + int32_t logstat0_mtime, logstat1_mtime; pNote->maxLines = maxLines; pNote->fileNum = maxNoteNum; @@ -184,13 +186,13 @@ static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxN strcpy(name, fn); strcat(name, ".0"); } - bool log0Exist = stat(name, &logstat0) >= 0; + bool log0Exist = taosStatFile(name, NULL, &logstat0_mtime) >= 0; if (strlen(fn) < NOTE_FILE_NAME_LEN + 50 - 2) { strcpy(name, fn); strcat(name, ".1"); } - bool log1Exist = stat(name, &logstat1) >= 0; + bool log1Exist = taosStatFile(name, NULL, &logstat1_mtime) >= 0; if (!log0Exist && !log1Exist) { pNote->flag = 0; @@ -199,15 +201,15 @@ static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxN } else if (!log0Exist) { pNote->flag = 1; } else { - pNote->flag = (logstat0.st_mtime > logstat1.st_mtime) ? 0 : 1; + pNote->flag = (logstat0_mtime > logstat1_mtime) ? 0 : 1; } char noteName[NOTE_FILE_NAME_LEN * 2] = {0}; sprintf(noteName, "%s.%d", pNote->name, pNote->flag); pthread_mutex_init(&pNote->mutex, NULL); - umask(0); - pNote->fd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + taosUmaskFile(0); + pNote->fd = taosOpenFileCreateWrite(noteName); if (pNote->fd < 0) { fprintf(stderr, "failed to open note file:%s reason:%s\n", noteName, strerror(errno)); @@ -216,12 +218,12 @@ static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxN taosLockNote(pNote->fd, pNote); // only an estimate for number of lines - struct stat filestat; - if (fstat(pNote->fd, &filestat) < 0) { + int64_t filestat_size; + if (taosFStatFile(pNote->fd, &filestat_size, NULL) < 0) { fprintf(stderr, "failed to fstat note file:%s reason:%s\n", noteName, strerror(errno)); return -1; } - size = (int32_t)filestat.st_size; + size = (int32_t)filestat_size; pNote->lines = size / 60; lseek(pNote->fd, 0, SEEK_END); @@ -231,7 +233,7 @@ static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxN void taosNotePrintBuffer(SNoteObj *pNote, char *buffer, int32_t len) { if (pNote->fd <= 0) return; - taosWrite(pNote->fd, buffer, len); + taosWriteFile(pNote->fd, buffer, len); if (pNote->maxLines > 0) { pNote->lines++; @@ -247,7 +249,7 @@ void taosNotePrint(SNoteObj *pNote, const char *const format, ...) { struct timeval timeSecs; time_t curTime; - gettimeofday(&timeSecs, NULL); + taosGetTimeOfDay(&timeSecs); curTime = timeSecs.tv_sec; ptm = localtime_r(&curTime, &Tm); len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index b86ebb38bc..915eaa8d4f 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -14,7 +14,7 @@ */ #include "os.h" -#include "taosdef.h" +#include "tdef.h" #include "tutil.h" #include "tulog.h" #include "tsched.h" diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 98fd9c094c..4905b2723e 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -16,7 +16,6 @@ #include "tskiplist.h" #include "os.h" #include "tcompare.h" -#include "tdataformat.h" #include "tulog.h" #include "tutil.h" diff --git a/source/util/src/tsocket.c b/source/util/src/tsocket.c deleted file mode 100644 index 8d69a87e77..0000000000 --- a/source/util/src/tsocket.c +++ /dev/null @@ -1,539 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tulog.h" -#include "tsocket.h" -#include "taoserror.h" - -#ifndef SIGPIPE - #define SIGPIPE EPIPE -#endif - -#define TCP_CONN_TIMEOUT 3000 // conn timeout - -int32_t taosGetFqdn(char *fqdn) { - char hostname[1024]; - hostname[1023] = '\0'; - if (gethostname(hostname, 1023) == -1) { - uError("failed to get hostname, reason:%s", strerror(errno)); - return -1; - } - - struct addrinfo hints = {0}; - struct addrinfo *result = NULL; -#ifdef __APPLE__ - // on macosx, hostname -f has the form of xxx.local - // which will block getaddrinfo for a few seconds if AI_CANONNAME is set - // thus, we choose AF_INET (ipv4 for the moment) to make getaddrinfo return - // immediately - hints.ai_family = AF_INET; -#else // __APPLE__ - hints.ai_flags = AI_CANONNAME; -#endif // __APPLE__ - int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); - if (!result) { - uError("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); - return -1; - } - -#ifdef __APPLE__ - // refer to comments above - strcpy(fqdn, hostname); -#else // __APPLE__ - strcpy(fqdn, result->ai_canonname); -#endif // __APPLE__ - freeaddrinfo(result); - return 0; -} - -uint32_t taosGetIpv4FromFqdn(const char *fqdn) { - struct addrinfo hints = {0}; - hints.ai_family = AF_INET; - hints.ai_socktype = SOCK_STREAM; - - struct addrinfo *result = NULL; - - int32_t ret = getaddrinfo(fqdn, NULL, &hints, &result); - if (result) { - struct sockaddr * sa = result->ai_addr; - struct sockaddr_in *si = (struct sockaddr_in *)sa; - struct in_addr ia = si->sin_addr; - uint32_t ip = ia.s_addr; - freeaddrinfo(result); - return ip; - } else { -#ifdef EAI_SYSTEM - if (ret == EAI_SYSTEM) { - uError("failed to get the ip address, fqdn:%s, since:%s", fqdn, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - } else { - uError("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret)); - } -#else - uError("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret)); -#endif - return 0xFFFFFFFF; - } -} - -// Function converting an IP address string to an uint32_t. -uint32_t ip2uint(const char *const ip_addr) { - char ip_addr_cpy[20]; - char ip[5]; - - tstrncpy(ip_addr_cpy, ip_addr, sizeof(ip_addr_cpy)); - - char *s_start, *s_end; - s_start = ip_addr_cpy; - s_end = ip_addr_cpy; - - int32_t k; - - for (k = 0; *s_start != '\0'; s_start = s_end) { - for (s_end = s_start; *s_end != '.' && *s_end != '\0'; s_end++) { - } - if (*s_end == '.') { - *s_end = '\0'; - s_end++; - } - ip[k++] = (char)atoi(s_start); - } - - ip[k] = '\0'; - - return *((uint32_t *)ip); -} - -int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) { - int32_t nleft, nwritten; - char * ptr = (char *)buf; - - nleft = nbytes; - - while (nleft > 0) { - nwritten = (int32_t)taosWriteSocket(fd, (char *)ptr, (size_t)nleft); - if (nwritten <= 0) { - if (errno == EINTR /* || errno == EAGAIN || errno == EWOULDBLOCK */) - continue; - else - return -1; - } else { - nleft -= nwritten; - ptr += nwritten; - } - - if (errno == SIGPIPE || errno == EPIPE) { - return -1; - } - } - - return (nbytes - nleft); -} - -int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) { - int32_t nleft, nread; - char * ptr = (char *)buf; - - nleft = nbytes; - - if (fd < 0) return -1; - - while (nleft > 0) { - nread = (int32_t)taosReadSocket(fd, ptr, (size_t)nleft); - if (nread == 0) { - break; - } else if (nread < 0) { - if (errno == EINTR/* || errno == EAGAIN || errno == EWOULDBLOCK*/) { - continue; - } else { - return -1; - } - } else { - nleft -= nread; - ptr += nread; - } - - if (errno == SIGPIPE || errno == EPIPE) { - return -1; - } - } - - return (nbytes - nleft); -} - -int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) { - taosSetNonblocking(fd, 1); - - int32_t nleft, nwritten, nready; - fd_set fset; - struct timeval tv; - - nleft = nbytes; - while (nleft > 0) { - tv.tv_sec = 30; - tv.tv_usec = 0; - FD_ZERO(&fset); - FD_SET(fd, &fset); - if ((nready = select((int32_t)(fd + 1), NULL, &fset, NULL, &tv)) == 0) { - errno = ETIMEDOUT; - uError("fd %d timeout, no enough space to write", fd); - break; - - } else if (nready < 0) { - if (errno == EINTR) continue; - - uError("select error, %d (%s)", errno, strerror(errno)); - return -1; - } - - nwritten = (int32_t)taosSend(fd, ptr, (size_t)nleft, MSG_NOSIGNAL); - if (nwritten <= 0) { - if (errno == EAGAIN || errno == EINTR) continue; - - uError("write error, %d (%s)", errno, strerror(errno)); - return -1; - } - - nleft -= nwritten; - ptr += nwritten; - } - - taosSetNonblocking(fd, 0); - - return (nbytes - nleft); -} - -int32_t taosReadn(SOCKET fd, char *ptr, int32_t nbytes) { - int32_t nread, nready, nleft = nbytes; - - fd_set fset; - struct timeval tv; - - while (nleft > 0) { - tv.tv_sec = 30; - tv.tv_usec = 0; - FD_ZERO(&fset); - FD_SET(fd, &fset); - if ((nready = select((int32_t)(fd + 1), NULL, &fset, NULL, &tv)) == 0) { - errno = ETIMEDOUT; - uError("fd %d timeout\n", fd); - break; - } else if (nready < 0) { - if (errno == EINTR) continue; - uError("select error, %d (%s)", errno, strerror(errno)); - return -1; - } - - if ((nread = (int32_t)taosReadSocket(fd, ptr, (size_t)nleft)) < 0) { - if (errno == EINTR) continue; - uError("read error, %d (%s)", errno, strerror(errno)); - return -1; - - } else if (nread == 0) { - uError("fd %d EOF", fd); - break; // EOF - } - - nleft -= nread; - ptr += nread; - } - - return (nbytes - nleft); -} - -SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { - struct sockaddr_in localAddr; - SOCKET sockFd; - int32_t bufSize = 1024000; - - uDebug("open udp socket:0x%x:%hu", ip, port); - - memset((char *)&localAddr, 0, sizeof(localAddr)); - localAddr.sin_family = AF_INET; - localAddr.sin_addr.s_addr = ip; - localAddr.sin_port = (uint16_t)htons(port); - - if ((sockFd = socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { - uError("failed to open udp socket: %d (%s)", errno, strerror(errno)); - taosCloseSocketNoCheck(sockFd); - return -1; - } - - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - uError("failed to set the send buffer size for UDP socket\n"); - taosCloseSocket(sockFd); - return -1; - } - - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - uError("failed to set the receive buffer size for UDP socket\n"); - taosCloseSocket(sockFd); - return -1; - } - - /* bind socket to local address */ - if (bind(sockFd, (struct sockaddr *)&localAddr, sizeof(localAddr)) < 0) { - uError("failed to bind udp socket: %d (%s), 0x%x:%hu", errno, strerror(errno), ip, port); - taosCloseSocket(sockFd); - return -1; - } - - return sockFd; -} - -SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { - SOCKET sockFd = 0; - int32_t ret; - struct sockaddr_in serverAddr, clientAddr; - int32_t bufSize = 1024 * 1024; - - sockFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); - - if (sockFd <= 2) { - uError("failed to open the socket: %d (%s)", errno, strerror(errno)); - taosCloseSocketNoCheck(sockFd); - return -1; - } - - /* set REUSEADDR option, so the portnumber can be re-used */ - int32_t reuse = 1; - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { - uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(sockFd); - return -1; - } - - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - uError("failed to set the send buffer size for TCP socket\n"); - taosCloseSocket(sockFd); - return -1; - } - - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - uError("failed to set the receive buffer size for TCP socket\n"); - taosCloseSocket(sockFd); - return -1; - } - - if (clientIp != 0) { - memset((char *)&clientAddr, 0, sizeof(clientAddr)); - clientAddr.sin_family = AF_INET; - clientAddr.sin_addr.s_addr = clientIp; - clientAddr.sin_port = 0; - - /* bind socket to client address */ - if (bind(sockFd, (struct sockaddr *)&clientAddr, sizeof(clientAddr)) < 0) { - uError("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)", clientIp, destIp, destPort, - strerror(errno)); - taosCloseSocket(sockFd); - return -1; - } - } - - memset((char *)&serverAddr, 0, sizeof(serverAddr)); - serverAddr.sin_family = AF_INET; - serverAddr.sin_addr.s_addr = destIp; - serverAddr.sin_port = (uint16_t)htons((uint16_t)destPort); - -#ifdef _TD_LINUX - taosSetNonblocking(sockFd, 1); - ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); - if (ret == -1) { - if (errno == EHOSTUNREACH) { - uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); - taosCloseSocket(sockFd); - return -1; - } else if (errno == EINPROGRESS || errno == EAGAIN || errno == EWOULDBLOCK) { - struct pollfd wfd[1]; - - wfd[0].fd = sockFd; - wfd[0].events = POLLOUT; - - int res = poll(wfd, 1, TCP_CONN_TIMEOUT); - if (res == -1 || res == 0) { - uError("failed to connect socket, ip:0x%x, port:%hu(poll error/conn timeout)", destIp, destPort); - taosCloseSocket(sockFd); // - return -1; - } - int optVal = -1, optLen = sizeof(int); - if ((0 != taosGetSockOpt(sockFd, SOL_SOCKET, SO_ERROR, &optVal, &optLen)) || (optVal != 0)) { - uError("failed to connect socket, ip:0x%x, port:%hu(connect host error)", destIp, destPort); - taosCloseSocket(sockFd); // - return -1; - } - ret = 0; - } else { // Other error - uError("failed to connect socket, ip:0x%x, port:%hu(target host cannot be reached)", destIp, destPort); - taosCloseSocket(sockFd); // - return -1; - } - } - taosSetNonblocking(sockFd, 0); - -#else - ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); -#endif - - if (ret != 0) { - uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); - taosCloseSocket(sockFd); - sockFd = -1; - } else { - taosKeepTcpAlive(sockFd); - } - - return sockFd; -} - -int32_t taosKeepTcpAlive(SOCKET sockFd) { - int32_t alive = 1; - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_KEEPALIVE, (void *)&alive, sizeof(alive)) < 0) { - uError("fd:%d setsockopt SO_KEEPALIVE failed: %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(sockFd); - return -1; - } - -#ifndef __APPLE__ - // all fails on macosx - int32_t probes = 3; - if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) { - uError("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(sockFd); - return -1; - } - - int32_t alivetime = 10; - if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPIDLE, (void *)&alivetime, sizeof(alivetime)) < 0) { - uError("fd:%d setsockopt SO_KEEPIDLE failed: %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(sockFd); - return -1; - } - - int32_t interval = 3; - if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPINTVL, (void *)&interval, sizeof(interval)) < 0) { - uError("fd:%d setsockopt SO_KEEPINTVL failed: %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(sockFd); - return -1; - } -#endif // __APPLE__ - - int32_t nodelay = 1; - if (taosSetSockOpt(sockFd, IPPROTO_TCP, TCP_NODELAY, (void *)&nodelay, sizeof(nodelay)) < 0) { - uError("fd:%d setsockopt TCP_NODELAY failed %d (%s)", sockFd, errno, strerror(errno)); - taosCloseSocket(sockFd); - return -1; - } - - struct linger linger = {0}; - linger.l_onoff = 1; - linger.l_linger = 3; - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_LINGER, (void *)&linger, sizeof(linger)) < 0) { - uError("setsockopt SO_LINGER failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(sockFd); - return -1; - } - - return 0; -} - -SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { - struct sockaddr_in serverAdd; - SOCKET sockFd; - int32_t reuse; - - uDebug("open tcp server socket:0x%x:%hu", ip, port); - - bzero((char *)&serverAdd, sizeof(serverAdd)); - serverAdd.sin_family = AF_INET; - serverAdd.sin_addr.s_addr = ip; - serverAdd.sin_port = (uint16_t)htons(port); - - if ((sockFd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { - uError("failed to open TCP socket: %d (%s)", errno, strerror(errno)); - taosCloseSocketNoCheck(sockFd); - return -1; - } - - /* set REUSEADDR option, so the portnumber can be re-used */ - reuse = 1; - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { - uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(sockFd); - return -1; - } - - /* bind socket to server address */ - if (bind(sockFd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) { - uError("bind tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); - taosCloseSocket(sockFd); - return -1; - } - - if (taosKeepTcpAlive(sockFd) < 0) { - uError("failed to set tcp server keep-alive option, 0x%x:%hu(%s)", ip, port, strerror(errno)); - taosCloseSocket(sockFd); - return -1; - } - - if (listen(sockFd, 1024) < 0) { - uError("listen tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); - taosCloseSocket(sockFd); - return -1; - } - - return sockFd; -} - -void tinet_ntoa(char *ipstr, uint32_t ip) { - sprintf(ipstr, "%d.%d.%d.%d", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, ip >> 24); -} - -#define COPY_SIZE 32768 -// sendfile shall be used - -int64_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len) { - int64_t leftLen; - int64_t readLen, writeLen; - char temp[COPY_SIZE]; - - leftLen = len; - - while (leftLen > 0) { - if (leftLen < COPY_SIZE) - readLen = leftLen; - else - readLen = COPY_SIZE; // 4K - - int64_t retLen = taosReadMsg(sfd, temp, (int32_t)readLen); - if (readLen != retLen) { - uError("read error, readLen:%" PRId64 " retLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", - readLen, retLen, len, leftLen, strerror(errno)); - return -1; - } - - writeLen = taosWriteMsg(dfd, temp, (int32_t)readLen); - - if (readLen != writeLen) { - uError("copy error, readLen:%" PRId64 " writeLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", - readLen, writeLen, len, leftLen, strerror(errno)); - return -1; - } - - leftLen -= readLen; - } - - return len; -} diff --git a/source/util/src/tthread.c b/source/util/src/tthread.c index 043b2de2f2..2ffefa25e6 100644 --- a/source/util/src/tthread.c +++ b/source/util/src/tthread.c @@ -15,8 +15,7 @@ #include "os.h" #include "tthread.h" -#include "tglobal.h" -#include "taosdef.h" +#include "tdef.h" #include "tutil.h" #include "tulog.h" #include "taoserror.h" diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 865e1159c1..d0a1fffa9e 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -18,7 +18,6 @@ #include "tsched.h" #include "ttimer.h" #include "tutil.h" -#include "monotonic.h" extern int8_t tscEmbedded; @@ -187,10 +186,6 @@ static void removeTimer(uintptr_t id) { unlockTimerList(list); } -static int64_t getMonotonicMs(void) { - return (int64_t) getMonotonicUs() / 1000; -} - static void addToWheel(tmr_obj_t* timer, uint32_t delay) { timerAddRef(timer); // select a wheel for the timer, we are not an accurate timer, @@ -206,7 +201,7 @@ static void addToWheel(tmr_obj_t* timer, uint32_t delay) { time_wheel_t* wheel = wheels + timer->wheel; timer->prev = NULL; - timer->expireAt = getMonotonicMs() + delay; + timer->expireAt = taosGetMonotonicMs() + delay; pthread_mutex_lock(&wheel->mutex); @@ -339,7 +334,7 @@ tmr_h taosTmrStart(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle } static void taosTimerLoopFunc(int signo) { - int64_t now = getMonotonicMs(); + int64_t now = taosGetMonotonicMs(); for (int i = 0; i < tListLen(wheels); i++) { // `expried` is a temporary expire list. @@ -506,7 +501,7 @@ static void taosTmrModuleInit(void) { pthread_mutex_init(&tmrCtrlMutex, NULL); - int64_t now = getMonotonicMs(); + int64_t now = taosGetMonotonicMs(); for (int i = 0; i < tListLen(wheels); i++) { time_wheel_t* wheel = wheels + i; if (pthread_mutex_init(&wheel->mutex, NULL) != 0) { @@ -537,9 +532,6 @@ static void taosTmrModuleInit(void) { } void* taosTmrInit(int maxNumOfTmrs, int resolution, int longest, const char* label) { - const char* ret = monotonicInit(); - tmrDebug("ttimer monotonic clock source:%s", ret); - pthread_once(&tmrModuleInit, taosTmrModuleInit); pthread_mutex_lock(&tmrCtrlMutex); diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index 5f8c92898f..9c79a7cca2 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -15,8 +15,7 @@ #include "os.h" #include "tcrc32c.h" -#include "tglobal.h" -#include "taosdef.h" +#include "tdef.h" #include "tutil.h" #include "tulog.h" #include "taoserror.h" diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 2d2ce38299..8f9b6a7252 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -371,7 +371,7 @@ void tscSaveSubscriptionProgress(void* sub) { char path[256]; sprintf(path, "%s/subscribe", tsDataDir); - if (!taosMkDir(path, 0777)) { + if (!taosMkDir(path)) { tscError("failed to create subscribe dir: %s", path); } diff --git a/include/util/tnettest.h b/src/kit/shell/inc/tnettest.h similarity index 100% rename from include/util/tnettest.h rename to src/kit/shell/inc/tnettest.h diff --git a/source/util/src/tnettest.c b/src/kit/shell/src/tnettest.c similarity index 100% rename from source/util/src/tnettest.c rename to src/kit/shell/src/tnettest.c diff --git a/src/tfs/src/tfs.c b/src/tfs/src/tfs.c index ac7dbc7fd2..b8e8972d93 100644 --- a/src/tfs/src/tfs.c +++ b/src/tfs/src/tfs.c @@ -252,7 +252,7 @@ int tfsMkdirAt(const char *rname, int level, int id) { char aname[TMPNAME_LEN]; snprintf(aname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), rname); - if (!taosMkDir(aname, 0755)) { + if (!taosMkDir(aname)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c index 946d1aea26..dbff08d730 100644 --- a/src/wal/src/walMgmt.c +++ b/src/wal/src/walMgmt.c @@ -139,7 +139,7 @@ void walClose(void *handle) { } static int32_t walInitObj(SWal *pWal) { - if (!taosMkDir(pWal->path, 0755)) { + if (!taosMkDir(pWal->path)) { wError("vgId:%d, path:%s, failed to create directory since %s", pWal->vgId, pWal->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index e991bf02aa..cae4291eb8 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -51,7 +51,7 @@ int32_t walRenew(void *handle) { } snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, pWal->fileId); - pWal->tfd = tfOpenM(pWal->name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + pWal->tfd = tfOpenCreateWrite(pWal->name); if (!tfValid(pWal->tfd)) { code = TAOS_SYSTEM_ERROR(errno); @@ -220,7 +220,7 @@ int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) { // open the existing WAL file in append mode pWal->fileId = 0; snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, pWal->fileId); - pWal->tfd = tfOpenM(pWal->name, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); + pWal->tfd = tfOpenCreateWriteAppend(pWal->name); if (!tfValid(pWal->tfd)) { wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); @@ -425,7 +425,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch return TAOS_SYSTEM_ERROR(errno); } - int64_t tfd = tfOpen(name, O_RDWR); + int64_t tfd = tfOpenReadWrite(name); if (!tfValid(tfd)) { wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno)); tfree(buffer); From 1c40f32e9cbfe4cf89a5c1dcc047ff3bc37ec080 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 4 Oct 2021 17:44:39 +0800 Subject: [PATCH 35/44] [TD-10430] resolve fix problems --- include/server/dnode/dnode.h | 2 +- include/server/mnode/mnode.h | 2 +- include/server/vnode/vnode.h | 2 +- source/os/CMakeLists.txt | 7 ++ source/server/CMakeLists.txt | 2 +- source/server/dnode/CMakeLists.txt | 4 - source/server/dnode/src/dnodeInt.c | 6 +- source/server/dnode/src/dnodeMain.c | 2 + source/server/mnode/CMakeLists.txt | 4 + source/server/mnode/inc/mnodeInt.h | 5 ++ .../src/vnode.c => mnode/src/mnodeMain.c} | 24 +++++- source/server/{main.c => server.c} | 19 ++++- source/server/vnode/CMakeLists.txt | 9 +- source/server/vnode/inc/vnodeInt.h | 5 ++ .../src/mnode.c => vnode/src/vnodeMain.c} | 18 +++- source/util/src/tstep.c | 85 +++++++++++++++++++ 16 files changed, 178 insertions(+), 18 deletions(-) rename source/server/{vnode/src/vnode.c => mnode/src/mnodeMain.c} (50%) rename source/server/{main.c => server.c} (59%) rename source/server/{mnode/src/mnode.c => vnode/src/vnodeMain.c} (57%) create mode 100644 source/util/src/tstep.c diff --git a/include/server/dnode/dnode.h b/include/server/dnode/dnode.h index 08658f3542..76a3d3ea1b 100644 --- a/include/server/dnode/dnode.h +++ b/include/server/dnode/dnode.h @@ -32,7 +32,7 @@ struct Dnode *dnodeCreateInstance(); * * @param dnode, instance of dnode module. */ -void dnodeCleanupInstance(struct Dnode *dnode); +void dnodeDropInstance(struct Dnode *dnode); /** * Send messages to other dnodes, such as create vnode message. diff --git a/include/server/mnode/mnode.h b/include/server/mnode/mnode.h index f6c067f784..7a06880a16 100644 --- a/include/server/mnode/mnode.h +++ b/include/server/mnode/mnode.h @@ -82,7 +82,7 @@ struct Mnode *mnodeCreateInstance(SMnodePara para); * * @param mnode, instance of mnode module. */ -void mnodeCleanupInstance(struct Mnode *vnode); +void mnodeDropInstance(struct Mnode *vnode); /** * Deploy mnode instances in dnode. diff --git a/include/server/vnode/vnode.h b/include/server/vnode/vnode.h index 8aebd5aa9b..ee015788d3 100644 --- a/include/server/vnode/vnode.h +++ b/include/server/vnode/vnode.h @@ -71,7 +71,7 @@ struct Vnode *vnodeCreateInstance(SVnodePara para); * * @param vnode, instance of vnode module. */ -void vnodeCleanupInstance(struct Vnode *vnode); +void vnodeDropInstance(struct Vnode *vnode); typedef struct { int32_t unused; diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index 9e5d6126ba..0eda9d637d 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -4,4 +4,11 @@ target_include_directories( os PUBLIC "${CMAKE_SOURCE_DIR}/include/os" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" +) +target_link_libraries( + os + PUBLIC pthread + PUBLIC dl + PUBLIC rt + PUBLIC m ) \ No newline at end of file diff --git a/source/server/CMakeLists.txt b/source/server/CMakeLists.txt index d80c7876db..1f6da54ebe 100644 --- a/source/server/CMakeLists.txt +++ b/source/server/CMakeLists.txt @@ -7,5 +7,5 @@ aux_source_directory(. TAOSD_SRC) add_executable(taosd ${TAOSD_SRC}) target_link_libraries( taosd - PUBLIC os + PUBLIC dnode ) \ No newline at end of file diff --git a/source/server/dnode/CMakeLists.txt b/source/server/dnode/CMakeLists.txt index cf7482eca6..e627ca94e8 100644 --- a/source/server/dnode/CMakeLists.txt +++ b/source/server/dnode/CMakeLists.txt @@ -2,11 +2,7 @@ aux_source_directory(src DNODE_SRC) add_library(dnode ${DNODE_SRC}) target_link_libraries( dnode - PUBLIC os PUBLIC cjson - PUBLIC util - PUBLIC common - PUBLIC transport PUBLIC mnode PUBLIC vnode ) diff --git a/source/server/dnode/src/dnodeInt.c b/source/server/dnode/src/dnodeInt.c index 2f805bb882..f38d9665ac 100644 --- a/source/server/dnode/src/dnodeInt.c +++ b/source/server/dnode/src/dnodeInt.c @@ -65,7 +65,7 @@ static int32_t dnodeInitVnodeModule(Dnode *dnode, struct Vnode** out) { static void dnodeCleanupVnodeModule(Dnode *dnode, struct Vnode **out) { struct Vnode *vnode = *out; *out = NULL; - vnodeCleanupInstance(vnode); + vnodeDropInstance(vnode); } static int32_t dnodeInitMnodeModule(Dnode *dnode, struct Mnode **out) { @@ -90,7 +90,7 @@ static int32_t dnodeInitMnodeModule(Dnode *dnode, struct Mnode **out) { static void dnodeCleanupMnodeModule(Dnode *dnode, struct Mnode **out) { struct Mnode *mnode = *out; *out = NULL; - mnodeCleanupInstance(mnode); + mnodeDropInstance(mnode); } Dnode *dnodeCreateInstance() { @@ -223,7 +223,7 @@ Dnode *dnodeCreateInstance() { return dnode; } -void dnodeCleanupInstance(Dnode *dnode) { +void dnodeDropInstance(Dnode *dnode) { if (dnode->main->runStatus != TD_RUN_STAT_STOPPED) { dnode->main->runStatus = TD_RUN_STAT_STOPPED; taosStepCleanup(dnode->steps); diff --git a/source/server/dnode/src/dnodeMain.c b/source/server/dnode/src/dnodeMain.c index 8836b633e4..7854fe2b4b 100644 --- a/source/server/dnode/src/dnodeMain.c +++ b/source/server/dnode/src/dnodeMain.c @@ -112,7 +112,9 @@ void dnodeCleanupMain(DnMain **out) { main->dnodeTimer = NULL; } +#if 0 taos_cleanup(); +#endif taosCloseLog(); taosStopCacheRefreshWorker(); diff --git a/source/server/mnode/CMakeLists.txt b/source/server/mnode/CMakeLists.txt index 689d6f7f2a..331a717bf5 100644 --- a/source/server/mnode/CMakeLists.txt +++ b/source/server/mnode/CMakeLists.txt @@ -4,4 +4,8 @@ target_include_directories( mnode PUBLIC "${CMAKE_SOURCE_DIR}/include/server/mnode" private "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +target_link_libraries( + mnode + PUBLIC transport ) \ No newline at end of file diff --git a/source/server/mnode/inc/mnodeInt.h b/source/server/mnode/inc/mnodeInt.h index 60b748f0ea..646331b076 100644 --- a/source/server/mnode/inc/mnodeInt.h +++ b/source/server/mnode/inc/mnodeInt.h @@ -20,6 +20,11 @@ extern "C" { #endif +#include "os.h" +#include "taosmsg.h" +#include "trpc.h" +#include "mnode.h" + #ifdef __cplusplus } #endif diff --git a/source/server/vnode/src/vnode.c b/source/server/mnode/src/mnodeMain.c similarity index 50% rename from source/server/vnode/src/vnode.c rename to source/server/mnode/src/mnodeMain.c index 6dea4a4e57..3747534e36 100644 --- a/source/server/vnode/src/vnode.c +++ b/source/server/mnode/src/mnodeMain.c @@ -11,4 +11,26 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "mnodeInt.h" + +struct Mnode *mnodeCreateInstance(SMnodePara para) { + return NULL; +} + +void mnodeDropInstance(struct Mnode *vnode) {} + +int32_t mnodeDeploy(struct Mnode *mnode, struct SMInfos *minfos) { return 0; } + +void mnodeUnDeploy(struct Mnode *mnode) {} + +bool mnodeIsServing(struct Mnode *mnode) { return false; } + +int32_t mnodeGetStatistics(struct Mnode *mnode, SMnodeStat *stat) { return 0; } + +int32_t mnodeRetriveAuth(struct Mnode *mnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { + return 0; +} + +void mnodeProcessMsg(struct Mnode *mnode, SRpcMsg *rpcMsg) {} diff --git a/source/server/main.c b/source/server/server.c similarity index 59% rename from source/server/main.c rename to source/server/server.c index 75449ecd1a..b5c8992b65 100644 --- a/source/server/main.c +++ b/source/server/server.c @@ -13,8 +13,25 @@ * along with this program. If not, see . */ #include "os.h" +#include "tulog.h" +#include "trpc.h" +#include "dnode.h" int main(int argc, char const *argv[]) { - printf("Hello world!\n"); + struct Dnode *dnode = dnodeCreateInstance(); + if (dnode == NULL) { + uInfo("Failed to start TDengine, please check the log at:%s", tsLogDir); + exit(EXIT_FAILURE); + } + + uInfo("Started TDengine service successfully."); + + // if (tsem_wait(&exitSem) != 0) { + // syslog(LOG_ERR, "failed to wait exit semphore: %s", strerror(errno)); + // } + + dnodeDropInstance(dnode); + + uInfo("TDengine is shut down!"); return 0; } diff --git a/source/server/vnode/CMakeLists.txt b/source/server/vnode/CMakeLists.txt index 75d24ed64e..03421b2628 100644 --- a/source/server/vnode/CMakeLists.txt +++ b/source/server/vnode/CMakeLists.txt @@ -9,9 +9,10 @@ target_include_directories( PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode" private "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_include_directories( +target_link_libraries( vnode - PRIVATE meta - PRIVATE tq - PRIVATE tsdb + PUBLIC transport + PUBLIC meta + PUBLIC tq + PUBLIC tsdb ) \ No newline at end of file diff --git a/source/server/vnode/inc/vnodeInt.h b/source/server/vnode/inc/vnodeInt.h index 855eb66c8a..12568d21b8 100644 --- a/source/server/vnode/inc/vnodeInt.h +++ b/source/server/vnode/inc/vnodeInt.h @@ -20,6 +20,11 @@ extern "C" { #endif +#include "os.h" +#include "taosmsg.h" +#include "trpc.h" +#include "vnode.h" + #ifdef __cplusplus } #endif diff --git a/source/server/mnode/src/mnode.c b/source/server/vnode/src/vnodeMain.c similarity index 57% rename from source/server/mnode/src/mnode.c rename to source/server/vnode/src/vnodeMain.c index 6dea4a4e57..0f26041787 100644 --- a/source/server/mnode/src/mnode.c +++ b/source/server/vnode/src/vnodeMain.c @@ -11,4 +11,20 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "vnodeInt.h" + +struct Vnode *vnodeCreateInstance(SVnodePara para) { + return NULL; +} + +void vnodeDropInstance(struct Vnode *vnode) {} + +int32_t vnodeGetStatistics(struct Vnode *vnode, SVnodeStat *stat) { return 0; } + +void vnodeGetStatus(struct Vnode *vnode, struct SStatusMsg *status) {} + +void vnodeSetAccess(struct Vnode *vnode, struct SVgroupAccess *access, int32_t numOfVnodes) {} + +void vnodeProcessMsg(struct Vnode *vnode, SRpcMsg *msg) {} diff --git a/source/util/src/tstep.c b/source/util/src/tstep.c new file mode 100644 index 0000000000..97cd3290da --- /dev/null +++ b/source/util/src/tstep.c @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tulog.h" +#include "tstep.h" + +SSteps *taosStepInit(int32_t maxsize) { + SSteps *steps = calloc(1, sizeof(SSteps)); + if (steps == NULL) return NULL; + + steps->maxsize = maxsize; + steps->cursize = 0; + steps->steps = calloc(maxsize, sizeof(SStepObj)); + + return steps; +} + +int32_t taosStepAdd(SSteps *steps, SStepObj *step) { + if (steps == NULL) return - 1; + + if (steps->cursize >= steps->maxsize) { + uError("failed to add step since up to the maxsize"); + return -1; + } + + steps->steps[steps->cursize++] = *step; + return 0; +} + +static void taosStepCleanupImp(SSteps *steps, int32_t pos) { + for (int32_t s = pos; s >= 0; s--) { + SStepObj *step = steps->steps + s; + uDebug("step:%s will cleanup", step->name); + if (step->cleanupFp != NULL) { + (*step->cleanupFp)(step->self); + } + } +} + +int32_t taosStepExec(SSteps *steps) { + if (steps == NULL) return -1; + + for (int32_t s = 0; s < steps->cursize; s++) { + SStepObj *step = steps->steps + s; + if (step->initFp == NULL) continue; + + if (step->reportFp != NULL) { + (*step->reportFp)(step->parent, step->name, "start initialize"); + } + + int32_t code = (*step->initFp)(step->parent, step->self); + if (code != 0) { + uDebug("step:%s will cleanup", step->name); + taosStepCleanupImp(steps, s); + return code; + } + + uInfo("step:%s is initialized", step->name); + + if (step->reportFp != NULL) { + (*step->reportFp)(step->parent, step->name, "initialize completed"); + } + } + + return 0; +} + +void taosStepCleanup(SSteps *steps) { + if (steps == NULL) return; + taosStepCleanupImp(steps, steps->cursize - 1); +} \ No newline at end of file From d20aa8f8f338fb4d5f7573282a3329524d34d829 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 4 Oct 2021 20:42:53 +0800 Subject: [PATCH 36/44] [TD-10430] check util and common files --- include/common/schema.h | 6 +- include/common/taosdef.h | 6 +- include/common/taoserror.h | 6 +- include/common/taosmsg.h | 6 +- include/common/tarithoperator.h | 6 +- include/common/tcmdtype.h | 6 +- include/common/tdataformat.h | 6 +- include/common/texpr.h | 6 +- include/common/tglobal.h | 10 +-- include/common/tlocale.h | 6 +- include/common/tname.h | 6 +- include/common/trow.h | 6 +- include/common/ttimezone.h | 6 +- include/common/ttoken.h | 6 +- include/common/ttokendef.h | 6 +- include/common/ttype.h | 6 +- include/common/tvariant.h | 6 +- include/os/osDef.h | 9 --- include/os/osSysinfo.h | 2 +- include/util/hashfunc.h | 6 +- include/util/tchecksum.h | 16 ++--- include/util/tcompare.h | 6 +- include/util/tconfig.h | 4 +- include/util/tdef.h | 9 +++ include/util/tfile.h | 6 +- include/util/tfunctional.h | 6 +- include/util/tidpool.h | 6 +- include/util/tkey.h | 6 +- include/util/tlog.h | 1 - include/util/tlosertree.h | 8 +-- include/util/tmd5.h | 8 +-- include/util/tmempool.h | 6 +- include/util/tnote.h | 6 +- include/util/tqueue.h | 6 +- include/util/tref.h | 6 +- include/util/tsched.h | 6 +- include/util/tscompression.h | 6 +- include/util/tskiplist.h | 6 +- include/util/tsocket.h | 48 ------------- include/util/tstrbuild.h | 6 +- include/util/tthread.h | 6 +- include/util/ttimer.h | 6 +- include/util/tulog.h | 6 +- include/util/tutil.h | 11 +-- include/util/tworker.h | 31 ++++---- source/common/src/tglobal.c | 22 +----- source/libs/transport/src/rpcMain.c | 1 - source/libs/transport/src/rpcTcp.c | 1 - source/libs/transport/src/rpcUdp.c | 1 - source/server/dnode/src/dnodeTelemetry.c | 1 - source/server/dnode/src/dnodeTrans.c | 2 +- source/util/src/hash.c | 1 + source/util/src/tcompare.c | 23 ++++-- source/util/src/tconfig.c | 1 - source/util/src/tlog.c | 2 +- source/util/src/tlosertree.c | 1 - source/util/src/ttimer.c | 3 + source/util/src/tworker.c | 91 +++++++++++++++--------- 58 files changed, 230 insertions(+), 275 deletions(-) delete mode 100644 include/util/tsocket.h diff --git a/include/common/schema.h b/include/common/schema.h index 2b19eca76c..ccc91f09ff 100644 --- a/include/common/schema.h +++ b/include/common/schema.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_SCHEMA_H_ -#define _TD_SCHEMA_H_ +#ifndef _TD_COMMON_SCHEMA_H_ +#define _TD_COMMON_SCHEMA_H_ #include "os.h" @@ -100,4 +100,4 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); } #endif -#endif /*_TD_SCHEMA_H_*/ \ No newline at end of file +#endif /*_TD_COMMON_SCHEMA_H_*/ \ No newline at end of file diff --git a/include/common/taosdef.h b/include/common/taosdef.h index dd7dfd9518..bae3dcf5e6 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_COMMON_TAOS_DEF_H -#define _TD_COMMON_TAOS_DEF_H +#ifndef _TD_COMMON_TAOS_DEF_H_ +#define _TD_COMMON_TAOS_DEF_H_ #ifdef __cplusplus extern "C" { @@ -76,4 +76,4 @@ extern char *qtypeStr[]; } #endif -#endif /*_TD_COMMON_TAOS_DEF_H*/ +#endif /*_TD_COMMON_TAOS_DEF_H_*/ diff --git a/include/common/taoserror.h b/include/common/taoserror.h index d59b88c7e6..7e30601037 100644 --- a/include/common/taoserror.h +++ b/include/common/taoserror.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TAOSERROR_H -#define TDENGINE_TAOSERROR_H +#ifndef _TD_COMMON_TAOS_ERROR_H_ +#define _TD_COMMON_TAOS_ERROR_H_ #ifdef __cplusplus extern "C" { @@ -463,4 +463,4 @@ int32_t* taosGetErrno(); } #endif -#endif //TDENGINE_TAOSERROR_H +#endif /*_TD_COMMON_TAOS_ERROR_H_*/ diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 76743f10a4..be2f690752 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TAOSMSG_H -#define TDENGINE_TAOSMSG_H +#ifndef _TD_COMMON_TAOS_MSG_H_ +#define _TD_COMMON_TAOS_MSG_H_ #ifdef __cplusplus extern "C" { @@ -961,4 +961,4 @@ typedef struct { } #endif -#endif +#endif /*_TD_COMMON_TAOS_MSG_H_*/ diff --git a/include/common/tarithoperator.h b/include/common/tarithoperator.h index 27e8871e2f..e47cb5c1cb 100644 --- a/include/common/tarithoperator.h +++ b/include/common/tarithoperator.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_QARITHMETICOPERATOR_H -#define TDENGINE_QARITHMETICOPERATOR_H +#ifndef _TD_COMMON_QARITHMETICOPERATOR_H_ +#define _TD_COMMON_QARITHMETICOPERATOR_H_ #ifdef __cplusplus extern "C" { @@ -29,4 +29,4 @@ _arithmetic_operator_fn_t getArithmeticOperatorFn(int32_t arithmeticOptr); } #endif -#endif // TDENGINE_TSYNTAXTREEFUNCTION_H +#endif /*_TD_COMMON_QARITHMETICOPERATOR_H_*/ diff --git a/include/common/tcmdtype.h b/include/common/tcmdtype.h index 918763ebb4..e37996b9c7 100644 --- a/include/common/tcmdtype.h +++ b/include/common/tcmdtype.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TSQLMSGTYPE_H -#define TDENGINE_TSQLMSGTYPE_H +#ifndef _TD_COMMON_SQLMSGTYPE_H_ +#define _TD_COMMON_SQLMSGTYPE_H_ #ifdef __cplusplus extern "C" { @@ -116,4 +116,4 @@ extern char *sqlCmd[]; } #endif -#endif // TDENGINE_TSQLMSGTYPE_H +#endif /*_TD_COMMON_SQLMSGTYPE_H_*/ diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index ab7baea2d7..e348381ae0 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -12,8 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef _TD_DATA_FORMAT_H_ -#define _TD_DATA_FORMAT_H_ +#ifndef _TD_COMMON_DATA_FORMAT_H_ +#define _TD_COMMON_DATA_FORMAT_H_ #include "os.h" #include "talgo.h" @@ -813,4 +813,4 @@ static FORCE_INLINE char *payloadNextCol(char *pCol) { return (char *)POINTER_SH } #endif -#endif // _TD_DATA_FORMAT_H_ +#endif /*_TD_COMMON_DATA_FORMAT_H_*/ diff --git a/include/common/texpr.h b/include/common/texpr.h index 0ede2e1e6f..e28f8644ed 100644 --- a/include/common/texpr.h +++ b/include/common/texpr.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TEXPR_H -#define TDENGINE_TEXPR_H +#ifndef _TD_COMMON_EXPR_H_ +#define _TD_COMMON_EXPR_H_ #ifdef __cplusplus extern "C" { @@ -104,4 +104,4 @@ void buildFilterSetFromBinary(void **q, const char *buf, int32_t len); } #endif -#endif // TDENGINE_TEXPR_H +#endif /*_TD_COMMON_EXPR_H_*/ diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 867d4ae2dd..ffabe0d935 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -13,13 +13,15 @@ * along with this program. If not, see . */ -#ifndef _TD_COMMON_GLOBAL_H -#define _TD_COMMON_GLOBAL_H +#ifndef _TD_COMMON_GLOBAL_H_ +#define _TD_COMMON_GLOBAL_H_ #ifdef __cplusplus extern "C" { #endif +#include "tdef.h" + // cluster extern char tsFirst[]; extern char tsSecond[]; @@ -186,7 +188,7 @@ extern char Compressor[]; extern int8_t tsDeadLockKillQuery; typedef struct { - char dir[128]; + char dir[TSDB_FILENAME_LEN]; int level; int primary; } SDiskCfg; @@ -208,4 +210,4 @@ void taosPrintDataDirCfg(); } #endif -#endif +#endif /*_TD_COMMON_GLOBAL_H_*/ diff --git a/include/common/tlocale.h b/include/common/tlocale.h index 91ae386622..d809c75d67 100644 --- a/include/common/tlocale.h +++ b/include/common/tlocale.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_COMMON_LOCALE_H -#define TDENGINE_COMMON_LOCALE_H +#ifndef _TD_COMMON_LOCALE_H_ +#define _TD_COMMON_LOCALE_H_ #ifdef __cplusplus extern "C" { @@ -24,6 +24,6 @@ void tsSetLocale(); #ifdef __cplusplus } -#endif +#endif /*_TD_COMMON_LOCALE_H_*/ #endif diff --git a/include/common/tname.h b/include/common/tname.h index b29a535ec2..5809d09b15 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_NAME_H -#define TDENGINE_NAME_H +#ifndef _TD_COMMON_NAME_H_ +#define _TD_COMMON_NAME_H_ #include "os.h" #include "taosmsg.h" @@ -136,4 +136,4 @@ int32_t tNameSetAcctId(SName* dst, const char* acct); int32_t tNameSetDbName(SName* dst, const char* acct, SStrToken* dbToken); -#endif // TDENGINE_NAME_H +#endif /*_TD_COMMON_NAME_H_*/ diff --git a/include/common/trow.h b/include/common/trow.h index c6e95d5d61..be4b7af32a 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_TROW_H_ -#define _TD_TROW_H_ +#ifndef _TD_COMMON_ROW_H_ +#define _TD_COMMON_ROW_H_ #ifdef __cplusplus extern "C" { @@ -24,4 +24,4 @@ extern "C" { } #endif -#endif /*_TD_TROW_H_*/ \ No newline at end of file +#endif /*_TD_COMMON_ROW_H_*/ \ No newline at end of file diff --git a/include/common/ttimezone.h b/include/common/ttimezone.h index 0ae983dba0..cc50a29d57 100644 --- a/include/common/ttimezone.h +++ b/include/common/ttimezone.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_COMMON_TIMEZONE_H -#define TDENGINE_COMMON_TIMEZONE_H +#ifndef _TD_COMMON_TIMEZONE_H_ +#define _TD_COMMON_TIMEZONE_H_ #ifdef __cplusplus extern "C" { @@ -26,4 +26,4 @@ void tsSetTimeZone(); } #endif -#endif +#endif /*_TD_COMMON_TIMEZONE_H_*/ diff --git a/include/common/ttoken.h b/include/common/ttoken.h index f62329183f..552e52636f 100644 --- a/include/common/ttoken.h +++ b/include/common/ttoken.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TTOKEN_H -#define TDENGINE_TTOKEN_H +#ifndef _TD_COMMON_TOKEN_H_ +#define _TD_COMMON_TOKEN_H_ #ifdef __cplusplus extern "C" { @@ -189,4 +189,4 @@ SStrToken taosTokenDup(SStrToken* pToken, char* buf, int32_t len); } #endif -#endif // TDENGINE_TTOKEN_H +#endif /*_TD_COMMON_TOKEN_H_*/ diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 9b0ad2cf13..5f9fe5134b 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TTOKENDEF_H -#define TDENGINE_TTOKENDEF_H +#ifndef _TD_COMMON_TOKEN_DEF_H_ +#define _TD_COMMON_TOKEN_DEF_H_ #define TK_ID 1 #define TK_BOOL 2 @@ -223,6 +223,6 @@ #define TK_FILE 306 #define TK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query -#endif +#endif /*_TD_COMMON_TOKEN_DEF_H_*/ diff --git a/include/common/ttype.h b/include/common/ttype.h index 42296f80ba..2150b818d7 100644 --- a/include/common/ttype.h +++ b/include/common/ttype.h @@ -1,5 +1,5 @@ -#ifndef TDENGINE_TTYPE_H -#define TDENGINE_TTYPE_H +#ifndef _TD_COMMON_TYPE_H_ +#define _TD_COMMON_TYPE_H_ #ifdef __cplusplus extern "C" { @@ -178,4 +178,4 @@ int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bo } #endif -#endif // TDENGINE_TTYPE_H +#endif /*_TD_COMMON_TYPE_H_*/ diff --git a/include/common/tvariant.h b/include/common/tvariant.h index c69a662846..f95eb281cf 100644 --- a/include/common/tvariant.h +++ b/include/common/tvariant.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TVARIANT_H -#define TDENGINE_TVARIANT_H +#ifndef _TD_COMMON_VARIANT_H_ +#define _TD_COMMON_VARIANT_H_ #include "tarray.h" #include "ttoken.h" @@ -61,4 +61,4 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type); } #endif -#endif // TDENGINE_TVARIANT_H +#endif /*_TD_COMMON_VARIANT_H_*/ diff --git a/include/os/osDef.h b/include/os/osDef.h index 8db9c6584f..2b3678ac67 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -102,15 +102,6 @@ extern "C" { #define UNUSED_FUNC #endif -// TODO: replace and remove code below -#define CHAR_BYTES sizeof(char) -#define SHORT_BYTES sizeof(int16_t) -#define INT_BYTES sizeof(int32_t) -#define LONG_BYTES sizeof(int64_t) -#define FLOAT_BYTES sizeof(float) -#define DOUBLE_BYTES sizeof(double) -#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) - #ifdef tListLen #undefine tListLen #endif diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 62b52827d6..688fb8f7ef 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -20,7 +20,7 @@ extern "C" { #endif -#define TSDB_LOCALE_LEN 64 +#define TSDB_LOCALE_LEN 64 #define TSDB_TIMEZONE_LEN 96 extern int64_t tsPageSize; diff --git a/include/util/hashfunc.h b/include/util/hashfunc.h index 04c2d1fcf0..3992d8b02e 100644 --- a/include/util/hashfunc.h +++ b/include/util/hashfunc.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_HASH_FUNC_H -#define _TD_UTIL_HASH_FUNC_H +#ifndef _TD_UTIL_HASHFUNC_H +#define _TD_UTIL_HASHFUNC_H #include "os.h" @@ -48,4 +48,4 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type); _equal_fn_t taosGetDefaultEqualFunction(int32_t type); -#endif /*_TD_UTIL_HASH_FUNC_H*/ +#endif /*_TD_UTIL_HASHFUNC_H*/ diff --git a/include/util/tchecksum.h b/include/util/tchecksum.h index 12ca3a5443..c1907b0c46 100644 --- a/include/util/tchecksum.h +++ b/include/util/tchecksum.h @@ -13,22 +13,14 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TCHECKSUM_H -#define TDENGINE_TCHECKSUM_H +#ifndef _TD_UTIL_CHECKSUM_H +#define _TD_UTIL_CHECKSUM_H #ifdef __cplusplus extern "C" { #endif -#include -#include -#include -#include -#include -#include -#include -#include - +#include "os.h" #include "tcrc32c.h" #include "tutil.h" @@ -64,4 +56,4 @@ static FORCE_INLINE int taosCheckChecksumWhole(const uint8_t *stream, uint32_t s } #endif -#endif // TDENGINE_TCHECKSUM_H +#endif /*_TD_UTIL_CHECKSUM_H*/ diff --git a/include/util/tcompare.h b/include/util/tcompare.h index 1125516d34..4b2c583b4b 100644 --- a/include/util/tcompare.h +++ b/include/util/tcompare.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TCOMPARE_H -#define TDENGINE_TCOMPARE_H +#ifndef _TD_UTIL_COMPARE_H +#define _TD_UTIL_COMPARE_H #ifdef __cplusplus extern "C" { @@ -93,4 +93,4 @@ int32_t compareWStrPatternComp(const void* pLeft, const void* pRight); } #endif -#endif // TDENGINE_TCOMPARE_H +#endif /*_TD_UTIL_COMPARE_H*/ diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 68fa4128ae..bc56d21b3a 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -87,8 +87,8 @@ bool taosReadGlobalCfg(); void taosPrintGlobalCfg(); void taosDumpGlobalCfg(); -void taosInitConfigOption(SGlobalCfg cfg); -SGlobalCfg * taosGetConfigOption(const char *option); +void taosInitConfigOption(SGlobalCfg cfg); +SGlobalCfg *taosGetConfigOption(const char *option); #ifdef __cplusplus } diff --git a/include/util/tdef.h b/include/util/tdef.h index b09f39fc32..04a06d1275 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -33,6 +33,15 @@ extern "C" { // Bytes for each type. extern const int32_t TYPE_BYTES[15]; +// TODO: replace and remove code below +#define CHAR_BYTES sizeof(char) +#define SHORT_BYTES sizeof(int16_t) +#define INT_BYTES sizeof(int32_t) +#define LONG_BYTES sizeof(int64_t) +#define FLOAT_BYTES sizeof(float) +#define DOUBLE_BYTES sizeof(double) +#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) + #define TSDB_KEYSIZE sizeof(TSKEY) #if LINUX diff --git a/include/util/tfile.h b/include/util/tfile.h index 155c127eff..ff62c9e341 100644 --- a/include/util/tfile.h +++ b/include/util/tfile.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TFILE_H -#define TDENGINE_TFILE_H +#ifndef _TD_UTIL_FILE_H +#define _TD_UTIL_FILE_H #ifdef __cplusplus extern "C" { @@ -44,4 +44,4 @@ int32_t tfFtruncate(int64_t tfd, int64_t length); } #endif -#endif // TDENGINE_TFILE_H +#endif /*_TD_UTIL_FILE_H*/ diff --git a/include/util/tfunctional.h b/include/util/tfunctional.h index 70f54e921d..c96f997f06 100644 --- a/include/util/tfunctional.h +++ b/include/util/tfunctional.h @@ -12,8 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef TD_TFUNCTIONAL_H -#define TD_TFUNCTIONAL_H +#ifndef _TD_UTIL_FUNCTIONAL_H +#define _TD_UTIL_FUNCTIONAL_H #ifdef __cplusplus extern "C" { @@ -53,4 +53,4 @@ void voidInvoke(tVoidSavedFunc* const pSavedFunc); } #endif -#endif +#endif /*_TD_UTIL_FUNCTIONAL_H*/ diff --git a/include/util/tidpool.h b/include/util/tidpool.h index e4439439ce..1a977fd04c 100644 --- a/include/util/tidpool.h +++ b/include/util/tidpool.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TIDPOOL_H -#define TDENGINE_TIDPOOL_H +#ifndef _TD_UTIL_IDPOOL_H +#define _TD_UTIL_IDPOOL_H #ifdef __cplusplus extern "C" { @@ -40,4 +40,4 @@ bool taosIdPoolMarkStatus(void *handle, int id); } #endif -#endif +#endif /*_TD_UTIL_IDPOOL_H*/ diff --git a/include/util/tkey.h b/include/util/tkey.h index b71cdb2868..197cfff958 100644 --- a/include/util/tkey.h +++ b/include/util/tkey.h @@ -12,8 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef TDENGINE_TKEY_H -#define TDENGINE_TKEY_H +#ifndef _TD_UTIL_KEY_H +#define _TD_UTIL_KEY_H #ifdef __cplusplus extern "C" { @@ -28,4 +28,4 @@ char * taosDesDecode(int64_t key, char *src, int len); } #endif -#endif \ No newline at end of file +#endif /*_TD_UTIL_KEY_H*/ \ No newline at end of file diff --git a/include/util/tlog.h b/include/util/tlog.h index a99085923e..5c6e59b103 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -20,7 +20,6 @@ extern "C" { #endif - // log extern int8_t tsAsyncLog; extern int32_t tsNumOfLogLines; diff --git a/include/util/tlosertree.h b/include/util/tlosertree.h index 58f2ca8c5c..d6ffde82ca 100644 --- a/include/util/tlosertree.h +++ b/include/util/tlosertree.h @@ -13,15 +13,13 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TLOSERTREE_H -#define TDENGINE_TLOSERTREE_H +#ifndef _TD_UTIL_LOSERTREE_H +#define _TD_UTIL_LOSERTREE_H #ifdef __cplusplus extern "C" { #endif -#include - typedef int (*__merge_compare_fn_t)(const void *, const void *, void *param); typedef struct SLoserTreeNode { @@ -51,4 +49,4 @@ void tLoserTreeDisplay(SLoserTreeInfo *pTree); } #endif -#endif // TDENGINE_TLOSERTREE_H +#endif /*_TD_UTIL_LOSERTREE_H*/ diff --git a/include/util/tmd5.h b/include/util/tmd5.h index d7fd038f37..329f4acf11 100644 --- a/include/util/tmd5.h +++ b/include/util/tmd5.h @@ -22,10 +22,8 @@ *********************************************************************** */ -#ifndef _taos_md5_header_ -#define _taos_md5_header_ - -#include +#ifndef _TD_UTIL_MD5_H +#define _TD_UTIL_MD5_H typedef struct { uint32_t i[2]; /* number of _bits_ handled mod 2^64 */ @@ -38,4 +36,4 @@ void MD5Init(MD5_CTX *mdContext); void MD5Update(MD5_CTX *mdContext, uint8_t *inBuf, unsigned int inLen); void MD5Final(MD5_CTX *mdContext); -#endif +#endif /*_TD_UTIL_MD5_H*/ diff --git a/include/util/tmempool.h b/include/util/tmempool.h index f2c6a0ef00..3e3db738a9 100644 --- a/include/util/tmempool.h +++ b/include/util/tmempool.h @@ -12,8 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef TDENGINE_TMEMPOOL_H -#define TDENGINE_TMEMPOOL_H +#ifndef _TD_UTIL_MEMPOOL_H +#define _TD_UTIL_MEMPOOL_H #ifdef __cplusplus extern "C" { @@ -33,4 +33,4 @@ void taosMemPoolCleanUp(mpool_h handle); } #endif -#endif +#endif /*_TD_UTIL_MEMPOOL_H*/ diff --git a/include/util/tnote.h b/include/util/tnote.h index 7511b61f41..e4f28d8cff 100644 --- a/include/util/tnote.h +++ b/include/util/tnote.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TNOTE_H -#define TDENGINE_TNOTE_H +#ifndef _TD_UTIL_NOTE_H +#define _TD_UTIL_NOTE_H #ifdef __cplusplus extern "C" { @@ -61,4 +61,4 @@ void taosNotePrintBuffer(SNoteObj *pNote, char *buffer, int32_t len); } #endif -#endif +#endif /*_TD_UTIL_NOTE_H*/ diff --git a/include/util/tqueue.h b/include/util/tqueue.h index c3051464e5..cd897435fa 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TAOS_QUEUE_H -#define TAOS_QUEUE_H +#ifndef _TD_UTIL_QUEUE_H +#define _TD_UTIL_QUEUE_H #ifdef __cplusplus extern "C" { @@ -71,6 +71,6 @@ int taosGetQsetItemsNumber(taos_qset param); } #endif -#endif +#endif /*_TD_UTIL_QUEUE_H*/ diff --git a/include/util/tref.h b/include/util/tref.h index 085c10c551..cc7d075f52 100644 --- a/include/util/tref.h +++ b/include/util/tref.h @@ -14,8 +14,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TREF_H -#define TDENGINE_TREF_H +#ifndef _TD_UTIL_REF_H +#define _TD_UTIL_REF_H #ifdef __cplusplus extern "C" { @@ -74,4 +74,4 @@ void demoIterateRefs(int rsetId) { } #endif -#endif // TDENGINE_TREF_H +#endif /*_TD_UTIL_REF_H*/ diff --git a/include/util/tsched.h b/include/util/tsched.h index a1591512c1..d60c0c5ad4 100644 --- a/include/util/tsched.h +++ b/include/util/tsched.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TSCHED_H -#define TDENGINE_TSCHED_H +#ifndef _TD_UTIL_SCHED_H +#define _TD_UTIL_SCHED_H #ifdef __cplusplus extern "C" { @@ -68,4 +68,4 @@ void taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg); } #endif -#endif // TDENGINE_TSCHED_H +#endif /*_TD_UTIL_SCHED_H*/ diff --git a/include/util/tscompression.h b/include/util/tscompression.h index 6f722533ac..1e76b5198a 100644 --- a/include/util/tscompression.h +++ b/include/util/tscompression.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TSCOMPRESSION_H -#define TDENGINE_TSCOMPRESSION_H +#ifndef _TD_UTIL_COMPRESSION_H +#define _TD_UTIL_COMPRESSION_H #ifdef __cplusplus extern "C" { @@ -366,4 +366,4 @@ static FORCE_INLINE int tsDecompressTimestamp(const char *const input, int compr } #endif -#endif // TDENGINE_TSCOMPRESSION_H \ No newline at end of file +#endif /*_TD_UTIL_COMPRESSION_H*/ \ No newline at end of file diff --git a/include/util/tskiplist.h b/include/util/tskiplist.h index 9aa225078b..8e991727a7 100644 --- a/include/util/tskiplist.h +++ b/include/util/tskiplist.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TSKIPLIST_H -#define TDENGINE_TSKIPLIST_H +#ifndef _TD_UTIL_SKILIST_H +#define _TD_UTIL_SKILIST_H #ifdef __cplusplus extern "C" { @@ -159,4 +159,4 @@ void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNod } #endif -#endif // TDENGINE_TSKIPLIST_H +#endif /*_TD_UTIL_SKILIST_H*/ diff --git a/include/util/tsocket.h b/include/util/tsocket.h deleted file mode 100644 index b4f5516959..0000000000 --- a/include/util/tsocket.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TSOCKET_H -#define TDENGINE_TSOCKET_H - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef WINDOWS -#include "wepoll.h" -#endif - -int32_t taosReadn(SOCKET sock, char *buffer, int32_t len); -int32_t taosWriteMsg(SOCKET fd, void *ptr, int32_t nbytes); -int32_t taosReadMsg(SOCKET fd, void *ptr, int32_t nbytes); -int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes); -int64_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len); -int32_t taosSetNonblocking(SOCKET sock, int32_t on); - -SOCKET taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); -SOCKET taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); -SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port); -int32_t taosKeepTcpAlive(SOCKET sockFd); - -int32_t taosGetFqdn(char *); -uint32_t taosGetIpv4FromFqdn(const char *); -void tinet_ntoa(char *ipstr, uint32_t ip); -uint32_t ip2uint(const char *const ip_addr); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_TSOCKET_H diff --git a/include/util/tstrbuild.h b/include/util/tstrbuild.h index 68d1914be3..48a302531c 100644 --- a/include/util/tstrbuild.h +++ b/include/util/tstrbuild.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_STRING_BUILDER_H -#define TDENGINE_STRING_BUILDER_H +#ifndef _TD_UTIL_STRING_BUILDER_H +#define _TD_UTIL_STRING_BUILDER_H #include #include @@ -49,4 +49,4 @@ void taosStringBuilderAppendDouble(SStringBuilder* sb, double v); } #endif -#endif \ No newline at end of file +#endif /*_TD_UTIL_STRING_BUILDER_H*/ \ No newline at end of file diff --git a/include/util/tthread.h b/include/util/tthread.h index 879c73ef89..0ff267dd1f 100644 --- a/include/util/tthread.h +++ b/include/util/tthread.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TTHREAD_H -#define TDENGINE_TTHREAD_H +#ifndef _TD_UTIL_THREAD_H +#define _TD_UTIL_THREAD_H #ifdef __cplusplus extern "C" { @@ -34,4 +34,4 @@ bool taosThreadRunning(pthread_t* pthread); } #endif -#endif // TDENGINE_TTHREAD_H +#endif /*_TD_UTIL_THREAD_H*/ diff --git a/include/util/ttimer.h b/include/util/ttimer.h index 6992f5ab6e..987d3f3cdc 100644 --- a/include/util/ttimer.h +++ b/include/util/ttimer.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TTIMER_H -#define TDENGINE_TTIMER_H +#ifndef _TD_UTIL_TIMER_H +#define _TD_UTIL_TIMER_H #ifdef __cplusplus extern "C" { @@ -48,4 +48,4 @@ void taosUninitTimer(); } #endif -#endif // TDENGINE_TTIMER_H +#endif /*_TD_UTIL_TIMER_H*/ diff --git a/include/util/tulog.h b/include/util/tulog.h index 566da40a10..ba59f4eb79 100644 --- a/include/util/tulog.h +++ b/include/util/tulog.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_COMMON_ULOG_H -#define TDENGINE_COMMON_ULOG_H +#ifndef _TD_UTIL_ULOG_H +#define _TD_UTIL_ULOG_H #ifdef __cplusplus extern "C" { @@ -39,4 +39,4 @@ extern int8_t tscEmbedded; } #endif -#endif +#endif /*_TD_UTIL_ULOG_H*/ diff --git a/include/util/tutil.h b/include/util/tutil.h index ba48fcdc04..b599fc095c 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TUTIL_H -#define TDENGINE_TUTIL_H +#ifndef _TD_UTIL_UTIL_H +#define _TD_UTIL_UTIL_H #ifdef __cplusplus extern "C" { @@ -23,6 +23,7 @@ extern "C" { #include "os.h" #include "tmd5.h" #include "tcrc32c.h" +#include "tdef.h" int32_t strdequote(char *src); int32_t strRmquote(char *z, int32_t len); @@ -45,16 +46,16 @@ int taosCheckVersion(char *input_client_version, char *input_server_version, in char * taosIpStr(uint32_t ipInt); uint32_t ip2uint(const char *const ip_addr); -static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target, int32_t keylen) { +static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) { MD5_CTX context; MD5Init(&context); MD5Update(&context, inBuf, (unsigned int)inLen); MD5Final(&context); - memcpy(target, context.digest, keylen); + memcpy(target, context.digest, TSDB_KEY_LEN); } #ifdef __cplusplus } #endif -#endif // TDENGINE_TUTIL_H +#endif /*_TD_UTIL_UTIL_H*/ diff --git a/include/util/tworker.h b/include/util/tworker.h index 7bc1eba2fd..156ced383e 100644 --- a/include/util/tworker.h +++ b/include/util/tworker.h @@ -13,30 +13,33 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TWORKER_H -#define TDENGINE_TWORKER_H +#ifndef _TD_UTIL_WORKER_H +#define _TD_UTIL_WORKER_H #ifdef __cplusplus extern "C" { #endif -typedef void *(*FWorkerThread)(void *pWorker); +typedef int32_t (*ProcessReqFp)(void *ahandle, void *msg); +typedef void (*SendRspFp)(void *ahandle, void *msg, int32_t qtype, int32_t code); + struct SWorkerPool; typedef struct { - pthread_t thread; // thread - int32_t id; // worker ID - struct SWorkerPool *pPool; + pthread_t thread; // thread + int32_t id; // worker ID + struct SWorkerPool *pool; } SWorker; typedef struct SWorkerPool { - int32_t max; // max number of workers - int32_t min; // min number of workers - int32_t num; // current number of workers - void * qset; - char * name; - SWorker *worker; - FWorkerThread workerFp; + int32_t max; // max number of workers + int32_t min; // min number of workers + int32_t num; // current number of workers + void * qset; + const char * name; + SWorker * workers; + ProcessReqFp reqFp; + SendRspFp rspFp; pthread_mutex_t mutex; } SWorkerPool; @@ -49,4 +52,4 @@ void tWorkerFreeQueue(SWorkerPool *pPool, void *pQueue); } #endif -#endif +#endif /*_TD_UTIL_WORKER_H*/ diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index fbf8926fb4..1cebf99351 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -21,7 +21,6 @@ #include "tlog.h" #include "tconfig.h" #include "tglobal.h" -#include "tsocket.h" #include "tcompare.h" #include "tutil.h" #include "ttimezone.h" @@ -385,25 +384,6 @@ static void taosCheckDataDirCfg() { } } -static int32_t taosCheckTmpDir(void) { -#if 0 - if (strlen(tsTempDir) <= 0){ - uError("tempDir is not set"); - return -1; - } - - DIR *dir = opendir(tsTempDir); - if (dir == NULL) { - uError("can not open tempDir:%s, error:%s", tsTempDir, strerror(errno)); - return -1; - } - - closedir(dir); -#endif - return 0; -} - - static void doInitGlobalConfig(void) { osInit(); srand(taosSafeRand()); @@ -1700,7 +1680,7 @@ int32_t taosCheckGlobalCfg() { taosCheckDataDirCfg(); - if (taosCheckTmpDir()) { + if (!taosDirExist(tsTempDir)) { return -1; } diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 9bf2773dde..fe58ee2746 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -22,7 +22,6 @@ #include "lz4.h" #include "tref.h" #include "taoserror.h" -#include "tsocket.h" #include "tglobal.h" #include "taosmsg.h" #include "trpc.h" diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index 1f657a5b8d..d853fdb9f1 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -14,7 +14,6 @@ */ #include "os.h" -#include "tsocket.h" #include "tutil.h" #include "taosdef.h" #include "taoserror.h" diff --git a/source/libs/transport/src/rpcUdp.c b/source/libs/transport/src/rpcUdp.c index 086a390cb8..5bc31c189a 100644 --- a/source/libs/transport/src/rpcUdp.c +++ b/source/libs/transport/src/rpcUdp.c @@ -14,7 +14,6 @@ */ #include "os.h" -#include "tsocket.h" #include "ttimer.h" #include "tutil.h" #include "taosdef.h" diff --git a/source/server/dnode/src/dnodeTelemetry.c b/source/server/dnode/src/dnodeTelemetry.c index 466470c8c7..089a614590 100644 --- a/source/server/dnode/src/dnodeTelemetry.c +++ b/source/server/dnode/src/dnodeTelemetry.c @@ -17,7 +17,6 @@ #include "os.h" #include "tbuffer.h" #include "tglobal.h" -#include "tsocket.h" #include "dnodeCfg.h" #include "dnodeTelemetry.h" #include "mnode.h" diff --git a/source/server/dnode/src/dnodeTrans.c b/source/server/dnode/src/dnodeTrans.c index d9d2c9ad47..4d03338835 100644 --- a/source/server/dnode/src/dnodeTrans.c +++ b/source/server/dnode/src/dnodeTrans.c @@ -242,7 +242,7 @@ static void dnodeProcessMsgFromShell(DnTrans *trans, SRpcMsg *pMsg, SRpcEpSet *p static int32_t dnodeAuthNetTest(char *user, char *spi, char *encrypt, char *secret, char *ckey) { if (strcmp(user, "nettestinternal") == 0) { char pass[32] = {0}; - taosEncryptPass((uint8_t *)user, strlen(user), pass, TSDB_KEY_LEN); + taosEncryptPass((uint8_t *)user, strlen(user), pass); *spi = 0; *encrypt = 0; *ckey = 0; diff --git a/source/util/src/hash.c b/source/util/src/hash.c index 2d48a7c3bf..644fc67eb5 100644 --- a/source/util/src/hash.c +++ b/source/util/src/hash.c @@ -16,6 +16,7 @@ #include "os.h" #include "hash.h" #include "tulog.h" +#include "tdef.h" #define EXT_SIZE 1024 diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index e31fbaaff1..29cd13a6d5 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -233,14 +233,20 @@ int patternMatch(const char *patterStr, const char *str, size_t size, const SPat int32_t i = 0; int32_t j = 0; + int32_t o = 0; + int32_t m = 0; while ((c = patterStr[i++]) != 0) { if (c == pInfo->matchAll) { /* Match "*" */ while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) { - if (c == pInfo->matchOne && (j > size || str[j++] == 0)) { - // empty string, return not match - return TSDB_PATTERN_NOWILDCARDMATCH; + if (c == pInfo->matchOne) { + if (j > size || str[j++] == 0) { + // empty string, return not match + return TSDB_PATTERN_NOWILDCARDMATCH; + } else { + ++o; + } } } @@ -249,9 +255,10 @@ int patternMatch(const char *patterStr, const char *str, size_t size, const SPat } char next[3] = {toupper(c), tolower(c), 0}; + m = o; while (1) { - size_t n = strcspn(str, next); - str += n; + size_t n = strcspn(str + m, next); + str += m + n; if (str[0] == 0 || (n >= size)) { break; @@ -261,12 +268,14 @@ int patternMatch(const char *patterStr, const char *str, size_t size, const SPat if (ret != TSDB_PATTERN_NOMATCH) { return ret; } + m = 0; } return TSDB_PATTERN_NOWILDCARDMATCH; } c1 = str[j++]; - + ++o; + if (j <= size) { if (c == '\\' && patterStr[i] == '_' && c1 == '_') { i++; continue; } if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) { @@ -292,7 +301,7 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c if (c == matchAll) { /* Match "%" */ while ((c = patterStr[i++]) == matchAll || c == matchOne) { - if (c == matchOne && (j > size || str[j++] == 0)) { + if (c == matchOne && (j >= size || str[j++] == 0)) { return TSDB_PATTERN_NOWILDCARDMATCH; } } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index b39e619ba2..b0256471cd 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -17,7 +17,6 @@ #include "os.h" #include "tconfig.h" #include "tulog.h" -#include "tsocket.h" #include "tutil.h" SGlobalCfg tsGlobalConfig[TSDB_CFG_MAX_NUM] = {{0}}; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 0f81f59604..f134730c5c 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -154,7 +154,7 @@ static bool taosLockLogFile(int32_t fd) { if (fd < 0) return false; if (tsLogObj.fileNum > 1) { - int32_t ret = taosUnLockFile(fd); + int32_t ret = taosLockFile(fd); if (ret == 0) { return true; } diff --git a/source/util/src/tlosertree.c b/source/util/src/tlosertree.c index 7be2ca7461..c6e3a98493 100644 --- a/source/util/src/tlosertree.c +++ b/source/util/src/tlosertree.c @@ -15,7 +15,6 @@ #include "os.h" #include "tlosertree.h" -// #include "taosmsg.h" #include "tulog.h" // set initial value for loser tree diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index d0a1fffa9e..4aac7b7abf 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -532,6 +532,9 @@ static void taosTmrModuleInit(void) { } void* taosTmrInit(int maxNumOfTmrs, int resolution, int longest, const char* label) { + const char* ret = taosMonotonicInit(); + tmrDebug("ttimer monotonic clock source:%s", ret); + pthread_once(&tmrModuleInit, taosTmrModuleInit); pthread_mutex_lock(&tmrCtrlMutex); diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 8b4053bccd..3fe34755d7 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -19,78 +19,101 @@ #include "tqueue.h" #include "tworker.h" -int32_t tWorkerInit(SWorkerPool *pPool) { - pPool->qset = taosOpenQset(); - pPool->worker = calloc(sizeof(SWorker), pPool->max); - pthread_mutex_init(&pPool->mutex, NULL); - for (int i = 0; i < pPool->max; ++i) { - SWorker *pWorker = pPool->worker + i; - pWorker->id = i; - pWorker->pPool = pPool; +static void *taosWorkerThreadFp(void *wparam) { + SWorker * worker = wparam; + SWorkerPool *pool = worker->pool; + void * msg = NULL; + int32_t qtype = 0; + void * ahandle = NULL; + int32_t code = 0; + + setThreadName(pool->name); + + while (1) { + if (taosReadQitemFromQset(pool->qset, &qtype, (void **)&msg, &ahandle) == 0) { + uDebug("pool:%s, worker:%d qset:%p, got no message and exiting", pool->name, worker->id, pool->qset); + break; + } + + code = (*pool->reqFp)(ahandle, msg); + (*pool->rspFp)(ahandle, msg, qtype, code); } - uInfo("worker:%s is initialized, min:%d max:%d", pPool->name, pPool->min, pPool->max); + return NULL; +} + +int32_t tWorkerInit(SWorkerPool *pool) { + pool->qset = taosOpenQset(); + pool->workers = calloc(sizeof(SWorker), pool->max); + pthread_mutex_init(&pool->mutex, NULL); + for (int i = 0; i < pool->max; ++i) { + SWorker *pWorker = pool->workers + i; + pWorker->id = i; + pWorker->pool = pool; + } + + uInfo("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max); return 0; } -void tWorkerCleanup(SWorkerPool *pPool) { - for (int i = 0; i < pPool->max; ++i) { - SWorker *pWorker = pPool->worker + i; +void tWorkerCleanup(SWorkerPool *pool) { + for (int i = 0; i < pool->max; ++i) { + SWorker *pWorker = pool->workers + i; if(taosCheckPthreadValid(pWorker->thread)) { - taosQsetThreadResume(pPool->qset); + taosQsetThreadResume(pool->qset); } } - for (int i = 0; i < pPool->max; ++i) { - SWorker *pWorker = pPool->worker + i; + for (int i = 0; i < pool->max; ++i) { + SWorker *pWorker = pool->workers + i; if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } } - free(pPool->worker); - taosCloseQset(pPool->qset); - pthread_mutex_destroy(&pPool->mutex); + free(pool->workers); + taosCloseQset(pool->qset); + pthread_mutex_destroy(&pool->mutex); - uInfo("worker:%s is closed", pPool->name); + uInfo("worker:%s is closed", pool->name); } -void *tWorkerAllocQueue(SWorkerPool *pPool, void *ahandle) { - pthread_mutex_lock(&pPool->mutex); +void *tWorkerAllocQueue(SWorkerPool *pool, void *ahandle) { + pthread_mutex_lock(&pool->mutex); taos_queue pQueue = taosOpenQueue(); if (pQueue == NULL) { - pthread_mutex_unlock(&pPool->mutex); + pthread_mutex_unlock(&pool->mutex); return NULL; } - taosAddIntoQset(pPool->qset, pQueue, ahandle); + taosAddIntoQset(pool->qset, pQueue, ahandle); // spawn a thread to process queue - if (pPool->num < pPool->max) { + if (pool->num < pool->max) { do { - SWorker *pWorker = pPool->worker + pPool->num; + SWorker *pWorker = pool->workers + pool->num; pthread_attr_t thAttr; pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&pWorker->thread, &thAttr, pPool->workerFp, pWorker) != 0) { - uError("worker:%s:%d failed to create thread to process since %s", pPool->name, pWorker->id, strerror(errno)); + if (pthread_create(&pWorker->thread, &thAttr, taosWorkerThreadFp, pWorker) != 0) { + uError("workers:%s:%d failed to create thread to process since %s", pool->name, pWorker->id, strerror(errno)); } pthread_attr_destroy(&thAttr); - pPool->num++; - uDebug("worker:%s:%d is launched, total:%d", pPool->name, pWorker->id, pPool->num); - } while (pPool->num < pPool->min); + pool->num++; + uDebug("workers:%s:%d is launched, total:%d", pool->name, pWorker->id, pool->num); + } while (pool->num < pool->min); } - pthread_mutex_unlock(&pPool->mutex); - uDebug("worker:%s, queue:%p is allocated, ahandle:%p", pPool->name, pQueue, ahandle); + pthread_mutex_unlock(&pool->mutex); + uDebug("workers:%s, queue:%p is allocated, ahandle:%p", pool->name, pQueue, ahandle); return pQueue; } -void tWorkerFreeQueue(SWorkerPool *pPool, void *pQueue) { +void tWorkerFreeQueue(SWorkerPool *pool, void *pQueue) { taosCloseQueue(pQueue); - uDebug("worker:%s, queue:%p is freed", pPool->name, pQueue); + uDebug("workers:%s, queue:%p is freed", pool->name, pQueue); } From 8ccd079f6eaeb1ac5d854b5e34131a77e788812f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 4 Oct 2021 20:47:39 +0800 Subject: [PATCH 37/44] rename tulog.h ulog.h --- include/common/taoserror.h | 3 - include/common/taosmsg.h | 3 - include/common/ttype.h | 2 - include/server/dnode/dnode.h | 4 ++ include/util/tdef.h | 2 + include/util/{tulog.h => ulog.h} | 0 source/common/src/tdataformat.c | 2 +- source/common/src/tglobal.c | 2 +- source/common/src/tlocale.c | 2 +- source/common/src/ttimezone.c | 2 +- source/server/server.c | 25 ++++++--- source/util/src/hash.c | 2 +- source/util/src/tcache.c | 2 +- source/util/src/tcompare.c | 2 +- source/util/src/tcompression.c | 2 +- source/util/src/tconfig.c | 2 +- source/util/src/tfile.c | 2 +- source/util/src/tidpool.c | 2 +- source/util/src/tlog.c | 2 +- source/util/src/tlosertree.c | 2 +- source/util/src/tmempool.c | 2 +- source/util/src/tqueue.c | 2 +- source/util/src/tref.c | 2 +- source/util/src/tsched.c | 2 +- source/util/src/tskiplist.c | 2 +- source/util/src/tstep.c | 2 +- source/util/src/tthread.c | 2 +- source/util/src/tutil.c | 2 +- source/util/src/tworker.c | 2 +- source/util/test/trefTest.c | 2 +- src/inc/module.h | 30 ---------- src/inc/vnode.h | 95 -------------------------------- 32 files changed, 47 insertions(+), 163 deletions(-) rename include/util/{tulog.h => ulog.h} (100%) delete mode 100644 src/inc/module.h delete mode 100644 src/inc/vnode.h diff --git a/include/common/taoserror.h b/include/common/taoserror.h index 7e30601037..c3f3e3b98b 100644 --- a/include/common/taoserror.h +++ b/include/common/taoserror.h @@ -20,9 +20,6 @@ extern "C" { #endif -#include -#include - #define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code)))) #define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index be2f690752..c296317758 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -20,9 +20,6 @@ extern "C" { #endif -#include -#include - #include "taosdef.h" #include "taoserror.h" #include "tdataformat.h" diff --git a/include/common/ttype.h b/include/common/ttype.h index 2150b818d7..941ad7ae84 100644 --- a/include/common/ttype.h +++ b/include/common/ttype.h @@ -5,8 +5,6 @@ extern "C" { #endif -#include -#include #include "taosdef.h" // this data type is internally used only in 'in' query to hold the values diff --git a/include/server/dnode/dnode.h b/include/server/dnode/dnode.h index 76a3d3ea1b..a41235e7c9 100644 --- a/include/server/dnode/dnode.h +++ b/include/server/dnode/dnode.h @@ -20,6 +20,10 @@ extern "C" { #endif +struct SRpcEpSet; +struct SRpcMsg; +struct Dnode; + /** * Initialize and start the dnode module. * diff --git a/include/util/tdef.h b/include/util/tdef.h index 04a06d1275..c67886f240 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -16,6 +16,8 @@ #ifndef _TD_UTIL_DEF_H #define _TD_UTIL_DEF_H +#include "os.h" + #ifdef __cplusplus extern "C" { #endif diff --git a/include/util/tulog.h b/include/util/ulog.h similarity index 100% rename from include/util/tulog.h rename to include/util/ulog.h diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 61378c79c4..6e76e3a8d0 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ #include "tdataformat.h" -#include "tulog.h" +#include "ulog.h" #include "talgo.h" #include "tcoding.h" #include "wchar.h" diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 1cebf99351..9eab95cd41 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -17,7 +17,7 @@ #include "os.h" #include "taosdef.h" #include "taoserror.h" -#include "tulog.h" +#include "ulog.h" #include "tlog.h" #include "tconfig.h" #include "tglobal.h" diff --git a/source/common/src/tlocale.c b/source/common/src/tlocale.c index 46a3d951db..a98a46b28a 100644 --- a/source/common/src/tlocale.c +++ b/source/common/src/tlocale.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "tulog.h" +#include "ulog.h" #include "tglobal.h" #include "tconfig.h" #include "tutil.h" diff --git a/source/common/src/ttimezone.c b/source/common/src/ttimezone.c index 62d4768db8..c45e39c20d 100644 --- a/source/common/src/ttimezone.c +++ b/source/common/src/ttimezone.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "tulog.h" +#include "ulog.h" #include "tglobal.h" #include "tconfig.h" #include "tutil.h" diff --git a/source/server/server.c b/source/server/server.c index b5c8992b65..0e50b1f97f 100644 --- a/source/server/server.c +++ b/source/server/server.c @@ -13,11 +13,22 @@ * along with this program. If not, see . */ #include "os.h" -#include "tulog.h" -#include "trpc.h" +#include "ulog.h" #include "dnode.h" +static bool stop = false; +static void sigintHandler(int32_t signum, void *info, void *ctx) { stop = true; } +static void setSignalHandler() { + taosSetSignal(SIGTERM, sigintHandler); + taosSetSignal(SIGHUP, sigintHandler); + taosSetSignal(SIGINT, sigintHandler); + taosSetSignal(SIGABRT, sigintHandler); + taosSetSignal(SIGBREAK, sigintHandler); +} + int main(int argc, char const *argv[]) { + setSignalHandler(); + struct Dnode *dnode = dnodeCreateInstance(); if (dnode == NULL) { uInfo("Failed to start TDengine, please check the log at:%s", tsLogDir); @@ -26,12 +37,12 @@ int main(int argc, char const *argv[]) { uInfo("Started TDengine service successfully."); - // if (tsem_wait(&exitSem) != 0) { - // syslog(LOG_ERR, "failed to wait exit semphore: %s", strerror(errno)); - // } + while (!stop) { + taosMsleep(100); + } - dnodeDropInstance(dnode); - uInfo("TDengine is shut down!"); + dnodeDropInstance(dnode); + return 0; } diff --git a/source/util/src/hash.c b/source/util/src/hash.c index 644fc67eb5..72d293a0e9 100644 --- a/source/util/src/hash.c +++ b/source/util/src/hash.c @@ -15,7 +15,7 @@ #include "os.h" #include "hash.h" -#include "tulog.h" +#include "ulog.h" #include "tdef.h" #define EXT_SIZE 1024 diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 589d3d4fa5..0c9a1a18ad 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "tulog.h" +#include "ulog.h" #include "ttimer.h" #include "tutil.h" #include "tcache.h" diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 29cd13a6d5..1f47ae6be1 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -18,7 +18,7 @@ #define _DEFAULT_SOURCE #include "tcompare.h" -#include "tulog.h" +#include "ulog.h" #include "hash.h" #include "regex.h" #include "os.h" diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 990008a840..4713de9772 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -53,7 +53,7 @@ #include "td_sz.h" #endif #include "tscompression.h" -#include "tulog.h" +#include "ulog.h" static const int TEST_NUMBER = 1; #define is_bigendian() ((*(char *)&TEST_NUMBER) == 0) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index b0256471cd..6b3f08a446 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tconfig.h" -#include "tulog.h" +#include "ulog.h" #include "tutil.h" SGlobalCfg tsGlobalConfig[TSDB_CFG_MAX_NUM] = {{0}}; diff --git a/source/util/src/tfile.c b/source/util/src/tfile.c index f4e1e5c115..2dafd689d3 100644 --- a/source/util/src/tfile.c +++ b/source/util/src/tfile.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" -#include "tulog.h" +#include "ulog.h" #include "tutil.h" #include "tref.h" diff --git a/source/util/src/tidpool.c b/source/util/src/tidpool.c index 61cecf54c0..00c43bb25f 100644 --- a/source/util/src/tidpool.c +++ b/source/util/src/tidpool.c @@ -14,7 +14,7 @@ */ #include "os.h" -#include "tulog.h" +#include "ulog.h" typedef struct { int maxId; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index f134730c5c..2fb84656b6 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "tulog.h" +#include "ulog.h" #include "tlog.h" #include "tnote.h" #include "tutil.h" diff --git a/source/util/src/tlosertree.c b/source/util/src/tlosertree.c index c6e3a98493..6155ba4c1a 100644 --- a/source/util/src/tlosertree.c +++ b/source/util/src/tlosertree.c @@ -15,7 +15,7 @@ #include "os.h" #include "tlosertree.h" -#include "tulog.h" +#include "ulog.h" // set initial value for loser tree void tLoserTreeInit(SLoserTreeInfo* pTree) { diff --git a/source/util/src/tmempool.c b/source/util/src/tmempool.c index 678c965eb1..a61c4abb14 100644 --- a/source/util/src/tmempool.c +++ b/source/util/src/tmempool.c @@ -14,7 +14,7 @@ */ #include "os.h" -#include "tulog.h" +#include "ulog.h" #include "tmempool.h" #include "tutil.h" diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 6a37f11ece..da1fb1837f 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -14,7 +14,7 @@ */ #include "os.h" -#include "tulog.h" +#include "ulog.h" #include "taoserror.h" #include "tqueue.h" diff --git a/source/util/src/tref.c b/source/util/src/tref.c index 7d64bd1f83..68f161bd3d 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -15,7 +15,7 @@ #include "os.h" #include "taoserror.h" -#include "tulog.h" +#include "ulog.h" #include "tutil.h" #define TSDB_REF_OBJECTS 50 diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 915eaa8d4f..915edc45ce 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -16,7 +16,7 @@ #include "os.h" #include "tdef.h" #include "tutil.h" -#include "tulog.h" +#include "ulog.h" #include "tsched.h" #include "ttimer.h" diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 4905b2723e..d02b148863 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -16,7 +16,7 @@ #include "tskiplist.h" #include "os.h" #include "tcompare.h" -#include "tulog.h" +#include "ulog.h" #include "tutil.h" static int initForwardBackwardPtr(SSkipList *pSkipList); diff --git a/source/util/src/tstep.c b/source/util/src/tstep.c index 97cd3290da..e307977e70 100644 --- a/source/util/src/tstep.c +++ b/source/util/src/tstep.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "tulog.h" +#include "ulog.h" #include "tstep.h" SSteps *taosStepInit(int32_t maxsize) { diff --git a/source/util/src/tthread.c b/source/util/src/tthread.c index 2ffefa25e6..5ed7fb5aa0 100644 --- a/source/util/src/tthread.c +++ b/source/util/src/tthread.c @@ -17,7 +17,7 @@ #include "tthread.h" #include "tdef.h" #include "tutil.h" -#include "tulog.h" +#include "ulog.h" #include "taoserror.h" // create new thread diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index 9c79a7cca2..c5027af7c7 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -17,7 +17,7 @@ #include "tcrc32c.h" #include "tdef.h" #include "tutil.h" -#include "tulog.h" +#include "ulog.h" #include "taoserror.h" int32_t strdequote(char *z) { diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 3fe34755d7..5370e78c09 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "tulog.h" +#include "ulog.h" #include "tqueue.h" #include "tworker.h" diff --git a/source/util/test/trefTest.c b/source/util/test/trefTest.c index e01da070af..54aa6a5f34 100644 --- a/source/util/test/trefTest.c +++ b/source/util/test/trefTest.c @@ -8,7 +8,7 @@ #include "tlog.h" #include "tglobal.h" #include "taoserror.h" -#include "tulog.h" +#include "ulog.h" typedef struct { int refNum; diff --git a/src/inc/module.h b/src/inc/module.h deleted file mode 100644 index b9b64c493e..0000000000 --- a/src/inc/module.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_MODULE -#define TDENGINE_MODULE - -#ifdef __cplusplus -extern "C" { -#endif - -int32_t moduleStart(); -void moduleStop(); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc/vnode.h b/src/inc/vnode.h deleted file mode 100644 index b3291645c0..0000000000 --- a/src/inc/vnode.h +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_VNODE_H -#define TDENGINE_VNODE_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "trpc.h" -#include "twal.h" - -typedef struct { - int32_t len; - void * rsp; - void * qhandle; // used by query and retrieve msg -} SRspRet; - -typedef struct { - int32_t code; - int32_t contLen; - void * rpcHandle; - void * rpcAhandle; - void * qhandle; - void * pVnode; - int8_t qtype; - int8_t msgType; - SRspRet rspRet; - char pCont[]; -} SVReadMsg; - -typedef struct { - int32_t code; - int32_t processedCount; - int32_t qtype; - void * pVnode; - SRpcMsg rpcMsg; - SRspRet rspRet; - char reserveForSync[24]; - SWalHead walHead; -} SVWriteMsg; - -// vnodeStatus -extern char *vnodeStatus[]; - -// vnodeMain -int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg); -int32_t vnodeDrop(int32_t vgId); -int32_t vnodeOpen(int32_t vgId); -int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg); -int32_t vnodeSync(int32_t vgId); -int32_t vnodeClose(int32_t vgId); -int32_t vnodeCompact(int32_t vgId); - -// vnodeMgmt -int32_t vnodeInitMgmt(); -void vnodeCleanupMgmt(); -void* vnodeAcquire(int32_t vgId); -void vnodeRelease(void *pVnode); -void* vnodeAcquireNotClose(int32_t vgId); -void* vnodeGetWal(void *pVnode); -int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes); -void vnodeBuildStatusMsg(void *pStatus); -void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes); - -// vnodeWrite -int32_t vnodeWriteToWQueue(void *pVnode, void *pHead, int32_t qtype, void *pRpcMsg); -void vnodeFreeFromWQueue(void *pVnode, SVWriteMsg *pWrite); -int32_t vnodeProcessWrite(void *pVnode, void *pHead, int32_t qtype, void *pRspRet); - -// vnodeSync -void vnodeConfirmForward(void *pVnode, uint64_t version, int32_t code, bool force); - -// vnodeRead -int32_t vnodeWriteToRQueue(void *pVnode, void *pCont, int32_t contLen, int8_t qtype, void *rparam); -void vnodeFreeFromRQueue(void *pVnode, SVReadMsg *pRead); -int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead); - -#ifdef __cplusplus -} -#endif - -#endif From 1f3b7197b1c7e72b5ac23cef00d0f63355a57092 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 4 Oct 2021 21:41:11 +0800 Subject: [PATCH 38/44] remove some system include files --- include/os/osFile.h | 2 - include/os/osSocket.h | 46 ++++++------------ include/os/osSysinfo.h | 2 + source/libs/transport/src/rpcMain.c | 2 +- source/os/src/osRand.c | 1 + source/os/src/osSemaphore.c | 1 + source/os/src/osSleep.c | 2 + source/os/src/osSocket.c | 61 +++++++++++++++++++----- source/os/src/osSysinfo.c | 6 +++ source/os/src/osTimer.c | 2 + source/server/dnode/src/dnodeTelemetry.c | 2 +- source/util/src/tfile.c | 4 +- source/util/src/tnote.c | 10 ++-- 13 files changed, 88 insertions(+), 53 deletions(-) diff --git a/include/os/osFile.h b/include/os/osFile.h index 83f90d4ba2..8d03759d82 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -24,10 +24,8 @@ extern "C" { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) typedef int32_t FileFd; -typedef SOCKET SocketFd; #else typedef int32_t FileFd; -typedef int32_t SocketFd; #endif #define FD_INITIALIZER ((int32_t)-1) diff --git a/include/os/osSocket.h b/include/os/osSocket.h index c503e667e6..d76b286d65 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -27,49 +27,33 @@ extern "C" { #include #else #include - #include - #include - #include #include - #include #endif -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - #define taosSend(sockfd, buf, len, flags) send((SOCKET)sockfd, buf, len, flags) - #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) sendto((SOCKET)sockfd, buf, len, flags, dest_addr, addrlen) - #define taosWriteSocket(fd, buf, len) send((SOCKET)fd, buf, len, 0) - #define taosReadSocket(fd, buf, len) recv((SOCKET)fd, buf, len, 0) - #define taosCloseSocketNoCheck(fd) closesocket((SOCKET)fd) - #define taosCloseSocket(fd) closesocket((SOCKET)fd) -#else - #define taosSend(sockfd, buf, len, flags) send(sockfd, buf, len, flags) - #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) sendto(sockfd, buf, len, flags, dest_addr, addrlen) - #define taosReadSocket(fd, buf, len) read(fd, buf, len) - #define taosWriteSocket(fd, buf, len) write(fd, buf, len) - #define taosCloseSocketNoCheck(x) close(x) - #define taosCloseSocket(x) \ - { \ - if ((x) > -1) { \ - close(x); \ - x = FD_INITIALIZER; \ - } \ - } -#endif - -#define TAOS_EPOLL_WAIT_TIME 500 +#define TAOS_EPOLL_WAIT_TIME 500 typedef int32_t SOCKET; -typedef SOCKET EpollFd; +typedef SOCKET EpollFd; #define EpollClose(pollFd) taosCloseSocket(pollFd) -void taosShutDownSocketRD(SOCKET fd); -void taosShutDownSocketWR(SOCKET fd); +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +typedef SOCKET SocketFd; +#else +typedef int32_t SocketFd; +#endif +int32_t taosSendto(SocketFd fd, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen); +int32_t taosWriteSocket(SocketFd fd, void *msg, int len); +int32_t taosReadSocket(SocketFd fd, void *msg, int len); +int32_t taosCloseSocketNoCheck(SocketFd fd); +int32_t taosCloseSocket(SocketFd fd); +void taosShutDownSocketRD(SOCKET fd); +void taosShutDownSocketWR(SOCKET fd); int32_t taosSetNonblocking(SOCKET sock, int32_t on); void taosIgnSIGPIPE(); void taosBlockSIGPIPE(); void taosSetMaskSIGPIPE(); int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); -int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t* optlen); +int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen); uint32_t taosInetAddr(char *ipAddr); const char *taosInetNtoa(struct in_addr ipInt); diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 688fb8f7ef..a3919890bd 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -66,6 +66,8 @@ typedef struct { SysNameInfo taosGetSysNameInfo(); +int64_t taosGetPid(); + #ifdef __cplusplus } #endif diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index fe58ee2746..800b146713 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -695,7 +695,7 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc) { pConn->sid = sid; pConn->tranId = (uint16_t)(taosRand() & 0xFFFF); pConn->ownId = htonl(pConn->sid); - pConn->linkUid = (uint32_t)((int64_t)pConn + (int64_t)getpid() + (int64_t)pConn->tranId); + pConn->linkUid = (uint32_t)((int64_t)pConn + taosGetPid() + (int64_t)pConn->tranId); pConn->spi = pRpc->spi; pConn->encrypt = pRpc->encrypt; if (pConn->spi) memcpy(pConn->secret, pRpc->secret, TSDB_KEY_LEN); diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index 247097f9ce..973323a346 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -18,6 +18,7 @@ #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #else #include +#include #endif uint32_t taosRand(void) { return rand(); } diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index c9533c8b8a..2385d10285 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -376,6 +376,7 @@ int32_t taosGetCurrentAPPName(char *name, int32_t *len) { */ #include +#include bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; } diff --git a/source/os/src/osSleep.c b/source/os/src/osSleep.c index 9c3231fd13..3b90fbdad8 100644 --- a/source/os/src/osSleep.c +++ b/source/os/src/osSleep.c @@ -22,6 +22,8 @@ void taosMsleep(int32_t ms) { Sleep(ms); } #else +#include + /* to make taosMsleep work, signal SIGALRM shall be blocked in the calling thread, diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 0b62df4524..fb1aeebe1c 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -17,16 +17,57 @@ #include "os.h" #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + #include "winsock2.h" + #include + #include + #include #else -#include -#include -#include -#include -#include -#include -#include -#include -#include + #include + #include + #include + #include + #include + #include + #include + #include + #include +#endif + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + +#define taosSend(sockfd, buf, len, flags) send((SOCKET)sockfd, buf, len, flags) +int32_t taosSendto(SocketFd fd, void *buf, int len, unsigned int flags, const struct sockaddr *to, int tolen) { + return sendto((SOCKET)sockfd, buf, len, flags, dest_addr, addrlen); +} +int32_t taosWriteSocket(SocketFd fd, void *buf, int len) { return send((SOCKET)fd, buf, len, 0); } +int32_t taosReadSocket(SocketFd fd, void *buf, int len) { return recv((SOCKET)fd, buf, len, 0)(); } +int32_t taosCloseSocketNoCheck(SocketFd fd) { return closesocket((SOCKET)fd); } +int32_t taosCloseSocket(SocketFd fd) { closesocket((SOCKET)fd) } + +#else + + #define taosSend(sockfd, buf, len, flags) send(sockfd, buf, len, flags) + int32_t taosSendto(SocketFd fd, void * buf, int len, unsigned int flags, const struct sockaddr * dest_addr, int addrlen) { + return sendto(fd, buf, len, flags, dest_addr, addrlen); + } + + int32_t taosWriteSocket(SocketFd fd, void *buf, int len) { + return write(fd, buf, len); + } + + int32_t taosReadSocket(SocketFd fd, void *buf, int len) { + return read(fd, buf, len); + } + + int32_t taosCloseSocketNoCheck(SocketFd fd) { + return close(fd); + } + + int32_t taosCloseSocket(SocketFd fd) { + if (fd > -1) { + close(fd); + } + } #endif void taosShutDownSocketRD(SOCKET fd) { @@ -226,8 +267,6 @@ uint64_t htonll(uint64_t val) { return (((uint64_t)htonl(val)) << 32) + htonl(va #endif - - #ifndef SIGPIPE #define SIGPIPE EPIPE #endif diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 54af2ab4ad..0344507f5e 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -485,6 +485,7 @@ char *taosGetCmdlineByPID(int pid) { #include #include #include +#include #define PROCESS_ITEM 12 @@ -1127,4 +1128,9 @@ SysNameInfo taosGetSysNameInfo() { return info; } + +int64_t taosGetPid() { + getpid(); +} + #endif \ No newline at end of file diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c index 2f18d76db9..b1bf1bcd2d 100644 --- a/source/os/src/osTimer.c +++ b/source/os/src/osTimer.c @@ -62,6 +62,7 @@ void taosUninitTimer() { #include #include +#include static void (*timer_callback)(int); static int timer_ms = 0; @@ -136,6 +137,7 @@ void taos_block_sigalrm(void) { */ #include +#include static void taosDeleteTimer(void *tharg) { timer_t *pTimer = tharg; diff --git a/source/server/dnode/src/dnodeTelemetry.c b/source/server/dnode/src/dnodeTelemetry.c index 089a614590..5d1d9b4aa4 100644 --- a/source/server/dnode/src/dnodeTelemetry.c +++ b/source/server/dnode/src/dnodeTelemetry.c @@ -210,7 +210,7 @@ static void dnodeSendTelemetryReport(DnTelem* telem) { "Content-Type: application/json\n" "Content-Length: "; - taosWriteSocket(fd, header, (int32_t)strlen(header)); + taosWriteSocket(fd, (void*)header, (int32_t)strlen(header)); int32_t contLen = (int32_t)(tbufTell(&bw) - 1); sprintf(buf, "%d\n\n", contLen); taosWriteSocket(fd, buf, (int32_t)strlen(buf)); diff --git a/source/util/src/tfile.c b/source/util/src/tfile.c index 2dafd689d3..5b61186b12 100644 --- a/source/util/src/tfile.c +++ b/source/util/src/tfile.c @@ -23,7 +23,7 @@ static int32_t tsFileRsetId = -1; static void tfCloseFile(void *p) { - close((int32_t)(uintptr_t)p); + taosCloseFile((int32_t)(uintptr_t)p); } int32_t tfInit() { @@ -48,7 +48,7 @@ static int64_t tfOpenImp(int32_t fd) { void * p = (void *)(int64_t)fd; int64_t rid = taosAddRef(tsFileRsetId, p); - if (rid < 0) close(fd); + if (rid < 0) taosCloseFile(fd); return rid; } diff --git a/source/util/src/tnote.c b/source/util/src/tnote.c index e4a326ba8f..50b1e0002d 100644 --- a/source/util/src/tnote.c +++ b/source/util/src/tnote.c @@ -100,7 +100,7 @@ static void *taosThreadToOpenNewNote(void *param) { } taosLockNote(fd, pNote); - (void)lseek(fd, 0, SEEK_SET); + (void)taosLSeekFile(fd, 0, SEEK_SET); int32_t oldFd = pNote->fd; pNote->fd = fd; @@ -142,10 +142,10 @@ static bool taosCheckNoteIsOpen(char *noteName, SNoteObj *pNote) { if (taosLockNote(fd, pNote)) { taosUnLockNote(fd, pNote); - close(fd); + taosCloseFile(fd); return false; } else { - close(fd); + taosCloseFile(fd); return true; } } @@ -226,7 +226,7 @@ static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxN size = (int32_t)filestat_size; pNote->lines = size / 60; - lseek(pNote->fd, 0, SEEK_END); + taosLSeekFile(pNote->fd, 0, SEEK_END); return 0; } @@ -271,6 +271,6 @@ void taosNotePrint(SNoteObj *pNote, const char *const format, ...) { static void taosCloseNoteByFd(int32_t fd, SNoteObj *pNote) { if (fd >= 0) { taosUnLockNote(fd, pNote); - close(fd); + taosCloseFile(fd); } } From 9fae7a3dfe166f1b39a2ba497dffe4a7b0e07670 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 4 Oct 2021 21:42:21 +0800 Subject: [PATCH 39/44] minor changes --- include/os/osThread.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/os/osThread.h b/include/os/osThread.h index ecb085cd06..79834dc9a5 100644 --- a/include/os/osThread.h +++ b/include/os/osThread.h @@ -21,7 +21,6 @@ extern "C" { #endif #include -#include #ifdef __cplusplus } From e168709b729ee1cf9af508c9ae77659419901ac3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 5 Oct 2021 13:16:54 +0800 Subject: [PATCH 40/44] [TD-10430] refact interface of dnode --- include/libs/transport/trpc.h | 7 +- include/server/dnode/dnode.h | 22 +- include/server/mnode/mnode.h | 41 +--- include/server/vnode/vnode.h | 34 +-- include/util/tstep.h | 29 +-- source/libs/transport/src/rpcMain.c | 14 +- source/libs/transport/src/rpcTcp.c | 14 +- source/server/dnode/inc/dnodeCfg.h | 3 +- source/server/dnode/inc/dnodeCheck.h | 3 +- source/server/dnode/inc/dnodeEps.h | 5 +- source/server/dnode/inc/dnodeInt.h | 2 + source/server/dnode/inc/dnodeMain.h | 17 +- source/server/dnode/inc/dnodeMnodeEps.h | 5 +- source/server/dnode/inc/dnodeStatus.h | 5 +- .../inc/{dnodeTelemetry.h => dnodeTelem.h} | 5 +- source/server/dnode/inc/dnodeTrans.h | 32 +-- source/server/dnode/src/dnodeCfg.c | 3 +- source/server/dnode/src/dnodeCheck.c | 3 +- source/server/dnode/src/dnodeEps.c | 11 +- source/server/dnode/src/dnodeInt.c | 193 +++------------ source/server/dnode/src/dnodeMain.c | 32 +-- source/server/dnode/src/dnodeMnodeEps.c | 11 +- source/server/dnode/src/dnodeStatus.c | 18 +- .../src/{dnodeTelemetry.c => dnodeTelem.c} | 14 +- source/server/dnode/src/dnodeTrans.c | 228 +++++++++--------- source/server/mnode/src/mnodeMain.c | 20 +- source/server/server.c | 6 +- source/server/vnode/src/vnodeMain.c | 14 +- source/util/src/tstep.c | 41 +++- 29 files changed, 315 insertions(+), 517 deletions(-) rename source/server/dnode/inc/{dnodeTelemetry.h => dnodeTelem.h} (89%) rename source/server/dnode/src/{dnodeTelemetry.c => dnodeTelem.c} (96%) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 2a0efab213..0ce2e3da14 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -64,9 +64,6 @@ typedef struct SRpcInit { int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS int idleTime; // milliseconds, 0 means idle timer is disabled - // owner of the rpc client/server, - void *owner; // set by the app when rpc init - // the following is for client app ecurity only char *user; // user name char spi; // security parameter index @@ -75,10 +72,10 @@ typedef struct SRpcInit { char *ckey; // ciphering key // call back to process incoming msg, code shall be ignored by server app - void (*cfp)(void *owner, SRpcMsg *, SRpcEpSet *); + void (*cfp)(SRpcMsg *, SRpcEpSet *); // call back to retrieve the client auth info, for server app only - int (*afp)(void *owner, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); + int (*afp)(char *tableId, char *spi, char *encrypt, char *secret, char *ckey); } SRpcInit; int32_t rpcInit(); diff --git a/include/server/dnode/dnode.h b/include/server/dnode/dnode.h index a41235e7c9..d7aaa0e008 100644 --- a/include/server/dnode/dnode.h +++ b/include/server/dnode/dnode.h @@ -22,58 +22,50 @@ extern "C" { struct SRpcEpSet; struct SRpcMsg; -struct Dnode; - /** * Initialize and start the dnode module. * - * @return Instance of dnode module. + * @return Error code. */ -struct Dnode *dnodeCreateInstance(); +int32_t dnodeInit(); /** * Stop and cleanup dnode module. - * - * @param dnode, instance of dnode module. */ -void dnodeDropInstance(struct Dnode *dnode); +void dnodeCleanup(); /** * Send messages to other dnodes, such as create vnode message. * - * @param dnode, the instance of Dnode module. * @param epSet, the endpoint list of the dnodes. * @param rpcMsg, message to be sent. */ -void dnodeSendMsgToDnode(struct Dnode *dnode, struct SRpcEpSet *epSet, struct SRpcMsg *rpcMsg); +void dnodeSendMsgToDnode(struct SRpcEpSet *epSet, struct SRpcMsg *rpcMsg); /** * Send messages to mnode, such as config message. * - * @param dnode, the instance of dnode module. * @param rpcMsg, message to be sent. */ -void dnodeSendMsgToMnode(struct Dnode *dnode, struct SRpcMsg *rpcMsg); +void dnodeSendMsgToMnode(struct SRpcMsg *rpcMsg); /** * Send redirect message to dnode or shell. * - * @param dnode, the instance of dnode module. * @param rpcMsg, message to be sent. * @param forShell, used to identify whether to send to shell or dnode. */ -void dnodeSendRedirectMsg(struct Dnode *dnode, struct SRpcMsg *rpcMsg, bool forShell); +void dnodeSendRedirectMsg(struct SRpcMsg *rpcMsg, bool forShell); /** * Get the corresponding endpoint information from dnodeId. * - * @param dnode, the instance of dnode module. * @param dnodeId, the id ot dnode. * @param ep, the endpoint of dnode. * @param fqdn, the fqdn of dnode. * @param port, the port of dnode. */ -void dnodeGetDnodeEp(struct Dnode *dnode, int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); +void dnodeGetDnodeEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); #ifdef __cplusplus } diff --git a/include/server/mnode/mnode.h b/include/server/mnode/mnode.h index 7a06880a16..e78994fc9c 100644 --- a/include/server/mnode/mnode.h +++ b/include/server/mnode/mnode.h @@ -20,34 +20,29 @@ extern "C" { #endif -struct Dnode; - typedef struct { /** * Send messages to other dnodes, such as create vnode message. * - * @param dnode, the instance of dnode module. * @param epSet, the endpoint list of the dnodes. * @param rpcMsg, message to be sent. */ - void (*SendMsgToDnode)(struct Dnode *dnode, struct SRpcEpSet *epSet, struct SRpcMsg *rpcMsg); + void (*SendMsgToDnode)(struct SRpcEpSet *epSet, struct SRpcMsg *rpcMsg); /** * Send messages to mnode, such as config message. * - * @param dnode, the instance of dnode module. * @param rpcMsg, message to be sent. */ - void (*SendMsgToMnode)(struct Dnode *dnode, struct SRpcMsg *rpcMsg); + void (*SendMsgToMnode)(struct SRpcMsg *rpcMsg); /** * Send redirect message to dnode or shell. * - * @param dnode, the instance of dnode module. * @param rpcMsg, message to be sent. * @param forShell, used to identify whether to send to shell or dnode. */ - void (*SendRedirectMsg)(struct Dnode *dnode, struct SRpcMsg *rpcMsg, bool forShell); + void (*SendRedirectMsg)(struct SRpcMsg *rpcMsg, bool forShell); /** * Get the corresponding endpoint information from dnodeId. @@ -58,12 +53,11 @@ typedef struct { * @param fqdn, the fqdn of dnode. * @param port, the port of dnode. */ - void (*GetDnodeEp)(struct Dnode *dnode, int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); + void (*GetDnodeEp)(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); } SMnodeFp; typedef struct { - struct Dnode *dnode; SMnodeFp fp; char clusterId[TSDB_CLUSTER_ID_LEN]; int32_t dnodeId; @@ -73,40 +67,34 @@ typedef struct { * Initialize and start mnode module. * * @param para, initialization parameters. - * @return Instance of mnode module. + * @return Error code. */ -struct Mnode *mnodeCreateInstance(SMnodePara para); +int32_t mnodeInit(SMnodePara para); /** * Stop and cleanup mnode module. - * - * @param mnode, instance of mnode module. */ -void mnodeDropInstance(struct Mnode *vnode); +void mnodeCleanup(); /** * Deploy mnode instances in dnode. * - * @param mnode, instance of mnode module. * @param minfos, server information used to deploy the mnode instance. * @return Error Code. */ -int32_t mnodeDeploy(struct Mnode *mnode, struct SMInfos *minfos); +int32_t mnodeDeploy(struct SMInfos *minfos); /** * Delete the mnode instance deployed in dnode. - * - * @param mnode, instance of mnode module. */ -void mnodeUnDeploy(struct Mnode *mnode); +void mnodeUnDeploy(); /** * Whether the mnode is in service. * - * @param mnode, instance of mnode module. * @return Server status. */ -bool mnodeIsServing(struct Mnode *mnode); +bool mnodeIsServing(); typedef struct { int64_t numOfDnode; @@ -124,16 +112,14 @@ typedef struct { /** * Get the statistical information of Mnode. * - * @param mnode, instance of mnode module. * @param stat, statistical information. * @return Error Code. */ -int32_t mnodeGetStatistics(struct Mnode *mnode, SMnodeStat *stat); +int32_t mnodeGetStatistics(SMnodeStat *stat); /** * Get the statistical information of Mnode. * - * @param mnode, instance of mnode module. * @param user, username. * @param spi, security parameter index. * @param encrypt, encrypt algorithm. @@ -141,16 +127,15 @@ int32_t mnodeGetStatistics(struct Mnode *mnode, SMnodeStat *stat); * @param ckey, ciphering key. * @return Error Code. */ -int32_t mnodeRetriveAuth(struct Mnode *mnode, char *user, char *spi, char *encrypt, char *secret, char *ckey); +int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey); /** * Interface for processing messages. * - * @param mnode, instance of mnode module. * @param rpcMsg, message to be processed. * @return Error code. */ -void mnodeProcessMsg(struct Mnode *mnode, SRpcMsg *rpcMsg); +void mnodeProcessMsg(SRpcMsg *rpcMsg); #ifdef __cplusplus } diff --git a/include/server/vnode/vnode.h b/include/server/vnode/vnode.h index ee015788d3..8fd4fd433f 100644 --- a/include/server/vnode/vnode.h +++ b/include/server/vnode/vnode.h @@ -20,41 +20,35 @@ extern "C" { #endif -struct Dnode; - typedef struct { /** * Send messages to other dnodes, such as create vnode message. * - * @param dnode, the instance of dnode module. * @param epSet, the endpoint list of dnodes. * @param rpcMsg, message to be sent. */ - void (*SendMsgToDnode)(struct Dnode *dnode, struct SRpcEpSet *epSet, struct SRpcMsg *rpcMsg); + void (*SendMsgToDnode)(struct SRpcEpSet *epSet, struct SRpcMsg *rpcMsg); /** * Send messages to mnode, such as config message. * - * @param dnode, the instance of dnode module. * @param rpcMsg, message to be sent. */ - void (*SendMsgToMnode)(struct Dnode *dnode, struct SRpcMsg *rpcMsg); + void (*SendMsgToMnode)(struct SRpcMsg *rpcMsg); /** * Get the corresponding endpoint information from dnodeId. * - * @param dnode, the instance of dnode module. * @param dnodeId, the id ot dnode. * @param ep, the endpoint of dnode. * @param fqdn, the fqdn of dnode. * @param port, the port of dnode. */ - void (*GetDnodeEp)(struct Dnode *dnode, int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); + void (*GetDnodeEp)(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); } SVnodeFp; typedef struct { - struct Dnode *dnode; SVnodeFp fp; } SVnodePara; @@ -62,16 +56,14 @@ typedef struct { * Start initialize vnode module. * * @param para, initialization parameters. - * @return Instance of vnode module. + * @return Error code. */ -struct Vnode *vnodeCreateInstance(SVnodePara para); +int32_t vnodeInit(SVnodePara para); /** * Cleanup vnode module. - * - * @param vnode, instance of vnode module. */ -void vnodeDropInstance(struct Vnode *vnode); +void vnodeCleanup(); typedef struct { int32_t unused; @@ -80,36 +72,32 @@ typedef struct { /** * Get the statistical information of vnode. * - * @param vnode, instance of vnode module. - * @param sta, statistical information. + * @param stat, statistical information. * @return Error Code. */ -int32_t vnodeGetStatistics(struct Vnode *vnode, SVnodeStat *stat); +int32_t vnodeGetStatistics(SVnodeStat *stat); /** * Get the status of all vnodes. * - * @param vnode, instance of vnode module. * @param status, status msg. */ -void vnodeGetStatus(struct Vnode *vnode, struct SStatusMsg *status); +void vnodeGetStatus(struct SStatusMsg *status); /** * Set access permissions for all vnodes. * - * @param vnode, instance of vnode module. * @param access, access permissions of vnodes. * @param numOfVnodes, the size of vnodes. */ -void vnodeSetAccess(struct Vnode *vnode, struct SVgroupAccess *access, int32_t numOfVnodes); +void vnodeSetAccess(struct SVgroupAccess *access, int32_t numOfVnodes); /** * Interface for processing messages. * - * @param vnode, instance of vnode module. * @param msg, message to be processed. */ -void vnodeProcessMsg(struct Vnode *vnode, SRpcMsg *msg); +void vnodeProcessMsg(SRpcMsg *msg); #ifdef __cplusplus } diff --git a/include/util/tstep.h b/include/util/tstep.h index 90dd5dd0fc..87e95edd97 100644 --- a/include/util/tstep.h +++ b/include/util/tstep.h @@ -20,29 +20,14 @@ extern "C" { #endif -typedef int32_t (*FnInitObj)(void *parent, void **self); -typedef void (*FnCleanupObj)(void **self); -typedef void (*FnReportProgress)(void *parent, const char *name, const char *desc); +typedef int32_t (*InitFp)(void **obj); +typedef void (*CleanupFp)(void **obj); +typedef void (*ReportFp)(char *name, char *desc); -typedef struct SStepObj { - const char * name; - void * parent; - void ** self; - FnInitObj initFp; - FnCleanupObj cleanupFp; - FnReportProgress reportFp; -} SStepObj; - -typedef struct SSteps { - int32_t cursize; - int32_t maxsize; - SStepObj *steps; -} SSteps; - -SSteps *taosStepInit(int32_t stepsize); -int32_t taosStepAdd(SSteps *steps, SStepObj *step); -int32_t taosStepExec(SSteps *steps); -void taosStepCleanup(SSteps *steps); +struct SSteps *taosStepInit(int32_t maxsize, ReportFp fp); +int32_t taosStepExec(struct SSteps *steps); +void taosStepCleanup(struct SSteps *steps); +int32_t taosStepAdd(struct SSteps *steps, char *name, void **obj, InitFp initFp, CleanupFp cleanupFp); #ifdef __cplusplus } diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 800b146713..1f4a1ba3cf 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -54,9 +54,8 @@ typedef struct { char secret[TSDB_KEY_LEN]; // secret for the link char ckey[TSDB_KEY_LEN]; // ciphering key - void *owner; - void (*cfp)(void * owner, SRpcMsg *, SRpcEpSet *); - int (*afp)(void * owner, char *user, char *spi, char *encrypt, char *secret, char *ckey); + void (*cfp)(SRpcMsg *, SRpcEpSet *); + int (*afp)(char *user, char *spi, char *encrypt, char *secret, char *ckey); int32_t refCount; void *idPool; // handle to ID pool @@ -259,7 +258,6 @@ void *rpcOpen(const SRpcInit *pInit) { if (pInit->secret) memcpy(pRpc->secret, pInit->secret, sizeof(pRpc->secret)); if (pInit->ckey) tstrncpy(pRpc->ckey, pInit->ckey, sizeof(pRpc->ckey)); pRpc->spi = pInit->spi; - pRpc->owner = pInit->owner; pRpc->cfp = pInit->cfp; pRpc->afp = pInit->afp; pRpc->refCount = 1; @@ -742,7 +740,7 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { if (pConn->user[0] == 0) { terrno = TSDB_CODE_RPC_AUTH_REQUIRED; } else { - terrno = (*pRpc->afp)(pRpc->owner, pConn->user, &pConn->spi, &pConn->encrypt, pConn->secret, pConn->ckey); + terrno = (*pRpc->afp)(pConn->user, &pConn->spi, &pConn->encrypt, pConn->secret, pConn->ckey); } if (terrno != 0) { @@ -1022,7 +1020,7 @@ static void doRpcReportBrokenLinkToServer(void *param, void *id) { SRpcMsg *pRpcMsg = (SRpcMsg *)(param); SRpcConn *pConn = (SRpcConn *)(pRpcMsg->handle); SRpcInfo *pRpc = pConn->pRpc; - (*(pRpc->cfp))(pRpc->owner, pRpcMsg, NULL); + (*(pRpc->cfp))(pRpcMsg, NULL); free(pRpcMsg); } static void rpcReportBrokenLinkToServer(SRpcConn *pConn) { @@ -1137,7 +1135,7 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) { if (pContext->epSet.inUse != pContext->oldInUse || pContext->redirect) pEpSet = &pContext->epSet; - (*pRpc->cfp)(pRpc->owner, pMsg, pEpSet); + (*pRpc->cfp)(pMsg, pEpSet); } // free the request message @@ -1161,7 +1159,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte rpcAddRef(pRpc); // add the refCount for requests // notify the server app - (*(pRpc->cfp))(pRpc->owner, &rpcMsg, NULL); + (*(pRpc->cfp))(&rpcMsg, NULL); } else { // it's a response rpcMsg.handle = pContext; diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index d853fdb9f1..d0710c883f 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -194,7 +194,7 @@ void taosStopTcpServer(void *handle) { pServerObj->stop = 1; if (pServerObj->fd >= 0) { - taosShutDownSocketRD(pServerObj->fd); + taosShutDownSocketRD(pServerObj->fd); } if (taosCheckPthreadValid(pServerObj->thread)) { if (taosComparePthread(pServerObj->thread, pthread_self())) { @@ -257,8 +257,8 @@ static void *taosAcceptTcpConnection(void *arg) { int32_t ret = taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to)); if (ret != 0) { taosCloseSocket(connFd); - tError("%s failed to set recv timeout fd(%s)for connection from:%hu", pServerObj->label, strerror(errno), - htons(caddr.sin_port)); + tError("%s failed to set recv timeout fd(%s)for connection from:%s:%hu", pServerObj->label, strerror(errno), + taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port)); continue; } @@ -270,12 +270,12 @@ static void *taosAcceptTcpConnection(void *arg) { if (pFdObj) { pFdObj->ip = caddr.sin_addr.s_addr; pFdObj->port = htons(caddr.sin_port); - tDebug("%s new TCP connection from %hu, fd:%d FD:%p numOfFds:%d", pServerObj->label, - pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds); + tDebug("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label, + taosInetNtoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds); } else { taosCloseSocket(connFd); - tError("%s failed to malloc FdObj(%s) for connection from:%hu", pServerObj->label, strerror(errno), - htons(caddr.sin_port)); + tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno), + taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port)); } // pick up next thread for next connection diff --git a/source/server/dnode/inc/dnodeCfg.h b/source/server/dnode/inc/dnodeCfg.h index 2284486e51..7a0d830a4b 100644 --- a/source/server/dnode/inc/dnodeCfg.h +++ b/source/server/dnode/inc/dnodeCfg.h @@ -22,7 +22,6 @@ extern "C" { #include "dnodeInt.h" typedef struct DnCfg { - Dnode * dnode; int32_t dnodeId; int32_t dropped; char clusterId[TSDB_CLUSTER_ID_LEN]; @@ -30,7 +29,7 @@ typedef struct DnCfg { pthread_mutex_t mutex; } DnCfg; -int32_t dnodeInitCfg(Dnode *dnode, DnCfg **cfg); +int32_t dnodeInitCfg(DnCfg **cfg); void dnodeCleanupCfg(DnCfg **cfg); void dnodeUpdateCfg(DnCfg *cfg, SDnodeCfg *data); int32_t dnodeGetDnodeId(DnCfg *cfg); diff --git a/source/server/dnode/inc/dnodeCheck.h b/source/server/dnode/inc/dnodeCheck.h index 84232fbc1d..db275259f5 100644 --- a/source/server/dnode/inc/dnodeCheck.h +++ b/source/server/dnode/inc/dnodeCheck.h @@ -22,10 +22,9 @@ extern "C" { #include "dnodeInt.h" typedef struct DnCheck { - Dnode *dnode; } DnCheck; -int32_t dnodeInitCheck(Dnode *dnode, DnCheck **check); +int32_t dnodeInitCheck(DnCheck **check); void dnodeCleanupCheck(DnCheck **check); #ifdef __cplusplus diff --git a/source/server/dnode/inc/dnodeEps.h b/source/server/dnode/inc/dnodeEps.h index aec412ee59..79c49bde40 100644 --- a/source/server/dnode/inc/dnodeEps.h +++ b/source/server/dnode/inc/dnodeEps.h @@ -23,7 +23,6 @@ extern "C" { #include "dnodeInt.h" typedef struct DnEps { - Dnode * dnode; int32_t dnodeId; int32_t dnodeNum; SDnodeEp * dnodeList; @@ -32,11 +31,11 @@ typedef struct DnEps { pthread_mutex_t mutex; } DnEps; -int32_t dnodeInitEps(Dnode *dnode, DnEps **eps); +int32_t dnodeInitEps(DnEps **eps); void dnodeCleanupEps(DnEps **eps); void dnodeUpdateEps(DnEps *eps, SDnodeEps *data); bool dnodeIsDnodeEpChanged(DnEps *eps, int32_t dnodeId, char *epstr); -void dnodeGetDnodeEp(Dnode *dnode, int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port); +void dnodeGetDnodeEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port); #ifdef __cplusplus } diff --git a/source/server/dnode/inc/dnodeInt.h b/source/server/dnode/inc/dnodeInt.h index b0e5a448e0..b32e23db14 100644 --- a/source/server/dnode/inc/dnodeInt.h +++ b/source/server/dnode/inc/dnodeInt.h @@ -52,6 +52,8 @@ typedef struct Dnode { struct Vnode * vnode; } Dnode; +Dnode* dnodeInst(); + #define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", 255, __VA_ARGS__); }} #define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", 255, __VA_ARGS__); }} #define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", 255, __VA_ARGS__); }} diff --git a/source/server/dnode/inc/dnodeMain.h b/source/server/dnode/inc/dnodeMain.h index f9c15db1c1..9fe5d64800 100644 --- a/source/server/dnode/inc/dnodeMain.h +++ b/source/server/dnode/inc/dnodeMain.h @@ -28,21 +28,20 @@ typedef enum { } RunStat; typedef struct DnMain { - Dnode * dnode; RunStat runStatus; void * dnodeTimer; SStartupStep startup; } DnMain; -int32_t dnodeInitMain(Dnode *dnode, DnMain **main); +int32_t dnodeInitMain(DnMain **main); void dnodeCleanupMain(DnMain **main); -int32_t dnodeInitStorage(Dnode *dnode, void **unused); -void dnodeCleanupStorage(void **unused); -void dnodeReportStartup(Dnode *dnode, char *name, char *desc); -void dnodeReportStartupFinished(Dnode *dnode, char *name, char *desc); -void dnodeProcessStartupReq(Dnode *dnode, SRpcMsg *pMsg); -void dnodeProcessCreateMnodeReq(Dnode *dnode, SRpcMsg *pMsg); -void dnodeProcessConfigDnodeReq(Dnode *dnode, SRpcMsg *pMsg); +int32_t dnodeInitStorage(); +void dnodeCleanupStorage(); +void dnodeReportStartup(char *name, char *desc); +void dnodeReportStartupFinished(char *name, char *desc); +void dnodeProcessStartupReq(SRpcMsg *pMsg); +void dnodeProcessCreateMnodeReq(SRpcMsg *pMsg); +void dnodeProcessConfigDnodeReq(SRpcMsg *pMsg); #ifdef __cplusplus } diff --git a/source/server/dnode/inc/dnodeMnodeEps.h b/source/server/dnode/inc/dnodeMnodeEps.h index 950fc3783d..b94b08c304 100644 --- a/source/server/dnode/inc/dnodeMnodeEps.h +++ b/source/server/dnode/inc/dnodeMnodeEps.h @@ -22,20 +22,19 @@ extern "C" { #include "dnodeInt.h" typedef struct DnMnEps { - Dnode * dnode; SRpcEpSet mnodeEpSet; SMInfos mnodeInfos; char file[PATH_MAX + 20]; pthread_mutex_t mutex; } DnMnEps; -int32_t dnodeInitMnodeEps(Dnode *dnode, DnMnEps **meps); +int32_t dnodeInitMnodeEps(DnMnEps **meps); void dnodeCleanupMnodeEps(DnMnEps **meps); void dnodeUpdateMnodeFromStatus(DnMnEps *meps, SMInfos *pMinfos); void dnodeUpdateMnodeFromPeer(DnMnEps *meps, SRpcEpSet *pEpSet); void dnodeGetEpSetForPeer(DnMnEps *meps, SRpcEpSet *epSet); void dnodeGetEpSetForShell(DnMnEps *meps, SRpcEpSet *epSet); -void dnodeSendRedirectMsg(Dnode *dnode, SRpcMsg *rpcMsg, bool forShell); +void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell); #ifdef __cplusplus } diff --git a/source/server/dnode/inc/dnodeStatus.h b/source/server/dnode/inc/dnodeStatus.h index ed7782fb36..cfd58578c0 100644 --- a/source/server/dnode/inc/dnodeStatus.h +++ b/source/server/dnode/inc/dnodeStatus.h @@ -22,15 +22,14 @@ extern "C" { #include "dnodeInt.h" typedef struct DnStatus { - Dnode * dnode; void * dnodeTimer; void * statusTimer; uint32_t rebootTime; } DnStatus; -int32_t dnodeInitStatus(Dnode *dnode, DnStatus **status); +int32_t dnodeInitStatus(DnStatus **status); void dnodeCleanupStatus(DnStatus **status); -void dnodeProcessStatusRsp(Dnode *dnode, SRpcMsg *pMsg); +void dnodeProcessStatusRsp(SRpcMsg *pMsg); #ifdef __cplusplus } diff --git a/source/server/dnode/inc/dnodeTelemetry.h b/source/server/dnode/inc/dnodeTelem.h similarity index 89% rename from source/server/dnode/inc/dnodeTelemetry.h rename to source/server/dnode/inc/dnodeTelem.h index e3e097d2f2..48b10399bd 100644 --- a/source/server/dnode/inc/dnodeTelemetry.h +++ b/source/server/dnode/inc/dnodeTelem.h @@ -26,7 +26,6 @@ extern "C" { * thus we use pthread_mutex_t/pthread_cond_t to simulate */ typedef struct DnTelem { - Dnode * dnode; bool enable; pthread_mutex_t lock; pthread_cond_t cond; @@ -35,8 +34,8 @@ typedef struct DnTelem { char email[TSDB_FQDN_LEN]; } DnTelem; -int32_t dnodeInitTelemetry(Dnode *dnode, DnTelem **telem); -void dnodeCleanupTelemetry(DnTelem **telem); +int32_t dnodeInitTelem(DnTelem **telem); +void dnodeCleanupTelem(DnTelem **telem); #ifdef __cplusplus } diff --git a/source/server/dnode/inc/dnodeTrans.h b/source/server/dnode/inc/dnodeTrans.h index ad2894c6f2..306624428b 100644 --- a/source/server/dnode/inc/dnodeTrans.h +++ b/source/server/dnode/inc/dnodeTrans.h @@ -21,30 +21,24 @@ extern "C" { #endif #include "dnodeInt.h" -typedef void (*RpcMsgCfp)(void *owner, SRpcMsg *pMsg, SRpcEpSet *pEpSet); -typedef void (*RpcMsgFp)(void *owner, SRpcMsg *pMsg); - -typedef struct DnMsgFp { - void * module; - RpcMsgFp fp; -} DnMsgFp; +typedef void (*RpcMsgFp)( SRpcMsg *pMsg); typedef struct DnTrans { - Dnode * dnode; - void * serverRpc; - void * clientRpc; - void * shellRpc; - int32_t queryReqNum; - int32_t submitReqNum; - DnMsgFp fpPeerMsg[TSDB_MSG_TYPE_MAX]; - DnMsgFp fpShellMsg[TSDB_MSG_TYPE_MAX]; + void * serverRpc; + void * clientRpc; + void * shellRpc; + int32_t queryReqNum; + int32_t submitReqNum; + RpcMsgFp peerMsgFp[TSDB_MSG_TYPE_MAX]; + RpcMsgFp shellMsgFp[TSDB_MSG_TYPE_MAX]; + } DnTrans; -int32_t dnodeInitTrans(Dnode *dnode, DnTrans **trans); +int32_t dnodeInitTrans(DnTrans **rans); void dnodeCleanupTrans(DnTrans **trans); -void dnodeSendMsgToMnode(Dnode *dnode, SRpcMsg *rpcMsg); -void dnodeSendMsgToDnode(Dnode *dnode, SRpcEpSet *epSet, SRpcMsg *rpcMsg); -void dnodeSendMsgToDnodeRecv(Dnode *dnode, SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet); +void dnodeSendMsgToMnode(SRpcMsg *rpcMsg); +void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg); +void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet); #ifdef __cplusplus } diff --git a/source/server/dnode/src/dnodeCfg.c b/source/server/dnode/src/dnodeCfg.c index 2162e3f791..77329e2ad1 100644 --- a/source/server/dnode/src/dnodeCfg.c +++ b/source/server/dnode/src/dnodeCfg.c @@ -103,11 +103,10 @@ static int32_t dnodeWriteCfg(DnCfg *cfg) { return 0; } -int32_t dnodeInitCfg(Dnode *dnode, DnCfg **out) { +int32_t dnodeInitCfg(DnCfg **out) { DnCfg* cfg = calloc(1, sizeof(DnCfg)); if (cfg == NULL) return -1; - cfg->dnode = dnode; cfg->dnodeId = 0; cfg->dropped = 0; cfg->clusterId[0] = 0; diff --git a/source/server/dnode/src/dnodeCheck.c b/source/server/dnode/src/dnodeCheck.c index 7c3b539d70..9175b4e519 100644 --- a/source/server/dnode/src/dnodeCheck.c +++ b/source/server/dnode/src/dnodeCheck.c @@ -145,11 +145,10 @@ static int32_t dnodeCheckAccess() { return 0; } static int32_t dnodeCheckVersion() { return 0; } static int32_t dnodeCheckDatafile() { return 0; } -int32_t dnodeInitCheck(Dnode *dnode, DnCheck **out) { +int32_t dnodeInitCheck(DnCheck **out) { DnCheck *check = calloc(1, sizeof(DnCheck)); if (check == NULL) return -1; - check->dnode = dnode; *out = check; if (dnodeCheckNetwork() != 0) { diff --git a/source/server/dnode/src/dnodeEps.c b/source/server/dnode/src/dnodeEps.c index 8595b1b339..e633da9fc9 100644 --- a/source/server/dnode/src/dnodeEps.c +++ b/source/server/dnode/src/dnodeEps.c @@ -182,15 +182,14 @@ static int32_t dnodeWriteEps(DnEps *eps) { return 0; } -int32_t dnodeInitEps(Dnode *dnode, DnEps **out) { +int32_t dnodeInitEps(DnEps **out) { DnEps *eps = calloc(1, sizeof(DnEps)); if (eps == NULL) return -1; eps->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (eps->dnodeHash == NULL) return -1; - eps->dnode = dnode; - eps->dnodeId = dnode->cfg->dnodeId; + eps->dnodeId = dnodeInst()->cfg->dnodeId; eps->dnodeNum = 0; snprintf(eps->file, sizeof(eps->file), "%s/dnodeEps.json", tsDnodeDir); pthread_mutex_init(&eps->mutex, NULL); @@ -269,10 +268,8 @@ bool dnodeIsDnodeEpChanged(DnEps *eps, int32_t dnodeId, char *epstr) { return changed; } -void dnodeGetDnodeEp(Dnode *dnode, int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port) { - assert(dnode != NULL); - - DnEps *eps = dnode->eps; +void dnodeGetDnodeEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port) { + DnEps *eps = dnodeInst()->eps; pthread_mutex_lock(&eps->mutex); SDnodeEp *ep = taosHashGet(eps->dnodeHash, &dnodeId, sizeof(int32_t)); diff --git a/source/server/dnode/src/dnodeInt.c b/source/server/dnode/src/dnodeInt.c index f38d9665ac..8c795be9de 100644 --- a/source/server/dnode/src/dnodeInt.c +++ b/source/server/dnode/src/dnodeInt.c @@ -28,202 +28,75 @@ #include "dnodeMain.h" #include "dnodeMnodeEps.h" #include "dnodeStatus.h" -#include "dnodeTelemetry.h" +#include "dnodeTelem.h" #include "dnodeTrans.h" #include "mnode.h" #include "vnode.h" -static int32_t dnodeInitRpcEnv(Dnode *dnode, void **unUsed) { return rpcInit(); } -static void dnodeCleanupRpcEnv(void **unUsed) { rpcCleanup(); } -#if 0 -static int32_t dnodeInitTfsEnv(Dnode *dnode, void **unUsed) { return tfInit(); } -static void dnodeCleanupTfsEnv(void **unUsed) { tfCleanup(); } -static int32_t dnodeInitScriptEnv(Dnode *dnode, void **unUsed) { return scriptEnvPoolInit(); } -static void dnodeCleanupScriptEnv(void **unUsed) { scriptEnvPoolCleanup(); } -static int32_t dnodeInitWalEnv(Dnode *dnode, void **unUsed) { return walInit(); } -static void dnodeCleanupWalEnv(void **unUsed) { walCleanUp(); } -static int32_t dnodeInitSyncEnv(Dnode *dnode, void **unUsed) { return syncInit(); } -static void dnodeCleanupSyncEnv(void **unUsed) { syncCleanUp(); } -#endif +static Dnode tsDnode = {0}; -static int32_t dnodeInitVnodeModule(Dnode *dnode, struct Vnode** out) { +Dnode *dnodeInst() { return &tsDnode; } + +static int32_t dnodeInitVnodeModule(void **unused) { SVnodePara para; para.fp.GetDnodeEp = dnodeGetDnodeEp; para.fp.SendMsgToDnode = dnodeSendMsgToDnode; para.fp.SendMsgToMnode = dnodeSendMsgToMnode; - para.dnode = dnode; - struct Vnode *vnode = vnodeCreateInstance(para); - if (vnode == NULL) { - return -1; - } - - *out = vnode; - return 0; + return vnodeInit(para); } -static void dnodeCleanupVnodeModule(Dnode *dnode, struct Vnode **out) { - struct Vnode *vnode = *out; - *out = NULL; - vnodeDropInstance(vnode); -} +static int32_t dnodeInitMnodeModule(void **unused) { + Dnode *dnode = dnodeInst(); -static int32_t dnodeInitMnodeModule(Dnode *dnode, struct Mnode **out) { SMnodePara para; para.fp.GetDnodeEp = dnodeGetDnodeEp; para.fp.SendMsgToDnode = dnodeSendMsgToDnode; para.fp.SendMsgToMnode = dnodeSendMsgToMnode; para.fp.SendRedirectMsg = dnodeSendRedirectMsg; - para.dnode = dnode; para.dnodeId = dnode->cfg->dnodeId; strncpy(para.clusterId, dnode->cfg->clusterId, sizeof(para.clusterId)); - struct Mnode *mnode = mnodeCreateInstance(para); - if (mnode == NULL) { - return -1; - } - - *out = mnode; - return 0; + return mnodeInit(para); } -static void dnodeCleanupMnodeModule(Dnode *dnode, struct Mnode **out) { - struct Mnode *mnode = *out; - *out = NULL; - mnodeDropInstance(mnode); -} +int32_t dnodeInit() { + struct SSteps *steps = taosStepInit(24, dnodeReportStartup); + if (steps == NULL) return -1; -Dnode *dnodeCreateInstance() { - Dnode *dnode = calloc(1, sizeof(Dnode)); - if (dnode == NULL) { - return NULL; - } + Dnode *dnode = dnodeInst(); - SSteps *steps = taosStepInit(24); - if (steps == NULL) { - return NULL; - } - - SStepObj step = {0}; - step.parent = dnode; - - step.name = "dnode-main"; - step.self = (void **)&dnode->main; - step.initFp = (FnInitObj)dnodeInitMain; - step.cleanupFp = (FnCleanupObj)dnodeCleanupMain; - step.reportFp = NULL; - taosStepAdd(steps, &step); - - step.name = "dnode-storage"; - step.self = NULL; - step.initFp = (FnInitObj)dnodeInitStorage; - step.cleanupFp = (FnCleanupObj)dnodeCleanupStorage; - step.reportFp = (FnReportProgress)dnodeReportStartup; - taosStepAdd(steps, &step); - -#if 0 - step.name = "dnode-tfs-env"; - step.self = NULL; - step.initFp = (FnInitObj)dnodeInitTfsEnv; - step.cleanupFp = (FnCleanupObj)dnodeCleanupTfsEnv; - taosStepAdd(steps, &step); -#endif - - step.name = "dnode-rpc-env"; - step.self = NULL; - step.initFp = (FnInitObj)dnodeInitRpcEnv; - step.cleanupFp = (FnCleanupObj)dnodeCleanupRpcEnv; - taosStepAdd(steps, &step); - - step.name = "dnode-check"; - step.self = (void **)&dnode->check; - step.initFp = (FnInitObj)dnodeInitCheck; - step.cleanupFp = (FnCleanupObj)dnodeCleanupCheck; - taosStepAdd(steps, &step); - - step.name = "dnode-cfg"; - step.self = (void **)&dnode->cfg; - step.initFp = (FnInitObj)dnodeInitCfg; - step.cleanupFp = (FnCleanupObj)dnodeCleanupCfg; - taosStepAdd(steps, &step); - - step.name = "dnode-deps"; - step.self = (void **)&dnode->eps; - step.initFp = (FnInitObj)dnodeInitEps; - step.cleanupFp = (FnCleanupObj)dnodeCleanupEps; - taosStepAdd(steps, &step); - - step.name = "dnode-meps"; - step.self = (void **)&dnode->meps; - step.initFp = (FnInitObj)dnodeInitMnodeEps; - step.cleanupFp = (FnCleanupObj)dnodeCleanupMnodeEps; - taosStepAdd(steps, &step); - -#if 0 - step.name = "dnode-wal"; - step.self = NULL; - step.initFp = (FnInitObj)dnodeInitWalEnv; - step.cleanupFp = (FnCleanupObj)dnodeCleanupWalEnv; - taosStepAdd(steps, &step); - - step.name = "dnode-sync"; - step.self = NULL; - step.initFp = (FnInitObj)dnodeInitSyncEnv; - step.cleanupFp = (FnCleanupObj)dnodeCleanupSyncEnv; - taosStepAdd(steps, &step); -#endif - - step.name = "dnode-vnode"; - step.self = (void **)&dnode->vnode; - step.initFp = (FnInitObj)dnodeInitVnodeModule; - step.cleanupFp = (FnCleanupObj)dnodeCleanupVnodeModule; - taosStepAdd(steps, &step); - - step.name = "dnode-mnode"; - step.self = (void **)&dnode->mnode; - step.initFp = (FnInitObj)dnodeInitMnodeModule; - step.cleanupFp = (FnCleanupObj)dnodeCleanupMnodeModule; - taosStepAdd(steps, &step); - - step.name = "dnode-trans"; - step.self = (void **)&dnode->trans; - step.initFp = (FnInitObj)dnodeInitTrans; - step.cleanupFp = (FnCleanupObj)dnodeCleanupTrans; - taosStepAdd(steps, &step); - - step.name = "dnode-status"; - step.self = (void **)&dnode->status; - step.initFp = (FnInitObj)dnodeInitStatus; - step.cleanupFp = (FnCleanupObj)dnodeCleanupStatus; - taosStepAdd(steps, &step); - - step.name = "dnode-telem"; - step.self = (void **)&dnode->telem; - step.initFp = (FnInitObj)dnodeInitTelemetry; - step.cleanupFp = (FnCleanupObj)dnodeCleanupTelemetry; - taosStepAdd(steps, &step); - -#if 0 - step.name = "dnode-script"; - step.self = NULL; - step.initFp = (FnInitObj)dnodeInitScriptEnv; - step.cleanupFp = (FnCleanupObj)dnodeCleanupScriptEnv; - taosStepAdd(steps, &step); -#endif + taosStepAdd(steps, "dnode-main", (void **)&dnode->main, (InitFp)dnodeInitMain, (CleanupFp)dnodeCleanupMain); + taosStepAdd(steps, "dnode-storage", NULL, (InitFp)dnodeInitStorage, (CleanupFp)dnodeCleanupStorage); + //taosStepAdd(steps, "dnode-tfs-env", NULL, (InitFp)tfInit, (CleanupFp)tfCleanup); + taosStepAdd(steps, "dnode-rpc-env", NULL, (InitFp)rpcInit, (CleanupFp)rpcCleanup); + taosStepAdd(steps, "dnode-check", (void **)&dnode->check, (InitFp)dnodeInitCheck, (CleanupFp)dnodeCleanupCheck); + taosStepAdd(steps, "dnode-cfg", (void **)&dnode->cfg, (InitFp)dnodeInitCfg, (CleanupFp)dnodeCleanupCfg); + taosStepAdd(steps, "dnode-deps", (void **)&dnode->eps, (InitFp)dnodeInitEps, (CleanupFp)dnodeCleanupEps); + taosStepAdd(steps, "dnode-meps", (void **)&dnode->meps, (InitFp)dnodeInitMnodeEps, (CleanupFp)dnodeCleanupMnodeEps); + //taosStepAdd(steps, "dnode-wal", NULL, (InitFp)walInit, (CleanupFp)walCleanUp); + //taosStepAdd(steps, "dnode-sync", NULL, (InitFp)syncInit, (CleanupFp)syncCleanUp); + taosStepAdd(steps, "dnode-vnode", NULL, (InitFp)dnodeInitVnodeModule, (CleanupFp)vnodeCleanup); + taosStepAdd(steps, "dnode-mnode", NULL, (InitFp)dnodeInitMnodeModule, (CleanupFp)mnodeCleanup); + taosStepAdd(steps, "dnode-trans", (void **)&dnode->trans, (InitFp)dnodeInitTrans, (CleanupFp)dnodeCleanupTrans); + taosStepAdd(steps, "dnode-status", (void **)&dnode->status, (InitFp)dnodeInitStatus, (CleanupFp)dnodeCleanupStatus); + taosStepAdd(steps, "dnode-telem", (void **)&dnode->meps, (InitFp)dnodeInitTelem, (CleanupFp)dnodeCleanupTelem); + //taosStepAdd(steps, "dnode-script", NULL, (InitFp)scriptEnvPoolInit, (CleanupFp)scriptEnvPoolCleanup); dnode->steps = steps; taosStepExec(dnode->steps); if (dnode->main) { dnode->main->runStatus = TD_RUN_STAT_RUNNING; - dnodeReportStartupFinished(dnode, "TDengine", "initialized successfully"); + dnodeReportStartupFinished("TDengine", "initialized successfully"); dInfo("TDengine is initialized successfully"); } - return dnode; + return 0; } -void dnodeDropInstance(Dnode *dnode) { +void dnodeCleanup() { + Dnode *dnode = dnodeInst(); if (dnode->main->runStatus != TD_RUN_STAT_STOPPED) { dnode->main->runStatus = TD_RUN_STAT_STOPPED; taosStepCleanup(dnode->steps); diff --git a/source/server/dnode/src/dnodeMain.c b/source/server/dnode/src/dnodeMain.c index 7854fe2b4b..2845192def 100644 --- a/source/server/dnode/src/dnodeMain.c +++ b/source/server/dnode/src/dnodeMain.c @@ -55,11 +55,10 @@ void dnodePrintDiskInfo() { dInfo("=================================="); } -int32_t dnodeInitMain(Dnode *dnode, DnMain **out) { +int32_t dnodeInitMain(DnMain **out) { DnMain* main = calloc(1, sizeof(DnMain)); if (main == NULL) return -1; - main->dnode = dnode; main->runStatus = TD_RUN_STAT_STOPPED; main->dnodeTimer = taosTmrInit(100, 200, 60000, "DND-TMR"); if (main->dnodeTimer == NULL) { @@ -75,9 +74,8 @@ int32_t dnodeInitMain(Dnode *dnode, DnMain **out) { taosResolveCRC(); taosInitGlobalCfg(); taosReadGlobalLogCfg(); -#if 0 taosSetCoreDump(tsEnableCoreFile); -#endif + if (!taosMkDir(tsLogDir)) { printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno)); @@ -121,7 +119,7 @@ void dnodeCleanupMain(DnMain **out) { free(main); } -int32_t dnodeInitStorage(Dnode *dnode, void **m) { +int32_t dnodeInitStorage() { #ifdef TD_TSZ // compress module init tsCompressInit(); @@ -191,7 +189,7 @@ int32_t dnodeInitStorage(Dnode *dnode, void **m) { return 0; } -void dnodeCleanupStorage(void **m) { +void dnodeCleanupStorage() { #if 0 // storage destroy tfsDestroy(); @@ -203,23 +201,26 @@ void dnodeCleanupStorage(void **m) { #endif } -void dnodeReportStartup(Dnode *dnode, char *name, char *desc) { +void dnodeReportStartup(char *name, char *desc) { + Dnode *dnode = dnodeInst(); SStartupStep *startup = &dnode->main->startup; tstrncpy(startup->name, name, strlen(startup->name)); tstrncpy(startup->desc, desc, strlen(startup->desc)); startup->finished = 0; } -void dnodeReportStartupFinished(Dnode *dnode, char *name, char *desc) { +void dnodeReportStartupFinished(char *name, char *desc) { + Dnode *dnode = dnodeInst(); SStartupStep *startup = &dnode->main->startup; tstrncpy(startup->name, name, strlen(startup->name)); tstrncpy(startup->desc, desc, strlen(startup->desc)); startup->finished = 1; } -void dnodeProcessStartupReq(Dnode *dnode, SRpcMsg *pMsg) { +void dnodeProcessStartupReq(SRpcMsg *pMsg) { dInfo("startup msg is received, cont:%s", (char *)pMsg->pCont); + Dnode *dnode = dnodeInst(); SStartupStep *pStep = rpcMallocCont(sizeof(SStartupStep)); memcpy(pStep, &dnode->main->startup, sizeof(SStartupStep)); @@ -230,10 +231,11 @@ void dnodeProcessStartupReq(Dnode *dnode, SRpcMsg *pMsg) { rpcFreeCont(pMsg->pCont); } -static int32_t dnodeStartMnode(Dnode *dnode, SRpcMsg *pMsg) { +static int32_t dnodeStartMnode(SRpcMsg *pMsg) { + Dnode *dnode = dnodeInst(); SCreateMnodeMsg *pCfg = pMsg->pCont; pCfg->dnodeId = htonl(pCfg->dnodeId); - if (pCfg->dnodeId != dnodeGetDnodeId(dnode->cfg)) { + if (pCfg->dnodeId != dnode->cfg->dnodeId) { dDebug("dnode:%d, in create meps msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId(dnode->cfg)); return TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED; @@ -252,11 +254,11 @@ static int32_t dnodeStartMnode(Dnode *dnode, SRpcMsg *pMsg) { if (mnodeIsServing(dnode->mnode)) return 0; - return mnodeDeploy(dnode->mnode, &pCfg->mnodes); + return mnodeDeploy(&pCfg->mnodes); } -void dnodeProcessCreateMnodeReq(Dnode *dnode, SRpcMsg *pMsg) { - int32_t code = dnodeStartMnode(dnode, pMsg); +void dnodeProcessCreateMnodeReq(SRpcMsg *pMsg) { + int32_t code = dnodeStartMnode(pMsg); SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0, .code = code}; @@ -264,7 +266,7 @@ void dnodeProcessCreateMnodeReq(Dnode *dnode, SRpcMsg *pMsg) { rpcFreeCont(pMsg->pCont); } -void dnodeProcessConfigDnodeReq(Dnode *dnode, SRpcMsg *pMsg) { +void dnodeProcessConfigDnodeReq(SRpcMsg *pMsg) { SCfgDnodeMsg *pCfg = pMsg->pCont; int32_t code = taosCfgDynamicOptions(pCfg->config); diff --git a/source/server/dnode/src/dnodeMnodeEps.c b/source/server/dnode/src/dnodeMnodeEps.c index 9858f9d21f..e072bb5bfd 100644 --- a/source/server/dnode/src/dnodeMnodeEps.c +++ b/source/server/dnode/src/dnodeMnodeEps.c @@ -179,7 +179,7 @@ PARSE_MINFOS_OVER: for (int32_t i = 0; i < mInfos.mnodeNum; ++i) { SMInfo *mInfo = &mInfos.mnodeInfos[i]; - dnodeGetDnodeEp(meps->dnode, mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL); + dnodeGetDnodeEp(mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL); } dnodeResetMnodeEps(meps, &mInfos); @@ -191,8 +191,8 @@ PARSE_MINFOS_OVER: return 0; } -void dnodeSendRedirectMsg(struct Dnode *dnode, SRpcMsg *rpcMsg, bool forShell) { - DnMnEps *meps = dnode->meps; +void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) { + DnMnEps *meps = dnodeInst()->meps; SRpcConnInfo connInfo = {0}; rpcGetConnInfo(rpcMsg->handle, &connInfo); @@ -222,17 +222,16 @@ void dnodeSendRedirectMsg(struct Dnode *dnode, SRpcMsg *rpcMsg, bool forShell) { rpcSendRedirectRsp(rpcMsg->handle, &epSet); } -int32_t dnodeInitMnodeEps(Dnode *dnode, DnMnEps **out) { +int32_t dnodeInitMnodeEps(DnMnEps **out) { DnMnEps *meps = calloc(1, sizeof(DnMnEps)); if (meps == NULL) return -1; - meps->dnode = dnode; snprintf(meps->file, sizeof(meps->file), "%s/mnodeEpSet.json", tsDnodeDir); pthread_mutex_init(&meps->mutex, NULL); *out = meps; dnodeResetMnodeEps(meps, NULL); - int32_t ret = dnodeReadMnodeEps(meps, dnode->eps); + int32_t ret = dnodeReadMnodeEps(meps, dnodeInst()->eps); if (ret == 0) { dInfo("dnode mInfos is initialized"); } diff --git a/source/server/dnode/src/dnodeStatus.c b/source/server/dnode/src/dnodeStatus.c index 58effb5b44..308be7d1b6 100644 --- a/source/server/dnode/src/dnodeStatus.c +++ b/source/server/dnode/src/dnodeStatus.c @@ -46,7 +46,7 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { return; } - Dnode *dnode = status->dnode; + Dnode *dnode = dnodeInst(); dnodeGetCfg(dnode->cfg, &pStatus->dnodeId, pStatus->clusterId); pStatus->dnodeId = htonl(dnodeGetDnodeId(dnode->cfg)); pStatus->version = htonl(tsVersion); @@ -76,16 +76,17 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { pStatus->clusterCfg.slaveQuery = tsEnableSlaveQuery; pStatus->clusterCfg.adjustMaster = tsEnableAdjustMaster; - vnodeGetStatus(dnode->vnode, pStatus); + vnodeGetStatus(pStatus); contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad); pStatus->openVnodes = htons(pStatus->openVnodes); SRpcMsg rpcMsg = {.ahandle = status, .pCont = pStatus, .contLen = contLen, .msgType = TSDB_MSG_TYPE_DM_STATUS}; - dnodeSendMsgToMnode(status->dnode, &rpcMsg); + dnodeSendMsgToMnode(&rpcMsg); } -void dnodeProcessStatusRsp(Dnode *dnode, SRpcMsg *pMsg) { +void dnodeProcessStatusRsp(SRpcMsg *pMsg) { + Dnode *dnode = dnodeInst(); DnStatus *status = pMsg->ahandle; if (pMsg->code != TSDB_CODE_SUCCESS) { @@ -114,7 +115,7 @@ void dnodeProcessStatusRsp(Dnode *dnode, SRpcMsg *pMsg) { pCfg->dnodeId = htonl(pCfg->dnodeId); dnodeUpdateCfg(dnode->cfg, pCfg); - vnodeSetAccess(dnode->vnode, pStatusRsp->vgAccess, pCfg->numOfVnodes); + vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes); SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess)); dnodeUpdateEps(dnode->eps, pEps); @@ -122,17 +123,14 @@ void dnodeProcessStatusRsp(Dnode *dnode, SRpcMsg *pMsg) { taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, status, status->dnodeTimer, &status->statusTimer); } -int32_t dnodeInitStatus(Dnode *dnode, DnStatus **out) { +int32_t dnodeInitStatus(DnStatus **out) { DnStatus *status = calloc(1, sizeof(DnStatus)); if (status == NULL) return -1; - - status->dnode = dnode; status->statusTimer = NULL; - status->dnodeTimer = dnode->main->dnodeTimer; + status->dnodeTimer = dnodeInst()->main->dnodeTimer; status->rebootTime = taosGetTimestampSec(); taosTmrReset(dnodeSendStatusMsg, 500, status, status->dnodeTimer, &status->statusTimer); *out = status; - dInfo("dnode status timer is initialized"); return TSDB_CODE_SUCCESS; } diff --git a/source/server/dnode/src/dnodeTelemetry.c b/source/server/dnode/src/dnodeTelem.c similarity index 96% rename from source/server/dnode/src/dnodeTelemetry.c rename to source/server/dnode/src/dnodeTelem.c index 5d1d9b4aa4..6b0bbfa6a0 100644 --- a/source/server/dnode/src/dnodeTelemetry.c +++ b/source/server/dnode/src/dnodeTelem.c @@ -18,7 +18,7 @@ #include "tbuffer.h" #include "tglobal.h" #include "dnodeCfg.h" -#include "dnodeTelemetry.h" +#include "dnodeTelem.h" #include "mnode.h" #define TELEMETRY_SERVER "telemetry.taosdata.com" @@ -163,7 +163,7 @@ static void dnodeAddVersionInfo(DnTelem* telem, SBufferWriter* bw) { static void dnodeAddRuntimeInfo(DnTelem* telem, SBufferWriter* bw) { SMnodeStat stat = {0}; - if (mnodeGetStatistics(telem->dnode->mnode, &stat) != 0) { + if (mnodeGetStatistics(&stat) != 0) { return; } @@ -192,9 +192,10 @@ static void dnodeSendTelemetryReport(DnTelem* telem) { return; } + Dnode *dnode = dnodeInst(); SBufferWriter bw = tbufInitWriter(NULL, false); dnodeBeginObject(&bw); - dnodeAddStringField(&bw, "instanceId", telem->dnode->cfg->clusterId); + dnodeAddStringField(&bw, "instanceId", dnode->cfg->clusterId); dnodeAddIntField(&bw, "reportVersion", 1); dnodeAddOsInfo(&bw); dnodeAddCpuInfo(&bw); @@ -243,7 +244,7 @@ static void* dnodeTelemThreadFp(void* param) { if (r == 0) break; if (r != ETIMEDOUT) continue; - if (mnodeIsServing(telem->dnode->mnode)) { + if (mnodeIsServing()) { dnodeSendTelemetryReport(telem); } end.tv_sec += REPORT_INTERVAL; @@ -265,11 +266,10 @@ static void dnodeGetEmail(DnTelem* telem, char* filepath) { taosCloseFile(fd); } -int32_t dnodeInitTelemetry(Dnode* dnode, DnTelem** out) { +int32_t dnodeInitTelem(DnTelem** out) { DnTelem* telem = calloc(1, sizeof(DnTelem)); if (telem == NULL) return -1; - telem->dnode = dnode; telem->enable = tsEnableTelemetryReporting; *out = telem; @@ -296,7 +296,7 @@ int32_t dnodeInitTelemetry(Dnode* dnode, DnTelem** out) { return 0; } -void dnodeCleanupTelemetry(DnTelem** out) { +void dnodeCleanupTelem(DnTelem** out) { DnTelem* telem = *out; *out = NULL; diff --git a/source/server/dnode/src/dnodeTrans.c b/source/server/dnode/src/dnodeTrans.c index 4d03338835..c6139101bb 100644 --- a/source/server/dnode/src/dnodeTrans.c +++ b/source/server/dnode/src/dnodeTrans.c @@ -29,13 +29,13 @@ #include "vnode.h" #include "mnode.h" -static void dnodeProcessPeerReq(DnTrans *trans, SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - Dnode * dnode = trans->dnode; +static void dnodeProcessPeerReq(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { + Dnode * dnode = dnodeInst(); SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0}; if (pMsg->pCont == NULL) return; if (pMsg->msgType == TSDB_MSG_TYPE_NETWORK_TEST) { - dnodeProcessStartupReq(dnode, pMsg); + dnodeProcessStartupReq(pMsg); return; } @@ -53,9 +53,9 @@ static void dnodeProcessPeerReq(DnTrans *trans, SRpcMsg *pMsg, SRpcEpSet *pEpSet return; } - DnMsgFp fp = trans->fpPeerMsg[pMsg->msgType]; - if (fp.fp != NULL) { - (*fp.fp)(fp.module, pMsg); + RpcMsgFp fp = dnode->trans->peerMsgFp[pMsg->msgType]; + if (fp != NULL) { + (*fp)(pMsg); } else { dDebug("RPC %p, peer req:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]); rspMsg.code = TSDB_CODE_DND_MSG_NOT_PROCESSED; @@ -65,41 +65,36 @@ static void dnodeProcessPeerReq(DnTrans *trans, SRpcMsg *pMsg, SRpcEpSet *pEpSet } int32_t dnodeInitServer(DnTrans *trans) { - struct Dnode *dnode = trans->dnode; - struct Vnode *vnode = dnode->vnode; - struct Mnode *mnode = dnode->mnode; - - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_CREATE_TABLE] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_DROP_TABLE] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_ALTER_TABLE] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_DROP_STABLE] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = vnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE] = vnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE] = vnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = vnodeProcessMsg; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_CREATE_VNODE] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_ALTER_VNODE] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_SYNC_VNODE] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_COMPACT_VNODE] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_DROP_VNODE] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_ALTER_STREAM] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = vnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = vnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE] = vnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE] = vnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = vnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = vnodeProcessMsg; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = (DnMsgFp){.module = dnode, .fp = (RpcMsgFp)dnodeProcessConfigDnodeReq}; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_CREATE_MNODE] = (DnMsgFp){.module = dnode, .fp = (RpcMsgFp)dnodeProcessCreateMnodeReq}; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeReq; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeProcessCreateMnodeReq; - trans->fpPeerMsg[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_DM_AUTH] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_DM_GRANT] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_DM_STATUS] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = mnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = mnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_DM_AUTH] = mnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_DM_GRANT] = mnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_DM_STATUS] = mnodeProcessMsg; SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = tsDnodeDnodePort; rpcInit.label = "DND-S"; rpcInit.numOfThreads = 1; - rpcInit.cfp = (RpcMsgCfp)dnodeProcessPeerReq; + rpcInit.cfp = dnodeProcessPeerReq; rpcInit.sessions = TSDB_MAX_VNODES << 4; rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.owner = trans; trans->serverRpc = rpcOpen(&rpcInit); if (trans->serverRpc == NULL) { @@ -119,8 +114,8 @@ void dnodeCleanupServer(DnTrans *trans) { } } -static void dnodeProcessRspFromPeer(DnTrans *trans, SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - Dnode *dnode = trans->dnode; +static void dnodeProcessRspFromPeer(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { + Dnode *dnode = dnodeInst(); if (dnode->main->runStatus == TD_RUN_STAT_STOPPED) { if (pMsg == NULL || pMsg->pCont == NULL) return; dTrace("msg:%p is ignored since dnode is stopping", pMsg); @@ -132,9 +127,9 @@ static void dnodeProcessRspFromPeer(DnTrans *trans, SRpcMsg *pMsg, SRpcEpSet *pE dnodeUpdateMnodeFromPeer(dnode->meps, pEpSet); } - DnMsgFp fp = trans->fpPeerMsg[pMsg->msgType]; - if (fp.fp != NULL) { - (*fp.fp)(fp.module, pMsg); + RpcMsgFp fp = dnode->trans->peerMsgFp[pMsg->msgType]; + if (fp != NULL) { + (*fp)(pMsg); } else { dDebug("RPC %p, peer rsp:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]); SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0}; @@ -147,43 +142,39 @@ static void dnodeProcessRspFromPeer(DnTrans *trans, SRpcMsg *pMsg, SRpcEpSet *pE } int32_t dnodeInitClient(DnTrans *trans) { - struct Dnode *dnode = trans->dnode; - struct Mnode *mnode = dnode->mnode; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP] = mnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE_RSP] = mnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP] = mnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE_RSP] = mnodeProcessMsg; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_DROP_TABLE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_DROP_STABLE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP] = mnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP] = mnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE_RSP] = mnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE_RSP] = mnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE_RSP] = mnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM_RSP] = mnodeProcessMsg; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_SYNC_VNODE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_COMPACT_VNODE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_DROP_VNODE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_ALTER_STREAM_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP] = mnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE_RSP] = mnodeProcessMsg; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_MD_CREATE_MNODE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - - trans->fpPeerMsg[TSDB_MSG_TYPE_DM_CONFIG_TABLE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_DM_CONFIG_VNODE_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_DM_AUTH_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_DM_GRANT_RSP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpPeerMsg[TSDB_MSG_TYPE_DM_STATUS_RSP] = (DnMsgFp){.module = dnode, .fp = (RpcMsgFp)dnodeProcessStatusRsp}; + trans->peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE_RSP] = mnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE_RSP] = mnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_DM_AUTH_RSP] = mnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_DM_GRANT_RSP] = mnodeProcessMsg; + trans->peerMsgFp[TSDB_MSG_TYPE_DM_STATUS_RSP] = dnodeProcessStatusRsp; char secret[TSDB_KEY_LEN] = "secret"; SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.label = "DND-C"; rpcInit.numOfThreads = 1; - rpcInit.cfp = (RpcMsgCfp)dnodeProcessRspFromPeer; + rpcInit.cfp = dnodeProcessRspFromPeer; rpcInit.sessions = TSDB_MAX_VNODES << 4; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.user = "t"; rpcInit.ckey = "key"; rpcInit.secret = secret; - rpcInit.owner = trans; trans->clientRpc = rpcOpen(&rpcInit); if (trans->clientRpc == NULL) { @@ -203,8 +194,8 @@ void dnodeCleanupClient(DnTrans *trans) { } } -static void dnodeProcessMsgFromShell(DnTrans *trans, SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - Dnode * dnode = trans->dnode; +static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { + Dnode * dnode = dnodeInst(); SRpcMsg rpcMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0}; if (pMsg->pCont == NULL) return; @@ -222,15 +213,16 @@ static void dnodeProcessMsgFromShell(DnTrans *trans, SRpcMsg *pMsg, SRpcEpSet *p return; } + DnTrans *trans = dnode->trans; if (pMsg->msgType == TSDB_MSG_TYPE_QUERY) { atomic_fetch_add_32(&trans->queryReqNum, 1); } else if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) { atomic_fetch_add_32(&trans->submitReqNum, 1); } else {} - DnMsgFp fp = trans->fpShellMsg[pMsg->msgType]; - if (fp.fp != NULL) { - (*fp.fp)(fp.module, pMsg); + RpcMsgFp fp = trans->shellMsgFp[pMsg->msgType]; + if (fp != NULL) { + (*fp)(pMsg); } else { dError("RPC %p, shell req:%s is not processed", pMsg->handle, taosMsg[pMsg->msgType]); rpcMsg.code = TSDB_CODE_DND_MSG_NOT_PROCESSED; @@ -254,32 +246,34 @@ static int32_t dnodeAuthNetTest(char *user, char *spi, char *encrypt, char *secr return -1; } -void dnodeSendMsgToDnode(Dnode *dnode, SRpcEpSet *epSet, SRpcMsg *rpcMsg) { +void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) { + Dnode *dnode = dnodeInst(); rpcSendRequest(dnode->trans->clientRpc, epSet, rpcMsg, NULL); } -void dnodeSendMsgToMnode(Dnode *dnode, SRpcMsg *rpcMsg) { +void dnodeSendMsgToMnode(SRpcMsg *rpcMsg) { + Dnode * dnode = dnodeInst(); SRpcEpSet epSet = {0}; dnodeGetEpSetForPeer(dnode->meps, &epSet); - dnodeSendMsgToDnode(dnode, &epSet, rpcMsg); + dnodeSendMsgToDnode(&epSet, rpcMsg); } -void dnodeSendMsgToMnodeRecv(Dnode *dnode, SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) { +void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) { + Dnode * dnode = dnodeInst(); SRpcEpSet epSet = {0}; dnodeGetEpSetForPeer(dnode->meps, &epSet); rpcSendRecv(dnode->trans->clientRpc, &epSet, rpcMsg, rpcRsp); } -void dnodeSendMsgToDnodeRecv(Dnode *dnode, SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet) { +void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet) { + Dnode *dnode = dnodeInst(); rpcSendRecv(dnode->trans->clientRpc, epSet, rpcMsg, rpcRsp); } -static int32_t dnodeRetrieveUserAuthInfo(void *owner, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - DnTrans *trans = owner; - +static int32_t dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) { if (dnodeAuthNetTest(user, spi, encrypt, secret, ckey) == 0) return 0; - int32_t code = mnodeRetriveAuth(trans->dnode->mnode, user, spi, encrypt, secret, ckey); + int32_t code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey); if (code != TSDB_CODE_APP_NOT_READY) return code; SAuthMsg *pMsg = rpcMallocCont(sizeof(SAuthMsg)); @@ -292,7 +286,7 @@ static int32_t dnodeRetrieveUserAuthInfo(void *owner, char *user, char *spi, cha dDebug("user:%s, send auth msg to mnodes", user); SRpcMsg rpcRsp = {0}; - dnodeSendMsgToMnodeRecv(trans->dnode, &rpcMsg, &rpcRsp); + dnodeSendMsgToMnodeRecv(&rpcMsg, &rpcRsp); if (rpcRsp.code != 0) { dError("user:%s, auth msg received from mnodes, error:%s", user, tstrerror(rpcRsp.code)); @@ -310,55 +304,51 @@ static int32_t dnodeRetrieveUserAuthInfo(void *owner, char *user, char *spi, cha } int32_t dnodeInitShell(DnTrans *trans) { - struct Dnode *dnode = trans->dnode; - struct Vnode *vnode = dnode->vnode; - struct Mnode *mnode = dnode->mnode; - - trans->fpShellMsg[TSDB_MSG_TYPE_SUBMIT] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_QUERY] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_FETCH] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = (DnMsgFp){.module = vnode, .fp = (RpcMsgFp)vnodeProcessMsg}; + trans->shellMsgFp[TSDB_MSG_TYPE_SUBMIT] = vnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_QUERY] = vnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_FETCH] = vnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = vnodeProcessMsg; // the following message shall be treated as mnode write - trans->fpShellMsg[TSDB_MSG_TYPE_CM_CREATE_ACCT] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_ALTER_ACCT] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_DROP_ACCT] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_CREATE_USER] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_ALTER_USER] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_DROP_USER] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_CREATE_DNODE] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_DROP_DNODE] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_CREATE_DB] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_CREATE_TP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_CREATE_FUNCTION] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_DROP_DB] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_SYNC_DB] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_DROP_TP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_DROP_FUNCTION] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_ALTER_DB] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_ALTER_TP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_CREATE_TABLE] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_DROP_TABLE] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_ALTER_TABLE] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_ALTER_STREAM] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_KILL_QUERY] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_KILL_STREAM] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_KILL_CONN] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_CONFIG_DNODE] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_COMPACT_VNODE] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TP] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_FUNCTION] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_SYNC_DB] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_TP] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_FUNCTION] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TP] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_COMPACT_VNODE] = mnodeProcessMsg; // the following message shall be treated as mnode query - trans->fpShellMsg[TSDB_MSG_TYPE_CM_HEARTBEAT] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_CONNECT] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_USE_DB] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_TABLE_META] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_STABLE_VGROUP] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_TABLES_META] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_SHOW] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_RETRIEVE] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; - trans->fpShellMsg[TSDB_MSG_TYPE_CM_RETRIEVE_FUNC] = (DnMsgFp){.module = mnode, .fp = (RpcMsgFp)mnodeProcessMsg}; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_CONNECT] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_USE_DB] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = mnodeProcessMsg; + trans->shellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE_FUNC] = mnodeProcessMsg; - trans->fpShellMsg[TSDB_MSG_TYPE_NETWORK_TEST] = (DnMsgFp){.module = dnode, .fp = (RpcMsgFp)dnodeProcessStartupReq}; + trans->shellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dnodeProcessStartupReq; int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); if (numOfThreads < 1) { @@ -370,12 +360,11 @@ int32_t dnodeInitShell(DnTrans *trans) { rpcInit.localPort = tsDnodeShellPort; rpcInit.label = "SHELL"; rpcInit.numOfThreads = numOfThreads; - rpcInit.cfp = (RpcMsgCfp)dnodeProcessMsgFromShell; + rpcInit.cfp = dnodeProcessMsgFromShell; rpcInit.sessions = tsMaxShellConns; rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.afp = dnodeRetrieveUserAuthInfo; - rpcInit.owner = trans; trans->shellRpc = rpcOpen(&rpcInit); if (trans->shellRpc == NULL) { @@ -394,11 +383,10 @@ void dnodeCleanupShell(DnTrans *trans) { } } -int32_t dnodeInitTrans(Dnode *dnode, DnTrans **out) { - DnTrans* trans = calloc(1, sizeof(DnTrans)); +int32_t dnodeInitTrans(DnTrans **out) { + DnTrans *trans = calloc(1, sizeof(DnTrans)); if (trans == NULL) return -1; - trans->dnode = dnode; *out = trans; if (dnodeInitClient(trans) != 0) { diff --git a/source/server/mnode/src/mnodeMain.c b/source/server/mnode/src/mnodeMain.c index 3747534e36..8e5b4692e3 100644 --- a/source/server/mnode/src/mnodeMain.c +++ b/source/server/mnode/src/mnodeMain.c @@ -15,22 +15,18 @@ #include "mnodeInt.h" -struct Mnode *mnodeCreateInstance(SMnodePara para) { - return NULL; -} +int32_t mnodeInit(SMnodePara para) { return 0; } -void mnodeDropInstance(struct Mnode *vnode) {} +void mnodeCleanup() {} -int32_t mnodeDeploy(struct Mnode *mnode, struct SMInfos *minfos) { return 0; } +int32_t mnodeDeploy(struct SMInfos *minfos) { return 0; } -void mnodeUnDeploy(struct Mnode *mnode) {} +void mnodeUnDeploy() {} -bool mnodeIsServing(struct Mnode *mnode) { return false; } +bool mnodeIsServing() { return false; } -int32_t mnodeGetStatistics(struct Mnode *mnode, SMnodeStat *stat) { return 0; } +int32_t mnodeGetStatistics(SMnodeStat *stat) { return 0; } -int32_t mnodeRetriveAuth(struct Mnode *mnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - return 0; -} +int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey) { return 0; } -void mnodeProcessMsg(struct Mnode *mnode, SRpcMsg *rpcMsg) {} +void mnodeProcessMsg(SRpcMsg *rpcMsg) {} diff --git a/source/server/server.c b/source/server/server.c index 0e50b1f97f..67e8e7bc58 100644 --- a/source/server/server.c +++ b/source/server/server.c @@ -29,8 +29,8 @@ static void setSignalHandler() { int main(int argc, char const *argv[]) { setSignalHandler(); - struct Dnode *dnode = dnodeCreateInstance(); - if (dnode == NULL) { + int32_t code = dnodeInit(); + if (code != 0) { uInfo("Failed to start TDengine, please check the log at:%s", tsLogDir); exit(EXIT_FAILURE); } @@ -42,7 +42,7 @@ int main(int argc, char const *argv[]) { } uInfo("TDengine is shut down!"); - dnodeDropInstance(dnode); + dnodeCleanup(); return 0; } diff --git a/source/server/vnode/src/vnodeMain.c b/source/server/vnode/src/vnodeMain.c index 0f26041787..ac14d2a756 100644 --- a/source/server/vnode/src/vnodeMain.c +++ b/source/server/vnode/src/vnodeMain.c @@ -15,16 +15,14 @@ #include "vnodeInt.h" -struct Vnode *vnodeCreateInstance(SVnodePara para) { - return NULL; -} +int32_t vnodeInit(SVnodePara para) { return 0; } -void vnodeDropInstance(struct Vnode *vnode) {} +void vnodeCleanup() {} -int32_t vnodeGetStatistics(struct Vnode *vnode, SVnodeStat *stat) { return 0; } +int32_t vnodeGetStatistics(SVnodeStat *stat) { return 0; } -void vnodeGetStatus(struct Vnode *vnode, struct SStatusMsg *status) {} +void vnodeGetStatus(struct SStatusMsg *status) {} -void vnodeSetAccess(struct Vnode *vnode, struct SVgroupAccess *access, int32_t numOfVnodes) {} +void vnodeSetAccess(struct SVgroupAccess *access, int32_t numOfVnodes) {} -void vnodeProcessMsg(struct Vnode *vnode, SRpcMsg *msg) {} +void vnodeProcessMsg(SRpcMsg *msg) {} diff --git a/source/util/src/tstep.c b/source/util/src/tstep.c index e307977e70..b04135194a 100644 --- a/source/util/src/tstep.c +++ b/source/util/src/tstep.c @@ -18,32 +18,47 @@ #include "ulog.h" #include "tstep.h" -SSteps *taosStepInit(int32_t maxsize) { +typedef struct SStepObj { + char * name; + void ** self; + InitFp initFp; + CleanupFp cleanupFp; +} SStep; + +typedef struct SSteps { + int32_t cursize; + int32_t maxsize; + SStep * steps; + ReportFp reportFp; +} SSteps; + +SSteps *taosStepInit(int32_t maxsize, ReportFp fp) { SSteps *steps = calloc(1, sizeof(SSteps)); if (steps == NULL) return NULL; steps->maxsize = maxsize; steps->cursize = 0; - steps->steps = calloc(maxsize, sizeof(SStepObj)); + steps->steps = calloc(maxsize, sizeof(SStep)); + steps->reportFp = fp; return steps; } -int32_t taosStepAdd(SSteps *steps, SStepObj *step) { - if (steps == NULL) return - 1; - +int32_t taosStepAdd(struct SSteps *steps, char *name, void **obj, InitFp initFp, CleanupFp cleanupFp) { + if (steps == NULL) return -1; if (steps->cursize >= steps->maxsize) { uError("failed to add step since up to the maxsize"); return -1; } - steps->steps[steps->cursize++] = *step; + SStep step = {.name = name, .self = obj, .initFp = initFp, .cleanupFp = cleanupFp}; + steps->steps[steps->cursize++] = step; return 0; } static void taosStepCleanupImp(SSteps *steps, int32_t pos) { for (int32_t s = pos; s >= 0; s--) { - SStepObj *step = steps->steps + s; + SStep *step = steps->steps + s; uDebug("step:%s will cleanup", step->name); if (step->cleanupFp != NULL) { (*step->cleanupFp)(step->self); @@ -55,14 +70,14 @@ int32_t taosStepExec(SSteps *steps) { if (steps == NULL) return -1; for (int32_t s = 0; s < steps->cursize; s++) { - SStepObj *step = steps->steps + s; + SStep *step = steps->steps + s; if (step->initFp == NULL) continue; - if (step->reportFp != NULL) { - (*step->reportFp)(step->parent, step->name, "start initialize"); + if (steps->reportFp != NULL) { + (*steps->reportFp)(step->name, "start initialize"); } - int32_t code = (*step->initFp)(step->parent, step->self); + int32_t code = (*step->initFp)(step->self); if (code != 0) { uDebug("step:%s will cleanup", step->name); taosStepCleanupImp(steps, s); @@ -71,8 +86,8 @@ int32_t taosStepExec(SSteps *steps) { uInfo("step:%s is initialized", step->name); - if (step->reportFp != NULL) { - (*step->reportFp)(step->parent, step->name, "initialize completed"); + if (steps->reportFp != NULL) { + (*steps->reportFp)(step->name, "initialize completed"); } } From 8d0870e35a17204228be7dd29c9010551559274c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 6 Oct 2021 10:40:26 +0800 Subject: [PATCH 41/44] minor changes --- source/server/dnode/src/dnodeInt.c | 6 +++--- source/server/dnode/src/dnodeMain.c | 10 ++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/source/server/dnode/src/dnodeInt.c b/source/server/dnode/src/dnodeInt.c index 8c795be9de..0da2834337 100644 --- a/source/server/dnode/src/dnodeInt.c +++ b/source/server/dnode/src/dnodeInt.c @@ -68,8 +68,8 @@ int32_t dnodeInit() { taosStepAdd(steps, "dnode-main", (void **)&dnode->main, (InitFp)dnodeInitMain, (CleanupFp)dnodeCleanupMain); taosStepAdd(steps, "dnode-storage", NULL, (InitFp)dnodeInitStorage, (CleanupFp)dnodeCleanupStorage); - //taosStepAdd(steps, "dnode-tfs-env", NULL, (InitFp)tfInit, (CleanupFp)tfCleanup); - taosStepAdd(steps, "dnode-rpc-env", NULL, (InitFp)rpcInit, (CleanupFp)rpcCleanup); + //taosStepAdd(steps, "dnode-tfs", NULL, (InitFp)tfInit, (CleanupFp)tfCleanup); + taosStepAdd(steps, "dnode-rpc", NULL, (InitFp)rpcInit, (CleanupFp)rpcCleanup); taosStepAdd(steps, "dnode-check", (void **)&dnode->check, (InitFp)dnodeInitCheck, (CleanupFp)dnodeCleanupCheck); taosStepAdd(steps, "dnode-cfg", (void **)&dnode->cfg, (InitFp)dnodeInitCfg, (CleanupFp)dnodeCleanupCfg); taosStepAdd(steps, "dnode-deps", (void **)&dnode->eps, (InitFp)dnodeInitEps, (CleanupFp)dnodeCleanupEps); @@ -80,7 +80,7 @@ int32_t dnodeInit() { taosStepAdd(steps, "dnode-mnode", NULL, (InitFp)dnodeInitMnodeModule, (CleanupFp)mnodeCleanup); taosStepAdd(steps, "dnode-trans", (void **)&dnode->trans, (InitFp)dnodeInitTrans, (CleanupFp)dnodeCleanupTrans); taosStepAdd(steps, "dnode-status", (void **)&dnode->status, (InitFp)dnodeInitStatus, (CleanupFp)dnodeCleanupStatus); - taosStepAdd(steps, "dnode-telem", (void **)&dnode->meps, (InitFp)dnodeInitTelem, (CleanupFp)dnodeCleanupTelem); + taosStepAdd(steps, "dnode-telem", (void **)&dnode->telem, (InitFp)dnodeInitTelem, (CleanupFp)dnodeCleanupTelem); //taosStepAdd(steps, "dnode-script", NULL, (InitFp)scriptEnvPoolInit, (CleanupFp)scriptEnvPoolCleanup); dnode->steps = steps; diff --git a/source/server/dnode/src/dnodeMain.c b/source/server/dnode/src/dnodeMain.c index 2845192def..73179d48b7 100644 --- a/source/server/dnode/src/dnodeMain.c +++ b/source/server/dnode/src/dnodeMain.c @@ -203,10 +203,12 @@ void dnodeCleanupStorage() { void dnodeReportStartup(char *name, char *desc) { Dnode *dnode = dnodeInst(); - SStartupStep *startup = &dnode->main->startup; - tstrncpy(startup->name, name, strlen(startup->name)); - tstrncpy(startup->desc, desc, strlen(startup->desc)); - startup->finished = 0; + if (dnode->main != NULL) { + SStartupStep *startup = &dnode->main->startup; + tstrncpy(startup->name, name, strlen(startup->name)); + tstrncpy(startup->desc, desc, strlen(startup->desc)); + startup->finished = 0; + } } void dnodeReportStartupFinished(char *name, char *desc) { From 951bcc7c94bdbb0cfb83fcf3b2042ec2d860bc3f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 6 Oct 2021 11:23:30 +0800 Subject: [PATCH 42/44] fix compile error --- source/server/vnode/src/{vnodeMain.c => vnodeInt.c} | 0 source/server/vnode/tq/src/tq.c | 7 ++++++- source/server/vnode/tsdb/CMakeLists.txt | 4 ++-- source/server/vnode/tsdb/src/tsdb.c | 7 ++++++- 4 files changed, 14 insertions(+), 4 deletions(-) rename source/server/vnode/src/{vnodeMain.c => vnodeInt.c} (100%) diff --git a/source/server/vnode/src/vnodeMain.c b/source/server/vnode/src/vnodeInt.c similarity index 100% rename from source/server/vnode/src/vnodeMain.c rename to source/server/vnode/src/vnodeInt.c diff --git a/source/server/vnode/tq/src/tq.c b/source/server/vnode/tq/src/tq.c index 6dea4a4e57..a539182115 100644 --- a/source/server/vnode/tq/src/tq.c +++ b/source/server/vnode/tq/src/tq.c @@ -11,4 +11,9 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "tq.h" + +int tqPushMsg(void * p) {return 0;} +int tqCommit(STQ *pTQ) {return 0;} diff --git a/source/server/vnode/tsdb/CMakeLists.txt b/source/server/vnode/tsdb/CMakeLists.txt index 4a1e154cd5..426fa2a317 100644 --- a/source/server/vnode/tsdb/CMakeLists.txt +++ b/source/server/vnode/tsdb/CMakeLists.txt @@ -5,8 +5,8 @@ target_include_directories( PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode/tsdb" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_include_directories( +target_link_libraries( tsdb - PRIVATE os + PUBLIC os PRIVATE common ) \ No newline at end of file diff --git a/source/server/vnode/tsdb/src/tsdb.c b/source/server/vnode/tsdb/src/tsdb.c index 6dea4a4e57..b79d6a850e 100644 --- a/source/server/vnode/tsdb/src/tsdb.c +++ b/source/server/vnode/tsdb/src/tsdb.c @@ -11,4 +11,9 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "tsdb.h" + +int tsdbInsert(STsdb *pTsdb, SSubmitReq *pMsg) { return 0; } +int tsdbCommit(STsdb *pTsdb) { return 0; } \ No newline at end of file From 4c7220e0db68d87e50a81c5074a4b9eebf73cbcf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 6 Oct 2021 18:24:48 +0800 Subject: [PATCH 43/44] rename some variables --- include/server/mnode/mnode.h | 2 +- source/server/dnode/inc/dnodeCfg.h | 18 +++++----- source/server/dnode/inc/dnodeCheck.h | 8 ++--- source/server/dnode/inc/dnodeEps.h | 12 +++---- source/server/dnode/inc/dnodeInt.h | 44 +++++++++++-------------- source/server/dnode/inc/dnodeMain.h | 8 ++--- source/server/dnode/inc/dnodeMnodeEps.h | 16 ++++----- source/server/dnode/inc/dnodeStatus.h | 8 ++--- source/server/dnode/inc/dnodeTelem.h | 8 ++--- source/server/dnode/inc/dnodeTrans.h | 9 +++-- source/server/dnode/src/dnodeCfg.c | 22 ++++++------- source/server/dnode/src/dnodeCheck.c | 8 ++--- source/server/dnode/src/dnodeEps.c | 22 ++++++------- source/server/dnode/src/dnodeInt.c | 13 ++++---- source/server/dnode/src/dnodeMain.c | 18 +++++----- source/server/dnode/src/dnodeMnodeEps.c | 26 +++++++-------- source/server/dnode/src/dnodeStatus.c | 16 ++++----- source/server/dnode/src/dnodeTelem.c | 20 +++++------ source/server/dnode/src/dnodeTrans.c | 38 ++++++++++----------- 19 files changed, 156 insertions(+), 160 deletions(-) diff --git a/include/server/mnode/mnode.h b/include/server/mnode/mnode.h index e78994fc9c..7ea50a92ce 100644 --- a/include/server/mnode/mnode.h +++ b/include/server/mnode/mnode.h @@ -118,7 +118,7 @@ typedef struct { int32_t mnodeGetStatistics(SMnodeStat *stat); /** - * Get the statistical information of Mnode. + * Get the auth information. * * @param user, username. * @param spi, security parameter index. diff --git a/source/server/dnode/inc/dnodeCfg.h b/source/server/dnode/inc/dnodeCfg.h index 7a0d830a4b..1565e7649a 100644 --- a/source/server/dnode/inc/dnodeCfg.h +++ b/source/server/dnode/inc/dnodeCfg.h @@ -21,21 +21,21 @@ extern "C" { #endif #include "dnodeInt.h" -typedef struct DnCfg { +typedef struct SDnCfg { int32_t dnodeId; int32_t dropped; char clusterId[TSDB_CLUSTER_ID_LEN]; char file[PATH_MAX + 20]; pthread_mutex_t mutex; -} DnCfg; +} SDnCfg; -int32_t dnodeInitCfg(DnCfg **cfg); -void dnodeCleanupCfg(DnCfg **cfg); -void dnodeUpdateCfg(DnCfg *cfg, SDnodeCfg *data); -int32_t dnodeGetDnodeId(DnCfg *cfg); -void dnodeGetClusterId(DnCfg *cfg, char *clusterId); -void dnodeGetCfg(DnCfg *cfg, int32_t *dnodeId, char *clusterId); -void dnodeSetDropped(DnCfg *cfg); +int32_t dnodeInitCfg(SDnCfg **cfg); +void dnodeCleanupCfg(SDnCfg **cfg); +void dnodeUpdateCfg(SDnCfg *cfg, SDnodeCfg *data); +int32_t dnodeGetDnodeId(SDnCfg *cfg); +void dnodeGetClusterId(SDnCfg *cfg, char *clusterId); +void dnodeGetCfg(SDnCfg *cfg, int32_t *dnodeId, char *clusterId); +void dnodeSetDropped(SDnCfg *cfg); #ifdef __cplusplus } diff --git a/source/server/dnode/inc/dnodeCheck.h b/source/server/dnode/inc/dnodeCheck.h index db275259f5..29172ba4df 100644 --- a/source/server/dnode/inc/dnodeCheck.h +++ b/source/server/dnode/inc/dnodeCheck.h @@ -21,11 +21,11 @@ extern "C" { #endif #include "dnodeInt.h" -typedef struct DnCheck { -} DnCheck; +typedef struct SDnCheck { +} SDnCheck; -int32_t dnodeInitCheck(DnCheck **check); -void dnodeCleanupCheck(DnCheck **check); +int32_t dnodeInitCheck(SDnCheck **check); +void dnodeCleanupCheck(SDnCheck **check); #ifdef __cplusplus } diff --git a/source/server/dnode/inc/dnodeEps.h b/source/server/dnode/inc/dnodeEps.h index 79c49bde40..80e2a2f488 100644 --- a/source/server/dnode/inc/dnodeEps.h +++ b/source/server/dnode/inc/dnodeEps.h @@ -22,19 +22,19 @@ extern "C" { #include "hash.h" #include "dnodeInt.h" -typedef struct DnEps { +typedef struct SDnEps { int32_t dnodeId; int32_t dnodeNum; SDnodeEp * dnodeList; SHashObj * dnodeHash; char file[PATH_MAX + 20]; pthread_mutex_t mutex; -} DnEps; +} SDnEps; -int32_t dnodeInitEps(DnEps **eps); -void dnodeCleanupEps(DnEps **eps); -void dnodeUpdateEps(DnEps *eps, SDnodeEps *data); -bool dnodeIsDnodeEpChanged(DnEps *eps, int32_t dnodeId, char *epstr); +int32_t dnodeInitEps(SDnEps **eps); +void dnodeCleanupEps(SDnEps **eps); +void dnodeUpdateEps(SDnEps *eps, SDnodeEps *data); +bool dnodeIsDnodeEpChanged(SDnEps *eps, int32_t dnodeId, char *epstr); void dnodeGetDnodeEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port); #ifdef __cplusplus diff --git a/source/server/dnode/inc/dnodeInt.h b/source/server/dnode/inc/dnodeInt.h index b32e23db14..9b56147ca0 100644 --- a/source/server/dnode/inc/dnodeInt.h +++ b/source/server/dnode/inc/dnodeInt.h @@ -27,32 +27,28 @@ extern "C" { #include "tstep.h" #include "dnode.h" -struct DnCfg; -struct DnCheck; -struct DnEps; -struct DnMnEps; -struct DnStatus; -struct DnTelem; -struct DnTrans; -struct DnMain; -struct Mnode; -struct Vnode; +struct SDnCfg; +struct SDnCheck; +struct SDnEps; +struct SDnMnEps; +struct SDnStatus; +struct SDnTelem; +struct SDnTrans; +struct SDnMain; -typedef struct Dnode { - struct SSteps * steps; - struct DnCfg * cfg; - struct DnCheck * check; - struct DnEps * eps; - struct DnMnEps * meps; - struct DnStatus *status; - struct DnTelem * telem; - struct DnTrans * trans; - struct DnMain * main; - struct Mnode * mnode; - struct Vnode * vnode; -} Dnode; +typedef struct SDnode { + struct SSteps* steps; + struct SDnCfg* cfg; + struct SDnCheck* check; + struct SDnEps* eps; + struct SDnMnEps* meps; + struct SDnStatus* status; + struct SDnTelem* telem; + struct SDnTrans* trans; + struct SDnMain* main; +} SDnode; -Dnode* dnodeInst(); +SDnode* dnodeInst(); #define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", 255, __VA_ARGS__); }} #define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", 255, __VA_ARGS__); }} diff --git a/source/server/dnode/inc/dnodeMain.h b/source/server/dnode/inc/dnodeMain.h index 9fe5d64800..08f6e10830 100644 --- a/source/server/dnode/inc/dnodeMain.h +++ b/source/server/dnode/inc/dnodeMain.h @@ -27,14 +27,14 @@ typedef enum { TD_RUN_STAT_STOPPED } RunStat; -typedef struct DnMain { +typedef struct SDnMain { RunStat runStatus; void * dnodeTimer; SStartupStep startup; -} DnMain; +} SDnMain; -int32_t dnodeInitMain(DnMain **main); -void dnodeCleanupMain(DnMain **main); +int32_t dnodeInitMain(SDnMain **main); +void dnodeCleanupMain(SDnMain **main); int32_t dnodeInitStorage(); void dnodeCleanupStorage(); void dnodeReportStartup(char *name, char *desc); diff --git a/source/server/dnode/inc/dnodeMnodeEps.h b/source/server/dnode/inc/dnodeMnodeEps.h index b94b08c304..a70c621046 100644 --- a/source/server/dnode/inc/dnodeMnodeEps.h +++ b/source/server/dnode/inc/dnodeMnodeEps.h @@ -21,19 +21,19 @@ extern "C" { #endif #include "dnodeInt.h" -typedef struct DnMnEps { +typedef struct SDnMnEps { SRpcEpSet mnodeEpSet; SMInfos mnodeInfos; char file[PATH_MAX + 20]; pthread_mutex_t mutex; -} DnMnEps; +} SDnMnEps; -int32_t dnodeInitMnodeEps(DnMnEps **meps); -void dnodeCleanupMnodeEps(DnMnEps **meps); -void dnodeUpdateMnodeFromStatus(DnMnEps *meps, SMInfos *pMinfos); -void dnodeUpdateMnodeFromPeer(DnMnEps *meps, SRpcEpSet *pEpSet); -void dnodeGetEpSetForPeer(DnMnEps *meps, SRpcEpSet *epSet); -void dnodeGetEpSetForShell(DnMnEps *meps, SRpcEpSet *epSet); +int32_t dnodeInitMnodeEps(SDnMnEps **meps); +void dnodeCleanupMnodeEps(SDnMnEps **meps); +void dnodeUpdateMnodeFromStatus(SDnMnEps *meps, SMInfos *pMinfos); +void dnodeUpdateMnodeFromPeer(SDnMnEps *meps, SRpcEpSet *pEpSet); +void dnodeGetEpSetForPeer(SDnMnEps *meps, SRpcEpSet *epSet); +void dnodeGetEpSetForShell(SDnMnEps *meps, SRpcEpSet *epSet); void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell); #ifdef __cplusplus diff --git a/source/server/dnode/inc/dnodeStatus.h b/source/server/dnode/inc/dnodeStatus.h index cfd58578c0..65a349ba6d 100644 --- a/source/server/dnode/inc/dnodeStatus.h +++ b/source/server/dnode/inc/dnodeStatus.h @@ -21,14 +21,14 @@ extern "C" { #endif #include "dnodeInt.h" -typedef struct DnStatus { +typedef struct SDnStatus { void * dnodeTimer; void * statusTimer; uint32_t rebootTime; -} DnStatus; +} SDnStatus; -int32_t dnodeInitStatus(DnStatus **status); -void dnodeCleanupStatus(DnStatus **status); +int32_t dnodeInitStatus(SDnStatus **status); +void dnodeCleanupStatus(SDnStatus **status); void dnodeProcessStatusRsp(SRpcMsg *pMsg); #ifdef __cplusplus diff --git a/source/server/dnode/inc/dnodeTelem.h b/source/server/dnode/inc/dnodeTelem.h index 48b10399bd..94356fffba 100644 --- a/source/server/dnode/inc/dnodeTelem.h +++ b/source/server/dnode/inc/dnodeTelem.h @@ -25,17 +25,17 @@ extern "C" { * sem_timedwait is NOT implemented on MacOSX * thus we use pthread_mutex_t/pthread_cond_t to simulate */ -typedef struct DnTelem { +typedef struct SDnTelem { bool enable; pthread_mutex_t lock; pthread_cond_t cond; volatile int32_t exit; pthread_t thread; char email[TSDB_FQDN_LEN]; -} DnTelem; +} SDnTelem; -int32_t dnodeInitTelem(DnTelem **telem); -void dnodeCleanupTelem(DnTelem **telem); +int32_t dnodeInitTelem(SDnTelem **telem); +void dnodeCleanupTelem(SDnTelem **telem); #ifdef __cplusplus } diff --git a/source/server/dnode/inc/dnodeTrans.h b/source/server/dnode/inc/dnodeTrans.h index 306624428b..d9016f0c7b 100644 --- a/source/server/dnode/inc/dnodeTrans.h +++ b/source/server/dnode/inc/dnodeTrans.h @@ -23,7 +23,7 @@ extern "C" { typedef void (*RpcMsgFp)( SRpcMsg *pMsg); -typedef struct DnTrans { +typedef struct SDnTrans { void * serverRpc; void * clientRpc; void * shellRpc; @@ -31,11 +31,10 @@ typedef struct DnTrans { int32_t submitReqNum; RpcMsgFp peerMsgFp[TSDB_MSG_TYPE_MAX]; RpcMsgFp shellMsgFp[TSDB_MSG_TYPE_MAX]; +} SDnTrans; -} DnTrans; - -int32_t dnodeInitTrans(DnTrans **rans); -void dnodeCleanupTrans(DnTrans **trans); +int32_t dnodeInitTrans(SDnTrans **rans); +void dnodeCleanupTrans(SDnTrans **trans); void dnodeSendMsgToMnode(SRpcMsg *rpcMsg); void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg); void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet); diff --git a/source/server/dnode/src/dnodeCfg.c b/source/server/dnode/src/dnodeCfg.c index 77329e2ad1..ee8a17db08 100644 --- a/source/server/dnode/src/dnodeCfg.c +++ b/source/server/dnode/src/dnodeCfg.c @@ -18,7 +18,7 @@ #include "cJSON.h" #include "dnodeCfg.h" -static int32_t dnodeReadCfg(DnCfg *cfg) { +static int32_t dnodeReadCfg(SDnCfg *cfg) { int32_t len = 0; int32_t maxLen = 200; char * content = calloc(1, maxLen + 1); @@ -76,7 +76,7 @@ PARSE_CFG_OVER: return 0; } -static int32_t dnodeWriteCfg(DnCfg *cfg) { +static int32_t dnodeWriteCfg(SDnCfg *cfg) { FILE *fp = fopen(cfg->file, "w"); if (!fp) { dError("failed to write %s since %s", cfg->file, strerror(errno)); @@ -103,8 +103,8 @@ static int32_t dnodeWriteCfg(DnCfg *cfg) { return 0; } -int32_t dnodeInitCfg(DnCfg **out) { - DnCfg* cfg = calloc(1, sizeof(DnCfg)); +int32_t dnodeInitCfg(SDnCfg **out) { + SDnCfg* cfg = calloc(1, sizeof(SDnCfg)); if (cfg == NULL) return -1; cfg->dnodeId = 0; @@ -127,15 +127,15 @@ int32_t dnodeInitCfg(DnCfg **out) { return ret; } -void dnodeCleanupCfg(DnCfg **out) { - DnCfg* cfg = *out; +void dnodeCleanupCfg(SDnCfg **out) { + SDnCfg* cfg = *out; *out = NULL; pthread_mutex_destroy(&cfg->mutex); free(cfg); } -void dnodeUpdateCfg(DnCfg *cfg, SDnodeCfg *data) { +void dnodeUpdateCfg(SDnCfg *cfg, SDnodeCfg *data) { if (cfg == NULL || cfg->dnodeId == 0) return; pthread_mutex_lock(&cfg->mutex); @@ -148,14 +148,14 @@ void dnodeUpdateCfg(DnCfg *cfg, SDnodeCfg *data) { pthread_mutex_unlock(&cfg->mutex); } -void dnodeSetDropped(DnCfg *cfg) { +void dnodeSetDropped(SDnCfg *cfg) { pthread_mutex_lock(&cfg->mutex); cfg->dropped = 1; dnodeWriteCfg(cfg); pthread_mutex_unlock(&cfg->mutex); } -int32_t dnodeGetDnodeId(DnCfg *cfg) { +int32_t dnodeGetDnodeId(SDnCfg *cfg) { int32_t dnodeId = 0; pthread_mutex_lock(&cfg->mutex); dnodeId = cfg->dnodeId; @@ -163,13 +163,13 @@ int32_t dnodeGetDnodeId(DnCfg *cfg) { return dnodeId; } -void dnodeGetClusterId(DnCfg *cfg, char *clusterId) { +void dnodeGetClusterId(SDnCfg *cfg, char *clusterId) { pthread_mutex_lock(&cfg->mutex); tstrncpy(clusterId, cfg->clusterId, TSDB_CLUSTER_ID_LEN); pthread_mutex_unlock(&cfg->mutex); } -void dnodeGetCfg(DnCfg *cfg, int32_t *dnodeId, char *clusterId) { +void dnodeGetCfg(SDnCfg *cfg, int32_t *dnodeId, char *clusterId) { pthread_mutex_lock(&cfg->mutex); *dnodeId = cfg->dnodeId; tstrncpy(clusterId, cfg->clusterId, TSDB_CLUSTER_ID_LEN); diff --git a/source/server/dnode/src/dnodeCheck.c b/source/server/dnode/src/dnodeCheck.c index 9175b4e519..8f561b1ea0 100644 --- a/source/server/dnode/src/dnodeCheck.c +++ b/source/server/dnode/src/dnodeCheck.c @@ -145,8 +145,8 @@ static int32_t dnodeCheckAccess() { return 0; } static int32_t dnodeCheckVersion() { return 0; } static int32_t dnodeCheckDatafile() { return 0; } -int32_t dnodeInitCheck(DnCheck **out) { - DnCheck *check = calloc(1, sizeof(DnCheck)); +int32_t dnodeInitCheck(SDnCheck **out) { + SDnCheck *check = calloc(1, sizeof(SDnCheck)); if (check == NULL) return -1; *out = check; @@ -195,8 +195,8 @@ int32_t dnodeInitCheck(DnCheck **out) { return 0; } -void dnodeCleanupCheck(DnCheck **out) { - DnCheck *check = *out; +void dnodeCleanupCheck(SDnCheck **out) { + SDnCheck *check = *out; *out = NULL; free(check); diff --git a/source/server/dnode/src/dnodeEps.c b/source/server/dnode/src/dnodeEps.c index e633da9fc9..317a9968fa 100644 --- a/source/server/dnode/src/dnodeEps.c +++ b/source/server/dnode/src/dnodeEps.c @@ -20,7 +20,7 @@ #include "dnodeEps.h" #include "dnodeCfg.h" -static void dnodePrintEps(DnEps *eps) { +static void dnodePrintEps(SDnEps *eps) { dDebug("print dnodeEp, dnodeNum:%d", eps->dnodeNum); for (int32_t i = 0; i < eps->dnodeNum; i++) { SDnodeEp *ep = &eps->dnodeList[i]; @@ -28,7 +28,7 @@ static void dnodePrintEps(DnEps *eps) { } } -static void dnodeResetEps(DnEps *eps, SDnodeEps *data) { +static void dnodeResetEps(SDnEps *eps, SDnodeEps *data) { assert(data != NULL); if (data->dnodeNum > eps->dnodeNum) { @@ -48,7 +48,7 @@ static void dnodeResetEps(DnEps *eps, SDnodeEps *data) { } } -static int32_t dnodeReadEps(DnEps *eps) { +static int32_t dnodeReadEps(SDnEps *eps) { int32_t len = 0; int32_t maxLen = 30000; char * content = calloc(1, maxLen + 1); @@ -145,7 +145,7 @@ PRASE_EPS_OVER: return 0; } -static int32_t dnodeWriteEps(DnEps *eps) { +static int32_t dnodeWriteEps(SDnEps *eps) { FILE *fp = fopen(eps->file, "w"); if (!fp) { dError("failed to write %s since %s", eps->file, strerror(errno)); @@ -182,8 +182,8 @@ static int32_t dnodeWriteEps(DnEps *eps) { return 0; } -int32_t dnodeInitEps(DnEps **out) { - DnEps *eps = calloc(1, sizeof(DnEps)); +int32_t dnodeInitEps(SDnEps **out) { + SDnEps *eps = calloc(1, sizeof(SDnEps)); if (eps == NULL) return -1; eps->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); @@ -203,8 +203,8 @@ int32_t dnodeInitEps(DnEps **out) { return ret; } -void dnodeCleanupEps(DnEps **out) { - DnEps *eps = *out; +void dnodeCleanupEps(SDnEps **out) { + SDnEps *eps = *out; *out = NULL; pthread_mutex_lock(&eps->mutex); @@ -225,7 +225,7 @@ void dnodeCleanupEps(DnEps **out) { free(eps); } -void dnodeUpdateEps(DnEps *eps, SDnodeEps *data) { +void dnodeUpdateEps(SDnEps *eps, SDnodeEps *data) { if (data == NULL || data->dnodeNum <= 0) return; data->dnodeNum = htonl(data->dnodeNum); @@ -250,7 +250,7 @@ void dnodeUpdateEps(DnEps *eps, SDnodeEps *data) { pthread_mutex_unlock(&eps->mutex); } -bool dnodeIsDnodeEpChanged(DnEps *eps, int32_t dnodeId, char *epstr) { +bool dnodeIsDnodeEpChanged(SDnEps *eps, int32_t dnodeId, char *epstr) { bool changed = false; pthread_mutex_lock(&eps->mutex); @@ -269,7 +269,7 @@ bool dnodeIsDnodeEpChanged(DnEps *eps, int32_t dnodeId, char *epstr) { } void dnodeGetDnodeEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port) { - DnEps *eps = dnodeInst()->eps; + SDnEps *eps = dnodeInst()->eps; pthread_mutex_lock(&eps->mutex); SDnodeEp *ep = taosHashGet(eps->dnodeHash, &dnodeId, sizeof(int32_t)); diff --git a/source/server/dnode/src/dnodeInt.c b/source/server/dnode/src/dnodeInt.c index 0da2834337..abb552cb02 100644 --- a/source/server/dnode/src/dnodeInt.c +++ b/source/server/dnode/src/dnodeInt.c @@ -33,9 +33,10 @@ #include "mnode.h" #include "vnode.h" -static Dnode tsDnode = {0}; - -Dnode *dnodeInst() { return &tsDnode; } +SDnode *dnodeInst() { + static SDnode inst = {0}; + return &inst; +} static int32_t dnodeInitVnodeModule(void **unused) { SVnodePara para; @@ -47,7 +48,7 @@ static int32_t dnodeInitVnodeModule(void **unused) { } static int32_t dnodeInitMnodeModule(void **unused) { - Dnode *dnode = dnodeInst(); + SDnode *dnode = dnodeInst(); SMnodePara para; para.fp.GetDnodeEp = dnodeGetDnodeEp; @@ -64,7 +65,7 @@ int32_t dnodeInit() { struct SSteps *steps = taosStepInit(24, dnodeReportStartup); if (steps == NULL) return -1; - Dnode *dnode = dnodeInst(); + SDnode *dnode = dnodeInst(); taosStepAdd(steps, "dnode-main", (void **)&dnode->main, (InitFp)dnodeInitMain, (CleanupFp)dnodeCleanupMain); taosStepAdd(steps, "dnode-storage", NULL, (InitFp)dnodeInitStorage, (CleanupFp)dnodeCleanupStorage); @@ -96,7 +97,7 @@ int32_t dnodeInit() { } void dnodeCleanup() { - Dnode *dnode = dnodeInst(); + SDnode *dnode = dnodeInst(); if (dnode->main->runStatus != TD_RUN_STAT_STOPPED) { dnode->main->runStatus = TD_RUN_STAT_STOPPED; taosStepCleanup(dnode->steps); diff --git a/source/server/dnode/src/dnodeMain.c b/source/server/dnode/src/dnodeMain.c index 73179d48b7..943ff61c13 100644 --- a/source/server/dnode/src/dnodeMain.c +++ b/source/server/dnode/src/dnodeMain.c @@ -55,8 +55,8 @@ void dnodePrintDiskInfo() { dInfo("=================================="); } -int32_t dnodeInitMain(DnMain **out) { - DnMain* main = calloc(1, sizeof(DnMain)); +int32_t dnodeInitMain(SDnMain **out) { + SDnMain* main = calloc(1, sizeof(SDnMain)); if (main == NULL) return -1; main->runStatus = TD_RUN_STAT_STOPPED; @@ -101,8 +101,8 @@ int32_t dnodeInitMain(DnMain **out) { return taosCheckGlobalCfg(); } -void dnodeCleanupMain(DnMain **out) { - DnMain *main = *out; +void dnodeCleanupMain(SDnMain **out) { + SDnMain *main = *out; *out = NULL; if (main->dnodeTimer != NULL) { @@ -202,7 +202,7 @@ void dnodeCleanupStorage() { } void dnodeReportStartup(char *name, char *desc) { - Dnode *dnode = dnodeInst(); + SDnode *dnode = dnodeInst(); if (dnode->main != NULL) { SStartupStep *startup = &dnode->main->startup; tstrncpy(startup->name, name, strlen(startup->name)); @@ -212,7 +212,7 @@ void dnodeReportStartup(char *name, char *desc) { } void dnodeReportStartupFinished(char *name, char *desc) { - Dnode *dnode = dnodeInst(); + SDnode *dnode = dnodeInst(); SStartupStep *startup = &dnode->main->startup; tstrncpy(startup->name, name, strlen(startup->name)); tstrncpy(startup->desc, desc, strlen(startup->desc)); @@ -222,7 +222,7 @@ void dnodeReportStartupFinished(char *name, char *desc) { void dnodeProcessStartupReq(SRpcMsg *pMsg) { dInfo("startup msg is received, cont:%s", (char *)pMsg->pCont); - Dnode *dnode = dnodeInst(); + SDnode *dnode = dnodeInst(); SStartupStep *pStep = rpcMallocCont(sizeof(SStartupStep)); memcpy(pStep, &dnode->main->startup, sizeof(SStartupStep)); @@ -234,7 +234,7 @@ void dnodeProcessStartupReq(SRpcMsg *pMsg) { } static int32_t dnodeStartMnode(SRpcMsg *pMsg) { - Dnode *dnode = dnodeInst(); + SDnode *dnode = dnodeInst(); SCreateMnodeMsg *pCfg = pMsg->pCont; pCfg->dnodeId = htonl(pCfg->dnodeId); if (pCfg->dnodeId != dnode->cfg->dnodeId) { @@ -254,7 +254,7 @@ static int32_t dnodeStartMnode(SRpcMsg *pMsg) { dDebug("meps index:%d, meps:%d:%s", i, pCfg->mnodes.mnodeInfos[i].mnodeId, pCfg->mnodes.mnodeInfos[i].mnodeEp); } - if (mnodeIsServing(dnode->mnode)) return 0; + if (mnodeIsServing()) return 0; return mnodeDeploy(&pCfg->mnodes); } diff --git a/source/server/dnode/src/dnodeMnodeEps.c b/source/server/dnode/src/dnodeMnodeEps.c index e072bb5bfd..ab5c76b580 100644 --- a/source/server/dnode/src/dnodeMnodeEps.c +++ b/source/server/dnode/src/dnodeMnodeEps.c @@ -22,7 +22,7 @@ #include "dnodeMnodeEps.h" #include "mnode.h" -static void dnodePrintMnodeEps(DnMnEps *meps) { +static void dnodePrintMnodeEps(SDnMnEps *meps) { SRpcEpSet *epset = &meps->mnodeEpSet; dInfo("print mnode eps, num:%d inuse:%d", epset->numOfEps, epset->inUse); for (int32_t i = 0; i < epset->numOfEps; i++) { @@ -30,7 +30,7 @@ static void dnodePrintMnodeEps(DnMnEps *meps) { } } -static void dnodeResetMnodeEps(DnMnEps *meps, SMInfos *mInfos) { +static void dnodeResetMnodeEps(SDnMnEps *meps, SMInfos *mInfos) { if (mInfos == NULL || mInfos->mnodeNum == 0) { meps->mnodeEpSet.numOfEps = 1; taosGetFqdnPortFromEp(tsFirst, meps->mnodeEpSet.fqdn[0], &meps->mnodeEpSet.port[0]); @@ -55,7 +55,7 @@ static void dnodeResetMnodeEps(DnMnEps *meps, SMInfos *mInfos) { dnodePrintMnodeEps(meps); } -static int32_t dnodeWriteMnodeEps(DnMnEps *meps) { +static int32_t dnodeWriteMnodeEps(SDnMnEps *meps) { FILE *fp = fopen(meps->file, "w"); if (!fp) { dError("failed to write %s since %s", meps->file, strerror(errno)); @@ -91,7 +91,7 @@ static int32_t dnodeWriteMnodeEps(DnMnEps *meps) { return 0; } -static int32_t dnodeReadMnodeEps(DnMnEps *meps, DnEps *deps) { +static int32_t dnodeReadMnodeEps(SDnMnEps *meps, SDnEps *deps) { int32_t len = 0; int32_t maxLen = 2000; char * content = calloc(1, maxLen + 1); @@ -192,7 +192,7 @@ PARSE_MINFOS_OVER: } void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) { - DnMnEps *meps = dnodeInst()->meps; + SDnMnEps *meps = dnodeInst()->meps; SRpcConnInfo connInfo = {0}; rpcGetConnInfo(rpcMsg->handle, &connInfo); @@ -222,8 +222,8 @@ void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) { rpcSendRedirectRsp(rpcMsg->handle, &epSet); } -int32_t dnodeInitMnodeEps(DnMnEps **out) { - DnMnEps *meps = calloc(1, sizeof(DnMnEps)); +int32_t dnodeInitMnodeEps(SDnMnEps **out) { + SDnMnEps *meps = calloc(1, sizeof(SDnMnEps)); if (meps == NULL) return -1; snprintf(meps->file, sizeof(meps->file), "%s/mnodeEpSet.json", tsDnodeDir); @@ -239,8 +239,8 @@ int32_t dnodeInitMnodeEps(DnMnEps **out) { return ret; } -void dnodeCleanupMnodeEps(DnMnEps **out) { - DnMnEps *meps = *out; +void dnodeCleanupMnodeEps(SDnMnEps **out) { + SDnMnEps *meps = *out; *out = NULL; if (meps != NULL) { @@ -249,7 +249,7 @@ void dnodeCleanupMnodeEps(DnMnEps **out) { } } -void dnodeUpdateMnodeFromStatus(DnMnEps *meps, SMInfos *mInfos) { +void dnodeUpdateMnodeFromStatus(SDnMnEps *meps, SMInfos *mInfos) { if (mInfos->mnodeNum <= 0 || mInfos->mnodeNum > TSDB_MAX_REPLICA) { dError("invalid mInfos since num:%d invalid", mInfos->mnodeNum); return; @@ -278,7 +278,7 @@ void dnodeUpdateMnodeFromStatus(DnMnEps *meps, SMInfos *mInfos) { pthread_mutex_unlock(&meps->mutex); } -void dnodeUpdateMnodeFromPeer(DnMnEps *meps, SRpcEpSet *ep) { +void dnodeUpdateMnodeFromPeer(SDnMnEps *meps, SRpcEpSet *ep) { if (ep->numOfEps <= 0) { dError("mInfos is changed, but content is invalid, discard it"); return; @@ -296,7 +296,7 @@ void dnodeUpdateMnodeFromPeer(DnMnEps *meps, SRpcEpSet *ep) { pthread_mutex_unlock(&meps->mutex); } -void dnodeGetEpSetForPeer(DnMnEps *meps, SRpcEpSet *epSet) { +void dnodeGetEpSetForPeer(SDnMnEps *meps, SRpcEpSet *epSet) { pthread_mutex_lock(&meps->mutex); *epSet = meps->mnodeEpSet; @@ -307,7 +307,7 @@ void dnodeGetEpSetForPeer(DnMnEps *meps, SRpcEpSet *epSet) { pthread_mutex_unlock(&meps->mutex); } -void dnodeGetEpSetForShell(DnMnEps *meps, SRpcEpSet *epSet) { +void dnodeGetEpSetForShell(SDnMnEps *meps, SRpcEpSet *epSet) { pthread_mutex_lock(&meps->mutex); *epSet = meps->mnodeEpSet; diff --git a/source/server/dnode/src/dnodeStatus.c b/source/server/dnode/src/dnodeStatus.c index 308be7d1b6..6abc886147 100644 --- a/source/server/dnode/src/dnodeStatus.c +++ b/source/server/dnode/src/dnodeStatus.c @@ -26,7 +26,7 @@ #include "vnode.h" static void dnodeSendStatusMsg(void *handle, void *tmrId) { - DnStatus *status = handle; + SDnStatus *status = handle; if (status->dnodeTimer == NULL) { dError("dnode timer is already released"); return; @@ -46,7 +46,7 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { return; } - Dnode *dnode = dnodeInst(); + SDnode *dnode = dnodeInst(); dnodeGetCfg(dnode->cfg, &pStatus->dnodeId, pStatus->clusterId); pStatus->dnodeId = htonl(dnodeGetDnodeId(dnode->cfg)); pStatus->version = htonl(tsVersion); @@ -86,8 +86,8 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { } void dnodeProcessStatusRsp(SRpcMsg *pMsg) { - Dnode *dnode = dnodeInst(); - DnStatus *status = pMsg->ahandle; + SDnode *dnode = dnodeInst(); + SDnStatus *status = pMsg->ahandle; if (pMsg->code != TSDB_CODE_SUCCESS) { dError("status rsp is received, error:%s", tstrerror(pMsg->code)); @@ -123,8 +123,8 @@ void dnodeProcessStatusRsp(SRpcMsg *pMsg) { taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, status, status->dnodeTimer, &status->statusTimer); } -int32_t dnodeInitStatus(DnStatus **out) { - DnStatus *status = calloc(1, sizeof(DnStatus)); +int32_t dnodeInitStatus(SDnStatus **out) { + SDnStatus *status = calloc(1, sizeof(SDnStatus)); if (status == NULL) return -1; status->statusTimer = NULL; status->dnodeTimer = dnodeInst()->main->dnodeTimer; @@ -135,8 +135,8 @@ int32_t dnodeInitStatus(DnStatus **out) { return TSDB_CODE_SUCCESS; } -void dnodeCleanupStatus(DnStatus **out) { - DnStatus *status = *out; +void dnodeCleanupStatus(SDnStatus **out) { + SDnStatus *status = *out; *out = NULL; if (status->statusTimer != NULL) { diff --git a/source/server/dnode/src/dnodeTelem.c b/source/server/dnode/src/dnodeTelem.c index 6b0bbfa6a0..b221746c83 100644 --- a/source/server/dnode/src/dnodeTelem.c +++ b/source/server/dnode/src/dnodeTelem.c @@ -154,14 +154,14 @@ static void dnodeAddMemoryInfo(SBufferWriter* bw) { fclose(fp); } -static void dnodeAddVersionInfo(DnTelem* telem, SBufferWriter* bw) { +static void dnodeAddVersionInfo(SDnTelem* telem, SBufferWriter* bw) { dnodeAddStringField(bw, "version", version); dnodeAddStringField(bw, "buildInfo", buildinfo); dnodeAddStringField(bw, "gitInfo", gitinfo); dnodeAddStringField(bw, "email", telem->email); } -static void dnodeAddRuntimeInfo(DnTelem* telem, SBufferWriter* bw) { +static void dnodeAddRuntimeInfo(SDnTelem* telem, SBufferWriter* bw) { SMnodeStat stat = {0}; if (mnodeGetStatistics(&stat) != 0) { return; @@ -179,7 +179,7 @@ static void dnodeAddRuntimeInfo(DnTelem* telem, SBufferWriter* bw) { dnodeAddIntField(bw, "compStorage", stat.compStorage); } -static void dnodeSendTelemetryReport(DnTelem* telem) { +static void dnodeSendTelemetryReport(SDnTelem* telem) { char buf[128] = {0}; uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); if (ip == 0xffffffff) { @@ -192,7 +192,7 @@ static void dnodeSendTelemetryReport(DnTelem* telem) { return; } - Dnode *dnode = dnodeInst(); + SDnode *dnode = dnodeInst(); SBufferWriter bw = tbufInitWriter(NULL, false); dnodeBeginObject(&bw); dnodeAddStringField(&bw, "instanceId", dnode->cfg->clusterId); @@ -227,7 +227,7 @@ static void dnodeSendTelemetryReport(DnTelem* telem) { } static void* dnodeTelemThreadFp(void* param) { - DnTelem* telem = param; + SDnTelem* telem = param; struct timespec end = {0}; clock_gettime(CLOCK_REALTIME, &end); @@ -253,7 +253,7 @@ static void* dnodeTelemThreadFp(void* param) { return NULL; } -static void dnodeGetEmail(DnTelem* telem, char* filepath) { +static void dnodeGetEmail(SDnTelem* telem, char* filepath) { int32_t fd = taosOpenFileRead(filepath); if (fd < 0) { return; @@ -266,8 +266,8 @@ static void dnodeGetEmail(DnTelem* telem, char* filepath) { taosCloseFile(fd); } -int32_t dnodeInitTelem(DnTelem** out) { - DnTelem* telem = calloc(1, sizeof(DnTelem)); +int32_t dnodeInitTelem(SDnTelem** out) { + SDnTelem* telem = calloc(1, sizeof(SDnTelem)); if (telem == NULL) return -1; telem->enable = tsEnableTelemetryReporting; @@ -296,8 +296,8 @@ int32_t dnodeInitTelem(DnTelem** out) { return 0; } -void dnodeCleanupTelem(DnTelem** out) { - DnTelem* telem = *out; +void dnodeCleanupTelem(SDnTelem** out) { + SDnTelem* telem = *out; *out = NULL; if (!telem->enable) { diff --git a/source/server/dnode/src/dnodeTrans.c b/source/server/dnode/src/dnodeTrans.c index c6139101bb..7a870f22c8 100644 --- a/source/server/dnode/src/dnodeTrans.c +++ b/source/server/dnode/src/dnodeTrans.c @@ -14,7 +14,7 @@ */ /* this file is mainly responsible for the communication between DNODEs. Each - * dnode works as both server and client. Dnode may send status, grant, config + * dnode works as both server and client. SDnode may send status, grant, config * messages to mnode, mnode may send create/alter/drop table/vnode messages * to dnode. All theses messages are handled from here */ @@ -30,7 +30,7 @@ #include "mnode.h" static void dnodeProcessPeerReq(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - Dnode * dnode = dnodeInst(); + SDnode * dnode = dnodeInst(); SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0}; if (pMsg->pCont == NULL) return; @@ -64,7 +64,7 @@ static void dnodeProcessPeerReq(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { } } -int32_t dnodeInitServer(DnTrans *trans) { +int32_t dnodeInitServer(SDnTrans *trans) { trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = vnodeProcessMsg; trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE] = vnodeProcessMsg; trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE] = vnodeProcessMsg; @@ -106,7 +106,7 @@ int32_t dnodeInitServer(DnTrans *trans) { return 0; } -void dnodeCleanupServer(DnTrans *trans) { +void dnodeCleanupServer(SDnTrans *trans) { if (trans->serverRpc) { rpcClose(trans->serverRpc); trans->serverRpc = NULL; @@ -115,7 +115,7 @@ void dnodeCleanupServer(DnTrans *trans) { } static void dnodeProcessRspFromPeer(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - Dnode *dnode = dnodeInst(); + SDnode *dnode = dnodeInst(); if (dnode->main->runStatus == TD_RUN_STAT_STOPPED) { if (pMsg == NULL || pMsg->pCont == NULL) return; dTrace("msg:%p is ignored since dnode is stopping", pMsg); @@ -141,7 +141,7 @@ static void dnodeProcessRspFromPeer(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { rpcFreeCont(pMsg->pCont); } -int32_t dnodeInitClient(DnTrans *trans) { +int32_t dnodeInitClient(SDnTrans *trans) { trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP] = mnodeProcessMsg; trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE_RSP] = mnodeProcessMsg; trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP] = mnodeProcessMsg; @@ -186,7 +186,7 @@ int32_t dnodeInitClient(DnTrans *trans) { return 0; } -void dnodeCleanupClient(DnTrans *trans) { +void dnodeCleanupClient(SDnTrans *trans) { if (trans->clientRpc) { rpcClose(trans->clientRpc); trans->clientRpc = NULL; @@ -195,7 +195,7 @@ void dnodeCleanupClient(DnTrans *trans) { } static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - Dnode * dnode = dnodeInst(); + SDnode * dnode = dnodeInst(); SRpcMsg rpcMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0}; if (pMsg->pCont == NULL) return; @@ -213,7 +213,7 @@ static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { return; } - DnTrans *trans = dnode->trans; + SDnTrans *trans = dnode->trans; if (pMsg->msgType == TSDB_MSG_TYPE_QUERY) { atomic_fetch_add_32(&trans->queryReqNum, 1); } else if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) { @@ -247,26 +247,26 @@ static int32_t dnodeAuthNetTest(char *user, char *spi, char *encrypt, char *secr } void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) { - Dnode *dnode = dnodeInst(); + SDnode *dnode = dnodeInst(); rpcSendRequest(dnode->trans->clientRpc, epSet, rpcMsg, NULL); } void dnodeSendMsgToMnode(SRpcMsg *rpcMsg) { - Dnode * dnode = dnodeInst(); + SDnode * dnode = dnodeInst(); SRpcEpSet epSet = {0}; dnodeGetEpSetForPeer(dnode->meps, &epSet); dnodeSendMsgToDnode(&epSet, rpcMsg); } void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) { - Dnode * dnode = dnodeInst(); + SDnode * dnode = dnodeInst(); SRpcEpSet epSet = {0}; dnodeGetEpSetForPeer(dnode->meps, &epSet); rpcSendRecv(dnode->trans->clientRpc, &epSet, rpcMsg, rpcRsp); } void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet) { - Dnode *dnode = dnodeInst(); + SDnode *dnode = dnodeInst(); rpcSendRecv(dnode->trans->clientRpc, epSet, rpcMsg, rpcRsp); } @@ -303,7 +303,7 @@ static int32_t dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, c return rpcRsp.code; } -int32_t dnodeInitShell(DnTrans *trans) { +int32_t dnodeInitShell(SDnTrans *trans) { trans->shellMsgFp[TSDB_MSG_TYPE_SUBMIT] = vnodeProcessMsg; trans->shellMsgFp[TSDB_MSG_TYPE_QUERY] = vnodeProcessMsg; trans->shellMsgFp[TSDB_MSG_TYPE_FETCH] = vnodeProcessMsg; @@ -376,15 +376,15 @@ int32_t dnodeInitShell(DnTrans *trans) { return 0; } -void dnodeCleanupShell(DnTrans *trans) { +void dnodeCleanupShell(SDnTrans *trans) { if (trans->shellRpc) { rpcClose(trans->shellRpc); trans->shellRpc = NULL; } } -int32_t dnodeInitTrans(DnTrans **out) { - DnTrans *trans = calloc(1, sizeof(DnTrans)); +int32_t dnodeInitTrans(SDnTrans **out) { + SDnTrans *trans = calloc(1, sizeof(SDnTrans)); if (trans == NULL) return -1; *out = trans; @@ -404,8 +404,8 @@ int32_t dnodeInitTrans(DnTrans **out) { return 0; } -void dnodeCleanupTrans(DnTrans **out) { - DnTrans* trans = *out; +void dnodeCleanupTrans(SDnTrans **out) { + SDnTrans* trans = *out; *out = NULL; dnodeCleanupShell(trans); From 0264c48ad79fd5124b5aa7b4c7ec306364b7ab04 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 8 Oct 2021 10:24:26 +0800 Subject: [PATCH 44/44] add cmake command for version.c.in --- source/util/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index df7efb3380..ffe84c63e8 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -11,3 +11,5 @@ target_link_libraries( PUBLIC zlib PUBLIC lz4_static ) + +CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/src/version.c.in" "${CMAKE_CURRENT_SOURCE_DIR}/src/version.c")