From 75bd0ebafe13149d39237a1f0cf8f7bebc24ef10 Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 19 Mar 2020 21:12:09 +0800 Subject: [PATCH 01/12] [TD-15] move sdb to mnode module --- src/CMakeLists.txt | 1 - src/dnode/CMakeLists.txt | 2 +- src/mnode/CMakeLists.txt | 2 +- src/{sdb/inc/sdbint.h => mnode/inc/mgmtSdb.h} | 0 .../src/sdbEngine.c => mnode/src/mgmtSdb.c} | 47 ++++++++++++++- src/sdb/CMakeLists.txt | 16 ----- src/sdb/src/sdbstr.c | 58 ------------------- src/{sdb => util}/inc/hashint.h | 0 src/{sdb => util}/inc/hashstr.h | 0 src/{sdb => util}/src/hashint.c | 0 src/{sdb => util}/src/hashstr.c | 0 11 files changed, 47 insertions(+), 79 deletions(-) rename src/{sdb/inc/sdbint.h => mnode/inc/mgmtSdb.h} (100%) rename src/{sdb/src/sdbEngine.c => mnode/src/mgmtSdb.c} (97%) delete mode 100644 src/sdb/CMakeLists.txt delete mode 100644 src/sdb/src/sdbstr.c rename src/{sdb => util}/inc/hashint.h (100%) rename src/{sdb => util}/inc/hashstr.h (100%) rename src/{sdb => util}/src/hashint.c (100%) rename src/{sdb => util}/src/hashstr.c (100%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a05cc0224a..326f00dbbd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -12,7 +12,6 @@ ADD_SUBDIRECTORY(client) ADD_SUBDIRECTORY(query) ADD_SUBDIRECTORY(kit) ADD_SUBDIRECTORY(plugins) -ADD_SUBDIRECTORY(sdb) ADD_SUBDIRECTORY(mnode) ADD_SUBDIRECTORY(vnode) ADD_SUBDIRECTORY(dnode) diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index 298eb6110c..d85782b463 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -13,7 +13,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_EXECUTABLE(taosd ${SRC}) - TARGET_LINK_LIBRARIES(taosd mnode sdb taos_static monitor http tsdb) + TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http tsdb) #IF (TD_CLUSTER) # TARGET_LINK_LIBRARIES(taosd dcluster) diff --git a/src/mnode/CMakeLists.txt b/src/mnode/CMakeLists.txt index 5bf4cfd604..bacd7497ad 100644 --- a/src/mnode/CMakeLists.txt +++ b/src/mnode/CMakeLists.txt @@ -13,7 +13,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(mnode ${SRC}) - TARGET_LINK_LIBRARIES(mnode trpc tutil sdb pthread) + TARGET_LINK_LIBRARIES(mnode trpc tutil pthread) IF (TD_CLUSTER) TARGET_LINK_LIBRARIES(mnode) diff --git a/src/sdb/inc/sdbint.h b/src/mnode/inc/mgmtSdb.h similarity index 100% rename from src/sdb/inc/sdbint.h rename to src/mnode/inc/mgmtSdb.h diff --git a/src/sdb/src/sdbEngine.c b/src/mnode/src/mgmtSdb.c similarity index 97% rename from src/sdb/src/sdbEngine.c rename to src/mnode/src/mgmtSdb.c index fbc41089d1..04802e77da 100644 --- a/src/sdb/src/sdbEngine.c +++ b/src/mnode/src/mgmtSdb.c @@ -13,11 +13,11 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" - #include "sdb.h" -#include "sdbint.h" #include "tutil.h" +#include "mgmtSdb.h" #define abs(x) (((x) < 0) ? -(x) : (x)) @@ -1015,3 +1015,46 @@ void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->id; } int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } + + +int32_t (*mpeerInitMnodesFp)(char *directory) = NULL; +void (*mpeerCleanUpMnodesFp)() = NULL; +int32_t (*mpeerForwardRequestFp)(SSdbTable *pTable, char type, void *cont, int32_t contLen) = NULL; + +char *sdbStatusStr[] = { + "offline", + "unsynced", + "syncing", + "serving", + "null" +}; + +char *sdbRoleStr[] = { + "unauthed", + "undecided", + "master", + "slave", + "null" +}; + +int32_t sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int32_t dataLen) { + if (mpeerForwardRequestFp) { + return mpeerForwardRequestFp(pTable, type, data, dataLen); + } else { + return 0; + } +} + +int32_t sdbInitPeers(char *directory) { + if (mpeerInitMnodesFp) { + return (*mpeerInitMnodesFp)(directory); + } else { + return 0; + } +} + +void sdbCleanUpPeers() { + if (mpeerCleanUpMnodesFp) { + (*mpeerCleanUpMnodesFp)(); + } +} diff --git a/src/sdb/CMakeLists.txt b/src/sdb/CMakeLists.txt deleted file mode 100644 index 47ea6e15b8..0000000000 --- a/src/sdb/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) -INCLUDE_DIRECTORIES(inc) - -IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) - AUX_SOURCE_DIRECTORY(src SRC) - ADD_LIBRARY(sdb ${SRC}) - TARGET_LINK_LIBRARIES(sdb trpc) - IF (TD_CLUSTER) - TARGET_LINK_LIBRARIES(sdb) - ENDIF() -ENDIF () diff --git a/src/sdb/src/sdbstr.c b/src/sdb/src/sdbstr.c deleted file mode 100644 index 59c01eb15a..0000000000 --- a/src/sdb/src/sdbstr.c +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#define _DEFAULT_SOURCE -#include "sdbint.h" - -int32_t (*mpeerInitMnodesFp)(char *directory) = NULL; -void (*mpeerCleanUpMnodesFp)() = NULL; -int32_t (*mpeerForwardRequestFp)(SSdbTable *pTable, char type, void *cont, int32_t contLen) = NULL; - -char *sdbStatusStr[] = { - "offline", - "unsynced", - "syncing", - "serving", - "null" -}; - -char *sdbRoleStr[] = { - "unauthed", - "undecided", - "master", - "slave", - "null" -}; - -int32_t sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int32_t dataLen) { - if (mpeerForwardRequestFp) { - return mpeerForwardRequestFp(pTable, type, data, dataLen); - } else { - return 0; - } -} - -int32_t sdbInitPeers(char *directory) { - if (mpeerInitMnodesFp) { - return (*mpeerInitMnodesFp)(directory); - } else { - return 0; - } -} - -void sdbCleanUpPeers() { - if (mpeerCleanUpMnodesFp) { - (*mpeerCleanUpMnodesFp)(); - } -} diff --git a/src/sdb/inc/hashint.h b/src/util/inc/hashint.h similarity index 100% rename from src/sdb/inc/hashint.h rename to src/util/inc/hashint.h diff --git a/src/sdb/inc/hashstr.h b/src/util/inc/hashstr.h similarity index 100% rename from src/sdb/inc/hashstr.h rename to src/util/inc/hashstr.h diff --git a/src/sdb/src/hashint.c b/src/util/src/hashint.c similarity index 100% rename from src/sdb/src/hashint.c rename to src/util/src/hashint.c diff --git a/src/sdb/src/hashstr.c b/src/util/src/hashstr.c similarity index 100% rename from src/sdb/src/hashstr.c rename to src/util/src/hashstr.c From 4b7e463be83f9ef6b8a88fdb9a9877d839ac7acf Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 19 Mar 2020 23:29:29 +0800 Subject: [PATCH 02/12] [TD-15] refactor sdb --- src/inc/mnode.h | 1 - src/inc/sdb.h | 131 ----------- src/inc/taosdef.h | 10 + src/kit/CMakeLists.txt | 4 +- src/mnode/inc/mgmtMnode.h | 7 +- src/mnode/inc/mgmtSdb.h | 139 ++++-------- src/mnode/src/mgmtChildTable.c | 16 +- src/mnode/src/mgmtDClient.c | 2 +- src/mnode/src/mgmtDServer.c | 2 +- src/mnode/src/mgmtDb.c | 12 +- src/mnode/src/mgmtDnode.c | 2 +- src/mnode/src/mgmtMain.c | 12 +- src/mnode/src/mgmtMnode.c | 70 ++++-- src/mnode/src/mgmtNormalTable.c | 15 +- src/mnode/src/mgmtSdb.c | 371 +++++++++++--------------------- src/mnode/src/mgmtShell.c | 42 ++-- src/mnode/src/mgmtSuperTable.c | 12 +- src/mnode/src/mgmtTable.c | 1 + src/mnode/src/mgmtUser.c | 10 +- src/mnode/src/mgmtVgroup.c | 4 +- src/util/inc/tglobalcfg.h | 6 - src/util/inc/tlog.h | 19 ++ src/util/src/tglobalcfg.c | 28 +-- 23 files changed, 294 insertions(+), 622 deletions(-) delete mode 100644 src/inc/sdb.h diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 35902df617..b7631674b3 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -25,7 +25,6 @@ extern "C" { #include "taosdef.h" #include "taosmsg.h" #include "taoserror.h" -#include "sdb.h" #include "tglobalcfg.h" #include "thash.h" #include "tidpool.h" diff --git a/src/inc/sdb.h b/src/inc/sdb.h deleted file mode 100644 index 4b4de1ac4b..0000000000 --- a/src/inc/sdb.h +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_SDB_H -#define TDENGINE_SDB_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "taosmsg.h" -#include "taosdef.h" - -extern uint16_t tsMgmtMgmtPort; -extern uint16_t tsMgmtSyncPort; -extern int tsMgmtPeerHBTimer; // seconds -extern char * sdbStatusStr[]; -extern char * sdbRoleStr[]; -extern int sdbMaster; -extern SRpcIpSet *pSdbIpList; -extern SRpcIpSet *pSdbPublicIpList; - -extern void (*sdbWorkAsMasterCallback)(); // this function pointer will be set by taosd - -enum _keytype { - SDB_KEYTYPE_STRING, SDB_KEYTYPE_UINT32, SDB_KEYTYPE_AUTO, SDB_KEYTYPE_RECYCLE, SDB_KEYTYPE_MAX -}; - -#define SDB_ROLE_UNAPPROVED 0 -#define SDB_ROLE_UNDECIDED 1 -#define SDB_ROLE_MASTER 2 -#define SDB_ROLE_SLAVE 3 - -#define SDB_STATUS_OFFLINE 0 -#define SDB_STATUS_UNSYNCED 1 -#define SDB_STATUS_SYNCING 2 -#define SDB_STATUS_SERVING 3 -#define SDB_STATUS_DELETED 4 - -enum _sdbaction { - SDB_TYPE_INSERT, - SDB_TYPE_DELETE, - SDB_TYPE_UPDATE, - SDB_TYPE_DECODE, - SDB_TYPE_ENCODE, - SDB_TYPE_BEFORE_BATCH_UPDATE, - SDB_TYPE_BATCH_UPDATE, - SDB_TYPE_AFTER_BATCH_UPDATE, - SDB_TYPE_RESET, - SDB_TYPE_DESTROY, - SDB_MAX_ACTION_TYPES -}; - -#define SDB_MAX_PEERS 4 -typedef struct { - uint32_t ip; - uint32_t publicIp; - char ipstr[20]; - char zone[12]; - char role; - int64_t createdTime; - uint64_t dbVersion; - int64_t lostTime; - char status; - char numOfMnodes; - int numOfDnodes; - char updateEnd[1]; - - // internal - int syncFd; - void *hbTimer; - void *pSync; -} SSdbPeer; - -extern SSdbPeer *sdbPeer[]; -#define sdbInited (sdbPeer[0]) -#define sdbStatus (sdbPeer[0]->status) - -void *sdbOpenTable(int maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, - void *(*appTool)(char, void *, char *, int, int *)); - -void *sdbGetRow(void *handle, void *key); - -int64_t sdbInsertRow(void *handle, void *row, int rowSize); - -int sdbDeleteRow(void *handle, void *key); - -int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated); - -void *sdbFetchRow(void *handle, void *pNode, void **ppRow); - -int sdbBatchUpdateRow(void *handle, void *row, int rowSize); - -int64_t sdbGetId(void *handle); - -int64_t sdbGetNumOfRows(void *handle); - -void sdbSaveSnapShot(void *handle); - -void sdbCloseTable(void *handle); - -int sdbRemovePeerByIp(uint32_t ip); - -int sdbInitPeers(char *directory); - -void sdbCleanUpPeers(); - -int64_t sdbGetVersion(); - -int32_t sdbGetRunStatus(); - -#define TSDB_MAX_NORMAL_TABLES 10000 -#define TSDB_MAX_SUPER_TABLES 1000 - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_SDB_H diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 7308715a87..49ed45559f 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -309,6 +309,16 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_SESSIONS_PER_VNODE (300) #define TSDB_SESSIONS_PER_DNODE (TSDB_SESSIONS_PER_VNODE * TSDB_MAX_VNODES) +#define TSDB_MAX_MNODES 5 +#define TSDB_MAX_DNODES 10 +#define TSDB_MAX_ACCOUNTS 10 +#define TSDB_MAX_USERS 20 +#define TSDB_MAX_DBS 100 +#define TSDB_MAX_VGROUPS 1000 +#define TSDB_MAX_SUPER_TABLES 100 +#define TSDB_MAX_NORMAL_TABLES 1000 +#define TSDB_MAX_CHILD_TABLES 100000 + enum { TSDB_PRECISION_MILLI, TSDB_PRECISION_MICRO, diff --git a/src/kit/CMakeLists.txt b/src/kit/CMakeLists.txt index 66e8cf7398..386c8a92f7 100644 --- a/src/kit/CMakeLists.txt +++ b/src/kit/CMakeLists.txt @@ -2,5 +2,5 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) ADD_SUBDIRECTORY(shell) -ADD_SUBDIRECTORY(taosdemo) -ADD_SUBDIRECTORY(taosdump) +#ADD_SUBDIRECTORY(taosdemo) +#ADD_SUBDIRECTORY(taosdump) diff --git a/src/mnode/inc/mgmtMnode.h b/src/mnode/inc/mgmtMnode.h index d768d2dd7c..d381b09055 100644 --- a/src/mnode/inc/mgmtMnode.h +++ b/src/mnode/inc/mgmtMnode.h @@ -20,9 +20,12 @@ extern "C" { #endif -bool mgmtCheckRedirect(void *handle); +int32_t mgmtInitMnodes(); +void mgmtCleanupMnodes(); -void mgmtGetMnodeIpList(SRpcIpSet *ipSet); +bool mgmtCheckRedirect(void *handle); +void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet); +void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet); int32_t mgmtAddMnode(uint32_t privateIp, uint32_t publicIp); int32_t mgmtRemoveMnode(uint32_t privateIp); diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h index 30ebe0909c..c64db286c6 100644 --- a/src/mnode/inc/mgmtSdb.h +++ b/src/mnode/inc/mgmtSdb.h @@ -13,8 +13,12 @@ * along with this program. If not, see . */ -#ifndef _sdbint_header_ -#define _sdbint_header_ +#ifndef TDENGINE_MNODE_SDB_H +#define TDENGINE_MNODE_SDB_H + +#ifdef __cplusplus +extern "C" { +#endif #include #include @@ -27,115 +31,46 @@ #include "hashint.h" #include "hashstr.h" -#include "sdb.h" #include "tchecksum.h" #include "tlog.h" #include "trpc.h" #include "tutil.h" -#define sdbError(...) \ - if (sdbDebugFlag & DEBUG_ERROR) { \ - tprintf("ERROR MND-SDB ", 255, __VA_ARGS__); \ - } -#define sdbWarn(...) \ - if (sdbDebugFlag & DEBUG_WARN) { \ - tprintf("WARN MND-SDB ", sdbDebugFlag, __VA_ARGS__); \ - } -#define sdbTrace(...) \ - if (sdbDebugFlag & DEBUG_TRACE) { \ - tprintf("MND-SDB ", sdbDebugFlag, __VA_ARGS__); \ - } -#define sdbPrint(...) \ - { tprintf("MND-SDB ", 255, __VA_ARGS__); } +enum _keytype { + SDB_KEYTYPE_STRING, + SDB_KEYTYPE_AUTO, + SDB_KEYTYPE_MAX +}; -#define mpeerError(...) \ - if (sdbDebugFlag & DEBUG_ERROR) { \ - tprintf("ERROR MND-MPEER ", 255, __VA_ARGS__); \ - } -#define mpeerWarn(...) \ - if (sdbDebugFlag & DEBUG_WARN) { \ - tprintf("WARN MND-MPEER ", sdbDebugFlag, __VA_ARGS__); \ - } -#define mpeerTrace(...) \ - if (sdbDebugFlag & DEBUG_TRACE) { \ - tprintf("MND-MPEER ", sdbDebugFlag, __VA_ARGS__); \ - } -#define mpeerPrint(...) \ - { tprintf("MND-MPEER ", 255, __VA_ARGS__); } +enum _sdbaction { + SDB_TYPE_INSERT, + SDB_TYPE_DELETE, + SDB_TYPE_UPDATE, + SDB_TYPE_DECODE, + SDB_TYPE_ENCODE, + SDB_TYPE_DESTROY, + SDB_MAX_ACTION_TYPES +}; -#define sdbLError(...) taosLogError(__VA_ARGS__) sdbError(__VA_ARGS__) -#define sdbLWarn(...) taosLogWarn(__VA_ARGS__) sdbWarn(__VA_ARGS__) -#define sdbLPrint(...) taosLogPrint(__VA_ARGS__) sdbPrint(__VA_ARGS__) +uint64_t sdbGetVersion(); +bool sdbInServerState(); +bool sdbIsMaster(); -#define SDB_MAX_PEERS 4 -#define SDB_DELIMITER 0xFFF00F00 -#define SDB_ENDCOMMIT 0xAFFFAAAF +void *sdbOpenTable(int32_t maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, + void *(*appTool)(char, void *, char *, int32_t, int32_t *)); +void sdbCloseTable(void *handle); -typedef struct { - uint64_t swVersion; - int16_t sdbFileVersion; - char reserved[6]; - TSCKSUM checkSum; -} SSdbHeader; - -typedef struct { - char type; - // short rowSize; - char *row; -} SSdbUpdate; - -typedef struct _SSdbTable { - SSdbHeader header; - int maxRows; - int dbId; - int32_t maxRowSize; - char name[TSDB_DB_NAME_LEN]; - char fn[128]; - int keyType; - uint32_t autoIndex; - int64_t numOfRows; - int64_t id; - int64_t size; - void * iHandle; - int fd; - void *(*appTool)(char, void *, char *, int, int *); - pthread_mutex_t mutex; - SSdbUpdate * update; - int numOfUpdates; - int updatePos; -} SSdbTable; - -typedef struct { - int64_t id; - int64_t offset; - int rowSize; - void * row; -} SRowMeta; - -typedef struct { - int32_t delimiter; - int32_t rowSize; - int64_t id; - char data[]; -} SRowHead; - -typedef struct { - uint8_t dbId; - char type; - uint64_t version; - short dataLen; - char data[]; -} SForwardMsg; - -extern SSdbTable *tableList[]; -extern int sdbMaxPeers; -extern int sdbNumOfTables; -extern int64_t sdbVersion; - -int sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int dataLen); -int mpeerRetrieveRows(int fd, SSdbTable *pTable, uint64_t version); -void sdbResetTable(SSdbTable *pTable); -extern const int16_t sdbFileVersion; +void *sdbGetRow(void *handle, void *key); +void *sdbFetchRow(void *handle, void *pNode, void **ppRow); +int64_t sdbGetId(void *handle); +int64_t sdbGetNumOfRows(void *handle); +int64_t sdbInsertRow(void *handle, void *row, int32_t rowSize); +int32_t sdbDeleteRow(void *handle, void *key); +int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, char isUpdated); +#ifdef __cplusplus +} #endif + +#endif \ No newline at end of file diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index 53304c7565..589beeaaba 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taosmsg.h" +#include "taosdef.h" #include "tschemautil.h" #include "tscompression.h" #include "tskiplist.h" @@ -26,10 +27,11 @@ #include "mgmtAcct.h" #include "mgmtChildTable.h" #include "mgmtDb.h" +#include "mgmtDClient.h" #include "mgmtGrant.h" #include "mgmtProfile.h" +#include "mgmtSdb.h" #include "mgmtShell.h" -#include "mgmtDClient.h" #include "mgmtSuperTable.h" #include "mgmtTable.h" #include "mgmtVgroup.h" @@ -56,7 +58,6 @@ static void mgmtChildTableActionInit() { mgmtChildTableActionFp[SDB_TYPE_UPDATE] = mgmtChildTableActionUpdate; mgmtChildTableActionFp[SDB_TYPE_ENCODE] = mgmtChildTableActionEncode; mgmtChildTableActionFp[SDB_TYPE_DECODE] = mgmtChildTableActionDecode; - mgmtChildTableActionFp[SDB_TYPE_RESET] = mgmtChildTableActionReset; mgmtChildTableActionFp[SDB_TYPE_DESTROY] = mgmtChildTableActionDestroy; } @@ -93,7 +94,7 @@ void *mgmtChildTableActionInsert(void *row, char *str, int32_t size, int32_t *ss return NULL; } - if (!sdbMaster) { + if (!sdbIsMaster()) { int32_t sid = taosAllocateId(pVgroup->idPool); if (sid != pTable->sid) { mError("ctable:%s, sid:%d is not matched from the master:%d", pTable->tableId, sid, pTable->sid); @@ -311,13 +312,6 @@ void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTab } void* mgmtCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t tid) { - int32_t numOfTables = sdbGetNumOfRows(tsChildTableSdb); - if (numOfTables >= tsMaxTables) { - mError("ctable:%s, numOfTables:%d exceed maxTables:%d", pCreate->tableId, numOfTables, tsMaxTables); - terrno = TSDB_CODE_TOO_MANY_TABLES; - return NULL; - } - char *pTagData = (char *) pCreate->schema; // it is a tag key SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTagData); if (pSuperTable == NULL) { @@ -338,7 +332,7 @@ void* mgmtCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t pTable->type = TSDB_CHILD_TABLE; pTable->createdTime = taosGetTimestampMs(); pTable->uid = (((uint64_t) pTable->vgId) << 40) + ((((uint64_t) pTable->sid) & ((1ul << 24) - 1ul)) << 16) + - ((uint64_t) sdbGetVersion() & ((1ul << 16) - 1ul)); + (sdbGetVersion() & ((1ul << 16) - 1ul)); pTable->sid = tid; pTable->vgId = pVgroup->vgId; pTable->superTable = pSuperTable; diff --git a/src/mnode/src/mgmtDClient.c b/src/mnode/src/mgmtDClient.c index da11ad2081..cbdcb8ff5d 100644 --- a/src/mnode/src/mgmtDClient.c +++ b/src/mnode/src/mgmtDClient.c @@ -42,7 +42,7 @@ int32_t mgmtInitDClient() { rpcInit.label = "MND-DC"; rpcInit.numOfThreads = 1; rpcInit.cfp = mgmtProcessRspFromDnode; - rpcInit.sessions = tsMaxDnodes * 5; + rpcInit.sessions = 100; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.user = "mgmtDClient"; diff --git a/src/mnode/src/mgmtDServer.c b/src/mnode/src/mgmtDServer.c index 177a45764c..b1d01b91f1 100644 --- a/src/mnode/src/mgmtDServer.c +++ b/src/mnode/src/mgmtDServer.c @@ -45,7 +45,7 @@ int32_t mgmtInitDServer() { rpcInit.label = "MND-DS"; rpcInit.numOfThreads = 1; rpcInit.cfp = mgmtProcessMsgFromDnode; - rpcInit.sessions = tsMaxDnodes * 5; + rpcInit.sessions = 100; rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.afp = mgmtDServerRetrieveAuth; diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index caf84737b0..5cd1259486 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -24,11 +24,12 @@ #include "mgmtBalance.h" #include "mgmtDb.h" #include "mgmtDnode.h" -#include "mgmtMnode.h" #include "mgmtGrant.h" #include "mgmtShell.h" +#include "mgmtMnode.h" #include "mgmtNormalTable.h" #include "mgmtChildTable.h" +#include "mgmtSdb.h" #include "mgmtSuperTable.h" #include "mgmtTable.h" #include "mgmtUser.h" @@ -62,7 +63,6 @@ static void mgmtDbActionInit() { mgmtDbActionFp[SDB_TYPE_UPDATE] = mgmtDbActionUpdate; mgmtDbActionFp[SDB_TYPE_ENCODE] = mgmtDbActionEncode; mgmtDbActionFp[SDB_TYPE_DECODE] = mgmtDbActionDecode; - mgmtDbActionFp[SDB_TYPE_RESET] = mgmtDbActionReset; mgmtDbActionFp[SDB_TYPE_DESTROY] = mgmtDbActionDestroy; } @@ -83,7 +83,7 @@ int32_t mgmtInitDbs() { SDbObj tObj; tsDbUpdateSize = tObj.updateEnd - (char *)&tObj; - tsDbSdb = sdbOpenTable(tsMaxDbs, tsDbUpdateSize, "dbs", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtDbAction); + tsDbSdb = sdbOpenTable(TSDB_MAX_DBS, tsDbUpdateSize, "dbs", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtDbAction); if (tsDbSdb == NULL) { mError("failed to init db data"); return -1; @@ -252,12 +252,6 @@ static int32_t mgmtCheckDbParams(SCMCreateDbMsg *pCreate) { } static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { - int32_t numOfDbs = sdbGetNumOfRows(tsDbSdb); - if (numOfDbs >= tsMaxDbs) { - mWarn("numOfDbs:%d, exceed tsMaxDbs:%d", numOfDbs, tsMaxDbs); - return TSDB_CODE_TOO_MANY_DATABASES; - } - int32_t code = mgmtCheckDbLimit(pAcct); if (code != 0) { return code; diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index c7dcefa5f4..77c314c56f 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -597,7 +597,7 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { return; } - mgmtGetMnodeIpList(&pRsp->ipList); + mgmtGetMnodePrivateIpList(&pRsp->ipList); pRsp->dnodeState.dnodeId = htonl(pDnode->dnodeId); pRsp->dnodeState.moduleStatus = htonl(pDnode->moduleStatus); diff --git a/src/mnode/src/mgmtMain.c b/src/mnode/src/mgmtMain.c index 66200e5a14..6aec06970e 100644 --- a/src/mnode/src/mgmtMain.c +++ b/src/mnode/src/mgmtMain.c @@ -25,6 +25,8 @@ #include "mgmtDClient.h" #include "mgmtDnode.h" #include "mgmtDServer.h" +#include "mgmtMnode.h" +#include "mgmtSdb.h" #include "mgmtVgroup.h" #include "mgmtUser.h" #include "mgmtTable.h" @@ -65,7 +67,7 @@ int32_t mgmtStartSystem() { return 0; } - tsMgmtTmr = taosTmrInit((tsMaxDnodes + tsMaxShellConns) * 3, 200, 3600000, "MND"); + tsMgmtTmr = taosTmrInit((tsMaxShellConns) * 3, 200, 3600000, "MND"); if (tsMgmtTmr == NULL) { mError("failed to init timer"); return -1; @@ -109,8 +111,8 @@ int32_t mgmtStartSystem() { return -1; } - if (sdbInitPeers(tsMnodeDir) < 0) { - mError("failed to init peers"); + if (mgmtInitMnodes() < 0) { + mError("failed to init mnodes"); return -1; } @@ -125,7 +127,7 @@ int32_t mgmtStartSystem() { void mgmtStopSystem() { - if (sdbMaster) { + if (sdbIsMaster()) { mTrace("it is a master mgmt node, it could not be stopped"); return; } @@ -136,7 +138,7 @@ void mgmtStopSystem() { void mgmtCleanUpSystem() { mPrint("starting to clean up mgmt"); - sdbCleanUpPeers(); + mgmtCleanupMnodes(); mgmtCleanupBalance(); mgmtCleanUpShell(); mgmtCleanupDClient(); diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index dc322e742f..ac18af0158 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -18,48 +18,81 @@ #include "trpc.h" #include "tschemautil.h" #include "mgmtMnode.h" +#include "mgmtSdb.h" #include "mgmtUser.h" -int32_t (*mgmtAddMnodeFp)(uint32_t privateIp, uint32_t publicIp) = NULL; -int32_t (*mgmtRemoveMnodeFp)(uint32_t privateIp) = NULL; -int32_t (*mgmtGetMnodesNumFp)() = NULL; -void * (*mgmtGetNextMnodeFp)(SShowObj *pShow, SMnodeObj **pMnode) = NULL; +int32_t (*mpeerAddMnodeFp)(uint32_t privateIp, uint32_t publicIp) = NULL; +int32_t (*mpeerRemoveMnodeFp)(uint32_t privateIp) = NULL; +int32_t (*mpeerGetMnodesNumFp)() = NULL; +void * (*mpeerGetNextMnodeFp)(SShowObj *pShow, SMnodeObj **pMnode) = NULL; +int32_t (*mpeerInitMnodesFp)() = NULL; +void (*mpeerCleanUpMnodesFp)() = NULL; static SMnodeObj tsMnodeObj = {0}; static int32_t mgmtGetMnodeMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static char *mgmtMnodeStatusStr[] = { + "offline", + "unsynced", + "syncing", + "serving", + "null" +}; + +static char *mgmtMnodeRoleStr[] = { + "unauthed", + "undecided", + "master", + "slave", + "null" +}; + +int32_t mgmtInitMnodes() { + if (mpeerInitMnodesFp) { + return (*mpeerInitMnodesFp)(); + } else { + return 0; + } +} + +void mgmtCleanupMnodes() { + if (mpeerCleanUpMnodesFp) { + (*mpeerCleanUpMnodesFp)(); + } +} + bool mgmtCheckRedirect(void *handle) { return false; } int32_t mgmtAddMnode(uint32_t privateIp, uint32_t publicIp) { - if (mgmtAddMnodeFp) { - return (*mgmtAddMnodeFp)(privateIp, publicIp); + if (mpeerAddMnodeFp) { + return (*mpeerAddMnodeFp)(privateIp, publicIp); } else { return 0; } } int32_t mgmtRemoveMnode(uint32_t privateIp) { - if (mgmtRemoveMnodeFp) { - return (*mgmtRemoveMnodeFp)(privateIp); + if (mpeerRemoveMnodeFp) { + return (*mpeerRemoveMnodeFp)(privateIp); } else { return 0; } } static int32_t mgmtGetMnodesNum() { - if (mgmtGetMnodesNumFp) { - return (*mgmtGetMnodesNumFp)(); + if (mpeerGetMnodesNumFp) { + return (*mpeerGetMnodesNumFp)(); } else { return 1; } } static void *mgmtGetNextMnode(SShowObj *pShow, SMnodeObj **pMnode) { - if (mgmtGetNextMnodeFp) { - return (*mgmtGetNextMnodeFp)(pShow, pMnode); + if (mpeerGetNextMnodeFp) { + return (*mpeerGetNextMnodeFp)(pShow, pMnode); } else { if (*pMnode == NULL) { *pMnode = &tsMnodeObj; @@ -149,11 +182,11 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, sdbStatusStr[(uint8_t)pMnode->status]); + strcpy(pWrite, mgmtMnodeStatusStr[pMnode->status]); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, sdbRoleStr[(uint8_t)pMnode->role]); + strcpy(pWrite, mgmtMnodeRoleStr[pMnode->role]); cols++; tinet_ntoa(ipstr, pMnode->publicIp); @@ -168,7 +201,14 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi return numOfRows; } -void mgmtGetMnodeIpList(SRpcIpSet *ipSet) { +void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet) { + ipSet->inUse = 0; + ipSet->port = htons(tsMnodeDnodePort); + ipSet->numOfIps = 1; + ipSet->ip[0] = htonl(inet_addr(tsMasterIp)); +} + +void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet) { ipSet->inUse = 0; ipSet->port = htons(tsMnodeDnodePort); ipSet->numOfIps = 1; diff --git a/src/mnode/src/mgmtNormalTable.c b/src/mnode/src/mgmtNormalTable.c index ca138bf42d..d42a596008 100644 --- a/src/mnode/src/mgmtNormalTable.c +++ b/src/mnode/src/mgmtNormalTable.c @@ -27,6 +27,7 @@ #include "mgmtDClient.h" #include "mgmtGrant.h" #include "mgmtNormalTable.h" +#include "mgmtSdb.h" #include "mgmtSuperTable.h" #include "mgmtTable.h" #include "mgmtVgroup.h" @@ -55,7 +56,6 @@ static void mgmtNormalTableActionInit() { mgmtNormalTableActionFp[SDB_TYPE_UPDATE] = mgmtNormalTableActionUpdate; mgmtNormalTableActionFp[SDB_TYPE_ENCODE] = mgmtNormalTableActionEncode; mgmtNormalTableActionFp[SDB_TYPE_DECODE] = mgmtNormalTableActionDecode; - mgmtNormalTableActionFp[SDB_TYPE_RESET] = mgmtNormalTableActionReset; mgmtNormalTableActionFp[SDB_TYPE_DESTROY] = mgmtNormalTableActionDestroy; } @@ -98,7 +98,7 @@ void *mgmtNormalTableActionInsert(void *row, char *str, int32_t size, int32_t *s return NULL; } - if (!sdbMaster) { + if (!sdbIsMaster()) { int32_t sid = taosAllocateId(pVgroup->idPool); if (sid != pTable->sid) { mError("sid:%d is not matched from the master:%d", sid, pTable->sid); @@ -222,7 +222,7 @@ int32_t mgmtInitNormalTables() { SNormalTableObj tObj; tsNormalTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - tsNormalTableSdb = sdbOpenTable(tsMaxTables, sizeof(SNormalTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, + tsNormalTableSdb = sdbOpenTable(TSDB_MAX_NORMAL_TABLES, sizeof(SNormalTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, "ntables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtNormalTableAction); if (tsNormalTableSdb == NULL) { mError("failed to init ntables data"); @@ -323,13 +323,6 @@ void *mgmtBuildCreateNormalTableMsg(SNormalTableObj *pTable) { } void *mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t sid) { - int32_t numOfTables = sdbGetNumOfRows(tsNormalTableSdb); - if (numOfTables >= TSDB_MAX_NORMAL_TABLES) { - mError("table:%s, numOfTables:%d exceed maxTables:%d", pCreate->tableId, numOfTables, TSDB_MAX_NORMAL_TABLES); - terrno = TSDB_CODE_TOO_MANY_TABLES; - return NULL; - } - SNormalTableObj *pTable = (SNormalTableObj *) calloc(sizeof(SNormalTableObj), 1); if (pTable == NULL) { mError("table:%s, failed to alloc memory", pCreate->tableId); @@ -341,7 +334,7 @@ void *mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t pTable->type = TSDB_NORMAL_TABLE; pTable->vgId = pVgroup->vgId; pTable->createdTime = taosGetTimestampMs(); - pTable->uid = (((uint64_t) pTable->createdTime) << 16) + ((uint64_t) sdbGetVersion() & ((1ul << 16) - 1ul)); + pTable->uid = (((uint64_t) pTable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul)); pTable->sid = sid; pTable->sversion = 0; pTable->numOfColumns = htons(pCreate->numOfColumns); diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 04802e77da..3564392ed1 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -15,60 +15,107 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "sdb.h" +#include "taosdef.h" #include "tutil.h" #include "mgmtSdb.h" #define abs(x) (((x) < 0) ? -(x) : (x)) +#define SDB_MAX_PEERS 4 +#define SDB_DELIMITER 0xFFF00F00 +#define SDB_ENDCOMMIT 0xAFFFAAAF +#define SDB_STATUS_OFFLINE 0 +#define SDB_STATUS_SERVING 1 + + +typedef struct { + uint64_t swVersion; + int16_t sdbFileVersion; + char reserved[6]; + TSCKSUM checkSum; +} SSdbHeader; + +typedef struct _SSdbTable { + SSdbHeader header; + int maxRows; + int dbId; + int32_t maxRowSize; + char name[TSDB_DB_NAME_LEN]; + char fn[128]; + int keyType; + uint32_t autoIndex; + int64_t numOfRows; + int64_t id; + int64_t size; + void * iHandle; + int fd; + void *(*appTool)(char, void *, char *, int, int *); + pthread_mutex_t mutex; +} SSdbTable; + +typedef struct { + int64_t id; + int64_t offset; + int rowSize; + void * row; +} SRowMeta; + +typedef struct { + int32_t delimiter; + int32_t rowSize; + int64_t id; + char data[]; +} SRowHead; + +typedef struct { + uint8_t dbId; + char type; + uint64_t version; + short dataLen; + char data[]; +} SForwardMsg; extern char version[]; -const int16_t sdbFileVersion = 0; -SRpcIpSet *pSdbIpList = NULL; -SRpcIpSet *pSdbPublicIpList = NULL; -SSdbPeer * sdbPeer[SDB_MAX_PEERS]; // first slot for self +const int16_t sdbFileVersion = 2; +int32_t (*mpeerForwardRequestFp)(SSdbTable *pTable, char type, void *cont, int32_t contLen) = NULL; -#ifdef CLUSTER -int sdbMaster = 0; -#else -int sdbMaster = 1; -#endif +static SSdbTable *sdbTableList[10] = {0}; +static int32_t sdbNumOfTables = 0; +static uint64_t sdbVersion = 0; +static int32_t sdbMaster = 0; +static int32_t sdbStatus = SDB_STATUS_OFFLINE; -void *(*sdbInitIndexFp[])(int maxRows, int dataSize) = {sdbOpenStrHash, sdbOpenIntHash, sdbOpenIntHash}; -void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash, sdbAddIntHash}; +// #ifdef CLUSTER +// int32_t sdbMaster = 0; +// #else +// int32_t sdbMaster = 1; +// #endif -void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash, sdbDeleteIntHash}; +static void *(*sdbInitIndexFp[])(int32_t maxRows, int32_t dataSize) = {sdbOpenStrHash, sdbOpenIntHash}; +static void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash}; +static void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash}; +static void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData}; +static void (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash}; +static void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = {sdbFetchStrHashData, sdbFetchIntHashData}; -void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData, sdbGetIntHashData}; +void sdbResetTable(SSdbTable *pTable); +void sdbSaveSnapShot(void *handle); -void (*sdbCleanUpIndexFp[])(void *handle) = { - sdbCloseStrHash, sdbCloseIntHash, sdbCloseIntHash, -}; +uint64_t sdbGetVersion() { return sdbVersion; } +bool sdbInServerState() { return sdbStatus == SDB_STATUS_SERVING; } +bool sdbIsMaster() { return sdbMaster; } +int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->id; } +int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } -void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = { - sdbFetchStrHashData, sdbFetchIntHashData, sdbFetchIntHashData, -}; - -SSdbTable *tableList[20]; -int sdbNumOfTables; -int64_t sdbVersion; - -int64_t sdbGetVersion() { - return sdbVersion; -}; - -int32_t sdbGetRunStatus() { - if (!tsIsCluster) { - return SDB_STATUS_SERVING; +static int32_t sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int32_t dataLen) { + if (mpeerForwardRequestFp) { + return mpeerForwardRequestFp(pTable, type, data, dataLen); + } else { + return 0; } - - if (sdbInited == NULL) { - return SDB_STATUS_OFFLINE; - } - return sdbStatus; } -void sdbFinishCommit(void *handle) { +static void sdbFinishCommit(void *handle) { SSdbTable *pTable = (SSdbTable *)handle; uint32_t sdbEcommit = SDB_ENDCOMMIT; @@ -78,7 +125,7 @@ void sdbFinishCommit(void *handle) { pTable->size += sizeof(sdbEcommit); } -int sdbOpenSdbFile(SSdbTable *pTable) { +static int32_t sdbOpenSdbFile(SSdbTable *pTable) { struct stat fstat, ofstat; uint64_t size; char * dirc = NULL; @@ -91,7 +138,7 @@ int sdbOpenSdbFile(SSdbTable *pTable) { memcpy(swVersion.cversion, version, sizeof(uint64_t)); // check sdb.db and .sdb.db status - char fn[128] = "\0"; + char fn[TSDB_FILENAME_LEN] = "\0"; dirc = strdup(pTable->fn); basec = strdup(pTable->fn); sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); @@ -168,27 +215,15 @@ int sdbOpenSdbFile(SSdbTable *pTable) { return pTable->fd; } -// TODO: Change here -void sdbAddIntoUpdateList(SSdbTable *pTable, char type, char *row) { - pTable->numOfUpdates++; - pTable->updatePos = pTable->numOfUpdates % pTable->maxRows; - - if (pTable->update[pTable->updatePos].type == SDB_TYPE_DELETE) - (*(pTable->appTool))(SDB_TYPE_DESTROY, pTable->update[pTable->updatePos].row, NULL, 0, NULL); - - pTable->update[pTable->updatePos].type = type; - pTable->update[pTable->updatePos].row = row; -} - -int sdbInitTableByFile(SSdbTable *pTable) { +static int32_t sdbInitTableByFile(SSdbTable *pTable) { SRowMeta rowMeta; - int numOfDels = 0; - int bytes = 0; + int32_t numOfDels = 0; + int32_t bytes = 0; int64_t oldId = 0; void * pMetaRow = NULL; - int total_size = 0; - int real_size = 0; - int maxAutoIndex = 0; + int32_t total_size = 0; + int32_t real_size = 0; + int32_t maxAutoIndex = 0; oldId = pTable->id; if (sdbOpenSdbFile(pTable) < 0) return -1; @@ -277,7 +312,7 @@ int sdbInitTableByFile(SSdbTable *pTable) { numOfDels++; } else { // Reset the object TODO: is it possible to merge reset and // update ?? - (*(pTable->appTool))(SDB_TYPE_RESET, pMetaRow, rowHead->data, rowHead->rowSize, NULL); + //(*(pTable->appTool))(SDB_TYPE_RESET, pMetaRow, rowHead->data, rowHead->rowSize, NULL); } numOfDels++; } @@ -293,9 +328,6 @@ int sdbInitTableByFile(SSdbTable *pTable) { sdbVersion += (pTable->id - oldId); if (numOfDels > pTable->maxRows / 4) sdbSaveSnapShot(pTable); - pTable->numOfUpdates = 0; - pTable->updatePos = 0; - tfree(rowHead); return 0; @@ -304,20 +336,12 @@ sdb_exit1: return -1; } -void *sdbOpenTable(int maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, - void *(*appTool)(char, void *, char *, int, int *)) { +void *sdbOpenTable(int32_t maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, + void *(*appTool)(char, void *, char *, int32_t, int32_t *)) { SSdbTable *pTable = (SSdbTable *)malloc(sizeof(SSdbTable)); if (pTable == NULL) return NULL; memset(pTable, 0, sizeof(SSdbTable)); - int size = sizeof(SSdbUpdate) * maxRows; - pTable->update = (SSdbUpdate *)malloc(size); - if (pTable->update == NULL) { - free(pTable); - return NULL; - }; - memset(pTable->update, 0, size); - strcpy(pTable->name, name); pTable->keyType = keyType; pTable->maxRows = maxRows; @@ -332,14 +356,14 @@ void *sdbOpenTable(int maxRows, int32_t maxRowSize, char *name, uint8_t keyType, if (sdbInitTableByFile(pTable) < 0) return NULL; pTable->dbId = sdbNumOfTables++; - tableList[pTable->dbId] = pTable; + sdbTableList[pTable->dbId] = pTable; sdbTrace("table:%s is initialized, numOfRows:%d, numOfTables:%d", pTable->name, pTable->numOfRows, sdbNumOfTables); return pTable; } -SRowMeta *sdbGetRowMeta(void *handle, void *key) { +static SRowMeta *sdbGetRowMeta(void *handle, void *key) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta; @@ -365,15 +389,14 @@ void *sdbGetRow(void *handle, void *key) { return pMeta->row; } -// row here must be encoded string (rowSize > 0) or the object it self (rowSize -// = 0) -int64_t sdbInsertRow(void *handle, void *row, int rowSize) { +// row here must be encoded string (rowSize > 0) or the object it self (rowSize = 0) +int64_t sdbInsertRow(void *handle, void *row, int32_t rowSize) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta rowMeta; int64_t id = -1; void * pObj = NULL; - int total_size = 0; - int real_size = 0; + int32_t total_size = 0; + int32_t real_size = 0; /* char action = SDB_TYPE_INSERT; */ if (pTable == NULL) { @@ -398,9 +421,6 @@ int64_t sdbInsertRow(void *handle, void *row, int rowSize) { case SDB_KEYTYPE_STRING: sdbError("table:%s, failed to insert record:%s sdbVersion:%" PRId64 " id:%" PRId64 , pTable->name, (char *)row, sdbVersion, pTable->id); break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbError("table:%s, failed to insert record:%s sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id); - break; case SDB_KEYTYPE_AUTO: sdbError("table:%s, failed to insert record:%d sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, *(int32_t *)row, sdbVersion, pTable->id); break; @@ -464,18 +484,12 @@ int64_t sdbInsertRow(void *handle, void *row, int rowSize) { pTable->size += real_size; sdbFinishCommit(pTable); - sdbAddIntoUpdateList(pTable, SDB_TYPE_INSERT, rowMeta.row); - pTable->numOfRows++; switch (pTable->keyType) { case SDB_KEYTYPE_STRING: sdbTrace("table:%s, a record is inserted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, pTable->name, (char *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbTrace("table:%s, a record is inserted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; case SDB_KEYTYPE_AUTO: sdbTrace("table:%s, a record is inserted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, pTable->name, *(int32_t *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); @@ -502,14 +516,14 @@ int64_t sdbInsertRow(void *handle, void *row, int rowSize) { } // row here can be object or null-terminated string -int sdbDeleteRow(void *handle, void *row) { +int32_t sdbDeleteRow(void *handle, void *row) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta = NULL; - int code = -1; + int32_t code = -1; void * pMetaRow = NULL; SRowHead * rowHead = NULL; - int rowSize = 0; - int total_size = 0; + int32_t rowSize = 0; + int32_t total_size = 0; /* char action = SDB_TYPE_DELETE; */ if (pTable == NULL) return -1; @@ -527,9 +541,6 @@ int sdbDeleteRow(void *handle, void *row) { case SDB_KEYTYPE_STRING: rowSize = strlen((char *)row) + 1; break; - case SDB_KEYTYPE_UINT32: - rowSize = sizeof(uint32_t); - break; case SDB_KEYTYPE_AUTO: rowSize = sizeof(uint64_t); break; @@ -568,17 +579,12 @@ int sdbDeleteRow(void *handle, void *row) { sdbFinishCommit(pTable); pTable->numOfRows--; - // TODO:Change the update list here - sdbAddIntoUpdateList(pTable, SDB_TYPE_DELETE, pMetaRow); + switch (pTable->keyType) { case SDB_KEYTYPE_STRING: sdbTrace("table:%s, a record is deleted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbTrace("table:%s, a record is deleted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id, pTable->numOfRows); - break; case SDB_KEYTYPE_AUTO: sdbTrace("table:%s, a record is deleted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); @@ -606,12 +612,12 @@ int sdbDeleteRow(void *handle, void *row) { } // row here can be the object or the string info (encoded string) -int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated) { +int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, char isUpdated) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta = NULL; - int code = -1; - int total_size = 0; - int real_size = 0; + int32_t code = -1; + int32_t total_size = 0; + int32_t real_size = 0; /* char action = SDB_TYPE_UPDATE; */ if (pTable == NULL || row == NULL) return -1; @@ -622,10 +628,6 @@ int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated) { sdbError("table:%s, failed to update record:%s, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, (char *) row, sdbVersion, pTable->id); break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbError("table:%s, failed to update record:%s, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, taosIpStr(*(int32_t *) row), sdbVersion, pTable->id); - break; case SDB_KEYTYPE_AUTO: sdbError("table:%s, failed to update record:%d, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, *(int32_t *) row, sdbVersion, pTable->id); @@ -694,10 +696,6 @@ int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated) { sdbTrace("table:%s, a record is updated:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbTrace("table:%s, a record is updated:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id, pTable->numOfRows); - break; case SDB_KEYTYPE_AUTO: sdbTrace("table:%s, a record is updated:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); @@ -708,7 +706,6 @@ int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated) { break; } - sdbAddIntoUpdateList(pTable, SDB_TYPE_UPDATE, pMetaRow); code = 0; } @@ -719,79 +716,6 @@ int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated) { return code; } -// row here must be the instruction string -int sdbBatchUpdateRow(void *handle, void *row, int rowSize) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta = NULL; - int total_size = 0; - /* char action = SDB_TYPE_BATCH_UPDATE; */ - - if (pTable == NULL || row == NULL || rowSize <= 0) return -1; - pMeta = sdbGetRowMeta(handle, row); - if (pMeta == NULL) { - sdbTrace("table:%s, record is not there, batch update failed", pTable->name); - return -1; - } - - void *pMetaRow = pMeta->row; - assert(pMetaRow != NULL); - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->name); - return -1; - } - - pthread_mutex_lock(&pTable->mutex); - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_BATCH_UPDATE, row, rowSize) == 0) { - /* // write action */ - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - - (*(pTable->appTool))(SDB_TYPE_BEFORE_BATCH_UPDATE, pMetaRow, NULL, 0, NULL); - - void *next_row = pMetaRow; - while (next_row != NULL) { - pTable->id++; - sdbVersion++; - - void *last_row = next_row; - next_row = (*(pTable->appTool))(SDB_TYPE_BATCH_UPDATE, last_row, (char *)row, rowSize, 0); - memset(rowHead, 0, sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM)); - - // update in current layer - pMeta->id = pTable->id; - pMeta->offset = pTable->size; - - // write to disk - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pMeta->id; - (*(pTable->appTool))(SDB_TYPE_ENCODE, last_row, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - taosCalcChecksumAppend(0, (uint8_t *)rowHead, sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); - pMeta->rowSize = rowHead->rowSize; - lseek(pTable->fd, pTable->size, SEEK_SET); - twrite(pTable->fd, rowHead, sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); - pTable->size += (sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); - - sdbAddIntoUpdateList(pTable, SDB_TYPE_UPDATE, last_row); - - if (next_row != NULL) { - pMeta = sdbGetRowMeta(handle, next_row); - } - } - - sdbFinishCommit(pTable); - - (*(pTable->appTool))(SDB_TYPE_AFTER_BATCH_UPDATE, pMetaRow, NULL, 0, NULL); - } - pthread_mutex_unlock(&pTable->mutex); - - tfree(rowHead); - - return 0; -} - void sdbCloseTable(void *handle) { SSdbTable *pTable = (SSdbTable *)handle; void * pNode = NULL; @@ -814,20 +738,19 @@ void sdbCloseTable(void *handle) { sdbNumOfTables--; sdbTrace("table:%s is closed, id:%" PRId64 " numOfTables:%d", pTable->name, pTable->id, sdbNumOfTables); - tfree(pTable->update); tfree(pTable); } void sdbResetTable(SSdbTable *pTable) { /* SRowHead rowHead; */ SRowMeta rowMeta; - int bytes; - int total_size = 0; - int real_size = 0; + int32_t bytes; + int32_t total_size = 0; + int32_t real_size = 0; SRowHead *rowHead = NULL; void * pMetaRow = NULL; int64_t oldId = pTable->id; - int oldNumOfRows = pTable->numOfRows; + int32_t oldNumOfRows = pTable->numOfRows; if (sdbOpenSdbFile(pTable) < 0) return; pTable->numOfRows = oldNumOfRows; @@ -911,24 +834,21 @@ void sdbResetTable(SSdbTable *pTable) { } sdbVersion += (pTable->id - oldId); - pTable->numOfUpdates = 0; - pTable->updatePos = 0; - + tfree(rowHead); sdbPrint("table:%s is updated, sdbVerion:%" PRId64 " id:%" PRId64, pTable->name, sdbVersion, pTable->id); } -// TODO:A problem here :use snapshot file to sync another node will cause -// problem +// TODO:A problem here :use snapshot file to sync another node will cause problem void sdbSaveSnapShot(void *handle) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta; void * pNode = NULL; - int total_size = 0; - int real_size = 0; - int size = 0; - int numOfRows = 0; + int32_t total_size = 0; + int32_t real_size = 0; + int32_t size = 0; + int32_t numOfRows = 0; uint32_t sdbEcommit = SDB_ENDCOMMIT; char * dirc = NULL; char * basec = NULL; @@ -942,7 +862,7 @@ void sdbSaveSnapShot(void *handle) { dirc = strdup(pTable->fn); basec = strdup(pTable->fn); sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); - int fd = open(fn, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + int32_t fd = open(fn, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); tfree(dirc); tfree(basec); @@ -1011,50 +931,3 @@ void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { return pNode; } - -int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->id; } - -int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } - - -int32_t (*mpeerInitMnodesFp)(char *directory) = NULL; -void (*mpeerCleanUpMnodesFp)() = NULL; -int32_t (*mpeerForwardRequestFp)(SSdbTable *pTable, char type, void *cont, int32_t contLen) = NULL; - -char *sdbStatusStr[] = { - "offline", - "unsynced", - "syncing", - "serving", - "null" -}; - -char *sdbRoleStr[] = { - "unauthed", - "undecided", - "master", - "slave", - "null" -}; - -int32_t sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int32_t dataLen) { - if (mpeerForwardRequestFp) { - return mpeerForwardRequestFp(pTable, type, data, dataLen); - } else { - return 0; - } -} - -int32_t sdbInitPeers(char *directory) { - if (mpeerInitMnodesFp) { - return (*mpeerInitMnodesFp)(directory); - } else { - return 0; - } -} - -void sdbCleanUpPeers() { - if (mpeerCleanUpMnodesFp) { - (*mpeerCleanUpMnodesFp)(); - } -} diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index e87c964496..c50ecc01c7 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -32,6 +32,7 @@ #include "mgmtMnode.h" #include "mgmtNormalTable.h" #include "mgmtProfile.h" +#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtSuperTable.h" #include "mgmtTable.h" @@ -63,7 +64,7 @@ int32_t mgmtInitShell() { mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_HEARTBEAT, mgmtProcessHeartBeatMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mgmtProcessConnectMsg); - tsMgmtTranQhandle = taosInitScheduler(tsMaxDnodes + tsMaxShellConns, 1, "mnodeT"); + tsMgmtTranQhandle = taosInitScheduler(tsMaxShellConns, 1, "mnodeT"); int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore / 4.0; if (numOfThreads < 1) { @@ -131,7 +132,7 @@ void mgmtAddToShellQueue(SQueuedMsg *queuedMsg) { } static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { - if (sdbGetRunStatus() != SDB_STATUS_SERVING) { + if (!sdbInServerState()) { mgmtProcessMsgWhileNotReady(rpcMsg); rpcFreeCont(rpcMsg->pCont); return; @@ -309,20 +310,10 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) { return; } - pHBRsp->ipList.inUse = 0; - pHBRsp->ipList.port = htons(tsMnodeShellPort); - pHBRsp->ipList.numOfIps = 0; - if (pSdbPublicIpList != NULL && pSdbIpList != NULL) { - pHBRsp->ipList.numOfIps = htons(pSdbPublicIpList->numOfIps); - if (connInfo.serverIp == tsPublicIpInt) { - for (int i = 0; i < pSdbPublicIpList->numOfIps; ++i) { - pHBRsp->ipList.ip[i] = htonl(pSdbPublicIpList->ip[i]); - } - } else { - for (int i = 0; i < pSdbIpList->numOfIps; ++i) { - pHBRsp->ipList.ip[i] = htonl(pSdbIpList->ip[i]); - } - } + if (connInfo.serverIp == tsPublicIpInt) { + mgmtGetMnodePublicIpList(&pHBRsp->ipList); + } else { + mgmtGetMnodePrivateIpList(&pHBRsp->ipList); } /* @@ -411,20 +402,11 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) { strcpy(pConnectRsp->serverVersion, version); pConnectRsp->writeAuth = pUser->writeAuth; pConnectRsp->superAuth = pUser->superAuth; - pConnectRsp->ipList.inUse = 0; - pConnectRsp->ipList.port = htons(tsMnodeShellPort); - pConnectRsp->ipList.numOfIps = 0; - if (pSdbPublicIpList != NULL && pSdbIpList != NULL) { - pConnectRsp->ipList.numOfIps = htons(pSdbPublicIpList->numOfIps); - if (connInfo.serverIp == tsPublicIpInt) { - for (int i = 0; i < pSdbPublicIpList->numOfIps; ++i) { - pConnectRsp->ipList.ip[i] = htonl(pSdbPublicIpList->ip[i]); - } - } else { - for (int i = 0; i < pSdbIpList->numOfIps; ++i) { - pConnectRsp->ipList.ip[i] = htonl(pSdbIpList->ip[i]); - } - } + + if (connInfo.serverIp == tsPublicIpInt) { + mgmtGetMnodePublicIpList(&pConnectRsp->ipList); + } else { + mgmtGetMnodePrivateIpList(&pConnectRsp->ipList); } connect_over: diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index b0d506980a..075633cd4d 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -31,6 +31,7 @@ #include "mgmtGrant.h" #include "mgmtShell.h" #include "mgmtSuperTable.h" +#include "mgmtSdb.h" #include "mgmtTable.h" #include "mgmtUser.h" #include "mgmtVgroup.h" @@ -63,7 +64,6 @@ static void mgmtSuperTableActionInit() { mgmtSuperTableActionFp[SDB_TYPE_UPDATE] = mgmtSuperTableActionUpdate; mgmtSuperTableActionFp[SDB_TYPE_ENCODE] = mgmtSuperTableActionEncode; mgmtSuperTableActionFp[SDB_TYPE_DECODE] = mgmtSuperTableActionDecode; - mgmtSuperTableActionFp[SDB_TYPE_RESET] = mgmtSuperTableActionReset; mgmtSuperTableActionFp[SDB_TYPE_DESTROY] = mgmtSuperTableActionDestroy; } @@ -164,7 +164,7 @@ int32_t mgmtInitSuperTables() { mgmtSuperTableActionInit(); - tsSuperTableSdb = sdbOpenTable(tsMaxTables, tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, + tsSuperTableSdb = sdbOpenTable(TSDB_MAX_SUPER_TABLES, tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, "stables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtSuperTableAction); if (tsSuperTableSdb == NULL) { mError("failed to init stables data"); @@ -201,12 +201,6 @@ void mgmtCleanUpSuperTables() { } int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { - int32_t numOfTables = sdbGetNumOfRows(tsSuperTableSdb); - if (numOfTables >= TSDB_MAX_SUPER_TABLES) { - mError("stable:%s, numOfTables:%d exceed maxTables:%d", pCreate->tableId, numOfTables, TSDB_MAX_SUPER_TABLES); - return TSDB_CODE_TOO_MANY_TABLES; - } - SSuperTableObj *pStable = (SSuperTableObj *)calloc(sizeof(SSuperTableObj), 1); if (pStable == NULL) { return TSDB_CODE_SERV_OUT_OF_MEMORY; @@ -217,7 +211,7 @@ int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { pStable->createdTime = taosGetTimestampMs(); pStable->vgId = 0; pStable->sid = 0; - pStable->uid = (((uint64_t) pStable->createdTime) << 16) + ((uint64_t) sdbGetVersion() & ((1ul << 16) - 1ul)); + pStable->uid = (((uint64_t) pStable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul)); pStable->sversion = 0; pStable->numOfColumns = htons(pCreate->numOfColumns); pStable->numOfTags = htons(pCreate->numOfTags); diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 6001c97a92..34c1a69cb1 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -24,6 +24,7 @@ #include "mgmtMnode.h" #include "mgmtNormalTable.h" #include "mgmtProfile.h" +#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtSuperTable.h" #include "mgmtUser.h" diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 22c6cbc1dc..33caa75988 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -21,6 +21,7 @@ #include "mgmtAcct.h" #include "mgmtGrant.h" #include "mgmtMnode.h" +#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtUser.h" @@ -59,7 +60,7 @@ int32_t mgmtInitUsers() { SUserObj tObj; tsUserUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - tsUserSdb = sdbOpenTable(tsMaxUsers, tsUserUpdateSize, "users", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtUserAction); + tsUserSdb = sdbOpenTable(TSDB_MAX_USERS, tsUserUpdateSize, "users", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtUserAction); if (tsUserSdb == NULL) { mError("failed to init user data"); return -1; @@ -106,12 +107,6 @@ static int32_t mgmtUpdateUser(SUserObj *pUser) { } static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { - int32_t numOfUsers = sdbGetNumOfRows(tsUserSdb); - if (numOfUsers >= tsMaxUsers) { - mWarn("numOfUsers:%d, exceed tsMaxUsers:%d", numOfUsers, tsMaxUsers); - return TSDB_CODE_TOO_MANY_USERS; - } - int32_t code = mgmtCheckUserLimit(pAcct); if (code != 0) { return code; @@ -257,7 +252,6 @@ static void mgmtUserActionInit() { mgmtUserActionFp[SDB_TYPE_UPDATE] = mgmtUserActionUpdate; mgmtUserActionFp[SDB_TYPE_ENCODE] = mgmtUserActionEncode; mgmtUserActionFp[SDB_TYPE_DECODE] = mgmtUserActionDecode; - mgmtUserActionFp[SDB_TYPE_RESET] = mgmtUserActionReset; mgmtUserActionFp[SDB_TYPE_DESTROY] = mgmtUserActionDestroy; } diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index be99148deb..62a2abd17d 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -25,6 +25,7 @@ #include "mgmtDClient.h" #include "mgmtDnode.h" #include "mgmtProfile.h" +#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtTable.h" #include "mgmtVgroup.h" @@ -58,7 +59,6 @@ static void mgmtVgroupActionInit() { mgmtVgroupActionFp[SDB_TYPE_UPDATE] = mgmtVgroupActionUpdate; mgmtVgroupActionFp[SDB_TYPE_ENCODE] = mgmtVgroupActionEncode; mgmtVgroupActionFp[SDB_TYPE_DECODE] = mgmtVgroupActionDecode; - mgmtVgroupActionFp[SDB_TYPE_RESET] = mgmtVgroupActionReset; mgmtVgroupActionFp[SDB_TYPE_DESTROY] = mgmtVgroupActionDestroy; } @@ -75,7 +75,7 @@ int32_t mgmtInitVgroups() { mgmtVgroupActionInit(); - tsVgroupSdb = sdbOpenTable(tsMaxVGroups, tsVgUpdateSize, "vgroups", SDB_KEYTYPE_AUTO, tsMnodeDir, mgmtVgroupAction); + tsVgroupSdb = sdbOpenTable(TSDB_MAX_VGROUPS, tsVgUpdateSize, "vgroups", SDB_KEYTYPE_AUTO, tsMnodeDir, mgmtVgroupAction); if (tsVgroupSdb == NULL) { mError("failed to init vgroups data"); return -1; diff --git a/src/util/inc/tglobalcfg.h b/src/util/inc/tglobalcfg.h index 993992ffcb..664eb16373 100644 --- a/src/util/inc/tglobalcfg.h +++ b/src/util/inc/tglobalcfg.h @@ -103,13 +103,7 @@ extern int tsReplications; extern int tsNumOfMPeers; extern int tsMaxShellConns; -extern int tsMaxAccounts; -extern int tsMaxUsers; -extern int tsMaxDbs; extern int tsMaxTables; -extern int tsMaxDnodes; -extern int tsMaxVGroups; -extern char tsMgmtZone[]; extern char tsLocalIp[]; extern char tsDefaultDB[]; diff --git a/src/util/inc/tlog.h b/src/util/inc/tlog.h index 350a2c700e..3a327b0a47 100644 --- a/src/util/inc/tlog.h +++ b/src/util/inc/tlog.h @@ -239,6 +239,25 @@ extern uint32_t cdebugFlag; #define monitorLWarn(...) taosLogWarn(__VA_ARGS__) monitorWarn(__VA_ARGS__) #define monitorLPrint(...) taosLogPrint(__VA_ARGS__) monitorPrint(__VA_ARGS__) +#define sdbError(...) \ + if (sdbDebugFlag & DEBUG_ERROR) { \ + tprintf("ERROR MND-SDB ", 255, __VA_ARGS__); \ + } +#define sdbWarn(...) \ + if (sdbDebugFlag & DEBUG_WARN) { \ + tprintf("WARN MND-SDB ", sdbDebugFlag, __VA_ARGS__); \ + } +#define sdbTrace(...) \ + if (sdbDebugFlag & DEBUG_TRACE) { \ + tprintf("MND-SDB ", sdbDebugFlag, __VA_ARGS__); \ + } +#define sdbPrint(...) \ + { tprintf("MND-SDB ", 255, __VA_ARGS__); } + +#define sdbLError(...) taosLogError(__VA_ARGS__) sdbError(__VA_ARGS__) +#define sdbLWarn(...) taosLogWarn(__VA_ARGS__) sdbWarn(__VA_ARGS__) +#define sdbLPrint(...) taosLogPrint(__VA_ARGS__) sdbPrint(__VA_ARGS__) + #ifdef __cplusplus } #endif diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index bbea30207c..02de88881d 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -112,13 +112,7 @@ int tsReplications = TSDB_REPLICA_MIN_NUM; int tsNumOfMPeers = 3; int tsMaxShellConns = 2000; -int tsMaxAccounts = 100; -int tsMaxUsers = 1000; -int tsMaxDbs = 1000; -int tsMaxTables = 650000; -int tsMaxDnodes = 1000; -int tsMaxVGroups = 1000; -char tsMgmtZone[16] = "rzone"; +int tsMaxTables = 100000; char tsLocalIp[TSDB_IPv4ADDR_LEN] = {0}; char tsDefaultDB[TSDB_DB_NAME_LEN] = {0}; @@ -612,28 +606,10 @@ static void doInitGlobalConfig() { 1, 8640000, 0, TSDB_CFG_UTYPE_SECOND); // mgmt configs - tsInitConfigOption(cfg++, "mgmtZone", tsMgmtZone, TSDB_CFG_VTYPE_STRING, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, - 0, 0, 16, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxAccounts", &tsMaxAccounts, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, - 1, 1000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxUsers", &tsMaxUsers, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, - 1, 1000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxDbs", &tsMaxDbs, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, - 1, 10000, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "maxTables", &tsMaxTables, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 1, 100000000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxDnodes", &tsMaxDnodes, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, - 1, 1000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxVGroups", &tsMaxVGroups, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, - 1, 1000000, 0, TSDB_CFG_UTYPE_NONE); - + tsInitConfigOption(cfg++, "minSlidingTime", &tsMinSlidingTime, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 10, 1000000, 0, TSDB_CFG_UTYPE_MS); From e9a07987fdd123cde221064372a0cc2fc1d7f8ce Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 20 Mar 2020 15:09:01 +0800 Subject: [PATCH 03/12] [TD-15] refactor sdb --- src/mnode/inc/mgmtMnode.h | 3 + src/mnode/inc/mgmtSdb.h | 34 ++-- src/mnode/src/mgmtChildTable.c | 17 +- src/mnode/src/mgmtDb.c | 4 +- src/mnode/src/mgmtMain.c | 2 +- src/mnode/src/mgmtMnode.c | 12 ++ src/mnode/src/mgmtNormalTable.c | 19 +- src/mnode/src/mgmtSdb.c | 326 +++++++++++++++----------------- src/mnode/src/mgmtShell.c | 2 +- src/mnode/src/mgmtSuperTable.c | 16 +- src/mnode/src/mgmtTable.c | 8 +- src/mnode/src/mgmtUser.c | 6 +- src/mnode/src/mgmtVgroup.c | 14 +- 13 files changed, 224 insertions(+), 239 deletions(-) diff --git a/src/mnode/inc/mgmtMnode.h b/src/mnode/inc/mgmtMnode.h index d381b09055..729d31544f 100644 --- a/src/mnode/inc/mgmtMnode.h +++ b/src/mnode/inc/mgmtMnode.h @@ -23,6 +23,9 @@ extern "C" { int32_t mgmtInitMnodes(); void mgmtCleanupMnodes(); +bool mgmtInServerStatus(); +bool mgmtIsMaster(); + bool mgmtCheckRedirect(void *handle); void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet); void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet); diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h index c64db286c6..7381ec6669 100644 --- a/src/mnode/inc/mgmtSdb.h +++ b/src/mnode/inc/mgmtSdb.h @@ -20,27 +20,11 @@ extern "C" { #endif -#include -#include -#include -#include -#include -#include -#include -#include - -#include "hashint.h" -#include "hashstr.h" -#include "tchecksum.h" -#include "tlog.h" -#include "trpc.h" -#include "tutil.h" - enum _keytype { SDB_KEYTYPE_STRING, SDB_KEYTYPE_AUTO, SDB_KEYTYPE_MAX -}; +} ESdbKeyType; enum _sdbaction { SDB_TYPE_INSERT, @@ -50,11 +34,15 @@ enum _sdbaction { SDB_TYPE_ENCODE, SDB_TYPE_DESTROY, SDB_MAX_ACTION_TYPES -}; +} ESdbType; + +typedef enum { + SDB_OPER_GLOBAL, + SDB_OPER_LOCAL, + SDB_OPER_DISK +} ESdbOper; uint64_t sdbGetVersion(); -bool sdbInServerState(); -bool sdbIsMaster(); void *sdbOpenTable(int32_t maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, void *(*appTool)(char, void *, char *, int32_t, int32_t *)); @@ -65,9 +53,9 @@ void *sdbFetchRow(void *handle, void *pNode, void **ppRow); int64_t sdbGetId(void *handle); int64_t sdbGetNumOfRows(void *handle); -int64_t sdbInsertRow(void *handle, void *row, int32_t rowSize); -int32_t sdbDeleteRow(void *handle, void *key); -int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, char isUpdated); +int32_t sdbInsertRow(void *handle, void *row, ESdbOper oper); +int32_t sdbDeleteRow(void *handle, void *key, ESdbOper oper); +int32_t sdbUpdateRow(void *handle, void *row, int32_t rowSize, ESdbOper oper); #ifdef __cplusplus } diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index 589beeaaba..be78983c91 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -29,6 +29,7 @@ #include "mgmtDb.h" #include "mgmtDClient.h" #include "mgmtGrant.h" +#include "mgmtMnode.h" #include "mgmtProfile.h" #include "mgmtSdb.h" #include "mgmtShell.h" @@ -94,7 +95,7 @@ void *mgmtChildTableActionInsert(void *row, char *str, int32_t size, int32_t *ss return NULL; } - if (!sdbIsMaster()) { + if (!mgmtIsMaster()) { int32_t sid = taosAllocateId(pVgroup->idPool); if (sid != pTable->sid) { mError("ctable:%s, sid:%d is not matched from the master:%d", pTable->tableId, sid, pTable->sid); @@ -214,7 +215,7 @@ int32_t mgmtInitChildTables() { SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); if (pDb == NULL) { mError("ctable:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsChildTableSdb, pTable); + sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_DISK); pNode = pLastNode; continue; } @@ -223,7 +224,7 @@ int32_t mgmtInitChildTables() { if (pVgroup == NULL) { mError("ctable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->tableId, pTable->vgId, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable); + sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_DISK); pNode = pLastNode; continue; } @@ -232,7 +233,7 @@ int32_t mgmtInitChildTables() { mError("ctable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it", pTable->tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable); + sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_DISK); pNode = pLastNode; continue; } @@ -240,7 +241,7 @@ int32_t mgmtInitChildTables() { if (pVgroup->tableList == NULL) { mError("ctable:%s, vgroup:%d tableList is null", pTable->tableId, pTable->vgId); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable); + sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_DISK); pNode = pLastNode; continue; } @@ -252,7 +253,7 @@ int32_t mgmtInitChildTables() { if (pSuperTable == NULL) { mError("ctable:%s, stable:%s not exist", pTable->tableId, pTable->superTableId); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable); + sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_DISK); pNode = pLastNode; continue; } @@ -337,7 +338,7 @@ void* mgmtCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t pTable->vgId = pVgroup->vgId; pTable->superTable = pSuperTable; - if (sdbInsertRow(tsChildTableSdb, pTable, 0) < 0) { + if (sdbInsertRow(tsChildTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { free(pTable); mError("ctable:%s, update sdb error", pCreate->tableId); terrno = TSDB_CODE_SDB_ERROR; @@ -481,7 +482,7 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) { } if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsChildTableSdb, pTable); + sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_LOCAL); pNode = pLastNode; numOfTables ++; continue; diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 5cd1259486..e889d200f5 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -279,7 +279,7 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { pDb->createdTime = taosGetTimestampMs(); pDb->cfg = *pCreate; - if (sdbInsertRow(tsDbSdb, pDb, 0) < 0) { + if (sdbInsertRow(tsDbSdb, pDb, SDB_OPER_GLOBAL) < 0) { code = TSDB_CODE_SDB_ERROR; tfree(pDb); } @@ -819,7 +819,7 @@ static void mgmtDropDb(void *handle, void *tmrId) { SDbObj *pDb = newMsg->ahandle; mPrint("db:%s, drop db from sdb", pDb->name); - int32_t code = sdbDeleteRow(tsDbSdb, pDb); + int32_t code = sdbDeleteRow(tsDbSdb, pDb, SDB_OPER_GLOBAL); if (code != 0) { code = TSDB_CODE_SDB_ERROR; } diff --git a/src/mnode/src/mgmtMain.c b/src/mnode/src/mgmtMain.c index 6aec06970e..ab0b99fa58 100644 --- a/src/mnode/src/mgmtMain.c +++ b/src/mnode/src/mgmtMain.c @@ -127,7 +127,7 @@ int32_t mgmtStartSystem() { void mgmtStopSystem() { - if (sdbIsMaster()) { + if (mgmtIsMaster()) { mTrace("it is a master mgmt node, it could not be stopped"); return; } diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index ac18af0158..8e459bf5af 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -29,6 +29,8 @@ int32_t (*mpeerInitMnodesFp)() = NULL; void (*mpeerCleanUpMnodesFp)() = NULL; static SMnodeObj tsMnodeObj = {0}; +static bool tsMnodeIsMaster = false; +static bool tsMnodeIsServing = false; static int32_t mgmtGetMnodeMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); @@ -52,6 +54,8 @@ int32_t mgmtInitMnodes() { if (mpeerInitMnodesFp) { return (*mpeerInitMnodesFp)(); } else { + tsMnodeIsServing = true; + tsMnodeIsMaster = true; return 0; } } @@ -62,6 +66,14 @@ void mgmtCleanupMnodes() { } } +bool mgmtInServerStatus() { + return tsMnodeIsServing; +} + +bool mgmtIsMaster() { + return tsMnodeIsMaster; +} + bool mgmtCheckRedirect(void *handle) { return false; } diff --git a/src/mnode/src/mgmtNormalTable.c b/src/mnode/src/mgmtNormalTable.c index d42a596008..df9fe0f439 100644 --- a/src/mnode/src/mgmtNormalTable.c +++ b/src/mnode/src/mgmtNormalTable.c @@ -26,6 +26,7 @@ #include "mgmtDb.h" #include "mgmtDClient.h" #include "mgmtGrant.h" +#include "mgmtMnode.h" #include "mgmtNormalTable.h" #include "mgmtSdb.h" #include "mgmtSuperTable.h" @@ -98,7 +99,7 @@ void *mgmtNormalTableActionInsert(void *row, char *str, int32_t size, int32_t *s return NULL; } - if (!sdbIsMaster()) { + if (!mgmtIsMaster()) { int32_t sid = taosAllocateId(pVgroup->idPool); if (sid != pTable->sid) { mError("sid:%d is not matched from the master:%d", sid, pTable->sid); @@ -237,7 +238,7 @@ int32_t mgmtInitNormalTables() { SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); if (pDb == NULL) { mError("ntable:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsNormalTableSdb, pTable); + sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_DISK); pNode = pLastNode; continue; } @@ -246,7 +247,7 @@ int32_t mgmtInitNormalTables() { if (pVgroup == NULL) { mError("ntable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->tableId, pTable->vgId, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable); + sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_DISK); pNode = pLastNode; continue; } @@ -255,7 +256,7 @@ int32_t mgmtInitNormalTables() { mError("ntable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it", pTable->tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable); + sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_DISK); pNode = pLastNode; continue; } @@ -263,7 +264,7 @@ int32_t mgmtInitNormalTables() { if (pVgroup->tableList == NULL) { mError("ntable:%s, vgroup:%d tableList is null", pTable->tableId, pTable->vgId); pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable); + sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_DISK); pNode = pLastNode; continue; } @@ -370,7 +371,7 @@ void *mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t mTrace("table:%s, stream sql len:%d sql:%s", pTable->tableId, pTable->sqlLen, pTable->sql); } - if (sdbInsertRow(tsNormalTableSdb, pTable, 0) < 0) { + if (sdbInsertRow(tsNormalTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { mError("table:%s, update sdb error", pTable->tableId); free(pTable); terrno = TSDB_CODE_SDB_ERROR; @@ -467,7 +468,7 @@ int32_t mgmtAddNormalTableColumn(SNormalTableObj *pTable, SSchema schema[], int3 pTable->sversion++; pAcct->acctInfo.numOfTimeSeries += ncols; - sdbUpdateRow(tsNormalTableSdb, pTable, 0, 1); + sdbUpdateRow(tsNormalTableSdb, pTable, tsNormalTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -496,7 +497,7 @@ int32_t mgmtDropNormalTableColumnByName(SNormalTableObj *pTable, char *colName) pTable->sversion++; pAcct->acctInfo.numOfTimeSeries--; - sdbUpdateRow(tsNormalTableSdb, pTable, 0, 1); + sdbUpdateRow(tsNormalTableSdb, pTable, tsNormalTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -555,7 +556,7 @@ void mgmtDropAllNormalTables(SDbObj *pDropDb) { if (pTable == NULL) break; if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsNormalTableSdb, pTable); + sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_LOCAL); pNode = pLastNode; numOfTables ++; continue; diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 3564392ed1..33bfe4622b 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -17,15 +17,18 @@ #include "os.h" #include "taosdef.h" #include "tutil.h" +#include "tchecksum.h" +#include "tlog.h" +#include "trpc.h" +#include "tutil.h" +#include "hashint.h" +#include "hashstr.h" #include "mgmtSdb.h" #define abs(x) (((x) < 0) ? -(x) : (x)) #define SDB_MAX_PEERS 4 #define SDB_DELIMITER 0xFFF00F00 #define SDB_ENDCOMMIT 0xAFFFAAAF -#define SDB_STATUS_OFFLINE 0 -#define SDB_STATUS_SERVING 1 - typedef struct { uint64_t swVersion; @@ -81,15 +84,6 @@ int32_t (*mpeerForwardRequestFp)(SSdbTable *pTable, char type, void *cont, int32 static SSdbTable *sdbTableList[10] = {0}; static int32_t sdbNumOfTables = 0; static uint64_t sdbVersion = 0; -static int32_t sdbMaster = 0; -static int32_t sdbStatus = SDB_STATUS_OFFLINE; - - -// #ifdef CLUSTER -// int32_t sdbMaster = 0; -// #else -// int32_t sdbMaster = 1; -// #endif static void *(*sdbInitIndexFp[])(int32_t maxRows, int32_t dataSize) = {sdbOpenStrHash, sdbOpenIntHash}; static void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash}; @@ -102,8 +96,6 @@ void sdbResetTable(SSdbTable *pTable); void sdbSaveSnapShot(void *handle); uint64_t sdbGetVersion() { return sdbVersion; } -bool sdbInServerState() { return sdbStatus == SDB_STATUS_SERVING; } -bool sdbIsMaster() { return sdbMaster; } int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->id; } int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } @@ -319,6 +311,10 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { pTable->size += real_size; if (pTable->id < abs(rowHead->id)) pTable->id = abs(rowHead->id); + + //TODO: check this valid + pTable->size += 4; + lseek(pTable->fd, 4, SEEK_CUR); } if (pTable->keyType == SDB_KEYTYPE_AUTO) { @@ -390,81 +386,67 @@ void *sdbGetRow(void *handle, void *key) { } // row here must be encoded string (rowSize > 0) or the object it self (rowSize = 0) -int64_t sdbInsertRow(void *handle, void *row, int32_t rowSize) { +int32_t sdbInsertRow(void *handle, void *row, ESdbOper oper) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta rowMeta; - int64_t id = -1; void * pObj = NULL; - int32_t total_size = 0; - int32_t real_size = 0; - /* char action = SDB_TYPE_INSERT; */ + int32_t total_size = 0; + int32_t real_size = 0; if (pTable == NULL) { sdbError("sdb tables is null"); return -1; } - - if ((pTable->keyType != SDB_KEYTYPE_AUTO) || *((int64_t *)row)) - if (sdbGetRow(handle, row)) { - if (strcmp(pTable->name, "mnode") == 0) { - /* - * The first mnode created when the system just start, so the insert action may failed - * see sdbPeer.c : sdbInitPeers - */ - pTable->id++; - sdbVersion++; - sdbPrint("table:%s, record:%s already exist, think it successed, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id); - return 0; - } else { - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbError("table:%s, failed to insert record:%s sdbVersion:%" PRId64 " id:%" PRId64 , pTable->name, (char *)row, sdbVersion, pTable->id); - break; - case SDB_KEYTYPE_AUTO: - sdbError("table:%s, failed to insert record:%d sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, *(int32_t *)row, sdbVersion, pTable->id); - break; - default: - sdbError("table:%s, failed to insert record sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, sdbVersion, pTable->id); - break; - } - return -1; - } + + if (sdbGetRow(handle, row)) { + switch (pTable->keyType) { + case SDB_KEYTYPE_STRING: + sdbError("table:%s, failed to insert record:%s sdbVersion:%" PRId64 " id:%" PRId64 , pTable->name, (char *)row, sdbVersion, pTable->id); + break; + case SDB_KEYTYPE_AUTO: + sdbError("table:%s, failed to insert record:%d sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, *(int32_t *)row, sdbVersion, pTable->id); + break; + default: + sdbError("table:%s, failed to insert record sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, sdbVersion, pTable->id); + break; } + return -1; + } total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); SRowHead *rowHead = (SRowHead *)malloc(total_size); if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb: %s", pTable->name); + sdbError("table:%s, failed to allocate row head memory", pTable->name); return -1; } memset(rowHead, 0, total_size); - if (rowSize == 0) { // object is created already + if (oper == SDB_OPER_GLOBAL) { pObj = row; - } else { // encoded string, to create object - pObj = (*(pTable->appTool))(SDB_TYPE_DECODE, NULL, row, rowSize, NULL); - } - (*(pTable->appTool))(SDB_TYPE_ENCODE, pObj, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); - + } else { + pObj = (*(pTable->appTool))(SDB_TYPE_DECODE, NULL, row, 0, NULL); + } + pthread_mutex_lock(&pTable->mutex); - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_INSERT, rowHead->data, rowHead->rowSize) == 0) { - pTable->id++; - sdbVersion++; - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - // TODO:here need to change - *((uint32_t *)pObj) = ++pTable->autoIndex; - (*(pTable->appTool))(SDB_TYPE_ENCODE, pObj, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - } + if (oper == SDB_OPER_GLOBAL) { + if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_INSERT, rowHead->data, rowHead->rowSize) != 0) { + sdbError("table:%s, failed to insert record", pTable->name); + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } + } + + if (oper == SDB_OPER_GLOBAL || oper == SDB_OPER_LOCAL) { + (*(pTable->appTool))(SDB_TYPE_ENCODE, pObj, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); + assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pTable->id; + rowHead->id = pTable->id + 1; if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum while inserting, sdb:%s", pTable->name); + sdbError("table:%s, failed to get checksum while inserting", pTable->name); pthread_mutex_unlock(&pTable->mutex); tfree(rowHead); return -1; @@ -477,14 +459,10 @@ int64_t sdbInsertRow(void *handle, void *row, int32_t rowSize) { rowMeta.row = pObj; (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pObj, &rowMeta); - /* Update the disk content */ - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ twrite(pTable->fd, rowHead, real_size); pTable->size += real_size; sdbFinishCommit(pTable); - pTable->numOfRows++; switch (pTable->keyType) { case SDB_KEYTYPE_STRING: sdbTrace("table:%s, a record is inserted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, @@ -499,32 +477,33 @@ int64_t sdbInsertRow(void *handle, void *row, int32_t rowSize) { pTable->name, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); break; } + } - id = rowMeta.id; - } else { - sdbError("table:%s, failed to insert record", pTable->name); + if (pTable->keyType == SDB_KEYTYPE_AUTO) { + *((uint32_t *)pObj) = ++pTable->autoIndex; } - tfree(rowHead); + pTable->numOfRows++; + pTable->id++; + sdbVersion++; pthread_mutex_unlock(&pTable->mutex); - /* callback function to update the MGMT layer */ - if (id >= 0 && pTable->appTool) (*pTable->appTool)(SDB_TYPE_INSERT, pObj, NULL, 0, NULL); + (*pTable->appTool)(SDB_TYPE_INSERT, pObj, NULL, 0, NULL); - return id; + tfree(rowHead); + + return 0; } // row here can be object or null-terminated string -int32_t sdbDeleteRow(void *handle, void *row) { +int32_t sdbDeleteRow(void *handle, void *row, ESdbOper oper) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta = NULL; - int32_t code = -1; void * pMetaRow = NULL; SRowHead * rowHead = NULL; - int32_t rowSize = 0; - int32_t total_size = 0; - /* char action = SDB_TYPE_DELETE; */ + int32_t rowSize = 0; + int32_t total_size = 0; if (pTable == NULL) return -1; @@ -558,67 +537,67 @@ int32_t sdbDeleteRow(void *handle, void *row) { pthread_mutex_lock(&pTable->mutex); - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_DELETE, (char *)row, rowSize) == 0) { - pTable->id++; - sdbVersion++; - - rowHead->delimiter = SDB_DELIMITER; - rowHead->rowSize = rowSize; - rowHead->id = -(pTable->id); - memcpy(rowHead->data, row, rowSize); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, total_size) < 0) { - sdbError("failed to get checksum while inserting, sdb:%s", pTable->name); + if (oper == SDB_OPER_GLOBAL) { + if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_DELETE, (char *)row, rowSize) == 0) { + sdbError("table:%s, failed to delete record", pTable->name); pthread_mutex_unlock(&pTable->mutex); tfree(rowHead); return -1; - } - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - twrite(pTable->fd, rowHead, total_size); - pTable->size += total_size; - sdbFinishCommit(pTable); - - pTable->numOfRows--; - - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is deleted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is deleted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - default: - sdbTrace("table:%s, a record is deleted, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, sdbVersion, pTable->id, pTable->numOfRows); - break; - } - - // Delete from current layer - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, row); - - code = 0; + } } + rowHead->delimiter = SDB_DELIMITER; + rowHead->rowSize = rowSize; + rowHead->id = -(pTable->id); + memcpy(rowHead->data, row, rowSize); + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, total_size) < 0) { + sdbError("failed to get checksum while inserting, sdb:%s", pTable->name); + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } + + twrite(pTable->fd, rowHead, total_size); + pTable->size += total_size; + sdbFinishCommit(pTable); + + switch (pTable->keyType) { + case SDB_KEYTYPE_STRING: + sdbTrace("table:%s, a record is deleted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", + pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); + break; + case SDB_KEYTYPE_AUTO: + sdbTrace("table:%s, a record is deleted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", + pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); + break; + default: + sdbTrace("table:%s, a record is deleted, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", + pTable->name, sdbVersion, pTable->id, pTable->numOfRows); + break; + } + + // Delete from current layer + (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, row); + + pTable->numOfRows--; + pTable->id++; + sdbVersion++; + pthread_mutex_unlock(&pTable->mutex); tfree(rowHead); - // callback function of the delete - if (code == 0 && pTable->appTool) (*pTable->appTool)(SDB_TYPE_DELETE, pMetaRow, NULL, 0, NULL); + (*pTable->appTool)(SDB_TYPE_DELETE, pMetaRow, NULL, 0, NULL); - return code; + return 0; } // row here can be the object or the string info (encoded string) -int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, char isUpdated) { +int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, ESdbOper oper) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta = NULL; - int32_t code = -1; - int32_t total_size = 0; - int32_t real_size = 0; - /* char action = SDB_TYPE_UPDATE; */ + int32_t total_size = 0; + int32_t real_size = 0; if (pTable == NULL || row == NULL) return -1; pMeta = sdbGetRowMeta(handle, row); @@ -651,8 +630,15 @@ int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, char isUpdated } memset(rowHead, 0, total_size); - if (!isUpdated) { - (*(pTable->appTool))(SDB_TYPE_UPDATE, pMetaRow, row, updateSize, NULL); // update in upper layer + pthread_mutex_lock(&pTable->mutex); + + if (oper == SDB_OPER_GLOBAL) { + if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_UPDATE, rowHead->data, rowHead->rowSize) == 0) { + sdbError("table:%s, failed to update record", pTable->name); + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } } if (pMetaRow != row) { @@ -663,57 +649,51 @@ int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, char isUpdated } real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - ; - pthread_mutex_lock(&pTable->mutex); - - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_UPDATE, rowHead->data, rowHead->rowSize) == 0) { - pTable->id++; - sdbVersion++; - - // write to the new position - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pTable->id; - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum, sdb:%s id:%d", pTable->name, rowHead->id); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - twrite(pTable->fd, rowHead, real_size); - - pMeta->id = pTable->id; - pMeta->offset = pTable->size; - pMeta->rowSize = rowHead->rowSize; - pTable->size += real_size; - - sdbFinishCommit(pTable); - - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is updated:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is updated:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - default: - sdbTrace("table:%s, a record is updated, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, pTable->name, sdbVersion, - pTable->id, pTable->numOfRows); - break; - } - - code = 0; + // write to the new position + rowHead->delimiter = SDB_DELIMITER; + rowHead->id = pTable->id; + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { + sdbError("failed to get checksum, sdb:%s id:%d", pTable->name, rowHead->id); + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; } + + twrite(pTable->fd, rowHead, real_size); + + pMeta->id = pTable->id; + pMeta->offset = pTable->size; + pMeta->rowSize = rowHead->rowSize; + pTable->size += real_size; + + sdbFinishCommit(pTable); + + switch (pTable->keyType) { + case SDB_KEYTYPE_STRING: + sdbTrace("table:%s, a record is updated:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, + pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); + break; + case SDB_KEYTYPE_AUTO: + sdbTrace("table:%s, a record is updated:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, + pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); + break; + default: + sdbTrace("table:%s, a record is updated, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, pTable->name, sdbVersion, + pTable->id, pTable->numOfRows); + break; + } + + pTable->id++; + sdbVersion++; pthread_mutex_unlock(&pTable->mutex); + (*(pTable->appTool))(SDB_TYPE_UPDATE, pMetaRow, row, updateSize, NULL); // update in upper layer + tfree(rowHead); - return code; + return 0; } void sdbCloseTable(void *handle) { diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index c50ecc01c7..501faaab97 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -132,7 +132,7 @@ void mgmtAddToShellQueue(SQueuedMsg *queuedMsg) { } static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { - if (!sdbInServerState()) { + if (!mgmtInServerStatus()) { mgmtProcessMsgWhileNotReady(rpcMsg); rpcFreeCont(rpcMsg->pCont); return; diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index 075633cd4d..9120854e47 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -181,7 +181,7 @@ int32_t mgmtInitSuperTables() { SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); if (pDb == NULL) { mError("super table:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsSuperTableSdb, pTable); + sdbDeleteRow(tsSuperTableSdb, pTable, SDB_OPER_DISK); pNode = pLastNode; continue; } @@ -233,7 +233,7 @@ int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { tschema[col].bytes = htons(tschema[col].bytes); } - if (sdbInsertRow(tsSuperTableSdb, pStable, 0) < 0) { + if (sdbInsertRow(tsSuperTableSdb, pStable, SDB_OPER_GLOBAL) < 0) { mError("stable:%s, update sdb error", pStable->tableId); return TSDB_CODE_SDB_ERROR; } @@ -319,7 +319,7 @@ int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t pStable->sversion++; pAcct->acctInfo.numOfTimeSeries += (ntags * pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1); + sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); mTrace("Succeed to add tag column %s to table %s", schema[0].name, pStable->tableId); return TSDB_CODE_SUCCESS; @@ -352,7 +352,7 @@ int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns); pStable->schema = realloc(pStable->schema, schemaSize); - sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1); + sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -384,7 +384,7 @@ int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char *oldTagN mgmtSuperTableActionEncode(pStable, msg, size, &rowSize); - int32_t ret = sdbUpdateRow(tsSuperTableSdb, msg, rowSize, 1); + int32_t ret = sdbUpdateRow(tsSuperTableSdb, msg, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); tfree(msg); if (ret < 0) { @@ -446,7 +446,7 @@ int32_t mgmtAddSuperTableColumn(SSuperTableObj *pStable, SSchema schema[], int32 pStable->sversion++; pAcct->acctInfo.numOfTimeSeries += (ncols * pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1); + sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -479,7 +479,7 @@ int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pStable, char *colName) { pStable->schema = realloc(pStable->schema, schemaSize); pAcct->acctInfo.numOfTimeSeries -= (pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1); + sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -618,7 +618,7 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) { } if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsSuperTableSdb, pTable); + sdbDeleteRow(tsSuperTableSdb, pTable, SDB_OPER_GLOBAL); pNode = pLastNode; numOfTables ++; continue; diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 34c1a69cb1..a907cd85f5 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -757,9 +757,9 @@ static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg) { if (rpcMsg->code != TSDB_CODE_SUCCESS) { if (pTable->type == TSDB_CHILD_TABLE) { - sdbDeleteRow(tsChildTableSdb, pTable); + sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_GLOBAL); } else if (pTable->type == TSDB_NORMAL_TABLE){ - sdbDeleteRow(tsNormalTableSdb, pTable); + sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_GLOBAL); } else {} mError("table:%s, failed to create in dnode, reason:%s", pTable->tableId, tstrerror(rpcMsg->code)); mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); @@ -813,14 +813,14 @@ static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg) { } if (pTable->type == TSDB_CHILD_TABLE) { - if (sdbDeleteRow(tsChildTableSdb, pTable) < 0) { + if (sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { mError("table:%s, update ctables sdb error", pTable->tableId); mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); free(queueMsg); return; } } else if (pTable->type == TSDB_NORMAL_TABLE){ - if (sdbDeleteRow(tsNormalTableSdb, pTable) < 0) { + if (sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { mError("table:%s, update ntables sdb error", pTable->tableId); mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); free(queueMsg); diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 33caa75988..174976f4c0 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -103,7 +103,7 @@ SUserObj *mgmtGetUser(char *name) { } static int32_t mgmtUpdateUser(SUserObj *pUser) { - return sdbUpdateRow(tsUserSdb, pUser, 0, 1); + return sdbUpdateRow(tsUserSdb, pUser, tsUserUpdateSize, SDB_OPER_GLOBAL); } static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { @@ -140,7 +140,7 @@ static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { } code = TSDB_CODE_SUCCESS; - if (sdbInsertRow(tsUserSdb, pUser, 0) < 0) { + if (sdbInsertRow(tsUserSdb, pUser, SDB_OPER_GLOBAL) < 0) { tfree(pUser); code = TSDB_CODE_SDB_ERROR; } @@ -161,7 +161,7 @@ static int32_t mgmtDropUser(SAcctObj *pAcct, char *name) { return TSDB_CODE_NO_RIGHTS; } - sdbDeleteRow(tsUserSdb, pUser); + sdbDeleteRow(tsUserSdb, pUser, SDB_OPER_GLOBAL); return 0; } diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index 62a2abd17d..d4df83c25c 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -112,7 +112,7 @@ int32_t mgmtInitVgroups() { if (tsIsCluster && pVgroup->vnodeGid[0].publicIp == 0) { pVgroup->vnodeGid[0].publicIp = inet_addr(tsPublicIp); pVgroup->vnodeGid[0].privateIp = inet_addr(tsPrivateIp); - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, 1); + sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, SDB_OPER_GLOBAL); } // mgmtSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes, pVgroup->vgId); @@ -161,7 +161,7 @@ void mgmtCreateVgroup(SQueuedMsg *pMsg) { mgmtAddVgroupIntoDb(pDb, pVgroup); // mgmtSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes, pVgroup->vgId); - sdbInsertRow(tsVgroupSdb, pVgroup, 0); + sdbInsertRow(tsVgroupSdb, pVgroup, SDB_OPER_GLOBAL); mPrint("vgroup:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { @@ -179,7 +179,7 @@ void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle) { } else { mTrace("vgroup:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes); mgmtSendDropVgroupMsg(pVgroup, NULL); - sdbDeleteRow(tsVgroupSdb, pVgroup); + sdbDeleteRow(tsVgroupSdb, pVgroup, SDB_OPER_GLOBAL); } } @@ -474,7 +474,7 @@ static void *mgmtVgroupActionDestroy(void *row, char *str, int32_t size, int32_t } void mgmtUpdateVgroup(SVgObj *pVgroup) { - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, 0); + sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, SDB_OPER_LOCAL); } void mgmtAddTableIntoVgroup(SVgObj *pVgroup, STableInfo *pTable) { @@ -607,7 +607,7 @@ static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen); mgmtAddToShellQueue(newMsg); } else { - sdbDeleteRow(tsVgroupSdb, pVgroup); + sdbDeleteRow(tsVgroupSdb, pVgroup, SDB_OPER_GLOBAL); mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); } @@ -661,7 +661,7 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) { if (queueMsg->received != queueMsg->expected) return; - sdbDeleteRow(tsVgroupSdb, pVgroup); + sdbDeleteRow(tsVgroupSdb, pVgroup, SDB_OPER_GLOBAL); SQueuedMsg *newMsg = calloc(1, sizeof(SQueuedMsg)); newMsg->msgType = queueMsg->msgType; @@ -691,7 +691,7 @@ void mgmtUpdateVgroupIp(SDnodeObj *pDnode) { pDnode->publicIp, taosIpStr(vnodeGid->publicIp)); vnodeGid->publicIp = pDnode->publicIp; vnodeGid->privateIp = pDnode->privateIp; - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, 1); + sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, SDB_OPER_GLOBAL); } } } From eb6764be24d27d7a3a93f6268cdfcd137c4bb4e6 Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 20 Mar 2020 18:13:03 +0800 Subject: [PATCH 04/12] [TD-15] refactor sdb --- src/mnode/inc/mgmtSdb.h | 45 ++++---- src/mnode/src/mgmtChildTable.c | 110 +++++++----------- src/mnode/src/mgmtDb.c | 147 +++++++++++------------- src/mnode/src/mgmtNormalTable.c | 143 ++++++++++-------------- src/mnode/src/mgmtSdb.c | 114 ++++++++++--------- src/mnode/src/mgmtSuperTable.c | 128 +++++++++------------ src/mnode/src/mgmtUser.c | 151 +++++++++++-------------- src/mnode/src/mgmtVgroup.c | 191 ++++++++++++++------------------ 8 files changed, 446 insertions(+), 583 deletions(-) diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h index 7381ec6669..bf5ccb2634 100644 --- a/src/mnode/inc/mgmtSdb.h +++ b/src/mnode/inc/mgmtSdb.h @@ -20,42 +20,49 @@ extern "C" { #endif -enum _keytype { +typedef enum { SDB_KEYTYPE_STRING, SDB_KEYTYPE_AUTO, SDB_KEYTYPE_MAX } ESdbKeyType; -enum _sdbaction { - SDB_TYPE_INSERT, - SDB_TYPE_DELETE, - SDB_TYPE_UPDATE, - SDB_TYPE_DECODE, - SDB_TYPE_ENCODE, - SDB_TYPE_DESTROY, - SDB_MAX_ACTION_TYPES -} ESdbType; - typedef enum { SDB_OPER_GLOBAL, SDB_OPER_LOCAL, SDB_OPER_DISK -} ESdbOper; +} ESdbOperType; -uint64_t sdbGetVersion(); +enum _sdbaction { + SDB_TYPE_INSERT, + SDB_TYPE_DELETE, + SDB_TYPE_UPDATE, +} ESdbForwardType; -void *sdbOpenTable(int32_t maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, - void *(*appTool)(char, void *, char *, int32_t, int32_t *)); +typedef struct { + char *tableName; + int32_t hashSessions; + int32_t maxRowSize; + ESdbKeyType keyType; + int32_t (*insertFp)(void *pObj); + int32_t (*deleteFp)(void *pObj); + int32_t (*updateFp)(void *pObj); + int32_t (*encodeFp)(void *pObj, void *pData, int32_t maxRowSize); + void * (*decodeFp)(void *pData); + int32_t (*destroyFp)(void *pObj); +} SSdbTableDesc; + +void *sdbOpenTable(SSdbTableDesc *desc); void sdbCloseTable(void *handle); +int32_t sdbInsertRow(void *handle, void *row, ESdbOperType oper); +int32_t sdbDeleteRow(void *handle, void *key, ESdbOperType oper); +int32_t sdbUpdateRow(void *handle, void *row, int32_t rowSize, ESdbOperType oper); + void *sdbGetRow(void *handle, void *key); void *sdbFetchRow(void *handle, void *pNode, void **ppRow); int64_t sdbGetId(void *handle); int64_t sdbGetNumOfRows(void *handle); - -int32_t sdbInsertRow(void *handle, void *row, ESdbOper oper); -int32_t sdbDeleteRow(void *handle, void *key, ESdbOper oper); -int32_t sdbUpdateRow(void *handle, void *row, int32_t rowSize, ESdbOper oper); +uint64_t sdbGetVersion(); #ifdef __cplusplus } diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index be78983c91..5b27acf1c4 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -39,67 +39,43 @@ void *tsChildTableSdb; int32_t tsChildTableUpdateSize; -void *(*mgmtChildTableActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); - -void *mgmtChildTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionReset(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); static void mgmtDestroyChildTable(SChildTableObj *pTable) { - free(pTable); + tfree(pTable); } -static void mgmtChildTableActionInit() { - mgmtChildTableActionFp[SDB_TYPE_INSERT] = mgmtChildTableActionInsert; - mgmtChildTableActionFp[SDB_TYPE_DELETE] = mgmtChildTableActionDelete; - mgmtChildTableActionFp[SDB_TYPE_UPDATE] = mgmtChildTableActionUpdate; - mgmtChildTableActionFp[SDB_TYPE_ENCODE] = mgmtChildTableActionEncode; - mgmtChildTableActionFp[SDB_TYPE_DECODE] = mgmtChildTableActionDecode; - mgmtChildTableActionFp[SDB_TYPE_DESTROY] = mgmtChildTableActionDestroy; -} - -void *mgmtChildTableActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *) row; - memcpy(pTable, str, tsChildTableUpdateSize); - return NULL; -} - -void *mgmtChildTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *)row; +static int32_t mgmtChildTableActionDestroy(void *pObj) { + SChildTableObj *pTable = (SChildTableObj *)pObj; mgmtDestroyChildTable(pTable); - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *) row; +static int32_t mgmtChildTableActionInsert(void *pObj) { + SChildTableObj *pTable = (SChildTableObj *) pObj; SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { mError("ctable:%s, not in vgroup:%d", pTable->tableId, pTable->vgId); - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { mError("ctable:%s, vgroup:%d not in db:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); - return NULL; + return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { mError("ctable:%s, account:%s not exists", pTable->tableId, pDb->cfg.acct); - return NULL; + return TSDB_CODE_INVALID_ACCT; } if (!mgmtIsMaster()) { int32_t sid = taosAllocateId(pVgroup->idPool); if (sid != pTable->sid) { mError("ctable:%s, sid:%d is not matched from the master:%d", pTable->tableId, sid, pTable->sid); - return NULL; + return TSDB_CODE_INVALID_SESSION_ID; } } @@ -114,30 +90,30 @@ void *mgmtChildTableActionInsert(void *row, char *str, int32_t size, int32_t *ss mgmtMoveVgroupToTail(pDb, pVgroup); } - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *) row; +static int32_t mgmtChildTableActionDelete(void *pObj) { + SChildTableObj *pTable = (SChildTableObj *) pObj; if (pTable->vgId == 0) { - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { mError("ctable:%s, vgroup:%d not in DB:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); - return NULL; + return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { mError("ctable:%s, account:%s not exists", pTable->tableId, pDb->cfg.acct); - return NULL; + return TSDB_CODE_INVALID_ACCT; } mgmtRestoreTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); @@ -150,56 +126,54 @@ void *mgmtChildTableActionDelete(void *row, char *str, int32_t size, int32_t *ss mgmtMoveVgroupToHead(pDb, pVgroup); } - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtChildTableActionReset(row, str, size, NULL); +static int32_t mgmtChildTableActionUpdate(void *pObj) { + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *) row; - assert(row != NULL && str != NULL); +static int32_t mgmtChildTableActionEncode(void *pObj, void *pData, int32_t maxRowSize) { + SChildTableObj *pTable = (SChildTableObj *) pObj; + assert(pObj != NULL && pData != NULL); - memcpy(str, pTable, tsChildTableUpdateSize); - *ssize = tsChildTableUpdateSize; - - return NULL; + memcpy(pData, pTable, tsChildTableUpdateSize); + return tsChildTableUpdateSize; } -void *mgmtChildTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - assert(str != NULL); +static void *mgmtChildTableActionDecode(void *pData) { + assert(pData != NULL); SChildTableObj *pTable = (SChildTableObj *)calloc(sizeof(SChildTableObj), 1); if (pTable == NULL) return NULL; - if (size < tsChildTableUpdateSize) { - mgmtDestroyChildTable(pTable); - return NULL; - } - memcpy(pTable, str, tsChildTableUpdateSize); + memcpy(pTable, pData, tsChildTableUpdateSize); return (void *)pTable; } -void *mgmtChildTableAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtChildTableActionFp[(uint8_t)action] != NULL) { - return (*(mgmtChildTableActionFp[(uint8_t)action]))(row, str, size, ssize); - } - return NULL; -} - int32_t mgmtInitChildTables() { void *pNode = NULL; void *pLastNode = NULL; SChildTableObj *pTable = NULL; - mgmtChildTableActionInit(); SChildTableObj tObj; tsChildTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - tsChildTableSdb = sdbOpenTable(tsMaxTables, tsChildTableUpdateSize, - "ctables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtChildTableAction); + SSdbTableDesc tableDesc = { + .tableName = "ctables", + .hashSessions = tsMaxTables, + .maxRowSize = tsChildTableUpdateSize, + .keyType = SDB_KEYTYPE_STRING, + .insertFp = mgmtChildTableActionInsert, + .deleteFp = mgmtChildTableActionDelete, + .updateFp = mgmtChildTableActionUpdate, + .encodeFp = mgmtChildTableActionEncode, + .decodeFp = mgmtChildTableActionDecode, + .destroyFp = mgmtChildTableActionDestroy, + }; + + tsChildTableSdb = sdbOpenTable(&tableDesc); if (tsChildTableSdb == NULL) { mError("failed to init child table data"); return -1; diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index e889d200f5..2549dd4c64 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -48,29 +48,57 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg); static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg); static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg); -static void *(*mgmtDbActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionReset(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); - -static void mgmtDbActionInit() { - mgmtDbActionFp[SDB_TYPE_INSERT] = mgmtDbActionInsert; - mgmtDbActionFp[SDB_TYPE_DELETE] = mgmtDbActionDelete; - mgmtDbActionFp[SDB_TYPE_UPDATE] = mgmtDbActionUpdate; - mgmtDbActionFp[SDB_TYPE_ENCODE] = mgmtDbActionEncode; - mgmtDbActionFp[SDB_TYPE_DECODE] = mgmtDbActionDecode; - mgmtDbActionFp[SDB_TYPE_DESTROY] = mgmtDbActionDestroy; +static int32_t mgmtDbActionDestroy(void *pObj) { + tfree(pObj); + return TSDB_CODE_SUCCESS; } -static void *mgmtDbAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtDbActionFp[(uint8_t)action] != NULL) { - return (*(mgmtDbActionFp[(uint8_t)action]))(row, str, size, ssize); +static int32_t mgmtDbActionInsert(void *pObj) { + SDbObj *pDb = (SDbObj *) pObj; + SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + + pDb->pHead = NULL; + pDb->pTail = NULL; + pDb->numOfVgroups = 0; + pDb->numOfTables = 0; + mgmtAddDbIntoAcct(pAcct, pDb); + + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtDbActionDelete(void *pObj) { + SDbObj *pDb = (SDbObj *) pObj; + SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + mgmtRemoveDbFromAcct(pAcct, pDb); + + mgmtDropAllNormalTables(pDb); + mgmtDropAllChildTables(pDb); + mgmtDropAllSuperTables(pDb); + + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtDbActionUpdate(void *pObj) { + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtDbActionEncode(void *pObj, void *pData, int32_t maxRowSize) { + SDbObj *pDb = (SDbObj *)pObj; + if (maxRowSize < tsDbUpdateSize) { + return -1; + } else { + memcpy(pData, pDb, tsDbUpdateSize); + return tsDbUpdateSize; } - return NULL; +} + +static void *mgmtDbActionDecode(void *pData) { + SDbObj *pDb = (SDbObj *) malloc(sizeof(SDbObj)); + if (pDb == NULL) return NULL; + memset(pDb, 0, sizeof(SDbObj)); + memcpy(pDb, pData, tsDbUpdateSize); + + return (void *)pDb; } int32_t mgmtInitDbs() { @@ -78,12 +106,23 @@ int32_t mgmtInitDbs() { SDbObj * pDb = NULL; SAcctObj *pAcct = NULL; - mgmtDbActionInit(); - SDbObj tObj; tsDbUpdateSize = tObj.updateEnd - (char *)&tObj; - tsDbSdb = sdbOpenTable(TSDB_MAX_DBS, tsDbUpdateSize, "dbs", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtDbAction); + SSdbTableDesc tableDesc = { + .tableName = "dbs", + .hashSessions = TSDB_MAX_DBS, + .maxRowSize = tsDbUpdateSize, + .keyType = SDB_KEYTYPE_STRING, + .insertFp = mgmtDbActionInsert, + .deleteFp = mgmtDbActionDelete, + .updateFp = mgmtDbActionUpdate, + .encodeFp = mgmtDbActionEncode, + .decodeFp = mgmtDbActionDecode, + .destroyFp = mgmtDbActionDestroy, + }; + + tsDbSdb = sdbOpenTable(&tableDesc); if (tsDbSdb == NULL) { mError("failed to init db data"); return -1; @@ -683,68 +722,6 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * return numOfRows; } -void *mgmtDbActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) row; - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - - pDb->pHead = NULL; - pDb->pTail = NULL; - pDb->numOfVgroups = 0; - pDb->numOfTables = 0; - mgmtAddDbIntoAcct(pAcct, pDb); - - return NULL; -} - -void *mgmtDbActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) row; - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - mgmtRemoveDbFromAcct(pAcct, pDb); - - mgmtDropAllNormalTables(pDb); - mgmtDropAllChildTables(pDb); - mgmtDropAllSuperTables(pDb); - - return NULL; -} - -void *mgmtDbActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtDbActionReset(row, str, size, ssize); -} - -void *mgmtDbActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) row; - if (size < tsDbUpdateSize) { - *ssize = -1; - } else { - memcpy(str, pDb, tsDbUpdateSize); - *ssize = tsDbUpdateSize; - } - - return NULL; -} -void *mgmtDbActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) malloc(sizeof(SDbObj)); - if (pDb == NULL) return NULL; - memset(pDb, 0, sizeof(SDbObj)); - - memcpy(pDb, str, tsDbUpdateSize); - - return (void *)pDb; -} - -void *mgmtDbActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) row; - memcpy(pDb, str, tsDbUpdateSize); - - return NULL; -} - -void *mgmtDbActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - tfree(row); - return NULL; -} - void mgmtAddSuperTableIntoDb(SDbObj *pDb) { atomic_add_fetch_32(&pDb->numOfSuperTables, 1); } diff --git a/src/mnode/src/mgmtNormalTable.c b/src/mnode/src/mgmtNormalTable.c index df9fe0f439..58578faf9a 100644 --- a/src/mnode/src/mgmtNormalTable.c +++ b/src/mnode/src/mgmtNormalTable.c @@ -35,75 +35,45 @@ void *tsNormalTableSdb; int32_t tsNormalTableUpdateSize; -void *(*mgmtNormalTableActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); - -void *mgmtNormalTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionReset(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); static void mgmtDestroyNormalTable(SNormalTableObj *pTable) { - free(pTable->schema); - free(pTable->sql); - free(pTable); + tfree(pTable->schema); + tfree(pTable->sql); + tfree(pTable); } -static void mgmtNormalTableActionInit() { - mgmtNormalTableActionFp[SDB_TYPE_INSERT] = mgmtNormalTableActionInsert; - mgmtNormalTableActionFp[SDB_TYPE_DELETE] = mgmtNormalTableActionDelete; - mgmtNormalTableActionFp[SDB_TYPE_UPDATE] = mgmtNormalTableActionUpdate; - mgmtNormalTableActionFp[SDB_TYPE_ENCODE] = mgmtNormalTableActionEncode; - mgmtNormalTableActionFp[SDB_TYPE_DECODE] = mgmtNormalTableActionDecode; - mgmtNormalTableActionFp[SDB_TYPE_DESTROY] = mgmtNormalTableActionDestroy; -} - -void *mgmtNormalTableActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *) row; - memcpy(pTable, str, tsNormalTableUpdateSize); - - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns) + pTable->sqlLen; - pTable->schema = realloc(pTable->schema, schemaSize); - pTable->sql = (char*)pTable->schema + sizeof(SSchema) * (pTable->numOfColumns); - memcpy(pTable->schema, str + tsNormalTableUpdateSize, schemaSize); - - return NULL; -} - -void *mgmtNormalTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *)row; +static int32_t mgmtNormalTableActionDestroy(void *pObj) { + SNormalTableObj *pTable = (SNormalTableObj *)pObj; mgmtDestroyNormalTable(pTable); - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtNormalTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *) row; +static int32_t mgmtNormalTableActionInsert(void *pObj) { + SNormalTableObj *pTable = (SNormalTableObj *) pObj; SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { mError("id:%s not in vgroup:%d", pTable->tableId, pTable->vgId); - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { mError("vgroup:%d not in DB:%s", pVgroup->vgId, pVgroup->dbName); - return NULL; + return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { mError("account not exists"); - return NULL; + return TSDB_CODE_INVALID_ACCT; } if (!mgmtIsMaster()) { int32_t sid = taosAllocateId(pVgroup->idPool); if (sid != pTable->sid) { mError("sid:%d is not matched from the master:%d", sid, pTable->sid); - return NULL; + return TSDB_CODE_INVALID_SESSION_ID; } } @@ -115,30 +85,30 @@ void *mgmtNormalTableActionInsert(void *row, char *str, int32_t size, int32_t *s mgmtMoveVgroupToTail(pDb, pVgroup); } - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtNormalTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *) row; +static int32_t mgmtNormalTableActionDelete(void *pObj) { + SNormalTableObj *pTable = (SNormalTableObj *) pObj; if (pTable->vgId == 0) { - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { mError("vgroup:%d not in DB:%s", pVgroup->vgId, pVgroup->dbName); - return NULL; + return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { mError("account not exists"); - return NULL; + return TSDB_CODE_INVALID_ACCT; } mgmtRestoreTimeSeries(pAcct, pTable->numOfColumns - 1); @@ -149,45 +119,46 @@ void *mgmtNormalTableActionDelete(void *row, char *str, int32_t size, int32_t *s mgmtMoveVgroupToHead(pDb, pVgroup); } - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtNormalTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtNormalTableActionReset(row, str, size, NULL); +static int32_t mgmtNormalTableActionUpdate(void *pObj) { + // SNormalTableObj *pTable = (SNormalTableObj *) pObj; + // memcpy(pTable, str, tsNormalTableUpdateSize); + + // int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns) + pTable->sqlLen; + // pTable->schema = realloc(pTable->schema, schemaSize); + // pTable->sql = (char*)pTable->schema + sizeof(SSchema) * (pTable->numOfColumns); + // memcpy(pTable->schema, str + tsNormalTableUpdateSize, schemaSize); + + return TSDB_CODE_SUCCESS; } -void *mgmtNormalTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *) row; - assert(row != NULL && str != NULL); +static int32_t mgmtNormalTableActionEncode(void *pObj, void *pData, int32_t maxRowSize) { + SNormalTableObj *pTable = (SNormalTableObj *) pObj; + assert(pObj != NULL && pData != NULL); int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - if (size < tsNormalTableUpdateSize + schemaSize + 1) { - *ssize = -1; - return NULL; + if (maxRowSize < tsNormalTableUpdateSize + schemaSize + 1) { + return -1; } - memcpy(str, pTable, tsNormalTableUpdateSize); - memcpy(str + tsNormalTableUpdateSize, pTable->schema, schemaSize); - memcpy(str + tsNormalTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen); - *ssize = tsNormalTableUpdateSize + schemaSize + pTable->sqlLen; - - return NULL; + memcpy(pData, pTable, tsNormalTableUpdateSize); + memcpy(pData + tsNormalTableUpdateSize, pTable->schema, schemaSize); + memcpy(pData + tsNormalTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen); + + return tsNormalTableUpdateSize + schemaSize + pTable->sqlLen; } -void *mgmtNormalTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - assert(str != NULL); +static void *mgmtNormalTableActionDecode(void *pData) { + assert(pData != NULL); SNormalTableObj *pTable = (SNormalTableObj *)malloc(sizeof(SNormalTableObj)); if (pTable == NULL) { return NULL; } memset(pTable, 0, sizeof(SNormalTableObj)); - - if (size < tsNormalTableUpdateSize) { - mgmtDestroyNormalTable(pTable); - return NULL; - } - memcpy(pTable, str, tsNormalTableUpdateSize); + memcpy(pTable, pData, tsNormalTableUpdateSize); int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); pTable->schema = (SSchema *)malloc(schemaSize); @@ -196,35 +167,39 @@ void *mgmtNormalTableActionDecode(void *row, char *str, int32_t size, int32_t *s return NULL; } - memcpy(pTable->schema, str + tsNormalTableUpdateSize, schemaSize); + memcpy(pTable->schema, pData + tsNormalTableUpdateSize, schemaSize); pTable->sql = (char *)malloc(pTable->sqlLen); if (pTable->sql == NULL) { mgmtDestroyNormalTable(pTable); return NULL; } - memcpy(pTable->sql, str + tsNormalTableUpdateSize + schemaSize, pTable->sqlLen); + memcpy(pTable->sql, pData + tsNormalTableUpdateSize + schemaSize, pTable->sqlLen); return (void *)pTable; } -void *mgmtNormalTableAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtNormalTableActionFp[(uint8_t)action] != NULL) { - return (*(mgmtNormalTableActionFp[(uint8_t)action]))(row, str, size, ssize); - } - return NULL; -} - int32_t mgmtInitNormalTables() { void *pNode = NULL; void *pLastNode = NULL; SNormalTableObj *pTable = NULL; - mgmtNormalTableActionInit(); SNormalTableObj tObj; tsNormalTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - tsNormalTableSdb = sdbOpenTable(TSDB_MAX_NORMAL_TABLES, sizeof(SNormalTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, - "ntables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtNormalTableAction); + SSdbTableDesc tableDesc = { + .tableName = "ntables", + .hashSessions = TSDB_MAX_NORMAL_TABLES, + .maxRowSize = sizeof(SNormalTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, + .keyType = SDB_KEYTYPE_STRING, + .insertFp = mgmtNormalTableActionInsert, + .deleteFp = mgmtNormalTableActionDelete, + .updateFp = mgmtNormalTableActionUpdate, + .encodeFp = mgmtNormalTableActionEncode, + .decodeFp = mgmtNormalTableActionDecode, + .destroyFp = mgmtNormalTableActionDestroy, + }; + + tsNormalTableSdb = sdbOpenTable(&tableDesc); if (tsNormalTableSdb == NULL) { mError("failed to init ntables data"); return -1; diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 33bfe4622b..8703c63541 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -16,8 +16,8 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taosdef.h" -#include "tutil.h" #include "tchecksum.h" +#include "tglobalcfg.h" #include "tlog.h" #include "trpc.h" #include "tutil.h" @@ -33,32 +33,37 @@ typedef struct { uint64_t swVersion; int16_t sdbFileVersion; - char reserved[6]; + char reserved[2]; TSCKSUM checkSum; } SSdbHeader; typedef struct _SSdbTable { - SSdbHeader header; - int maxRows; - int dbId; - int32_t maxRowSize; - char name[TSDB_DB_NAME_LEN]; - char fn[128]; - int keyType; - uint32_t autoIndex; - int64_t numOfRows; - int64_t id; - int64_t size; - void * iHandle; - int fd; - void *(*appTool)(char, void *, char *, int, int *); + SSdbHeader header; + char name[TSDB_DB_NAME_LEN]; + char fn[TSDB_FILENAME_LEN]; + ESdbKeyType keyType; + int32_t dbId; + int32_t hashSessions; + int32_t maxRowSize; + uint32_t autoIndex; + int64_t numOfRows; + int64_t id; + int64_t size; + void * iHandle; + int32_t fd; + int32_t (*insertFp)(void *pObj); + int32_t (*deleteFp)(void *pObj); + int32_t (*updateFp)(void *pObj); + void * (*decodeFp)(void *pData); // return pObj + int32_t (*encodeFp)(void *pObj, void *pData, int32_t maxRowSize); // return size of pData + int32_t (*destroyFp)(void *pObj); pthread_mutex_t mutex; } SSdbTable; typedef struct { int64_t id; int64_t offset; - int rowSize; + int32_t rowSize; void * row; } SRowMeta; @@ -71,9 +76,9 @@ typedef struct { typedef struct { uint8_t dbId; - char type; + int8_t type; + int16_t dataLen; uint64_t version; - short dataLen; char data[]; } SForwardMsg; @@ -283,7 +288,7 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { // TODO: Get rid of the rowMeta.offset and rowSize rowMeta.offset = pTable->size; rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = (*(pTable->appTool))(SDB_TYPE_DECODE, NULL, rowHead->data, rowHead->rowSize, NULL); + rowMeta.row = (*pTable->decodeFp)(rowHead->data); (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); if (pTable->keyType == SDB_KEYTYPE_AUTO) { pTable->autoIndex++; @@ -299,7 +304,7 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { if (rowHead->id < 0) { // Delete the object (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); - (*(pTable->appTool))(SDB_TYPE_DESTROY, pMetaRow, NULL, 0, NULL); + (*pTable->destroyFp)(pMetaRow); pTable->numOfRows--; numOfDels++; } else { // Reset the object TODO: is it possible to merge reset and @@ -322,7 +327,7 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { } sdbVersion += (pTable->id - oldId); - if (numOfDels > pTable->maxRows / 4) sdbSaveSnapShot(pTable); + if (numOfDels > pTable->hashSessions / 4) sdbSaveSnapShot(pTable); tfree(rowHead); return 0; @@ -332,20 +337,25 @@ sdb_exit1: return -1; } -void *sdbOpenTable(int32_t maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, - void *(*appTool)(char, void *, char *, int32_t, int32_t *)) { - SSdbTable *pTable = (SSdbTable *)malloc(sizeof(SSdbTable)); +void *sdbOpenTable(SSdbTableDesc *pDesc) { + SSdbTable *pTable = (SSdbTable *)calloc(1, sizeof(SSdbTable)); if (pTable == NULL) return NULL; - memset(pTable, 0, sizeof(SSdbTable)); - strcpy(pTable->name, name); - pTable->keyType = keyType; - pTable->maxRows = maxRows; - pTable->maxRowSize = maxRowSize; - pTable->appTool = appTool; - sprintf(pTable->fn, "%s/%s.db", directory, pTable->name); + pTable->keyType = pDesc->keyType; + pTable->hashSessions = pDesc->hashSessions; + pTable->maxRowSize = pDesc->maxRowSize; + pTable->insertFp = pDesc->insertFp; + pTable->deleteFp = pDesc->deleteFp; + pTable->updateFp = pDesc->updateFp; + pTable->encodeFp = pDesc->encodeFp; + pTable->decodeFp = pDesc->decodeFp; + pTable->destroyFp = pDesc->destroyFp; + strcpy(pTable->name, pDesc->tableName); + sprintf(pTable->fn, "%s/%s.db", tsMnodeDir, pTable->name); - if (sdbInitIndexFp[keyType] != NULL) pTable->iHandle = (*sdbInitIndexFp[keyType])(maxRows, sizeof(SRowMeta)); + if (sdbInitIndexFp[pTable->keyType] != NULL) { + pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SRowMeta)); + } pthread_mutex_init(&pTable->mutex, NULL); @@ -386,7 +396,7 @@ void *sdbGetRow(void *handle, void *key) { } // row here must be encoded string (rowSize > 0) or the object it self (rowSize = 0) -int32_t sdbInsertRow(void *handle, void *row, ESdbOper oper) { +int32_t sdbInsertRow(void *handle, void *row, ESdbOperType oper) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta rowMeta; void * pObj = NULL; @@ -424,7 +434,7 @@ int32_t sdbInsertRow(void *handle, void *row, ESdbOper oper) { if (oper == SDB_OPER_GLOBAL) { pObj = row; } else { - pObj = (*(pTable->appTool))(SDB_TYPE_DECODE, NULL, row, 0, NULL); + pObj = (*pTable->decodeFp)(row); } pthread_mutex_lock(&pTable->mutex); @@ -439,7 +449,7 @@ int32_t sdbInsertRow(void *handle, void *row, ESdbOper oper) { } if (oper == SDB_OPER_GLOBAL || oper == SDB_OPER_LOCAL) { - (*(pTable->appTool))(SDB_TYPE_ENCODE, pObj, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); + rowHead->rowSize = (*pTable->encodeFp)(pObj, rowHead->data, pTable->maxRowSize); assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); @@ -489,7 +499,7 @@ int32_t sdbInsertRow(void *handle, void *row, ESdbOper oper) { pthread_mutex_unlock(&pTable->mutex); - (*pTable->appTool)(SDB_TYPE_INSERT, pObj, NULL, 0, NULL); + (*pTable->insertFp)(pObj); tfree(rowHead); @@ -497,7 +507,7 @@ int32_t sdbInsertRow(void *handle, void *row, ESdbOper oper) { } // row here can be object or null-terminated string -int32_t sdbDeleteRow(void *handle, void *row, ESdbOper oper) { +int32_t sdbDeleteRow(void *handle, void *row, ESdbOperType oper) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta = NULL; void * pMetaRow = NULL; @@ -587,13 +597,13 @@ int32_t sdbDeleteRow(void *handle, void *row, ESdbOper oper) { tfree(rowHead); - (*pTable->appTool)(SDB_TYPE_DELETE, pMetaRow, NULL, 0, NULL); + (*pTable->deleteFp)(pMetaRow); return 0; } // row here can be the object or the string info (encoded string) -int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, ESdbOper oper) { +int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, ESdbOperType oper) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta = NULL; int32_t total_size = 0; @@ -645,7 +655,7 @@ int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, ESdbOper oper) memcpy(rowHead->data, row, updateSize); rowHead->rowSize = updateSize; } else { - (*(pTable->appTool))(SDB_TYPE_ENCODE, pMetaRow, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); + rowHead->rowSize = (*pTable->encodeFp)(pMetaRow, rowHead->data, pTable->maxRowSize); } real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); @@ -689,7 +699,7 @@ int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, ESdbOper oper) pthread_mutex_unlock(&pTable->mutex); - (*(pTable->appTool))(SDB_TYPE_UPDATE, pMetaRow, row, updateSize, NULL); // update in upper layer + (*pTable->updateFp)(pMetaRow); // update in upper layer tfree(rowHead); @@ -706,7 +716,7 @@ void sdbCloseTable(void *handle) { while (1) { pNode = sdbFetchRow(handle, pNode, &row); if (row == NULL) break; - (*(pTable->appTool))(SDB_TYPE_DESTROY, row, NULL, 0, NULL); + (*pTable->destroyFp)(row); } if (sdbCleanUpIndexFp[pTable->keyType]) (*sdbCleanUpIndexFp[pTable->keyType])(pTable->iHandle); @@ -724,13 +734,13 @@ void sdbCloseTable(void *handle) { void sdbResetTable(SSdbTable *pTable) { /* SRowHead rowHead; */ SRowMeta rowMeta; - int32_t bytes; - int32_t total_size = 0; - int32_t real_size = 0; + int32_t bytes; + int32_t total_size = 0; + int32_t real_size = 0; SRowHead *rowHead = NULL; void * pMetaRow = NULL; int64_t oldId = pTable->id; - int32_t oldNumOfRows = pTable->numOfRows; + int32_t oldNumOfRows = pTable->numOfRows; if (sdbOpenSdbFile(pTable) < 0) return; pTable->numOfRows = oldNumOfRows; @@ -792,19 +802,19 @@ void sdbResetTable(SSdbTable *pTable) { // TODO:Get rid of the rowMeta.offset and rowSize rowMeta.offset = pTable->size; rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = (*(pTable->appTool))(SDB_TYPE_DECODE, NULL, rowHead->data, rowHead->rowSize, NULL); + rowMeta.row = (*pTable->decodeFp)(rowHead->data); (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); pTable->numOfRows++; - (*pTable->appTool)(SDB_TYPE_INSERT, rowMeta.row, NULL, 0, NULL); + (*pTable->insertFp)(rowMeta.row); } } else { // already exists if (rowHead->id < 0) { // Delete the object (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); - (*(pTable->appTool))(SDB_TYPE_DESTROY, pMetaRow, NULL, 0, NULL); + (*pTable->destroyFp)(pMetaRow); pTable->numOfRows--; } else { // update the object - (*(pTable->appTool))(SDB_TYPE_UPDATE, pMetaRow, rowHead->data, rowHead->rowSize, NULL); + (*pTable->updateFp)(pMetaRow); } } } @@ -866,7 +876,7 @@ void sdbSaveSnapShot(void *handle) { rowHead->delimiter = SDB_DELIMITER; rowHead->id = pMeta->id; - (*(pTable->appTool))(SDB_TYPE_ENCODE, pMeta->row, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); + rowHead->rowSize = (*pTable->encodeFp)(pMeta->row, rowHead->data, pTable->maxRowSize); real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { sdbError("failed to get checksum while save sdb %s snapshot", pTable->name); diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index 9120854e47..de64f9f54e 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -39,105 +39,73 @@ static void *tsSuperTableSdb; static int32_t tsSuperTableUpdateSize; -static void *(*mgmtSuperTableActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionReset(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); static int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mgmtGetShowSuperTableMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn); static void mgmtDestroySuperTable(SSuperTableObj *pTable) { - free(pTable->schema); - free(pTable); + tfree(pTable->schema); + tfree(pTable); } -static void mgmtSuperTableActionInit() { - SSuperTableObj tObj; - tsSuperTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - - mgmtSuperTableActionFp[SDB_TYPE_INSERT] = mgmtSuperTableActionInsert; - mgmtSuperTableActionFp[SDB_TYPE_DELETE] = mgmtSuperTableActionDelete; - mgmtSuperTableActionFp[SDB_TYPE_UPDATE] = mgmtSuperTableActionUpdate; - mgmtSuperTableActionFp[SDB_TYPE_ENCODE] = mgmtSuperTableActionEncode; - mgmtSuperTableActionFp[SDB_TYPE_DECODE] = mgmtSuperTableActionDecode; - mgmtSuperTableActionFp[SDB_TYPE_DESTROY] = mgmtSuperTableActionDestroy; -} - -void *mgmtSuperTableActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SSuperTableObj *pTable = (SSuperTableObj *) row; - memcpy(pTable, str, tsSuperTableUpdateSize); - - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); - pTable->schema = realloc(pTable->schema, schemaSize); - memcpy(pTable->schema, str + tsSuperTableUpdateSize, schemaSize); - - return NULL; -} - -void *mgmtSuperTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - SSuperTableObj *pTable = (SSuperTableObj *) row; +static int32_t mgmtSuperTableActionDestroy(void *pObj) { + SSuperTableObj *pTable = (SSuperTableObj *) pObj; mgmtDestroySuperTable(pTable); - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - STableInfo *pTable = (STableInfo *) row; +static int32_t mgmtSuperTableActionInsert(void *pObj) { + STableInfo *pTable = (STableInfo *) pObj; SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); if (pDb) { mgmtAddSuperTableIntoDb(pDb); } - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - STableInfo *pTable = (STableInfo *) row; +static int32_t mgmtSuperTableActionDelete(void *pObj) { + STableInfo *pTable = (STableInfo *) pObj; SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); if (pDb) { mgmtRemoveSuperTableFromDb(pDb); } - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtSuperTableActionReset(row, str, size, NULL); +static int32_t mgmtSuperTableActionUpdate(void *pObj) { + SSuperTableObj *pTable = (SSuperTableObj *) pObj; + memcpy(pTable, pObj, tsSuperTableUpdateSize); + + int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); + pTable->schema = realloc(pTable->schema, schemaSize); + memcpy(pTable->schema, pObj + tsSuperTableUpdateSize, schemaSize); + + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SSuperTableObj *pTable = (SSuperTableObj *) row; - assert(row != NULL && str != NULL); +static int32_t mgmtSuperTableActionEncode(void *pObj, void *pData, int32_t maxRowSize) { + SSuperTableObj *pTable = (SSuperTableObj *) pObj; + assert(pObj != NULL && pData != NULL); int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); - if (size < tsSuperTableUpdateSize + schemaSize + 1) { - *ssize = -1; - return NULL; + if (maxRowSize < tsSuperTableUpdateSize + schemaSize + 1) { + return TSDB_CODE_INVALID_MSG_LEN; } - memcpy(str, pTable, tsSuperTableUpdateSize); - memcpy(str + tsSuperTableUpdateSize, pTable->schema, schemaSize); - *ssize = tsSuperTableUpdateSize + schemaSize; - - return NULL; + memcpy(pData, pTable, tsSuperTableUpdateSize); + memcpy(pData + tsSuperTableUpdateSize, pTable->schema, schemaSize); + return tsSuperTableUpdateSize + schemaSize; } -void *mgmtSuperTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - assert(str != NULL); +static void *mgmtSuperTableActionDecode(void *pData) { + assert(pData != NULL); SSuperTableObj *pTable = (SSuperTableObj *) malloc(sizeof(SSuperTableObj)); if (pTable == NULL) { return NULL; } memset(pTable, 0, sizeof(SSuperTableObj)); - - if (size < tsSuperTableUpdateSize) { - mgmtDestroySuperTable(pTable); - return NULL; - } - memcpy(pTable, str, tsSuperTableUpdateSize); + memcpy(pTable, pData, tsSuperTableUpdateSize); int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); pTable->schema = malloc(schemaSize); @@ -146,26 +114,32 @@ void *mgmtSuperTableActionDecode(void *row, char *str, int32_t size, int32_t *ss return NULL; } - memcpy(pTable->schema, str + tsSuperTableUpdateSize, schemaSize); + memcpy(pTable->schema, pData + tsSuperTableUpdateSize, schemaSize); return (void *) pTable; } -void *mgmtSuperTableAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtSuperTableActionFp[(uint8_t) action] != NULL) { - return (*(mgmtSuperTableActionFp[(uint8_t) action]))(row, str, size, ssize); - } - return NULL; -} - int32_t mgmtInitSuperTables() { void *pNode = NULL; void *pLastNode = NULL; SSuperTableObj *pTable = NULL; - mgmtSuperTableActionInit(); + SSuperTableObj tObj; + tsSuperTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - tsSuperTableSdb = sdbOpenTable(TSDB_MAX_SUPER_TABLES, tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, - "stables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtSuperTableAction); + SSdbTableDesc tableDesc = { + .tableName = "stables", + .hashSessions = TSDB_MAX_SUPER_TABLES, + .maxRowSize = tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, + .keyType = SDB_KEYTYPE_STRING, + .insertFp = mgmtSuperTableActionInsert, + .deleteFp = mgmtSuperTableActionDelete, + .updateFp = mgmtSuperTableActionUpdate, + .encodeFp = mgmtSuperTableActionEncode, + .decodeFp = mgmtSuperTableActionDecode, + .destroyFp = mgmtSuperTableActionDestroy, + }; + + tsSuperTableSdb = sdbOpenTable(&tableDesc); if (tsSuperTableSdb == NULL) { mError("failed to init stables data"); return -1; @@ -365,8 +339,8 @@ int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char *oldTagN return TSDB_CODE_INVALID_MSG_TYPE; } - int32_t rowSize = 0; - uint32_t len = strlen(newTagName); + int32_t rowSize = 0; + uint32_t len = strlen(newTagName); if (col >= pStable->numOfTags || len >= TSDB_COL_NAME_LEN || mgmtFindSuperTableTagIndex(pStable, newTagName) >= 0) { return TSDB_CODE_APP_ERROR; @@ -382,7 +356,7 @@ int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char *oldTagN if (msg == NULL) return TSDB_CODE_APP_ERROR; memset(msg, 0, size); - mgmtSuperTableActionEncode(pStable, msg, size, &rowSize); + // mgmtSuperTableActionEncode(pStable, msg, size, &rowSize); int32_t ret = sdbUpdateRow(tsSuperTableSdb, msg, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); tfree(msg); diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 174976f4c0..2ca37a4572 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -18,6 +18,7 @@ #include "trpc.h" #include "tschemautil.h" #include "ttime.h" +#include "tutil.h" #include "mgmtAcct.h" #include "mgmtGrant.h" #include "mgmtMnode.h" @@ -38,16 +39,53 @@ static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg); static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg); static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg); -static void *(*mgmtUserActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionReset(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserAction(char action, void *row, char *str, int32_t size, int32_t *ssize); -static void mgmtUserActionInit(); +static int32_t mgmtUserActionDestroy(void *pObj) { + tfree(pObj); + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtUserActionInsert(void *pObj) { + SUserObj *pUser = (SUserObj *) pObj; + SAcctObj *pAcct = mgmtGetAcct(pUser->acct); + + pUser->pAcct = pAcct; + mgmtAddUserIntoAcct(pAcct, pUser); + + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtUserActionDelete(void *pObj) { + SUserObj *pUser = (SUserObj *) pObj; + SAcctObj *pAcct = mgmtGetAcct(pUser->acct); + + mgmtRemoveUserFromAcct(pAcct, pUser); + + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtUserActionUpdate(void *pObj) { + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtUserActionEncode(void *pObj, void *pData, int32_t maxRowSize) { + SUserObj *pUser = (SUserObj *) pObj; + + if (maxRowSize < tsUserUpdateSize) { + return -1; + } else { + memcpy(pData, pUser, tsUserUpdateSize); + return tsUserUpdateSize; + } +} + +static void *mgmtUserActionDecode(void *pData) { + SUserObj *pUser = (SUserObj *) malloc(sizeof(SUserObj)); + if (pUser == NULL) return NULL; + memset(pUser, 0, sizeof(SUserObj)); + memcpy(pUser, pData, tsUserUpdateSize); + + return pUser; +} int32_t mgmtInitUsers() { void *pNode = NULL; @@ -55,12 +93,23 @@ int32_t mgmtInitUsers() { SAcctObj *pAcct = NULL; int32_t numOfUsers = 0; - mgmtUserActionInit(); - SUserObj tObj; tsUserUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - tsUserSdb = sdbOpenTable(TSDB_MAX_USERS, tsUserUpdateSize, "users", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtUserAction); + SSdbTableDesc tableDesc = { + .tableName = "users", + .hashSessions = TSDB_MAX_USERS, + .maxRowSize = tsUserUpdateSize, + .keyType = SDB_KEYTYPE_STRING, + .insertFp = mgmtUserActionInsert, + .deleteFp = mgmtUserActionDelete, + .updateFp = mgmtUserActionUpdate, + .encodeFp = mgmtUserActionEncode, + .decodeFp = mgmtUserActionDecode, + .destroyFp = mgmtUserActionDestroy, + }; + + tsUserSdb = sdbOpenTable(&tableDesc); if (tsUserSdb == NULL) { mError("failed to init user data"); return -1; @@ -246,82 +295,6 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void return numOfRows; } -static void mgmtUserActionInit() { - mgmtUserActionFp[SDB_TYPE_INSERT] = mgmtUserActionInsert; - mgmtUserActionFp[SDB_TYPE_DELETE] = mgmtUserActionDelete; - mgmtUserActionFp[SDB_TYPE_UPDATE] = mgmtUserActionUpdate; - mgmtUserActionFp[SDB_TYPE_ENCODE] = mgmtUserActionEncode; - mgmtUserActionFp[SDB_TYPE_DECODE] = mgmtUserActionDecode; - mgmtUserActionFp[SDB_TYPE_DESTROY] = mgmtUserActionDestroy; -} - -static void *mgmtUserAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtUserActionFp[(uint8_t) action] != NULL) { - return (*(mgmtUserActionFp[(uint8_t) action]))(row, str, size, ssize); - } - return NULL; -} - -static void *mgmtUserActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *) row; - SAcctObj *pAcct = mgmtGetAcct(pUser->acct); - - pUser->pAcct = pAcct; - mgmtAddUserIntoAcct(pAcct, pUser); - - return NULL; -} - -static void *mgmtUserActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *) row; - SAcctObj *pAcct = mgmtGetAcct(pUser->acct); - - mgmtRemoveUserFromAcct(pAcct, pUser); - - return NULL; -} - -static void *mgmtUserActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtUserActionReset(row, str, size, ssize); -} - -static void *mgmtUserActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *) row; - - if (size < tsUserUpdateSize) { - *ssize = -1; - } else { - memcpy(str, pUser, tsUserUpdateSize); - *ssize = tsUserUpdateSize; - } - - return NULL; -} - -static void *mgmtUserActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *) malloc(sizeof(SUserObj)); - if (pUser == NULL) return NULL; - memset(pUser, 0, sizeof(SUserObj)); - - memcpy(pUser, str, tsUserUpdateSize); - - return (void *)pUser; -} - -static void *mgmtUserActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *)row; - - memcpy(pUser, str, tsUserUpdateSize); - - return NULL; -} - -static void *mgmtUserActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - tfree(row); - - return NULL; -} - SUserObj *mgmtGetUserFromConn(void *pConn) { SRpcConnInfo connInfo; if (rpcGetConnInfo(pConn, &connInfo) == 0) { diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index d4df83c25c..12ac6fef57 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -33,15 +33,6 @@ static void *tsVgroupSdb = NULL; static int32_t tsVgUpdateSize = 0; -static void *(*mgmtVgroupActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionReset(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); - static int32_t mgmtGetVgroupMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg); @@ -50,32 +41,98 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg); static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle); static void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle); -static void mgmtVgroupActionInit() { - SVgObj tObj; - tsVgUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - - mgmtVgroupActionFp[SDB_TYPE_INSERT] = mgmtVgroupActionInsert; - mgmtVgroupActionFp[SDB_TYPE_DELETE] = mgmtVgroupActionDelete; - mgmtVgroupActionFp[SDB_TYPE_UPDATE] = mgmtVgroupActionUpdate; - mgmtVgroupActionFp[SDB_TYPE_ENCODE] = mgmtVgroupActionEncode; - mgmtVgroupActionFp[SDB_TYPE_DECODE] = mgmtVgroupActionDecode; - mgmtVgroupActionFp[SDB_TYPE_DESTROY] = mgmtVgroupActionDestroy; +static int32_t mgmtVgroupActionDestroy(void *pObj) { + SVgObj *pVgroup = (SVgObj *) pObj; + if (pVgroup->idPool) { + taosIdPoolCleanUp(pVgroup->idPool); + pVgroup->idPool = NULL; + } + if (pVgroup->tableList) tfree(pVgroup->tableList); + tfree(pObj); + return TSDB_CODE_SUCCESS; } -static void *mgmtVgroupAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtVgroupActionFp[(uint8_t) action] != NULL) { - return (*(mgmtVgroupActionFp[(uint8_t) action]))(row, str, size, ssize); +static int32_t mgmtVgroupActionInsert(void *pObj) { + SVgObj *pVgroup = pObj; + for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { + pVgroup->vnodeGid[i].vnode = pVgroup->vgId; } - return NULL; + + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtVgroupActionDelete(void *pObj) { + SVgObj *pVgroup = pObj; + SDbObj *pDb = mgmtGetDb(pVgroup->dbName); + + if (pDb != NULL) { + mgmtRemoveVgroupFromDb(pDb, pVgroup); + } + + // mgmtUnSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes); + tfree(pVgroup->tableList); + + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtVgroupActionUpdate(void *pObj) { + SVgObj *pVgroup = (SVgObj *) pObj; + int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool); + + SDbObj *pDb = mgmtGetDb(pVgroup->dbName); + if (pDb != NULL) { + if (pDb->cfg.maxSessions != oldTables) { + mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxSessions); + taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxSessions); + int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions; + pVgroup->tableList = (STableInfo **)realloc(pVgroup->tableList, size); + } + } + + mTrace("vgroup:%d update, numOfVnode:%d", pVgroup->vgId, pVgroup->numOfVnodes); + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtVgroupActionEncode(void *pObj, void *pData, int32_t maxRowSize) { + SVgObj *pVgroup = (SVgObj *) pObj; + if (maxRowSize < tsVgUpdateSize) { + return -1; + } else { + memcpy(pData, pVgroup, tsVgUpdateSize); + return tsVgUpdateSize; + } +} + +static void *mgmtVgroupActionDecode(void *pObj) { + SVgObj *pVgroup = (SVgObj *) malloc(sizeof(SVgObj)); + if (pVgroup == NULL) return NULL; + memset(pVgroup, 0, sizeof(SVgObj)); + memcpy(pVgroup, pObj, tsVgUpdateSize); + + return pVgroup; } int32_t mgmtInitVgroups() { void *pNode = NULL; SVgObj *pVgroup = NULL; - mgmtVgroupActionInit(); + SVgObj tObj; + tsVgUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - tsVgroupSdb = sdbOpenTable(TSDB_MAX_VGROUPS, tsVgUpdateSize, "vgroups", SDB_KEYTYPE_AUTO, tsMnodeDir, mgmtVgroupAction); + SSdbTableDesc tableDesc = { + .tableName = "vgroups", + .hashSessions = TSDB_MAX_VGROUPS, + .maxRowSize = tsVgUpdateSize, + .keyType = SDB_KEYTYPE_AUTO, + .insertFp = mgmtVgroupActionInsert, + .deleteFp = mgmtVgroupActionDelete, + .updateFp = mgmtVgroupActionUpdate, + .encodeFp = mgmtVgroupActionEncode, + .decodeFp = mgmtVgroupActionDecode, + .destroyFp = mgmtVgroupActionDestroy, + }; + + tsVgroupSdb = sdbOpenTable(&tableDesc); if (tsVgroupSdb == NULL) { mError("failed to init vgroups data"); return -1; @@ -389,90 +446,6 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo return numOfRows; } -static void *mgmtVgroupActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = row; - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - pVgroup->vnodeGid[i].vnode = pVgroup->vgId; - } - - return NULL; -} - -static void *mgmtVgroupActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = row; - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - - if (pDb != NULL) { - mgmtRemoveVgroupFromDb(pDb, pVgroup); - } - - // mgmtUnSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes); - tfree(pVgroup->tableList); - - return NULL; -} - -static void *mgmtVgroupActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - mgmtVgroupActionReset(row, str, size, ssize); - - SVgObj *pVgroup = (SVgObj *) row; - int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool); - - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - if (pDb != NULL) { - if (pDb->cfg.maxSessions != oldTables) { - mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxSessions); - taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxSessions); - int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions; - pVgroup->tableList = (STableInfo **)realloc(pVgroup->tableList, size); - } - } - - mTrace("vgroup:%d update, numOfVnode:%d", pVgroup->vgId, pVgroup->numOfVnodes); - - return NULL; -} - -static void *mgmtVgroupActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = (SVgObj *) row; - if (size < tsVgUpdateSize) { - *ssize = -1; - } else { - memcpy(str, pVgroup, tsVgUpdateSize); - *ssize = tsVgUpdateSize; - } - - return NULL; -} - -static void *mgmtVgroupActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = (SVgObj *) malloc(sizeof(SVgObj)); - if (pVgroup == NULL) return NULL; - memset(pVgroup, 0, sizeof(SVgObj)); - - int32_t tsVgUpdateSize = pVgroup->updateEnd - (int8_t *) pVgroup; - memcpy(pVgroup, str, tsVgUpdateSize); - - return (void *) pVgroup; -} - -static void *mgmtVgroupActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = (SVgObj *) row; - memcpy(pVgroup, str, tsVgUpdateSize); - return NULL; -} - -static void *mgmtVgroupActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = (SVgObj *) row; - if (pVgroup->idPool) { - taosIdPoolCleanUp(pVgroup->idPool); - pVgroup->idPool = NULL; - } - if (pVgroup->tableList) tfree(pVgroup->tableList); - tfree(row); - return NULL; -} - void mgmtUpdateVgroup(SVgObj *pVgroup) { sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, SDB_OPER_LOCAL); } From d0b0ac956e5c30a44fbd8630a57ca57fe13e2b26 Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 20 Mar 2020 23:18:00 +0800 Subject: [PATCH 05/12] [TD-15] fix bug in sdbInsertRow --- src/mnode/src/mgmtSdb.c | 184 +++++++++++++++++++-------------------- src/mnode/src/mgmtUser.c | 3 +- 2 files changed, 89 insertions(+), 98 deletions(-) diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 8703c63541..ebdbba5605 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -222,7 +222,6 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { int32_t real_size = 0; int32_t maxAutoIndex = 0; - oldId = pTable->id; if (sdbOpenSdbFile(pTable) < 0) return -1; total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); @@ -234,7 +233,6 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { sdbTrace("open sdb file:%s for read", pTable->fn); - // Loop to read sdb file row by row while (1) { memset(rowHead, 0, total_size); @@ -259,9 +257,6 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { continue; } - // sdbTrace("%s id:%ld rowSize:%d", pTable->name, rowHead->id, - // rowHead->rowSize); - bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM)); if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) { // TODO: Here may cause pTable->size not end of the file @@ -276,40 +271,22 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { continue; } - // Check if the the object exists already + if (pTable->keyType == SDB_KEYTYPE_AUTO) { + maxAutoIndex = MAX(maxAutoIndex, *(int32_t *) rowHead->data); + } pMetaRow = sdbGetRow(pTable, rowHead->data); - if (pMetaRow == NULL) { // New object + if (pMetaRow == NULL) { if (rowHead->id < 0) { - /* assert(0); */ sdbError("error sdb negative id:%d, sdb:%s, skip", rowHead->id, pTable->name); } else { - rowMeta.id = rowHead->id; - // TODO: Get rid of the rowMeta.offset and rowSize - rowMeta.offset = pTable->size; - rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = (*pTable->decodeFp)(rowHead->data); - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - pTable->autoIndex++; - maxAutoIndex = MAX(maxAutoIndex, *(int32_t*)rowHead->data); - } - pTable->numOfRows++; + sdbInsertRow(pTable, rowHead->data, SDB_OPER_DISK); } - } else { // already exists - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - pTable->autoIndex++; - maxAutoIndex = MAX(maxAutoIndex, *(int32_t *) rowHead->data); - } - - if (rowHead->id < 0) { // Delete the object - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); - (*pTable->destroyFp)(pMetaRow); - pTable->numOfRows--; - numOfDels++; - } else { // Reset the object TODO: is it possible to merge reset and - // update ?? - //(*(pTable->appTool))(SDB_TYPE_RESET, pMetaRow, rowHead->data, rowHead->rowSize, NULL); + } else { + if (rowHead->id < 0) { + sdbDeleteRow(pTable, rowHead->data, SDB_OPER_DISK); + } else { + sdbUpdateRow(pTable, rowHead->data, rowHead->rowSize, SDB_OPER_DISK); } numOfDels++; } @@ -317,7 +294,6 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { pTable->size += real_size; if (pTable->id < abs(rowHead->id)) pTable->id = abs(rowHead->id); - //TODO: check this valid pTable->size += 4; lseek(pTable->fd, 4, SEEK_CUR); } @@ -326,8 +302,10 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { pTable->autoIndex = maxAutoIndex; } - sdbVersion += (pTable->id - oldId); - if (numOfDels > pTable->hashSessions / 4) sdbSaveSnapShot(pTable); + sdbVersion += pTable->id; + if (numOfDels > pTable->hashSessions / 4) { + sdbSaveSnapShot(pTable); + } tfree(rowHead); return 0; @@ -390,12 +368,13 @@ void *sdbGetRow(void *handle, void *key) { pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key); pthread_mutex_unlock(&pTable->mutex); - if (pMeta == NULL) return NULL; + if (pMeta == NULL) { + return NULL; + } return pMeta->row; } -// row here must be encoded string (rowSize > 0) or the object it self (rowSize = 0) int32_t sdbInsertRow(void *handle, void *row, ESdbOperType oper) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta rowMeta; @@ -448,7 +427,7 @@ int32_t sdbInsertRow(void *handle, void *row, ESdbOperType oper) { } } - if (oper == SDB_OPER_GLOBAL || oper == SDB_OPER_LOCAL) { + if (oper != SDB_OPER_DISK) { rowHead->rowSize = (*pTable->encodeFp)(pObj, rowHead->data, pTable->maxRowSize); assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); @@ -462,43 +441,46 @@ int32_t sdbInsertRow(void *handle, void *row, ESdbOperType oper) { return -1; } - // update in SDB layer - rowMeta.id = pTable->id; - rowMeta.offset = pTable->size; - rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = pObj; - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pObj, &rowMeta); - twrite(pTable->fd, rowHead, real_size); pTable->size += real_size; sdbFinishCommit(pTable); + } - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is inserted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, (char *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is inserted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, *(int32_t *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - default: - sdbTrace("table:%s, a record is inserted, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - } - } + // update in SDB layer + rowMeta.id = pTable->id; + rowMeta.offset = pTable->size; + rowMeta.rowSize = rowHead->rowSize; + rowMeta.row = pObj; + (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pObj, &rowMeta); if (pTable->keyType == SDB_KEYTYPE_AUTO) { *((uint32_t *)pObj) = ++pTable->autoIndex; } pTable->numOfRows++; - pTable->id++; - sdbVersion++; + if (oper != SDB_OPER_DISK) { + pTable->id++; + sdbVersion++; + } + pthread_mutex_unlock(&pTable->mutex); + switch (pTable->keyType) { + case SDB_KEYTYPE_STRING: + sdbTrace("table:%s, a record is inserted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, + pTable->name, (char *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); + break; + case SDB_KEYTYPE_AUTO: + sdbTrace("table:%s, a record is inserted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, + pTable->name, *(int32_t *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); + break; + default: + sdbTrace("table:%s, a record is inserted, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, + pTable->name, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); + break; + } + (*pTable->insertFp)(pObj); tfree(rowHead); @@ -556,20 +538,22 @@ int32_t sdbDeleteRow(void *handle, void *row, ESdbOperType oper) { } } - rowHead->delimiter = SDB_DELIMITER; - rowHead->rowSize = rowSize; - rowHead->id = -(pTable->id); - memcpy(rowHead->data, row, rowSize); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, total_size) < 0) { - sdbError("failed to get checksum while inserting, sdb:%s", pTable->name); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } + if (oper != SDB_OPER_DISK) { + rowHead->delimiter = SDB_DELIMITER; + rowHead->rowSize = rowSize; + rowHead->id = -(pTable->id); + memcpy(rowHead->data, row, rowSize); + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, total_size) < 0) { + sdbError("failed to get checksum while inserting, sdb:%s", pTable->name); + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } - twrite(pTable->fd, rowHead, total_size); - pTable->size += total_size; - sdbFinishCommit(pTable); + twrite(pTable->fd, rowHead, total_size); + pTable->size += total_size; + sdbFinishCommit(pTable); + } switch (pTable->keyType) { case SDB_KEYTYPE_STRING: @@ -590,14 +574,18 @@ int32_t sdbDeleteRow(void *handle, void *row, ESdbOperType oper) { (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, row); pTable->numOfRows--; - pTable->id++; - sdbVersion++; + if (oper != SDB_OPER_DISK) { + pTable->id++; + sdbVersion++; + } + pthread_mutex_unlock(&pTable->mutex); tfree(rowHead); (*pTable->deleteFp)(pMetaRow); + (*pTable->destroyFp)(pMetaRow); return 0; } @@ -661,23 +649,25 @@ int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, ESdbOperType o real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); // write to the new position - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pTable->id; - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum, sdb:%s id:%d", pTable->name, rowHead->id); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; + if (oper != SDB_OPER_DISK) { + rowHead->delimiter = SDB_DELIMITER; + rowHead->id = pTable->id; + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { + sdbError("failed to get checksum, sdb:%s id:%d", pTable->name, rowHead->id); + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } + + twrite(pTable->fd, rowHead, real_size); + + pMeta->id = pTable->id; + pMeta->offset = pTable->size; + pMeta->rowSize = rowHead->rowSize; + pTable->size += real_size; + + sdbFinishCommit(pTable); } - - twrite(pTable->fd, rowHead, real_size); - - pMeta->id = pTable->id; - pMeta->offset = pTable->size; - pMeta->rowSize = rowHead->rowSize; - pTable->size += real_size; - - sdbFinishCommit(pTable); switch (pTable->keyType) { case SDB_KEYTYPE_STRING: @@ -694,8 +684,10 @@ int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, ESdbOperType o break; } - pTable->id++; - sdbVersion++; + if (oper != SDB_OPER_DISK) { + pTable->id++; + sdbVersion++; + } pthread_mutex_unlock(&pTable->mutex); diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 2ca37a4572..3321c41513 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -176,8 +176,7 @@ static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { return code; } - pUser = malloc(sizeof(SUserObj)); - memset(pUser, 0, sizeof(SUserObj)); + pUser = calloc(1, sizeof(SUserObj)); strcpy(pUser->user, name); taosEncryptPass((uint8_t*) pass, strlen(pass), pUser->pass); strcpy(pUser->acct, pAcct->user); From d93d1093b849a9363cb9c3e15b90d9cdf6a326f6 Mon Sep 17 00:00:00 2001 From: slguan Date: Sun, 22 Mar 2020 10:12:41 +0800 Subject: [PATCH 06/12] [TD-15] refact the interface of sdb --- src/mnode/inc/mgmtSdb.h | 43 +-- src/mnode/src/mgmtSdb.c | 800 +++++++++++++++++++++------------------- 2 files changed, 438 insertions(+), 405 deletions(-) diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h index bf5ccb2634..e33bfce824 100644 --- a/src/mnode/inc/mgmtSdb.h +++ b/src/mnode/inc/mgmtSdb.h @@ -21,46 +21,47 @@ extern "C" { #endif typedef enum { - SDB_KEYTYPE_STRING, - SDB_KEYTYPE_AUTO, - SDB_KEYTYPE_MAX + SDB_KEY_TYPE_STRING, + SDB_KEY_TYPE_AUTO } ESdbKeyType; typedef enum { - SDB_OPER_GLOBAL, - SDB_OPER_LOCAL, - SDB_OPER_DISK + SDB_OPER_TYPE_GLOBAL, + SDB_OPER_TYPE_LOCAL } ESdbOperType; -enum _sdbaction { - SDB_TYPE_INSERT, - SDB_TYPE_DELETE, - SDB_TYPE_UPDATE, -} ESdbForwardType; +typedef struct { + ESdbOperType type; + int32_t maxRowSize; + int32_t rowSize; + void * rowData; + void * pObj; + void * table; + int64_t version; +} SSdbOperDesc; typedef struct { char *tableName; int32_t hashSessions; int32_t maxRowSize; ESdbKeyType keyType; - int32_t (*insertFp)(void *pObj); - int32_t (*deleteFp)(void *pObj); - int32_t (*updateFp)(void *pObj); - int32_t (*encodeFp)(void *pObj, void *pData, int32_t maxRowSize); - void * (*decodeFp)(void *pData); - int32_t (*destroyFp)(void *pObj); + int32_t (*insertFp)(SSdbOperDesc *pOper); + int32_t (*deleteFp)(SSdbOperDesc *pOper); + int32_t (*updateFp)(SSdbOperDesc *pOper); + int32_t (*encodeFp)(SSdbOperDesc *pOper); + int32_t (*decodeFp)(SSdbOperDesc *pDesc); + int32_t (*destroyFp)(SSdbOperDesc *pDesc); } SSdbTableDesc; void *sdbOpenTable(SSdbTableDesc *desc); void sdbCloseTable(void *handle); -int32_t sdbInsertRow(void *handle, void *row, ESdbOperType oper); -int32_t sdbDeleteRow(void *handle, void *key, ESdbOperType oper); -int32_t sdbUpdateRow(void *handle, void *row, int32_t rowSize, ESdbOperType oper); +int32_t sdbInsertRow(SSdbOperDesc *pOper); +int32_t sdbDeleteRow(SSdbOperDesc *pOper); +int32_t sdbUpdateRow(SSdbOperDesc *pOper); void *sdbGetRow(void *handle, void *key); void *sdbFetchRow(void *handle, void *pNode, void **ppRow); -int64_t sdbGetId(void *handle); int64_t sdbGetNumOfRows(void *handle); uint64_t sdbGetVersion(); diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index ebdbba5605..f957dc4a83 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taosdef.h" +#include "taoserror.h" #include "tchecksum.h" #include "tglobalcfg.h" #include "tlog.h" @@ -39,29 +40,29 @@ typedef struct { typedef struct _SSdbTable { SSdbHeader header; - char name[TSDB_DB_NAME_LEN]; - char fn[TSDB_FILENAME_LEN]; + char tableName[TSDB_DB_NAME_LEN]; + char fileName[TSDB_FILENAME_LEN]; ESdbKeyType keyType; - int32_t dbId; + int32_t tableId; int32_t hashSessions; int32_t maxRowSize; - uint32_t autoIndex; - int64_t numOfRows; - int64_t id; - int64_t size; - void * iHandle; + int32_t autoIndex; int32_t fd; - int32_t (*insertFp)(void *pObj); - int32_t (*deleteFp)(void *pObj); - int32_t (*updateFp)(void *pObj); - void * (*decodeFp)(void *pData); // return pObj - int32_t (*encodeFp)(void *pObj, void *pData, int32_t maxRowSize); // return size of pData - int32_t (*destroyFp)(void *pObj); + int64_t numOfRows; + int64_t version; + int64_t fileSize; + void * iHandle; + int32_t (*insertFp)(SSdbOperDesc *pDesc); + int32_t (*deleteFp)(SSdbOperDesc *pOper); + int32_t (*updateFp)(SSdbOperDesc *pOper); + int32_t (*decodeFp)(SSdbOperDesc *pOper); + int32_t (*encodeFp)(SSdbOperDesc *pOper); + int32_t (*destroyFp)(SSdbOperDesc *pOper); pthread_mutex_t mutex; } SSdbTable; typedef struct { - int64_t id; + int64_t version; int64_t offset; int32_t rowSize; void * row; @@ -70,21 +71,27 @@ typedef struct { typedef struct { int32_t delimiter; int32_t rowSize; - int64_t id; + int64_t version; char data[]; } SRowHead; +typedef enum { + SDB_FORWARD_TYPE_INSERT, + SDB_FORWARD_TYPE_DELETE, + SDB_FORWARD_TYPE_UPDATE +} ESdbForwardType; + typedef struct { - uint8_t dbId; - int8_t type; - int16_t dataLen; - uint64_t version; - char data[]; + ESdbForwardType type; + int32_t tableId; + int64_t version; + int32_t rowSize; + void * rowData; } SForwardMsg; extern char version[]; const int16_t sdbFileVersion = 2; -int32_t (*mpeerForwardRequestFp)(SSdbTable *pTable, char type, void *cont, int32_t contLen) = NULL; +int32_t (*mpeerForwardRequestFp)(SForwardMsg *forwardMsg) = NULL; static SSdbTable *sdbTableList[10] = {0}; static int32_t sdbNumOfTables = 0; @@ -101,25 +108,36 @@ void sdbResetTable(SSdbTable *pTable); void sdbSaveSnapShot(void *handle); uint64_t sdbGetVersion() { return sdbVersion; } -int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->id; } +int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->version; } int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } -static int32_t sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int32_t dataLen) { +static char *sdbGetkeyStr(SSdbTable *pTable, void *row) { + static char str[16]; + switch (pTable->keyType) { + case SDB_KEY_TYPE_STRING: + return (char *)row; + case SDB_KEY_TYPE_AUTO: + sprintf(str, "%d", *(int32_t *)row); + return str; + default: + return "unknown"; + } +} + +static int32_t sdbForwardDbReqToPeer(SForwardMsg *forwardMsg) { if (mpeerForwardRequestFp) { - return mpeerForwardRequestFp(pTable, type, data, dataLen); + return mpeerForwardRequestFp(forwardMsg); } else { return 0; } } -static void sdbFinishCommit(void *handle) { - SSdbTable *pTable = (SSdbTable *)handle; - uint32_t sdbEcommit = SDB_ENDCOMMIT; - +static void sdbFinishCommit(SSdbTable *pTable) { + uint32_t sdbEcommit = SDB_ENDCOMMIT; off_t offset = lseek(pTable->fd, 0, SEEK_END); - assert(offset == pTable->size); + assert(offset == pTable->fileSize); twrite(pTable->fd, &sdbEcommit, sizeof(sdbEcommit)); - pTable->size += sizeof(sdbEcommit); + pTable->fileSize += sizeof(sdbEcommit); } static int32_t sdbOpenSdbFile(SSdbTable *pTable) { @@ -136,40 +154,40 @@ static int32_t sdbOpenSdbFile(SSdbTable *pTable) { // check sdb.db and .sdb.db status char fn[TSDB_FILENAME_LEN] = "\0"; - dirc = strdup(pTable->fn); - basec = strdup(pTable->fn); + dirc = strdup(pTable->fileName); + basec = strdup(pTable->fileName); sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); tfree(dirc); tfree(basec); if (stat(fn, &ofstat) == 0) { // .sdb.db file exists - if (stat(pTable->fn, &fstat) == 0) { + if (stat(pTable->fileName, &fstat) == 0) { remove(fn); } else { - remove(pTable->fn); - rename(fn, pTable->fn); + remove(pTable->fileName); + rename(fn, pTable->fileName); } } - pTable->fd = open(pTable->fn, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + pTable->fd = open(pTable->fileName, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); if (pTable->fd < 0) { - sdbError("failed to open file:%s", pTable->fn); + sdbError("table:%s, failed to open file:%s", pTable->tableName, pTable->fileName); return -1; } - pTable->size = 0; - stat(pTable->fn, &fstat); + pTable->fileSize = 0; + stat(pTable->fileName, &fstat); size = sizeof(pTable->header); if (fstat.st_size == 0) { pTable->header.swVersion = swVersion.iversion; pTable->header.sdbFileVersion = sdbFileVersion; if (taosCalcChecksumAppend(0, (uint8_t *)(&pTable->header), size) < 0) { - sdbError("failed to get file header checksum, file:%s", pTable->fn); + sdbError("table:%s, failed to get file header checksum, file:%s", pTable->tableName, pTable->fileName); tclose(pTable->fd); return -1; } twrite(pTable->fd, &(pTable->header), size); - pTable->size += size; + pTable->fileSize += size; sdbFinishCommit(pTable); } else { uint32_t sdbEcommit = 0; @@ -186,25 +204,25 @@ static int32_t sdbOpenSdbFile(SSdbTable *pTable) { ssize_t tsize = read(pTable->fd, &(pTable->header), size); if (tsize < size) { - sdbError("failed to read sdb file header, file:%s", pTable->fn); + sdbError("table:%s, failed to read sdb file header, file:%s", pTable->tableName, pTable->fileName); tclose(pTable->fd); return -1; } if (pTable->header.swVersion != swVersion.iversion) { - sdbWarn("sdb file:%s version not match software version", pTable->fn); + sdbWarn("table:%s, sdb file:%s version not match software version", pTable->tableName, pTable->fileName); } if (!taosCheckChecksumWhole((uint8_t *)(&pTable->header), size)) { - sdbError("sdb file header is broken since checksum mismatch, file:%s", pTable->fn); + sdbError("table:%s, sdb file header is broken since checksum mismatch, file:%s", pTable->tableName, pTable->fileName); tclose(pTable->fd); return -1; } - pTable->size += size; + pTable->fileSize += size; // skip end commit symbol lseek(pTable->fd, sizeof(sdbEcommit), SEEK_CUR); - pTable->size += sizeof(sdbEcommit); + pTable->fileSize += sizeof(sdbEcommit); } pTable->numOfRows = 0; @@ -213,106 +231,170 @@ static int32_t sdbOpenSdbFile(SSdbTable *pTable) { } static int32_t sdbInitTableByFile(SSdbTable *pTable) { - SRowMeta rowMeta; - int32_t numOfDels = 0; - int32_t bytes = 0; - int64_t oldId = 0; - void * pMetaRow = NULL; - int32_t total_size = 0; - int32_t real_size = 0; - int32_t maxAutoIndex = 0; - - if (sdbOpenSdbFile(pTable) < 0) return -1; - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); + sdbTrace("table:%s, open sdb file:%s for read", pTable->tableName, pTable->fileName); + if (sdbOpenSdbFile(pTable) < 0) { + sdbError("table:%s, failed to open sdb file:%s for read", pTable->tableName, pTable->fileName); + return -1; + } + + int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); SRowHead *rowHead = (SRowHead *)malloc(total_size); if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->name); + sdbError("table:%s, failed to allocate row head memory, sdb:%s", pTable->tableName, pTable->tableName); return -1; } - sdbTrace("open sdb file:%s for read", pTable->fn); - + int32_t numOfChanged = 0; + int32_t maxAutoIndex = 0; while (1) { memset(rowHead, 0, total_size); - bytes = read(pTable->fd, rowHead, sizeof(SRowHead)); + int32_t bytes = read(pTable->fd, rowHead, sizeof(SRowHead)); if (bytes < 0) { - sdbError("failed to read sdb file:%s", pTable->fn); - goto sdb_exit1; + sdbError("table:%s, failed to read sdb file:%s", pTable->tableName, pTable->fileName); + tfree(rowHead); + return -1; } if (bytes == 0) break; if (bytes < sizeof(SRowHead) || rowHead->delimiter != SDB_DELIMITER) { - pTable->size++; + pTable->fileSize++; lseek(pTable->fd, -(bytes - 1), SEEK_CUR); continue; } if (rowHead->rowSize < 0 || rowHead->rowSize > pTable->maxRowSize) { - sdbError("error row size in sdb file:%s, id:%d rowSize:%d maxRowSize:%d", - pTable->fn, rowHead->id, rowHead->rowSize, pTable->maxRowSize); - pTable->size += sizeof(SRowHead); + sdbError("table:%s, error row size in sdb filesize:%d, version:%d rowSize:%d maxRowSize:%d", pTable->tableName, + pTable->fileSize, rowHead->version, rowHead->rowSize, pTable->maxRowSize); + pTable->fileSize += sizeof(SRowHead); continue; } bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM)); if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) { - // TODO: Here may cause pTable->size not end of the file - sdbError("failed to read sdb file:%s id:%d rowSize:%d", pTable->fn, rowHead->id, rowHead->rowSize); + // TODO: Here may cause pTable->fileSize not end of the file + sdbError("table:%s, failed to read sdb file, version:%d rowSize:%d", pTable->tableName, rowHead->version, + rowHead->rowSize); break; } - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); if (!taosCheckChecksumWhole((uint8_t *)rowHead, real_size)) { - sdbError("error sdb checksum, sdb:%s id:%d, skip", pTable->name, rowHead->id); - pTable->size += real_size; + sdbError("table:%s, error sdb checksum, version:%d, skip", pTable->tableName, rowHead->version); + pTable->fileSize += real_size; continue; } - - if (pTable->keyType == SDB_KEYTYPE_AUTO) { + + if (pTable->keyType == SDB_KEY_TYPE_AUTO) { maxAutoIndex = MAX(maxAutoIndex, *(int32_t *) rowHead->data); } - pMetaRow = sdbGetRow(pTable, rowHead->data); + void *pMetaRow = sdbGetRow(pTable, rowHead->data); if (pMetaRow == NULL) { - if (rowHead->id < 0) { - sdbError("error sdb negative id:%d, sdb:%s, skip", rowHead->id, pTable->name); + if (rowHead->version < 0) { + sdbError("table:%s, error sdb negative version:%d, record:%s, skip", pTable->tableName, rowHead->version, + sdbGetkeyStr(pTable, rowHead->data)); } else { - sdbInsertRow(pTable, rowHead->data, SDB_OPER_DISK); + SRowMeta rowMeta; + rowMeta.version = rowHead->version; + rowMeta.offset = pTable->fileSize; + rowMeta.rowSize = rowHead->rowSize; + SSdbOperDesc oper = { + .table = pTable, + .rowData = rowHead->data, + .rowSize = rowHead->rowSize + }; + int32_t code = (*pTable->decodeFp)(&oper); + if (code == TSDB_CODE_SUCCESS) { + rowMeta.row = oper.pObj; + (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); + pTable->numOfRows++; + sdbTrace("table:%s, read record:%s and insert, numOfRows:%d version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version, sdbVersion); + } else { + sdbError("table:%s, failed to decode record:%s, numOfRows:%d version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version, sdbVersion); + } } } else { - if (rowHead->id < 0) { - sdbDeleteRow(pTable, rowHead->data, SDB_OPER_DISK); + if (rowHead->version < 0) { + SSdbOperDesc oper = { + .table = pTable, + .pObj = pMetaRow + }; + (*pTable->destroyFp)(&oper); + (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); + pTable->numOfRows--; + sdbTrace("table:%s, read record:%s and delete, numOfRows:%d version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version, sdbVersion); } else { - sdbUpdateRow(pTable, rowHead->data, rowHead->rowSize, SDB_OPER_DISK); + SRowMeta rowMeta; + rowMeta.version = rowHead->version; + rowMeta.offset = pTable->fileSize; + rowMeta.rowSize = rowHead->rowSize; + SSdbOperDesc oper = { + .table = pTable, + .rowData = rowHead->data, + .rowSize = rowHead->rowSize, + .pObj = pMetaRow + }; + + (*pTable->destroyFp)(&oper); + (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); + + int32_t code = (*pTable->decodeFp)(&oper); + if (code == TSDB_CODE_SUCCESS) { + rowMeta.row = oper.pObj; + (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); + sdbTrace("table:%s, read record:%s and update, numOfRows:%d version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version, sdbVersion); + } else { + sdbError("table:%s, failed to decode record:%s, numOfRows:%d version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version, sdbVersion); + } + } + numOfChanged++; + if (pTable->version < abs(rowHead->version)) { + pTable->version = abs(rowHead->version); } - numOfDels++; } - pTable->size += real_size; - if (pTable->id < abs(rowHead->id)) pTable->id = abs(rowHead->id); - - pTable->size += 4; + pTable->fileSize += real_size; + pTable->fileSize += 4; lseek(pTable->fd, 4, SEEK_CUR); } - if (pTable->keyType == SDB_KEYTYPE_AUTO) { + void *pNode = NULL; + while (1) { + SRowMeta * pMeta; + pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); + if (pMeta == NULL) break; + + SSdbOperDesc oper = { + .pObj = pMeta->row, + .table = pTable, + .version = pMeta->version, + }; + + int32_t code = (*pTable->insertFp)(&oper); + if (code != TSDB_CODE_SUCCESS) { + sdbError("table:%s, failed to insert record:%s", pTable->tableName, sdbGetkeyStr(pTable, rowHead->data)); + } + } + + sdbVersion += pTable->version; + + if (pTable->keyType == SDB_KEY_TYPE_AUTO) { pTable->autoIndex = maxAutoIndex; } - sdbVersion += pTable->id; - if (numOfDels > pTable->hashSessions / 4) { + if (numOfChanged > pTable->hashSessions / 4) { sdbSaveSnapShot(pTable); } tfree(rowHead); return 0; - -sdb_exit1: - tfree(rowHead); - return -1; } void *sdbOpenTable(SSdbTableDesc *pDesc) { @@ -328,8 +410,8 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) { pTable->encodeFp = pDesc->encodeFp; pTable->decodeFp = pDesc->decodeFp; pTable->destroyFp = pDesc->destroyFp; - strcpy(pTable->name, pDesc->tableName); - sprintf(pTable->fn, "%s/%s.db", tsMnodeDir, pTable->name); + strcpy(pTable->tableName, pDesc->tableName); + sprintf(pTable->fileName, "%s/%s.db", tsMnodeDir, pTable->tableName); if (sdbInitIndexFp[pTable->keyType] != NULL) { pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SRowMeta)); @@ -339,10 +421,11 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) { if (sdbInitTableByFile(pTable) < 0) return NULL; - pTable->dbId = sdbNumOfTables++; - sdbTableList[pTable->dbId] = pTable; + pTable->tableId = sdbNumOfTables++; + sdbTableList[pTable->tableId] = pTable; - sdbTrace("table:%s is initialized, numOfRows:%d, numOfTables:%d", pTable->name, pTable->numOfRows, sdbNumOfTables); + sdbTrace("table:%s is initialized, numOfRows:%d, numOfTables:%d, version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, pTable->numOfRows, sdbNumOfTables, pTable->version, sdbVersion); return pTable; } @@ -375,323 +458,267 @@ void *sdbGetRow(void *handle, void *key) { return pMeta->row; } -int32_t sdbInsertRow(void *handle, void *row, ESdbOperType oper) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta rowMeta; - void * pObj = NULL; - int32_t total_size = 0; - int32_t real_size = 0; - +int32_t sdbInsertRow(SSdbOperDesc *pOper) { + SSdbTable *pTable = (SSdbTable *)pOper->table; if (pTable == NULL) { sdbError("sdb tables is null"); - return -1; - } - - if (sdbGetRow(handle, row)) { - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbError("table:%s, failed to insert record:%s sdbVersion:%" PRId64 " id:%" PRId64 , pTable->name, (char *)row, sdbVersion, pTable->id); - break; - case SDB_KEYTYPE_AUTO: - sdbError("table:%s, failed to insert record:%d sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, *(int32_t *)row, sdbVersion, pTable->id); - break; - default: - sdbError("table:%s, failed to insert record sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, sdbVersion, pTable->id); - break; - } - return -1; + return TSDB_CODE_OTHERS; } - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("table:%s, failed to allocate row head memory", pTable->name); - return -1; + if (sdbGetRow(pTable, pOper->pObj)) { + sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj)); + return TSDB_CODE_ALREADY_THERE; } - memset(rowHead, 0, total_size); - if (oper == SDB_OPER_GLOBAL) { - pObj = row; - } else { - pObj = (*pTable->decodeFp)(row); - } - pthread_mutex_lock(&pTable->mutex); - if (oper == SDB_OPER_GLOBAL) { - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_INSERT, rowHead->data, rowHead->rowSize) != 0) { - sdbError("table:%s, failed to insert record", pTable->name); + if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + SForwardMsg forward = { + .type = SDB_FORWARD_TYPE_INSERT, + .tableId = pTable->tableId, + .version = pTable->version + 1, + .rowSize = pOper->rowSize, + .rowData = pOper->rowData, + }; + + if (sdbForwardDbReqToPeer(&forward) != 0) { + sdbError("table:%s, failed to forward record:%s version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), pOper->version, sdbVersion); pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; + return TSDB_CODE_OTHERS; } } - if (oper != SDB_OPER_DISK) { - rowHead->rowSize = (*pTable->encodeFp)(pObj, rowHead->data, pTable->maxRowSize); - assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); + int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); + SRowHead *rowHead = (SRowHead *)calloc(1, total_size); + if (rowHead == NULL) { + pthread_mutex_unlock(&pTable->mutex); + sdbError("table:%s, failed to allocate row head memory for record:%s version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), pOper->version, sdbVersion); + return -1; + } + + if (pTable->keyType == SDB_KEY_TYPE_AUTO) { + *((uint32_t *)pOper->pObj) = ++pTable->autoIndex; + } + pTable->version++; + sdbVersion++; - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pTable->id + 1; - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("table:%s, failed to get checksum while inserting", pTable->name); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } + pOper->rowData = rowHead->data; + (*pTable->encodeFp)(pOper); + rowHead->rowSize = pOper->rowSize; - twrite(pTable->fd, rowHead, real_size); - pTable->size += real_size; - sdbFinishCommit(pTable); + assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); + + int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + rowHead->delimiter = SDB_DELIMITER; + rowHead->version = pTable->version; + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { + sdbError("table:%s, failed to get checksum while inserting", pTable->tableName); + pTable->version--; + sdbVersion--; + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; } + twrite(pTable->fd, rowHead, real_size); + pTable->fileSize += real_size; + sdbFinishCommit(pTable); + tfree(rowHead); + // update in SDB layer - rowMeta.id = pTable->id; - rowMeta.offset = pTable->size; - rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = pObj; - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pObj, &rowMeta); - - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - *((uint32_t *)pObj) = ++pTable->autoIndex; - } + SRowMeta rowMeta; + rowMeta.version = pTable->version; + rowMeta.offset = pTable->fileSize; + rowMeta.rowSize = pOper->rowSize; + rowMeta.row = pOper->pObj; + (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj, &rowMeta); pTable->numOfRows++; - - if (oper != SDB_OPER_DISK) { - pTable->id++; - sdbVersion++; - } pthread_mutex_unlock(&pTable->mutex); - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is inserted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, (char *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is inserted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, *(int32_t *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - default: - sdbTrace("table:%s, a record is inserted, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - } + sdbTrace("table:%s, a record is inserted:%s, sdbversion:%" PRId64 " version:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pOper->version, pOper->rowSize, pTable->numOfRows, pTable->fileSize); - (*pTable->insertFp)(pObj); - - tfree(rowHead); + (*pTable->insertFp)(pOper); return 0; } // row here can be object or null-terminated string -int32_t sdbDeleteRow(void *handle, void *row, ESdbOperType oper) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta = NULL; - void * pMetaRow = NULL; - SRowHead * rowHead = NULL; - int32_t rowSize = 0; - int32_t total_size = 0; - +int32_t sdbDeleteRow(SSdbOperDesc *pOper) { + SSdbTable *pTable = (SSdbTable *)pOper->table; if (pTable == NULL) return -1; - pMeta = sdbGetRowMeta(handle, row); + SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj); if (pMeta == NULL) { - sdbTrace("table:%s, record is not there, delete failed", pTable->name); + sdbTrace("table:%s, record is not there, delete failed", pTable->tableName); return -1; } - pMetaRow = pMeta->row; + void * pMetaRow = pMeta->row; assert(pMetaRow != NULL); + pthread_mutex_lock(&pTable->mutex); + + if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + SForwardMsg forward = { + .type = SDB_FORWARD_TYPE_DELETE, + .tableId = pTable->tableId, + .version = pTable->version + 1, + .rowSize = pOper->rowSize, + .rowData = pOper->rowData, + }; + + if (sdbForwardDbReqToPeer(&forward) == 0) { + sdbError("table:%s, failed to delete record", pTable->tableName); + pthread_mutex_unlock(&pTable->mutex); + return -1; + } + } + + int32_t total_size = sizeof(SRowHead) + pOper->rowSize + sizeof(TSCKSUM); + SRowHead *rowHead = (SRowHead *)calloc(1, total_size); + if (rowHead == NULL) { + sdbError("failed to allocate row head memory, sdb:%s", pTable->tableName); + pthread_mutex_unlock(&pTable->mutex); + return -1; + } + + pTable->version++; + sdbVersion++; + + int32_t rowSize = 0; switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - rowSize = strlen((char *)row) + 1; + case SDB_KEY_TYPE_STRING: + rowSize = strlen((char *)pOper->rowData) + 1; break; - case SDB_KEYTYPE_AUTO: + case SDB_KEY_TYPE_AUTO: rowSize = sizeof(uint64_t); break; default: return -1; } - total_size = sizeof(SRowHead) + rowSize + sizeof(TSCKSUM); - rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->name); + rowHead->delimiter = SDB_DELIMITER; + rowHead->rowSize = rowSize; + rowHead->version = -(pTable->version); + memcpy(rowHead->data, pOper->rowData, pOper->rowSize); + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, total_size) < 0) { + sdbError("failed to get checksum while inserting, sdb:%s", pTable->tableName); + pTable->version--; + sdbVersion--; + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); return -1; } - memset(rowHead, 0, total_size); - pthread_mutex_lock(&pTable->mutex); - - if (oper == SDB_OPER_GLOBAL) { - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_DELETE, (char *)row, rowSize) == 0) { - sdbError("table:%s, failed to delete record", pTable->name); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - } - - if (oper != SDB_OPER_DISK) { - rowHead->delimiter = SDB_DELIMITER; - rowHead->rowSize = rowSize; - rowHead->id = -(pTable->id); - memcpy(rowHead->data, row, rowSize); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, total_size) < 0) { - sdbError("failed to get checksum while inserting, sdb:%s", pTable->name); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - - twrite(pTable->fd, rowHead, total_size); - pTable->size += total_size; - sdbFinishCommit(pTable); - } - - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is deleted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is deleted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - default: - sdbTrace("table:%s, a record is deleted, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, sdbVersion, pTable->id, pTable->numOfRows); - break; - } - - // Delete from current layer - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, row); - - pTable->numOfRows--; - - if (oper != SDB_OPER_DISK) { - pTable->id++; - sdbVersion++; - } - - pthread_mutex_unlock(&pTable->mutex); + twrite(pTable->fd, rowHead, total_size); + pTable->fileSize += total_size; + sdbFinishCommit(pTable); tfree(rowHead); - (*pTable->deleteFp)(pMetaRow); - (*pTable->destroyFp)(pMetaRow); + sdbTrace("table:%s, a record is deleted:%s, sdbversion:%" PRId64 " id:%" PRId64 " numOfRows:%d", + pTable->tableName, sdbGetkeyStr(pTable, pOper->rowData), sdbVersion, pTable->version, pTable->numOfRows); + + // Delete from current layer + (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj); + + pTable->numOfRows--; + + pthread_mutex_unlock(&pTable->mutex); + + (*pTable->deleteFp)(pOper); + (*pTable->destroyFp)(pOper); return 0; } // row here can be the object or the string info (encoded string) -int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, ESdbOperType oper) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta = NULL; - int32_t total_size = 0; - int32_t real_size = 0; +int32_t sdbUpdateRow(SSdbOperDesc *pOper) { + SSdbTable *pTable = (SSdbTable *)pOper->table; + if (pTable == NULL) return -1; - if (pTable == NULL || row == NULL) return -1; - pMeta = sdbGetRowMeta(handle, row); + SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->rowData); if (pMeta == NULL) { - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbError("table:%s, failed to update record:%s, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, (char *) row, sdbVersion, pTable->id); - break; - case SDB_KEYTYPE_AUTO: - sdbError("table:%s, failed to update record:%d, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, *(int32_t *) row, sdbVersion, pTable->id); - break; - default: - sdbError("table:%s, failed to update record, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, sdbVersion, pTable->id); - break; - } + sdbError("table:%s, failed to update record:%s, record is not there, sdbversion:%" PRId64 " id:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, pOper->rowData), sdbVersion, pTable->version); return -1; } void *pMetaRow = pMeta->row; assert(pMetaRow != NULL); - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->name); - return -1; - } - memset(rowHead, 0, total_size); - pthread_mutex_lock(&pTable->mutex); - if (oper == SDB_OPER_GLOBAL) { - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_UPDATE, rowHead->data, rowHead->rowSize) == 0) { - sdbError("table:%s, failed to update record", pTable->name); + if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + SForwardMsg forward = { + .type = SDB_FORWARD_TYPE_UPDATE, + .tableId = pTable->tableId, + .version = pOper->version + 1, + .rowSize = pOper->rowSize, + .rowData = pOper->rowData, + }; + if (sdbForwardDbReqToPeer(&forward) == 0) { + sdbError("table:%s, failed to update record", pTable->tableName); pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); return -1; } } - if (pMetaRow != row) { - memcpy(rowHead->data, row, updateSize); - rowHead->rowSize = updateSize; + int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); + SRowHead *rowHead = (SRowHead *)calloc(1, total_size); + if (rowHead == NULL) { + sdbError("table:%s, failed to allocate row head memory", pTable->tableName); + return -1; + } + + if (pMetaRow != pOper->pObj) { + memcpy(rowHead->data, pOper->rowData, pOper->rowSize); + rowHead->rowSize = pOper->rowSize; } else { - rowHead->rowSize = (*pTable->encodeFp)(pMetaRow, rowHead->data, pTable->maxRowSize); + SSdbOperDesc oper = { + .table = pTable, + .rowData = rowHead->data, + .maxRowSize = pTable->maxRowSize, + .pObj = pOper->pObj + }; + (*pTable->encodeFp)(&oper); + rowHead->rowSize = oper.rowSize; } - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + pTable->version++; + sdbVersion++; - // write to the new position - if (oper != SDB_OPER_DISK) { - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pTable->id; - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum, sdb:%s id:%d", pTable->name, rowHead->id); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - - twrite(pTable->fd, rowHead, real_size); - - pMeta->id = pTable->id; - pMeta->offset = pTable->size; - pMeta->rowSize = rowHead->rowSize; - pTable->size += real_size; - - sdbFinishCommit(pTable); - } - - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is updated:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is updated:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - default: - sdbTrace("table:%s, a record is updated, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, pTable->name, sdbVersion, - pTable->id, pTable->numOfRows); - break; - } - - if (oper != SDB_OPER_DISK) { - pTable->id++; - sdbVersion++; + int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + rowHead->delimiter = SDB_DELIMITER; + rowHead->version = pTable->version; + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { + sdbError("failed to get checksum, sdb:%s version:%d", pTable->tableName, rowHead->version); + pTable->version--; + sdbVersion--; + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; } + + twrite(pTable->fd, rowHead, real_size); + pTable->fileSize += real_size; + sdbFinishCommit(pTable); + + sdbTrace("table:%s, a record is updated:%s, sdbversion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, pOper->rowData), sdbVersion, pTable->version, pTable->numOfRows); + pMeta->version = pTable->version; + pMeta->offset = pTable->fileSize; + pMeta->rowSize = rowHead->rowSize; + pthread_mutex_unlock(&pTable->mutex); - (*pTable->updateFp)(pMetaRow); // update in upper layer + (*pTable->updateFp)(pOper); // update in upper layer tfree(rowHead); @@ -708,7 +735,12 @@ void sdbCloseTable(void *handle) { while (1) { pNode = sdbFetchRow(handle, pNode, &row); if (row == NULL) break; - (*pTable->destroyFp)(row); + + SSdbOperDesc oper = { + .table = pTable, + .rowData = row, + }; + (*pTable->destroyFp)(&oper); } if (sdbCleanUpIndexFp[pTable->keyType]) (*sdbCleanUpIndexFp[pTable->keyType])(pTable->iHandle); @@ -718,20 +750,20 @@ void sdbCloseTable(void *handle) { pthread_mutex_destroy(&pTable->mutex); sdbNumOfTables--; - sdbTrace("table:%s is closed, id:%" PRId64 " numOfTables:%d", pTable->name, pTable->id, sdbNumOfTables); + sdbTrace("table:%s is closed, id:%" PRId64 " numOfTables:%d", pTable->tableName, pTable->version, sdbNumOfTables); tfree(pTable); } void sdbResetTable(SSdbTable *pTable) { - /* SRowHead rowHead; */ + /* SRowMeta rowMeta; int32_t bytes; int32_t total_size = 0; int32_t real_size = 0; SRowHead *rowHead = NULL; void * pMetaRow = NULL; - int64_t oldId = pTable->id; + int64_t oldId = pTable->version; int32_t oldNumOfRows = pTable->numOfRows; if (sdbOpenSdbFile(pTable) < 0) return; @@ -740,18 +772,18 @@ void sdbResetTable(SSdbTable *pTable) { total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); rowHead = (SRowHead *)malloc(total_size); if (rowHead == NULL) { - sdbError("failed to allocate row head memory for reset, sdb:%s", pTable->name); + sdbError("failed to allocate row head memory for reset, sdb:%s", pTable->tableName); return; } - sdbPrint("open sdb file:%s for reset table", pTable->fn); + sdbPrint("open sdb file:%s for reset table", pTable->fileName); while (1) { memset(rowHead, 0, total_size); bytes = read(pTable->fd, rowHead, sizeof(SRowHead)); if (bytes < 0) { - sdbError("failed to read sdb file:%s", pTable->fn); + sdbError("failed to read sdb file:%s", pTable->fileName); tfree(rowHead); return; } @@ -759,40 +791,40 @@ void sdbResetTable(SSdbTable *pTable) { if (bytes == 0) break; if (bytes < sizeof(SRowHead) || rowHead->delimiter != SDB_DELIMITER) { - pTable->size++; + pTable->fileSize++; lseek(pTable->fd, -(bytes - 1), SEEK_CUR); continue; } if (rowHead->rowSize < 0 || rowHead->rowSize > pTable->maxRowSize) { - sdbError("error row size in sdb file:%s for reset, id:%d rowSize:%d maxRowSize:%d", - pTable->fn, rowHead->id, rowHead->rowSize, pTable->maxRowSize); - pTable->size += sizeof(SRowHead); + sdbError("error row size in sdb file:%s for reset, version:%d rowSize:%d maxRowSize:%d", + pTable->fileName, rowHead->version, rowHead->rowSize, pTable->maxRowSize); + pTable->fileSize += sizeof(SRowHead); continue; } bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM)); if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) { - sdbError("failed to read sdb file:%s for reset, id:%d rowSize:%d", pTable->fn, rowHead->id, rowHead->rowSize); + sdbError("failed to read sdb file:%s for reset, version:%d rowSize:%d", pTable->fileName, rowHead->version, rowHead->rowSize); break; } real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); if (!taosCheckChecksumWhole((uint8_t *)rowHead, real_size)) { - sdbError("error sdb checksum, sdb:%s id:%d, skip", pTable->name, rowHead->id); - pTable->size += real_size; + sdbError("error sdb checksum, sdb:%s version:%d, skip", pTable->tableName, rowHead->version); + pTable->fileSize += real_size; continue; } - if (abs(rowHead->id) > oldId) { // not operated + if (abs(rowHead->version) > oldId) { // not operated pMetaRow = sdbGetRow(pTable, rowHead->data); if (pMetaRow == NULL) { // New object - if (rowHead->id < 0) { - sdbError("error sdb negative id:%d, sdb:%s, skip", rowHead->id, pTable->name); + if (rowHead->version < 0) { + sdbError("error sdb negative version:%d, sdb:%s, skip", rowHead->version, pTable->tableName); } else { - rowMeta.id = rowHead->id; + rowMeta.version = rowHead->version; // TODO:Get rid of the rowMeta.offset and rowSize - rowMeta.offset = pTable->size; + rowMeta.offset = pTable->fileSize; rowMeta.rowSize = rowHead->rowSize; rowMeta.row = (*pTable->decodeFp)(rowHead->data); (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); @@ -801,7 +833,7 @@ void sdbResetTable(SSdbTable *pTable) { (*pTable->insertFp)(rowMeta.row); } } else { // already exists - if (rowHead->id < 0) { // Delete the object + if (rowHead->version < 0) { // Delete the object (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); (*pTable->destroyFp)(pMetaRow); pTable->numOfRows--; @@ -811,38 +843,39 @@ void sdbResetTable(SSdbTable *pTable) { } } - pTable->size += real_size; - if (pTable->id < abs(rowHead->id)) pTable->id = abs(rowHead->id); + pTable->fileSize += real_size; + if (pTable->version < abs(rowHead->version)) pTable->version = abs(rowHead->version); } - sdbVersion += (pTable->id - oldId); + sdbVersion += (pTable->version - oldId); tfree(rowHead); - sdbPrint("table:%s is updated, sdbVerion:%" PRId64 " id:%" PRId64, pTable->name, sdbVersion, pTable->id); + sdbPrint("table:%s is updated, sdbVerion:%" PRId64 " id:%" PRId64, pTable->tableName, sdbVersion, pTable->version); + */ } // TODO:A problem here :use snapshot file to sync another node will cause problem void sdbSaveSnapShot(void *handle) { + /* SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta; void * pNode = NULL; - int32_t total_size = 0; - int32_t real_size = 0; - int32_t size = 0; - int32_t numOfRows = 0; + int32_t total_size = 0; + int32_t real_size = 0; + int32_t size = 0; + int32_t numOfRows = 0; uint32_t sdbEcommit = SDB_ENDCOMMIT; char * dirc = NULL; char * basec = NULL; - /* char action = SDB_TYPE_INSERT; */ if (pTable == NULL) return; - sdbTrace("Table:%s, save the snapshop", pTable->name); + sdbTrace("Table:%s, save the snapshop", pTable->tableName); char fn[128] = "\0"; - dirc = strdup(pTable->fn); - basec = strdup(pTable->fn); + dirc = strdup(pTable->fileName); + basec = strdup(pTable->fileName); sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); int32_t fd = open(fn, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); tfree(dirc); @@ -851,7 +884,7 @@ void sdbSaveSnapShot(void *handle) { total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); SRowHead *rowHead = (SRowHead *)malloc(total_size); if (rowHead == NULL) { - sdbError("failed to allocate memory while saving SDB snapshot, sdb:%s", pTable->name); + sdbError("failed to allocate memory while saving SDB snapshot, sdb:%s", pTable->tableName); return; } memset(rowHead, 0, size); @@ -867,36 +900,35 @@ void sdbSaveSnapShot(void *handle) { if (pMeta == NULL) break; rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pMeta->id; + rowHead->version = pMeta->id; rowHead->rowSize = (*pTable->encodeFp)(pMeta->row, rowHead->data, pTable->maxRowSize); real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum while save sdb %s snapshot", pTable->name); + sdbError("failed to get checksum while save sdb %s snapshot", pTable->tableName); tfree(rowHead); return; } - /* write(fd, &action, sizeof(action)); */ - /* size += sizeof(action); */ - twrite(fd, rowHead, real_size); - size += real_size; - twrite(fd, &sdbEcommit, sizeof(sdbEcommit)); - size += sizeof(sdbEcommit); - numOfRows++; + twrite(fd, rowHead, real_size); + size += real_size; + twrite(fd, &sdbEcommit, sizeof(sdbEcommit)); + size += sizeof(sdbEcommit); + numOfRows++; } tfree(rowHead); // Remove the old file tclose(pTable->fd); - remove(pTable->fn); + remove(pTable->fileName); // Rename the .sdb.db file to sdb.db file - rename(fn, pTable->fn); + rename(fn, pTable->fileName); pTable->fd = fd; - pTable->size = size; + pTable->fileSize = size; pTable->numOfRows = numOfRows; fdatasync(pTable->fd); + */ } void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { From 2e3e7f324035683a388680f8d1a6d3cdd3c83622 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 23 Mar 2020 00:05:02 +0800 Subject: [PATCH 07/12] [TD-15] fix sdb error while delete user --- src/mnode/inc/mgmtSdb.h | 6 +-- src/mnode/src/mgmtSdb.c | 64 +++++++++++------------ src/mnode/src/mgmtUser.c | 110 +++++++++++++++++++++------------------ 3 files changed, 93 insertions(+), 87 deletions(-) diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h index e33bfce824..ccddc11e04 100644 --- a/src/mnode/inc/mgmtSdb.h +++ b/src/mnode/inc/mgmtSdb.h @@ -32,12 +32,12 @@ typedef enum { typedef struct { ESdbOperType type; + void * table; + void * pObj; + int64_t version; int32_t maxRowSize; int32_t rowSize; void * rowData; - void * pObj; - void * table; - int64_t version; } SSdbOperDesc; typedef struct { diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index f957dc4a83..ea05cb59f4 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -290,6 +290,8 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { maxAutoIndex = MAX(maxAutoIndex, *(int32_t *) rowHead->data); } + pTable->version = MAX(pTable->version, abs(rowHead->version)); + void *pMetaRow = sdbGetRow(pTable, rowHead->data); if (pMetaRow == NULL) { if (rowHead->version < 0) { @@ -310,11 +312,11 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { rowMeta.row = oper.pObj; (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); pTable->numOfRows++; - sdbTrace("table:%s, read record:%s and insert, numOfRows:%d version:%" PRId64 " sdbversion:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version, sdbVersion); + sdbTrace("table:%s, read new record:%s, numOfRows:%d version:%" PRId64 , + pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version); } else { - sdbError("table:%s, failed to decode record:%s, numOfRows:%d version:%" PRId64 " sdbversion:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version, sdbVersion); + sdbError("table:%s, failed to decode record:%s, numOfRows:%d version:%" PRId64 , + pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version); } } } else { @@ -326,8 +328,8 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { (*pTable->destroyFp)(&oper); (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); pTable->numOfRows--; - sdbTrace("table:%s, read record:%s and delete, numOfRows:%d version:%" PRId64 " sdbversion:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version, sdbVersion); + sdbTrace("table:%s, read deleted record:%s, numOfRows:%d version:%" PRId64 , + pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version); } else { SRowMeta rowMeta; rowMeta.version = rowHead->version; @@ -347,17 +349,14 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { if (code == TSDB_CODE_SUCCESS) { rowMeta.row = oper.pObj; (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); - sdbTrace("table:%s, read record:%s and update, numOfRows:%d version:%" PRId64 " sdbversion:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version, sdbVersion); + sdbTrace("table:%s, read updated record:%s, numOfRows:%d version:%" PRId64 , + pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version); } else { - sdbError("table:%s, failed to decode record:%s, numOfRows:%d version:%" PRId64 " sdbversion:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version, sdbVersion); + sdbError("table:%s, failed to decode record:%s, numOfRows:%d version:%" PRId64 , + pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version); } } numOfChanged++; - if (pTable->version < abs(rowHead->version)) { - pTable->version = abs(rowHead->version); - } } pTable->fileSize += real_size; @@ -424,7 +423,7 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) { pTable->tableId = sdbNumOfTables++; sdbTableList[pTable->tableId] = pTable; - sdbTrace("table:%s is initialized, numOfRows:%d, numOfTables:%d, version:%" PRId64 " sdbversion:%" PRId64, + sdbTrace("table:%s, is initialized, numOfRows:%d numOfTables:%d version:%" PRId64 " sdbversion:%" PRId64, pTable->tableName, pTable->numOfRows, sdbNumOfTables, pTable->version, sdbVersion); return pTable; @@ -470,6 +469,7 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) { return TSDB_CODE_ALREADY_THERE; } + pOper->maxRowSize = pTable->maxRowSize; pthread_mutex_lock(&pTable->mutex); if (pOper->type == SDB_OPER_TYPE_GLOBAL) { @@ -507,12 +507,11 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) { pOper->rowData = rowHead->data; (*pTable->encodeFp)(pOper); rowHead->rowSize = pOper->rowSize; - + rowHead->delimiter = SDB_DELIMITER; + rowHead->version = pTable->version; assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - rowHead->delimiter = SDB_DELIMITER; - rowHead->version = pTable->version; if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { sdbError("table:%s, failed to get checksum while inserting", pTable->tableName); pTable->version--; @@ -540,7 +539,7 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) { pthread_mutex_unlock(&pTable->mutex); sdbTrace("table:%s, a record is inserted:%s, sdbversion:%" PRId64 " version:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pOper->version, pOper->rowSize, pTable->numOfRows, pTable->fileSize); + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version, pOper->rowSize, pTable->numOfRows, pTable->fileSize); (*pTable->insertFp)(pOper); @@ -568,11 +567,11 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { .type = SDB_FORWARD_TYPE_DELETE, .tableId = pTable->tableId, .version = pTable->version + 1, - .rowSize = pOper->rowSize, - .rowData = pOper->rowData, + .rowSize = pMeta->rowSize, + .rowData = pMeta->row, }; - if (sdbForwardDbReqToPeer(&forward) == 0) { + if (sdbForwardDbReqToPeer(&forward) != 0) { sdbError("table:%s, failed to delete record", pTable->tableName); pthread_mutex_unlock(&pTable->mutex); return -1; @@ -593,7 +592,7 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { int32_t rowSize = 0; switch (pTable->keyType) { case SDB_KEY_TYPE_STRING: - rowSize = strlen((char *)pOper->rowData) + 1; + rowSize = strlen((char *)pOper->pObj) + 1; break; case SDB_KEY_TYPE_AUTO: rowSize = sizeof(uint64_t); @@ -605,8 +604,9 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { rowHead->delimiter = SDB_DELIMITER; rowHead->rowSize = rowSize; rowHead->version = -(pTable->version); - memcpy(rowHead->data, pOper->rowData, pOper->rowSize); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, total_size) < 0) { + memcpy(rowHead->data, pOper->pObj, rowSize); + int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { sdbError("failed to get checksum while inserting, sdb:%s", pTable->tableName); pTable->version--; sdbVersion--; @@ -615,14 +615,14 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { return -1; } - twrite(pTable->fd, rowHead, total_size); - pTable->fileSize += total_size; + twrite(pTable->fd, rowHead, real_size); + pTable->fileSize += real_size; sdbFinishCommit(pTable); tfree(rowHead); sdbTrace("table:%s, a record is deleted:%s, sdbversion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->tableName, sdbGetkeyStr(pTable, pOper->rowData), sdbVersion, pTable->version, pTable->numOfRows); + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version, pTable->numOfRows); // Delete from current layer (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj); @@ -642,10 +642,10 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { SSdbTable *pTable = (SSdbTable *)pOper->table; if (pTable == NULL) return -1; - SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->rowData); + SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj); if (pMeta == NULL) { sdbError("table:%s, failed to update record:%s, record is not there, sdbversion:%" PRId64 " id:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, pOper->rowData), sdbVersion, pTable->version); + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version); return -1; } @@ -658,11 +658,11 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { SForwardMsg forward = { .type = SDB_FORWARD_TYPE_UPDATE, .tableId = pTable->tableId, - .version = pOper->version + 1, + .version = pTable->version + 1, .rowSize = pOper->rowSize, .rowData = pOper->rowData, }; - if (sdbForwardDbReqToPeer(&forward) == 0) { + if (sdbForwardDbReqToPeer(&forward) != 0) { sdbError("table:%s, failed to update record", pTable->tableName); pthread_mutex_unlock(&pTable->mutex); return -1; @@ -710,7 +710,7 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { sdbFinishCommit(pTable); sdbTrace("table:%s, a record is updated:%s, sdbversion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, pOper->rowData), sdbVersion, pTable->version, pTable->numOfRows); + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version, pTable->numOfRows); pMeta->version = pTable->version; pMeta->offset = pTable->fileSize; diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 6e72a6c8c2..11285764b8 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -38,23 +38,20 @@ static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg); static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg); static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg); -static int32_t mgmtUserActionDestroy(void *pObj) { - tfree(pObj); +static int32_t mgmtUserActionDestroy(SSdbOperDesc *pOper) { + tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mgmtUserActionInsert(void *pObj) { - SUserObj *pUser = (SUserObj *) pObj; +static int32_t mgmtUserActionInsert(SSdbOperDesc *pOper) { + SUserObj *pUser = pOper->pObj; SAcctObj *pAcct = mgmtGetAcct(pUser->acct); - - pUser->pAcct = pAcct; mgmtAddUserIntoAcct(pAcct, pUser); - return TSDB_CODE_SUCCESS; } -static int32_t mgmtUserActionDelete(void *pObj) { - SUserObj *pUser = (SUserObj *) pObj; +static int32_t mgmtUserActionDelete(SSdbOperDesc *pOper) { + SUserObj *pUser = pOper->pObj; SAcctObj *pAcct = mgmtGetAcct(pUser->acct); mgmtRemoveUserFromAcct(pAcct, pUser); @@ -62,36 +59,32 @@ static int32_t mgmtUserActionDelete(void *pObj) { return TSDB_CODE_SUCCESS; } -static int32_t mgmtUserActionUpdate(void *pObj) { +static int32_t mgmtUserActionUpdate(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mgmtUserActionEncode(void *pObj, void *pData, int32_t maxRowSize) { - SUserObj *pUser = (SUserObj *) pObj; +static int32_t mgmtUserActionEncode(SSdbOperDesc *pOper) { + SUserObj *pUser = pOper->pObj; - if (maxRowSize < tsUserUpdateSize) { + if (pOper->maxRowSize < tsUserUpdateSize) { return -1; } else { - memcpy(pData, pUser, tsUserUpdateSize); - return tsUserUpdateSize; + memcpy(pOper->rowData, pUser, tsUserUpdateSize); + pOper->rowSize = tsUserUpdateSize; + return TSDB_CODE_SUCCESS; } } -static void *mgmtUserActionDecode(void *pData) { - SUserObj *pUser = (SUserObj *) malloc(sizeof(SUserObj)); - if (pUser == NULL) return NULL; - memset(pUser, 0, sizeof(SUserObj)); - memcpy(pUser, pData, tsUserUpdateSize); +static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) { + SUserObj *pUser = (SUserObj *) calloc(1, sizeof(SUserObj)); + if (pUser == NULL) return -1; - return pUser; + memcpy(pUser, pOper->rowData, tsUserUpdateSize); + pOper->pObj = pUser; + return TSDB_CODE_SUCCESS; } int32_t mgmtInitUsers() { - void *pNode = NULL; - SUserObj *pUser = NULL; - SAcctObj *pAcct = NULL; - int32_t numOfUsers = 0; - SUserObj tObj; tsUserUpdateSize = tObj.updateEnd - (int8_t *)&tObj; @@ -99,7 +92,7 @@ int32_t mgmtInitUsers() { .tableName = "users", .hashSessions = TSDB_MAX_USERS, .maxRowSize = tsUserUpdateSize, - .keyType = SDB_KEYTYPE_STRING, + .keyType = SDB_KEY_TYPE_STRING, .insertFp = mgmtUserActionInsert, .deleteFp = mgmtUserActionDelete, .updateFp = mgmtUserActionUpdate, @@ -114,20 +107,7 @@ int32_t mgmtInitUsers() { return -1; } - while (1) { - pNode = sdbFetchRow(tsUserSdb, pNode, (void **)&pUser); - if (pUser == NULL) break; - - pUser->prev = NULL; - pUser->next = NULL; - - pAcct = mgmtGetAcct(pUser->acct); - mgmtAddUserIntoAcct(pAcct, pUser); - - numOfUsers++; - } - - pAcct = mgmtGetAcct("root"); + SAcctObj *pAcct = mgmtGetAcct("root"); mgmtCreateUser(pAcct, "root", "taosdata"); mgmtCreateUser(pAcct, "monitor", tsInternalPass); mgmtCreateUser(pAcct, "_root", tsInternalPass); @@ -151,7 +131,20 @@ SUserObj *mgmtGetUser(char *name) { } static int32_t mgmtUpdateUser(SUserObj *pUser) { - return sdbUpdateRow(tsUserSdb, pUser, tsUserUpdateSize, SDB_OPER_GLOBAL); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsUserSdb, + .pObj = pUser, + .rowSize = tsUserUpdateSize + }; + + int32_t code = sdbUpdateRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + tfree(pUser); + code = TSDB_CODE_SDB_ERROR; + } + + return code; } static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { @@ -186,8 +179,15 @@ static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { pUser->superAuth = 1; } - code = TSDB_CODE_SUCCESS; - if (sdbInsertRow(tsUserSdb, pUser, SDB_OPER_GLOBAL) < 0) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsUserSdb, + .pObj = pUser, + .rowSize = sizeof(SUserObj) + }; + + code = sdbInsertRow(&oper); + if (code != TSDB_CODE_SUCCESS) { tfree(pUser); code = TSDB_CODE_SDB_ERROR; } @@ -208,9 +208,19 @@ static int32_t mgmtDropUser(SAcctObj *pAcct, char *name) { return TSDB_CODE_NO_RIGHTS; } - sdbDeleteRow(tsUserSdb, pUser, SDB_OPER_GLOBAL); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsUserSdb, + .pObj = pUser + }; - return 0; + int32_t code = sdbDeleteRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + tfree(pUser); + code = TSDB_CODE_SDB_ERROR; + } + + return code; } static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { @@ -359,7 +369,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { memset(pUser->pass, 0, sizeof(pUser->pass)); taosEncryptPass((uint8_t*)pAlter->pass, strlen(pAlter->pass), pUser->pass); code = mgmtUpdateUser(pUser); - mLPrint("user:%s password is altered by %s, code:%d", pAlter->user, pUser->user, code); + mLPrint("user:%s password is altered by %s, result:%d", pUser->user, pOperUser->user, tstrerror(code)); } else { code = TSDB_CODE_NO_RIGHTS; } @@ -394,10 +404,6 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { } if (hasRight) { - //if (pAlter->privilege == 1) { // super - // pUser->superAuth = 1; - // pUser->writeAuth = 1; - //} if (pAlter->privilege == 2) { // read pUser->superAuth = 0; pUser->writeAuth = 0; @@ -408,7 +414,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { } code = mgmtUpdateUser(pUser); - mLPrint("user:%s privilege is altered by %s, code:%d", pAlter->user, pUser->user, code); + mLPrint("user:%s privilege is altered by %s, result:%d", pUser->user, pOperUser->user, tstrerror(code)); } else { code = TSDB_CODE_NO_RIGHTS; } From 4271952deaf1d1c0ea3d4ec4ccb2e54352def555 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 23 Mar 2020 15:52:56 +0800 Subject: [PATCH 08/12] [TD-15] refact the code to delete the database --- src/mnode/src/mgmtDb.c | 342 ++++++++++++++++++++++----------------- src/mnode/src/mgmtSdb.c | 197 +--------------------- src/mnode/src/mgmtUser.c | 13 +- 3 files changed, 210 insertions(+), 342 deletions(-) diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index e502d82a19..a4eeb88d91 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -15,11 +15,15 @@ #define _DEFAULT_SOURCE #include "os.h" - -#include "mgmtDb.h" +#include "taoserror.h" +#include "tstatus.h" +#include "tutil.h" +#include "name.h" +#include "mnode.h" #include "mgmtAcct.h" #include "mgmtBalance.h" #include "mgmtChildTable.h" +#include "mgmtDb.h" #include "mgmtDnode.h" #include "mgmtGrant.h" #include "mgmtShell.h" @@ -31,19 +35,13 @@ #include "mgmtTable.h" #include "mgmtUser.h" #include "mgmtVgroup.h" -#include "mnode.h" - -#include "taoserror.h" -#include "tstatus.h" -#include "tutil.h" -#include "name.h" static void *tsDbSdb = NULL; static int32_t tsDbUpdateSize; static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate); static void mgmtDropDb(void *handle, void *tmrId); -static void mgmtSetDbDirty(SDbObj *pDb); +static int32_t mgmtSetDbDirty(SDbObj *pDb); static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *pConn); @@ -51,64 +49,73 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg); static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg); static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg); -static int32_t mgmtDbActionDestroy(void *pObj) { - tfree(pObj); +static int32_t mgmtDbActionDestroy(SSdbOperDesc *pOper) { + tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mgmtDbActionInsert(void *pObj) { - SDbObj *pDb = (SDbObj *) pObj; +static int32_t mgmtDbActionInsert(SSdbOperDesc *pOper) { + SDbObj *pDb = pOper->pObj; SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); pDb->pHead = NULL; pDb->pTail = NULL; + pDb->prev = NULL; + pDb->next = NULL; pDb->numOfVgroups = 0; pDb->numOfTables = 0; - mgmtAddDbIntoAcct(pAcct, pDb); + pDb->numOfSuperTables = 0; + + if (pAcct != NULL) { + mgmtAddDbIntoAcct(pAcct, pDb); + } + else { + mError("db:%s, acct:%s info not exist in sdb", pDb->name, pDb->cfg.acct); + return TSDB_CODE_INVALID_ACCT; + } return TSDB_CODE_SUCCESS; } -static int32_t mgmtDbActionDelete(void *pObj) { - SDbObj *pDb = (SDbObj *) pObj; +static int32_t mgmtDbActionDelete(SSdbOperDesc *pOper) { + SDbObj *pDb = pOper->pObj; SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - mgmtRemoveDbFromAcct(pAcct, pDb); + mgmtRemoveDbFromAcct(pAcct, pDb); mgmtDropAllNormalTables(pDb); mgmtDropAllChildTables(pDb); mgmtDropAllSuperTables(pDb); - + mgmtDropAllVgroups(pDb); + return TSDB_CODE_SUCCESS; } -static int32_t mgmtDbActionUpdate(void *pObj) { +static int32_t mgmtDbActionUpdate(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mgmtDbActionEncode(void *pObj, void *pData, int32_t maxRowSize) { - SDbObj *pDb = (SDbObj *)pObj; - if (maxRowSize < tsDbUpdateSize) { +static int32_t mgmtDbActionEncode(SSdbOperDesc *pOper) { + SDbObj *pDb = pOper->pObj; + + if (pOper->maxRowSize < tsDbUpdateSize) { return -1; } else { - memcpy(pData, pDb, tsDbUpdateSize); - return tsDbUpdateSize; + memcpy(pOper->rowData, pDb, tsDbUpdateSize); + pOper->rowSize = tsDbUpdateSize; + return TSDB_CODE_SUCCESS; } } -static void *mgmtDbActionDecode(void *pData) { - SDbObj *pDb = (SDbObj *) malloc(sizeof(SDbObj)); - if (pDb == NULL) return NULL; - memset(pDb, 0, sizeof(SDbObj)); - memcpy(pDb, pData, tsDbUpdateSize); - - return (void *)pDb; +static int32_t mgmtDbActionDecode(SSdbOperDesc *pOper) { + SDbObj *pDb = (SDbObj *) calloc(1, sizeof(SDbObj)); + if (pDb == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; + + memcpy(pDb, pOper->rowData, tsDbUpdateSize); + pOper->pObj = pDb; + return TSDB_CODE_SUCCESS; } int32_t mgmtInitDbs() { - void * pNode = NULL; - SDbObj * pDb = NULL; - SAcctObj *pAcct = NULL; - SDbObj tObj; tsDbUpdateSize = tObj.updateEnd - (char *)&tObj; @@ -116,7 +123,7 @@ int32_t mgmtInitDbs() { .tableName = "dbs", .hashSessions = TSDB_MAX_DBS, .maxRowSize = tsDbUpdateSize, - .keyType = SDB_KEYTYPE_STRING, + .keyType = SDB_KEY_TYPE_STRING, .insertFp = mgmtDbActionInsert, .deleteFp = mgmtDbActionDelete, .updateFp = mgmtDbActionUpdate, @@ -131,25 +138,6 @@ int32_t mgmtInitDbs() { return -1; } - while (1) { - pNode = sdbFetchRow(tsDbSdb, pNode, (void **)&pDb); - if (pDb == NULL) break; - - pDb->pHead = NULL; - pDb->pTail = NULL; - pDb->prev = NULL; - pDb->next = NULL; - pDb->numOfTables = 0; - pDb->numOfVgroups = 0; - pDb->numOfSuperTables = 0; - pAcct = mgmtGetAcct(pDb->cfg.acct); - if (pAcct != NULL) - mgmtAddDbIntoAcct(pAcct, pDb); - else { - mError("db:%s acct:%s info not exist in sdb", pDb->name, pDb->cfg.acct); - } - } - mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_DB, mgmtProcessCreateDbMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_ALTER_DB, mgmtProcessAlterDbMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_DB, mgmtProcessDropDbMsg); @@ -250,14 +238,14 @@ static int32_t mgmtCheckDBParams(SCMCreateDbMsg *pCreate) { static int32_t mgmtCheckDbParams(SCMCreateDbMsg *pCreate) { // assign default parameters - if (pCreate->maxSessions < 0) pCreate->maxSessions = tsSessionsPerVnode; // - if (pCreate->cacheBlockSize < 0) pCreate->cacheBlockSize = tsCacheBlockSize; // - if (pCreate->daysPerFile < 0) pCreate->daysPerFile = tsDaysPerFile; // - if (pCreate->daysToKeep < 0) pCreate->daysToKeep = tsDaysToKeep; // - if (pCreate->daysToKeep1 < 0) pCreate->daysToKeep1 = pCreate->daysToKeep; // - if (pCreate->daysToKeep2 < 0) pCreate->daysToKeep2 = pCreate->daysToKeep; // - if (pCreate->commitTime < 0) pCreate->commitTime = tsCommitTime; // - if (pCreate->compression < 0) pCreate->compression = tsCompression; // + if (pCreate->maxSessions < 0) pCreate->maxSessions = tsSessionsPerVnode; + if (pCreate->cacheBlockSize < 0) pCreate->cacheBlockSize = tsCacheBlockSize; + if (pCreate->daysPerFile < 0) pCreate->daysPerFile = tsDaysPerFile; + if (pCreate->daysToKeep < 0) pCreate->daysToKeep = tsDaysToKeep; + if (pCreate->daysToKeep1 < 0) pCreate->daysToKeep1 = pCreate->daysToKeep; + if (pCreate->daysToKeep2 < 0) pCreate->daysToKeep2 = pCreate->daysToKeep; + if (pCreate->commitTime < 0) pCreate->commitTime = tsCommitTime; + if (pCreate->compression < 0) pCreate->compression = tsCompression; if (pCreate->commitLog < 0) pCreate->commitLog = tsCommitLog; if (pCreate->replications < 0) pCreate->replications = tsReplications; // if (pCreate->rowsInFileBlock < 0) pCreate->rowsInFileBlock = tsRowsInFileBlock; // @@ -321,9 +309,17 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { pDb->createdTime = taosGetTimestampMs(); pDb->cfg = *pCreate; - if (sdbInsertRow(tsDbSdb, pDb, SDB_OPER_GLOBAL) < 0) { - code = TSDB_CODE_SDB_ERROR; + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsDbSdb, + .pObj = pDb, + .rowSize = sizeof(SDbObj) + }; + + code = sdbInsertRow(&oper); + if (code != TSDB_CODE_SUCCESS) { tfree(pDb); + code = TSDB_CODE_SDB_ERROR; } return code; @@ -337,72 +333,6 @@ bool mgmtCheckIsMonitorDB(char *db, char *monitordb) { return (strncasecmp(dbName, monitordb, len) == 0 && len == strlen(monitordb)); } -static int32_t mgmtAlterDb(SAcctObj *pAcct, SCMAlterDbMsg *pAlter) { - return 0; -// int32_t code = TSDB_CODE_SUCCESS; -// -// SDbObj *pDb = (SDbObj *) sdbGetRow(tsDbSdb, pAlter->db); -// if (pDb == NULL) { -// mTrace("db:%s is not exist", pAlter->db); -// return TSDB_CODE_INVALID_DB; -// } -// -// int32_t oldReplicaNum = pDb->cfg.replications; -// if (pAlter->daysToKeep > 0) { -// mTrace("db:%s daysToKeep:%d change to %d", pDb->name, pDb->cfg.daysToKeep, pAlter->daysToKeep); -// pDb->cfg.daysToKeep = pAlter->daysToKeep; -// } else if (pAlter->replications > 0) { -// mTrace("db:%s replica:%d change to %d", pDb->name, pDb->cfg.replications, pAlter->replications); -// if (pAlter->replications < TSDB_REPLICA_MIN_NUM || pAlter->replications > TSDB_REPLICA_MAX_NUM) { -// mError("invalid db option replica: %d valid range: %d--%d", pAlter->replications, TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); -// return TSDB_CODE_INVALID_OPTION; -// } -// pDb->cfg.replications = pAlter->replications; -// } else if (pAlter->maxSessions > 0) { -// mTrace("db:%s tables:%d change to %d", pDb->name, pDb->cfg.maxSessions, pAlter->maxSessions); -// if (pAlter->maxSessions < TSDB_MIN_TABLES_PER_VNODE || pAlter->maxSessions > TSDB_MAX_TABLES_PER_VNODE) { -// mError("invalid db option tables: %d valid range: %d--%d", pAlter->maxSessions, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); -// return TSDB_CODE_INVALID_OPTION; -// } -// if (pAlter->maxSessions < pDb->cfg.maxSessions) { -// mError("invalid db option tables: %d should larger than original:%d", pAlter->maxSessions, pDb->cfg.maxSessions); -// return TSDB_CODE_INVALID_OPTION; -// } -// return TSDB_CODE_INVALID_OPTION; -// //The modification of tables needs to rewrite the head file, so disable this option -// //pDb->cfg.maxSessions = pAlter->maxSessions; -// } else { -// mError("db:%s alter msg, replica:%d, keep:%d, tables:%d, origin replica:%d keep:%d", pDb->name, -// pAlter->replications, pAlter->maxSessions, pAlter->daysToKeep, -// pDb->cfg.replications, pDb->cfg.daysToKeep); -// return TSDB_CODE_INVALID_OPTION; -// } -// -// if (sdbUpdateRow(tsDbSdb, pDb, tsDbUpdateSize, 1) < 0) { -// return TSDB_CODE_SDB_ERROR; -// } -// -// SVgObj *pVgroup = pDb->pHead; -// while (pVgroup != NULL) { -// balanceUpdateVgroupState(pVgroup, TSDB_VG_LB_STATUS_UPDATE, 0); -// if (oldReplicaNum < pDb->cfg.replications) { -// if (!balanceAddVnode(pVgroup, NULL, NULL)) { -// mWarn("db:%s vgroup:%d not enough dnode to add vnode", pAlter->db, pVgroup->vgId); -// code = TSDB_CODE_NO_ENOUGH_DNODES; -// } -// } -// if (pAlter->maxSessions > 0) { -// //rebuild meterList in mgmtVgroup.c -// mgmtUpdateVgroup(pVgroup); -// } -//// mgmtSendCreateVnodeMsg(pVgroup); -// pVgroup = pVgroup->next; -// } -// mgmtStartBalanceTimer(10); -// -// return code; -} - int32_t mgmtAddVgroupIntoDb(SDbObj *pDb, SVgObj *pVgroup) { pVgroup->next = pDb->pHead; pVgroup->prev = NULL; @@ -472,7 +402,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "created_time"); + strcpy(pSchema[cols].name, "create time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -600,7 +530,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) return 0; } -char *mgmtGetDbStr(char *src) { +static char *mgmtGetDbStr(char *src) { char *pos = strstr(src, TS_PATH_DELIMITER); return ++pos; @@ -740,8 +670,23 @@ void mgmtRemoveTableFromDb(SDbObj *pDb) { atomic_add_fetch_32(&pDb->numOfTables, -1); } -static void mgmtSetDbDirty(SDbObj *pDb) { +static int32_t mgmtSetDbDirty(SDbObj *pDb) { + if (pDb->dirty) return TSDB_CODE_SUCCESS; + pDb->dirty = true; + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsDbSdb, + .pObj = pDb, + .rowSize = tsDbUpdateSize + }; + + int32_t code = sdbUpdateRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + return TSDB_CODE_SDB_ERROR; + } + + return code; } static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { @@ -766,32 +711,121 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { } else { code = mgmtCreateDb(pMsg->pUser->pAcct, pCreate); if (code == TSDB_CODE_SUCCESS) { - mLPrint("DB:%s is created by %s", pCreate->db, pMsg->pUser->user); + mLPrint("db:%s, is created by %s", pCreate->db, pMsg->pUser->user); } } mgmtSendSimpleResp(pMsg->thandle, code); } +static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { + SDbCfg newCfg = pDb->cfg; + int32_t daysToKeep = htonl(pAlter->daysToKeep); + int32_t maxSessions = htonl(pAlter->maxSessions) + 1; + int8_t replications = pAlter->replications; + + terrno = TSDB_CODE_SUCCESS; + + if (daysToKeep > 0 && daysToKeep != pDb->cfg.daysToKeep) { + mTrace("db:%s, daysToKeep:%d change to %d", pDb->name, pDb->cfg.daysToKeep, daysToKeep); + newCfg.daysToKeep = daysToKeep; + } else if (replications > 0 && replications != pDb->cfg.replications) { + mTrace("db:%s, replica:%d change to %d", pDb->name, pDb->cfg.replications, replications); + if (replications < TSDB_REPLICA_MIN_NUM || replications > TSDB_REPLICA_MAX_NUM) { + mError("invalid db option replica: %d valid range: %d--%d", replications, TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); + terrno = TSDB_CODE_INVALID_OPTION; + } + newCfg.replications = replications; + } else if (maxSessions > 0 && maxSessions != pDb->cfg.maxSessions) { + mTrace("db:%s, tables:%d change to %d", pDb->name, pDb->cfg.maxSessions, maxSessions); + if (maxSessions < TSDB_MIN_TABLES_PER_VNODE || maxSessions > TSDB_MAX_TABLES_PER_VNODE) { + mError("invalid db option tables: %d valid range: %d--%d", maxSessions, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); + terrno = TSDB_CODE_INVALID_OPTION; + } + if (maxSessions < pDb->cfg.maxSessions) { + mError("invalid db option tables: %d should larger than original:%d", maxSessions, pDb->cfg.maxSessions); + terrno = TSDB_CODE_INVALID_OPTION; + } + newCfg.maxSessions = maxSessions; + } else { + } + + return newCfg; +} + +static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) { + SDbCfg newCfg = mgmtGetAlterDbOption(pDb, pAlter); + if (terrno != TSDB_CODE_SUCCESS) { + return terrno; + } + + if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) { + pDb->cfg = newCfg; + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsDbSdb, + .pObj = pDb, + .rowSize = tsDbUpdateSize + }; + + int32_t code = sdbUpdateRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + return TSDB_CODE_SDB_ERROR; + } + } + + return TSDB_CODE_SUCCESS; +} + static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg) { if (mgmtCheckRedirect(pMsg->thandle)) return; SCMAlterDbMsg *pAlter = pMsg->pCont; - pAlter->daysPerFile = htonl(pAlter->daysPerFile); - pAlter->daysToKeep = htonl(pAlter->daysToKeep); - pAlter->maxSessions = htonl(pAlter->maxSessions) + 1; + mTrace("db:%s, alter db msg is received from thandle:%p", pAlter->db, pMsg->thandle); - int32_t code; - if (!pMsg->pUser->writeAuth) { - code = TSDB_CODE_NO_RIGHTS; - } else { - code = mgmtAlterDb(pMsg->pUser->pAcct, pAlter); - if (code == TSDB_CODE_SUCCESS) { - mLPrint("DB:%s is altered by %s", pAlter->db, pMsg->pUser->user); - } + if (mgmtCheckExpired()) { + mError("db:%s, failed to alter, grant expired", pAlter->db); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED); + return; } - mgmtSendSimpleResp(pMsg->thandle, code); + if (!pMsg->pUser->writeAuth) { + mError("db:%s, failed to alter, no rights", pAlter->db); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS); + return; + } + + SDbObj *pDb = mgmtGetDb(pAlter->db); + if (pDb == NULL) { + mError("db:%s, failed to alter, invalid db", pAlter->db); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_DB); + return; + } + + int32_t code = mgmtAlterDb(pDb, pAlter); + if (code != TSDB_CODE_SUCCESS) { + mError("db:%s, failed to alter, invalid db option", pAlter->db); + mgmtSendSimpleResp(pMsg->thandle, code); + } + + SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg)); + memcpy(newMsg, pMsg, sizeof(SQueuedMsg)); + pMsg->pCont = NULL; + + SVgObj *pVgroup = pDb->pHead; + if (pVgroup != NULL) { + mPrint("vgroup:%d, will be altered", pVgroup->vgId); + newMsg->ahandle = pVgroup; + newMsg->expected = pVgroup->numOfVnodes; + mgmtAlterVgroup(pVgroup, newMsg); + return; + } + + mTrace("db:%s, all vgroups is altered", pDb->name); + + mgmtSendSimpleResp(newMsg->thandle, TSDB_CODE_SUCCESS); + rpcFreeCont(newMsg->pCont); + free(newMsg); } static void mgmtDropDb(void *handle, void *tmrId) { @@ -799,7 +833,12 @@ static void mgmtDropDb(void *handle, void *tmrId) { SDbObj *pDb = newMsg->ahandle; mPrint("db:%s, drop db from sdb", pDb->name); - int32_t code = sdbDeleteRow(tsDbSdb, pDb, SDB_OPER_GLOBAL); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsDbSdb, + .pObj = pDb + }; + int32_t code = sdbDeleteRow(&oper); if (code != 0) { code = TSDB_CODE_SDB_ERROR; } @@ -846,7 +885,12 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) { return; } - mgmtSetDbDirty(pDb); + int32_t code = mgmtSetDbDirty(pDb); + if (code != TSDB_CODE_SUCCESS) { + mError("db:%s, failed to drop, reason:%s", pDrop->db, tstrerror(code)); + mgmtSendSimpleResp(pMsg->thandle, code); + return; + } SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg)); memcpy(newMsg, pMsg, sizeof(SQueuedMsg)); diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index ea05cb59f4..eee8c0f1a0 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -104,9 +104,6 @@ static void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, s static void (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash}; static void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = {sdbFetchStrHashData, sdbFetchIntHashData}; -void sdbResetTable(SSdbTable *pTable); -void sdbSaveSnapShot(void *handle); - uint64_t sdbGetVersion() { return sdbVersion; } int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->version; } int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } @@ -388,10 +385,6 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { pTable->autoIndex = maxAutoIndex; } - if (numOfChanged > pTable->hashSessions / 4) { - sdbSaveSnapShot(pTable); - } - tfree(rowHead); return 0; } @@ -538,7 +531,7 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) { pthread_mutex_unlock(&pTable->mutex); - sdbTrace("table:%s, a record is inserted:%s, sdbversion:%" PRId64 " version:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, + sdbTrace("table:%s, inserte record:%s, sdbversion:%" PRId64 " version:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version, pOper->rowSize, pTable->numOfRows, pTable->fileSize); (*pTable->insertFp)(pOper); @@ -578,7 +571,7 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { } } - int32_t total_size = sizeof(SRowHead) + pOper->rowSize + sizeof(TSCKSUM); + int32_t total_size = sizeof(SRowHead) + pMeta->rowSize + sizeof(TSCKSUM); SRowHead *rowHead = (SRowHead *)calloc(1, total_size); if (rowHead == NULL) { sdbError("failed to allocate row head memory, sdb:%s", pTable->tableName); @@ -621,7 +614,7 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { tfree(rowHead); - sdbTrace("table:%s, a record is deleted:%s, sdbversion:%" PRId64 " id:%" PRId64 " numOfRows:%d", + sdbTrace("table:%s, delete record:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version, pTable->numOfRows); // Delete from current layer @@ -644,7 +637,7 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj); if (pMeta == NULL) { - sdbError("table:%s, failed to update record:%s, record is not there, sdbversion:%" PRId64 " id:%" PRId64, + sdbError("table:%s, failed to update record:%s, record is not there, sdbversion:%" PRId64 " version:%" PRId64, pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version); return -1; } @@ -697,7 +690,7 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { rowHead->delimiter = SDB_DELIMITER; rowHead->version = pTable->version; if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum, sdb:%s version:%d", pTable->tableName, rowHead->version); + sdbError("table:%s, failed to get checksum, version:%d", pTable->tableName, rowHead->version); pTable->version--; sdbVersion--; pthread_mutex_unlock(&pTable->mutex); @@ -709,7 +702,7 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { pTable->fileSize += real_size; sdbFinishCommit(pTable); - sdbTrace("table:%s, a record is updated:%s, sdbversion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, + sdbTrace("table:%s, update record:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%" PRId64, pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version, pTable->numOfRows); pMeta->version = pTable->version; @@ -750,187 +743,11 @@ void sdbCloseTable(void *handle) { pthread_mutex_destroy(&pTable->mutex); sdbNumOfTables--; - sdbTrace("table:%s is closed, id:%" PRId64 " numOfTables:%d", pTable->tableName, pTable->version, sdbNumOfTables); + sdbTrace("table:%s, is closed, version:%" PRId64 " numOfTables:%d", pTable->tableName, pTable->version, sdbNumOfTables); tfree(pTable); } -void sdbResetTable(SSdbTable *pTable) { - /* - SRowMeta rowMeta; - int32_t bytes; - int32_t total_size = 0; - int32_t real_size = 0; - SRowHead *rowHead = NULL; - void * pMetaRow = NULL; - int64_t oldId = pTable->version; - int32_t oldNumOfRows = pTable->numOfRows; - - if (sdbOpenSdbFile(pTable) < 0) return; - pTable->numOfRows = oldNumOfRows; - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory for reset, sdb:%s", pTable->tableName); - return; - } - - sdbPrint("open sdb file:%s for reset table", pTable->fileName); - - while (1) { - memset(rowHead, 0, total_size); - - bytes = read(pTable->fd, rowHead, sizeof(SRowHead)); - if (bytes < 0) { - sdbError("failed to read sdb file:%s", pTable->fileName); - tfree(rowHead); - return; - } - - if (bytes == 0) break; - - if (bytes < sizeof(SRowHead) || rowHead->delimiter != SDB_DELIMITER) { - pTable->fileSize++; - lseek(pTable->fd, -(bytes - 1), SEEK_CUR); - continue; - } - - if (rowHead->rowSize < 0 || rowHead->rowSize > pTable->maxRowSize) { - sdbError("error row size in sdb file:%s for reset, version:%d rowSize:%d maxRowSize:%d", - pTable->fileName, rowHead->version, rowHead->rowSize, pTable->maxRowSize); - pTable->fileSize += sizeof(SRowHead); - continue; - } - - bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM)); - if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) { - sdbError("failed to read sdb file:%s for reset, version:%d rowSize:%d", pTable->fileName, rowHead->version, rowHead->rowSize); - break; - } - - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - if (!taosCheckChecksumWhole((uint8_t *)rowHead, real_size)) { - sdbError("error sdb checksum, sdb:%s version:%d, skip", pTable->tableName, rowHead->version); - pTable->fileSize += real_size; - continue; - } - - if (abs(rowHead->version) > oldId) { // not operated - pMetaRow = sdbGetRow(pTable, rowHead->data); - if (pMetaRow == NULL) { // New object - if (rowHead->version < 0) { - sdbError("error sdb negative version:%d, sdb:%s, skip", rowHead->version, pTable->tableName); - } else { - rowMeta.version = rowHead->version; - // TODO:Get rid of the rowMeta.offset and rowSize - rowMeta.offset = pTable->fileSize; - rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = (*pTable->decodeFp)(rowHead->data); - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); - pTable->numOfRows++; - - (*pTable->insertFp)(rowMeta.row); - } - } else { // already exists - if (rowHead->version < 0) { // Delete the object - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); - (*pTable->destroyFp)(pMetaRow); - pTable->numOfRows--; - } else { // update the object - (*pTable->updateFp)(pMetaRow); - } - } - } - - pTable->fileSize += real_size; - if (pTable->version < abs(rowHead->version)) pTable->version = abs(rowHead->version); - } - - sdbVersion += (pTable->version - oldId); - - tfree(rowHead); - - sdbPrint("table:%s is updated, sdbVerion:%" PRId64 " id:%" PRId64, pTable->tableName, sdbVersion, pTable->version); - */ -} - -// TODO:A problem here :use snapshot file to sync another node will cause problem -void sdbSaveSnapShot(void *handle) { - /* - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta; - void * pNode = NULL; - int32_t total_size = 0; - int32_t real_size = 0; - int32_t size = 0; - int32_t numOfRows = 0; - uint32_t sdbEcommit = SDB_ENDCOMMIT; - char * dirc = NULL; - char * basec = NULL; - - if (pTable == NULL) return; - - sdbTrace("Table:%s, save the snapshop", pTable->tableName); - - char fn[128] = "\0"; - dirc = strdup(pTable->fileName); - basec = strdup(pTable->fileName); - sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); - int32_t fd = open(fn, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); - tfree(dirc); - tfree(basec); - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate memory while saving SDB snapshot, sdb:%s", pTable->tableName); - return; - } - memset(rowHead, 0, size); - - // Write the header - twrite(fd, &(pTable->header), sizeof(SSdbHeader)); - size += sizeof(SSdbHeader); - twrite(fd, &sdbEcommit, sizeof(sdbEcommit)); - size += sizeof(sdbEcommit); - - while (1) { - pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); - if (pMeta == NULL) break; - - rowHead->delimiter = SDB_DELIMITER; - rowHead->version = pMeta->id; - rowHead->rowSize = (*pTable->encodeFp)(pMeta->row, rowHead->data, pTable->maxRowSize); - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum while save sdb %s snapshot", pTable->tableName); - tfree(rowHead); - return; - } - - twrite(fd, rowHead, real_size); - size += real_size; - twrite(fd, &sdbEcommit, sizeof(sdbEcommit)); - size += sizeof(sdbEcommit); - numOfRows++; - } - - tfree(rowHead); - - // Remove the old file - tclose(pTable->fd); - remove(pTable->fileName); - // Rename the .sdb.db file to sdb.db file - rename(fn, pTable->fileName); - pTable->fd = fd; - pTable->fileSize = size; - pTable->numOfRows = numOfRows; - - fdatasync(pTable->fd); - */ -} - void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta; diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 11285764b8..e55ba9ea60 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -46,7 +46,15 @@ static int32_t mgmtUserActionDestroy(SSdbOperDesc *pOper) { static int32_t mgmtUserActionInsert(SSdbOperDesc *pOper) { SUserObj *pUser = pOper->pObj; SAcctObj *pAcct = mgmtGetAcct(pUser->acct); - mgmtAddUserIntoAcct(pAcct, pUser); + + if (pAcct != NULL) { + mgmtAddUserIntoAcct(pAcct, pUser); + } + else { + mError("user:%s, acct:%s info not exist in sdb", pUser->user, pUser->acct); + return TSDB_CODE_INVALID_ACCT; + } + return TSDB_CODE_SUCCESS; } @@ -77,7 +85,7 @@ static int32_t mgmtUserActionEncode(SSdbOperDesc *pOper) { static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) { SUserObj *pUser = (SUserObj *) calloc(1, sizeof(SUserObj)); - if (pUser == NULL) return -1; + if (pUser == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; memcpy(pUser, pOper->rowData, tsUserUpdateSize); pOper->pObj = pUser; @@ -216,7 +224,6 @@ static int32_t mgmtDropUser(SAcctObj *pAcct, char *name) { int32_t code = sdbDeleteRow(&oper); if (code != TSDB_CODE_SUCCESS) { - tfree(pUser); code = TSDB_CODE_SDB_ERROR; } From 00e18b70d7e18c2aab8394c08ba9e869a66eacf9 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 23 Mar 2020 19:07:57 +0800 Subject: [PATCH 09/12] [TD-15] refact code of tidpool --- src/inc/mnode.h | 3 + src/mnode/inc/mgmtDb.h | 10 +- src/mnode/inc/mgmtProfile.h | 2 + src/mnode/inc/mgmtVgroup.h | 5 +- src/mnode/src/mgmtAcct.c | 6 +- src/mnode/src/mgmtDb.c | 39 ++--- src/mnode/src/mgmtDnode.c | 5 - src/mnode/src/mgmtProfile.c | 10 ++ src/mnode/src/mgmtVgroup.c | 339 +++++++++++++++++++----------------- src/util/inc/tidpool.h | 6 +- src/util/src/tidpool.c | 139 ++++++--------- 11 files changed, 283 insertions(+), 281 deletions(-) diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 1452e1f53a..cde26dad4a 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -159,6 +159,8 @@ typedef struct { SSchema* schema; } SNormalTableObj; +struct _db_obj; + typedef struct _vg_obj { uint32_t vgId; char dbName[TSDB_DB_NAME_LEN + 1]; @@ -172,6 +174,7 @@ typedef struct _vg_obj { int8_t reserved[14]; int8_t updateEnd[1]; struct _vg_obj *prev, *next; + struct _db_obj *pDb; void * idPool; STableInfo ** tableList; } SVgObj; diff --git a/src/mnode/inc/mgmtDb.h b/src/mnode/inc/mgmtDb.h index 32bb9d9ec6..f2049f836c 100644 --- a/src/mnode/inc/mgmtDb.h +++ b/src/mnode/inc/mgmtDb.h @@ -22,11 +22,11 @@ extern "C" { #include "mnode.h" -int32_t mgmtAddVgroupIntoDb(SDbObj *pDb, SVgObj *pVgroup); -int32_t mgmtAddVgroupIntoDbTail(SDbObj *pDb, SVgObj *pVgroup); -int32_t mgmtRemoveVgroupFromDb(SDbObj *pDb, SVgObj *pVgroup); -int32_t mgmtMoveVgroupToTail(SDbObj *pDb, SVgObj *pVgroup); -int32_t mgmtMoveVgroupToHead(SDbObj *pDb, SVgObj *pVgroup); +void mgmtAddVgroupIntoDb(SVgObj *pVgroup); +void mgmtAddVgroupIntoDbTail(SVgObj *pVgroup); +void mgmtRemoveVgroupFromDb(SVgObj *pVgroup); +void mgmtMoveVgroupToTail(SVgObj *pVgroup); +void mgmtMoveVgroupToHead(SVgObj *pVgroup); int32_t mgmtInitDbs(); void mgmtCleanUpDbs(); diff --git a/src/mnode/inc/mgmtProfile.h b/src/mnode/inc/mgmtProfile.h index 7f9fd9622c..8c2e073c2d 100644 --- a/src/mnode/inc/mgmtProfile.h +++ b/src/mnode/inc/mgmtProfile.h @@ -28,6 +28,8 @@ bool mgmtCheckQhandle(uint64_t qhandle); void mgmtSaveQhandle(void *qhandle); void mgmtFreeQhandle(void *qhandle); +void mgmtFreeQueuedMsg(SQueuedMsg *pMsg); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mgmtVgroup.h b/src/mnode/inc/mgmtVgroup.h index b7c68b5f80..3379a93f28 100644 --- a/src/mnode/inc/mgmtVgroup.h +++ b/src/mnode/inc/mgmtVgroup.h @@ -28,13 +28,12 @@ int32_t mgmtInitVgroups(); void mgmtCleanUpVgroups(); SVgObj *mgmtGetVgroup(int32_t vgId); SVgObj *mgmtGetVgroupByVnode(uint32_t dnode, int32_t vnode); +void mgmtDropAllVgroups(SDbObj *pDropDb); void mgmtCreateVgroup(SQueuedMsg *pMsg); void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle); void mgmtUpdateVgroup(SVgObj *pVgroup); -void mgmtUpdateVgroupIp(SDnodeObj *pDnode); - -void mgmtSetVgroupIdPool(); +void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle); SVgObj *mgmtGetAvailableVgroup(SDbObj *pDb); void mgmtAddTableIntoVgroup(SVgObj *pVgroup, STableInfo *pTable); diff --git a/src/mnode/src/mgmtAcct.c b/src/mnode/src/mgmtAcct.c index b1ee72386e..0075f53319 100644 --- a/src/mnode/src/mgmtAcct.c +++ b/src/mnode/src/mgmtAcct.c @@ -85,8 +85,10 @@ int32_t mgmtRemoveUserFromAcct(SAcctObj *pAcct, SUserObj *pUser) { pUser->prev->next = pUser->next; } - if (pUser->next) pUser->next->prev = pUser->prev; - + if (pUser->next) { + pUser->next->prev = pUser->prev; + } + if (pUser->prev == NULL) { pAcct->pUser = pUser->next; } diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index a4eeb88d91..e8ab5aaa8c 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -333,7 +333,8 @@ bool mgmtCheckIsMonitorDB(char *db, char *monitordb) { return (strncasecmp(dbName, monitordb, len) == 0 && len == strlen(monitordb)); } -int32_t mgmtAddVgroupIntoDb(SDbObj *pDb, SVgObj *pVgroup) { +void mgmtAddVgroupIntoDb(SVgObj *pVgroup) { + SDbObj *pDb = pVgroup->pDb; pVgroup->next = pDb->pHead; pVgroup->prev = NULL; @@ -342,11 +343,10 @@ int32_t mgmtAddVgroupIntoDb(SDbObj *pDb, SVgObj *pVgroup) { pDb->pHead = pVgroup; pDb->numOfVgroups++; - - return 0; } -int32_t mgmtAddVgroupIntoDbTail(SDbObj *pDb, SVgObj *pVgroup) { +void mgmtAddVgroupIntoDbTail(SVgObj *pVgroup) { + SDbObj *pDb = pVgroup->pDb; pVgroup->next = NULL; pVgroup->prev = pDb->pTail; @@ -355,32 +355,25 @@ int32_t mgmtAddVgroupIntoDbTail(SDbObj *pDb, SVgObj *pVgroup) { pDb->pTail = pVgroup; pDb->numOfVgroups++; - - return 0; } -int32_t mgmtRemoveVgroupFromDb(SDbObj *pDb, SVgObj *pVgroup) { +void mgmtRemoveVgroupFromDb(SVgObj *pVgroup) { + SDbObj *pDb = pVgroup->pDb; if (pVgroup->prev) pVgroup->prev->next = pVgroup->next; if (pVgroup->next) pVgroup->next->prev = pVgroup->prev; if (pVgroup->prev == NULL) pDb->pHead = pVgroup->next; if (pVgroup->next == NULL) pDb->pTail = pVgroup->prev; pDb->numOfVgroups--; - - return 0; } -int32_t mgmtMoveVgroupToTail(SDbObj *pDb, SVgObj *pVgroup) { - mgmtRemoveVgroupFromDb(pDb, pVgroup); - mgmtAddVgroupIntoDbTail(pDb, pVgroup); - - return 0; +void mgmtMoveVgroupToTail(SVgObj *pVgroup) { + mgmtRemoveVgroupFromDb(pVgroup); + mgmtAddVgroupIntoDbTail(pVgroup); } -int32_t mgmtMoveVgroupToHead(SDbObj *pDb, SVgObj *pVgroup) { - mgmtRemoveVgroupFromDb(pDb, pVgroup); - mgmtAddVgroupIntoDb(pDb, pVgroup); - - return 0; +void mgmtMoveVgroupToHead(SVgObj *pVgroup) { + mgmtRemoveVgroupFromDb(pVgroup); + mgmtAddVgroupIntoDb(pVgroup); } void mgmtCleanUpDbs() { @@ -600,7 +593,7 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * if (strcmp(pUser->user, "root") == 0) { #endif pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.maxSessions - 1; // table num can be created should minus 1 + *(int32_t *)pWrite = pDb->cfg.maxSessions; // table num can be created should minus 1 cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -613,9 +606,9 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; #ifdef _TD_ARM_32_ - *(int32_t *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions - 1)); + *(int32_t *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions)); #else - *(float *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions - 1)); + *(float *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions)); #endif cols++; @@ -721,7 +714,7 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { SDbCfg newCfg = pDb->cfg; int32_t daysToKeep = htonl(pAlter->daysToKeep); - int32_t maxSessions = htonl(pAlter->maxSessions) + 1; + int32_t maxSessions = htonl(pAlter->maxSessions); int8_t replications = pAlter->replications; terrno = TSDB_CODE_SUCCESS; diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index f088ac75a4..0d83e262ae 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -566,11 +566,6 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { mgmtSetDnodeMaxVnodes(pDnode); } - if (lastPrivateIp != pDnode->privateIp || lastPublicIp != pDnode->publicIp) { - mgmtUpdateVgroupIp(pDnode); - //mgmtUpdateMnodeIp(); - } - int32_t openVnodes = htons(pStatus->openVnodes); for (int32_t j = 0; j < openVnodes; ++j) { pDnode->vload[j].vgId = htonl(pStatus->load[j].vgId); diff --git a/src/mnode/src/mgmtProfile.c b/src/mnode/src/mgmtProfile.c index b20e66dd87..1b98ac9596 100644 --- a/src/mnode/src/mgmtProfile.c +++ b/src/mnode/src/mgmtProfile.c @@ -762,3 +762,13 @@ int32_t mgmtInitProfile() { void mgmtCleanUpProfile() { } + +void mgmtFreeQueuedMsg(SQueuedMsg *pMsg) { + if (pMsg != NULL) { + if (pMsg->pCont != NULL) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + } + free(pMsg); + } +} \ No newline at end of file diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index 5b1968259f..b37b363077 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -23,62 +23,90 @@ #include "mgmtDb.h" #include "mgmtDClient.h" #include "mgmtDnode.h" +#include "mgmtDServer.h" #include "mgmtProfile.h" #include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtTable.h" #include "mgmtVgroup.h" -static void *tsVgroupSdb = NULL; +static void *tsVgroupSdb = NULL; static int32_t tsVgUpdateSize = 0; static int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg); static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg); +static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) ; static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle); static void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle); -static int32_t mgmtVgroupActionDestroy(void *pObj) { - SVgObj *pVgroup = (SVgObj *) pObj; +static int32_t mgmtVgroupActionDestroy(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; if (pVgroup->idPool) { taosIdPoolCleanUp(pVgroup->idPool); pVgroup->idPool = NULL; } - if (pVgroup->tableList) tfree(pVgroup->tableList); - tfree(pObj); + if (pVgroup->tableList) { + tfree(pVgroup->tableList); + } + tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mgmtVgroupActionInsert(void *pObj) { - SVgObj *pVgroup = pObj; +static int32_t mgmtVgroupActionInsert(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + SDbObj *pDb = mgmtGetDb(pVgroup->dbName); + if (pDb == NULL) { + return TSDB_CODE_INVALID_DB; + } + + pVgroup->pDb = pDb; + pVgroup->prev = NULL; + pVgroup->next = NULL; + + int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions; + pVgroup->tableList = (STableInfo **)calloc(pDb->cfg.maxSessions, sizeof(STableInfo *)); + if (pVgroup->tableList == NULL) { + mError("vgroup:%d, failed to malloc(size:%d) for the tableList of vgroups", pVgroup->vgId, size); + return -1; + } + + pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions); + if (pVgroup->idPool == NULL) { + mError("vgroup:%d, failed to taosInitIdPool for vgroups", pVgroup->vgId); + tfree(pVgroup->tableList); + return -1; + } + for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { + SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].dnodeId); + pVgroup->vnodeGid[i].privateIp = pDnode->privateIp; + pVgroup->vnodeGid[i].publicIp = pDnode->publicIp; pVgroup->vnodeGid[i].vnode = pVgroup->vgId; } + mgmtAddVgroupIntoDb(pVgroup); + return TSDB_CODE_SUCCESS; } -static int32_t mgmtVgroupActionDelete(void *pObj) { - SVgObj *pVgroup = pObj; - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - - if (pDb != NULL) { - mgmtRemoveVgroupFromDb(pDb, pVgroup); +static int32_t mgmtVgroupActionDelete(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + + if (pVgroup->pDb != NULL) { + mgmtRemoveVgroupFromDb(pVgroup); } - // mgmtUnSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes); - tfree(pVgroup->tableList); - return TSDB_CODE_SUCCESS; } -static int32_t mgmtVgroupActionUpdate(void *pObj) { - SVgObj *pVgroup = (SVgObj *) pObj; +static int32_t mgmtVgroupActionUpdate(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool); - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); + SDbObj *pDb = pVgroup->pDb; if (pDb != NULL) { if (pDb->cfg.maxSessions != oldTables) { mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxSessions); @@ -88,33 +116,31 @@ static int32_t mgmtVgroupActionUpdate(void *pObj) { } } - mTrace("vgroup:%d update, numOfVnode:%d", pVgroup->vgId, pVgroup->numOfVnodes); + mTrace("vgroup:%d, is updated, tables:%d numOfVnode:%d", pVgroup->vgId, pDb->cfg.maxSessions, pVgroup->numOfVnodes); return TSDB_CODE_SUCCESS; } -static int32_t mgmtVgroupActionEncode(void *pObj, void *pData, int32_t maxRowSize) { - SVgObj *pVgroup = (SVgObj *) pObj; - if (maxRowSize < tsVgUpdateSize) { +static int32_t mgmtVgroupActionEncode(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + if (pOper->maxRowSize < tsVgUpdateSize) { return -1; } else { - memcpy(pData, pVgroup, tsVgUpdateSize); - return tsVgUpdateSize; + memcpy(pOper->rowData, pVgroup, tsVgUpdateSize); + pOper->rowSize = tsVgUpdateSize; + return TSDB_CODE_SUCCESS; } } -static void *mgmtVgroupActionDecode(void *pObj) { - SVgObj *pVgroup = (SVgObj *) malloc(sizeof(SVgObj)); - if (pVgroup == NULL) return NULL; - memset(pVgroup, 0, sizeof(SVgObj)); - memcpy(pVgroup, pObj, tsVgUpdateSize); +static int32_t mgmtVgroupActionDecode(SSdbOperDesc *pOper) { + SVgObj *pVgroup = (SVgObj *) calloc(1, sizeof(SVgObj)); + if (pVgroup == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; - return pVgroup; + memcpy(pVgroup, pOper->rowData, tsVgUpdateSize); + pOper->pObj = pVgroup; + return TSDB_CODE_SUCCESS; } int32_t mgmtInitVgroups() { - void *pNode = NULL; - SVgObj *pVgroup = NULL; - SVgObj tObj; tsVgUpdateSize = tObj.updateEnd - (int8_t *)&tObj; @@ -122,7 +148,7 @@ int32_t mgmtInitVgroups() { .tableName = "vgroups", .hashSessions = TSDB_MAX_VGROUPS, .maxRowSize = tsVgUpdateSize, - .keyType = SDB_KEYTYPE_AUTO, + .keyType = SDB_KEY_TYPE_AUTO, .insertFp = mgmtVgroupActionInsert, .deleteFp = mgmtVgroupActionDelete, .updateFp = mgmtVgroupActionUpdate, @@ -137,47 +163,11 @@ int32_t mgmtInitVgroups() { return -1; } - while (1) { - pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); - if (pVgroup == NULL) break; - - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - if (pDb == NULL) continue; - - pVgroup->prev = NULL; - pVgroup->next = NULL; - - int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions; - pVgroup->tableList = (STableInfo **)malloc(size); - if (pVgroup->tableList == NULL) { - mError("failed to malloc(size:%d) for the tableList of vgroups", size); - return -1; - } - - memset(pVgroup->tableList, 0, size); - - pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions); - if (pVgroup->idPool == NULL) { - mError("failed to taosInitIdPool for vgroups"); - free(pVgroup->tableList); - return -1; - } - - taosIdPoolReinit(pVgroup->idPool); - - if (tsIsCluster && pVgroup->vnodeGid[0].publicIp == 0) { - pVgroup->vnodeGid[0].publicIp = inet_addr(tsPublicIp); - pVgroup->vnodeGid[0].privateIp = inet_addr(tsPrivateIp); - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, SDB_OPER_GLOBAL); - } - - // mgmtSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes, pVgroup->vgId); - } - mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_VGROUP, mgmtGetVgroupMeta); mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_VGROUP, mgmtRetrieveVgroups); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP, mgmtProcessCreateVnodeRsp); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_VNODE_RSP, mgmtProcessDropVnodeRsp); + mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_VNODE, mgmtProcessVnodeCfgMsg); mTrace("vgroup is initialized"); return 0; @@ -196,32 +186,41 @@ void mgmtCreateVgroup(SQueuedMsg *pMsg) { if (pDb == NULL) { mError("failed to create vgroup, db not found"); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_DB); + mgmtFreeQueuedMsg(pMsg); return; } - SVgObj *pVgroup = (SVgObj *)calloc(sizeof(SVgObj), 1); + SVgObj *pVgroup = (SVgObj *)calloc(1, sizeof(SVgObj)); strcpy(pVgroup->dbName, pDb->name); pVgroup->numOfVnodes = pDb->cfg.replications; + pVgroup->createdTime = taosGetTimestampMs(); if (mgmtAllocVnodes(pVgroup) != 0) { mError("db:%s, no enough dnode to alloc %d vnodes to vgroup", pDb->name, pVgroup->numOfVnodes); free(pVgroup); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_ENOUGH_DNODES); + mgmtFreeQueuedMsg(pMsg); return; } - pVgroup->createdTime = taosGetTimestampMs(); - pVgroup->tableList = (STableInfo **) calloc(sizeof(STableInfo *), pDb->cfg.maxSessions); - pVgroup->numOfTables = 0; - pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup, + .rowSize = sizeof(SVgObj) + }; - mgmtAddVgroupIntoDb(pDb, pVgroup); - // mgmtSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes, pVgroup->vgId); - - sdbInsertRow(tsVgroupSdb, pVgroup, SDB_OPER_GLOBAL); + int32_t code = sdbInsertRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + tfree(pVgroup); + code = TSDB_CODE_SDB_ERROR; + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SDB_ERROR); + mgmtFreeQueuedMsg(pMsg); + return; + } mPrint("vgroup:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - mPrint("vgroup:%d, dnode:%d vnode:%d", pVgroup->vgId, pVgroup->vnodeGid[i].dnodeId, pVgroup->vnodeGid[i].vnode); + mPrint("vgroup:%d, index:%d, dnode:%d vnode:%d", pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId, pVgroup->vnodeGid[i].vnode); } pMsg->ahandle = pVgroup; @@ -235,28 +234,12 @@ void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle) { } else { mTrace("vgroup:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes); mgmtSendDropVgroupMsg(pVgroup, NULL); - sdbDeleteRow(tsVgroupSdb, pVgroup, SDB_OPER_GLOBAL); - } -} - -void mgmtSetVgroupIdPool() { - void * pNode = NULL; - SVgObj *pVgroup = NULL; - SDbObj *pDb; - - while (1) { - pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); - if (pVgroup == NULL || pVgroup->idPool == 0) break; - - taosIdPoolSetFreeList(pVgroup->idPool); - pVgroup->numOfTables = taosIdPoolNumOfUsed(pVgroup->idPool); - - pDb = mgmtGetDb(pVgroup->dbName); - pDb->numOfTables += pVgroup->numOfTables; - if (pVgroup->numOfTables >= pDb->cfg.maxSessions - 1) - mgmtAddVgroupIntoDbTail(pDb, pVgroup); - else - mgmtAddVgroupIntoDb(pDb, pVgroup); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup + }; + sdbDeleteRow(&oper); } } @@ -287,7 +270,7 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { pShow->bytes[cols] = 9; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "vgroup_status"); + strcpy(pSchema[cols].name, "vgroup status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -327,13 +310,13 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { pShow->bytes[cols] = 9; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "vnode_status"); + strcpy(pSchema[cols].name, "vnode status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 16; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "public_ip"); + strcpy(pSchema[cols].name, "public ip"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; } @@ -445,25 +428,36 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo return numOfRows; } -void mgmtUpdateVgroup(SVgObj *pVgroup) { - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, SDB_OPER_LOCAL); -} - void mgmtAddTableIntoVgroup(SVgObj *pVgroup, STableInfo *pTable) { - pVgroup->numOfTables++; - if (pTable->sid >= 0) + if (pTable->sid >= 0 && pVgroup->tableList[pTable->sid] == NULL) { pVgroup->tableList[pTable->sid] = pTable; + taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); + pVgroup->numOfTables++; + pVgroup->pDb->numOfTables++; + } + + if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxSessions) + mgmtAddVgroupIntoDbTail(pVgroup); + else + mgmtAddVgroupIntoDb(pVgroup); } void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, STableInfo *pTable) { - pVgroup->numOfTables--; - if (pTable->sid >= 0) + if (pTable->sid >= 0 && pVgroup->tableList[pTable->sid] != NULL) { pVgroup->tableList[pTable->sid] = NULL; - taosFreeId(pVgroup->idPool, pTable->sid); + taosFreeId(pVgroup->idPool, pTable->sid); + pVgroup->numOfTables--; + pVgroup->pDb->numOfTables--; + } + + if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxSessions) + mgmtAddVgroupIntoDbTail(pVgroup); + else + mgmtAddVgroupIntoDb(pVgroup); } SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); + SDbObj *pDb = pVgroup->pDb; if (pDb == NULL) return NULL; SMDCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SMDCreateVnodeMsg)); @@ -487,6 +481,7 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { SVnodeDesc *vpeerDesc = pVnode->vpeerDesc; for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) { + vpeerDesc[j].vnode = htonl(pVgroup->vnodeGid[j].vnode); vpeerDesc[j].ip = htonl(pVgroup->vnodeGid[j].privateIp); } @@ -576,13 +571,23 @@ static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { newMsg->contLen = queueMsg->contLen; newMsg->pCont = rpcMallocCont(newMsg->contLen); memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen); + queueMsg->pCont = NULL; mgmtAddToShellQueue(newMsg); } else { - sdbDeleteRow(tsVgroupSdb, pVgroup, SDB_OPER_GLOBAL); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup + }; + int32_t code = sdbDeleteRow(&oper); + if (code != 0) { + code = TSDB_CODE_SDB_ERROR; + } + mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); } - free(queueMsg); + mgmtFreeQueuedMsg(queueMsg); } static SMDDropVnodeMsg *mgmtBuildDropVnodeMsg(int32_t vgId) { @@ -632,7 +637,15 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) { if (queueMsg->received != queueMsg->expected) return; - sdbDeleteRow(tsVgroupSdb, pVgroup, SDB_OPER_GLOBAL); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup + }; + int32_t code = sdbDeleteRow(&oper); + if (code != 0) { + code = TSDB_CODE_SDB_ERROR; + } SQueuedMsg *newMsg = calloc(1, sizeof(SQueuedMsg)); newMsg->msgType = queueMsg->msgType; @@ -644,50 +657,64 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) { memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen); mgmtAddToShellQueue(newMsg); - free(queueMsg); + queueMsg->pCont = NULL; + mgmtFreeQueuedMsg(queueMsg); } -void mgmtUpdateVgroupIp(SDnodeObj *pDnode) { - void * pNode = NULL; +static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) { + if (mgmtCheckRedirect(rpcMsg->handle)) return; + + SDMConfigVnodeMsg *pCfg = (SDMConfigVnodeMsg *) rpcMsg->pCont; + pCfg->dnode = htonl(pCfg->dnode); + pCfg->vnode = htonl(pCfg->vnode); + + SVgObj *pVgroup = mgmtGetVgroupByVnode(pCfg->dnode, pCfg->vnode); + if (pVgroup == NULL) { + mTrace("dnode:%s, vnode:%d, no vgroup info", taosIpStr(pCfg->dnode), pCfg->vnode); + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_VNODE); + return; + } + + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SUCCESS); + + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); + mgmtSendCreateVnodeMsg(pVgroup, &ipSet, NULL); +} + +void mgmtDropAllVgroups(SDbObj *pDropDb) { + void *pNode = NULL; + void *pLastNode = NULL; + int32_t numOfTables = 0; + int32_t dbNameLen = strlen(pDropDb->name); SVgObj *pVgroup = NULL; + while (1) { pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); if (pVgroup == NULL) break; - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SVnodeGid *vnodeGid = pVgroup->vnodeGid + i; - if (vnodeGid->dnodeId == pDnode->dnodeId) { - mPrint("vgroup:%d, dnode:%d, privateIp:%s change to %s, publicIp:%s change to %s", - pVgroup->vgId, vnodeGid->dnodeId, pDnode->privateIp, taosIpStr(vnodeGid->privateIp), - pDnode->publicIp, taosIpStr(vnodeGid->publicIp)); - vnodeGid->publicIp = pDnode->publicIp; - vnodeGid->privateIp = pDnode->privateIp; - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, SDB_OPER_GLOBAL); - } + if (strncmp(pDropDb->name, pVgroup->dbName, dbNameLen) == 0) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsVgroupSdb, + .pObj = pVgroup, + }; + sdbDeleteRow(&oper); + pNode = pLastNode; + numOfTables++; + continue; } } + + mTrace("db:%s, all vgroups is dropped from sdb", pDropDb->name, numOfTables); } -//static void mgmtProcessVnodeCfgMsg(int8_t msgType, int8_t *pCont, int32_t contLen, void *pConn) { -// if (!sdbMaster) { -// mgmtSendRspToDnode(pConn, msgType + 1, TSDB_CODE_REDIRECT, NULL, 0); -// return; -// } -// -// SDMConfigVnodeMsg *pCfg = (SDMConfigVnodeMsg *) pCont; -// pCfg->dnode = htonl(pCfg->dnode); -// pCfg->vnode = htonl(pCfg->vnode); -// -// SVgObj *pVgroup = mgmtGetVgroupByVnode(pCfg->dnode, pCfg->vnode); -// if (pVgroup == NULL) { -// mTrace("dnode:%s, vnode:%d, no vgroup info", taosIpStr(pCfg->dnode), pCfg->vnode); -// mgmtSendRspToDnode(pConn, msgType + 1, TSDB_CODE_NOT_ACTIVE_VNODE, NULL, 0); -// return; -// } -// -// mgmtSendRspToDnode(pConn, msgType + 1, TSDB_CODE_SUCCESS, NULL, 0); -// -// SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); -// mgmtSendCreateVnodeMsg(pVgroup, pCfg->vnode, &ipSet, NULL); -//} -// \ No newline at end of file +void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle) { + assert(ahandle != NULL); + + if (pVgroup->numOfVnodes != pVgroup->pDb->cfg.replications) { + // TODO: + // mgmtSendAlterVgroupMsg(pVgroup, NULL); + } else { + mgmtAddToShellQueue(ahandle); + } +} \ No newline at end of file diff --git a/src/util/inc/tidpool.h b/src/util/inc/tidpool.h index 08f9d36467..bf35251631 100644 --- a/src/util/inc/tidpool.h +++ b/src/util/inc/tidpool.h @@ -34,11 +34,7 @@ void taosIdPoolCleanUp(void *handle); int taosIdPoolNumOfUsed(void *handle); -void taosIdPoolReinit(void *handle); - -void taosIdPoolMarkStatus(void *handle, int id, int status); - -void taosIdPoolSetFreeList(void *handle); +void taosIdPoolMarkStatus(void *handle, int id); #ifdef __cplusplus } diff --git a/src/util/src/tidpool.c b/src/util/src/tidpool.c index c50c38aa3c..c3f429ac43 100644 --- a/src/util/src/tidpool.c +++ b/src/util/src/tidpool.c @@ -25,78 +25,74 @@ typedef struct { } id_pool_t; void *taosInitIdPool(int maxId) { - id_pool_t *pIdPool; - int * idList, i; - - if (maxId < 3) maxId = 3; - - pIdPool = (id_pool_t *)malloc(sizeof(id_pool_t)); + id_pool_t *pIdPool = calloc(1, sizeof(id_pool_t)); if (pIdPool == NULL) return NULL; - idList = (int *)malloc(sizeof(int) * (size_t)maxId); - if (idList == NULL) { + pIdPool->freeList = malloc(sizeof(int) * (size_t)maxId); + if (pIdPool->freeList == NULL) { free(pIdPool); return NULL; } - memset(pIdPool, 0, sizeof(id_pool_t)); pIdPool->maxId = maxId; - pIdPool->numOfFree = maxId - 1; + pIdPool->numOfFree = maxId; pIdPool->freeSlot = 0; - pIdPool->freeList = idList; pthread_mutex_init(&pIdPool->mutex, NULL); - for (i = 1; i < maxId; ++i) idList[i - 1] = i; + for (int i = 0; i < maxId; ++i) { + pIdPool->freeList[i] = -1; + } pTrace("pool:%p is setup, maxId:%d", pIdPool, pIdPool->maxId); - return (void *)pIdPool; + return pIdPool; } int taosAllocateId(void *handle) { - id_pool_t *pIdPool; - int id = -1; - if (handle == NULL) return id; - - pIdPool = (id_pool_t *)handle; - - if (pIdPool->maxId < 3) pError("pool:%p is messed up, maxId:%d", pIdPool, pIdPool->maxId); - - if (pthread_mutex_lock(&pIdPool->mutex) != 0) perror("lock pIdPool Mutex"); - - if (pIdPool->numOfFree > 0) { - id = pIdPool->freeList[pIdPool->freeSlot]; - pIdPool->freeSlot = (pIdPool->freeSlot + 1) % pIdPool->maxId; - pIdPool->numOfFree--; + id_pool_t *pIdPool = handle; + if (handle == NULL) { + return -1; } - if (pthread_mutex_unlock(&pIdPool->mutex) != 0) perror("unlock pIdPool Mutex"); + int slot = -1; + pthread_mutex_lock(&pIdPool->mutex); - return id; + if (pIdPool->numOfFree > 0) { + for (int i = 0; i < pIdPool->maxId; ++i) { + slot = (i + pIdPool->freeSlot) % pIdPool->maxId; + if (pIdPool->freeList[slot] == -1) { + pIdPool->freeList[slot] = slot; + pIdPool->freeSlot = slot + 1; + pIdPool->numOfFree--; + break; + } + } + } + + pthread_mutex_unlock(&pIdPool->mutex); + return slot; } void taosFreeId(void *handle, int id) { - id_pool_t *pIdPool; - int slot; + id_pool_t *pIdPool = handle; + if (handle == NULL) return; - pIdPool = (id_pool_t *)handle; - if (pIdPool->freeList == NULL || pIdPool->maxId == 0) return; - if (id <= 0 || id >= pIdPool->maxId) return; - if (pthread_mutex_lock(&pIdPool->mutex) != 0) perror("lock pIdPool Mutex"); + pthread_mutex_lock(&pIdPool->mutex); - slot = (pIdPool->freeSlot + pIdPool->numOfFree) % pIdPool->maxId; - pIdPool->freeList[slot] = id; - pIdPool->numOfFree++; + int slot = id % pIdPool->maxId; + if (pIdPool->freeList[slot] != -1) { + pIdPool->freeList[slot] = -1; + pIdPool->numOfFree++; + } - if (pthread_mutex_unlock(&pIdPool->mutex) != 0) perror("unlock pIdPool Mutex"); + pthread_mutex_unlock(&pIdPool->mutex); } void taosIdPoolCleanUp(void *handle) { - id_pool_t *pIdPool; + id_pool_t *pIdPool = handle; - if (handle == NULL) return; - pIdPool = (id_pool_t *)handle; + if (pIdPool == NULL) return; pTrace("pool:%p is cleaned", pIdPool); @@ -110,42 +106,21 @@ void taosIdPoolCleanUp(void *handle) { } int taosIdPoolNumOfUsed(void *handle) { - id_pool_t *pIdPool = (id_pool_t *)handle; - - return pIdPool->maxId - pIdPool->numOfFree - 1; + id_pool_t *pIdPool = handle; + return pIdPool->maxId - pIdPool->numOfFree; } -void taosIdPoolReinit(void *handle) { - id_pool_t *pIdPool; +void taosIdPoolMarkStatus(void *handle, int id) { + id_pool_t *pIdPool = handle; + pthread_mutex_lock(&pIdPool->mutex); - pIdPool = (id_pool_t *)handle; - pIdPool->numOfFree = 0; - pIdPool->freeSlot = 0; - - for (int i = 0; i < pIdPool->maxId; ++i) pIdPool->freeList[i] = 0; -} - -void taosIdPoolMarkStatus(void *handle, int id, int status) { - id_pool_t *pIdPool = (id_pool_t *)handle; - - pIdPool->freeList[id] = status; -} - -void taosIdPoolSetFreeList(void *handle) { - id_pool_t *pIdPool; - int pos = 0; - - pIdPool = (id_pool_t *)handle; - pIdPool->numOfFree = 0; - pIdPool->freeSlot = 0; - - for (int i = 1; i < pIdPool->maxId; ++i) { - if (pIdPool->freeList[i] == 0) { - pIdPool->freeList[pos] = i; - pIdPool->numOfFree++; - pos++; - } + int slot = id % pIdPool->maxId; + if (pIdPool->freeList[slot] == -1) { + pIdPool->freeList[slot] = slot; + pIdPool->numOfFree--; } + + pthread_mutex_unlock(&pIdPool->mutex); } int taosUpdateIdPool(id_pool_t *handle, int maxId) { @@ -154,18 +129,18 @@ int taosUpdateIdPool(id_pool_t *handle, int maxId) { return -1; } - int *idList, i; - idList = (int *)malloc(sizeof(int) * (size_t)maxId); + int *idList = malloc(sizeof(int) * maxId); if (idList == NULL) { return -1; } - for (i = 1; i < maxId; ++i) { - idList[i - 1] = i; + + pthread_mutex_lock(&pIdPool->mutex); + + for (int i = 0; i < maxId; ++i) { + idList[i] = -1; } - if (pthread_mutex_lock(&pIdPool->mutex) != 0) perror("lock pIdPool Mutex"); - - memcpy(idList, pIdPool->freeList, sizeof(int) * (size_t)pIdPool->maxId); + memcpy(idList, pIdPool->freeList, sizeof(int) * pIdPool->maxId); pIdPool->numOfFree += (maxId - pIdPool->maxId); pIdPool->maxId = maxId; @@ -173,7 +148,7 @@ int taosUpdateIdPool(id_pool_t *handle, int maxId) { pIdPool->freeList = idList; free(oldIdList); - if (pthread_mutex_unlock(&pIdPool->mutex) != 0) perror("unlock pIdPool Mutex"); + pthread_mutex_unlock(&pIdPool->mutex); return 0; } From 51171c75de64ec31a2185c9547360a28d45ce373 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 23 Mar 2020 22:58:09 +0800 Subject: [PATCH 10/12] [TD-15] --- src/mnode/inc/mgmtChildTable.h | 1 + src/mnode/src/mgmtChildTable.c | 123 +++++++++++++++++------ src/mnode/src/mgmtDnode.c | 3 - src/mnode/src/mgmtNormalTable.c | 115 ++++++++++++++------- src/mnode/src/mgmtSuperTable.c | 170 +++++++++++++++----------------- src/mnode/src/mgmtTable.c | 43 ++++---- 6 files changed, 272 insertions(+), 183 deletions(-) diff --git a/src/mnode/inc/mgmtChildTable.h b/src/mnode/inc/mgmtChildTable.h index b16dd58f67..9252a7d485 100644 --- a/src/mnode/inc/mgmtChildTable.h +++ b/src/mnode/inc/mgmtChildTable.h @@ -39,6 +39,7 @@ int32_t mgmtModifyChildTableTagValueByName(SChildTableObj *pTable, char *tagName int32_t mgmtGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp); void mgmtDropAllChildTables(SDbObj *pDropDb); +void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable); #ifdef __cplusplus } diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index 9d3c3c634d..d717a4e5e9 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -42,14 +42,14 @@ static void mgmtDestroyChildTable(SChildTableObj *pTable) { tfree(pTable); } -static int32_t mgmtChildTableActionDestroy(void *pObj) { - SChildTableObj *pTable = (SChildTableObj *)pObj; +static int32_t mgmtChildTableActionDestroy(SSdbOperDesc *pOper) { + SChildTableObj *pTable = pOper->pObj; mgmtDestroyChildTable(pTable); return TSDB_CODE_SUCCESS; } -static int32_t mgmtChildTableActionInsert(void *pObj) { - SChildTableObj *pTable = (SChildTableObj *) pObj; +static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) { + SChildTableObj *pTable = pOper->pObj; SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { @@ -84,15 +84,15 @@ static int32_t mgmtChildTableActionInsert(void *pObj) { mgmtAddTableIntoDb(pDb); mgmtAddTableIntoVgroup(pVgroup, (STableInfo *) pTable); - if (pVgroup->numOfTables >= pDb->cfg.maxSessions - 1 && pDb->numOfVgroups > 1) { - mgmtMoveVgroupToTail(pDb, pVgroup); + if (pVgroup->numOfTables >= pDb->cfg.maxSessions && pDb->numOfVgroups > 1) { + mgmtMoveVgroupToTail(pVgroup); } return TSDB_CODE_SUCCESS; } -static int32_t mgmtChildTableActionDelete(void *pObj) { - SChildTableObj *pTable = (SChildTableObj *) pObj; +static int32_t mgmtChildTableActionDelete(SSdbOperDesc *pOper) { + SChildTableObj *pTable = pOper->pObj; if (pTable->vgId == 0) { return TSDB_CODE_INVALID_VGROUP_ID; } @@ -121,33 +121,35 @@ static int32_t mgmtChildTableActionDelete(void *pObj) { mgmtRemoveTableFromSuperTable(pTable->superTable); if (pVgroup->numOfTables > 0) { - mgmtMoveVgroupToHead(pDb, pVgroup); + mgmtMoveVgroupToHead(pVgroup); } return TSDB_CODE_SUCCESS; } -static int32_t mgmtChildTableActionUpdate(void *pObj) { +static int32_t mgmtChildTableActionUpdate(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mgmtChildTableActionEncode(void *pObj, void *pData, int32_t maxRowSize) { - SChildTableObj *pTable = (SChildTableObj *) pObj; - assert(pObj != NULL && pData != NULL); +static int32_t mgmtChildTableActionEncode(SSdbOperDesc *pOper) { + SChildTableObj *pTable = pOper->pObj; + assert(pTable != NULL && pOper->rowData != NULL); - memcpy(pData, pTable, tsChildTableUpdateSize); + memcpy(pOper->rowData, pTable, tsChildTableUpdateSize); return tsChildTableUpdateSize; } -static void *mgmtChildTableActionDecode(void *pData) { - assert(pData != NULL); +static int32_t mgmtChildTableActionDecode(SSdbOperDesc *pOper) { + assert(pOper->rowData != NULL); - SChildTableObj *pTable = (SChildTableObj *)calloc(sizeof(SChildTableObj), 1); - if (pTable == NULL) return NULL; + pOper->pObj = calloc(1, sizeof(SChildTableObj)); + if (pOper->pObj == NULL) { + return TSDB_CODE_SERV_OUT_OF_MEMORY; + } - memcpy(pTable, pData, tsChildTableUpdateSize); + memcpy(pOper->pObj, pOper->rowData, tsChildTableUpdateSize); - return (void *)pTable; + return TSDB_CODE_SUCCESS; } int32_t mgmtInitChildTables() { @@ -162,7 +164,7 @@ int32_t mgmtInitChildTables() { .tableName = "ctables", .hashSessions = tsMaxTables, .maxRowSize = tsChildTableUpdateSize, - .keyType = SDB_KEYTYPE_STRING, + .keyType = SDB_KEY_TYPE_STRING, .insertFp = mgmtChildTableActionInsert, .deleteFp = mgmtChildTableActionDelete, .updateFp = mgmtChildTableActionUpdate, @@ -187,7 +189,11 @@ int32_t mgmtInitChildTables() { SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); if (pDb == NULL) { mError("ctable:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_DISK); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -196,7 +202,11 @@ int32_t mgmtInitChildTables() { if (pVgroup == NULL) { mError("ctable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->tableId, pTable->vgId, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_DISK); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -205,7 +215,11 @@ int32_t mgmtInitChildTables() { mError("ctable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it", pTable->tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_DISK); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -213,19 +227,27 @@ int32_t mgmtInitChildTables() { if (pVgroup->tableList == NULL) { mError("ctable:%s, vgroup:%d tableList is null", pTable->tableId, pTable->vgId); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_DISK); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } pVgroup->tableList[pTable->sid] = (STableInfo*)pTable; - taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid, 1); + taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTable->superTableId); if (pSuperTable == NULL) { mError("ctable:%s, stable:%s not exist", pTable->tableId, pTable->superTableId); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_DISK); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -310,7 +332,13 @@ void* mgmtCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t pTable->vgId = pVgroup->vgId; pTable->superTable = pSuperTable; - if (sdbInsertRow(tsChildTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbInsertRow(&desc); + + if (sdbInsertRow(&desc) < 0) { free(pTable); mError("ctable:%s, update sdb error", pCreate->tableId); terrno = TSDB_CODE_SDB_ERROR; @@ -453,12 +481,45 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) { } if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_LOCAL); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsChildTableSdb, + .pObj = pTable, + }; + sdbDeleteRow(&oper); pNode = pLastNode; - numOfTables ++; + numOfTables++; continue; } } - mTrace("db:%s, all child tables:%d is dropped", pDropDb->name, numOfTables); + mTrace("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables); +} + +void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) { + void *pNode = NULL; + void *pLastNode = NULL; + int32_t numOfTables = 0; + SChildTableObj *pTable = NULL; + + while (1) { + pNode = sdbFetchRow(tsChildTableSdb, pNode, (void **)&pTable); + if (pTable == NULL) { + break; + } + + if (pTable->superTable == pStable) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsChildTableSdb, + .pObj = pTable, + }; + sdbDeleteRow(&oper); + pNode = pLastNode; + numOfTables++; + continue; + } + } + + mTrace("stable:%s, all child tables:%d is dropped from sdb", pStable->tableId, numOfTables); } \ No newline at end of file diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index 0d83e262ae..0aab083471 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -547,9 +547,6 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { return ; } - uint32_t lastPrivateIp = pDnode->privateIp; - uint32_t lastPublicIp = pDnode->publicIp; - pDnode->privateIp = htonl(pStatus->privateIp); pDnode->publicIp = htonl(pStatus->publicIp); pDnode->lastReboot = htonl(pStatus->lastReboot); diff --git a/src/mnode/src/mgmtNormalTable.c b/src/mnode/src/mgmtNormalTable.c index 9140a4c752..0ffd4fb28e 100644 --- a/src/mnode/src/mgmtNormalTable.c +++ b/src/mnode/src/mgmtNormalTable.c @@ -42,14 +42,14 @@ static void mgmtDestroyNormalTable(SNormalTableObj *pTable) { tfree(pTable); } -static int32_t mgmtNormalTableActionDestroy(void *pObj) { - SNormalTableObj *pTable = (SNormalTableObj *)pObj; +static int32_t mgmtNormalTableActionDestroy(SSdbOperDesc *pOper) { + SNormalTableObj *pTable = pOper->pObj; mgmtDestroyNormalTable(pTable); return TSDB_CODE_SUCCESS; } -static int32_t mgmtNormalTableActionInsert(void *pObj) { - SNormalTableObj *pTable = (SNormalTableObj *) pObj; +static int32_t mgmtNormalTableActionInsert(SSdbOperDesc *pOper) { + SNormalTableObj *pTable = pOper->pObj; SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { @@ -82,14 +82,14 @@ static int32_t mgmtNormalTableActionInsert(void *pObj) { mgmtAddTableIntoVgroup(pVgroup, (STableInfo *) pTable); if (pVgroup->numOfTables >= pDb->cfg.maxSessions - 1 && pDb->numOfVgroups > 1) { - mgmtMoveVgroupToTail(pDb, pVgroup); + mgmtMoveVgroupToTail(pVgroup); } return TSDB_CODE_SUCCESS; } -static int32_t mgmtNormalTableActionDelete(void *pObj) { - SNormalTableObj *pTable = (SNormalTableObj *) pObj; +static int32_t mgmtNormalTableActionDelete(SSdbOperDesc *pOper) { + SNormalTableObj *pTable = pOper->pObj; if (pTable->vgId == 0) { return TSDB_CODE_INVALID_VGROUP_ID; } @@ -116,13 +116,13 @@ static int32_t mgmtNormalTableActionDelete(void *pObj) { mgmtRemoveTableFromVgroup(pVgroup, (STableInfo *) pTable); if (pVgroup->numOfTables > 0) { - mgmtMoveVgroupToHead(pDb, pVgroup); + mgmtMoveVgroupToHead(pVgroup); } return TSDB_CODE_SUCCESS; } -static int32_t mgmtNormalTableActionUpdate(void *pObj) { +static int32_t mgmtNormalTableActionUpdate(SSdbOperDesc *pOper) { // SNormalTableObj *pTable = (SNormalTableObj *) pObj; // memcpy(pTable, str, tsNormalTableUpdateSize); @@ -134,48 +134,49 @@ static int32_t mgmtNormalTableActionUpdate(void *pObj) { return TSDB_CODE_SUCCESS; } -static int32_t mgmtNormalTableActionEncode(void *pObj, void *pData, int32_t maxRowSize) { - SNormalTableObj *pTable = (SNormalTableObj *) pObj; - assert(pObj != NULL && pData != NULL); +static int32_t mgmtNormalTableActionEncode(SSdbOperDesc *pOper) { + SNormalTableObj *pTable = pOper->pObj; + assert(pOper->pObj != NULL && pOper->rowData != NULL); int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - if (maxRowSize < tsNormalTableUpdateSize + schemaSize + 1) { + if (pOper->maxRowSize < tsNormalTableUpdateSize + schemaSize + 1) { return -1; } - memcpy(pData, pTable, tsNormalTableUpdateSize); - memcpy(pData + tsNormalTableUpdateSize, pTable->schema, schemaSize); - memcpy(pData + tsNormalTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen); + memcpy(pOper->rowData, pTable, tsNormalTableUpdateSize); + memcpy(pOper->rowData + tsNormalTableUpdateSize, pTable->schema, schemaSize); + memcpy(pOper->rowData + tsNormalTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen); return tsNormalTableUpdateSize + schemaSize + pTable->sqlLen; } -static void *mgmtNormalTableActionDecode(void *pData) { - assert(pData != NULL); +static int32_t mgmtNormalTableActionDecode(SSdbOperDesc *pOper) { + assert(pOper->rowData != NULL); SNormalTableObj *pTable = (SNormalTableObj *)malloc(sizeof(SNormalTableObj)); if (pTable == NULL) { - return NULL; + return -1; } memset(pTable, 0, sizeof(SNormalTableObj)); - memcpy(pTable, pData, tsNormalTableUpdateSize); + memcpy(pTable, pOper->rowData, tsNormalTableUpdateSize); int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); pTable->schema = (SSchema *)malloc(schemaSize); if (pTable->schema == NULL) { mgmtDestroyNormalTable(pTable); - return NULL; + return -1; } - memcpy(pTable->schema, pData + tsNormalTableUpdateSize, schemaSize); + memcpy(pTable->schema, pOper->rowData + tsNormalTableUpdateSize, schemaSize); pTable->sql = (char *)malloc(pTable->sqlLen); if (pTable->sql == NULL) { mgmtDestroyNormalTable(pTable); - return NULL; + return -1; } - memcpy(pTable->sql, pData + tsNormalTableUpdateSize + schemaSize, pTable->sqlLen); - return (void *)pTable; + memcpy(pTable->sql, pOper->rowData + tsNormalTableUpdateSize + schemaSize, pTable->sqlLen); + + return TSDB_CODE_SUCCESS; } int32_t mgmtInitNormalTables() { @@ -190,7 +191,7 @@ int32_t mgmtInitNormalTables() { .tableName = "ntables", .hashSessions = TSDB_MAX_NORMAL_TABLES, .maxRowSize = sizeof(SNormalTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, - .keyType = SDB_KEYTYPE_STRING, + .keyType = SDB_KEY_TYPE_STRING, .insertFp = mgmtNormalTableActionInsert, .deleteFp = mgmtNormalTableActionDelete, .updateFp = mgmtNormalTableActionUpdate, @@ -213,7 +214,11 @@ int32_t mgmtInitNormalTables() { SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); if (pDb == NULL) { mError("ntable:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_DISK); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -222,7 +227,11 @@ int32_t mgmtInitNormalTables() { if (pVgroup == NULL) { mError("ntable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->tableId, pTable->vgId, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_DISK); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -231,7 +240,12 @@ int32_t mgmtInitNormalTables() { mError("ntable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it", pTable->tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_DISK); + + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -239,14 +253,18 @@ int32_t mgmtInitNormalTables() { if (pVgroup->tableList == NULL) { mError("ntable:%s, vgroup:%d tableList is null", pTable->tableId, pTable->vgId); pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_DISK); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } mgmtAddTableIntoVgroup(pVgroup, (STableInfo *)pTable); //pVgroup->tableList[pTable->sid] = (STableInfo*)pTable; - taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid, 1); + taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); pTable->sql = (char *)pTable->schema + sizeof(SSchema) * pTable->numOfColumns; @@ -346,7 +364,11 @@ void *mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t mTrace("table:%s, stream sql len:%d sql:%s", pTable->tableId, pTable->sqlLen, pTable->sql); } - if (sdbInsertRow(tsNormalTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + if (sdbInsertRow(&desc) < 0) { mError("table:%s, update sdb error", pTable->tableId); free(pTable); terrno = TSDB_CODE_SDB_ERROR; @@ -443,7 +465,14 @@ int32_t mgmtAddNormalTableColumn(SNormalTableObj *pTable, SSchema schema[], int3 pTable->sversion++; pAcct->acctInfo.numOfTimeSeries += ncols; - sdbUpdateRow(tsNormalTableSdb, pTable, tsNormalTableUpdateSize, SDB_OPER_GLOBAL); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + desc.rowData = pTable; + desc.rowSize = tsNormalTableUpdateSize; + sdbUpdateRow(&desc); + return TSDB_CODE_SUCCESS; } @@ -472,7 +501,14 @@ int32_t mgmtDropNormalTableColumnByName(SNormalTableObj *pTable, char *colName) pTable->sversion++; pAcct->acctInfo.numOfTimeSeries--; - sdbUpdateRow(tsNormalTableSdb, pTable, tsNormalTableUpdateSize, SDB_OPER_GLOBAL); + + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + desc.rowData = pTable; + desc.rowSize = tsNormalTableUpdateSize; + sdbUpdateRow(&desc); return TSDB_CODE_SUCCESS; } @@ -533,12 +569,17 @@ void mgmtDropAllNormalTables(SDbObj *pDropDb) { if (pTable == NULL) break; if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_LOCAL); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsNormalTableSdb, + .pObj = pTable, + }; + sdbDeleteRow(&oper); pNode = pLastNode; - numOfTables ++; + numOfTables++; continue; } } - mTrace("db:%s, all normal tables:%d is dropped", pDropDb->name, numOfTables); + mTrace("db:%s, all normal tables:%d is dropped from sdb", pDropDb->name, numOfTables); } diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index a4949a74c4..d3a0dcb6bc 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -32,93 +32,84 @@ #include "name.h" #include "tsqlfunction.h" -static void *tsSuperTableSdb; +static void *tsSuperTableSdb; static int32_t tsSuperTableUpdateSize; static int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mgmtGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static void mgmtDestroySuperTable(SSuperTableObj *pTable) { - tfree(pTable->schema); - tfree(pTable); +static void mgmtDestroySuperTable(SSuperTableObj *pStable) { + tfree(pStable->schema); + tfree(pStable); } -static int32_t mgmtSuperTableActionDestroy(void *pObj) { - SSuperTableObj *pTable = (SSuperTableObj *) pObj; - mgmtDestroySuperTable(pTable); +static int32_t mgmtSuperTableActionDestroy(SSdbOperDesc *pOper) { + mgmtDestroySuperTable(pOper->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mgmtSuperTableActionInsert(void *pObj) { - STableInfo *pTable = (STableInfo *) pObj; - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb) { +static int32_t mgmtSuperTableActionInsert(SSdbOperDesc *pOper) { + STableInfo *pStable = pOper->pObj; + SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId); + if (pDb != NULL) { mgmtAddSuperTableIntoDb(pDb); } return TSDB_CODE_SUCCESS; } -static int32_t mgmtSuperTableActionDelete(void *pObj) { - STableInfo *pTable = (STableInfo *) pObj; - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb) { +static int32_t mgmtSuperTableActionDelete(SSdbOperDesc *pOper) { + STableInfo *pStable = pOper->pObj; + SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId); + if (pDb != NULL) { mgmtRemoveSuperTableFromDb(pDb); + mgmtDropAllChildTablesInStable((SSuperTableObj *)pStable); } return TSDB_CODE_SUCCESS; } -static int32_t mgmtSuperTableActionUpdate(void *pObj) { - SSuperTableObj *pTable = (SSuperTableObj *) pObj; - memcpy(pTable, pObj, tsSuperTableUpdateSize); - - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); - pTable->schema = realloc(pTable->schema, schemaSize); - memcpy(pTable->schema, pObj + tsSuperTableUpdateSize, schemaSize); - +static int32_t mgmtSuperTableActionUpdate(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mgmtSuperTableActionEncode(void *pObj, void *pData, int32_t maxRowSize) { - SSuperTableObj *pTable = (SSuperTableObj *) pObj; - assert(pObj != NULL && pData != NULL); +static int32_t mgmtSuperTableActionEncode(SSdbOperDesc *pOper) { + SSuperTableObj *pStable = pOper->pObj; + assert(pOper->pObj != NULL && pOper->rowData != NULL); - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); + int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); - if (maxRowSize < tsSuperTableUpdateSize + schemaSize + 1) { + if (pOper->maxRowSize < tsSuperTableUpdateSize + schemaSize) { return TSDB_CODE_INVALID_MSG_LEN; } - memcpy(pData, pTable, tsSuperTableUpdateSize); - memcpy(pData + tsSuperTableUpdateSize, pTable->schema, schemaSize); - return tsSuperTableUpdateSize + schemaSize; + memcpy(pOper->rowData, pStable, tsSuperTableUpdateSize); + memcpy(pOper->rowData + tsSuperTableUpdateSize, pStable->schema, schemaSize); + pOper->rowSize = tsSuperTableUpdateSize + schemaSize; + + return TSDB_CODE_SUCCESS; } -static void *mgmtSuperTableActionDecode(void *pData) { - assert(pData != NULL); +static int32_t mgmtSuperTableActionDecode(SSdbOperDesc *pOper) { + assert(pOper->rowData != NULL); - SSuperTableObj *pTable = (SSuperTableObj *) malloc(sizeof(SSuperTableObj)); - if (pTable == NULL) { - return NULL; - } - memset(pTable, 0, sizeof(SSuperTableObj)); - memcpy(pTable, pData, tsSuperTableUpdateSize); + SSuperTableObj *pStable = (SSuperTableObj *) calloc(1, sizeof(SSuperTableObj)); + if (pStable == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); - pTable->schema = malloc(schemaSize); - if (pTable->schema == NULL) { - mgmtDestroySuperTable(pTable); - return NULL; + memcpy(pStable, pOper->rowData, tsSuperTableUpdateSize); + + int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); + pStable->schema = malloc(schemaSize); + if (pStable->schema == NULL) { + mgmtDestroySuperTable(pStable); + return -1; } - memcpy(pTable->schema, pData + tsSuperTableUpdateSize, schemaSize); - return (void *) pTable; + memcpy(pStable->schema, pOper->rowData + tsSuperTableUpdateSize, schemaSize); + pOper->pObj = pStable; + + return TSDB_CODE_SUCCESS; } int32_t mgmtInitSuperTables() { - void *pNode = NULL; - void *pLastNode = NULL; - SSuperTableObj *pTable = NULL; - SSuperTableObj tObj; tsSuperTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; @@ -126,7 +117,7 @@ int32_t mgmtInitSuperTables() { .tableName = "stables", .hashSessions = TSDB_MAX_SUPER_TABLES, .maxRowSize = tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, - .keyType = SDB_KEYTYPE_STRING, + .keyType = SDB_KEY_TYPE_STRING, .insertFp = mgmtSuperTableActionInsert, .deleteFp = mgmtSuperTableActionDelete, .updateFp = mgmtSuperTableActionUpdate, @@ -141,24 +132,6 @@ int32_t mgmtInitSuperTables() { return -1; } - pNode = NULL; - while (1) { - pNode = sdbFetchRow(tsSuperTableSdb, pNode, (void **) &pTable); - if (pTable == NULL) { - break; - } - - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb == NULL) { - mError("super table:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsSuperTableSdb, pTable, SDB_OPER_DISK); - pNode = pLastNode; - continue; - } - - mgmtAddSuperTableIntoDb(pDb); - } - mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_METRIC, mgmtGetShowSuperTableMeta); mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_METRIC, mgmtRetrieveShowSuperTables); @@ -171,7 +144,7 @@ void mgmtCleanUpSuperTables() { } int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { - SSuperTableObj *pStable = (SSuperTableObj *)calloc(sizeof(SSuperTableObj), 1); + SSuperTableObj *pStable = (SSuperTableObj *)calloc(1, sizeof(SSuperTableObj)); if (pStable == NULL) { return TSDB_CODE_SERV_OUT_OF_MEMORY; } @@ -203,13 +176,21 @@ int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { tschema[col].bytes = htons(tschema[col].bytes); } - if (sdbInsertRow(tsSuperTableSdb, pStable, SDB_OPER_GLOBAL) < 0) { - mError("stable:%s, update sdb error", pStable->tableId); - return TSDB_CODE_SDB_ERROR; - } + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsSuperTableSdb, + .pObj = pStable, + .rowSize = sizeof(SSuperTableObj) + schemaSize + }; - mPrint("stable:%s, is created, tags:%d cols:%d", pStable->tableId, pStable->numOfTags, pStable->numOfColumns); - return TSDB_CODE_SUCCESS; + int32_t code = sdbInsertRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + mgmtDestroySuperTable(pStable); + return TSDB_CODE_SDB_ERROR; + } else { + mLPrint("stable:%s, is created, tags:%d cols:%d", pStable->tableId, pStable->numOfTags, pStable->numOfColumns); + return TSDB_CODE_SUCCESS; + } } int32_t mgmtDropSuperTable(SQueuedMsg *newMsg, SDbObj *pDb, SSuperTableObj *pStable) { @@ -217,10 +198,14 @@ int32_t mgmtDropSuperTable(SQueuedMsg *newMsg, SDbObj *pDb, SSuperTableObj *pSta mError("stable:%s, numOfTables:%d not 0", pStable->tableId, pStable->numOfTables); return TSDB_CODE_OTHERS; } else { - //TODO: drop child tables - mError("stable:%s, is dropped from sdb", pStable->tableId); - mgmtRemoveSuperTableFromDb(pDb); - return TSDB_CODE_OTHERS; + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsSuperTableSdb, + .pObj = pStable + }; + int32_t code = sdbDeleteRow(&oper); + mLPrint("stable:%s, is dropped from sdb, result:%s", pStable->tableId, tstrerror(code)); + return code; } } @@ -229,10 +214,9 @@ void* mgmtGetSuperTable(char *tableId) { } void *mgmtGetSuperTableVgroup(SSuperTableObj *pStable) { - //TODO get vgroup of dnodes SCMSuperTableInfoRsp *rsp = rpcMallocCont(sizeof(SCMSuperTableInfoRsp) + sizeof(uint32_t) * mgmtGetDnodesNum()); - rsp->numOfDnodes = 1; - rsp->dnodeIps[0] = 0; + rsp->numOfDnodes = htonl(1); + rsp->dnodeIps[0] = htonl(inet_addr(tsPrivateIp)); return rsp; } @@ -289,7 +273,7 @@ int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t pStable->sversion++; pAcct->acctInfo.numOfTimeSeries += (ntags * pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); + // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); mTrace("Succeed to add tag column %s to table %s", schema[0].name, pStable->tableId); return TSDB_CODE_SUCCESS; @@ -322,7 +306,7 @@ int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns); pStable->schema = realloc(pStable->schema, schemaSize); - sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); + // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -354,7 +338,8 @@ int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char *oldTagN // mgmtSuperTableActionEncode(pStable, msg, size, &rowSize); - int32_t ret = sdbUpdateRow(tsSuperTableSdb, msg, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); + int32_t ret = 0; + // int32_t ret = sdbUpdateRow(tsSuperTableSdb, msg, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); tfree(msg); if (ret < 0) { @@ -416,7 +401,7 @@ int32_t mgmtAddSuperTableColumn(SSuperTableObj *pStable, SSchema schema[], int32 pStable->sversion++; pAcct->acctInfo.numOfTimeSeries += (ncols * pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); + // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -449,7 +434,7 @@ int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pStable, char *colName) { pStable->schema = realloc(pStable->schema, schemaSize); pAcct->acctInfo.numOfTimeSeries -= (pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); + // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -588,14 +573,19 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) { } if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsSuperTableSdb, pTable, SDB_OPER_GLOBAL); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsSuperTableSdb, + .pObj = pTable, + }; + sdbDeleteRow(&oper); pNode = pLastNode; numOfTables ++; continue; } } - mTrace("db:%s, all super tables:%d is dropped", pDropDb->name, numOfTables); + mTrace("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables); } void mgmtAddTableIntoSuperTable(SSuperTableObj *pStable) { diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 255b2a6f27..918547fa12 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -78,8 +78,6 @@ int32_t mgmtInitTables() { return code; } - mgmtSetVgroupIdPool(); - mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_TABLE, mgmtProcessCreateTableMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_TABLE, mgmtProcessDropTableMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_ALTER_TABLE, mgmtProcessAlterTableMsg); @@ -202,13 +200,13 @@ int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "table_name"); + strcpy(pSchema[cols].name, "table name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "created_time"); + strcpy(pSchema[cols].name, "create time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -220,7 +218,7 @@ int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "stable_name"); + strcpy(pSchema[cols].name, "stable name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -238,9 +236,6 @@ int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) return 0; } -/* - * remove the hole in result set - */ static void mgmtVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) { if (rows < capacity) { for (int32_t i = 0; i < numOfCols; ++i) { @@ -434,11 +429,13 @@ void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { pTable = mgmtCreateChildTable(pCreate, pVgroup, sid); if (pTable == NULL) { mgmtSendSimpleResp(pMsg->thandle, terrno); + mgmtFreeQueuedMsg(newMsg); return; } pMDCreate = mgmtBuildCreateChildTableMsg(pCreate, (SChildTableObj *) pTable); if (pMDCreate == NULL) { mgmtSendSimpleResp(pMsg->thandle, terrno); + mgmtFreeQueuedMsg(newMsg); return; } } else { @@ -446,11 +443,13 @@ void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { pTable = mgmtCreateNormalTable(pCreate, pVgroup, sid); if (pTable == NULL) { mgmtSendSimpleResp(pMsg->thandle, terrno); + mgmtFreeQueuedMsg(newMsg); return; } pMDCreate = mgmtBuildCreateNormalTableMsg((SNormalTableObj *) pTable); if (pMDCreate == NULL) { mgmtSendSimpleResp(pMsg->thandle, terrno); + mgmtFreeQueuedMsg(newMsg); return; } } @@ -761,9 +760,9 @@ static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg) { if (rpcMsg->code != TSDB_CODE_SUCCESS) { if (pTable->type == TSDB_CHILD_TABLE) { - sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_GLOBAL); + // sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_GLOBAL); } else if (pTable->type == TSDB_NORMAL_TABLE){ - sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_GLOBAL); + // sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_GLOBAL); } else {} mError("table:%s, failed to create in dnode, reason:%s", pTable->tableId, tstrerror(rpcMsg->code)); mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); @@ -817,19 +816,19 @@ static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg) { } if (pTable->type == TSDB_CHILD_TABLE) { - if (sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { - mError("table:%s, update ctables sdb error", pTable->tableId); - mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); - free(queueMsg); - return; - } + // if (sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { + // mError("table:%s, update ctables sdb error", pTable->tableId); + // mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); + // free(queueMsg); + // return; + // } } else if (pTable->type == TSDB_NORMAL_TABLE){ - if (sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { - mError("table:%s, update ntables sdb error", pTable->tableId); - mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); - free(queueMsg); - return; - } + // if (sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { + // mError("table:%s, update ntables sdb error", pTable->tableId); + // mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); + // free(queueMsg); + // return; + // } } if (pVgroup->numOfTables <= 0) { From d2e3ed8b7ddd7d6a3f6932c30584931a638284db Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 24 Mar 2020 12:49:22 +0800 Subject: [PATCH 11/12] [TD-15] refact code of sdb --- src/inc/mnode.h | 20 ++--- src/mnode/src/mgmtChildTable.c | 49 ++++-------- src/mnode/src/mgmtDb.c | 4 +- src/mnode/src/mgmtNormalTable.c | 66 ++++------------- src/mnode/src/mgmtSdb.c | 39 +++++----- src/mnode/src/mgmtSuperTable.c | 2 +- src/mnode/src/mgmtTable.c | 127 +++++++++++++++++++++----------- src/mnode/src/mgmtUser.c | 4 +- src/mnode/src/mgmtVgroup.c | 8 +- src/util/src/tidpool.c | 35 ++++----- 10 files changed, 165 insertions(+), 189 deletions(-) diff --git a/src/inc/mnode.h b/src/inc/mnode.h index cde26dad4a..792a8ec1cc 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -53,8 +53,8 @@ typedef struct { int8_t numOfMnodes; int32_t numOfDnodes; char mnodeName[TSDB_DNODE_NAME_LEN + 1]; - char reserved[7]; - char updateEnd[1]; + int8_t reserved[15]; + int8_t updateEnd[1]; int syncFd; void *hbTimer; void *pSync; @@ -78,8 +78,8 @@ typedef struct { float lbScore; // calc in balance function int32_t customScore; // config by user char dnodeName[TSDB_DNODE_NAME_LEN + 1]; - char reserved[7]; - char updateEnd[1]; + int8_t reserved[15]; + int8_t updateEnd[1]; SVnodeLoad vload[TSDB_MAX_VNODES]; int32_t status; uint32_t lastReboot; // time stamp for last reboot @@ -120,7 +120,7 @@ typedef struct SSuperTableObj { int32_t sversion; int32_t numOfColumns; int32_t numOfTags; - int8_t reserved[5]; + int8_t reserved[15]; int8_t updateEnd[1]; int32_t numOfTables; int16_t nextColId; @@ -152,7 +152,7 @@ typedef struct { int32_t sversion; int32_t numOfColumns; int32_t sqlLen; - int8_t reserved[3]; + int8_t reserved[7]; int8_t updateEnd[1]; char* sql; //null-terminated string int16_t nextColId; @@ -167,7 +167,6 @@ typedef struct _vg_obj { int64_t createdTime; SVnodeGid vnodeGid[TSDB_VNODES_SUPPORT]; int32_t numOfVnodes; - int32_t numOfTables; int32_t lbIp; int32_t lbTime; int8_t lbStatus; @@ -175,6 +174,7 @@ typedef struct _vg_obj { int8_t updateEnd[1]; struct _vg_obj *prev, *next; struct _db_obj *pDb; + int32_t numOfTables; void * idPool; STableInfo ** tableList; } SVgObj; @@ -184,8 +184,8 @@ typedef struct _db_obj { int8_t dirty; int64_t createdTime; SDbCfg cfg; - char reserved[15]; - char updateEnd[1]; + int8_t reserved[15]; + int8_t updateEnd[1]; struct _db_obj *prev, *next; int32_t numOfVgroups; int32_t numOfTables; @@ -203,7 +203,7 @@ typedef struct _user_obj { int64_t createdTime; int8_t superAuth; int8_t writeAuth; - int8_t reserved[16]; + int8_t reserved[13]; int8_t updateEnd[1]; struct _user_obj *prev, *next; struct _acctObj * pAcct; diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index d717a4e5e9..9076681caa 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -43,8 +43,7 @@ static void mgmtDestroyChildTable(SChildTableObj *pTable) { } static int32_t mgmtChildTableActionDestroy(SSdbOperDesc *pOper) { - SChildTableObj *pTable = pOper->pObj; - mgmtDestroyChildTable(pTable); + mgmtDestroyChildTable(pOper->pObj); return TSDB_CODE_SUCCESS; } @@ -69,14 +68,6 @@ static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) { return TSDB_CODE_INVALID_ACCT; } - if (!mgmtIsMaster()) { - int32_t sid = taosAllocateId(pVgroup->idPool); - if (sid != pTable->sid) { - mError("ctable:%s, sid:%d is not matched from the master:%d", pTable->tableId, sid, pTable->sid); - return TSDB_CODE_INVALID_SESSION_ID; - } - } - pTable->superTable = mgmtGetSuperTable(pTable->superTableId); mgmtAddTableIntoSuperTable(pTable->superTable); @@ -84,10 +75,6 @@ static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) { mgmtAddTableIntoDb(pDb); mgmtAddTableIntoVgroup(pVgroup, (STableInfo *) pTable); - if (pVgroup->numOfTables >= pDb->cfg.maxSessions && pDb->numOfVgroups > 1) { - mgmtMoveVgroupToTail(pVgroup); - } - return TSDB_CODE_SUCCESS; } @@ -117,13 +104,8 @@ static int32_t mgmtChildTableActionDelete(SSdbOperDesc *pOper) { mgmtRestoreTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); mgmtRemoveTableFromDb(pDb); mgmtRemoveTableFromVgroup(pVgroup, (STableInfo *) pTable); - mgmtRemoveTableFromSuperTable(pTable->superTable); - if (pVgroup->numOfTables > 0) { - mgmtMoveVgroupToHead(pVgroup); - } - return TSDB_CODE_SUCCESS; } @@ -136,7 +118,9 @@ static int32_t mgmtChildTableActionEncode(SSdbOperDesc *pOper) { assert(pTable != NULL && pOper->rowData != NULL); memcpy(pOper->rowData, pTable, tsChildTableUpdateSize); - return tsChildTableUpdateSize; + pOper->rowSize = tsChildTableUpdateSize; + + return TSDB_CODE_SUCCESS; } static int32_t mgmtChildTableActionDecode(SSdbOperDesc *pOper) { @@ -158,7 +142,7 @@ int32_t mgmtInitChildTables() { SChildTableObj *pTable = NULL; SChildTableObj tObj; - tsChildTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; + tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; SSdbTableDesc tableDesc = { .tableName = "ctables", @@ -236,9 +220,6 @@ int32_t mgmtInitChildTables() { continue; } - pVgroup->tableList[pTable->sid] = (STableInfo*)pTable; - taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); - SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTable->superTableId); if (pSuperTable == NULL) { mError("ctable:%s, stable:%s not exist", pTable->tableId, pTable->superTableId); @@ -251,12 +232,6 @@ int32_t mgmtInitChildTables() { pNode = pLastNode; continue; } - - pTable->superTable = pSuperTable; - mgmtAddTableIntoSuperTable(pSuperTable); - - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - mgmtAddTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); } mTrace("child table is initialized"); @@ -268,8 +243,13 @@ void mgmtCleanUpChildTables() { } void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTable) { - char *pTagData = pMsg->schema + TSDB_TABLE_ID_LEN + 1; - int32_t tagDataLen = htonl(pMsg->contLen) - sizeof(SCMCreateTableMsg) - TSDB_TABLE_ID_LEN - 1; + char * pTagData = NULL; + int32_t tagDataLen = 0; + if (pMsg != NULL) { + pTagData = pMsg->schema + TSDB_TABLE_ID_LEN + 1; + tagDataLen = htonl(pMsg->contLen) - sizeof(SCMCreateTableMsg) - TSDB_TABLE_ID_LEN - 1; + } + int32_t totalCols = pTable->superTable->numOfColumns + pTable->superTable->numOfTags; int32_t contLen = sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema) + tagDataLen; @@ -302,7 +282,10 @@ void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTab pSchema++; } - memcpy(pCreate->data + totalCols * sizeof(SSchema), pTagData, tagDataLen); + if (pMsg != NULL) { + memcpy(pCreate->data + totalCols * sizeof(SSchema), pTagData, tagDataLen); + } + return pCreate; } diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index e8ab5aaa8c..0cc793f2c0 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -117,7 +117,7 @@ static int32_t mgmtDbActionDecode(SSdbOperDesc *pOper) { int32_t mgmtInitDbs() { SDbObj tObj; - tsDbUpdateSize = tObj.updateEnd - (char *)&tObj; + tsDbUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; SSdbTableDesc tableDesc = { .tableName = "dbs", @@ -276,8 +276,6 @@ static int32_t mgmtCheckDbParams(SCMCreateDbMsg *pCreate) { pCreate->blocksPerTable = TSDB_MIN_AVG_BLOCKS; } - pCreate->maxSessions++; - return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mgmtNormalTable.c b/src/mnode/src/mgmtNormalTable.c index 0ffd4fb28e..4e5e4ce7cc 100644 --- a/src/mnode/src/mgmtNormalTable.c +++ b/src/mnode/src/mgmtNormalTable.c @@ -33,7 +33,7 @@ #include "mgmtTable.h" #include "mgmtVgroup.h" -void *tsNormalTableSdb; +void *tsNormalTableSdb; int32_t tsNormalTableUpdateSize; static void mgmtDestroyNormalTable(SNormalTableObj *pTable) { @@ -43,8 +43,7 @@ static void mgmtDestroyNormalTable(SNormalTableObj *pTable) { } static int32_t mgmtNormalTableActionDestroy(SSdbOperDesc *pOper) { - SNormalTableObj *pTable = pOper->pObj; - mgmtDestroyNormalTable(pTable); + mgmtDestroyNormalTable(pOper->pObj); return TSDB_CODE_SUCCESS; } @@ -53,38 +52,26 @@ static int32_t mgmtNormalTableActionInsert(SSdbOperDesc *pOper) { SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - mError("id:%s not in vgroup:%d", pTable->tableId, pTable->vgId); + mError("ntable:%s not in vgroup:%d", pTable->tableId, pTable->vgId); return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { - mError("vgroup:%d not in DB:%s", pVgroup->vgId, pVgroup->dbName); + mError("ntable:%s, vgroup:%d not in DB:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { - mError("account not exists"); + mError("ntable:%s, account:%s not exists", pTable->tableId, pDb->cfg.acct); return TSDB_CODE_INVALID_ACCT; } - if (!mgmtIsMaster()) { - int32_t sid = taosAllocateId(pVgroup->idPool); - if (sid != pTable->sid) { - mError("sid:%d is not matched from the master:%d", sid, pTable->sid); - return TSDB_CODE_INVALID_SESSION_ID; - } - } - mgmtAddTimeSeries(pAcct, pTable->numOfColumns - 1); mgmtAddTableIntoDb(pDb); mgmtAddTableIntoVgroup(pVgroup, (STableInfo *) pTable); - if (pVgroup->numOfTables >= pDb->cfg.maxSessions - 1 && pDb->numOfVgroups > 1) { - mgmtMoveVgroupToTail(pVgroup); - } - return TSDB_CODE_SUCCESS; } @@ -101,7 +88,7 @@ static int32_t mgmtNormalTableActionDelete(SSdbOperDesc *pOper) { SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { - mError("vgroup:%d not in DB:%s", pVgroup->vgId, pVgroup->dbName); + mError("ntable:%s, vgroup:%d not in DB:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); return TSDB_CODE_INVALID_DB; } @@ -115,22 +102,10 @@ static int32_t mgmtNormalTableActionDelete(SSdbOperDesc *pOper) { mgmtRemoveTableFromDb(pDb); mgmtRemoveTableFromVgroup(pVgroup, (STableInfo *) pTable); - if (pVgroup->numOfTables > 0) { - mgmtMoveVgroupToHead(pVgroup); - } - return TSDB_CODE_SUCCESS; } static int32_t mgmtNormalTableActionUpdate(SSdbOperDesc *pOper) { - // SNormalTableObj *pTable = (SNormalTableObj *) pObj; - // memcpy(pTable, str, tsNormalTableUpdateSize); - - // int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns) + pTable->sqlLen; - // pTable->schema = realloc(pTable->schema, schemaSize); - // pTable->sql = (char*)pTable->schema + sizeof(SSchema) * (pTable->numOfColumns); - // memcpy(pTable->schema, str + tsNormalTableUpdateSize, schemaSize); - return TSDB_CODE_SUCCESS; } @@ -139,25 +114,24 @@ static int32_t mgmtNormalTableActionEncode(SSdbOperDesc *pOper) { assert(pOper->pObj != NULL && pOper->rowData != NULL); int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - if (pOper->maxRowSize < tsNormalTableUpdateSize + schemaSize + 1) { - return -1; + if (pOper->maxRowSize < tsNormalTableUpdateSize + schemaSize) { + return TSDB_CODE_INVALID_MSG_LEN; } memcpy(pOper->rowData, pTable, tsNormalTableUpdateSize); memcpy(pOper->rowData + tsNormalTableUpdateSize, pTable->schema, schemaSize); memcpy(pOper->rowData + tsNormalTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen); - - return tsNormalTableUpdateSize + schemaSize + pTable->sqlLen; + + pOper->rowSize = tsNormalTableUpdateSize + schemaSize + pTable->sqlLen; + return TSDB_CODE_SUCCESS; } static int32_t mgmtNormalTableActionDecode(SSdbOperDesc *pOper) { assert(pOper->rowData != NULL); - SNormalTableObj *pTable = (SNormalTableObj *)malloc(sizeof(SNormalTableObj)); - if (pTable == NULL) { - return -1; - } - memset(pTable, 0, sizeof(SNormalTableObj)); + SNormalTableObj *pTable = (SNormalTableObj *)calloc(1, sizeof(SNormalTableObj)); + if (pTable == NULL) TSDB_CODE_SERV_OUT_OF_MEMORY; + memcpy(pTable, pOper->rowData, tsNormalTableUpdateSize); int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); @@ -176,6 +150,7 @@ static int32_t mgmtNormalTableActionDecode(SSdbOperDesc *pOper) { } memcpy(pTable->sql, pOper->rowData + tsNormalTableUpdateSize + schemaSize, pTable->sqlLen); + pOper->pObj = pTable; return TSDB_CODE_SUCCESS; } @@ -185,7 +160,7 @@ int32_t mgmtInitNormalTables() { SNormalTableObj *pTable = NULL; SNormalTableObj tObj; - tsNormalTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; + tsNormalTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; SSdbTableDesc tableDesc = { .tableName = "ntables", @@ -261,15 +236,6 @@ int32_t mgmtInitNormalTables() { pNode = pLastNode; continue; } - - mgmtAddTableIntoVgroup(pVgroup, (STableInfo *)pTable); - //pVgroup->tableList[pTable->sid] = (STableInfo*)pTable; - taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); - - pTable->sql = (char *)pTable->schema + sizeof(SSchema) * pTable->numOfColumns; - - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - mgmtAddTimeSeries(pAcct, pTable->numOfColumns - 1); } mTrace("ntables is initialized"); diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index eee8c0f1a0..e950761a31 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -309,11 +309,11 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { rowMeta.row = oper.pObj; (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); pTable->numOfRows++; - sdbTrace("table:%s, read new record:%s, numOfRows:%d version:%" PRId64 , - pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version); + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read new record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); } else { - sdbError("table:%s, failed to decode record:%s, numOfRows:%d version:%" PRId64 , - pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version); + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, failed to decode record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); } } } else { @@ -325,8 +325,8 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { (*pTable->destroyFp)(&oper); (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); pTable->numOfRows--; - sdbTrace("table:%s, read deleted record:%s, numOfRows:%d version:%" PRId64 , - pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version); + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read deleted record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); } else { SRowMeta rowMeta; rowMeta.version = rowHead->version; @@ -346,11 +346,11 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { if (code == TSDB_CODE_SUCCESS) { rowMeta.row = oper.pObj; (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); - sdbTrace("table:%s, read updated record:%s, numOfRows:%d version:%" PRId64 , - pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version); + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read updated record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); } else { - sdbError("table:%s, failed to decode record:%s, numOfRows:%d version:%" PRId64 , - pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version); + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, failed to decode record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); } } numOfChanged++; @@ -416,8 +416,8 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) { pTable->tableId = sdbNumOfTables++; sdbTableList[pTable->tableId] = pTable; - sdbTrace("table:%s, is initialized, numOfRows:%d numOfTables:%d version:%" PRId64 " sdbversion:%" PRId64, - pTable->tableName, pTable->numOfRows, sdbNumOfTables, pTable->version, sdbVersion); + sdbTrace("table:%s, is initialized, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d numOfTables:%d", + pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbNumOfTables); return pTable; } @@ -531,8 +531,8 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) { pthread_mutex_unlock(&pTable->mutex); - sdbTrace("table:%s, inserte record:%s, sdbversion:%" PRId64 " version:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version, pOper->rowSize, pTable->numOfRows, pTable->fileSize); + sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, insert record:%s, rowSize:%d fileSize:%" PRId64, + pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); (*pTable->insertFp)(pOper); @@ -614,8 +614,9 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { tfree(rowHead); - sdbTrace("table:%s, delete record:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d", - pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version, pTable->numOfRows); + + sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, delete record:%s, rowSize:%d fileSize:%" PRId64, + pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); // Delete from current layer (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj); @@ -702,9 +703,9 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { pTable->fileSize += real_size; sdbFinishCommit(pTable); - sdbTrace("table:%s, update record:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version, pTable->numOfRows); - + sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, update record:%s, rowSize:%d fileSize:%" PRId64, + pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); + pMeta->version = pTable->version; pMeta->offset = pTable->fileSize; pMeta->rowSize = rowHead->rowSize; diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index d3a0dcb6bc..f2b54ec000 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -111,7 +111,7 @@ static int32_t mgmtSuperTableActionDecode(SSdbOperDesc *pOper) { int32_t mgmtInitSuperTables() { SSuperTableObj tObj; - tsSuperTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; + tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; SSdbTableDesc tableDesc = { .tableName = "stables", diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 918547fa12..675de57924 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -22,6 +22,7 @@ #include "mgmtDClient.h" #include "mgmtDb.h" #include "mgmtDnode.h" +#include "mgmtDServer.h" #include "mgmtGrant.h" #include "mgmtMnode.h" #include "mgmtNormalTable.h" @@ -61,6 +62,7 @@ static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg); static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessGetTableMeta(STableInfo *pTable, void *thandle); +static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg); int32_t mgmtInitTables() { int32_t code = mgmtInitSuperTables(); @@ -90,6 +92,7 @@ int32_t mgmtInitTables() { mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_TABLE_RSP, mgmtProcessDropTableRsp); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP, mgmtProcessAlterTableRsp); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_STABLE_RSP, mgmtProcessDropStableRsp); + mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_TABLE, mgmtProcessTableCfgMsg); return TSDB_CODE_SUCCESS; } @@ -760,9 +763,19 @@ static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg) { if (rpcMsg->code != TSDB_CODE_SUCCESS) { if (pTable->type == TSDB_CHILD_TABLE) { - // sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_GLOBAL); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsChildTableSdb, + .pObj = pTable + }; + sdbDeleteRow(&oper); } else if (pTable->type == TSDB_NORMAL_TABLE){ - // sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_GLOBAL); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsNormalTableSdb, + .pObj = pTable + }; + sdbDeleteRow(&oper); } else {} mError("table:%s, failed to create in dnode, reason:%s", pTable->tableId, tstrerror(rpcMsg->code)); mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); @@ -816,19 +829,31 @@ static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg) { } if (pTable->type == TSDB_CHILD_TABLE) { - // if (sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { - // mError("table:%s, update ctables sdb error", pTable->tableId); - // mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); - // free(queueMsg); - // return; - // } + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsChildTableSdb, + .pObj = pTable + }; + int32_t code = sdbDeleteRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + mError("table:%s, update ctables sdb error", pTable->tableId); + mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); + free(queueMsg); + return; + } } else if (pTable->type == TSDB_NORMAL_TABLE){ - // if (sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { - // mError("table:%s, update ntables sdb error", pTable->tableId); - // mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); - // free(queueMsg); - // return; - // } + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsNormalTableSdb, + .pObj = pTable + }; + int32_t code = sdbDeleteRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + mError("table:%s, update ntables sdb error", pTable->tableId); + mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); + free(queueMsg); + return; + } } if (pVgroup->numOfTables <= 0) { @@ -844,32 +869,48 @@ static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg) { mTrace("drop stable rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code); } -// -// -//static void mgmtProcessTableCfgMsg(int8_t msgType, int8_t *pCont, int32_t contLen, void *thandle) { -// SDMConfigTableMsg *pCfg = (SDMConfigTableMsg *) pCont; -// pCfg->dnode = htonl(pCfg->dnode); -// pCfg->vnode = htonl(pCfg->vnode); -// pCfg->sid = htonl(pCfg->sid); -// mTrace("dnode:%s, vnode:%d, sid:%d, receive table config msg", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); -// -// if (!sdbMaster) { -// mError("dnode:%s, vnode:%d, sid:%d, not master, redirect it", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); -// mgmtSendRspToDnode(thandle, msgType + 1, TSDB_CODE_REDIRECT, NULL, 0); -// return; -// } -// -// STableInfo *pTable = mgmtGetTableByPos(pCfg->dnode, pCfg->vnode, pCfg->sid); -// if (pTable == NULL) { -// mError("dnode:%s, vnode:%d, sid:%d, table not found", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); -// mgmtSendRspToDnode(thandle, msgType + 1, TSDB_CODE_INVALID_TABLE, NULL, 0); -// return; -// } -// -// mgmtSendRspToDnode(thandle, msgType + 1, TSDB_CODE_SUCCESS, NULL, 0); -// -// //TODO -// SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); -// mgmtSendCreateTableMsg(NULL, &ipSet, NULL); -//} -// \ No newline at end of file +static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) { + if (mgmtCheckRedirect(rpcMsg->handle)) return; + + SDMConfigTableMsg *pCfg = (SDMConfigTableMsg *) rpcMsg->pCont; + pCfg->dnode = htonl(pCfg->dnode); + pCfg->vnode = htonl(pCfg->vnode); + pCfg->sid = htonl(pCfg->sid); + mTrace("dnode:%s, vnode:%d, sid:%d, receive table config msg", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); + + STableInfo *pTable = mgmtGetTableByPos(pCfg->dnode, pCfg->vnode, pCfg->sid); + if (pTable == NULL) { + mError("dnode:%s, vnode:%d, sid:%d, table not found", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_TABLE); + return; + } + + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SUCCESS); + + SMDCreateTableMsg *pMDCreate = NULL; + if (pTable->type == TSDB_CHILD_TABLE) { + mTrace("table:%s, is a child table, vgroup:%d sid:%d", pTable->tableId, pCfg->vnode, pCfg->sid); + pMDCreate = mgmtBuildCreateChildTableMsg(NULL, (SChildTableObj *) pTable); + if (pMDCreate == NULL) { + return; + } + } else if (pTable->type == TSDB_NORMAL_TABLE) { + mTrace("table:%s, is a normal table, vgroup:%d sid:%d", pTable->tableId, pCfg->vnode, pCfg->sid); + pMDCreate = mgmtBuildCreateNormalTableMsg((SNormalTableObj *) pTable); + if (pMDCreate == NULL) { + return; + } + } else { + mError("table:%s, invalid msg type, vgroup:%d sid:%d", pTable->tableId, pCfg->vnode, pCfg->sid); + } + + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); + SRpcMsg rpcRsp = { + .handle = NULL, + .pCont = pMDCreate, + .contLen = htonl(pMDCreate->contLen), + .code = 0, + .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE + }; + mgmtSendMsgToDnode(&ipSet, &rpcRsp); +} diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index e55ba9ea60..576b326669 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -94,7 +94,7 @@ static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) { int32_t mgmtInitUsers() { SUserObj tObj; - tsUserUpdateSize = tObj.updateEnd - (int8_t *)&tObj; + tsUserUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; SSdbTableDesc tableDesc = { .tableName = "users", @@ -471,7 +471,7 @@ static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) { if (hasRight) { code = mgmtDropUser(pUser->pAcct, pDrop->user); if (code == TSDB_CODE_SUCCESS) { - mLPrint("user:%s is dropped by %s", pDrop->user, pUser->user); + mLPrint("user:%s is dropped by %s, result:%d", pUser->user, pOperUser->user, tstrerror(code)); } } else { code = TSDB_CODE_NO_RIGHTS; diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index b37b363077..9e52574b8c 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -142,7 +142,7 @@ static int32_t mgmtVgroupActionDecode(SSdbOperDesc *pOper) { int32_t mgmtInitVgroups() { SVgObj tObj; - tsVgUpdateSize = tObj.updateEnd - (int8_t *)&tObj; + tsVgUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; SSdbTableDesc tableDesc = { .tableName = "vgroups", @@ -433,13 +433,10 @@ void mgmtAddTableIntoVgroup(SVgObj *pVgroup, STableInfo *pTable) { pVgroup->tableList[pTable->sid] = pTable; taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); pVgroup->numOfTables++; - pVgroup->pDb->numOfTables++; } if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxSessions) mgmtAddVgroupIntoDbTail(pVgroup); - else - mgmtAddVgroupIntoDb(pVgroup); } void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, STableInfo *pTable) { @@ -447,13 +444,10 @@ void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, STableInfo *pTable) { pVgroup->tableList[pTable->sid] = NULL; taosFreeId(pVgroup->idPool, pTable->sid); pVgroup->numOfTables--; - pVgroup->pDb->numOfTables--; } if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxSessions) mgmtAddVgroupIntoDbTail(pVgroup); - else - mgmtAddVgroupIntoDb(pVgroup); } SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { diff --git a/src/util/src/tidpool.c b/src/util/src/tidpool.c index c3f429ac43..5ed6b8cf23 100644 --- a/src/util/src/tidpool.c +++ b/src/util/src/tidpool.c @@ -15,12 +15,13 @@ #include "os.h" #include "tlog.h" +#include typedef struct { int maxId; int numOfFree; int freeSlot; - int * freeList; + bool * freeList; pthread_mutex_t mutex; } id_pool_t; @@ -28,7 +29,7 @@ void *taosInitIdPool(int maxId) { id_pool_t *pIdPool = calloc(1, sizeof(id_pool_t)); if (pIdPool == NULL) return NULL; - pIdPool->freeList = malloc(sizeof(int) * (size_t)maxId); + pIdPool->freeList = calloc(maxId, sizeof(bool)); if (pIdPool->freeList == NULL) { free(pIdPool); return NULL; @@ -40,10 +41,6 @@ void *taosInitIdPool(int maxId) { pthread_mutex_init(&pIdPool->mutex, NULL); - for (int i = 0; i < maxId; ++i) { - pIdPool->freeList[i] = -1; - } - pTrace("pool:%p is setup, maxId:%d", pIdPool, pIdPool->maxId); return pIdPool; @@ -61,8 +58,8 @@ int taosAllocateId(void *handle) { if (pIdPool->numOfFree > 0) { for (int i = 0; i < pIdPool->maxId; ++i) { slot = (i + pIdPool->freeSlot) % pIdPool->maxId; - if (pIdPool->freeList[slot] == -1) { - pIdPool->freeList[slot] = slot; + if (!pIdPool->freeList[slot]) { + pIdPool->freeList[slot] = true; pIdPool->freeSlot = slot + 1; pIdPool->numOfFree--; break; @@ -71,7 +68,7 @@ int taosAllocateId(void *handle) { } pthread_mutex_unlock(&pIdPool->mutex); - return slot; + return slot + 1; } void taosFreeId(void *handle, int id) { @@ -80,9 +77,9 @@ void taosFreeId(void *handle, int id) { pthread_mutex_lock(&pIdPool->mutex); - int slot = id % pIdPool->maxId; - if (pIdPool->freeList[slot] != -1) { - pIdPool->freeList[slot] = -1; + int slot = (id - 1) % pIdPool->maxId; + if (pIdPool->freeList[slot]) { + pIdPool->freeList[slot] = false; pIdPool->numOfFree++; } @@ -114,9 +111,9 @@ void taosIdPoolMarkStatus(void *handle, int id) { id_pool_t *pIdPool = handle; pthread_mutex_lock(&pIdPool->mutex); - int slot = id % pIdPool->maxId; - if (pIdPool->freeList[slot] == -1) { - pIdPool->freeList[slot] = slot; + int slot = (id - 1) % pIdPool->maxId; + if (!pIdPool->freeList[slot]) { + pIdPool->freeList[slot] = true; pIdPool->numOfFree--; } @@ -129,18 +126,14 @@ int taosUpdateIdPool(id_pool_t *handle, int maxId) { return -1; } - int *idList = malloc(sizeof(int) * maxId); + int *idList = calloc(maxId, sizeof(bool)); if (idList == NULL) { return -1; } pthread_mutex_lock(&pIdPool->mutex); - for (int i = 0; i < maxId; ++i) { - idList[i] = -1; - } - - memcpy(idList, pIdPool->freeList, sizeof(int) * pIdPool->maxId); + memcpy(idList, pIdPool->freeList, sizeof(bool) * pIdPool->maxId); pIdPool->numOfFree += (maxId - pIdPool->maxId); pIdPool->maxId = maxId; From c541d35462d5884f174fdb9808265981e4d9e79d Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 24 Mar 2020 15:34:22 +0800 Subject: [PATCH 12/12] [TD-15] sdb may crash while received close signal --- src/mnode/src/mgmtSdb.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index e950761a31..0b1ac3be97 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -722,18 +722,20 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { void sdbCloseTable(void *handle) { SSdbTable *pTable = (SSdbTable *)handle; void * pNode = NULL; - void * row; - + if (pTable == NULL) return; while (1) { - pNode = sdbFetchRow(handle, pNode, &row); - if (row == NULL) break; + SRowMeta * pMeta; + pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); + if (pMeta == NULL) break; SSdbOperDesc oper = { + .pObj = pMeta->row, .table = pTable, - .rowData = row, + .version = pMeta->version, }; + (*pTable->destroyFp)(&oper); }