From a12b0e71755e2d3fce52c3c646e427aa86e02f69 Mon Sep 17 00:00:00 2001 From: lichuang Date: Thu, 20 May 2021 15:12:34 +0800 Subject: [PATCH] [TD-1920]add mnode compact wal functions --- src/inc/mnode.h | 1 + src/mnode/inc/mnodeAcct.h | 2 ++ src/mnode/inc/mnodeCluster.h | 2 ++ src/mnode/inc/mnodeDb.h | 2 ++ src/mnode/inc/mnodeDnode.h | 1 + src/mnode/inc/mnodeMnode.h | 1 + src/mnode/inc/mnodeSdb.h | 1 + src/mnode/inc/mnodeTable.h | 1 + src/mnode/inc/mnodeUser.h | 2 ++ src/mnode/inc/mnodeVgroup.h | 1 + src/mnode/src/mnodeAcct.c | 26 +++++++++++++++ src/mnode/src/mnodeCluster.c | 24 ++++++++++++++ src/mnode/src/mnodeDb.c | 27 ++++++++++++++++ src/mnode/src/mnodeDnode.c | 27 ++++++++++++++++ src/mnode/src/mnodeMain.c | 16 ++++++++-- src/mnode/src/mnodeMnode.c | 27 ++++++++++++++++ src/mnode/src/mnodeSdb.c | 27 ++++++++++++++++ src/mnode/src/mnodeTable.c | 62 ++++++++++++++++++++++++++++++++++++ src/mnode/src/mnodeUser.c | 27 ++++++++++++++++ src/mnode/src/mnodeVgroup.c | 27 ++++++++++++++++ 20 files changed, 301 insertions(+), 3 deletions(-) diff --git a/src/inc/mnode.h b/src/inc/mnode.h index c66dc60a1b..203ac57469 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -74,6 +74,7 @@ void mnodeProcessPeerRsp(SRpcMsg *pMsg); int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey); int32_t mnodeCompactWal(); +int32_t mnodeCompactComponents(); #ifdef __cplusplus } diff --git a/src/mnode/inc/mnodeAcct.h b/src/mnode/inc/mnodeAcct.h index 595dcca413..522070e909 100644 --- a/src/mnode/inc/mnodeAcct.h +++ b/src/mnode/inc/mnodeAcct.h @@ -35,6 +35,8 @@ void mnodeDropDbFromAcct(SAcctObj *pAcct, SDbObj *pDb); void mnodeAddUserToAcct(SAcctObj *pAcct, SUserObj *pUser); void mnodeDropUserFromAcct(SAcctObj *pAcct, SUserObj *pUser); +int32_t mnodeCompactAccts(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeCluster.h b/src/mnode/inc/mnodeCluster.h index a5af544dc2..db258ae6d6 100644 --- a/src/mnode/inc/mnodeCluster.h +++ b/src/mnode/inc/mnodeCluster.h @@ -25,6 +25,8 @@ void mnodeCleanupCluster(); void mnodeUpdateClusterId(); const char* mnodeGetClusterId(); +int32_t mnodeCompactCluster(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeDb.h b/src/mnode/inc/mnodeDb.h index da0865833d..0fa1a15e2d 100644 --- a/src/mnode/inc/mnodeDb.h +++ b/src/mnode/inc/mnodeDb.h @@ -41,6 +41,8 @@ void mnodeDecDbRef(SDbObj *pDb); bool mnodeCheckIsMonitorDB(char *db, char *monitordb); void mnodeDropAllDbs(SAcctObj *pAcct); +int32_t mnodeCompactDbs(); + // util func void mnodeAddSuperTableIntoDb(SDbObj *pDb); void mnodeRemoveSuperTableFromDb(SDbObj *pDb); diff --git a/src/mnode/inc/mnodeDnode.h b/src/mnode/inc/mnodeDnode.h index fa1995254e..2db794a173 100644 --- a/src/mnode/inc/mnodeDnode.h +++ b/src/mnode/inc/mnodeDnode.h @@ -77,6 +77,7 @@ void * mnodeGetDnodeByEp(char *ep); void mnodeUpdateDnode(SDnodeObj *pDnode); int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg); +int32_t mnodeCompactDnodes(); extern int32_t tsAccessSquence; #ifdef __cplusplus diff --git a/src/mnode/inc/mnodeMnode.h b/src/mnode/inc/mnodeMnode.h index ffdec02eb6..66e9eb1e0e 100644 --- a/src/mnode/inc/mnodeMnode.h +++ b/src/mnode/inc/mnodeMnode.h @@ -50,6 +50,7 @@ char* mnodeGetMnodeMasterEp(); void mnodeGetMnodeInfos(void *mnodes); void mnodeUpdateMnodeEpSet(SMInfos *pMnodes); +int32_t mnodeCompactMnodes(); #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 13dd8bd06b..2ae0c47902 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -92,6 +92,7 @@ void sdbUpdateMnodeRoles(); int32_t sdbGetReplicaNum(); int32_t sdbInsertRow(SSdbRow *pRow); +int32_t sdbInsertCompactRow(SSdbRow *pRow); int32_t sdbDeleteRow(SSdbRow *pRow); int32_t sdbUpdateRow(SSdbRow *pRow); int32_t sdbInsertRowToQueue(SSdbRow *pRow); diff --git a/src/mnode/inc/mnodeTable.h b/src/mnode/inc/mnodeTable.h index bf04f26a90..c583a60c7a 100644 --- a/src/mnode/inc/mnodeTable.h +++ b/src/mnode/inc/mnodeTable.h @@ -36,6 +36,7 @@ void mnodeCancelGetNextSuperTable(void *pIter); void mnodeDropAllChildTables(SDbObj *pDropDb); void mnodeDropAllSuperTables(SDbObj *pDropDb); void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup); +int32_t mnodeCompactTables(); #ifdef __cplusplus } diff --git a/src/mnode/inc/mnodeUser.h b/src/mnode/inc/mnodeUser.h index 156bc7aaeb..b8f0805120 100644 --- a/src/mnode/inc/mnodeUser.h +++ b/src/mnode/inc/mnodeUser.h @@ -33,6 +33,8 @@ char * mnodeGetUserFromMsg(void *pMnodeMsg); int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg); void mnodeDropAllUsers(SAcctObj *pAcct); +int32_t mnodeCompactUsers(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h index 7b798c23f8..73b0e6ae1b 100644 --- a/src/mnode/inc/mnodeVgroup.h +++ b/src/mnode/inc/mnodeVgroup.h @@ -32,6 +32,7 @@ void mnodeDropAllDbVgroups(SDbObj *pDropDb); void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb); void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode); //void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb); +int32_t mnodeCompactVgroups(); void * mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup); void mnodeCancelGetNextVgroup(void *pIter); diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index afe474df6b..64cfa28917 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -238,6 +238,32 @@ static int32_t mnodeCreateRootAcct() { return sdbInsertRow(&row); } +int32_t mnodeCompactAccts() { + void *pIter = NULL; + SAcctObj *pAcct = NULL; + + mInfo("start to compact accts table..."); + + while (1) { + pIter = mnodeGetNextAcct(pIter, &pAcct); + if (pAcct == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsAcctSdb, + .pObj = pAcct, + }; + + mInfo("compact accts %s", pAcct->user); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact accts table..."); + + return 0; +} + #ifndef _ACCT int32_t acctInit() { return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 169d2ebd9d..8b9af6d493 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -237,3 +237,27 @@ static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, pShow->numOfReads += numOfRows; return numOfRows; } + +int32_t mnodeCompactCluster() { + SClusterObj *pCluster = NULL; + void *pIter; + + mInfo("start to compact cluster table..."); + + pIter = mnodeGetNextCluster(NULL, &pCluster); + while (pCluster) { + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsClusterSdb, + .pObj = pCluster, + }; + + sdbInsertCompactRow(&row); + + pIter = mnodeGetNextCluster(pIter, &pCluster); + } + + mInfo("end to compact cluster table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 8af20aa862..b5663bc00c 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -1271,3 +1271,30 @@ void mnodeDropAllDbs(SAcctObj *pAcct) { mInfo("acct:%s, all dbs:%d is dropped from sdb", pAcct->user, numOfDbs); } + +int32_t mnodeCompactDbs() { + void *pIter = NULL; + SDbObj *pDb = NULL; + + mInfo("start to compact dbs table..."); + + while (1) { + pIter = mnodeGetNextDb(pIter, &pDb); + if (pDb == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsDbSdb, + .pObj = pDb, + .rowSize = sizeof(SDbObj), + }; + + mInfo("compact dbs %s", pDb->name); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact dbs table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 3525bcac18..51f16e4bc6 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -1270,3 +1270,30 @@ char* dnodeRoles[] = { "vnode", "any" }; + +int32_t mnodeCompactDnodes() { + SDnodeObj *pDnode = NULL; + void * pIter = NULL; + + mInfo("start to compact dnodes table..."); + + while (1) { + pIter = mnodeGetNextDnode(pIter, &pDnode); + if (pDnode == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsDnodeSdb, + .pObj = pDnode, + .rowSize = sizeof(SDnodeObj), + }; + + mInfo("compact dnode %d", pDnode->dnodeId); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact dnodes table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 76eebe0d01..01b83bec19 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -57,11 +57,16 @@ static SStep tsMnodeSteps[] = { {"show", mnodeInitShow, mnodeCleanUpShow} }; -/* static SStep tsMnodeCompactSteps[] = { - {"cluster", mnodeInitCluster, NULL}, + {"cluster", mnodeCompactCluster, NULL}, + {"dnodes", mnodeCompactDnodes, NULL}, + {"mnodes", mnodeCompactMnodes, NULL}, + {"accts", mnodeCompactAccts, NULL}, + {"users", mnodeCompactUsers, NULL}, + {"dbs", mnodeCompactDbs, NULL}, + {"vgroups", mnodeCompactVgroups, NULL}, + {"tables", mnodeCompactTables, NULL}, }; -*/ static void mnodeInitTimer(); static void mnodeCleanupTimer(); @@ -77,6 +82,11 @@ static int32_t mnodeInitComponents() { return dnodeStepInit(tsMnodeSteps, stepSize); } +int32_t mnodeCompactComponents() { + int32_t stepSize = sizeof(tsMnodeCompactSteps) / sizeof(SStep); + return dnodeStepInit(tsMnodeCompactSteps, stepSize); +} + int32_t mnodeStartSystem() { if (tsMgmtIsRunning) { mInfo("mnode module already started..."); diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index ca6d6400ae..ddc9ea59c4 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -566,3 +566,30 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo return numOfRows; } + +int32_t mnodeCompactMnodes() { + void *pIter = NULL; + SMnodeObj *pMnode = NULL; + + mInfo("start to compact mnodes table..."); + + while (1) { + pIter = mnodeGetNextMnode(pIter, &pMnode); + if (pMnode == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsMnodeSdb, + .pObj = pMnode, + .rowSize = sizeof(SMnodeObj), + }; + + mInfo("compact mnode %d", pMnode->mnodeId); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact mnodes table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 64c7713164..79081945d9 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -732,6 +732,12 @@ static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void * } } +int32_t sdbInsertCompactRow(SSdbRow *pRow) { + SSdbTable *pTable = pRow->pTable; + if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; + return sdbWriteRowToQueue(pRow, SDB_ACTION_INSERT); +} + int32_t sdbInsertRow(SSdbRow *pRow) { SSdbTable *pTable = pRow->pTable; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; @@ -1148,5 +1154,26 @@ int32_t sdbGetReplicaNum() { int32_t mnodeCompactWal() { sdbInfo("vgId:1, start compact mnode wal..."); + // close wal + walFsync(tsSdbMgmt.wal, true); + walClose(tsSdbMgmt.wal); + + // change wal to wal_bak dir + char temp[TSDB_FILENAME_LEN] = {0}; + SWalCfg walCfg = {.vgId = 1, .walLevel = TAOS_WAL_FSYNC, .keep = TAOS_WAL_KEEP, .fsyncPeriod = 0}; + sprintf(temp, "%s/wal_tmp", tsMnodeDir); + if (mkdir(temp, 0755) != 0 && errno != EEXIST) { + return -1; + } + tsSdbMgmt.wal = walOpen(temp, &walCfg); + walRenew(tsSdbMgmt.wal); + + // compact memory tables info to wal tmp dir + mnodeCompactComponents(); + + // close wal + walFsync(tsSdbMgmt.wal, true); + walClose(tsSdbMgmt.wal); + return 0; } \ No newline at end of file diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 2a8e941fcb..2e5d80b787 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -3242,3 +3242,65 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro return numOfRows; } + +static int32_t mnodeCompactSuperTables() { + void *pIter = NULL; + SSTableObj *pTable = NULL; + + mInfo("start to compact super table..."); + + while (1) { + pIter = mnodeGetNextSuperTable(pIter, &pTable); + if (pTable == NULL) break; + + int32_t schemaSize = (pTable->numOfColumns + pTable->numOfTags) * sizeof(SSchema); + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsSuperTableSdb, + .pObj = pTable, + .rowSize = sizeof(SSTableObj) + schemaSize, + }; + + mInfo("compact super %ld", pTable->uid); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact super table..."); + + return 0; +} + +static int32_t mnodeCompactChildTables() { + void *pIter = NULL; + SCTableObj *pTable = NULL; + + mInfo("start to compact child table..."); + + while (1) { + pIter = mnodeGetNextChildTable(pIter, &pTable); + if (pTable == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pObj = pTable, + .pTable = tsChildTableSdb, + }; + + mInfo("compact child %ld:%d", pTable->uid, pTable->tid); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact child table..."); + + return 0; +} + +int32_t mnodeCompactTables() { + mnodeCompactSuperTables(); + + mnodeCompactChildTables(); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index e77c1b3e59..c5c54791cf 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -617,3 +617,30 @@ static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg) { return mnodeRetriveAuth(pAuthMsg->user, &pAuthRsp->spi, &pAuthRsp->encrypt, pAuthRsp->secret, pAuthRsp->ckey); } + +int32_t mnodeCompactUsers() { + void *pIter = NULL; + SUserObj *pUser = NULL; + + mInfo("start to compact users table..."); + + while (1) { + pIter = mnodeGetNextUser(pIter, &pUser); + if (pUser == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsUserSdb, + .pObj = pUser, + .rowSize = sizeof(SUserObj), + }; + + mInfo("compact users %s", pUser->user); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact users table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 7222c8d1a0..67532ad85a 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -1302,3 +1302,30 @@ void mnodeSetVgidVer(int8_t *cver, uint64_t iver) { cver[1] = (int8_t)((int32_t)(iver % 100000) / 100); cver[2] = (int8_t)(iver % 100); } + +int32_t mnodeCompactVgroups() { + void *pIter = NULL; + SVgObj *pVgroup = NULL; + + mInfo("start to compact vgroups table..."); + + while (1) { + pIter = mnodeGetNextVgroup(pIter, &pVgroup); + if (pVgroup == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsVgroupSdb, + .pObj = pVgroup, + .rowSize = sizeof(SVgObj), + }; + + mInfo("compact vgroups %d", pVgroup->vgId); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact vgroups table..."); + + return 0; +} \ No newline at end of file