From 7316fb89796120fc274700f109612eb1eade2cbd Mon Sep 17 00:00:00 2001 From: lichuang Date: Thu, 20 May 2021 11:06:28 +0800 Subject: [PATCH 1/5] [TD-1920]add mnode compact wal functions --- src/common/inc/tglobal.h | 1 + src/common/src/tglobal.c | 1 + src/dnode/src/dnodeMain.c | 1 + src/dnode/src/dnodeSystem.c | 2 ++ src/inc/mnode.h | 2 ++ src/mnode/inc/mnodeSdb.h | 1 + src/mnode/src/mnodeMain.c | 6 ++++++ src/mnode/src/mnodeSdb.c | 12 ++++++++++++ 8 files changed, 26 insertions(+) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 2f4aa4c2b2..abb4d39ee1 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -142,6 +142,7 @@ extern int32_t tsMonitorInterval; extern int8_t tsEnableStream; // internal +extern int8_t tsCompactMnodeWal; extern int8_t tsPrintAuth; extern int8_t tscEmbedded; extern char configDir[]; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index db97c3a5af..ab036b021c 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -176,6 +176,7 @@ int32_t tsMonitorInterval = 30; // seconds int8_t tsEnableStream = 1; // internal +int8_t tsCompactMnodeWal = 0; int8_t tsPrintAuth = 0; int8_t tscEmbedded = 0; char configDir[TSDB_FILENAME_LEN] = {0}; diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 410e6bb188..48680e20d9 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -40,6 +40,7 @@ #include "dnodeShell.h" #include "dnodeTelemetry.h" #include "module.h" +#include "mnode.h" #if !defined(_MODULE) || !defined(_TD_LINUX) int32_t moduleStart() { return 0; } diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index e49b3eba99..ee37ffdcbb 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -42,6 +42,8 @@ int32_t main(int32_t argc, char *argv[]) { } } else if (strcmp(argv[i], "-C") == 0) { dump_config = 1; + } else if (strcmp(argv[i], "--compact-mnode-wal") == 0) { + tsCompactMnodeWal = 1; } else if (strcmp(argv[i], "-V") == 0) { #ifdef _ACCT char *versionStr = "enterprise"; diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 2495a42ba2..c66dc60a1b 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -73,6 +73,8 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg); void mnodeProcessPeerRsp(SRpcMsg *pMsg); int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey); +int32_t mnodeCompactWal(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index e4e4a7a054..13dd8bd06b 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -106,6 +106,7 @@ int32_t sdbGetId(void *pTable); uint64_t sdbGetVersion(); bool sdbCheckRowDeleted(void *pTable, void *pRow); +int32_t mnodeCompactWal(); #ifdef __cplusplus } #endif diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 7ef0488c42..76eebe0d01 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -57,6 +57,12 @@ static SStep tsMnodeSteps[] = { {"show", mnodeInitShow, mnodeCleanUpShow} }; +/* +static SStep tsMnodeCompactSteps[] = { + {"cluster", mnodeInitCluster, NULL}, +}; +*/ + static void mnodeInitTimer(); static void mnodeCleanupTimer(); static bool mnodeNeedStart() ; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 505d3c519c..64c7713164 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -450,6 +450,12 @@ int32_t sdbInit() { } tsSdbMgmt.status = SDB_STATUS_SERVING; + + if (tsCompactMnodeWal) { + mnodeCompactWal(); + exit(EXIT_SUCCESS); + } + return TSDB_CODE_SUCCESS; } @@ -1138,3 +1144,9 @@ static void *sdbWorkerFp(void *pWorker) { int32_t sdbGetReplicaNum() { return tsSdbMgmt.cfg.replica; } + +int32_t mnodeCompactWal() { + sdbInfo("vgId:1, start compact mnode wal..."); + + return 0; +} \ No newline at end of file From a12b0e71755e2d3fce52c3c646e427aa86e02f69 Mon Sep 17 00:00:00 2001 From: lichuang Date: Thu, 20 May 2021 15:12:34 +0800 Subject: [PATCH 2/5] [TD-1920]add mnode compact wal functions --- src/inc/mnode.h | 1 + src/mnode/inc/mnodeAcct.h | 2 ++ src/mnode/inc/mnodeCluster.h | 2 ++ src/mnode/inc/mnodeDb.h | 2 ++ src/mnode/inc/mnodeDnode.h | 1 + src/mnode/inc/mnodeMnode.h | 1 + src/mnode/inc/mnodeSdb.h | 1 + src/mnode/inc/mnodeTable.h | 1 + src/mnode/inc/mnodeUser.h | 2 ++ src/mnode/inc/mnodeVgroup.h | 1 + src/mnode/src/mnodeAcct.c | 26 +++++++++++++++ src/mnode/src/mnodeCluster.c | 24 ++++++++++++++ src/mnode/src/mnodeDb.c | 27 ++++++++++++++++ src/mnode/src/mnodeDnode.c | 27 ++++++++++++++++ src/mnode/src/mnodeMain.c | 16 ++++++++-- src/mnode/src/mnodeMnode.c | 27 ++++++++++++++++ src/mnode/src/mnodeSdb.c | 27 ++++++++++++++++ src/mnode/src/mnodeTable.c | 62 ++++++++++++++++++++++++++++++++++++ src/mnode/src/mnodeUser.c | 27 ++++++++++++++++ src/mnode/src/mnodeVgroup.c | 27 ++++++++++++++++ 20 files changed, 301 insertions(+), 3 deletions(-) diff --git a/src/inc/mnode.h b/src/inc/mnode.h index c66dc60a1b..203ac57469 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -74,6 +74,7 @@ void mnodeProcessPeerRsp(SRpcMsg *pMsg); int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey); int32_t mnodeCompactWal(); +int32_t mnodeCompactComponents(); #ifdef __cplusplus } diff --git a/src/mnode/inc/mnodeAcct.h b/src/mnode/inc/mnodeAcct.h index 595dcca413..522070e909 100644 --- a/src/mnode/inc/mnodeAcct.h +++ b/src/mnode/inc/mnodeAcct.h @@ -35,6 +35,8 @@ void mnodeDropDbFromAcct(SAcctObj *pAcct, SDbObj *pDb); void mnodeAddUserToAcct(SAcctObj *pAcct, SUserObj *pUser); void mnodeDropUserFromAcct(SAcctObj *pAcct, SUserObj *pUser); +int32_t mnodeCompactAccts(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeCluster.h b/src/mnode/inc/mnodeCluster.h index a5af544dc2..db258ae6d6 100644 --- a/src/mnode/inc/mnodeCluster.h +++ b/src/mnode/inc/mnodeCluster.h @@ -25,6 +25,8 @@ void mnodeCleanupCluster(); void mnodeUpdateClusterId(); const char* mnodeGetClusterId(); +int32_t mnodeCompactCluster(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeDb.h b/src/mnode/inc/mnodeDb.h index da0865833d..0fa1a15e2d 100644 --- a/src/mnode/inc/mnodeDb.h +++ b/src/mnode/inc/mnodeDb.h @@ -41,6 +41,8 @@ void mnodeDecDbRef(SDbObj *pDb); bool mnodeCheckIsMonitorDB(char *db, char *monitordb); void mnodeDropAllDbs(SAcctObj *pAcct); +int32_t mnodeCompactDbs(); + // util func void mnodeAddSuperTableIntoDb(SDbObj *pDb); void mnodeRemoveSuperTableFromDb(SDbObj *pDb); diff --git a/src/mnode/inc/mnodeDnode.h b/src/mnode/inc/mnodeDnode.h index fa1995254e..2db794a173 100644 --- a/src/mnode/inc/mnodeDnode.h +++ b/src/mnode/inc/mnodeDnode.h @@ -77,6 +77,7 @@ void * mnodeGetDnodeByEp(char *ep); void mnodeUpdateDnode(SDnodeObj *pDnode); int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg); +int32_t mnodeCompactDnodes(); extern int32_t tsAccessSquence; #ifdef __cplusplus diff --git a/src/mnode/inc/mnodeMnode.h b/src/mnode/inc/mnodeMnode.h index ffdec02eb6..66e9eb1e0e 100644 --- a/src/mnode/inc/mnodeMnode.h +++ b/src/mnode/inc/mnodeMnode.h @@ -50,6 +50,7 @@ char* mnodeGetMnodeMasterEp(); void mnodeGetMnodeInfos(void *mnodes); void mnodeUpdateMnodeEpSet(SMInfos *pMnodes); +int32_t mnodeCompactMnodes(); #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 13dd8bd06b..2ae0c47902 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -92,6 +92,7 @@ void sdbUpdateMnodeRoles(); int32_t sdbGetReplicaNum(); int32_t sdbInsertRow(SSdbRow *pRow); +int32_t sdbInsertCompactRow(SSdbRow *pRow); int32_t sdbDeleteRow(SSdbRow *pRow); int32_t sdbUpdateRow(SSdbRow *pRow); int32_t sdbInsertRowToQueue(SSdbRow *pRow); diff --git a/src/mnode/inc/mnodeTable.h b/src/mnode/inc/mnodeTable.h index bf04f26a90..c583a60c7a 100644 --- a/src/mnode/inc/mnodeTable.h +++ b/src/mnode/inc/mnodeTable.h @@ -36,6 +36,7 @@ void mnodeCancelGetNextSuperTable(void *pIter); void mnodeDropAllChildTables(SDbObj *pDropDb); void mnodeDropAllSuperTables(SDbObj *pDropDb); void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup); +int32_t mnodeCompactTables(); #ifdef __cplusplus } diff --git a/src/mnode/inc/mnodeUser.h b/src/mnode/inc/mnodeUser.h index 156bc7aaeb..b8f0805120 100644 --- a/src/mnode/inc/mnodeUser.h +++ b/src/mnode/inc/mnodeUser.h @@ -33,6 +33,8 @@ char * mnodeGetUserFromMsg(void *pMnodeMsg); int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg); void mnodeDropAllUsers(SAcctObj *pAcct); +int32_t mnodeCompactUsers(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h index 7b798c23f8..73b0e6ae1b 100644 --- a/src/mnode/inc/mnodeVgroup.h +++ b/src/mnode/inc/mnodeVgroup.h @@ -32,6 +32,7 @@ void mnodeDropAllDbVgroups(SDbObj *pDropDb); void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb); void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode); //void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb); +int32_t mnodeCompactVgroups(); void * mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup); void mnodeCancelGetNextVgroup(void *pIter); diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index afe474df6b..64cfa28917 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -238,6 +238,32 @@ static int32_t mnodeCreateRootAcct() { return sdbInsertRow(&row); } +int32_t mnodeCompactAccts() { + void *pIter = NULL; + SAcctObj *pAcct = NULL; + + mInfo("start to compact accts table..."); + + while (1) { + pIter = mnodeGetNextAcct(pIter, &pAcct); + if (pAcct == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsAcctSdb, + .pObj = pAcct, + }; + + mInfo("compact accts %s", pAcct->user); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact accts table..."); + + return 0; +} + #ifndef _ACCT int32_t acctInit() { return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 169d2ebd9d..8b9af6d493 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -237,3 +237,27 @@ static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, pShow->numOfReads += numOfRows; return numOfRows; } + +int32_t mnodeCompactCluster() { + SClusterObj *pCluster = NULL; + void *pIter; + + mInfo("start to compact cluster table..."); + + pIter = mnodeGetNextCluster(NULL, &pCluster); + while (pCluster) { + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsClusterSdb, + .pObj = pCluster, + }; + + sdbInsertCompactRow(&row); + + pIter = mnodeGetNextCluster(pIter, &pCluster); + } + + mInfo("end to compact cluster table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 8af20aa862..b5663bc00c 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -1271,3 +1271,30 @@ void mnodeDropAllDbs(SAcctObj *pAcct) { mInfo("acct:%s, all dbs:%d is dropped from sdb", pAcct->user, numOfDbs); } + +int32_t mnodeCompactDbs() { + void *pIter = NULL; + SDbObj *pDb = NULL; + + mInfo("start to compact dbs table..."); + + while (1) { + pIter = mnodeGetNextDb(pIter, &pDb); + if (pDb == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsDbSdb, + .pObj = pDb, + .rowSize = sizeof(SDbObj), + }; + + mInfo("compact dbs %s", pDb->name); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact dbs table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 3525bcac18..51f16e4bc6 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -1270,3 +1270,30 @@ char* dnodeRoles[] = { "vnode", "any" }; + +int32_t mnodeCompactDnodes() { + SDnodeObj *pDnode = NULL; + void * pIter = NULL; + + mInfo("start to compact dnodes table..."); + + while (1) { + pIter = mnodeGetNextDnode(pIter, &pDnode); + if (pDnode == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsDnodeSdb, + .pObj = pDnode, + .rowSize = sizeof(SDnodeObj), + }; + + mInfo("compact dnode %d", pDnode->dnodeId); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact dnodes table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 76eebe0d01..01b83bec19 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -57,11 +57,16 @@ static SStep tsMnodeSteps[] = { {"show", mnodeInitShow, mnodeCleanUpShow} }; -/* static SStep tsMnodeCompactSteps[] = { - {"cluster", mnodeInitCluster, NULL}, + {"cluster", mnodeCompactCluster, NULL}, + {"dnodes", mnodeCompactDnodes, NULL}, + {"mnodes", mnodeCompactMnodes, NULL}, + {"accts", mnodeCompactAccts, NULL}, + {"users", mnodeCompactUsers, NULL}, + {"dbs", mnodeCompactDbs, NULL}, + {"vgroups", mnodeCompactVgroups, NULL}, + {"tables", mnodeCompactTables, NULL}, }; -*/ static void mnodeInitTimer(); static void mnodeCleanupTimer(); @@ -77,6 +82,11 @@ static int32_t mnodeInitComponents() { return dnodeStepInit(tsMnodeSteps, stepSize); } +int32_t mnodeCompactComponents() { + int32_t stepSize = sizeof(tsMnodeCompactSteps) / sizeof(SStep); + return dnodeStepInit(tsMnodeCompactSteps, stepSize); +} + int32_t mnodeStartSystem() { if (tsMgmtIsRunning) { mInfo("mnode module already started..."); diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index ca6d6400ae..ddc9ea59c4 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -566,3 +566,30 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo return numOfRows; } + +int32_t mnodeCompactMnodes() { + void *pIter = NULL; + SMnodeObj *pMnode = NULL; + + mInfo("start to compact mnodes table..."); + + while (1) { + pIter = mnodeGetNextMnode(pIter, &pMnode); + if (pMnode == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsMnodeSdb, + .pObj = pMnode, + .rowSize = sizeof(SMnodeObj), + }; + + mInfo("compact mnode %d", pMnode->mnodeId); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact mnodes table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 64c7713164..79081945d9 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -732,6 +732,12 @@ static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void * } } +int32_t sdbInsertCompactRow(SSdbRow *pRow) { + SSdbTable *pTable = pRow->pTable; + if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; + return sdbWriteRowToQueue(pRow, SDB_ACTION_INSERT); +} + int32_t sdbInsertRow(SSdbRow *pRow) { SSdbTable *pTable = pRow->pTable; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; @@ -1148,5 +1154,26 @@ int32_t sdbGetReplicaNum() { int32_t mnodeCompactWal() { sdbInfo("vgId:1, start compact mnode wal..."); + // close wal + walFsync(tsSdbMgmt.wal, true); + walClose(tsSdbMgmt.wal); + + // change wal to wal_bak dir + char temp[TSDB_FILENAME_LEN] = {0}; + SWalCfg walCfg = {.vgId = 1, .walLevel = TAOS_WAL_FSYNC, .keep = TAOS_WAL_KEEP, .fsyncPeriod = 0}; + sprintf(temp, "%s/wal_tmp", tsMnodeDir); + if (mkdir(temp, 0755) != 0 && errno != EEXIST) { + return -1; + } + tsSdbMgmt.wal = walOpen(temp, &walCfg); + walRenew(tsSdbMgmt.wal); + + // compact memory tables info to wal tmp dir + mnodeCompactComponents(); + + // close wal + walFsync(tsSdbMgmt.wal, true); + walClose(tsSdbMgmt.wal); + return 0; } \ No newline at end of file diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 2a8e941fcb..2e5d80b787 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -3242,3 +3242,65 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro return numOfRows; } + +static int32_t mnodeCompactSuperTables() { + void *pIter = NULL; + SSTableObj *pTable = NULL; + + mInfo("start to compact super table..."); + + while (1) { + pIter = mnodeGetNextSuperTable(pIter, &pTable); + if (pTable == NULL) break; + + int32_t schemaSize = (pTable->numOfColumns + pTable->numOfTags) * sizeof(SSchema); + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsSuperTableSdb, + .pObj = pTable, + .rowSize = sizeof(SSTableObj) + schemaSize, + }; + + mInfo("compact super %ld", pTable->uid); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact super table..."); + + return 0; +} + +static int32_t mnodeCompactChildTables() { + void *pIter = NULL; + SCTableObj *pTable = NULL; + + mInfo("start to compact child table..."); + + while (1) { + pIter = mnodeGetNextChildTable(pIter, &pTable); + if (pTable == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pObj = pTable, + .pTable = tsChildTableSdb, + }; + + mInfo("compact child %ld:%d", pTable->uid, pTable->tid); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact child table..."); + + return 0; +} + +int32_t mnodeCompactTables() { + mnodeCompactSuperTables(); + + mnodeCompactChildTables(); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index e77c1b3e59..c5c54791cf 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -617,3 +617,30 @@ static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg) { return mnodeRetriveAuth(pAuthMsg->user, &pAuthRsp->spi, &pAuthRsp->encrypt, pAuthRsp->secret, pAuthRsp->ckey); } + +int32_t mnodeCompactUsers() { + void *pIter = NULL; + SUserObj *pUser = NULL; + + mInfo("start to compact users table..."); + + while (1) { + pIter = mnodeGetNextUser(pIter, &pUser); + if (pUser == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsUserSdb, + .pObj = pUser, + .rowSize = sizeof(SUserObj), + }; + + mInfo("compact users %s", pUser->user); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact users table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 7222c8d1a0..67532ad85a 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -1302,3 +1302,30 @@ void mnodeSetVgidVer(int8_t *cver, uint64_t iver) { cver[1] = (int8_t)((int32_t)(iver % 100000) / 100); cver[2] = (int8_t)(iver % 100); } + +int32_t mnodeCompactVgroups() { + void *pIter = NULL; + SVgObj *pVgroup = NULL; + + mInfo("start to compact vgroups table..."); + + while (1) { + pIter = mnodeGetNextVgroup(pIter, &pVgroup); + if (pVgroup == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsVgroupSdb, + .pObj = pVgroup, + .rowSize = sizeof(SVgObj), + }; + + mInfo("compact vgroups %d", pVgroup->vgId); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact vgroups table..."); + + return 0; +} \ No newline at end of file From 99f7359390539e659b7de331344015582aa1fbbe Mon Sep 17 00:00:00 2001 From: lichuang Date: Thu, 20 May 2021 16:04:53 +0800 Subject: [PATCH 3/5] [TD-1920]add mnode compact wal functions --- src/mnode/src/mnodeCluster.c | 2 +- src/mnode/src/mnodeMain.c | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 8b9af6d493..553e8446ab 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -249,7 +249,7 @@ int32_t mnodeCompactCluster() { SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsClusterSdb, - .pObj = pCluster, + .pObj = pCluster, }; sdbInsertCompactRow(&row); diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 01b83bec19..d9abe16120 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -59,6 +59,8 @@ static SStep tsMnodeSteps[] = { static SStep tsMnodeCompactSteps[] = { {"cluster", mnodeCompactCluster, NULL}, + +/* {"dnodes", mnodeCompactDnodes, NULL}, {"mnodes", mnodeCompactMnodes, NULL}, {"accts", mnodeCompactAccts, NULL}, @@ -66,6 +68,7 @@ static SStep tsMnodeCompactSteps[] = { {"dbs", mnodeCompactDbs, NULL}, {"vgroups", mnodeCompactVgroups, NULL}, {"tables", mnodeCompactTables, NULL}, +*/ }; static void mnodeInitTimer(); From 9958b3d8d27130525b1c3b0a4f95bf08d23b620a Mon Sep 17 00:00:00 2001 From: lichuang Date: Thu, 20 May 2021 18:19:27 +0800 Subject: [PATCH 4/5] [TD-1920]add mnode compact wal functions --- src/common/inc/tglobal.h | 2 ++ src/common/src/tglobal.c | 2 ++ src/dnode/src/dnodeMain.c | 11 +++++++++++ src/mnode/src/mnodeMain.c | 4 +--- src/mnode/src/mnodeSdb.c | 33 +++++++++++++++++++++++++-------- 5 files changed, 41 insertions(+), 11 deletions(-) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index abb4d39ee1..e07c3611d7 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -149,6 +149,8 @@ extern char configDir[]; extern char tsVnodeDir[]; extern char tsDnodeDir[]; extern char tsMnodeDir[]; +extern char tsMnodeBakDir[]; +extern char tsMnodeTmpDir[]; extern char tsDataDir[]; extern char tsLogDir[]; extern char tsScriptDir[]; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index ab036b021c..d6bbc288ad 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -183,6 +183,8 @@ char configDir[TSDB_FILENAME_LEN] = {0}; char tsVnodeDir[TSDB_FILENAME_LEN] = {0}; char tsDnodeDir[TSDB_FILENAME_LEN] = {0}; char tsMnodeDir[TSDB_FILENAME_LEN] = {0}; +char tsMnodeTmpDir[TSDB_FILENAME_LEN] = {0}; +char tsMnodeBakDir[TSDB_FILENAME_LEN] = {0}; char tsDataDir[TSDB_FILENAME_LEN] = {0}; char tsScriptDir[TSDB_FILENAME_LEN] = {0}; char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/"; diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 48680e20d9..ee4dfcf852 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -217,6 +217,17 @@ static int32_t dnodeInitStorage() { sprintf(tsDnodeDir, "%s/dnode", tsDataDir); // sprintf(tsVnodeBakDir, "%s/vnode_bak", tsDataDir); + if (tsCompactMnodeWal == 1) { + sprintf(tsMnodeTmpDir, "%s/mnode_tmp", tsDataDir); + tfsRmdir(tsMnodeTmpDir); + if (dnodeCreateDir(tsMnodeTmpDir) < 0) { + dError("failed to create dir: %s, reason: %s", tsMnodeTmpDir, strerror(errno)); + return -1; + } + + sprintf(tsMnodeBakDir, "%s/mnode_bak", tsDataDir); + //tfsRmdir(tsMnodeBakDir); + } //TODO(dengyihao): no need to init here if (dnodeCreateDir(tsMnodeDir) < 0) { dError("failed to create dir: %s, reason: %s", tsMnodeDir, strerror(errno)); diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index d9abe16120..d3511a4e62 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -59,8 +59,6 @@ static SStep tsMnodeSteps[] = { static SStep tsMnodeCompactSteps[] = { {"cluster", mnodeCompactCluster, NULL}, - -/* {"dnodes", mnodeCompactDnodes, NULL}, {"mnodes", mnodeCompactMnodes, NULL}, {"accts", mnodeCompactAccts, NULL}, @@ -68,7 +66,7 @@ static SStep tsMnodeCompactSteps[] = { {"dbs", mnodeCompactDbs, NULL}, {"vgroups", mnodeCompactVgroups, NULL}, {"tables", mnodeCompactTables, NULL}, -*/ + }; static void mnodeInitTimer(); diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 79081945d9..9e3b87671e 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -20,6 +20,7 @@ #include "tutil.h" #include "tref.h" #include "tbn.h" +#include "tfs.h" #include "tqueue.h" #include "twal.h" #include "tsync.h" @@ -1154,26 +1155,42 @@ int32_t sdbGetReplicaNum() { int32_t mnodeCompactWal() { sdbInfo("vgId:1, start compact mnode wal..."); - // close wal + // close old wal walFsync(tsSdbMgmt.wal, true); walClose(tsSdbMgmt.wal); - // change wal to wal_bak dir - char temp[TSDB_FILENAME_LEN] = {0}; + // reset version,then compacted wal log can start from version 1 + tsSdbMgmt.version = 0; + + // change wal to wal_tmp dir SWalCfg walCfg = {.vgId = 1, .walLevel = TAOS_WAL_FSYNC, .keep = TAOS_WAL_KEEP, .fsyncPeriod = 0}; - sprintf(temp, "%s/wal_tmp", tsMnodeDir); - if (mkdir(temp, 0755) != 0 && errno != EEXIST) { - return -1; - } + char temp[TSDB_FILENAME_LEN] = {0}; + sprintf(temp, "%s/wal", tsMnodeTmpDir); tsSdbMgmt.wal = walOpen(temp, &walCfg); walRenew(tsSdbMgmt.wal); // compact memory tables info to wal tmp dir - mnodeCompactComponents(); + if (mnodeCompactComponents() != 0) { + tfsRmdir(tsMnodeTmpDir); + return -1; + } // close wal walFsync(tsSdbMgmt.wal, true); walClose(tsSdbMgmt.wal); + // rename old wal to wal_bak + if (taosRename(tsMnodeDir, tsMnodeBakDir) != 0) { + return -1; + } + + // rename wal_tmp to wal + if (taosRename(tsMnodeTmpDir, tsMnodeDir) != 0) { + return -1; + } + + // del wal_tmp dir + sdbInfo("vgId:1, compact mnode wal success"); + return 0; } \ No newline at end of file From e38362c037c864f1752975fdf792202983a3c306 Mon Sep 17 00:00:00 2001 From: lichuang Date: Fri, 21 May 2021 09:43:05 +0800 Subject: [PATCH 5/5] [TD-1920]fix compile error --- src/mnode/src/mnodeTable.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 2e5d80b787..4d3125a2d1 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -3261,7 +3261,7 @@ static int32_t mnodeCompactSuperTables() { .rowSize = sizeof(SSTableObj) + schemaSize, }; - mInfo("compact super %ld", pTable->uid); + mInfo("compact super %" PRIu64, pTable->uid); sdbInsertCompactRow(&row); } @@ -3287,7 +3287,7 @@ static int32_t mnodeCompactChildTables() { .pTable = tsChildTableSdb, }; - mInfo("compact child %ld:%d", pTable->uid, pTable->tid); + mInfo("compact child %" PRIu64 ":%d", pTable->uid, pTable->tid); sdbInsertCompactRow(&row); }