xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(wal/level): enable alter replica from fake walLevel 1

This commit is contained in:
Minglei Jin 2024-06-12 15:16:58 +08:00
parent bee01f9dba
commit 568799a6a3
7 changed files with 62 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/libs/wal/wal.h
View File

@ -60,6 +60,7 @@ typedef struct {
EWalType level; // wal level
int32_t encryptAlgorithm;
char encryptKey[ENCRYPT_KEY_LEN + 1];
int8_t clearFiles;
} SWalCfg;
typedef struct {

14
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -144,7 +144,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
}
#if defined(TD_ENTERPRISE)
pCfg->tsdbCfg.encryptAlgorithm = pCreate->encryptAlgorithm;
if(pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4){
if (pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4) {
strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
#else
@ -160,7 +160,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->walCfg.level = pCreate->walLevel;
#if defined(TD_ENTERPRISE)
pCfg->walCfg.encryptAlgorithm = pCreate->encryptAlgorithm;
if(pCfg->walCfg.encryptAlgorithm == DND_CA_SM4){
if (pCfg->walCfg.encryptAlgorithm == DND_CA_SM4) {
strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
#else
@ -169,7 +169,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
#if defined(TD_ENTERPRISE)
pCfg->tdbEncryptAlgorithm = pCreate->encryptAlgorithm;
if(pCfg->tdbEncryptAlgorithm == DND_CA_SM4){
if (pCfg->tdbEncryptAlgorithm == DND_CA_SM4) {
strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
#else
@ -304,8 +304,8 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return -1;
}
if(req.encryptAlgorithm == DND_CA_SM4){
if(strlen(tsEncryptKey) == 0){
if (req.encryptAlgorithm == DND_CA_SM4) {
if (strlen(tsEncryptKey) == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("vgId:%d, failed to create vnode since encrypt key is empty", req.vgId);
return -1;
@ -737,7 +737,9 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
.diskPrimary = pVnode->diskPrimary,
};
tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path));
vmCloseVnode(pMgmt, pVnode, false);
bool commitAndRemoveWal = vnodeShouldRemoveWal(pVnode->pImpl);
vmCloseVnode(pMgmt, pVnode, commitAndRemoveWal);
int32_t diskPrimary = wrapperCfg.diskPrimary;
char path[TSDB_FILENAME_LEN] = {0};

11
source/dnode/vnode/inc/vnode.h
View File

@ -52,6 +52,7 @@ extern const SVnodeCfg vnodeCfgDefault;
int32_t vnodeInit(int32_t nthreads);
void vnodeCleanup();
int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, int32_t diskPrimary, STfs *pTfs);
bool vnodeShouldRemoveWal(SVnode *pVnode);
int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t diskPrimary, STfs *pTfs);
int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnodeHashRangeReq *pReq,
int32_t diskPrimary, STfs *pTfs);
@ -181,7 +182,7 @@ void tsdbReaderSetNotifyCb(STsdbReader *pReader, TsdReaderNotifyCbFn not
int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables);
int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr,
SArray *pFuncTypeList, SColumnInfo* pkCol, int32_t numOfPks);
SArray *pFuncTypeList, SColumnInfo *pkCol, int32_t numOfPks);
int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
SArray *pTableUids);
void *tsdbCacherowsReaderClose(void *pReader);
@ -218,8 +219,8 @@ typedef struct STqReader {
STqReader *tqReaderOpen(SVnode *pVnode);
void tqReaderClose(STqReader *);
bool tqGetTablePrimaryKey(STqReader* pReader);
void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid);
bool tqGetTablePrimaryKey(STqReader *pReader);
void tqSetTablePrimaryKey(STqReader *pReader, int64_t uid);
void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList);
int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList, const char *id);
@ -278,8 +279,8 @@ struct STsdbCfg {
int32_t keep2; // just for save config, don't use in tsdbRead/tsdbCommit/..., and use STsdbKeepCfg in STsdb instead
int32_t keepTimeOffset; // just for save config, use STsdbKeepCfg in STsdb instead
SRetention retentions[TSDB_RETENTION_MAX];
int32_t encryptAlgorithm;
char encryptKey[ENCRYPT_KEY_LEN + 1];
int32_t encryptAlgorithm;
char encryptKey[ENCRYPT_KEY_LEN + 1];
};
typedef struct {

27
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -13,9 +13,9 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tglobal.h"
#include "tutil.h"
#include "vnd.h"
#include "tglobal.h"
const SVnodeCfg vnodeCfgDefault = {.vgId = -1,
.dbname = "",
@ -47,6 +47,7 @@ const SVnodeCfg vnodeCfgDefault = {.vgId = -1,
.segSize = 0,
.retentionSize = -1,
.level = TAOS_WAL_WRITE,
.clearFiles = 0,
},
.hashBegin = 0,
.hashEnd = 0,
@ -142,6 +143,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
if (tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.clearFiles", pCfg->walCfg.clearFiles) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sttTrigger) < 0) return -1;
@ -249,12 +251,11 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
tjsonGetNumberValue(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm, code);
if (code < 0) return -1;
#if defined(TD_ENTERPRISE)
if(pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4){
if(tsEncryptKey[0] == 0){
if (pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4) {
if (tsEncryptKey[0] == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
}
else{
} else {
strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
@ -273,15 +274,16 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
if (code < 0) return -1;
tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code);
if (code < 0) return -1;
tjsonGetNumberValue(pJson, "wal.clearFiles", pCfg->walCfg.clearFiles, code);
if (code < 0) return -1;
tjsonGetNumberValue(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm, code);
if (code < 0) return -1;
#if defined(TD_ENTERPRISE)
if(pCfg->walCfg.encryptAlgorithm == DND_CA_SM4){
if(tsEncryptKey[0] == 0){
if (pCfg->walCfg.encryptAlgorithm == DND_CA_SM4) {
if (tsEncryptKey[0] == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
}
else{
} else {
strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
@ -289,12 +291,11 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
tjsonGetNumberValue(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm, code);
if (code < 0) return -1;
#if defined(TD_ENTERPRISE)
if(pCfg->tdbEncryptAlgorithm == DND_CA_SM4){
if(tsEncryptKey[0] == 0){
if (pCfg->tdbEncryptAlgorithm == DND_CA_SM4) {
if (tsEncryptKey[0] == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
}
else{
} else {
strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}

25
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -81,6 +81,8 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, int32_t diskPrimary, STfs
return 0;
}
bool vnodeShouldRemoveWal(SVnode *pVnode) { return pVnode->config.walCfg.clearFiles == 1; }
int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t diskPrimary, STfs *pTfs) {
SVnodeInfo info = {0};
char dir[TSDB_FILENAME_LEN] = {0};
@ -129,6 +131,12 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t
}
pCfg->changeVersion = pReq->changeVersion;
if (info.config.walCfg.clearFiles) {
info.config.walCfg.clearFiles = 0;
vInfo("vgId:%d, reset wal clearFiles", pReq->vgId);
}
vInfo("vgId:%d, save config while alter, replicas:%d totalReplicas:%d selfIndex:%d changeVersion:%d", pReq->vgId,
pCfg->replicaNum, pCfg->totalReplicaNum, pCfg->myIndex, pCfg->changeVersion);
@ -486,15 +494,14 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC
if (tsEnableMonitor && pVnode->monitor.insertCounter == NULL) {
taos_counter_t *counter = NULL;
int32_t label_count = 7;
const char *sample_labels[] = {VNODE_METRIC_TAG_NAME_SQL_TYPE, VNODE_METRIC_TAG_NAME_CLUSTER_ID,
VNODE_METRIC_TAG_NAME_DNODE_ID, VNODE_METRIC_TAG_NAME_DNODE_EP,
VNODE_METRIC_TAG_NAME_VGROUP_ID, VNODE_METRIC_TAG_NAME_USERNAME,
VNODE_METRIC_TAG_NAME_RESULT};
counter = taos_counter_new(VNODE_METRIC_SQL_COUNT, "counter for insert sql",
label_count, sample_labels);
vInfo("vgId:%d, new metric:%p",TD_VID(pVnode), counter);
if(taos_collector_registry_register_metric(counter) == 1){
int32_t label_count = 7;
const char *sample_labels[] = {VNODE_METRIC_TAG_NAME_SQL_TYPE, VNODE_METRIC_TAG_NAME_CLUSTER_ID,
VNODE_METRIC_TAG_NAME_DNODE_ID, VNODE_METRIC_TAG_NAME_DNODE_EP,
VNODE_METRIC_TAG_NAME_VGROUP_ID, VNODE_METRIC_TAG_NAME_USERNAME,
VNODE_METRIC_TAG_NAME_RESULT};
counter = taos_counter_new(VNODE_METRIC_SQL_COUNT, "counter for insert sql", label_count, sample_labels);
vInfo("vgId:%d, new metric:%p", TD_VID(pVnode), counter);
if (taos_collector_registry_register_metric(counter) == 1) {
taos_counter_destroy(counter);
counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT);
vInfo("vgId:%d, get metric from registry:%p", TD_VID(pVnode), counter);

3
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -2021,6 +2021,9 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pRe
}
if (pVnode->config.walCfg.level != req.walLevel) {
if (pVnode->config.walCfg.level == 0) {
pVnode->config.walCfg.clearFiles = 1;
}
pVnode->config.walCfg.level = req.walLevel;
walChanged = true;
}

18
source/libs/parser/src/parTranslater.c
View File

@ -6898,6 +6898,19 @@ static int32_t checkDbTbPrefixSuffixOptions(STranslateContext* pCxt, int32_t tbP
}
static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) {
if (pOptions->replica > 1) {
SDbCfgInfo dbCfg = {0};
int32_t code = getDBCfg(pCxt, pDbName, &dbCfg);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
if (pOptions->walLevel == 0 || dbCfg.walLevel == 0) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option, wal_level 0 should be used with replica 1");
}
}
int32_t daysPerFile = pOptions->daysPerFile;
int32_t s3KeepLocal = pOptions->s3KeepLocal;
int64_t daysToKeep0 = pOptions->keep[0];
@ -6926,11 +6939,6 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa
"Invalid option, with_arbitrator should be used with replica 2");
}
if (pOptions->replica > 1 && pOptions->walLevel == 0) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option, wal_level 0 should be used with replica 1");
}
return TSDB_CODE_SUCCESS;
}