Merge pull request #26117 from taosdata/fix/TD-30545

fix(wal/level): enable alter replica from fake walLevel 1

include/libs/wal/wal.h

@@ -60,6 +60,7 @@ typedef struct {
   EWalType level;  // wal level
   int32_t  encryptAlgorithm;
   char     encryptKey[ENCRYPT_KEY_LEN + 1];
+  int8_t   clearFiles;
 } SWalCfg;
 
 typedef struct {

include/util/taoserror.h

@@ -336,6 +336,7 @@ int32_t  taosGetErrSize();
 #define TSDB_CODE_MND_DB_IN_CREATING            TAOS_DEF_ERROR_CODE(0, 0x0396) //
 #define TSDB_CODE_MND_INVALID_SYS_TABLENAME     TAOS_DEF_ERROR_CODE(0, 0x039A)
 #define TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE  TAOS_DEF_ERROR_CODE(0, 0x039B)
+#define TSDB_CODE_MND_INVALID_WAL_LEVEL         TAOS_DEF_ERROR_CODE(0, 0x039C)
 
 // mnode-node
 #define TSDB_CODE_MND_MNODE_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03A0)

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -144,7 +144,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
   }
 #if defined(TD_ENTERPRISE)
   pCfg->tsdbCfg.encryptAlgorithm = pCreate->encryptAlgorithm;
-  if(pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4){
+  if (pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4) {
     strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
   }
 #else
@@ -160,7 +160,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
   pCfg->walCfg.level = pCreate->walLevel;
 #if defined(TD_ENTERPRISE)
   pCfg->walCfg.encryptAlgorithm = pCreate->encryptAlgorithm;
-  if(pCfg->walCfg.encryptAlgorithm == DND_CA_SM4){
+  if (pCfg->walCfg.encryptAlgorithm == DND_CA_SM4) {
     strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
   }
 #else
@@ -169,7 +169,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
 
 #if defined(TD_ENTERPRISE)
   pCfg->tdbEncryptAlgorithm = pCreate->encryptAlgorithm;
-  if(pCfg->tdbEncryptAlgorithm == DND_CA_SM4){
+  if (pCfg->tdbEncryptAlgorithm == DND_CA_SM4) {
     strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
   }
 #else
@@ -278,7 +278,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
       req.keepTimeOffset, req.s3ChunkSize, req.s3KeepLocal, req.s3Compact, req.isTsma, req.precision, req.compression,
       req.minRows, req.maxRows, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize,
       req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix,
-      req.replica, req.selfIndex, req.learnerReplica, req.learnerSelfIndex, req.strict, req.changeVersion, 
+      req.replica, req.selfIndex, req.learnerReplica, req.learnerSelfIndex, req.strict, req.changeVersion,
       req.encryptAlgorithm);
 
   for (int32_t i = 0; i < req.replica; ++i) {
@@ -304,8 +304,8 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
     return -1;
   }
 
-  if(req.encryptAlgorithm == DND_CA_SM4){
+  if (req.encryptAlgorithm == DND_CA_SM4) {
-    if(strlen(tsEncryptKey) == 0){
+    if (strlen(tsEncryptKey) == 0) {
       terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
       dError("vgId:%d, failed to create vnode since encrypt key is empty", req.vgId);
       return -1;
@@ -482,7 +482,9 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
       .diskPrimary = pVnode->diskPrimary,
   };
   tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path));
-  vmCloseVnode(pMgmt, pVnode, false);
+
+  bool commitAndRemoveWal = vnodeShouldRemoveWal(pVnode->pImpl);
+  vmCloseVnode(pMgmt, pVnode, commitAndRemoveWal);
 
   int32_t diskPrimary = wrapperCfg.diskPrimary;
   char    path[TSDB_FILENAME_LEN] = {0};
@@ -737,7 +739,9 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
       .diskPrimary = pVnode->diskPrimary,
   };
   tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path));
-  vmCloseVnode(pMgmt, pVnode, false);
+
+  bool commitAndRemoveWal = vnodeShouldRemoveWal(pVnode->pImpl);
+  vmCloseVnode(pMgmt, pVnode, commitAndRemoveWal);
 
   int32_t diskPrimary = wrapperCfg.diskPrimary;
   char    path[TSDB_FILENAME_LEN] = {0};

source/dnode/mnode/impl/src/mndDb.c

@@ -495,6 +495,16 @@ static int32_t mndCheckInChangeDbCfg(SMnode *pMnode, SDbCfg *pOldCfg, SDbCfg *pN
 #else
   if (pNewCfg->replications != 1 && pNewCfg->replications != 3) return -1;
 #endif
+
+  if (pNewCfg->walLevel == 0 && pOldCfg->replications > 1) {
+    terrno = TSDB_CODE_MND_INVALID_WAL_LEVEL;
+    return -1;
+  }
+  if (pNewCfg->replications > 1 && pOldCfg->walLevel == 0) {
+    terrno = TSDB_CODE_MND_INVALID_WAL_LEVEL;
+    return -1;
+  }
+
   if (pNewCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pNewCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1;
   if (pNewCfg->minRows < TSDB_MIN_MINROWS_FBLOCK || pNewCfg->minRows > TSDB_MAX_MINROWS_FBLOCK) return -1;
   if (pNewCfg->maxRows < TSDB_MIN_MAXROWS_FBLOCK || pNewCfg->maxRows > TSDB_MAX_MAXROWS_FBLOCK) return -1;

source/dnode/vnode/inc/vnode.h

@@ -52,6 +52,7 @@ extern const SVnodeCfg vnodeCfgDefault;
 int32_t vnodeInit(int32_t nthreads);
 void    vnodeCleanup();
 int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, int32_t diskPrimary, STfs *pTfs);
+bool    vnodeShouldRemoveWal(SVnode *pVnode);
 int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t diskPrimary, STfs *pTfs);
 int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnodeHashRangeReq *pReq,
                             int32_t diskPrimary, STfs *pTfs);
@@ -181,7 +182,7 @@ void         tsdbReaderSetNotifyCb(STsdbReader *pReader, TsdReaderNotifyCbFn not
 int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables);
 int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
                                 SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr,
-                                SArray *pFuncTypeList, SColumnInfo* pkCol, int32_t numOfPks);
+                                SArray *pFuncTypeList, SColumnInfo *pkCol, int32_t numOfPks);
 int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
                               SArray *pTableUids);
 void   *tsdbCacherowsReaderClose(void *pReader);
@@ -218,8 +219,8 @@ typedef struct STqReader {
 STqReader *tqReaderOpen(SVnode *pVnode);
 void       tqReaderClose(STqReader *);
 
-bool tqGetTablePrimaryKey(STqReader* pReader);
+bool tqGetTablePrimaryKey(STqReader *pReader);
-void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid);
+void tqSetTablePrimaryKey(STqReader *pReader, int64_t uid);
 
 void    tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList);
 int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList, const char *id);
@@ -278,8 +279,8 @@ struct STsdbCfg {
   int32_t keep2;  // just for save config, don't use in tsdbRead/tsdbCommit/..., and use STsdbKeepCfg in STsdb instead
   int32_t keepTimeOffset;  // just for save config, use STsdbKeepCfg in STsdb instead
   SRetention retentions[TSDB_RETENTION_MAX];
-  int32_t encryptAlgorithm;
+  int32_t    encryptAlgorithm;
-  char    encryptKey[ENCRYPT_KEY_LEN + 1];
+  char       encryptKey[ENCRYPT_KEY_LEN + 1];
 };
 
 typedef struct {

source/dnode/vnode/src/vnd/vnodeCfg.c

@@ -13,9 +13,9 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "tglobal.h"
 #include "tutil.h"
 #include "vnd.h"
-#include "tglobal.h"
 
 const SVnodeCfg vnodeCfgDefault = {.vgId = -1,
                                    .dbname = "",
@@ -47,6 +47,7 @@ const SVnodeCfg vnodeCfgDefault = {.vgId = -1,
                                            .segSize = 0,
                                            .retentionSize = -1,
                                            .level = TAOS_WAL_WRITE,
+                                           .clearFiles = 0,
                                        },
                                    .hashBegin = 0,
                                    .hashEnd = 0,
@@ -142,6 +143,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
   if (tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "wal.clearFiles", pCfg->walCfg.clearFiles) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sttTrigger) < 0) return -1;
@@ -249,12 +251,11 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
   tjsonGetNumberValue(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm, code);
   if (code < 0) return -1;
 #if defined(TD_ENTERPRISE)
-  if(pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4){
+  if (pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4) {
-    if(tsEncryptKey[0] == 0){
+    if (tsEncryptKey[0] == 0) {
       terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
       return -1;
-    }
+    } else {
-    else{
       strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
     }
   }
@@ -273,15 +274,16 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
   if (code < 0) return -1;
   tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code);
   if (code < 0) return -1;
+  tjsonGetNumberValue(pJson, "wal.clearFiles", pCfg->walCfg.clearFiles, code);
+  if (code < 0) return -1;
   tjsonGetNumberValue(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm, code);
   if (code < 0) return -1;
 #if defined(TD_ENTERPRISE)
-  if(pCfg->walCfg.encryptAlgorithm == DND_CA_SM4){
+  if (pCfg->walCfg.encryptAlgorithm == DND_CA_SM4) {
-    if(tsEncryptKey[0] == 0){
+    if (tsEncryptKey[0] == 0) {
       terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
       return -1;
-    }
+    } else {
-    else{
       strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
     }
   }
@@ -289,12 +291,11 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
   tjsonGetNumberValue(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm, code);
   if (code < 0) return -1;
 #if defined(TD_ENTERPRISE)
-  if(pCfg->tdbEncryptAlgorithm == DND_CA_SM4){
+  if (pCfg->tdbEncryptAlgorithm == DND_CA_SM4) {
-    if(tsEncryptKey[0] == 0){
+    if (tsEncryptKey[0] == 0) {
       terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
       return -1;
-    }
+    } else {
-    else{
       strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
     }
   }

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -81,6 +81,8 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, int32_t diskPrimary, STfs
   return 0;
 }
 
+bool vnodeShouldRemoveWal(SVnode *pVnode) { return pVnode->config.walCfg.clearFiles == 1; }
+
 int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t diskPrimary, STfs *pTfs) {
   SVnodeInfo info = {0};
   char       dir[TSDB_FILENAME_LEN] = {0};
@@ -129,6 +131,12 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t
   }
   pCfg->changeVersion = pReq->changeVersion;
 
+  if (info.config.walCfg.clearFiles) {
+    info.config.walCfg.clearFiles = 0;
+
+    vInfo("vgId:%d, reset wal clearFiles", pReq->vgId);
+  }
+
   vInfo("vgId:%d, save config while alter, replicas:%d totalReplicas:%d selfIndex:%d changeVersion:%d", pReq->vgId,
         pCfg->replicaNum, pCfg->totalReplicaNum, pCfg->myIndex, pCfg->changeVersion);
 
@@ -486,15 +494,14 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC
 
   if (tsEnableMonitor && pVnode->monitor.insertCounter == NULL) {
     taos_counter_t *counter = NULL;
-    int32_t label_count = 7;
+    int32_t         label_count = 7;
-    const char *sample_labels[] = {VNODE_METRIC_TAG_NAME_SQL_TYPE, VNODE_METRIC_TAG_NAME_CLUSTER_ID,
+    const char     *sample_labels[] = {VNODE_METRIC_TAG_NAME_SQL_TYPE,  VNODE_METRIC_TAG_NAME_CLUSTER_ID,
-                                  VNODE_METRIC_TAG_NAME_DNODE_ID, VNODE_METRIC_TAG_NAME_DNODE_EP,
+                                       VNODE_METRIC_TAG_NAME_DNODE_ID,  VNODE_METRIC_TAG_NAME_DNODE_EP,
-                                  VNODE_METRIC_TAG_NAME_VGROUP_ID, VNODE_METRIC_TAG_NAME_USERNAME,
+                                       VNODE_METRIC_TAG_NAME_VGROUP_ID, VNODE_METRIC_TAG_NAME_USERNAME,
-                                  VNODE_METRIC_TAG_NAME_RESULT};
+                                       VNODE_METRIC_TAG_NAME_RESULT};
-    counter = taos_counter_new(VNODE_METRIC_SQL_COUNT, "counter for insert sql",
+    counter = taos_counter_new(VNODE_METRIC_SQL_COUNT, "counter for insert sql", label_count, sample_labels);
-                                                label_count, sample_labels);
+    vInfo("vgId:%d, new metric:%p", TD_VID(pVnode), counter);
-    vInfo("vgId:%d, new metric:%p",TD_VID(pVnode), counter);
+    if (taos_collector_registry_register_metric(counter) == 1) {
-    if(taos_collector_registry_register_metric(counter) == 1){
       taos_counter_destroy(counter);
       counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT);
       vInfo("vgId:%d, get metric from registry:%p", TD_VID(pVnode), counter);

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -2023,6 +2023,9 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pRe
   }
 
   if (pVnode->config.walCfg.level != req.walLevel) {
+    if (pVnode->config.walCfg.level == 0) {
+      pVnode->config.walCfg.clearFiles = 1;
+    }
     pVnode->config.walCfg.level = req.walLevel;
     walChanged = true;
   }

source/libs/parser/src/parTranslater.c

@@ -7302,7 +7302,17 @@ static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStm
                                      "Invalid option, wal_level 0 should be used with replica 1");
     }
   }
-
+#if 0
+  if (pStmt->pOptions->replica > 1 && pStmt->pOptions->walLevel < 1) {
+    SDbCfgInfo dbCfg = {0};
+    dbCfg.walLevel = -1;
+    int32_t code = getDBCfg(pCxt, pStmt->dbName, &dbCfg);
+    if (TSDB_CODE_SUCCESS == code && dbCfg.walLevel == 0) {
+      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
+                                     "Invalid option, wal_level 0 should be used with replica 1");
+    }
+  }
+#endif
   int32_t code = checkDatabaseOptions(pCxt, pStmt->dbName, pStmt->pOptions);
   if (TSDB_CODE_SUCCESS != code) {
     return code;

source/libs/wal/src/walMgmt.c

@@ -232,6 +232,12 @@ void walClose(SWal *pWal) {
   pWal->pRefHash = NULL;
   taosThreadMutexUnlock(&pWal->mutex);
 
+  if (pWal->cfg.level == TAOS_WAL_SKIP) {
+    wInfo("vgId:%d, remove all wals, path:%s", pWal->cfg.vgId, pWal->path);
+    taosRemoveDir(pWal->path);
+    taosMkDir(pWal->path);
+  }
+
   taosRemoveRef(tsWal.refSetId, pWal->refId);
 }
 

source/util/src/terror.c

@@ -253,6 +253,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ENCRYPT_KEY,      "The cluster has not b
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_CREATING,           "Database in creating status")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SYS_TABLENAME,    "Invalid system table name")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE, "Encryption is not allowed to be changed after database is created")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_WAL_LEVEL,        "Invalid option, wal_level 0 should be used with replica 1")
 
 // mnode-node
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST,      "Mnode already exists")