From 8fba6c53150dde0c8ce409e498b36eed9972b9a9 Mon Sep 17 00:00:00 2001
From: ShikiSuen <shikisuen@outlook.com>
Date: Wed, 16 Feb 2022 17:24:20 +0800
Subject: [PATCH] LMConsolidator // Add another option to allow
 force-consolidate, etc.

---
 Source/Modules/FileHandlers/LMConsolidator.h               | 2 +-
 Source/Modules/FileHandlers/LMConsolidator.mm              | 4 ++--
 .../SubLanguageModels/AssociatedPhrases.mm                 | 7 +------
 .../SubLanguageModels/PhraseReplacementMap.mm              | 7 +------
 .../LangModelRelated/SubLanguageModels/UserPhrasesLM.mm    | 7 +------
 Source/Modules/LangModelRelated/mgrLangModel.mm            | 4 +++-
 6 files changed, 9 insertions(+), 22 deletions(-)

diff --git a/Source/Modules/FileHandlers/LMConsolidator.h b/Source/Modules/FileHandlers/LMConsolidator.h
index 1007c92b..3dd481ca 100644
--- a/Source/Modules/FileHandlers/LMConsolidator.h
+++ b/Source/Modules/FileHandlers/LMConsolidator.h
@@ -37,7 +37,7 @@ class LMConsolidator
 public:
     static bool CheckPragma(const char *path);
     static bool FixEOF(const char *path);
-    static bool ConsolidateContent(const char *path, bool shouldsort);
+    static bool ConsolidateContent(const char *path, bool shouldsort, bool shouldCheckPragma);
 };
 
 } // namespace vChewing
diff --git a/Source/Modules/FileHandlers/LMConsolidator.mm b/Source/Modules/FileHandlers/LMConsolidator.mm
index 7cd0f147..d3350e2f 100644
--- a/Source/Modules/FileHandlers/LMConsolidator.mm
+++ b/Source/Modules/FileHandlers/LMConsolidator.mm
@@ -71,8 +71,8 @@ bool LMConsolidator::FixEOF(const char *path)
 } // END: EOF FIXER.
 
 // CONTENT CONSOLIDATOR. CREDIT: Shiki Suen.
-bool LMConsolidator::ConsolidateContent(const char *path, bool shouldsort) {
-    if (LMConsolidator::CheckPragma(path) && !shouldsort){
+bool LMConsolidator::ConsolidateContent(const char *path, bool shouldsort, bool shouldCheckPragma) {
+    if (LMConsolidator::CheckPragma(path) && !shouldsort && shouldCheckPragma){
         return true;
     }
 
diff --git a/Source/Modules/LangModelRelated/SubLanguageModels/AssociatedPhrases.mm b/Source/Modules/LangModelRelated/SubLanguageModels/AssociatedPhrases.mm
index 53ae599a..4970f8a3 100644
--- a/Source/Modules/LangModelRelated/SubLanguageModels/AssociatedPhrases.mm
+++ b/Source/Modules/LangModelRelated/SubLanguageModels/AssociatedPhrases.mm
@@ -59,12 +59,7 @@ bool AssociatedPhrases::open(const char *path)
     }
 
     LMConsolidator::FixEOF(path);
-
-    if (Preferences.shouldAutoSortAssociatedPhrasesOnLoad) {
-        LMConsolidator::ConsolidateContent(path, true);
-    } else {
-        LMConsolidator::ConsolidateContent(path, false);
-    }
+    LMConsolidator::ConsolidateContent(path, Preferences.shouldAutoSortAssociatedPhrasesOnLoad, true);
 
     fd = ::open(path, O_RDONLY);
     if (fd == -1) {
diff --git a/Source/Modules/LangModelRelated/SubLanguageModels/PhraseReplacementMap.mm b/Source/Modules/LangModelRelated/SubLanguageModels/PhraseReplacementMap.mm
index 67aa2870..64d2071e 100644
--- a/Source/Modules/LangModelRelated/SubLanguageModels/PhraseReplacementMap.mm
+++ b/Source/Modules/LangModelRelated/SubLanguageModels/PhraseReplacementMap.mm
@@ -54,12 +54,7 @@ bool PhraseReplacementMap::open(const char *path)
     }
 
     LMConsolidator::FixEOF(path);
-
-    if (Preferences.shouldAutoSortPhraseReplacementMapOnLoad) {
-        LMConsolidator::ConsolidateContent(path, true);
-    } else {
-        LMConsolidator::ConsolidateContent(path, false);
-    }
+    LMConsolidator::ConsolidateContent(path, Preferences.shouldAutoSortPhraseReplacementMapOnLoad, true);
 
     fd = ::open(path, O_RDONLY);
     if (fd == -1) {
diff --git a/Source/Modules/LangModelRelated/SubLanguageModels/UserPhrasesLM.mm b/Source/Modules/LangModelRelated/SubLanguageModels/UserPhrasesLM.mm
index 2219117d..f23206f8 100644
--- a/Source/Modules/LangModelRelated/SubLanguageModels/UserPhrasesLM.mm
+++ b/Source/Modules/LangModelRelated/SubLanguageModels/UserPhrasesLM.mm
@@ -60,12 +60,7 @@ bool UserPhrasesLM::open(const char *path)
     }
 
     LMConsolidator::FixEOF(path);
-
-    if (Preferences.shouldAutoSortUserPhrasesAndExclListOnLoad) {
-        LMConsolidator::ConsolidateContent(path, true);
-    } else {
-        LMConsolidator::ConsolidateContent(path, false);
-    }
+    LMConsolidator::ConsolidateContent(path, Preferences.shouldAutoSortUserPhrasesAndExclListOnLoad, true);
 
     fd = ::open(path, O_RDONLY);
     if (fd == -1) {
diff --git a/Source/Modules/LangModelRelated/mgrLangModel.mm b/Source/Modules/LangModelRelated/mgrLangModel.mm
index 9555b122..bb93347f 100644
--- a/Source/Modules/LangModelRelated/mgrLangModel.mm
+++ b/Source/Modules/LangModelRelated/mgrLangModel.mm
@@ -252,7 +252,9 @@ static void LTLoadLanguageModelFile(NSString *filenameWithoutExtension, vChewing
     // }
     [currentMarkedPhrase appendString:userPhrase];
     if (areWeDuplicating) {
-        [currentMarkedPhrase appendString:@"\tFreqOverrideByDuplication"];
+        // Do not use ASCII characters to comment here.
+        // Otherwise, it will be scrambled by HYPY2BPMF module shipped in the vChewing Phrase Editor.
+        [currentMarkedPhrase appendString:@"\t#𝙾𝚟𝚎𝚛𝚛𝚒𝚍𝚎"];
     }
     [currentMarkedPhrase appendString:@"\n"];