Zonble: Implementing AllUnigrams to autosort candidates.

- This brings a little side effect that the user candidate won't reflect duplicated user phrases anymore. However, it is meaningless to feel entangled with that since we can later introduce a file-content deduplicator for user phrase data file.
- Though this is not a hard requirement, the CNS11643 support need this module to boost the efficiency of the dev process.

Source/Engine/LanguageModel/vChewingLM.cpp

@@ -9,7 +9,6 @@
 #include "vChewingLM.h"
 #include <algorithm>
 #include <iterator>
-#include <unordered_set>
 
 using namespace vChewing;
 
@@ -46,7 +45,8 @@ void vChewingLM::loadUserPhrases(const char* userPhrasesDataPath,
     }
 }
 
-void vChewingLM::loadPhraseReplacementMap(const char* phraseReplacementPath) {
+void vChewingLM::loadPhraseReplacementMap(const char* phraseReplacementPath)
+{
     if (phraseReplacementPath) {
         m_phraseReplacement.close();
         m_phraseReplacement.open(phraseReplacementPath);
@@ -60,79 +60,37 @@ const vector<Bigram> vChewingLM::bigramsForKeys(const string& preceedingKey, con
 
 const vector<Unigram> vChewingLM::unigramsForKey(const string& key)
 {
-    vector<Unigram> unigrams;
+    vector<Unigram> allUnigrams;
     vector<Unigram> userUnigrams;
 
-    // Use unordered_set so that you don't have to do O(n*m)
     unordered_set<string> excludedValues;
-    unordered_set<string> userValues;
+    unordered_set<string> insertedValues;
 
     if (m_excludedPhrases.hasUnigramsForKey(key)) {
         vector<Unigram> excludedUnigrams = m_excludedPhrases.unigramsForKey(key);
         transform(excludedUnigrams.begin(), excludedUnigrams.end(),
                   inserter(excludedValues, excludedValues.end()),
-                  [](const Unigram &u) { return u.keyValue.value; });
+                  [](const Unigram& u) { return u.keyValue.value; });
     }
 
     if (m_userPhrases.hasUnigramsForKey(key)) {
         vector<Unigram> rawUserUnigrams = m_userPhrases.unigramsForKey(key);
-        vector<Unigram> filterredUserUnigrams;
+        userUnigrams = filterAndTransformUnigrams(rawUserUnigrams, excludedValues, insertedValues);
-
-        for (auto&& unigram : rawUserUnigrams) {
-            if (excludedValues.find(unigram.keyValue.value) == excludedValues.end()) {
-                filterredUserUnigrams.push_back(unigram);
-            }
-        }
-
-        transform(filterredUserUnigrams.begin(), filterredUserUnigrams.end(),
-                  inserter(userValues, userValues.end()),
-                  [](const Unigram &u) { return u.keyValue.value; });
-
-        if (m_phraseReplacementEnabled) {
-            for (auto&& unigram : filterredUserUnigrams) {
-                string value = unigram.keyValue.value;
-                string replacement = m_phraseReplacement.valueForKey(value);
-                if (replacement != "") {
-                    unigram.keyValue.value = replacement;
-                }
-                unigrams.push_back(unigram);
-            }
-        } else {
-            unigrams = filterredUserUnigrams;
-        }
     }
 
     if (m_languageModel.hasUnigramsForKey(key)) {
-        vector<Unigram> globalUnigrams = m_languageModel.unigramsForKey(key);
+        vector<Unigram> rawGlobalUnigrams = m_languageModel.unigramsForKey(key);
-
+        allUnigrams = filterAndTransformUnigrams(rawGlobalUnigrams, excludedValues, insertedValues);
-        for (auto&& unigram : globalUnigrams) {
-            string value = unigram.keyValue.value;
-            if (excludedValues.find(value) == excludedValues.end() &&
-                userValues.find(value) == userValues.end()) {
-                if (m_phraseReplacementEnabled) {
-                    string replacement = m_phraseReplacement.valueForKey(value);
-                    if (replacement != "") {
-                        unigram.keyValue.value = replacement;
-                    }
-                }
-                unigrams.push_back(unigram);
-            }
-        }
     }
 
-    unigrams.insert(unigrams.begin(), userUnigrams.begin(), userUnigrams.end());
+    allUnigrams.insert(allUnigrams.begin(), userUnigrams.begin(), userUnigrams.end());
-    return unigrams;
+    return allUnigrams;
 }
 
 bool vChewingLM::hasUnigramsForKey(const string& key)
 {
-    if (key == " ") {
-        return true;
-    }
-
     if (!m_excludedPhrases.hasUnigramsForKey(key)) {
-        return m_userPhrases.hasUnigramsForKey(key) ||
+        return m_userPhrases.hasUnigramsForKey(key) || m_languageModel.hasUnigramsForKey(key);
-        m_languageModel.hasUnigramsForKey(key);
     }
 
     return unigramsForKey(key).size() > 0;
@@ -140,10 +98,31 @@ bool vChewingLM::hasUnigramsForKey(const string& key)
 
 void vChewingLM::setPhraseReplacementEnabled(bool enabled)
 {
-        m_phraseReplacementEnabled = enabled;
+    m_phraseReplacementEnabled = enabled;
 }
 
 bool vChewingLM::phraseReplacementEnabled()
 {
     return m_phraseReplacementEnabled;
 }
+
+const vector<Unigram> vChewingLM::filterAndTransformUnigrams(vector<Unigram> unigrams, const unordered_set<string>& excludedValues, unordered_set<string>& insertedValues)
+{
+    vector<Unigram> results;
+
+    for (auto&& unigram : unigrams) {
+        string value = unigram.keyValue.value;
+        if (m_phraseReplacementEnabled) {
+            string replacement = m_phraseReplacement.valueForKey(value);
+            if (replacement != "") {
+                value = replacement;
+                unigram.keyValue.value = value;
+            }
+        }
+        if (excludedValues.find(value) == excludedValues.end() && insertedValues.find(value) == insertedValues.end()) {
+            results.push_back(unigram);
+            insertedValues.insert(value);
+        }
+    }
+    return results;
+}

Source/Engine/LanguageModel/vChewingLM.h

@@ -10,9 +10,10 @@
 #define VCHEWINGLM_H
 
 #include <stdio.h>
-#include "FastLM.h"
 #include "UserPhrasesLM.h"
+#include "FastLM.h"
 #include "PhraseReplacementMap.h"
+#include <unordered_set>
 
 namespace vChewing {
 
@@ -23,9 +24,8 @@ public:
     vChewingLM();
     ~vChewingLM();
 
-    void loadLanguageModel(const char* languageModelDataPath);
+    void loadLanguageModel(const char* languageModelPath);
-    void loadUserPhrases(const char* userPhrasesDataPath,
+    void loadUserPhrases(const char* userPhrasesPath, const char* excludedPhrasesPath);
-                         const char* excludedPhrasesDataPath);
     void loadPhraseReplacementMap(const char* phraseReplacementPath);
 
     const vector<Bigram> bigramsForKeys(const string& preceedingKey, const string& key);
@@ -36,6 +36,10 @@ public:
     bool phraseReplacementEnabled();
 
 protected:
+    const vector<Unigram> filterAndTransformUnigrams(vector<Unigram> unigrams,
+        const std::unordered_set<string>& excludedValues,
+        std::unordered_set<string>& insertedValues);
+
     FastLM m_languageModel;
     UserPhrasesLM m_userPhrases;
     UserPhrasesLM m_excludedPhrases;