Use a parseless phrase db to speed up LM loading

We take advantage of the fact that no one is able to modify the phrase
databases shipped with the binary (guranteed by macOS's integrity check
for notarized apps), and we can simply pre-sort the phrases in the
database files.

With this change, we can speed up McBopomofo's language model loading
during the app initialization by about 500-800x on a 2018 Intel MacBook
Pro. The LM loading used to take 300-400 ms, but now it's done within a
sub-millisecond range (0.5-0.6 ms). Microbenchmarking shows that
ParselessLM is about 16000x faster than FastLM. We amortize the latency
during the query time, and even by deferring the parsing, ParselessLM is
only ~1.5x slower than FastLM, and both LM classes serve queries unedr 6
microseconds (that's 0.006 ms), which means the tradeoff only
contributes to neglible overall latency.

This PR requires some small changes to the phrase db cooking scripts.
Python 3 is now used and the (value, reading, score) tuples are
rearranged to (reading, value, score) and sorted by reading ("key"). A
header is added to the phrase databases to call out the fact that these
are pre-sorted.

clang-format is used to apply WebKit C++ style to the new code. This
also applies to KeyValueBlobReader that was added recently.

Microbenchmark result below:

```
---------------------------------------------------------------------
Benchmark                           Time             CPU   Iterations
---------------------------------------------------------------------
BM_ParselessLMOpenClose         17710 ns        17199 ns        33422
BM_FastLMOpenClose          376520248 ns    367526500 ns            2
BM_ParselessLMFindUnigrams       5967 ns         5899 ns       113729
BM_FastLMFindUnigrams            2268 ns         2265 ns       307038
```

McBopomofo.xcodeproj/project.pbxproj

@@ -7,7 +7,6 @@
 	objects = {
 
 /* Begin PBXBuildFile section */
-		6A0421A815FEF3F50061ED63 /* FastLM.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 6A0421A615FEF3F50061ED63 /* FastLM.cpp */; };
 		6A0D4EA715FC0D2D00ABF4B3 /* Cocoa.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 6A0D4EA615FC0D2D00ABF4B3 /* Cocoa.framework */; };
 		6A0D4ED215FC0D6400ABF4B3 /* InputMethodController.mm in Sources */ = {isa = PBXBuildFile; fileRef = 6A0D4EC715FC0D6400ABF4B3 /* InputMethodController.mm */; };
 		6A0D4ED315FC0D6400ABF4B3 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 6A0D4EC815FC0D6400ABF4B3 /* main.m */; };
@@ -33,6 +32,8 @@
 		6ACA41FF15FC1D9000935EF6 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 6ACA41F415FC1D9000935EF6 /* main.m */; };
 		6ACA420215FC1E5200935EF6 /* McBopomofo.app in Resources */ = {isa = PBXBuildFile; fileRef = 6A0D4EA215FC0D2D00ABF4B3 /* McBopomofo.app */; };
 		6ACC3D3F27914F2400F1B140 /* KeyValueBlobReader.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 6ACC3D3E27914F2400F1B140 /* KeyValueBlobReader.cpp */; };
+		6ACC3D442793701600F1B140 /* ParselessPhraseDB.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 6ACC3D402793701600F1B140 /* ParselessPhraseDB.cpp */; };
+		6ACC3D452793701600F1B140 /* ParselessLM.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 6ACC3D422793701600F1B140 /* ParselessLM.cpp */; };
 		6AD7CBC815FE555000691B5B /* data-plain-bpmf.txt in Resources */ = {isa = PBXBuildFile; fileRef = 6AD7CBC715FE555000691B5B /* data-plain-bpmf.txt */; };
 		6AE210B215FC63CC003659FE /* PlainBopomofo.tiff in Resources */ = {isa = PBXBuildFile; fileRef = 6AE210B015FC63CC003659FE /* PlainBopomofo.tiff */; };
 		6AE210B315FC63CC003659FE /* PlainBopomofo@2x.tiff in Resources */ = {isa = PBXBuildFile; fileRef = 6AE210B115FC63CC003659FE /* PlainBopomofo@2x.tiff */; };
@@ -74,8 +75,6 @@
 /* End PBXContainerItemProxy section */
 
 /* Begin PBXFileReference section */
-		6A0421A615FEF3F50061ED63 /* FastLM.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = FastLM.cpp; sourceTree = "<group>"; };
-		6A0421A715FEF3F50061ED63 /* FastLM.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FastLM.h; sourceTree = "<group>"; };
 		6A0D4EA215FC0D2D00ABF4B3 /* McBopomofo.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = McBopomofo.app; sourceTree = BUILT_PRODUCTS_DIR; };
 		6A0D4EA615FC0D2D00ABF4B3 /* Cocoa.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Cocoa.framework; path = System/Library/Frameworks/Cocoa.framework; sourceTree = SDKROOT; };
 		6A0D4EA915FC0D2D00ABF4B3 /* AppKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = AppKit.framework; path = System/Library/Frameworks/AppKit.framework; sourceTree = SDKROOT; };
@@ -162,6 +161,10 @@
 		6ACA41F815FC1D9000935EF6 /* zh-Hant */ = {isa = PBXFileReference; lastKnownFileType = file.xib; name = "zh-Hant"; path = "zh-Hant.lproj/MainMenu.xib"; sourceTree = "<group>"; };
 		6ACC3D3C27914AAB00F1B140 /* KeyValueBlobReader.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = KeyValueBlobReader.h; sourceTree = "<group>"; };
 		6ACC3D3E27914F2400F1B140 /* KeyValueBlobReader.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = KeyValueBlobReader.cpp; sourceTree = "<group>"; };
+		6ACC3D402793701600F1B140 /* ParselessPhraseDB.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ParselessPhraseDB.cpp; sourceTree = "<group>"; };
+		6ACC3D412793701600F1B140 /* ParselessPhraseDB.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ParselessPhraseDB.h; sourceTree = "<group>"; };
+		6ACC3D422793701600F1B140 /* ParselessLM.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ParselessLM.cpp; sourceTree = "<group>"; };
+		6ACC3D432793701600F1B140 /* ParselessLM.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ParselessLM.h; sourceTree = "<group>"; };
 		6AD7CBC715FE555000691B5B /* data-plain-bpmf.txt */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = "data-plain-bpmf.txt"; sourceTree = "<group>"; };
 		6AE210B015FC63CC003659FE /* PlainBopomofo.tiff */ = {isa = PBXFileReference; lastKnownFileType = image.tiff; path = PlainBopomofo.tiff; sourceTree = "<group>"; };
 		6AE210B115FC63CC003659FE /* PlainBopomofo@2x.tiff */ = {isa = PBXFileReference; lastKnownFileType = image.tiff; path = "PlainBopomofo@2x.tiff"; sourceTree = "<group>"; };
@@ -289,12 +292,14 @@
 				6A0D4F1315FC0EB100ABF4B3 /* Gramambular */,
 				6A0D4F1F15FC0EB100ABF4B3 /* Mandarin */,
 				6A0D4F2215FC0EB100ABF4B3 /* OpenVanilla */,
-				6A0421A615FEF3F50061ED63 /* FastLM.cpp */,
-				6A0421A715FEF3F50061ED63 /* FastLM.h */,
 				6ACC3D3E27914F2400F1B140 /* KeyValueBlobReader.cpp */,
 				6ACC3D3C27914AAB00F1B140 /* KeyValueBlobReader.h */,
 				D41355DC278EA3ED005E5CBD /* UserPhrasesLM.cpp */,
 				D41355DD278EA3ED005E5CBD /* UserPhrasesLM.h */,
+				6ACC3D422793701600F1B140 /* ParselessLM.cpp */,
+				6ACC3D432793701600F1B140 /* ParselessLM.h */,
+				6ACC3D402793701600F1B140 /* ParselessPhraseDB.cpp */,
+				6ACC3D412793701600F1B140 /* ParselessPhraseDB.h */,
 				D44FB74B2792189A003C80A6 /* PhraseReplacementMap.cpp */,
 				D44FB74C2792189A003C80A6 /* PhraseReplacementMap.h */,
 				D41355D9278E6D17005E5CBD /* McBopomofoLM.cpp */,
@@ -592,6 +597,7 @@
 			buildActionMask = 2147483647;
 			files = (
 				D427F76C278CA2B0004A2160 /* AppDelegate.swift in Sources */,
+				6ACC3D442793701600F1B140 /* ParselessPhraseDB.cpp in Sources */,
 				D44FB74727919D35003C80A6 /* EmacsKeyHelper.swift in Sources */,
 				6A0D4ED315FC0D6400ABF4B3 /* main.m in Sources */,
 				D44FB74D2792189A003C80A6 /* PhraseReplacementMap.cpp in Sources */,
@@ -602,9 +608,9 @@
 				D41355DB278E6D17005E5CBD /* McBopomofoLM.cpp in Sources */,
 				D47F7DD3278C1263002F9DD7 /* UserOverrideModel.cpp in Sources */,
 				6A0D4F4515FC0EB100ABF4B3 /* Mandarin.cpp in Sources */,
+				6ACC3D452793701600F1B140 /* ParselessLM.cpp in Sources */,
 				D41355DE278EA3ED005E5CBD /* UserPhrasesLM.cpp in Sources */,
 				6ACC3D3F27914F2400F1B140 /* KeyValueBlobReader.cpp in Sources */,
-				6A0421A815FEF3F50061ED63 /* FastLM.cpp in Sources */,
 				D41355D8278D74B5005E5CBD /* LanguageModelManager.mm in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;

Source/Engine/CMakeLists.txt

@@ -1,9 +1,16 @@
 cmake_minimum_required(VERSION 3.17)
-project(KeyValueBlobReader)
+project(McBopomofoLMLib)
 
 set(CMAKE_CXX_STANDARD 17)
 
-add_library(KeyValueBlobReader KeyValueBlobReader.cpp KeyValueBlobReader.h)
+include_directories("Gramambular")
+add_library(McBopomofoLMLib
+        KeyValueBlobReader.cpp
+        KeyValueBlobReader.h
+        ParselessPhraseDB.cpp
+        ParselessPhraseDB.h
+        ParselessLM.cpp
+        ParselessLM.h)
 
 # Let CMake fetch Google Test for us.
 # https://github.com/google/googletest/tree/main/googletest#incorporating-into-an-existing-cmake-project
@@ -19,6 +26,17 @@ set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
 FetchContent_MakeAvailable(googletest)
 
 # Test target declarations.
-add_executable(KeyValueBlobReadTest KeyValueBlobReaderTest.cpp)
+add_executable(McBopomofoLMLibTest
-target_link_libraries(KeyValueBlobReadTest gtest_main KeyValueBlobReader)
+        KeyValueBlobReaderTest.cpp
-add_test(NAME KeyValueBlobReadTest COMMAND KeyValueBlobReadTest)
+        ParselessLMTest.cpp
+        ParselessPhraseDBTest.cpp)
+target_link_libraries(McBopomofoLMLibTest gtest_main McBopomofoLMLib)
+include(GoogleTest)
+gtest_discover_tests(McBopomofoLMLibTest)
+
+# Benchmark target.
+find_package(benchmark REQUIRED)
+add_executable(ParselessLMBenchmark
+        FastLM.cpp
+        ParselessLMBenchmark.cpp)
+target_link_libraries(ParselessLMBenchmark McBopomofoLMLib benchmark::benchmark)

Source/Engine/KeyValueBlobReader.cpp

@@ -25,116 +25,118 @@
 
 namespace McBopomofo {
 
-KeyValueBlobReader::State KeyValueBlobReader::Next(KeyValue* out) {
+KeyValueBlobReader::State KeyValueBlobReader::Next(KeyValue* out)
-  static auto new_line = [](char c) { return c == '\n' || c == '\r'; };
+{
-  static auto blank = [](char c) { return c == ' ' || c == '\t'; };
+    static auto new_line = [](char c) { return c == '\n' || c == '\r'; };
-  static auto blank_or_newline = [](char c) { return blank(c) || new_line(c); };
+    static auto blank = [](char c) { return c == ' ' || c == '\t'; };
-  static auto content_char = [](char c) {
+    static auto blank_or_newline
-    return !blank(c) && !new_line(c);
+        = [](char c) { return blank(c) || new_line(c); };
-  };
+    static auto content_char = [](char c) { return !blank(c) && !new_line(c); };
 
-  if (state_ == State::ERROR) {
+    if (state_ == State::ERROR) {
+        return state_;
+    }
+
+    const char* key_begin = nullptr;
+    size_t key_length = 0;
+    const char* value_begin = nullptr;
+    size_t value_length = 0;
+
+    while (true) {
+        state_ = SkipUntilNot(blank_or_newline);
+        if (state_ != State::CAN_CONTINUE) {
+            return state_;
+        }
+
+        // Check if it's a comment line; if so, read until end of line.
+        if (*current_ != '#') {
+            break;
+        }
+        state_ = SkipUntil(new_line);
+        if (state_ != State::CAN_CONTINUE) {
+            return state_;
+        }
+    }
+
+    // No need to check whether* current_ is a content_char, since content_char
+    // is defined as not blank and not new_line.
+
+    key_begin = current_;
+    state_ = SkipUntilNot(content_char);
+    if (state_ != State::CAN_CONTINUE) {
+        goto error;
+    }
+    key_length = current_ - key_begin;
+
+    // There should be at least one blank character after the key string.
+    if (!blank(*current_)) {
+        goto error;
+    }
+
+    state_ = SkipUntilNot(blank);
+    if (state_ != State::CAN_CONTINUE) {
+        goto error;
+    }
+
+    if (!content_char(*current_)) {
+        goto error;
+    }
+
+    value_begin = current_;
+    // value must only contain content characters, blanks not are allowed.
+    // also, there's no need to check the state after this, since we will always
+    // emit the value. This also avoids the situation where trailing spaces in a
+    // line would become part of the value.
+    SkipUntilNot(content_char);
+    value_length = current_ - value_begin;
+
+    // Unconditionally skip until the end of the line. This prevents the case
+    // like "foo bar baz\n" where baz should not be treated as the Next key.
+    SkipUntil(new_line);
+
+    if (out != nullptr) {
+        *out = KeyValue { std::string_view { key_begin, key_length },
+            std::string_view { value_begin, value_length } };
+    }
+    state_ = State::HAS_PAIR;
     return state_;
-  }
-
-  const char* key_begin = nullptr;
-  size_t key_length = 0;
-  const char* value_begin = nullptr;
-  size_t value_length = 0;
-
-  while (true) {
-    state_ = SkipUntilNot(blank_or_newline);
-    if (state_ != State::CAN_CONTINUE) {
-      return state_;
-    }
-
-    // Check if it's a comment line; if so, read until end of line.
-    if (*current_ != '#') {
-      break;
-    }
-    state_ = SkipUntil(new_line);
-    if (state_ != State::CAN_CONTINUE) {
-      return state_;
-    }
-  }
-
-  // No need to check whether* current_ is a content_char, since content_char
-  // is defined as not blank and not new_line.
-
-  key_begin = current_;
-  state_ = SkipUntilNot(content_char);
-  if (state_ != State::CAN_CONTINUE) {
-    goto error;
-  }
-  key_length = current_ - key_begin;
-
-  // There should be at least one blank character after the key string.
-  if (!blank(*current_)) {
-    goto error;
-  }
-
-  state_ = SkipUntilNot(blank);
-  if (state_ != State::CAN_CONTINUE) {
-    goto error;
-  }
-
-  if (!content_char(*current_)) {
-    goto error;
-  }
-
-  value_begin = current_;
-  // value must only contain content characters, blanks not are allowed.
-  // also, there's no need to check the state after this, since we will always
-  // emit the value. This also avoids the situation where trailing spaces in a
-  // line would become part of the value.
-  SkipUntilNot(content_char);
-  value_length = current_ - value_begin;
-
-  // Unconditionally skip until the end of the line. This prevents the case
-  // like "foo bar baz\n" where baz should not be treated as the Next key.
-  SkipUntil(new_line);
-
-  if (out != nullptr) {
-    *out = KeyValue{
-      std::string_view{key_begin, key_length},
-      std::string_view{value_begin, value_length}};
-  }
-  state_ = State::HAS_PAIR;
-  return state_;
 
 error:
-  state_ = State::ERROR;
+    state_ = State::ERROR;
-  return State::ERROR;
+    return state_;
 }
 
 KeyValueBlobReader::State KeyValueBlobReader::SkipUntilNot(
-    const std::function<bool(char)>& f) {
+    const std::function<bool(char)>& f)
-  while (current_ != end_ &&* current_) {
+{
-    if (!f(*current_)) {
+    while (current_ != end_ && *current_) {
-      return State::CAN_CONTINUE;
+        if (!f(*current_)) {
+            return State::CAN_CONTINUE;
+        }
+        ++current_;
     }
-    ++current_;
-  }
 
-  return State::END;
+    return State::END;
 }
 
 KeyValueBlobReader::State KeyValueBlobReader::SkipUntil(
-    const std::function<bool(char)>& f) {
+    const std::function<bool(char)>& f)
-  while (current_ != end_ &&* current_) {
+{
-    if (f(*current_)) {
+    while (current_ != end_ && *current_) {
-      return State::CAN_CONTINUE;
+        if (f(*current_)) {
+            return State::CAN_CONTINUE;
+        }
+        ++current_;
     }
-    ++current_;
-  }
 
-  return State::END;
+    return State::END;
 }
 
-std::ostream& operator<<(std::ostream& os,
+std::ostream& operator<<(
-                         const KeyValueBlobReader::KeyValue& kv) {
+    std::ostream& os, const KeyValueBlobReader::KeyValue& kv)
-  os << "(key: " << kv.key << ", value: " << kv.value << ")";
+{
-  return os;
+    os << "(key: " << kv.key << ", value: " << kv.value << ")";
+    return os;
 }
 
-}  // namespace McBopomofo
+} // namespace McBopomofo

Source/Engine/KeyValueBlobReader.h

@@ -46,49 +46,61 @@
 namespace McBopomofo {
 
 class KeyValueBlobReader {
- public:
+public:
-  enum class State : int {
+    enum class State : int {
-    // There are no more key-value pairs in this blob.
+        // There are no more key-value pairs in this blob.
-    END = 0,
+        END = 0,
-    // The reader has produced a new key-value pair.
+        // The reader has produced a new key-value pair.
-    HAS_PAIR = 1,
+        HAS_PAIR = 1,
-    // An error is encountered and the parsing stopped.
+        // An error is encountered and the parsing stopped.
-    ERROR = -1,
+        ERROR = -1,
-    // Internal-only state: the parser can continue parsing.
+        // Internal-only state: the parser can continue parsing.
-    CAN_CONTINUE = 2
+        CAN_CONTINUE = 2
-  };
+    };
 
-  struct KeyValue {
+    struct KeyValue {
-    constexpr KeyValue() : key(""), value("") {}
+        constexpr KeyValue()
-    constexpr KeyValue(std::string_view k, std::string_view v)
+            : key("")
-        : key(k), value(v) {}
+            , value("")
+        {
+        }
+        constexpr KeyValue(std::string_view k, std::string_view v)
+            : key(k)
+            , value(v)
+        {
+        }
 
-    bool operator==(const KeyValue& another) const {
+        bool operator==(const KeyValue& another) const
-      return key == another.key && value == another.value;
+        {
+            return key == another.key && value == another.value;
+        }
+
+        std::string_view key;
+        std::string_view value;
+    };
+
+    KeyValueBlobReader(const char* blob, size_t size)
+        : current_(blob)
+        , end_(blob + size)
+    {
     }
 
-    std::string_view key;
+    // Parse the next key-value pair and return the state of the reader. If
-    std::string_view value;
+    // `out` is passed, out will be set to the produced key-value pair if there
-  };
+    // is one.
+    State Next(KeyValue* out = nullptr);
 
-  KeyValueBlobReader(const char* blob, size_t size)
+private:
-      : current_(blob), end_(blob + size) {}
+    State SkipUntil(const std::function<bool(char)>& f);
+    State SkipUntilNot(const std::function<bool(char)>& f);
 
-  // Parse the next key-value pair and return the state of the reader. If `out`
+    const char* current_;
-  // is passed, out will be set to the produced key-value pair if there is one.
+    const char* end_;
-  State Next(KeyValue* out = nullptr);
+    State state_ = State::CAN_CONTINUE;
-
- private:
-  State SkipUntil(const std::function<bool(char)>& f);
-  State SkipUntilNot(const std::function<bool(char)>& f);
-
-  const char* current_;
-  const char* end_;
-  State state_ = State::CAN_CONTINUE;
 };
 
 std::ostream& operator<<(std::ostream&, const KeyValueBlobReader::KeyValue&);
 
-}  // namespace McBopomofo
+} // namespace McBopomofo
 
-#endif  // SOURCE_ENGINE_KEYVALUEBLOBREADER_H_
+#endif // SOURCE_ENGINE_KEYVALUEBLOBREADER_H_

Source/Engine/KeyValueBlobReaderTest.cpp

@@ -21,9 +21,9 @@
 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 // OTHER DEALINGS IN THE SOFTWARE.
 
-#include "KeyValueBlobReader.h"
-
 #include <string>
+
+#include "KeyValueBlobReader.h"
 #include "gtest/gtest.h"
 
 namespace McBopomofo {
@@ -31,156 +31,175 @@ namespace McBopomofo {
 using State = KeyValueBlobReader::State;
 using KeyValue = KeyValueBlobReader::KeyValue;
 
-TEST(KeyValueBlobReaderTest, EmptyBlob) {
+TEST(KeyValueBlobReaderTest, EmptyBlob)
-  std::string empty;
+{
-  KeyValueBlobReader reader(empty.c_str(), empty.length());
+    std::string empty;
-  EXPECT_EQ(reader.Next(), State::END);
+    KeyValueBlobReader reader(empty.c_str(), empty.length());
+    EXPECT_EQ(reader.Next(), State::END);
 }
 
-TEST(KeyValueBlobReaderTest, EmptyBlobIdempotency) {
+TEST(KeyValueBlobReaderTest, EmptyBlobIdempotency)
-  char empty[0];
+{
-  KeyValueBlobReader reader(empty, 0);
+    char empty[0];
-  EXPECT_EQ(reader.Next(), State::END);
+    KeyValueBlobReader reader(empty, 0);
-  EXPECT_EQ(reader.Next(), State::END);
+    EXPECT_EQ(reader.Next(), State::END);
+    EXPECT_EQ(reader.Next(), State::END);
 }
 
-TEST(KeyValueBlobReaderTest, BlankBlob) {
+TEST(KeyValueBlobReaderTest, BlankBlob)
-  std::string blank = " ";
+{
-  KeyValueBlobReader reader(blank.c_str(), blank.length());
+    std::string blank = " ";
-  EXPECT_EQ(reader.Next(), State::END);
+    KeyValueBlobReader reader(blank.c_str(), blank.length());
+    EXPECT_EQ(reader.Next(), State::END);
 }
 
-TEST(KeyValueBlobReaderTest, KeyWithoutValueIsInvalid) {
+TEST(KeyValueBlobReaderTest, KeyWithoutValueIsInvalid)
-  std::string empty = "hello";
+{
-  KeyValueBlobReader reader(empty.c_str(), empty.length());
+    std::string empty = "hello";
-  EXPECT_EQ(reader.Next(), State::ERROR);
+    KeyValueBlobReader reader(empty.c_str(), empty.length());
+    EXPECT_EQ(reader.Next(), State::ERROR);
 }
 
-TEST(KeyValueBlobReaderTest, ErrorStateMakesNoMoreProgress) {
+TEST(KeyValueBlobReaderTest, ErrorStateMakesNoMoreProgress)
-  std::string empty = "hello";
+{
-  KeyValueBlobReader reader(empty.c_str(), empty.length());
+    std::string empty = "hello";
-  EXPECT_EQ(reader.Next(), State::ERROR);
+    KeyValueBlobReader reader(empty.c_str(), empty.length());
-  EXPECT_EQ(reader.Next(), State::ERROR);
+    EXPECT_EQ(reader.Next(), State::ERROR);
+    EXPECT_EQ(reader.Next(), State::ERROR);
 }
 
-TEST(KeyValueBlobReaderTest, KeyValueSeparatedByNullCharIsInvalid) {
+TEST(KeyValueBlobReaderTest, KeyValueSeparatedByNullCharIsInvalid)
-  char bad[] = {'h', 0, 'w'};
+{
-  KeyValueBlobReader reader(bad, sizeof(bad));
+    char bad[] = { 'h', 0, 'w' };
-  EXPECT_EQ(reader.Next(), State::ERROR);
+    KeyValueBlobReader reader(bad, sizeof(bad));
+    EXPECT_EQ(reader.Next(), State::ERROR);
 }
 
-TEST(KeyValueBlobReaderTest, SingleKeyValuePair) {
+TEST(KeyValueBlobReaderTest, SingleKeyValuePair)
-  std::string empty = "hello world\n";
+{
-  KeyValueBlobReader reader(empty.c_str(), empty.length());
+    std::string empty = "hello world\n";
-  KeyValue keyValue;
+    KeyValueBlobReader reader(empty.c_str(), empty.length());
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    KeyValue keyValue;
-  EXPECT_EQ(keyValue, (KeyValue{"hello", "world"}));
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(reader.Next(), State::END);
+    EXPECT_EQ(keyValue, (KeyValue { "hello", "world" }));
+    EXPECT_EQ(reader.Next(), State::END);
 }
 
-TEST(KeyValueBlobReaderTest, SingleKeyValuePairFromButterWithoutNullEnding) {
+TEST(KeyValueBlobReaderTest, SingleKeyValuePairFromButterWithoutNullEnding)
-  char small[] = {'p', ' ', 'q'};
+{
-  KeyValueBlobReader reader(small, sizeof(small));
+    char small[] = { 'p', ' ', 'q' };
-  KeyValue keyValue;
+    KeyValueBlobReader reader(small, sizeof(small));
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    KeyValue keyValue;
-  EXPECT_EQ(keyValue, (KeyValue{"p", "q"}));
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(reader.Next(), State::END);
+    EXPECT_EQ(keyValue, (KeyValue { "p", "q" }));
+    EXPECT_EQ(reader.Next(), State::END);
 }
 
-TEST(KeyValueBlobReaderTest, NullCharInTheMiddleTerminatesParsing) {
+TEST(KeyValueBlobReaderTest, NullCharInTheMiddleTerminatesParsing)
-  char small[] = {'p', ' ', 'q', ' ', 0, '\n', 'r', ' ', 's'};
+{
-  KeyValueBlobReader reader(small, sizeof(small));
+    char small[] = { 'p', ' ', 'q', ' ', 0, '\n', 'r', ' ', 's' };
-  KeyValue keyValue;
+    KeyValueBlobReader reader(small, sizeof(small));
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    KeyValue keyValue;
-  EXPECT_EQ(keyValue, (KeyValue{"p", "q"}));
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(reader.Next(), State::END);
+    EXPECT_EQ(keyValue, (KeyValue { "p", "q" }));
+    EXPECT_EQ(reader.Next(), State::END);
 }
 
-TEST(KeyValueBlobReaderTest, SingleKeyValuePairWithoutLFAtEnd) {
+TEST(KeyValueBlobReaderTest, SingleKeyValuePairWithoutLFAtEnd)
-  std::string simple = "hello world";
+{
-  KeyValueBlobReader reader(simple.c_str(), simple.length());
+    std::string simple = "hello world";
-  KeyValue keyValue;
+    KeyValueBlobReader reader(simple.c_str(), simple.length());
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    KeyValue keyValue;
-  EXPECT_EQ(keyValue, (KeyValue{"hello", "world"}));
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(reader.Next(), State::END);
+    EXPECT_EQ(keyValue, (KeyValue { "hello", "world" }));
+    EXPECT_EQ(reader.Next(), State::END);
 }
 
-TEST(KeyValueBlobReaderTest, EncodingAgnostic1) {
+TEST(KeyValueBlobReaderTest, EncodingAgnostic1)
-  std::string simple = u8"smile ☺️";
+{
-  KeyValueBlobReader reader(simple.c_str(), simple.length());
+    std::string simple = u8"smile ☺️";
-  KeyValue keyValue;
+    KeyValueBlobReader reader(simple.c_str(), simple.length());
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    KeyValue keyValue;
-  EXPECT_EQ(keyValue, (KeyValue{"smile", u8"☺️"}));
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(reader.Next(), State::END);
+    EXPECT_EQ(keyValue, (KeyValue { "smile", u8"☺️" }));
+    EXPECT_EQ(reader.Next(), State::END);
 }
 
-TEST(KeyValueBlobReaderTest, EncodingAgnostic2) {
+TEST(KeyValueBlobReaderTest, EncodingAgnostic2)
-  std::string simple = "Nobel-Laureate "
+{
-                       "\xe9\x81\x94\xe8\xb3\xb4\xe5\x96\x87\xe5\x98\x9b";
+    std::string simple = "Nobel-Laureate "
-  KeyValueBlobReader reader(simple.c_str(), simple.length());
+                         "\xe9\x81\x94\xe8\xb3\xb4\xe5\x96\x87\xe5\x98\x9b";
-  KeyValue keyValue;
+    KeyValueBlobReader reader(simple.c_str(), simple.length());
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    KeyValue keyValue;
-  EXPECT_EQ(keyValue, (
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-      KeyValue{"Nobel-Laureate",
+    EXPECT_EQ(keyValue,
-               "\xe9\x81\x94\xe8\xb3\xb4\xe5\x96\x87\xe5\x98\x9b"}));
+        (KeyValue { "Nobel-Laureate",
-  EXPECT_EQ(reader.Next(), State::END);
+            "\xe9\x81\x94\xe8\xb3\xb4\xe5\x96\x87\xe5\x98\x9b" }));
+    EXPECT_EQ(reader.Next(), State::END);
 }
 
-TEST(KeyValueBlobReaderTest, ValueDoesNotIncludeSpace) {
+TEST(KeyValueBlobReaderTest, ValueDoesNotIncludeSpace)
-  std::string simple = "hello world and all\nanother value";
+{
-  KeyValueBlobReader reader(simple.c_str(), simple.length());
+    std::string simple = "hello world and all\nanother value";
-  KeyValue keyValue;
+    KeyValueBlobReader reader(simple.c_str(), simple.length());
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    KeyValue keyValue;
-  EXPECT_EQ(keyValue, (KeyValue{"hello", "world"}));
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    EXPECT_EQ(keyValue, (KeyValue { "hello", "world" }));
-  EXPECT_EQ(keyValue, (KeyValue{"another", "value"}));
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(reader.Next(), State::END);
+    EXPECT_EQ(keyValue, (KeyValue { "another", "value" }));
+    EXPECT_EQ(reader.Next(), State::END);
 }
 
-TEST(KeyValueBlobReaderTest, TrailingSpaceInValueIsIgnored) {
+TEST(KeyValueBlobReaderTest, TrailingSpaceInValueIsIgnored)
-  std::string simple = "\thello   world        \n\n    foo bar \t\t\t   \n\n\n";
+{
-  KeyValueBlobReader reader(simple.c_str(), simple.length());
+    std::string simple
-  KeyValue keyValue;
+        = "\thello   world        \n\n    foo bar \t\t\t   \n\n\n";
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    KeyValueBlobReader reader(simple.c_str(), simple.length());
-  EXPECT_EQ(keyValue, (KeyValue{"hello", "world"}));
+    KeyValue keyValue;
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(keyValue, (KeyValue{"foo", "bar"}));
+    EXPECT_EQ(keyValue, (KeyValue { "hello", "world" }));
-  EXPECT_EQ(reader.Next(), State::END);
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    EXPECT_EQ(keyValue, (KeyValue { "foo", "bar" }));
+    EXPECT_EQ(reader.Next(), State::END);
 }
 
-TEST(KeyValueBlobReaderTest, WindowsCRLFSupported) {
+TEST(KeyValueBlobReaderTest, WindowsCRLFSupported)
-  std::string simple = "lorem ipsum\r\nhello world";
+{
-  KeyValueBlobReader reader(simple.c_str(), simple.length());
+    std::string simple = "lorem ipsum\r\nhello world";
-  KeyValue keyValue;
+    KeyValueBlobReader reader(simple.c_str(), simple.length());
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    KeyValue keyValue;
-  EXPECT_EQ(keyValue, (KeyValue{"lorem", "ipsum"}));
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    EXPECT_EQ(keyValue, (KeyValue { "lorem", "ipsum" }));
-  EXPECT_EQ(keyValue, (KeyValue{"hello", "world"}));
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(reader.Next(), State::END);
+    EXPECT_EQ(keyValue, (KeyValue { "hello", "world" }));
+    EXPECT_EQ(reader.Next(), State::END);
 }
 
-TEST(KeyValueBlobReaderTest, MultipleKeyValuePair) {
+TEST(KeyValueBlobReaderTest, MultipleKeyValuePair)
-  std::string multi = "\n   \nhello world\n  foo \t bar  ";
+{
-  KeyValueBlobReader reader(multi.c_str(), multi.length());
+    std::string multi = "\n   \nhello world\n  foo \t bar  ";
-  KeyValue keyValue;
+    KeyValueBlobReader reader(multi.c_str(), multi.length());
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    KeyValue keyValue;
-  EXPECT_EQ(keyValue, (KeyValue{"hello", "world"}));
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    EXPECT_EQ(keyValue, (KeyValue { "hello", "world" }));
-  EXPECT_EQ(keyValue, (KeyValue{"foo", "bar"}));
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(reader.Next(), State::END);
+    EXPECT_EQ(keyValue, (KeyValue { "foo", "bar" }));
+    EXPECT_EQ(reader.Next(), State::END);
 }
 
-TEST(KeyValueBlobReaderTest, ReadUntilNullChar) {
+TEST(KeyValueBlobReaderTest, ReadUntilNullChar)
-  char buf[] = {'p', '\t', 'q', '\n', 0, 'r', ' ', 's'};
+{
-  KeyValueBlobReader reader(buf, sizeof(buf));
+    char buf[] = { 'p', '\t', 'q', '\n', 0, 'r', ' ', 's' };
-  KeyValue keyValue;
+    KeyValueBlobReader reader(buf, sizeof(buf));
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    KeyValue keyValue;
-  EXPECT_EQ(keyValue, (KeyValue{"p", "q"}));
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(reader.Next(), State::END);
+    EXPECT_EQ(keyValue, (KeyValue { "p", "q" }));
+    EXPECT_EQ(reader.Next(), State::END);
 }
 
-TEST(KeyValueBlobReaderTest, MultipleKeyValuePairWithComments) {
+TEST(KeyValueBlobReaderTest, MultipleKeyValuePairWithComments)
-  std::string text = R"(
+{
+    std::string text = R"(
 # comment1
 # comment2
 
@@ -195,19 +214,20 @@ TEST(KeyValueBlobReaderTest, MultipleKeyValuePairWithComments) {
 # comment5
 )";
 
-  KeyValueBlobReader reader(text.c_str(), text.length());
+    KeyValueBlobReader reader(text.c_str(), text.length());
-  KeyValue keyValue;
+    KeyValue keyValue;
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(keyValue, (KeyValue{"hello", "World"}));
+    EXPECT_EQ(keyValue, (KeyValue { "hello", "World" }));
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(keyValue, (KeyValue{"caffè", "latte"}));
+    EXPECT_EQ(keyValue, (KeyValue { "caffè", "latte" }));
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(keyValue, (KeyValue{"foo", "bar"}));
+    EXPECT_EQ(keyValue, (KeyValue { "foo", "bar" }));
-  EXPECT_EQ(reader.Next(), State::END);
+    EXPECT_EQ(reader.Next(), State::END);
 }
 
-TEST(KeyValueBlobReaderTest, ValueCommentSupported) {
+TEST(KeyValueBlobReaderTest, ValueCommentSupported)
-  std::string text = R"(
+{
+    std::string text = R"(
   # empty
 
   hello world#peace
@@ -217,19 +237,19 @@ hello world#peace  // peace
   foo bar
 )";
 
-  KeyValueBlobReader reader(text.c_str(), text.length());
+    KeyValueBlobReader reader(text.c_str(), text.length());
-  KeyValue keyValue;
+    KeyValue keyValue;
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(keyValue, (KeyValue{"hello", "world#peace"}));
+    EXPECT_EQ(keyValue, (KeyValue { "hello", "world#peace" }));
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(keyValue, (KeyValue{"hello", "world#peace"}));
+    EXPECT_EQ(keyValue, (KeyValue { "hello", "world#peace" }));
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(keyValue, (KeyValue{"hello", "world#peace"}));
+    EXPECT_EQ(keyValue, (KeyValue { "hello", "world#peace" }));
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(keyValue, (KeyValue{"caffè", "latte"}));
+    EXPECT_EQ(keyValue, (KeyValue { "caffè", "latte" }));
-  EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
+    EXPECT_EQ(reader.Next(&keyValue), State::HAS_PAIR);
-  EXPECT_EQ(keyValue, (KeyValue{"foo", "bar"}));
+    EXPECT_EQ(keyValue, (KeyValue { "foo", "bar" }));
-  EXPECT_EQ(reader.Next(), State::END);
+    EXPECT_EQ(reader.Next(), State::END);
 }
 
-}  // namespace McBopomofo
+} // namespace McBopomofo

Source/Engine/McBopomofoLM.h

@@ -25,8 +25,8 @@
 #define MCBOPOMOFOLM_H
 
 #include <stdio.h>
-#include "FastLM.h"
 #include "UserPhrasesLM.h"
+#include "ParselessLM.h"
 #include "PhraseReplacementMap.h"
 
 namespace McBopomofo {
@@ -51,7 +51,7 @@ public:
     bool phraseReplacementEnabled();
 
 protected:
-    FastLM m_languageModel;
+    ParselessLM m_languageModel;
     UserPhrasesLM m_userPhrases;
     UserPhrasesLM m_excludedPhrases;
     PhraseReplacementMap m_phraseReplacement;

Source/Engine/ParselessLM.cpp

@@ -0,0 +1,143 @@
+// Copyright (c) 2022 and onwards The McBopomofo Authors.
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+
+#include "ParselessLM.h"
+
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include <memory>
+
+McBopomofo::ParselessLM::~ParselessLM() { close(); }
+
+bool McBopomofo::ParselessLM::open(const std::string_view& path)
+{
+    if (data_) {
+        return false;
+    }
+
+    fd_ = ::open(path.data(), O_RDONLY);
+    if (fd_ == -1) {
+        return false;
+    }
+
+    struct stat sb;
+    if (fstat(fd_, &sb) == -1) {
+        ::close(fd_);
+        fd_ = -1;
+        return false;
+    }
+
+    length_ = static_cast<size_t>(sb.st_size);
+
+    data_ = mmap(NULL, length_, PROT_READ, MAP_SHARED, fd_, 0);
+    if (data_ == nullptr) {
+        ::close(fd_);
+        fd_ = -1;
+        length_ = 0;
+        return false;
+    }
+
+    db_ = std::unique_ptr<ParselessPhraseDB>(new ParselessPhraseDB(
+        static_cast<char*>(data_), length_, /*validate_pragme=*/
+        true));
+    return true;
+}
+
+void McBopomofo::ParselessLM::close()
+{
+    if (data_ != nullptr) {
+        munmap(data_, length_);
+        ::close(fd_);
+        fd_ = -1;
+        length_ = 0;
+        data_ = nullptr;
+    }
+}
+
+const std::vector<Formosa::Gramambular::Bigram>
+McBopomofo::ParselessLM::bigramsForKeys(
+    const std::string& preceedingKey, const std::string& key)
+{
+    return std::vector<Formosa::Gramambular::Bigram>();
+}
+
+const std::vector<Formosa::Gramambular::Unigram>
+McBopomofo::ParselessLM::unigramsForKey(const std::string& key)
+{
+    if (db_ == nullptr) {
+        return std::vector<Formosa::Gramambular::Unigram>();
+    }
+
+    std::vector<Formosa::Gramambular::Unigram> results;
+    for (const auto& row : db_->findRows(key + " ")) {
+        Formosa::Gramambular::Unigram unigram;
+
+        // Move ahead until we encounter the first space. This is the key.
+        auto it = row.begin();
+        while (it != row.end() && *it != ' ') {
+            ++it;
+        }
+
+        unigram.keyValue.key = std::string(row.begin(), it);
+
+        // Read past the space.
+        if (it != row.end()) {
+            ++it;
+        }
+
+        if (it != row.end()) {
+            // Now it is the start of the value portion.
+            auto value_begin = it;
+
+            // Move ahead until we encounter the second space. This is the
+            // value.
+            while (it != row.end() && *it != ' ') {
+                ++it;
+            }
+            unigram.keyValue.value = std::string(value_begin, it);
+        }
+
+        // Read past the space. The remainder, if it exists, is the score.
+        if (it != row.end()) {
+            ++it;
+        }
+
+        if (it != row.end()) {
+            unigram.score = std::stod(std::string(it, row.end()));
+        }
+        results.push_back(unigram);
+    }
+    return results;
+}
+
+bool McBopomofo::ParselessLM::hasUnigramsForKey(const std::string& key)
+{
+    if (db_ == nullptr) {
+        return false;
+    }
+
+    return db_->findFirstMatchingLine(key + " ") != nullptr;
+}

Source/Engine/ParselessLM.h

@@ -0,0 +1,58 @@
+// Copyright (c) 2022 and onwards The McBopomofo Authors.
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+
+#ifndef SOURCE_ENGINE_PARSELESSLM_H_
+#define SOURCE_ENGINE_PARSELESSLM_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "LanguageModel.h"
+#include "ParselessPhraseDB.h"
+
+namespace McBopomofo {
+
+class ParselessLM : public Formosa::Gramambular::LanguageModel {
+public:
+    ~ParselessLM() override;
+
+    bool open(const std::string_view& path);
+    void close();
+
+    const std::vector<Formosa::Gramambular::Bigram> bigramsForKeys(
+        const std::string& preceedingKey, const std::string& key) override;
+    const std::vector<Formosa::Gramambular::Unigram> unigramsForKey(
+        const std::string& key) override;
+    bool hasUnigramsForKey(const std::string& key) override;
+
+private:
+    int fd_ = -1;
+    void* data_ = nullptr;
+    size_t length_ = 0;
+    std::unique_ptr<ParselessPhraseDB> db_;
+};
+
+}; // namespace McBopomofo
+
+#endif // SOURCE_ENGINE_PARSELESSLM_H_

Source/Engine/ParselessLMBenchmark.cpp

@@ -0,0 +1,89 @@
+// Copyright (c) 2022 and onwards The McBopomofo Authors.
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+
+#include <benchmark/benchmark.h>
+
+#include <cassert>
+#include <filesystem>
+
+#include "FastLM.h"
+#include "ParselessLM.h"
+
+namespace {
+
+using FastLM = Formosa::Gramambular::FastLM;
+using ParselessLM = McBopomofo::ParselessLM;
+
+static const char* kDataPath = "data.txt";
+static const char* kLegacyDataPath = "data-legacy.txt";
+static const char* kUnigramSearchKey = "ㄕˋ-ㄕˊ";
+
+static void BM_ParselessLMOpenClose(benchmark::State& state)
+{
+    assert(std::filesystem::exists(kDataPath));
+    for (auto _ : state) {
+        ParselessLM lm;
+        lm.open(kDataPath);
+        lm.close();
+    }
+}
+BENCHMARK(BM_ParselessLMOpenClose);
+
+static void BM_FastLMOpenClose(benchmark::State& state)
+{
+    assert(std::filesystem::exists(kLegacyDataPath));
+    for (auto _ : state) {
+        FastLM lm;
+        lm.open(kLegacyDataPath);
+        lm.close();
+    }
+}
+BENCHMARK(BM_FastLMOpenClose);
+
+static void BM_ParselessLMFindUnigrams(benchmark::State& state)
+{
+    assert(std::filesystem::exists(kDataPath));
+    ParselessLM lm;
+    lm.open(kDataPath);
+    for (auto _ : state) {
+        lm.unigramsForKey(kUnigramSearchKey);
+    }
+    lm.close();
+}
+BENCHMARK(BM_ParselessLMFindUnigrams);
+
+static void BM_FastLMFindUnigrams(benchmark::State& state)
+{
+    assert(std::filesystem::exists(kLegacyDataPath));
+    FastLM lm;
+    lm.open(kLegacyDataPath);
+    for (auto _ : state) {
+        lm.unigramsForKey(kUnigramSearchKey);
+    }
+    lm.close();
+}
+BENCHMARK(BM_FastLMFindUnigrams);
+
+}; // namespace
+
+BENCHMARK_MAIN();

Source/Engine/ParselessLMTest.cpp

@@ -0,0 +1,59 @@
+// Copyright (c) 2022 and onwards The McBopomofo Authors.
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+
+#include <filesystem>
+#include <iostream>
+
+#include "ParselessLM.h"
+#include "gtest/gtest.h"
+
+namespace McBopomofo {
+
+TEST(ParselessLMTest, SanityCheckTest)
+{
+    constexpr const char* data_path = "data.txt";
+    if (!std::filesystem::exists(data_path)) {
+        GTEST_SKIP();
+    }
+
+    ParselessLM lm;
+    bool status = lm.open(data_path);
+    ASSERT_TRUE(status);
+
+    ASSERT_TRUE(lm.hasUnigramsForKey("ㄕ"));
+    ASSERT_TRUE(lm.hasUnigramsForKey("ㄕˋ-ㄕˊ"));
+    ASSERT_TRUE(lm.hasUnigramsForKey("_punctuation_list"));
+
+    auto unigrams = lm.unigramsForKey("ㄕ");
+    ASSERT_GT(unigrams.size(), 0);
+
+    unigrams = lm.unigramsForKey("ㄕˋ-ㄕˊ");
+    ASSERT_GT(unigrams.size(), 0);
+
+    unigrams = lm.unigramsForKey("_punctuation_list");
+    ASSERT_GT(unigrams.size(), 0);
+
+    lm.close();
+}
+
+}; // namespace McBopomofo

Source/Engine/ParselessPhraseDB.cpp

@@ -0,0 +1,166 @@
+// Copyright (c) 2022 and onwards The McBopomofo Authors.
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+
+#include "ParselessPhraseDB.h"
+
+#include <cassert>
+#include <cstring>
+
+namespace McBopomofo {
+
+ParselessPhraseDB::ParselessPhraseDB(
+    const char* buf, size_t length, bool validate_pragma)
+    : begin_(buf)
+    , end_(buf + length)
+{
+    assert(buf != nullptr);
+    assert(length > 0);
+
+    if (validate_pragma) {
+        assert(length > SORTED_PRAGMA_HEADER.length());
+
+        std::string_view header(buf, SORTED_PRAGMA_HEADER.length());
+        assert(header == SORTED_PRAGMA_HEADER);
+
+        uint32_t x = 5381;
+        for (const auto& i : header) {
+            x = x * 33 + i;
+        }
+
+        assert(x == uint32_t { 3012373384 });
+
+        begin_ += header.length();
+    }
+}
+
+std::vector<std::string_view> ParselessPhraseDB::findRows(
+    const std::string_view& key)
+{
+    std::vector<std::string_view> rows;
+
+    const char* ptr = findFirstMatchingLine(key);
+    if (ptr == nullptr) {
+        return rows;
+    }
+
+    while (ptr + key.length() <= end_
+        && memcmp(ptr, key.data(), key.length()) == 0) {
+        const char* eol = ptr;
+
+        while (eol != end_ && *eol != '\n') {
+            ++eol;
+        }
+
+        rows.emplace_back(ptr, eol - ptr);
+        if (eol == end_) {
+            break;
+        }
+
+        ptr = ++eol;
+    }
+
+    return rows;
+}
+
+// Implements a binary search that returns the pointer to the first matching
+// row. In its core it's just a standard binary search, but we use backtracking
+// to locate the line start. We also check the previous line to see if the
+// current line is actually the first matching line: if the previous line is
+// less to the key and the current line starts exactly with the key, then
+// the current line is the first matching line.
+const char* ParselessPhraseDB::findFirstMatchingLine(
+    const std::string_view& key)
+{
+    if (key.empty()) {
+        return begin_;
+    }
+
+    const char* top = begin_;
+    const char* bottom = end_;
+
+    while (top < bottom) {
+        const char* mid = top + (bottom - top) / 2;
+        const char* ptr = mid;
+
+        if (ptr != begin_) {
+            --ptr;
+        }
+
+        while (ptr != begin_ && *ptr != '\n') {
+            --ptr;
+        }
+
+        const char* prev = nullptr;
+        if (*ptr == '\n') {
+            prev = ptr;
+            ++ptr;
+        }
+
+        // ptr is now in the "current" line we're interested in.
+        if (ptr + key.length() > end_) {
+            // not enough data to compare at this point, bail.
+            break;
+        }
+
+        int current_cmp = memcmp(ptr, key.data(), key.length());
+
+        if (current_cmp > 0) {
+            bottom = mid - 1;
+            continue;
+        }
+
+        if (current_cmp < 0) {
+            top = mid + 1;
+            continue;
+        }
+
+        if (!prev) {
+            return ptr;
+        }
+
+        // Move the prev so that it reaches the previous line.
+        if (prev != begin_) {
+            --prev;
+        }
+        while (prev != begin_ && *prev != '\n') {
+            --prev;
+        }
+        if (*prev == '\n') {
+            ++prev;
+        }
+
+        int prev_cmp = memcmp(prev, key.data(), key.length());
+
+        // This is the first occurrence.
+        if (prev_cmp < 0 && current_cmp == 0) {
+            return ptr;
+        }
+
+        // This is not, which means ptr is "larger" than the keyData.
+        bottom = mid - 1;
+    }
+
+    return nullptr;
+}
+
+}; // namespace McBopomofo

Source/Engine/ParselessPhraseDB.h

@@ -0,0 +1,60 @@
+// Copyright (c) 2022 and onwards The McBopomofo Authors.
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+
+#ifndef SOURCE_ENGINE_PARSELESSPHRASEDB_H_
+#define SOURCE_ENGINE_PARSELESSPHRASEDB_H_
+
+#include <cstddef>
+#include <string>
+#include <vector>
+
+namespace McBopomofo {
+
+constexpr std::string_view SORTED_PRAGMA_HEADER
+    = "# format org.openvanilla.mcbopomofo.sorted\n";
+
+// Defines phrase database that consists of (key, value, score) rows that are
+// pre-sorted by the byte value of the keys. It is way faster than FastLM
+// because it does not need to parse anything. Instead, it relies on the fact
+// that the database is already sorted, and binary search is used to find the
+// rows.
+class ParselessPhraseDB {
+public:
+    ParselessPhraseDB(
+        const char* buf, size_t length, bool validate_pragma = false);
+
+    // Find the rows that match the key. Note that prefix match is used. If you
+    // need exact match, the key will need to have a delimiter (usually a space)
+    // at the end.
+    std::vector<std::string_view> findRows(const std::string_view& key);
+
+    const char* findFirstMatchingLine(const std::string_view& key);
+
+private:
+    const char* begin_;
+    const char* end_;
+};
+
+}; // namespace McBopomofo
+
+#endif // SOURCE_ENGINE_PARSELESSPHRASEDB_H_

Source/Engine/ParselessPhraseDBTest.cpp

@@ -0,0 +1,198 @@
+// Copyright (c) 2022 and onwards The McBopomofo Authors.
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+
+#include <cstdio>
+#include <filesystem>
+#include <map>
+#include <sstream>
+#include <vector>
+
+#include "ParselessPhraseDB.h"
+#include "gtest/gtest-death-test.h"
+#include "gtest/gtest.h"
+
+using StringViews = std::vector<std::string_view>;
+
+namespace McBopomofo {
+
+static bool VectorsEqual(
+    const std::vector<std::string_view>& a, const std::vector<std::string>& b)
+{
+    if (a.size() != b.size()) {
+        return false;
+    }
+
+    size_t s = a.size();
+    for (size_t i = 0; i < s; i++) {
+        if (a[i] != b[i]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+TEST(ParselessPhraseDBTest, Simple)
+{
+    std::string data = "a 1";
+    ParselessPhraseDB db(data.c_str(), data.length());
+
+    const char* first = db.findFirstMatchingLine("a");
+    EXPECT_NE(first, nullptr);
+    EXPECT_EQ(memcmp(first, "a 1", 3), 0);
+
+    first = db.findFirstMatchingLine("a ");
+    EXPECT_NE(first, nullptr);
+    EXPECT_EQ(memcmp(first, "a 1", 3), 0);
+
+    first = db.findFirstMatchingLine("a 1");
+    EXPECT_NE(first, nullptr);
+    EXPECT_EQ(memcmp(first, "a 1", 3), 0);
+}
+
+TEST(ParselessPhraseDBTest, NotFound)
+{
+    std::string data = "a 1\na 2\na 3\nb 1";
+    ParselessPhraseDB db(data.c_str(), data.length());
+    EXPECT_EQ(db.findFirstMatchingLine("c"), nullptr);
+    EXPECT_EQ(db.findFirstMatchingLine("A"), nullptr);
+}
+
+TEST(ParselessPhraseDBTest, FindRowsLongerExample)
+{
+    std::string data = "a 1\na 2\na 3\nb 42\nb 1\nb 2\nc 7\nd 1";
+    ParselessPhraseDB db(data.c_str(), data.length());
+
+    EXPECT_EQ(db.findRows("a"), (StringViews { "a 1", "a 2", "a 3" }));
+    EXPECT_EQ(db.findRows("b"), (StringViews { "b 42", "b 1", "b 2" }));
+    EXPECT_EQ(db.findRows("c"), (StringViews { "c 7" }));
+    EXPECT_EQ(db.findRows("d"), (StringViews { "d 1" }));
+    EXPECT_EQ(db.findRows("e"), (StringViews {}));
+    EXPECT_EQ(db.findRows("A"), (StringViews {}));
+}
+
+TEST(ParselessPhraseDBTest, FindFirstMatchingLineLongerExample)
+{
+    std::string data = "a 1\na 2\na 3\nb 42\nb 1\nb 2\nc 7\nd 1";
+    ParselessPhraseDB db(data.c_str(), data.length());
+
+    const char* first = db.findFirstMatchingLine("a");
+    EXPECT_NE(first, nullptr);
+    EXPECT_EQ(memcmp(first, "a 1", 3), 0);
+
+    db.findFirstMatchingLine("a 1");
+    EXPECT_NE(first, nullptr);
+    EXPECT_EQ(memcmp(first, "a 1", 3), 0);
+
+    first = db.findFirstMatchingLine("b");
+    EXPECT_NE(first, nullptr);
+    EXPECT_EQ(memcmp(first, "b 42", 4), 0);
+
+    first = db.findFirstMatchingLine("c");
+    EXPECT_NE(first, nullptr);
+    EXPECT_EQ(memcmp(first, "c 7", 3), 0);
+
+    first = db.findFirstMatchingLine("d");
+    EXPECT_NE(first, nullptr);
+    EXPECT_EQ(memcmp(first, "d 1", 3), 0);
+
+    first = db.findFirstMatchingLine("d 1");
+    EXPECT_NE(first, nullptr);
+    EXPECT_EQ(memcmp(first, "d 1", 3), 0);
+
+    first = db.findFirstMatchingLine("d 2");
+    EXPECT_EQ(first, nullptr);
+}
+
+TEST(ParselessPhraseDBTest, InvalidConstructorArguments)
+{
+    EXPECT_DEATH((ParselessPhraseDB { nullptr, 1 }), "buf != nullptr");
+    EXPECT_DEATH((ParselessPhraseDB { nullptr, 0 }), "buf != nullptr");
+    EXPECT_DEATH((ParselessPhraseDB { "", 0 }), "length > 0");
+    EXPECT_DEATH((ParselessPhraseDB { "a", 1, /*validate_pragma=*/true }),
+        "length > SORTED_PRAGMA_HEADER\\.length\\(\\)");
+}
+
+TEST(ParselessPhraseDBTest, PragmaGuard)
+{
+    std::string buf1 = std::string(SORTED_PRAGMA_HEADER) + "a";
+    std::string buf2 = "#" + buf1;
+    std::string buf3 = buf1;
+    buf3[3] = 'x';
+
+    ParselessPhraseDB { buf1.c_str(), buf1.length(), /*validate_pragma=*/true };
+    EXPECT_DEATH(
+        (ParselessPhraseDB { buf2.c_str(), buf2.length(), /*validate_pragma=*/
+            true }),
+        "==");
+    EXPECT_DEATH(
+        (ParselessPhraseDB { buf3.c_str(), buf3.length(), /*validate_pragma=*/
+            true }),
+        "==");
+}
+
+TEST(ParselessPhraseDBTest, StressTest)
+{
+    constexpr const char* data_path = "data.txt";
+    if (!std::filesystem::exists(data_path)) {
+        GTEST_SKIP();
+    }
+
+    FILE* f = fopen(data_path, "r");
+    ASSERT_NE(f, nullptr);
+    int status = fseek(f, 0L, SEEK_END);
+    ASSERT_EQ(status, 0);
+    size_t length = ftell(f);
+    std::unique_ptr<char[]> buf(new char[length]);
+    status = fseek(f, 0L, SEEK_SET);
+    ASSERT_EQ(status, 0);
+    size_t items_read = fread(buf.get(), length, 1, f);
+    ASSERT_EQ(items_read, 1);
+    fclose(f);
+
+    std::stringstream sstr(std::string(buf.get(), length));
+    std::string line;
+    std::map<std::string, std::vector<std::string>> key_to_lines;
+
+    // Skip the pragma line.
+    std::getline(sstr, line);
+
+    while (!sstr.eof()) {
+        std::getline(sstr, line);
+        if (line == "") {
+            continue;
+        }
+
+        std::stringstream linest(line);
+        std::string key;
+        linest >> key;
+        key_to_lines[key].push_back(line);
+    }
+
+    ParselessPhraseDB db(buf.get(), length, /*validate_pragma=*/true);
+    for (const auto& it : key_to_lines) {
+        std::vector<std::string_view> rows = db.findRows(it.first + " ");
+        ASSERT_TRUE(VectorsEqual(rows, it.second));
+    }
+}
+
+}; // namespace McBopomofo

Source/LanguageModelManager.h

@@ -1,5 +1,4 @@
 #import <Foundation/Foundation.h>
-#import "FastLM.h"
 #import "UserOverrideModel.h"
 #import "McBopomofoLM.h"