floraachy
/
vChewing-macOS
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #269 from lukhnos/mandarin-refactoring 

Refactor the Bopomofo syllable library for better thread safety
This commit is contained in:
Weizhong Yang a.k.a zonble 2022-02-02 17:57:14 +08:00 committed by GitHub
parent 063164d0ab 59d6ace3e5
commit 8e1e3d184b
5 changed files with 1809 additions and 1496 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/continuous-integration-workflow.yml vendored
View File

@ -15,6 +15,12 @@ jobs:
- name: Run McBopomofoLMLibTest
run: make runTest
working-directory: Source/Engine/build
- name: Build MandarinTest
run: cmake -S . -B build
working-directory: Source/Engine/Mandarin
- name: Run MandarinTest
run: make runTest
working-directory: Source/Engine/Mandarin/build
- name: Test McBopomofo App Bundle
run: xcodebuild -scheme McBopomofo -configuration Debug test
- name: Test CandidateUI

31
Source/Engine/Mandarin/CMakeLists.txt Normal file
View File

@ -0,0 +1,31 @@
cmake_minimum_required(VERSION 3.17)
project(Mandarin)
set(CMAKE_CXX_STANDARD 17)
add_library(MandarinLib Mandarin.h Mandarin.cpp)
# Let CMake fetch Google Test for us.
# https://github.com/google/googletest/tree/main/googletest#incorporating-into-an-existing-cmake-project
include(FetchContent)
FetchContent_Declare(
googletest
# Specify the commit you depend on and update it regularly.
URL https://github.com/google/googletest/archive/609281088cfefc76f9d0ce82e1ff6c30cc3591e5.zip
)
# For Windows: Prevent overriding the parent project's compiler/linker settings
set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
FetchContent_MakeAvailable(googletest)
# Test target declarations.
add_executable(MandarinTest MandarinTest.cpp)
target_link_libraries(MandarinTest gtest_main MandarinLib)
include(GoogleTest)
gtest_discover_tests(MandarinTest)
add_custom_target(
runTest
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/MandarinTest
)
add_dependencies(runTest MandarinTest)

1178
Source/Engine/Mandarin/Mandarin.cpp
View File

File diff suppressed because it is too large Load Diff

530
Source/Engine/Mandarin/Mandarin.h
View File

@ -1,7 +1,4 @@
//
// Mandarin.h
//
// Copyright (c) 2006-2010 Lukhnos D. Liu (http://lukhnos.org)
// Copyright (c) 2006 and onwards Lukhnos Liu
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
@ -23,151 +20,109 @@
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.
//
#ifndef Mandarin_h
#define Mandarin_h
#ifndef MANDARIN_H_
#define MANDARIN_H_
#include <iostream>
#include <map>
#include <string>
#include <vector>
#include <map>
namespace Formosa {
namespace Mandarin {
using namespace std;
class BopomofoSyllable {
public:
typedef unsigned int Component;
BopomofoSyllable(Component syllable = 0)
: m_syllable(syllable)
{
}
typedef uint16_t Component;
BopomofoSyllable(const BopomofoSyllable& another)
: m_syllable(another.m_syllable)
{
}
explicit BopomofoSyllable(Component syllable = 0) : syllable_(syllable) {}
~BopomofoSyllable()
{
}
BopomofoSyllable(const BopomofoSyllable&) = default;
BopomofoSyllable(BopomofoSyllable&& another) = default;
BopomofoSyllable& operator=(const BopomofoSyllable&) = default;
BopomofoSyllable& operator=(BopomofoSyllable&&) = default;
BopomofoSyllable& operator=(const BopomofoSyllable& another)
{
m_syllable = another.m_syllable;
return *this;
}
// takes the ASCII-form, "v"-tolerant, TW-style Hanyu Pinyin (fong, pong, bong acceptable)
static const BopomofoSyllable FromHanyuPinyin(const string& str);
// takes the ASCII-form, "v"-tolerant, TW-style Hanyu Pinyin (fong, pong, bong
// acceptable)
static const BopomofoSyllable FromHanyuPinyin(const std::string& str);
// TO DO: Support accented vowels
const string HanyuPinyinString(bool includesTone, bool useVForUUmlaut) const;
// const string HanyuPinyinString(bool includesTone, bool useVForUUmlaut, bool composeAccentedVowel) const;
const std::string HanyuPinyinString(bool includesTone,
bool useVForUUmlaut) const;
// const std::string HanyuPinyinString(bool includesTone, bool useVForUUmlaut,
// bool composeAccentedVowel) const;
// PHT = Pai-hua-tsi
static const BopomofoSyllable FromPHT(const string& str);
const string PHTString(bool includesTone) const;
static const BopomofoSyllable FromPHT(const std::string& str);
const std::string PHTString(bool includesTone) const;
static const BopomofoSyllable FromComposedString(const string& str);
const string composedString() const;
static const BopomofoSyllable FromComposedString(const std::string& str);
const std::string composedString() const;
void clear()
{
m_syllable = 0;
void clear() { syllable_ = 0; }
bool isEmpty() const { return !syllable_; }
bool hasConsonant() const { return !!(syllable_ & ConsonantMask); }
bool hasMiddleVowel() const { return !!(syllable_ & MiddleVowelMask); }
bool hasVowel() const { return !!(syllable_ & VowelMask); }
bool hasToneMarker() const { return !!(syllable_ & ToneMarkerMask); }
Component consonantComponent() const { return syllable_ & ConsonantMask; }
Component middleVowelComponent() const {
return syllable_ & MiddleVowelMask;
}
bool isEmpty() const
{
return !m_syllable;
Component vowelComponent() const { return syllable_ & VowelMask; }
Component toneMarkerComponent() const { return syllable_ & ToneMarkerMask; }
bool operator==(const BopomofoSyllable& another) const {
return syllable_ == another.syllable_;
}
bool hasConsonant() const
{
return !!(m_syllable & ConsonantMask);
bool operator!=(const BopomofoSyllable& another) const {
return syllable_ != another.syllable_;
}
bool hasMiddleVowel() const
{
return !!(m_syllable & MiddleVowelMask);
}
bool hasVowel() const
{
return !!(m_syllable & VowelMask);
}
bool hasToneMarker() const
{
return !!(m_syllable & ToneMarkerMask);
}
Component consonantComponent() const
{
return m_syllable & ConsonantMask;
}
Component middleVowelComponent() const
{
return m_syllable & MiddleVowelMask;
}
Component vowelComponent() const
{
return m_syllable & VowelMask;
}
Component toneMarkerComponent() const
{
return m_syllable & ToneMarkerMask;
}
bool operator==(const BopomofoSyllable& another) const
{
return m_syllable == another.m_syllable;
}
bool operator!=(const BopomofoSyllable& another) const
{
return m_syllable != another.m_syllable;
}
bool isOverlappingWith(const BopomofoSyllable& another) const
{
#define IOW_SAND(mask) ((m_syllable & mask) && (another.m_syllable & mask))
return IOW_SAND(ConsonantMask) || IOW_SAND(MiddleVowelMask) || IOW_SAND(VowelMask) || IOW_SAND(ToneMarkerMask);
bool isOverlappingWith(const BopomofoSyllable& another) const {
#define IOW_SAND(mask) ((syllable_ & mask) && (another.syllable_ & mask))
return IOW_SAND(ConsonantMask) || IOW_SAND(MiddleVowelMask) ||
IOW_SAND(VowelMask) || IOW_SAND(ToneMarkerMask);
#undef IOW_SAND
}
// consonants J, Q, X all require the existence of vowel I or UE
bool belongsToJQXClass() const
{
Component consonant = m_syllable & ConsonantMask;
bool belongsToJQXClass() const {
Component consonant = syllable_ & ConsonantMask;
return (consonant == J || consonant == Q || consonant == X);
}
// zi, ci, si, chi, chi, shi, ri
bool belongsToZCSRClass() const
{
Component consonant = m_syllable & ConsonantMask;
bool belongsToZCSRClass() const {
Component consonant = syllable_ & ConsonantMask;
return (consonant >= ZH && consonant <= S);
}
Component maskType() const
{
Component maskType() const {
Component mask = 0;
mask |= (m_syllable & ConsonantMask) ? ConsonantMask : 0;
mask |= (m_syllable & MiddleVowelMask) ? MiddleVowelMask : 0;
mask |= (m_syllable & VowelMask) ? VowelMask : 0;
mask |= (m_syllable & ToneMarkerMask) ? ToneMarkerMask : 0;
mask |= (syllable_ & ConsonantMask) ? ConsonantMask : 0;
mask |= (syllable_ & MiddleVowelMask) ? MiddleVowelMask : 0;
mask |= (syllable_ & VowelMask) ? VowelMask : 0;
mask |= (syllable_ & ToneMarkerMask) ? ToneMarkerMask : 0;
return mask;
}
const BopomofoSyllable operator+(const BopomofoSyllable& another) const
{
Component newSyllable = m_syllable;
#define OP_SOVER(mask) if (another.m_syllable & mask) newSyllable = (newSyllable & ~mask) | (another.m_syllable & mask)
const BopomofoSyllable operator+(const BopomofoSyllable& another) const {
Component newSyllable = syllable_;
#define OP_SOVER(mask) \
if (another.syllable_ & mask) { \
newSyllable = (newSyllable & ~mask) | (another.syllable_ & mask); \
}
OP_SOVER(ConsonantMask);
OP_SOVER(MiddleVowelMask);
OP_SOVER(VowelMask);
@ -176,9 +131,11 @@ namespace Formosa {
return BopomofoSyllable(newSyllable);
}
BopomofoSyllable& operator+=(const BopomofoSyllable& another)
{
#define OPE_SOVER(mask) if (another.m_syllable & mask) m_syllable = (m_syllable & ~mask) | (another.m_syllable & mask)
BopomofoSyllable& operator+=(const BopomofoSyllable& another) {
#define OPE_SOVER(mask) \
if (another.syllable_ & mask) { \
syllable_ = (syllable_ & ~mask) | (another.syllable_ & mask); \
}
OPE_SOVER(ConsonantMask);
OPE_SOVER(MiddleVowelMask);
OPE_SOVER(VowelMask);
@ -187,83 +144,74 @@ namespace Formosa {
return *this;
}
short absoluteOrder() const
{
uint16_t absoluteOrder() const {
// turn BPMF syllable into a 4*14*4*22 number
return (short)(m_syllable & ConsonantMask) +
(short)((m_syllable & MiddleVowelMask) >> 5) * 22 +
(short)((m_syllable & VowelMask) >> 7) * 22 * 4 +
(short)((m_syllable & ToneMarkerMask) >> 11) * 22 * 4 * 14;
return (uint16_t)(syllable_ & ConsonantMask) +
(uint16_t)((syllable_ & MiddleVowelMask) >> 5) * 22 +
(uint16_t)((syllable_ & VowelMask) >> 7) * 22 * 4 +
(uint16_t)((syllable_ & ToneMarkerMask) >> 11) * 22 * 4 * 14;
}
const string absoluteOrderString() const
{
const std::string absoluteOrderString() const {
// 5*14*4*22 = 6160, we use a 79*79 encoding to represent that
short order = absoluteOrder();
char low = 48 + (char)(order % 79);
char high = 48 + (char)(order / 79);
string result(2, ' ');
uint16_t order = absoluteOrder();
char low = 48 + static_cast<char>(order % 79);
char high = 48 + static_cast<char>(order / 79);
std::string result(2, ' ');
result[0] = low;
result[1] = high;
return result;
}
static BopomofoSyllable FromAbsoluteOrder(short order)
{
return BopomofoSyllable(
(order % 22) |
((order / 22) % 4) << 5 |
static BopomofoSyllable FromAbsoluteOrder(uint16_t order) {
return BopomofoSyllable((order % 22) | ((order / 22) % 4) << 5 |
((order / (22 * 4)) % 14) << 7 |
((order / (22 * 4 * 14)) % 5) << 11
);
((order / (22 * 4 * 14)) % 5) << 11);
}
static BopomofoSyllable FromAbsoluteOrderString(const string& str)
{
if (str.length() != 2)
return BopomofoSyllable();
static BopomofoSyllable FromAbsoluteOrderString(const std::string& str) {
if (str.length() != 2) return BopomofoSyllable();
return FromAbsoluteOrder((short)(str[1] - 48) * 79 + (short)(str[0] - 48));
return FromAbsoluteOrder((uint16_t)(str[1] - 48) * 79 +
(uint16_t)(str[0] - 48));
}
friend ostream& operator<<(ostream& stream, const BopomofoSyllable& syllable);
friend std::ostream& operator<<(std::ostream& stream,
const BopomofoSyllable& syllable);
static const Component
static constexpr Component
ConsonantMask = 0x001f, // 0000 0000 0001 1111, 21 consonants
MiddleVowelMask = 0x0060, // 0000 0000 0110 0000, 3 middle vowels
VowelMask = 0x0780, // 0000 0111 1000 0000, 13 vowels
ToneMarkerMask = 0x3800, // 0011 1000 0000 0000, 5 tones (tone1 = 0x00)
B = 0x0001, P = 0x0002, M = 0x0003, F = 0x0004,
D = 0x0005, T = 0x0006, N = 0x0007, L = 0x0008,
G = 0x0009, K = 0x000a, H = 0x000b,
J = 0x000c, Q = 0x000d, X = 0x000e,
ZH = 0x000f, CH = 0x0010, SH = 0x0011, R = 0x0012,
Z = 0x0013, C = 0x0014, S = 0x0015,
I = 0x0020, U = 0x0040, UE = 0x0060, // ue = u umlaut (we use the German convention here as an ersatz to the /ju:/ sound)
A = 0x0080, O = 0x0100, ER = 0x0180, E = 0x0200,
AI = 0x0280, EI = 0x0300, AO = 0x0380, OU = 0x0400,
AN = 0x0480, EN = 0x0500, ANG = 0x0580, ENG = 0x0600,
ERR = 0x0680,
Tone1 = 0x0000, Tone2 = 0x0800, Tone3 = 0x1000, Tone4 = 0x1800, Tone5 = 0x2000;
B = 0x0001, P = 0x0002, M = 0x0003, F = 0x0004, D = 0x0005, T = 0x0006,
N = 0x0007, L = 0x0008, G = 0x0009, K = 0x000a, H = 0x000b, J = 0x000c,
Q = 0x000d, X = 0x000e, ZH = 0x000f, CH = 0x0010, SH = 0x0011, R = 0x0012,
Z = 0x0013, C = 0x0014, S = 0x0015, I = 0x0020, U = 0x0040,
UE = 0x0060, // ue = u umlaut (we use the German convention here as an
// ersatz to the /ju:/ sound)
A = 0x0080, O = 0x0100, ER = 0x0180, E = 0x0200, AI = 0x0280, EI = 0x0300,
AO = 0x0380, OU = 0x0400, AN = 0x0480, EN = 0x0500, ANG = 0x0580,
ENG = 0x0600, ERR = 0x0680, Tone1 = 0x0000, Tone2 = 0x0800,
Tone3 = 0x1000, Tone4 = 0x1800, Tone5 = 0x2000;
protected:
Component m_syllable;
Component syllable_;
};
inline ostream& operator<<(ostream& stream, const BopomofoSyllable& syllable)
{
inline std::ostream& operator<<(std::ostream& stream,
const BopomofoSyllable& syllable) {
stream << syllable.composedString();
return stream;
}
typedef BopomofoSyllable BPMF;
typedef map<char, vector<BPMF::Component> > BopomofoKeyToComponentMap;
typedef map<BPMF::Component, char> BopomofoComponentToKeyMap;
typedef std::map<char, std::vector<BPMF::Component> > BopomofoKeyToComponentMap;
typedef std::map<BPMF::Component, char> BopomofoComponentToKeyMap;
class BopomofoKeyboardLayout {
public:
static void FinalizeLayouts();
static const BopomofoKeyboardLayout* StandardLayout();
static const BopomofoKeyboardLayout* ETenLayout();
static const BopomofoKeyboardLayout* HsuLayout();
@ -271,42 +219,41 @@ namespace Formosa {
static const BopomofoKeyboardLayout* IBMLayout();
static const BopomofoKeyboardLayout* HanyuPinyinLayout();
// recognizes (case-insensitive): standard, eten, hsu, eten26, ibm
static const BopomofoKeyboardLayout* LayoutForName(const string& name);
BopomofoKeyboardLayout(const BopomofoKeyToComponentMap& ktcm, const string& name)
: m_keyToComponent(ktcm)
, m_name(name)
{
for (BopomofoKeyToComponentMap::const_iterator miter = m_keyToComponent.begin() ; miter != m_keyToComponent.end() ; ++miter)
for (vector<BPMF::Component>::const_iterator viter = (*miter).second.begin() ; viter != (*miter).second.end() ; ++viter)
BopomofoKeyboardLayout(const BopomofoKeyToComponentMap& ktcm,
const std::string& name)
: m_keyToComponent(ktcm), m_name(name) {
for (BopomofoKeyToComponentMap::const_iterator miter =
m_keyToComponent.begin();
miter != m_keyToComponent.end(); ++miter)
for (std::vector<BPMF::Component>::const_iterator viter =
(*miter).second.begin();
viter != (*miter).second.end(); ++viter)
m_componentToKey[*viter] = (*miter).first;
}
const string name() const
{
return m_name;
}
const std::string name() const { return m_name; }
char componentToKey(BPMF::Component component) const
{
BopomofoComponentToKeyMap::const_iterator iter = m_componentToKey.find(component);
char componentToKey(BPMF::Component component) const {
BopomofoComponentToKeyMap::const_iterator iter =
m_componentToKey.find(component);
return (iter == m_componentToKey.end()) ? 0 : (*iter).second;
}
const vector<BPMF::Component> keyToComponents(char key) const
{
const std::vector<BPMF::Component> keyToComponents(char key) const {
BopomofoKeyToComponentMap::const_iterator iter = m_keyToComponent.find(key);
return (iter == m_keyToComponent.end()) ? vector<BPMF::Component>() : (*iter).second;
return (iter == m_keyToComponent.end()) ? std::vector<BPMF::Component>()
: (*iter).second;
}
const string keySequenceFromSyllable(BPMF syllable) const
{
string sequence;
const std::string keySequenceFromSyllable(BPMF syllable) const {
std::string sequence;
BPMF::Component c;
char k;
#define STKS_COMBINE(component) if ((c = component)) { if ((k = componentToKey(c))) sequence += string(1, k); }
#define STKS_COMBINE(component) \
if ((c = component)) { \
if ((k = componentToKey(c))) sequence += std::string(1, k); \
}
STKS_COMBINE(syllable.consonantComponent());
STKS_COMBINE(syllable.middleVowelComponent());
STKS_COMBINE(syllable.vowelComponent());
@ -315,19 +262,17 @@ namespace Formosa {
return sequence;
}
const BPMF syllableFromKeySequence(const string& sequence) const
{
const BPMF syllableFromKeySequence(const std::string& sequence) const {
BPMF syllable;
for (string::const_iterator iter = sequence.begin() ; iter != sequence.end() ; ++iter)
{
for (std::string::const_iterator iter = sequence.begin();
iter != sequence.end(); ++iter) {
bool beforeSeqHasIorUE = sequenceContainsIorUE(sequence.begin(), iter);
bool aheadSeqHasIorUE = sequenceContainsIorUE(iter + 1, sequence.end());
vector<BPMF::Component> components = keyToComponents(*iter);
std::vector<BPMF::Component> components = keyToComponents(*iter);
if (!components.size())
continue;
if (!components.size()) continue;
if (components.size() == 1) {
syllable += BPMF(components[0]);
@ -339,25 +284,24 @@ namespace Formosa {
BPMF ending = components.size() > 2 ? BPMF(components[2]) : follow;
// apply the I/UE + E rule
if (head.vowelComponent() == BPMF::E && follow.vowelComponent() != BPMF::E)
{
if (head.vowelComponent() == BPMF::E &&
follow.vowelComponent() != BPMF::E) {
syllable += beforeSeqHasIorUE ? head : follow;
continue;
}
if (head.vowelComponent() != BPMF::E && follow.vowelComponent() == BPMF::E)
{
if (head.vowelComponent() != BPMF::E &&
follow.vowelComponent() == BPMF::E) {
syllable += beforeSeqHasIorUE ? follow : head;
continue;
}
// apply the J/Q/X + I/UE rule, only two components are allowed in the components vector here
// apply the J/Q/X + I/UE rule, only two components are allowed in the
// components vector here
if (head.belongsToJQXClass() && !follow.belongsToJQXClass()) {
if (!syllable.isEmpty()) {
if (ending != follow)
syllable += ending;
}
else {
if (ending != follow) syllable += ending;
} else {
syllable += aheadSeqHasIorUE ? head : follow;
}
@ -366,10 +310,8 @@ namespace Formosa {
if (!head.belongsToJQXClass() && follow.belongsToJQXClass()) {
if (!syllable.isEmpty()) {
if (ending != follow)
syllable += ending;
}
else {
if (ending != follow) syllable += ending;
} else {
syllable += aheadSeqHasIorUE ? follow : head;
}
@ -378,30 +320,30 @@ namespace Formosa {
// the nasty issue of only one char in the buffer
if (iter == sequence.begin() && iter + 1 == sequence.end()) {
if (head.hasVowel() || follow.hasToneMarker() || head.belongsToZCSRClass())
if (head.hasVowel() || follow.hasToneMarker() ||
head.belongsToZCSRClass()) {
syllable += head;
else {
if (follow.hasVowel() || ending.hasToneMarker())
} else {
if (follow.hasVowel() || ending.hasToneMarker()) {
syllable += follow;
else
} else {
syllable += ending;
}
}
continue;
}
if (!(syllable.maskType() & head.maskType()) && !endAheadOrAheadHasToneMarkKey(iter + 1, sequence.end())) {
if (!(syllable.maskType() & head.maskType()) &&
!endAheadOrAheadHasToneMarkKey(iter + 1, sequence.end())) {
syllable += head;
}
else {
if (endAheadOrAheadHasToneMarkKey(iter + 1, sequence.end()) && head.belongsToZCSRClass() && syllable.isEmpty()) {
} else {
if (endAheadOrAheadHasToneMarkKey(iter + 1, sequence.end()) &&
head.belongsToZCSRClass() && syllable.isEmpty()) {
syllable += head;
}
else if (syllable.maskType() < follow.maskType()) {
} else if (syllable.maskType() < follow.maskType()) {
syllable += follow;
}
else {
} else {
syllable += ending;
}
}
@ -410,24 +352,23 @@ namespace Formosa {
// heuristics for Hsu keyboard layout
if (this == HsuLayout()) {
// fix the left out L to ERR when it has sound, and GI, GUE -> JI, JUE
if (syllable.vowelComponent() == BPMF::ENG && !syllable.hasConsonant() && !syllable.hasMiddleVowel()) {
if (syllable.vowelComponent() == BPMF::ENG && !syllable.hasConsonant() &&
!syllable.hasMiddleVowel()) {
syllable += BPMF(BPMF::ERR);
}
else if (syllable.consonantComponent() == BPMF::G && (syllable.middleVowelComponent() == BPMF::I || syllable.middleVowelComponent() == BPMF::UE)) {
} else if (syllable.consonantComponent() == BPMF::G &&
(syllable.middleVowelComponent() == BPMF::I ||
syllable.middleVowelComponent() == BPMF::UE)) {
syllable += BPMF(BPMF::J);
}
}
return syllable;
}
protected:
bool endAheadOrAheadHasToneMarkKey(string::const_iterator ahead, string::const_iterator end) const
{
if (ahead == end)
return true;
bool endAheadOrAheadHasToneMarkKey(std::string::const_iterator ahead,
std::string::const_iterator end) const {
if (ahead == end) return true;
char tone1 = componentToKey(BPMF::Tone1);
char tone2 = componentToKey(BPMF::Tone2);
@ -438,70 +379,57 @@ namespace Formosa {
if (tone1)
if (*ahead == tone1) return true;
if (*ahead == tone2 || *ahead == tone3 || *ahead == tone4 || *ahead == tone5)
if (*ahead == tone2 || *ahead == tone3 || *ahead == tone4 ||
*ahead == tone5)
return true;
return false;
}
bool sequenceContainsIorUE(string::const_iterator start, string::const_iterator end) const
{
bool sequenceContainsIorUE(std::string::const_iterator start,
std::string::const_iterator end) const {
char iChar = componentToKey(BPMF::I);
char ueChar = componentToKey(BPMF::UE);
for (; start != end; ++start)
if (*start == iChar || *start == ueChar)
return true;
if (*start == iChar || *start == ueChar) return true;
return false;
}
string m_name;
std::string m_name;
BopomofoKeyToComponentMap m_keyToComponent;
BopomofoComponentToKeyMap m_componentToKey;
static const BopomofoKeyboardLayout* c_StandardLayout;
static const BopomofoKeyboardLayout* c_ETenLayout;
static const BopomofoKeyboardLayout* c_HsuLayout;
static const BopomofoKeyboardLayout* c_ETen26Layout;
static const BopomofoKeyboardLayout* c_IBMLayout;
// this is essentially an empty layout, but we use pointer semantic to tell the differences--and pass on the responsibility to BopomofoReadingBuffer
static const BopomofoKeyboardLayout* c_HanyuPinyinLayout;
};
class BopomofoReadingBuffer {
public:
BopomofoReadingBuffer(const BopomofoKeyboardLayout* layout)
: m_layout(layout)
, m_pinyinMode(false)
{
explicit BopomofoReadingBuffer(const BopomofoKeyboardLayout* layout)
: layout_(layout), pinyin_mode_(false) {
if (layout == BopomofoKeyboardLayout::HanyuPinyinLayout()) {
m_pinyinMode = true;
m_pinyinSequence = "";
pinyin_mode_ = true;
pinyin_sequence_ = "";
}
}
void setKeyboardLayout(const BopomofoKeyboardLayout* layout)
{
m_layout = layout;
void setKeyboardLayout(const BopomofoKeyboardLayout* layout) {
layout_ = layout;
if (layout == BopomofoKeyboardLayout::HanyuPinyinLayout()) {
m_pinyinMode = true;
m_pinyinSequence = "";
pinyin_mode_ = true;
pinyin_sequence_ = "";
}
}
bool isValidKey(char k) const
{
if (!m_pinyinMode) {
return m_layout ? (m_layout->keyToComponents(k)).size() > 0 : false;
bool isValidKey(char k) const {
if (!pinyin_mode_) {
return layout_ ? (layout_->keyToComponents(k)).size() > 0 : false;
}
char lk = tolower(k);
if (lk >= 'a' && lk <= 'z') {
// if a tone marker is already in place
if (m_pinyinSequence.length()) {
char lastc = m_pinyinSequence[m_pinyinSequence.length() - 1];
if (pinyin_sequence_.length()) {
char lastc = pinyin_sequence_[pinyin_sequence_.length() - 1];
if (lastc >= '2' && lastc <= '5') {
return false;
}
@ -510,98 +438,84 @@ namespace Formosa {
return true;
}
if (m_pinyinSequence.length() && (lk >= '2' && lk <= '5')) {
if (pinyin_sequence_.length() && (lk >= '2' && lk <= '5')) {
return true;
}
return false;
}
bool combineKey(char k)
{
if (!isValidKey(k))
return false;
bool combineKey(char k) {
if (!isValidKey(k)) return false;
if (m_pinyinMode) {
m_pinyinSequence += string(1, tolower(k));
m_syllable = BPMF::FromHanyuPinyin(m_pinyinSequence);
if (pinyin_mode_) {
pinyin_sequence_ += std::string(1, tolower(k));
syllable_ = BPMF::FromHanyuPinyin(pinyin_sequence_);
return true;
}
string sequence = m_layout->keySequenceFromSyllable(m_syllable) + string(1, k);
m_syllable = m_layout->syllableFromKeySequence(sequence);
std::string sequence =
layout_->keySequenceFromSyllable(syllable_) + std::string(1, k);
syllable_ = layout_->syllableFromKeySequence(sequence);
return true;
}
void clear()
{
m_pinyinSequence.clear();
m_syllable.clear();
void clear() {
pinyin_sequence_.clear();
syllable_.clear();
}
void backspace()
{
if (!m_layout)
return;
void backspace() {
if (!layout_) return;
if (m_pinyinMode) {
if (m_pinyinSequence.length()) {
m_pinyinSequence = m_pinyinSequence.substr(0, m_pinyinSequence.length() - 1);
if (pinyin_mode_) {
if (pinyin_sequence_.length()) {
pinyin_sequence_ =
pinyin_sequence_.substr(0, pinyin_sequence_.length() - 1);
}
m_syllable = BPMF::FromHanyuPinyin(m_pinyinSequence);
syllable_ = BPMF::FromHanyuPinyin(pinyin_sequence_);
return;
}
string sequence = m_layout->keySequenceFromSyllable(m_syllable);
std::string sequence = layout_->keySequenceFromSyllable(syllable_);
if (sequence.length()) {
sequence = sequence.substr(0, sequence.length() - 1);
m_syllable = m_layout->syllableFromKeySequence(sequence);
syllable_ = layout_->syllableFromKeySequence(sequence);
}
}
bool isEmpty() const
{
return m_syllable.isEmpty();
bool isEmpty() const { return syllable_.isEmpty(); }
const std::string composedString() const {
if (pinyin_mode_) {
return pinyin_sequence_;
}
const string composedString() const
{
if (m_pinyinMode) {
return m_pinyinSequence;
return syllable_.composedString();
}
return m_syllable.composedString();
const BPMF syllable() const { return syllable_; }
const std::string standardLayoutQueryString() const {
return BopomofoKeyboardLayout::StandardLayout()->keySequenceFromSyllable(
syllable_);
}
const BPMF syllable() const
{
return m_syllable;
const std::string absoluteOrderQueryString() const {
return syllable_.absoluteOrderString();
}
const string standardLayoutQueryString() const
{
return BopomofoKeyboardLayout::StandardLayout()->keySequenceFromSyllable(m_syllable);
}
const string absoluteOrderQueryString() const
{
return m_syllable.absoluteOrderString();
}
bool hasToneMarker() const
{
return m_syllable.hasToneMarker();
}
bool hasToneMarker() const { return syllable_.hasToneMarker(); }
protected:
const BopomofoKeyboardLayout* m_layout;
BPMF m_syllable;
const BopomofoKeyboardLayout* layout_;
BPMF syllable_;
bool m_pinyinMode;
string m_pinyinSequence;
bool pinyin_mode_;
std::string pinyin_sequence_;
};
}
}
} // namespace Mandarin
} // namespace Formosa
#endif
#endif // MANDARIN_H_

120
Source/Engine/Mandarin/MandarinTest.cpp Normal file
View File

@ -0,0 +1,120 @@
// Copyright (c) 2022 and onwards Lukhnos Liu
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use,
// copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following
// conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.
#include "Mandarin.h"
#include "gtest/gtest.h"
namespace Formosa {
namespace Mandarin {
static std::string RoundTrip(const std::string& composedString) {
return BopomofoSyllable::FromComposedString(composedString).composedString();
}
TEST(MandarinTest, FromComposedString) {
ASSERT_EQ(RoundTrip("ㄅ"), "ㄅ");
ASSERT_EQ(RoundTrip("ㄅㄧ"), "ㄅㄧ");
ASSERT_EQ(RoundTrip("ㄅㄧˇ"), "ㄅㄧˇ");
ASSERT_EQ(RoundTrip("ㄅㄧˇㄆ"), "ㄆㄧˇ");
ASSERT_EQ(RoundTrip("ㄅㄧˇㄆ"), "ㄆㄧˇ");
ASSERT_EQ(RoundTrip("e"), "");
ASSERT_EQ(RoundTrip("é"), "");
ASSERT_EQ(RoundTrip("ㄅéㄆ"), "ㄅ");
ASSERT_EQ(RoundTrip("ㄅeㄆ"), "ㄅ");
}
TEST(MandarinTest, SimpleCompositions) {
BopomofoSyllable syllable;
syllable += BopomofoSyllable(BopomofoSyllable::X);
syllable += BopomofoSyllable(BopomofoSyllable::I);
ASSERT_EQ(syllable.composedString(), "ㄒㄧ");
syllable.clear();
syllable += BopomofoSyllable(BopomofoSyllable::Z);
syllable += BopomofoSyllable(BopomofoSyllable::ANG);
syllable += BopomofoSyllable(BopomofoSyllable::Tone4);
ASSERT_EQ(syllable.composedString(), "ㄗㄤˋ");
}
TEST(MandarinTest, StandardLayout) {
BopomofoReadingBuffer buf(BopomofoKeyboardLayout::StandardLayout());
buf.combineKey('w');
buf.combineKey('9');
buf.combineKey('6');
ASSERT_EQ(buf.composedString(), "ㄊㄞˊ");
}
TEST(MandarinTest, StandardLayoutCombination) {
BopomofoReadingBuffer buf(BopomofoKeyboardLayout::StandardLayout());
buf.combineKey('w');
buf.combineKey('9');
buf.combineKey('6');
ASSERT_EQ(buf.composedString(), "ㄊㄞˊ");
buf.backspace();
ASSERT_EQ(buf.composedString(), "ㄊㄞ");
buf.combineKey('y');
ASSERT_EQ(buf.composedString(), "ㄗㄞ");
buf.combineKey('4');
ASSERT_EQ(buf.composedString(), "ㄗㄞˋ");
buf.combineKey('3');
ASSERT_EQ(buf.composedString(), "ㄗㄞˇ");
}
TEST(MandarinTest, ETenLayout) {
BopomofoReadingBuffer buf(BopomofoKeyboardLayout::ETenLayout());
buf.combineKey('x');
buf.combineKey('8');
ASSERT_EQ(buf.composedString(), "ㄨㄢ");
}
TEST(MandarinTest, ETen26Layout) {
BopomofoReadingBuffer buf(BopomofoKeyboardLayout::ETen26Layout());
buf.combineKey('q');
buf.combineKey('m'); // AN in the vowel state
buf.combineKey('k'); // Tone 4 in the tone state
ASSERT_EQ(buf.composedString(), "ㄗㄢˋ");
}
TEST(MandarinTest, HsuLayout) {
BopomofoReadingBuffer buf(BopomofoKeyboardLayout::HsuLayout());
buf.combineKey('f');
buf.combineKey('a'); // EI when in the vowel state
buf.combineKey('f'); // Tone 3 when in the tone state
ASSERT_EQ(buf.composedString(), "ㄈㄟˇ");
}
TEST(MandarinTest, IBMLayout) {
BopomofoReadingBuffer buf(BopomofoKeyboardLayout::IBMLayout());
buf.combineKey('9');
buf.combineKey('s');
buf.combineKey('g');
buf.combineKey('m');
ASSERT_EQ(buf.composedString(), "ㄍㄨㄛˊ");
}
} // namespace Mandarin
} // namespace Formosa