Merge pull request #298 from lukhnos/gramambular-modernization
Gramambular modernization
This commit is contained in:
Lukhnos Liu
GitHub
commit
ccd035a666
|
|
@ -24,6 +24,12 @@ jobs:
|
|||
- name: Run MandarinTest
|
||||
run: make runTest
|
||||
working-directory: Source/Engine/Mandarin/build
|
||||
- name: Build GramambularTest
|
||||
run: cmake -S . -B build
|
||||
working-directory: Source/Engine/Gramambular
|
||||
- name: Run GramambularTest
|
||||
run: make runTest
|
||||
working-directory: Source/Engine/Gramambular/build
|
||||
- name: Test McBopomofo App Bundle
|
||||
run: xcodebuild -scheme McBopomofo -configuration Debug test
|
||||
- name: Test CandidateUI
|
||||
|
|
|
|||
|
|
@ -24,6 +24,12 @@ jobs:
|
|||
- name: Run MandarinTest
|
||||
run: make runTest
|
||||
working-directory: Source/Engine/Mandarin/build
|
||||
- name: Build GramambularTest
|
||||
run: cmake -S . -B build
|
||||
working-directory: Source/Engine/Gramambular
|
||||
- name: Run GramambularTest
|
||||
run: make runTest
|
||||
working-directory: Source/Engine/Gramambular/build
|
||||
- name: Test McBopomofo App Bundle
|
||||
run: xcodebuild -scheme McBopomofo -configuration Debug test
|
||||
- name: Test CandidateUI
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@
|
|||
6A6ED16C2797650A0012872E /* template-data.txt in Resources */ = {isa = PBXBuildFile; fileRef = 6A6ED1652797650A0012872E /* template-data.txt */; };
|
||||
6A6ED16D2797650A0012872E /* template-exclude-phrases-plain-bpmf.txt in Resources */ = {isa = PBXBuildFile; fileRef = 6A6ED1672797650A0012872E /* template-exclude-phrases-plain-bpmf.txt */; };
|
||||
6A6ED16E2797650A0012872E /* template-exclude-phrases.txt in Resources */ = {isa = PBXBuildFile; fileRef = 6A6ED1692797650A0012872E /* template-exclude-phrases.txt */; };
|
||||
6A74B14927C16845001988F4 /* Grid.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 6A74B14827C16845001988F4 /* Grid.cpp */; };
|
||||
6ACA41FA15FC1D9000935EF6 /* InfoPlist.strings in Resources */ = {isa = PBXBuildFile; fileRef = 6ACA41EA15FC1D9000935EF6 /* InfoPlist.strings */; };
|
||||
6ACA41FB15FC1D9000935EF6 /* License.rtf in Resources */ = {isa = PBXBuildFile; fileRef = 6ACA41EC15FC1D9000935EF6 /* License.rtf */; };
|
||||
6ACA41FC15FC1D9000935EF6 /* Localizable.strings in Resources */ = {isa = PBXBuildFile; fileRef = 6ACA41EE15FC1D9000935EF6 /* Localizable.strings */; };
|
||||
|
|
@ -59,8 +60,8 @@
|
|||
D485D3B92796A8A000657FF3 /* PreferencesTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D485D3B82796A8A000657FF3 /* PreferencesTests.swift */; };
|
||||
D485D3C02796CE3200657FF3 /* VersionUpdateTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D485D3BF2796CE3200657FF3 /* VersionUpdateTests.swift */; };
|
||||
D4A13D5A27A59F0B003BE359 /* InputMethodController.swift in Sources */ = {isa = PBXBuildFile; fileRef = D4A13D5927A59D5C003BE359 /* InputMethodController.swift */; };
|
||||
D4C9CAB127AAC9690058DFEA /* NSStringUtils in Frameworks */ = {isa = PBXBuildFile; productRef = D4C9CAB027AAC9690058DFEA /* NSStringUtils */; };
|
||||
D4A8E43627A9E982002F7A07 /* KeyHandlerPlainBopomofoTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D4A8E43527A9E982002F7A07 /* KeyHandlerPlainBopomofoTests.swift */; };
|
||||
D4C9CAB127AAC9690058DFEA /* NSStringUtils in Frameworks */ = {isa = PBXBuildFile; productRef = D4C9CAB027AAC9690058DFEA /* NSStringUtils */; };
|
||||
D4E33D8A27A838CF006DB1CF /* Localizable.strings in Resources */ = {isa = PBXBuildFile; fileRef = D4E33D8827A838CF006DB1CF /* Localizable.strings */; };
|
||||
D4E33D8F27A838F0006DB1CF /* InfoPlist.strings in Resources */ = {isa = PBXBuildFile; fileRef = D4E33D8D27A838F0006DB1CF /* InfoPlist.strings */; };
|
||||
D4E569DC27A34D0E00AC2CEF /* KeyHandler.mm in Sources */ = {isa = PBXBuildFile; fileRef = D4E569DB27A34CC100AC2CEF /* KeyHandler.mm */; };
|
||||
|
|
@ -131,6 +132,7 @@
|
|||
6A6ED170279765140012872E /* zh-Hant */ = {isa = PBXFileReference; lastKnownFileType = text; name = "zh-Hant"; path = "zh-Hant.lproj/template-exclude-phrases-plain-bpmf.txt"; sourceTree = "<group>"; };
|
||||
6A6ED171279765170012872E /* zh-Hant */ = {isa = PBXFileReference; lastKnownFileType = text; name = "zh-Hant"; path = "zh-Hant.lproj/template-exclude-phrases.txt"; sourceTree = "<group>"; };
|
||||
6A6ED1722797651A0012872E /* zh-Hant */ = {isa = PBXFileReference; lastKnownFileType = text; name = "zh-Hant"; path = "zh-Hant.lproj/template-phrases-replacement.txt"; sourceTree = "<group>"; };
|
||||
6A74B14827C16845001988F4 /* Grid.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = Grid.cpp; path = Grid.cpp; sourceTree = "<group>"; };
|
||||
6A93050C279877FF00D370DA /* McBopomofoInstaller-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "McBopomofoInstaller-Bridging-Header.h"; sourceTree = "<group>"; };
|
||||
6ACA41CB15FC1D7500935EF6 /* McBopomofoInstaller.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = McBopomofoInstaller.app; sourceTree = BUILT_PRODUCTS_DIR; };
|
||||
6ACA41EB15FC1D9000935EF6 /* en */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = en; path = en.lproj/InfoPlist.strings; sourceTree = "<group>"; };
|
||||
|
|
@ -186,8 +188,8 @@
|
|||
D485D3BF2796CE3200657FF3 /* VersionUpdateTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VersionUpdateTests.swift; sourceTree = "<group>"; };
|
||||
D495583A27A5C6C4006ADE1C /* LanguageModelManager+Privates.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "LanguageModelManager+Privates.h"; sourceTree = "<group>"; };
|
||||
D4A13D5927A59D5C003BE359 /* InputMethodController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InputMethodController.swift; sourceTree = "<group>"; };
|
||||
D4C9CAAF27AAC8EC0058DFEA /* NSStringUtils */ = {isa = PBXFileReference; lastKnownFileType = wrapper; name = NSStringUtils; path = Packages/NSStringUtils; sourceTree = "<group>"; };
|
||||
D4A8E43527A9E982002F7A07 /* KeyHandlerPlainBopomofoTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeyHandlerPlainBopomofoTests.swift; sourceTree = "<group>"; };
|
||||
D4C9CAAF27AAC8EC0058DFEA /* NSStringUtils */ = {isa = PBXFileReference; lastKnownFileType = wrapper; name = NSStringUtils; path = Packages/NSStringUtils; sourceTree = "<group>"; };
|
||||
D4E33D8927A838CF006DB1CF /* Base */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = Base; path = Base.lproj/Localizable.strings; sourceTree = "<group>"; };
|
||||
D4E33D8B27A838D5006DB1CF /* en */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = en; path = en.lproj/Localizable.strings; sourceTree = "<group>"; };
|
||||
D4E33D8C27A838D8006DB1CF /* zh-Hant */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = "zh-Hant"; path = "zh-Hant.lproj/Localizable.strings"; sourceTree = "<group>"; };
|
||||
|
|
@ -329,6 +331,7 @@
|
|||
6A0D4F1415FC0EB100ABF4B3 /* Bigram.h */,
|
||||
6A0D4F1515FC0EB100ABF4B3 /* BlockReadingBuilder.h */,
|
||||
6A0D4F1615FC0EB100ABF4B3 /* Gramambular.h */,
|
||||
6A74B14827C16845001988F4 /* Grid.cpp */,
|
||||
6A0D4F1715FC0EB100ABF4B3 /* Grid.h */,
|
||||
6A0D4F1815FC0EB100ABF4B3 /* KeyValuePair.h */,
|
||||
6A0D4F1915FC0EB100ABF4B3 /* LanguageModel.h */,
|
||||
|
|
@ -669,6 +672,7 @@
|
|||
D41355DE278EA3ED005E5CBD /* UserPhrasesLM.cpp in Sources */,
|
||||
6ACC3D3F27914F2400F1B140 /* KeyValueBlobReader.cpp in Sources */,
|
||||
D41355D8278D74B5005E5CBD /* LanguageModelManager.mm in Sources */,
|
||||
6A74B14927C16845001988F4 /* Grid.cpp in Sources */,
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -25,16 +25,16 @@
|
|||
// OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
|
||||
#ifndef Bigram_h
|
||||
#define Bigram_h
|
||||
#ifndef BIGRAM_H_
|
||||
#define BIGRAM_H_
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "KeyValuePair.h"
|
||||
|
||||
namespace Formosa {
|
||||
namespace Gramambular {
|
||||
class Bigram {
|
||||
namespace Gramambular {
|
||||
class Bigram {
|
||||
public:
|
||||
Bigram();
|
||||
|
||||
|
|
@ -42,65 +42,60 @@ namespace Formosa {
|
|||
KeyValuePair keyValue;
|
||||
double score;
|
||||
|
||||
bool operator==(const Bigram& inAnother) const;
|
||||
bool operator<(const Bigram& inAnother) const;
|
||||
};
|
||||
bool operator==(const Bigram& another) const;
|
||||
bool operator<(const Bigram& another) const;
|
||||
};
|
||||
|
||||
inline ostream& operator<<(ostream& inStream, const Bigram& inGram)
|
||||
{
|
||||
streamsize p = inStream.precision();
|
||||
inStream.precision(6);
|
||||
inStream << "(" << inGram.keyValue << "|" <<inGram.preceedingKeyValue << "," << inGram.score << ")";
|
||||
inStream.precision(p);
|
||||
return inStream;
|
||||
}
|
||||
inline std::ostream& operator<<(std::ostream& stream, const Bigram& gram) {
|
||||
std::streamsize p = stream.precision();
|
||||
stream.precision(6);
|
||||
stream << "(" << gram.keyValue << "|" << gram.preceedingKeyValue << ","
|
||||
<< gram.score << ")";
|
||||
stream.precision(p);
|
||||
return stream;
|
||||
}
|
||||
|
||||
inline ostream& operator<<(ostream& inStream, const vector<Bigram>& inGrams)
|
||||
{
|
||||
inStream << "[" << inGrams.size() << "]=>{";
|
||||
inline std::ostream& operator<<(std::ostream& stream,
|
||||
const std::vector<Bigram>& grams) {
|
||||
stream << "[" << grams.size() << "]=>{";
|
||||
|
||||
size_t index = 0;
|
||||
|
||||
for (vector<Bigram>::const_iterator gi = inGrams.begin() ; gi != inGrams.end() ; ++gi, ++index) {
|
||||
inStream << index << "=>";
|
||||
inStream << *gi;
|
||||
if (gi + 1 != inGrams.end()) {
|
||||
inStream << ",";
|
||||
for (std::vector<Bigram>::const_iterator gi = grams.begin();
|
||||
gi != grams.end(); ++gi, ++index) {
|
||||
stream << index << "=>";
|
||||
stream << *gi;
|
||||
if (gi + 1 != grams.end()) {
|
||||
stream << ",";
|
||||
}
|
||||
}
|
||||
|
||||
inStream << "}";
|
||||
return inStream;
|
||||
}
|
||||
|
||||
inline Bigram::Bigram()
|
||||
: score(0.0)
|
||||
{
|
||||
}
|
||||
|
||||
inline bool Bigram::operator==(const Bigram& inAnother) const
|
||||
{
|
||||
return preceedingKeyValue == inAnother.preceedingKeyValue && keyValue == inAnother.keyValue && score == inAnother.score;
|
||||
}
|
||||
|
||||
inline bool Bigram::operator<(const Bigram& inAnother) const
|
||||
{
|
||||
if (preceedingKeyValue < inAnother.preceedingKeyValue) {
|
||||
return true;
|
||||
}
|
||||
else if (preceedingKeyValue == inAnother.preceedingKeyValue) {
|
||||
if (keyValue < inAnother.keyValue) {
|
||||
return true;
|
||||
}
|
||||
else if (keyValue == inAnother.keyValue) {
|
||||
return score < inAnother.score;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
stream << "}";
|
||||
return stream;
|
||||
}
|
||||
|
||||
inline Bigram::Bigram() : score(0.0) {}
|
||||
|
||||
inline bool Bigram::operator==(const Bigram& another) const {
|
||||
return preceedingKeyValue == another.preceedingKeyValue &&
|
||||
keyValue == another.keyValue && score == another.score;
|
||||
}
|
||||
|
||||
inline bool Bigram::operator<(const Bigram& another) const {
|
||||
if (preceedingKeyValue < another.preceedingKeyValue) {
|
||||
return true;
|
||||
} else if (preceedingKeyValue == another.preceedingKeyValue) {
|
||||
if (keyValue < another.keyValue) {
|
||||
return true;
|
||||
} else if (keyValue == another.keyValue) {
|
||||
return score < another.score;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
} // namespace Gramambular
|
||||
} // namespace Formosa
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -25,123 +25,113 @@
|
|||
// OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
|
||||
#ifndef BlockReadingBuilder_h
|
||||
#define BlockReadingBuilder_h
|
||||
#ifndef BLOCKREADINGBUILDER_H_
|
||||
#define BLOCKREADINGBUILDER_H_
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "Grid.h"
|
||||
#include "LanguageModel.h"
|
||||
|
||||
namespace Formosa {
|
||||
namespace Gramambular {
|
||||
using namespace std;
|
||||
namespace Gramambular {
|
||||
|
||||
class BlockReadingBuilder {
|
||||
class BlockReadingBuilder {
|
||||
public:
|
||||
BlockReadingBuilder(LanguageModel *inLM);
|
||||
explicit BlockReadingBuilder(LanguageModel* lm);
|
||||
void clear();
|
||||
|
||||
size_t length() const;
|
||||
size_t cursorIndex() const;
|
||||
void setCursorIndex(size_t inNewIndex);
|
||||
void insertReadingAtCursor(const string& inReading);
|
||||
void setCursorIndex(size_t newIndex);
|
||||
void insertReadingAtCursor(const std::string& reading);
|
||||
bool deleteReadingBeforeCursor(); // backspace
|
||||
bool deleteReadingAfterCursor(); // delete
|
||||
|
||||
bool removeHeadReadings(size_t count);
|
||||
|
||||
void setJoinSeparator(const string& separator);
|
||||
const string joinSeparator() const;
|
||||
void setJoinSeparator(const std::string& separator);
|
||||
const std::string joinSeparator() const;
|
||||
|
||||
vector<string> readings() const;
|
||||
std::vector<std::string> readings() const;
|
||||
|
||||
Grid& grid();
|
||||
|
||||
protected:
|
||||
void build();
|
||||
|
||||
static const string Join(vector<string>::const_iterator begin, vector<string>::const_iterator end, const string& separator);
|
||||
static const std::string Join(std::vector<std::string>::const_iterator begin,
|
||||
std::vector<std::string>::const_iterator end,
|
||||
const std::string& separator);
|
||||
|
||||
//最多使用六個字組成一個詞
|
||||
// 最多使用六個字組成一個詞
|
||||
static const size_t MaximumBuildSpanLength = 6;
|
||||
|
||||
size_t m_cursorIndex;
|
||||
vector<string> m_readings;
|
||||
std::vector<std::string> m_readings;
|
||||
|
||||
Grid m_grid;
|
||||
LanguageModel *m_LM;
|
||||
string m_joinSeparator;
|
||||
};
|
||||
LanguageModel* m_LM;
|
||||
std::string m_joinSeparator;
|
||||
};
|
||||
|
||||
inline BlockReadingBuilder::BlockReadingBuilder(LanguageModel *inLM)
|
||||
: m_LM(inLM)
|
||||
, m_cursorIndex(0)
|
||||
{
|
||||
}
|
||||
inline BlockReadingBuilder::BlockReadingBuilder(LanguageModel* lm)
|
||||
: m_LM(lm), m_cursorIndex(0) {}
|
||||
|
||||
inline void BlockReadingBuilder::clear()
|
||||
{
|
||||
inline void BlockReadingBuilder::clear() {
|
||||
m_cursorIndex = 0;
|
||||
m_readings.clear();
|
||||
m_grid.clear();
|
||||
}
|
||||
}
|
||||
|
||||
inline size_t BlockReadingBuilder::length() const
|
||||
{
|
||||
return m_readings.size();
|
||||
}
|
||||
inline size_t BlockReadingBuilder::length() const { return m_readings.size(); }
|
||||
|
||||
inline size_t BlockReadingBuilder::cursorIndex() const
|
||||
{
|
||||
return m_cursorIndex;
|
||||
}
|
||||
inline size_t BlockReadingBuilder::cursorIndex() const { return m_cursorIndex; }
|
||||
|
||||
inline void BlockReadingBuilder::setCursorIndex(size_t inNewIndex)
|
||||
{
|
||||
m_cursorIndex = inNewIndex > m_readings.size() ? m_readings.size() : inNewIndex;
|
||||
}
|
||||
inline void BlockReadingBuilder::setCursorIndex(size_t newIndex) {
|
||||
m_cursorIndex = newIndex > m_readings.size() ? m_readings.size() : newIndex;
|
||||
}
|
||||
|
||||
inline void BlockReadingBuilder::insertReadingAtCursor(const string& inReading)
|
||||
{
|
||||
m_readings.insert(m_readings.begin() + m_cursorIndex, inReading);
|
||||
inline void BlockReadingBuilder::insertReadingAtCursor(
|
||||
const std::string& reading) {
|
||||
m_readings.insert(m_readings.begin() + m_cursorIndex, reading);
|
||||
|
||||
m_grid.expandGridByOneAtLocation(m_cursorIndex);
|
||||
build();
|
||||
m_cursorIndex++;
|
||||
}
|
||||
}
|
||||
|
||||
inline vector<string> BlockReadingBuilder::readings() const
|
||||
{
|
||||
inline std::vector<std::string> BlockReadingBuilder::readings() const {
|
||||
return m_readings;
|
||||
}
|
||||
}
|
||||
|
||||
inline bool BlockReadingBuilder::deleteReadingBeforeCursor()
|
||||
{
|
||||
inline bool BlockReadingBuilder::deleteReadingBeforeCursor() {
|
||||
if (!m_cursorIndex) {
|
||||
return false;
|
||||
}
|
||||
|
||||
m_readings.erase(m_readings.begin() + m_cursorIndex - 1, m_readings.begin() + m_cursorIndex);
|
||||
m_readings.erase(m_readings.begin() + m_cursorIndex - 1,
|
||||
m_readings.begin() + m_cursorIndex);
|
||||
m_cursorIndex--;
|
||||
m_grid.shrinkGridByOneAtLocation(m_cursorIndex);
|
||||
build();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
inline bool BlockReadingBuilder::deleteReadingAfterCursor()
|
||||
{
|
||||
inline bool BlockReadingBuilder::deleteReadingAfterCursor() {
|
||||
if (m_cursorIndex == m_readings.size()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
m_readings.erase(m_readings.begin() + m_cursorIndex, m_readings.begin() + m_cursorIndex + 1);
|
||||
m_readings.erase(m_readings.begin() + m_cursorIndex,
|
||||
m_readings.begin() + m_cursorIndex + 1);
|
||||
m_grid.shrinkGridByOneAtLocation(m_cursorIndex);
|
||||
build();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
inline bool BlockReadingBuilder::removeHeadReadings(size_t count)
|
||||
{
|
||||
inline bool BlockReadingBuilder::removeHeadReadings(size_t count) {
|
||||
if (count > length()) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -156,25 +146,20 @@ namespace Formosa {
|
|||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
inline void BlockReadingBuilder::setJoinSeparator(const string& separator)
|
||||
{
|
||||
inline void BlockReadingBuilder::setJoinSeparator(
|
||||
const std::string& separator) {
|
||||
m_joinSeparator = separator;
|
||||
}
|
||||
}
|
||||
|
||||
inline const string BlockReadingBuilder::joinSeparator() const
|
||||
{
|
||||
inline const std::string BlockReadingBuilder::joinSeparator() const {
|
||||
return m_joinSeparator;
|
||||
}
|
||||
}
|
||||
|
||||
inline Grid& BlockReadingBuilder::grid()
|
||||
{
|
||||
return m_grid;
|
||||
}
|
||||
inline Grid& BlockReadingBuilder::grid() { return m_grid; }
|
||||
|
||||
inline void BlockReadingBuilder::build()
|
||||
{
|
||||
inline void BlockReadingBuilder::build() {
|
||||
if (!m_LM) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -184,8 +169,7 @@ namespace Formosa {
|
|||
|
||||
if (m_cursorIndex < MaximumBuildSpanLength) {
|
||||
begin = 0;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
begin = m_cursorIndex - MaximumBuildSpanLength;
|
||||
}
|
||||
|
||||
|
|
@ -193,25 +177,29 @@ namespace Formosa {
|
|||
end = m_readings.size();
|
||||
}
|
||||
|
||||
for (size_t p = begin ; p < end ; p++) {
|
||||
for (size_t q = 1 ; q <= MaximumBuildSpanLength && p+q <= end ; q++) {
|
||||
string combinedReading = Join(m_readings.begin() + p, m_readings.begin() + p + q, m_joinSeparator);
|
||||
if (!m_grid.hasNodeAtLocationSpanningLengthMatchingKey(p, q, combinedReading)) {
|
||||
vector<Unigram> unigrams = m_LM->unigramsForKey(combinedReading);
|
||||
for (size_t p = begin; p < end; p++) {
|
||||
for (size_t q = 1; q <= MaximumBuildSpanLength && p + q <= end; q++) {
|
||||
std::string combinedReading = Join(
|
||||
m_readings.begin() + p, m_readings.begin() + p + q, m_joinSeparator);
|
||||
if (!m_grid.hasNodeAtLocationSpanningLengthMatchingKey(p, q,
|
||||
combinedReading)) {
|
||||
std::vector<Unigram> unigrams = m_LM->unigramsForKey(combinedReading);
|
||||
|
||||
if (unigrams.size() > 0) {
|
||||
Node n(combinedReading, unigrams, vector<Bigram>());
|
||||
Node n(combinedReading, unigrams, std::vector<Bigram>());
|
||||
m_grid.insertNode(n, p, q);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inline const string BlockReadingBuilder::Join(vector<string>::const_iterator begin, vector<string>::const_iterator end, const string& separator)
|
||||
{
|
||||
string result;
|
||||
for (vector<string>::const_iterator iter = begin ; iter != end ; ) {
|
||||
inline const std::string BlockReadingBuilder::Join(
|
||||
std::vector<std::string>::const_iterator begin,
|
||||
std::vector<std::string>::const_iterator end,
|
||||
const std::string& separator) {
|
||||
std::string result;
|
||||
for (std::vector<std::string>::const_iterator iter = begin; iter != end;) {
|
||||
result += *iter;
|
||||
++iter;
|
||||
if (iter != end) {
|
||||
|
|
@ -219,8 +207,8 @@ namespace Formosa {
|
|||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace Gramambular
|
||||
} // namespace Formosa
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -0,0 +1,31 @@
|
|||
cmake_minimum_required(VERSION 3.17)
|
||||
project(Gramambular)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
|
||||
add_library(GramambularLib Bigram.h BlockReadingBuilder.h Gramambular.h Grid.h Grid.cpp KeyValuePair.h LanguageModel.h Node.h NodeAnchor.h Span.h Unigram.h Walker.h)
|
||||
|
||||
# Let CMake fetch Google Test for us.
|
||||
# https://github.com/google/googletest/tree/main/googletest#incorporating-into-an-existing-cmake-project
|
||||
include(FetchContent)
|
||||
|
||||
FetchContent_Declare(
|
||||
googletest
|
||||
# Specify the commit you depend on and update it regularly.
|
||||
URL https://github.com/google/googletest/archive/609281088cfefc76f9d0ce82e1ff6c30cc3591e5.zip
|
||||
)
|
||||
# For Windows: Prevent overriding the parent project's compiler/linker settings
|
||||
set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
|
||||
FetchContent_MakeAvailable(googletest)
|
||||
|
||||
# Test target declarations.
|
||||
add_executable(GramambularTest GramambularTest.cpp)
|
||||
target_link_libraries(GramambularTest gtest_main GramambularLib)
|
||||
include(GoogleTest)
|
||||
gtest_discover_tests(GramambularTest)
|
||||
|
||||
add_custom_target(
|
||||
runTest
|
||||
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/GramambularTest
|
||||
)
|
||||
add_dependencies(runTest GramambularTest)
|
||||
|
|
@ -25,8 +25,8 @@
|
|||
// OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
|
||||
#ifndef Gramambular_h
|
||||
#define Gramambular_h
|
||||
#ifndef GRAMAMBULAR_H_
|
||||
#define GRAMAMBULAR_H_
|
||||
|
||||
#include "Bigram.h"
|
||||
#include "BlockReadingBuilder.h"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,247 @@
|
|||
// Copyright (c) 2022 and onwards Lukhnos Liu
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the "Software"), to deal in the Software without
|
||||
// restriction, including without limitation the rights to use,
|
||||
// copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following
|
||||
// conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be
|
||||
// included in all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
// OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <sstream>
|
||||
#include <vector>
|
||||
|
||||
#include "Gramambular.h"
|
||||
#include "gtest/gtest.h"
|
||||
|
||||
namespace Formosa {
|
||||
namespace Gramambular {
|
||||
|
||||
const char* SampleData = R"(
|
||||
#
|
||||
# The sample is from libtabe (http://sourceforge.net/projects/libtabe/)
|
||||
# last updated in 2002. The project was originally initiated by
|
||||
# Pai-Hsiang Hsiao in 1999.
|
||||
#
|
||||
# Libtabe is a frequency table of Taiwanese Mandarin words. The database
|
||||
# itself is, according to the tar file, released under the BSD License.
|
||||
#
|
||||
|
||||
ㄙ 絲 -9.495858
|
||||
ㄙ 思 -9.006414
|
||||
ㄙ 私 -99.000000
|
||||
ㄙ 斯 -8.091803
|
||||
ㄙ 司 -99.000000
|
||||
ㄙ 嘶 -13.513987
|
||||
ㄙ 撕 -12.259095
|
||||
ㄍㄠ 高 -7.171551
|
||||
ㄎㄜ 顆 -10.574273
|
||||
ㄎㄜ 棵 -11.504072
|
||||
ㄎㄜ 刻 -10.450457
|
||||
ㄎㄜ 科 -7.171052
|
||||
ㄎㄜ 柯 -99.000000
|
||||
ㄍㄠ 膏 -11.928720
|
||||
ㄍㄠ 篙 -13.624335
|
||||
ㄍㄠ 糕 -12.390804
|
||||
ㄉㄜ˙ 的 -3.516024
|
||||
ㄉㄧˊ 的 -3.516024
|
||||
ㄉㄧˋ 的 -3.516024
|
||||
ㄓㄨㄥ 中 -5.809297
|
||||
ㄉㄜ˙ 得 -7.427179
|
||||
ㄍㄨㄥ 共 -8.381971
|
||||
ㄍㄨㄥ 供 -8.501463
|
||||
ㄐㄧˋ 既 -99.000000
|
||||
ㄐㄧㄣ 今 -8.034095
|
||||
ㄍㄨㄥ 紅 -8.858181
|
||||
ㄐㄧˋ 際 -7.608341
|
||||
ㄐㄧˋ 季 -99.000000
|
||||
ㄐㄧㄣ 金 -7.290109
|
||||
ㄐㄧˋ 騎 -10.939895
|
||||
ㄓㄨㄥ 終 -99.000000
|
||||
ㄐㄧˋ 記 -99.000000
|
||||
ㄐㄧˋ 寄 -99.000000
|
||||
ㄐㄧㄣ 斤 -99.000000
|
||||
ㄐㄧˋ 繼 -9.715317
|
||||
ㄐㄧˋ 計 -7.926683
|
||||
ㄐㄧˋ 暨 -8.373022
|
||||
ㄓㄨㄥ 鐘 -9.877580
|
||||
ㄐㄧㄣ 禁 -10.711079
|
||||
ㄍㄨㄥ 公 -7.877973
|
||||
ㄍㄨㄥ 工 -7.822167
|
||||
ㄍㄨㄥ 攻 -99.000000
|
||||
ㄍㄨㄥ 功 -99.000000
|
||||
ㄍㄨㄥ 宮 -99.000000
|
||||
ㄓㄨㄥ 鍾 -9.685671
|
||||
ㄐㄧˋ 繫 -10.425662
|
||||
ㄍㄨㄥ 弓 -99.000000
|
||||
ㄍㄨㄥ 恭 -99.000000
|
||||
ㄐㄧˋ 劑 -8.888722
|
||||
ㄐㄧˋ 祭 -10.204425
|
||||
ㄐㄧㄣ 浸 -11.378321
|
||||
ㄓㄨㄥ 盅 -99.000000
|
||||
ㄐㄧˋ 忌 -99.000000
|
||||
ㄐㄧˋ 技 -8.450826
|
||||
ㄐㄧㄣ 筋 -11.074890
|
||||
ㄍㄨㄥ 躬 -99.000000
|
||||
ㄐㄧˋ 冀 -12.045357
|
||||
ㄓㄨㄥ 忠 -99.000000
|
||||
ㄐㄧˋ 妓 -99.000000
|
||||
ㄐㄧˋ 濟 -9.517568
|
||||
ㄐㄧˋ 薊 -12.021587
|
||||
ㄐㄧㄣ 巾 -99.000000
|
||||
ㄐㄧㄣ 襟 -12.784206
|
||||
ㄋㄧㄢˊ 年 -6.086515
|
||||
ㄐㄧㄤˇ 講 -9.164384
|
||||
ㄐㄧㄤˇ 獎 -8.690941
|
||||
ㄐㄧㄤˇ 蔣 -10.127828
|
||||
ㄋㄧㄢˊ 黏 -11.336864
|
||||
ㄋㄧㄢˊ 粘 -11.285740
|
||||
ㄐㄧㄤˇ 槳 -12.492933
|
||||
ㄍㄨㄥㄙ 公司 -6.299461
|
||||
ㄎㄜㄐㄧˋ 科技 -6.736613
|
||||
ㄐㄧˋㄍㄨㄥ 濟公 -13.336653
|
||||
ㄐㄧㄤˇㄐㄧㄣ 獎金 -10.344678
|
||||
ㄋㄧㄢˊㄓㄨㄥ 年終 -11.668947
|
||||
ㄋㄧㄢˊㄓㄨㄥ 年中 -11.373044
|
||||
ㄍㄠㄎㄜㄐㄧˋ 高科技 -9.842421
|
||||
)";
|
||||
|
||||
class SimpleLM : public LanguageModel {
|
||||
public:
|
||||
SimpleLM(const char* input, bool swapKeyValue = false) {
|
||||
std::stringstream sstream(input);
|
||||
while (sstream.good()) {
|
||||
std::string line;
|
||||
getline(sstream, line);
|
||||
|
||||
if (!line.size() || (line.size() && line[0] == '#')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::stringstream linestream(line);
|
||||
std::string col0;
|
||||
std::string col1;
|
||||
std::string col2;
|
||||
linestream >> col0;
|
||||
linestream >> col1;
|
||||
linestream >> col2;
|
||||
|
||||
Unigram u;
|
||||
|
||||
if (swapKeyValue) {
|
||||
u.keyValue.key = col1;
|
||||
u.keyValue.value = col0;
|
||||
} else {
|
||||
u.keyValue.key = col0;
|
||||
u.keyValue.value = col1;
|
||||
}
|
||||
|
||||
u.score = atof(col2.c_str());
|
||||
|
||||
m_db[u.keyValue.key].push_back(u);
|
||||
}
|
||||
}
|
||||
|
||||
const std::vector<Bigram> bigramsForKeys(const std::string& preceedingKey,
|
||||
const std::string& key) override {
|
||||
return std::vector<Bigram>();
|
||||
}
|
||||
|
||||
const std::vector<Unigram> unigramsForKey(const std::string& key) override {
|
||||
std::map<std::string, std::vector<Unigram> >::const_iterator f =
|
||||
m_db.find(key);
|
||||
return f == m_db.end() ? std::vector<Unigram>() : (*f).second;
|
||||
}
|
||||
|
||||
bool hasUnigramsForKey(const std::string& key) override {
|
||||
std::map<std::string, std::vector<Unigram> >::const_iterator f =
|
||||
m_db.find(key);
|
||||
return f != m_db.end();
|
||||
}
|
||||
|
||||
protected:
|
||||
std::map<std::string, std::vector<Unigram> > m_db;
|
||||
};
|
||||
|
||||
TEST(GramambularTest, InputTest) {
|
||||
SimpleLM lm(SampleData);
|
||||
|
||||
BlockReadingBuilder builder(&lm);
|
||||
builder.insertReadingAtCursor("ㄍㄠ");
|
||||
builder.insertReadingAtCursor("ㄐㄧˋ");
|
||||
builder.setCursorIndex(1);
|
||||
builder.insertReadingAtCursor("ㄎㄜ");
|
||||
builder.setCursorIndex(0);
|
||||
builder.deleteReadingAfterCursor();
|
||||
builder.insertReadingAtCursor("ㄍㄠ");
|
||||
builder.setCursorIndex(builder.length());
|
||||
builder.insertReadingAtCursor("ㄍㄨㄥ");
|
||||
builder.insertReadingAtCursor("ㄙ");
|
||||
builder.insertReadingAtCursor("ㄉㄜ˙");
|
||||
builder.insertReadingAtCursor("ㄋㄧㄢˊ");
|
||||
builder.insertReadingAtCursor("ㄓㄨㄥ");
|
||||
builder.insertReadingAtCursor("ㄐㄧㄤˇ");
|
||||
builder.insertReadingAtCursor("ㄐㄧㄣ");
|
||||
|
||||
Walker walker(&builder.grid());
|
||||
|
||||
std::vector<NodeAnchor> walked =
|
||||
walker.reverseWalk(builder.grid().width(), 0.0);
|
||||
reverse(walked.begin(), walked.end());
|
||||
|
||||
std::vector<std::string> composed;
|
||||
for (std::vector<NodeAnchor>::iterator wi = walked.begin();
|
||||
wi != walked.end(); ++wi) {
|
||||
composed.push_back((*wi).node->currentKeyValue().value);
|
||||
}
|
||||
ASSERT_EQ(composed,
|
||||
(std::vector<std::string>{"高科技", "公司", "的", "年中", "獎金"}));
|
||||
}
|
||||
|
||||
TEST(GramambularTest, WordSegmentationTest) {
|
||||
SimpleLM lm2(SampleData, true);
|
||||
BlockReadingBuilder builder2(&lm2);
|
||||
builder2.insertReadingAtCursor("高");
|
||||
builder2.insertReadingAtCursor("科");
|
||||
builder2.insertReadingAtCursor("技");
|
||||
builder2.insertReadingAtCursor("公");
|
||||
builder2.insertReadingAtCursor("司");
|
||||
builder2.insertReadingAtCursor("的");
|
||||
builder2.insertReadingAtCursor("年");
|
||||
builder2.insertReadingAtCursor("終");
|
||||
builder2.insertReadingAtCursor("獎");
|
||||
builder2.insertReadingAtCursor("金");
|
||||
Walker walker2(&builder2.grid());
|
||||
|
||||
std::vector<NodeAnchor> walked =
|
||||
walker2.reverseWalk(builder2.grid().width(), 0.0);
|
||||
reverse(walked.begin(), walked.end());
|
||||
|
||||
std::vector<std::string> segmented;
|
||||
for (std::vector<NodeAnchor>::iterator wi = walked.begin();
|
||||
wi != walked.end(); ++wi) {
|
||||
segmented.push_back((*wi).node->currentKeyValue().key);
|
||||
}
|
||||
ASSERT_EQ(segmented,
|
||||
(std::vector<std::string>{"高科技", "公司", "的", "年終", "獎金"}));
|
||||
}
|
||||
|
||||
} // namespace Gramambular
|
||||
} // namespace Formosa
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
// Copyright (c) 2007 and onwards Lukhnos Liu
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the "Software"), to deal in the Software without
|
||||
// restriction, including without limitation the rights to use,
|
||||
// copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following
|
||||
// conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be
|
||||
// included in all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
// OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
#include "Grid.h"
|
||||
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
|
||||
namespace Formosa {
|
||||
namespace Gramambular {
|
||||
|
||||
std::string Grid::dumpDOT() {
|
||||
std::stringstream sst;
|
||||
sst << "digraph {" << std::endl;
|
||||
sst << "graph [ rankdir=LR ];" << std::endl;
|
||||
sst << "BOS;" << std::endl;
|
||||
|
||||
for (size_t p = 0; p < m_spans.size(); p++) {
|
||||
Span& span = m_spans[p];
|
||||
for (size_t ni = 0; ni <= span.maximumLength(); ni++) {
|
||||
Node* np = span.nodeOfLength(ni);
|
||||
if (np) {
|
||||
if (!p) {
|
||||
sst << "BOS -> " << np->currentKeyValue().value << ";" << std::endl;
|
||||
}
|
||||
|
||||
sst << np->currentKeyValue().value << ";" << std::endl;
|
||||
|
||||
if (p + ni < m_spans.size()) {
|
||||
Span& dstSpan = m_spans[p + ni];
|
||||
for (size_t q = 0; q <= dstSpan.maximumLength(); q++) {
|
||||
Node* dn = dstSpan.nodeOfLength(q);
|
||||
if (dn) {
|
||||
sst << np->currentKeyValue().value << " -> "
|
||||
<< dn->currentKeyValue().value << ";" << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (p + ni == m_spans.size()) {
|
||||
sst << np->currentKeyValue().value << " -> "
|
||||
<< "EOS;" << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sst << "EOS;" << std::endl;
|
||||
sst << "}";
|
||||
return sst.str();
|
||||
}
|
||||
|
||||
} // namespace Gramambular
|
||||
} // namespace Formosa
|
||||
|
|
@ -25,125 +25,124 @@
|
|||
// OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
|
||||
#ifndef Grid_h
|
||||
#define Grid_h
|
||||
#ifndef GRID_H_
|
||||
#define GRID_H_
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "NodeAnchor.h"
|
||||
#include "Span.h"
|
||||
|
||||
namespace Formosa {
|
||||
namespace Gramambular {
|
||||
namespace Gramambular {
|
||||
|
||||
class Grid {
|
||||
class Grid {
|
||||
public:
|
||||
void clear();
|
||||
void insertNode(const Node& inNode, size_t inLocation, size_t inSpanningLength);
|
||||
bool hasNodeAtLocationSpanningLengthMatchingKey(size_t inLocation, size_t inSpanningLength, const string& inKey);
|
||||
void insertNode(const Node& node, size_t location, size_t spanningLength);
|
||||
bool hasNodeAtLocationSpanningLengthMatchingKey(size_t location,
|
||||
size_t spanningLength,
|
||||
const std::string& key);
|
||||
|
||||
void expandGridByOneAtLocation(size_t inLocation);
|
||||
void shrinkGridByOneAtLocation(size_t inLocation);
|
||||
void expandGridByOneAtLocation(size_t location);
|
||||
void shrinkGridByOneAtLocation(size_t location);
|
||||
|
||||
size_t width() const;
|
||||
vector<NodeAnchor> nodesEndingAt(size_t inLocation);
|
||||
vector<NodeAnchor> nodesCrossingOrEndingAt(size_t inLocation);
|
||||
std::vector<NodeAnchor> nodesEndingAt(size_t location);
|
||||
std::vector<NodeAnchor> nodesCrossingOrEndingAt(size_t location);
|
||||
|
||||
// "Freeze" the node with the unigram that represents the selected candidate value.
|
||||
// After this, the node that contains the unigram will always be evaluated to that
|
||||
// unigram, while all other overlapping nodes will be reset to their initial state
|
||||
// (that is, if any of those nodes were "frozen" or fixed, they will be unfrozen.)
|
||||
NodeAnchor fixNodeSelectedCandidate(size_t location, const string& value);
|
||||
// "Freeze" the node with the unigram that represents the selected candidate
|
||||
// value. After this, the node that contains the unigram will always be
|
||||
// evaluated to that unigram, while all other overlapping nodes will be reset
|
||||
// to their initial state (that is, if any of those nodes were "frozen" or
|
||||
// fixed, they will be unfrozen.)
|
||||
NodeAnchor fixNodeSelectedCandidate(size_t location,
|
||||
const std::string& value);
|
||||
|
||||
// Similar to fixNodeSelectedCandidate, but instead of "freezing" the node, only
|
||||
// boost the unigram that represents the value with an overriding score. This
|
||||
// has the same side effect as fixNodeSelectedCandidate, which is that all other
|
||||
// overlapping nodes will be reset to their initial state.
|
||||
void overrideNodeScoreForSelectedCandidate(size_t location, const string& value, float overridingScore);
|
||||
// Similar to fixNodeSelectedCandidate, but instead of "freezing" the node,
|
||||
// only boost the unigram that represents the value with an overriding score.
|
||||
// This has the same side effect as fixNodeSelectedCandidate, which is that
|
||||
// all other overlapping nodes will be reset to their initial state.
|
||||
void overrideNodeScoreForSelectedCandidate(size_t location,
|
||||
const std::string& value,
|
||||
float overridingScore);
|
||||
|
||||
const string dumpDOT();
|
||||
std::string dumpDOT();
|
||||
|
||||
protected:
|
||||
vector<Span> m_spans;
|
||||
};
|
||||
std::vector<Span> m_spans;
|
||||
};
|
||||
|
||||
inline void Grid::clear()
|
||||
{
|
||||
m_spans.clear();
|
||||
}
|
||||
inline void Grid::clear() { m_spans.clear(); }
|
||||
|
||||
inline void Grid::insertNode(const Node& inNode, size_t inLocation, size_t inSpanningLength)
|
||||
{
|
||||
if (inLocation >= m_spans.size()) {
|
||||
size_t diff = inLocation - m_spans.size() + 1;
|
||||
inline void Grid::insertNode(const Node& node, size_t location,
|
||||
size_t spanningLength) {
|
||||
if (location >= m_spans.size()) {
|
||||
size_t diff = location - m_spans.size() + 1;
|
||||
|
||||
for (size_t i = 0 ; i < diff ; i++) {
|
||||
for (size_t i = 0; i < diff; i++) {
|
||||
m_spans.push_back(Span());
|
||||
}
|
||||
}
|
||||
|
||||
m_spans[inLocation].insertNodeOfLength(inNode, inSpanningLength);
|
||||
}
|
||||
m_spans[location].insertNodeOfLength(node, spanningLength);
|
||||
}
|
||||
|
||||
inline bool Grid::hasNodeAtLocationSpanningLengthMatchingKey(size_t inLocation, size_t inSpanningLength, const string& inKey)
|
||||
{
|
||||
if (inLocation > m_spans.size()) {
|
||||
inline bool Grid::hasNodeAtLocationSpanningLengthMatchingKey(
|
||||
size_t location, size_t spanningLength, const std::string& key) {
|
||||
if (location > m_spans.size()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const Node *n = m_spans[inLocation].nodeOfLength(inSpanningLength);
|
||||
const Node* n = m_spans[location].nodeOfLength(spanningLength);
|
||||
if (!n) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return inKey == n->key();
|
||||
}
|
||||
return key == n->key();
|
||||
}
|
||||
|
||||
inline void Grid::expandGridByOneAtLocation(size_t inLocation)
|
||||
{
|
||||
if (!inLocation || inLocation == m_spans.size()) {
|
||||
m_spans.insert(m_spans.begin() + inLocation, Span());
|
||||
}
|
||||
else {
|
||||
m_spans.insert(m_spans.begin() + inLocation, Span());
|
||||
for (size_t i = 0 ; i < inLocation ; i++) {
|
||||
inline void Grid::expandGridByOneAtLocation(size_t location) {
|
||||
if (!location || location == m_spans.size()) {
|
||||
m_spans.insert(m_spans.begin() + location, Span());
|
||||
} else {
|
||||
m_spans.insert(m_spans.begin() + location, Span());
|
||||
for (size_t i = 0; i < location; i++) {
|
||||
// zaps overlapping spans
|
||||
m_spans[i].removeNodeOfLengthGreaterThan(inLocation - i);
|
||||
}
|
||||
m_spans[i].removeNodeOfLengthGreaterThan(location - i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inline void Grid::shrinkGridByOneAtLocation(size_t inLocation)
|
||||
{
|
||||
if (inLocation >= m_spans.size()) {
|
||||
inline void Grid::shrinkGridByOneAtLocation(size_t location) {
|
||||
if (location >= m_spans.size()) {
|
||||
return;
|
||||
}
|
||||
|
||||
m_spans.erase(m_spans.begin() + inLocation);
|
||||
for (size_t i = 0 ; i < inLocation ; i++) {
|
||||
m_spans.erase(m_spans.begin() + location);
|
||||
for (size_t i = 0; i < location; i++) {
|
||||
// zaps overlapping spans
|
||||
m_spans[i].removeNodeOfLengthGreaterThan(inLocation - i);
|
||||
}
|
||||
m_spans[i].removeNodeOfLengthGreaterThan(location - i);
|
||||
}
|
||||
}
|
||||
|
||||
inline size_t Grid::width() const
|
||||
{
|
||||
return m_spans.size();
|
||||
}
|
||||
inline size_t Grid::width() const { return m_spans.size(); }
|
||||
|
||||
inline vector<NodeAnchor> Grid::nodesEndingAt(size_t inLocation)
|
||||
{
|
||||
vector<NodeAnchor> result;
|
||||
inline std::vector<NodeAnchor> Grid::nodesEndingAt(size_t location) {
|
||||
std::vector<NodeAnchor> result;
|
||||
|
||||
if (m_spans.size() && inLocation <= m_spans.size()) {
|
||||
for (size_t i = 0 ; i < inLocation ; i++) {
|
||||
if (m_spans.size() && location <= m_spans.size()) {
|
||||
for (size_t i = 0; i < location; i++) {
|
||||
Span& span = m_spans[i];
|
||||
if (i + span.maximumLength() >= inLocation) {
|
||||
Node *np = span.nodeOfLength(inLocation - i);
|
||||
if (i + span.maximumLength() >= location) {
|
||||
Node* np = span.nodeOfLength(location - i);
|
||||
if (np) {
|
||||
NodeAnchor na;
|
||||
na.node = np;
|
||||
na.location = i;
|
||||
na.spanningLength = inLocation - i;
|
||||
na.spanningLength = location - i;
|
||||
|
||||
result.push_back(na);
|
||||
}
|
||||
|
|
@ -152,30 +151,27 @@ namespace Formosa {
|
|||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
inline vector<NodeAnchor> Grid::nodesCrossingOrEndingAt(size_t inLocation)
|
||||
{
|
||||
vector<NodeAnchor> result;
|
||||
inline std::vector<NodeAnchor> Grid::nodesCrossingOrEndingAt(size_t location) {
|
||||
std::vector<NodeAnchor> result;
|
||||
|
||||
if (m_spans.size() && inLocation <= m_spans.size()) {
|
||||
for (size_t i = 0 ; i < inLocation ; i++) {
|
||||
if (m_spans.size() && location <= m_spans.size()) {
|
||||
for (size_t i = 0; i < location; i++) {
|
||||
Span& span = m_spans[i];
|
||||
|
||||
if (i + span.maximumLength() >= inLocation) {
|
||||
|
||||
for (size_t j = 1, m = span.maximumLength(); j <= m ; j++) {
|
||||
|
||||
if (i + j < inLocation) {
|
||||
if (i + span.maximumLength() >= location) {
|
||||
for (size_t j = 1, m = span.maximumLength(); j <= m; j++) {
|
||||
if (i + j < location) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Node *np = span.nodeOfLength(j);
|
||||
Node* np = span.nodeOfLength(j);
|
||||
if (np) {
|
||||
NodeAnchor na;
|
||||
na.node = np;
|
||||
na.location = i;
|
||||
na.spanningLength = inLocation - i;
|
||||
na.spanningLength = location - i;
|
||||
|
||||
result.push_back(na);
|
||||
}
|
||||
|
|
@ -185,12 +181,13 @@ namespace Formosa {
|
|||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
// For nodes found at the location, fix their currently-selected candidate using the supplied string value.
|
||||
inline NodeAnchor Grid::fixNodeSelectedCandidate(size_t location, const string& value)
|
||||
{
|
||||
vector<NodeAnchor> nodes = nodesCrossingOrEndingAt(location);
|
||||
// For nodes found at the location, fix their currently-selected candidate using
|
||||
// the supplied string value.
|
||||
inline NodeAnchor Grid::fixNodeSelectedCandidate(size_t location,
|
||||
const std::string& value) {
|
||||
std::vector<NodeAnchor> nodes = nodesCrossingOrEndingAt(location);
|
||||
NodeAnchor node;
|
||||
for (auto nodeAnchor : nodes) {
|
||||
auto candidates = nodeAnchor.node->candidates();
|
||||
|
|
@ -202,16 +199,16 @@ namespace Formosa {
|
|||
if (candidates[i].value == value) {
|
||||
const_cast<Node*>(nodeAnchor.node)->selectCandidateAtIndex(i);
|
||||
node = nodeAnchor;
|
||||
break;;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return node;
|
||||
}
|
||||
}
|
||||
|
||||
inline void Grid::overrideNodeScoreForSelectedCandidate(size_t location, const string& value, float overridingScore)
|
||||
{
|
||||
vector<NodeAnchor> nodes = nodesCrossingOrEndingAt(location);
|
||||
inline void Grid::overrideNodeScoreForSelectedCandidate(
|
||||
size_t location, const std::string& value, float overridingScore) {
|
||||
std::vector<NodeAnchor> nodes = nodesCrossingOrEndingAt(location);
|
||||
for (auto nodeAnchor : nodes) {
|
||||
auto candidates = nodeAnchor.node->candidates();
|
||||
|
||||
|
|
@ -220,53 +217,15 @@ namespace Formosa {
|
|||
|
||||
for (size_t i = 0, c = candidates.size(); i < c; ++i) {
|
||||
if (candidates[i].value == value) {
|
||||
const_cast<Node*>(nodeAnchor.node)->selectFloatingCandidateAtIndex(i, overridingScore);
|
||||
const_cast<Node*>(nodeAnchor.node)
|
||||
->selectFloatingCandidateAtIndex(i, overridingScore);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inline const string Grid::dumpDOT()
|
||||
{
|
||||
stringstream sst;
|
||||
sst << "digraph {" << endl;
|
||||
sst << "graph [ rankdir=LR ];" << endl;
|
||||
sst << "BOS;" << endl;
|
||||
|
||||
for (size_t p = 0 ; p < m_spans.size() ; p++) {
|
||||
Span& span = m_spans[p];
|
||||
for (size_t ni = 0 ; ni <= span.maximumLength() ; ni++) {
|
||||
Node* np = span.nodeOfLength(ni);
|
||||
if (np) {
|
||||
if (!p) {
|
||||
sst << "BOS -> " << np->currentKeyValue().value << ";" << endl;
|
||||
}
|
||||
|
||||
sst << np->currentKeyValue().value << ";" << endl;
|
||||
|
||||
if (p + ni < m_spans.size()) {
|
||||
Span& dstSpan = m_spans[p+ni];
|
||||
for (size_t q = 0 ; q <= dstSpan.maximumLength() ; q++) {
|
||||
Node *dn = dstSpan.nodeOfLength(q);
|
||||
if (dn) {
|
||||
sst << np->currentKeyValue().value << " -> " << dn->currentKeyValue().value << ";" << endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (p + ni == m_spans.size()) {
|
||||
sst << np->currentKeyValue().value << " -> " << "EOS;" << endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sst << "EOS;" << endl;
|
||||
sst << "}";
|
||||
return sst.str();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Gramambular
|
||||
} // namespace Formosa
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -25,47 +25,43 @@
|
|||
// OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
|
||||
#ifndef KeyValuePair_h
|
||||
#define KeyValuePair_h
|
||||
#ifndef KEYVALUEPAIR_H_
|
||||
#define KEYVALUEPAIR_H_
|
||||
|
||||
#include <ostream>
|
||||
#include <string>
|
||||
|
||||
namespace Formosa {
|
||||
namespace Gramambular {
|
||||
using namespace std;
|
||||
namespace Gramambular {
|
||||
|
||||
class KeyValuePair {
|
||||
class KeyValuePair {
|
||||
public:
|
||||
string key;
|
||||
string value;
|
||||
std::string key;
|
||||
std::string value;
|
||||
|
||||
bool operator==(const KeyValuePair& inAnother) const;
|
||||
bool operator<(const KeyValuePair& inAnother) const;
|
||||
};
|
||||
bool operator==(const KeyValuePair& another) const;
|
||||
bool operator<(const KeyValuePair& another) const;
|
||||
};
|
||||
|
||||
inline ostream& operator<<(ostream& inStream, const KeyValuePair& inPair)
|
||||
{
|
||||
inStream << "(" << inPair.key << "," << inPair.value << ")";
|
||||
return inStream;
|
||||
}
|
||||
|
||||
inline bool KeyValuePair::operator==(const KeyValuePair& inAnother) const
|
||||
{
|
||||
return key == inAnother.key && value == inAnother.value;
|
||||
}
|
||||
|
||||
inline bool KeyValuePair::operator<(const KeyValuePair& inAnother) const
|
||||
{
|
||||
if (key < inAnother.key) {
|
||||
return true;
|
||||
}
|
||||
else if (key == inAnother.key) {
|
||||
return value < inAnother.value;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
inline std::ostream& operator<<(std::ostream& stream,
|
||||
const KeyValuePair& pair) {
|
||||
stream << "(" << pair.key << "," << pair.value << ")";
|
||||
return stream;
|
||||
}
|
||||
|
||||
inline bool KeyValuePair::operator==(const KeyValuePair& another) const {
|
||||
return key == another.key && value == another.value;
|
||||
}
|
||||
|
||||
inline bool KeyValuePair::operator<(const KeyValuePair& another) const {
|
||||
if (key < another.key) {
|
||||
return true;
|
||||
} else if (key == another.key) {
|
||||
return value < another.value;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
} // namespace Gramambular
|
||||
} // namespace Formosa
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -25,28 +25,28 @@
|
|||
// OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
|
||||
#ifndef LanguageModel_h
|
||||
#define LanguageModel_h
|
||||
#ifndef LANGUAGEMODEL_H_
|
||||
#define LANGUAGEMODEL_H_
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "Bigram.h"
|
||||
#include "Unigram.h"
|
||||
|
||||
namespace Formosa {
|
||||
namespace Gramambular {
|
||||
namespace Gramambular {
|
||||
|
||||
using namespace std;
|
||||
|
||||
class LanguageModel {
|
||||
class LanguageModel {
|
||||
public:
|
||||
virtual ~LanguageModel() {}
|
||||
|
||||
virtual const vector<Bigram> bigramsForKeys(const string &preceedingKey, const string& key) = 0;
|
||||
virtual const vector<Unigram> unigramsForKey(const string &key) = 0;
|
||||
virtual bool hasUnigramsForKey(const string& key) = 0;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
virtual const std::vector<Bigram> bigramsForKeys(
|
||||
const std::string& preceedingKey, const std::string& key) = 0;
|
||||
virtual const std::vector<Unigram> unigramsForKey(const std::string& key) = 0;
|
||||
virtual bool hasUnigramsForKey(const std::string& key) = 0;
|
||||
};
|
||||
} // namespace Gramambular
|
||||
} // namespace Formosa
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -25,75 +25,75 @@
|
|||
// OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
|
||||
#ifndef Node_h
|
||||
#define Node_h
|
||||
#ifndef NODE_H_
|
||||
#define NODE_H_
|
||||
|
||||
#include <limits>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "LanguageModel.h"
|
||||
|
||||
namespace Formosa {
|
||||
namespace Gramambular {
|
||||
using namespace std;
|
||||
namespace Gramambular {
|
||||
|
||||
class Node {
|
||||
class Node {
|
||||
public:
|
||||
Node();
|
||||
Node(const string& inKey, const vector<Unigram>& inUnigrams, const vector<Bigram>& inBigrams);
|
||||
Node(const std::string& key, const std::vector<Unigram>& unigrams,
|
||||
const std::vector<Bigram>& bigrams);
|
||||
|
||||
void primeNodeWithPreceedingKeyValues(const vector<KeyValuePair>& inKeyValues);
|
||||
void primeNodeWithPreceedingKeyValues(
|
||||
const std::vector<KeyValuePair>& keyValues);
|
||||
|
||||
bool isCandidateFixed() const;
|
||||
const vector<KeyValuePair>& candidates() const;
|
||||
void selectCandidateAtIndex(size_t inIndex = 0, bool inFix = true);
|
||||
const std::vector<KeyValuePair>& candidates() const;
|
||||
void selectCandidateAtIndex(size_t index = 0, bool fix = true);
|
||||
void resetCandidate();
|
||||
void selectFloatingCandidateAtIndex(size_t index, double score);
|
||||
|
||||
const string& key() const;
|
||||
const std::string& key() const;
|
||||
double score() const;
|
||||
double scoreForCandidate(string &candidate) const;
|
||||
double scoreForCandidate(const std::string& candidate) const;
|
||||
const KeyValuePair currentKeyValue() const;
|
||||
double highestUnigramScore() const;
|
||||
|
||||
protected:
|
||||
const LanguageModel* m_LM;
|
||||
|
||||
string m_key;
|
||||
std::string m_key;
|
||||
double m_score;
|
||||
|
||||
vector<Unigram> m_unigrams;
|
||||
vector<KeyValuePair> m_candidates;
|
||||
map<string, size_t> m_valueUnigramIndexMap;
|
||||
map<KeyValuePair, vector<Bigram> > m_preceedingGramBigramMap;
|
||||
std::vector<Unigram> m_unigrams;
|
||||
std::vector<KeyValuePair> m_candidates;
|
||||
std::map<std::string, size_t> m_valueUnigramIndexMap;
|
||||
std::map<KeyValuePair, std::vector<Bigram> > m_preceedingGramBigramMap;
|
||||
|
||||
bool m_candidateFixed;
|
||||
size_t m_selectedUnigramIndex;
|
||||
|
||||
friend ostream& operator<<(ostream& inStream, const Node& inNode);
|
||||
};
|
||||
friend std::ostream& operator<<(std::ostream& stream, const Node& node);
|
||||
};
|
||||
|
||||
inline ostream& operator<<(ostream& inStream, const Node& inNode)
|
||||
{
|
||||
inStream << "(node,key:" << inNode.m_key << ",fixed:" << (inNode.m_candidateFixed ? "true" : "false")
|
||||
<< ",selected:" << inNode.m_selectedUnigramIndex
|
||||
<< "," << inNode.m_unigrams << ")";
|
||||
return inStream;
|
||||
}
|
||||
inline std::ostream& operator<<(std::ostream& stream, const Node& node) {
|
||||
stream << "(node,key:" << node.m_key
|
||||
<< ",fixed:" << (node.m_candidateFixed ? "true" : "false")
|
||||
<< ",selected:" << node.m_selectedUnigramIndex << ","
|
||||
<< node.m_unigrams << ")";
|
||||
return stream;
|
||||
}
|
||||
|
||||
inline Node::Node()
|
||||
: m_candidateFixed(false)
|
||||
, m_selectedUnigramIndex(0)
|
||||
, m_score(0.0)
|
||||
{
|
||||
}
|
||||
inline Node::Node()
|
||||
: m_candidateFixed(false), m_selectedUnigramIndex(0), m_score(0.0) {}
|
||||
|
||||
inline Node::Node(const string& inKey, const vector<Unigram>& inUnigrams, const vector<Bigram>& inBigrams)
|
||||
: m_key(inKey)
|
||||
, m_unigrams(inUnigrams)
|
||||
, m_candidateFixed(false)
|
||||
, m_selectedUnigramIndex(0)
|
||||
, m_score(0.0)
|
||||
{
|
||||
inline Node::Node(const std::string& key, const std::vector<Unigram>& unigrams,
|
||||
const std::vector<Bigram>& bigrams)
|
||||
: m_key(key),
|
||||
m_unigrams(unigrams),
|
||||
m_candidateFixed(false),
|
||||
m_selectedUnigramIndex(0),
|
||||
m_score(0.0) {
|
||||
stable_sort(m_unigrams.begin(), m_unigrams.end(), Unigram::ScoreCompare);
|
||||
|
||||
if (m_unigrams.size()) {
|
||||
|
|
@ -101,33 +101,39 @@ namespace Formosa {
|
|||
}
|
||||
|
||||
size_t i = 0;
|
||||
for (vector<Unigram>::const_iterator ui = m_unigrams.begin() ; ui != m_unigrams.end() ; ++ui) {
|
||||
for (std::vector<Unigram>::const_iterator ui = m_unigrams.begin();
|
||||
ui != m_unigrams.end(); ++ui) {
|
||||
m_valueUnigramIndexMap[(*ui).keyValue.value] = i;
|
||||
i++;
|
||||
|
||||
m_candidates.push_back((*ui).keyValue);
|
||||
}
|
||||
|
||||
for (vector<Bigram>::const_iterator bi = inBigrams.begin() ; bi != inBigrams.end() ; ++bi) {
|
||||
for (std::vector<Bigram>::const_iterator bi = bigrams.begin();
|
||||
bi != bigrams.end(); ++bi) {
|
||||
m_preceedingGramBigramMap[(*bi).preceedingKeyValue].push_back(*bi);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inline void Node::primeNodeWithPreceedingKeyValues(const vector<KeyValuePair>& inKeyValues)
|
||||
{
|
||||
inline void Node::primeNodeWithPreceedingKeyValues(
|
||||
const std::vector<KeyValuePair>& keyValues) {
|
||||
size_t newIndex = m_selectedUnigramIndex;
|
||||
double max = m_score;
|
||||
|
||||
if (!isCandidateFixed()) {
|
||||
for (vector<KeyValuePair>::const_iterator kvi = inKeyValues.begin() ; kvi != inKeyValues.end() ; ++kvi) {
|
||||
map<KeyValuePair, vector<Bigram> >::const_iterator f = m_preceedingGramBigramMap.find(*kvi);
|
||||
for (std::vector<KeyValuePair>::const_iterator kvi = keyValues.begin();
|
||||
kvi != keyValues.end(); ++kvi) {
|
||||
std::map<KeyValuePair, std::vector<Bigram> >::const_iterator f =
|
||||
m_preceedingGramBigramMap.find(*kvi);
|
||||
if (f != m_preceedingGramBigramMap.end()) {
|
||||
const vector<Bigram>& bigrams = (*f).second;
|
||||
const std::vector<Bigram>& bigrams = (*f).second;
|
||||
|
||||
for (vector<Bigram>::const_iterator bi = bigrams.begin() ; bi != bigrams.end() ; ++bi) {
|
||||
for (std::vector<Bigram>::const_iterator bi = bigrams.begin();
|
||||
bi != bigrams.end(); ++bi) {
|
||||
const Bigram& bigram = *bi;
|
||||
if (bigram.score > max) {
|
||||
map<string, size_t>::const_iterator uf = m_valueUnigramIndexMap.find((*bi).keyValue.value);
|
||||
std::map<std::string, size_t>::const_iterator uf =
|
||||
m_valueUnigramIndexMap.find((*bi).keyValue.value);
|
||||
if (uf != m_valueUnigramIndexMap.end()) {
|
||||
newIndex = (*uf).second;
|
||||
max = bigram.score;
|
||||
|
|
@ -145,41 +151,34 @@ namespace Formosa {
|
|||
if (newIndex != m_selectedUnigramIndex) {
|
||||
m_selectedUnigramIndex = newIndex;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inline bool Node::isCandidateFixed() const
|
||||
{
|
||||
return m_candidateFixed;
|
||||
}
|
||||
inline bool Node::isCandidateFixed() const { return m_candidateFixed; }
|
||||
|
||||
inline const vector<KeyValuePair>& Node::candidates() const
|
||||
{
|
||||
inline const std::vector<KeyValuePair>& Node::candidates() const {
|
||||
return m_candidates;
|
||||
}
|
||||
}
|
||||
|
||||
inline void Node::selectCandidateAtIndex(size_t inIndex, bool inFix)
|
||||
{
|
||||
if (inIndex >= m_unigrams.size()) {
|
||||
inline void Node::selectCandidateAtIndex(size_t index, bool fix) {
|
||||
if (index >= m_unigrams.size()) {
|
||||
m_selectedUnigramIndex = 0;
|
||||
}
|
||||
else {
|
||||
m_selectedUnigramIndex = inIndex;
|
||||
} else {
|
||||
m_selectedUnigramIndex = index;
|
||||
}
|
||||
|
||||
m_candidateFixed = inFix;
|
||||
m_candidateFixed = fix;
|
||||
m_score = 99;
|
||||
}
|
||||
}
|
||||
|
||||
inline void Node::resetCandidate()
|
||||
{
|
||||
inline void Node::resetCandidate() {
|
||||
m_selectedUnigramIndex = 0;
|
||||
m_candidateFixed = 0;
|
||||
if (m_unigrams.size()) {
|
||||
m_score = m_unigrams[0].score;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inline void Node::selectFloatingCandidateAtIndex(size_t index, double score) {
|
||||
inline void Node::selectFloatingCandidateAtIndex(size_t index, double score) {
|
||||
if (index >= m_unigrams.size()) {
|
||||
m_selectedUnigramIndex = 0;
|
||||
} else {
|
||||
|
|
@ -187,45 +186,36 @@ namespace Formosa {
|
|||
}
|
||||
m_candidateFixed = false;
|
||||
m_score = score;
|
||||
}
|
||||
}
|
||||
|
||||
inline const string& Node::key() const
|
||||
{
|
||||
return m_key;
|
||||
}
|
||||
inline const std::string& Node::key() const { return m_key; }
|
||||
|
||||
inline double Node::score() const
|
||||
{
|
||||
return m_score;
|
||||
}
|
||||
inline double Node::score() const { return m_score; }
|
||||
|
||||
inline double Node::scoreForCandidate(string &candidate) const
|
||||
{
|
||||
inline double Node::scoreForCandidate(const std::string& candidate) const {
|
||||
for (auto unigram : m_unigrams) {
|
||||
if (unigram.keyValue.value == candidate) {
|
||||
return unigram.score;
|
||||
}
|
||||
}
|
||||
return 0.0;
|
||||
}
|
||||
}
|
||||
|
||||
inline double Node::highestUnigramScore() const {
|
||||
inline double Node::highestUnigramScore() const {
|
||||
if (m_unigrams.empty()) {
|
||||
return 0.0;
|
||||
}
|
||||
return m_unigrams[0].score;
|
||||
}
|
||||
|
||||
inline const KeyValuePair Node::currentKeyValue() const
|
||||
{
|
||||
if(m_selectedUnigramIndex >= m_unigrams.size()) {
|
||||
return KeyValuePair();
|
||||
}
|
||||
else {
|
||||
return m_candidates[m_selectedUnigramIndex];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inline const KeyValuePair Node::currentKeyValue() const {
|
||||
if (m_selectedUnigramIndex >= m_unigrams.size()) {
|
||||
return KeyValuePair();
|
||||
} else {
|
||||
return m_candidates[m_selectedUnigramIndex];
|
||||
}
|
||||
}
|
||||
} // namespace Gramambular
|
||||
} // namespace Formosa
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -25,55 +25,48 @@
|
|||
// OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
|
||||
#ifndef NodeAnchor_h
|
||||
#define NodeAnchor_h
|
||||
#ifndef NODEANCHOR_H_
|
||||
#define NODEANCHOR_H_
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "Node.h"
|
||||
|
||||
namespace Formosa {
|
||||
namespace Gramambular {
|
||||
class NodeAnchor {
|
||||
public:
|
||||
NodeAnchor();
|
||||
const Node *node;
|
||||
size_t location;
|
||||
size_t spanningLength;
|
||||
double accumulatedScore;
|
||||
};
|
||||
namespace Gramambular {
|
||||
|
||||
inline NodeAnchor::NodeAnchor()
|
||||
: node(0)
|
||||
, location(0)
|
||||
, spanningLength(0)
|
||||
, accumulatedScore(0.0)
|
||||
{
|
||||
}
|
||||
struct NodeAnchor {
|
||||
const Node* node = nullptr;
|
||||
size_t location = 0;
|
||||
size_t spanningLength = 0;
|
||||
double accumulatedScore = 0.0;
|
||||
};
|
||||
|
||||
inline ostream& operator<<(ostream& inStream, const NodeAnchor& inAnchor)
|
||||
{
|
||||
inStream << "{@(" << inAnchor.location << "," << inAnchor.spanningLength << "),";
|
||||
if (inAnchor.node) {
|
||||
inStream << *(inAnchor.node);
|
||||
}
|
||||
else {
|
||||
inStream << "null";
|
||||
}
|
||||
inStream << "}";
|
||||
return inStream;
|
||||
}
|
||||
|
||||
inline ostream& operator<<(ostream& inStream, const vector<NodeAnchor>& inAnchor)
|
||||
{
|
||||
for (vector<NodeAnchor>::const_iterator i = inAnchor.begin() ; i != inAnchor.end() ; ++i) {
|
||||
inStream << *i;
|
||||
if (i + 1 != inAnchor.end()) {
|
||||
inStream << "<-";
|
||||
}
|
||||
}
|
||||
|
||||
return inStream;
|
||||
}
|
||||
inline std::ostream& operator<<(std::ostream& stream,
|
||||
const NodeAnchor& anchor) {
|
||||
stream << "{@(" << anchor.location << "," << anchor.spanningLength << "),";
|
||||
if (anchor.node) {
|
||||
stream << *(anchor.node);
|
||||
} else {
|
||||
stream << "null";
|
||||
}
|
||||
stream << "}";
|
||||
return stream;
|
||||
}
|
||||
|
||||
inline std::ostream& operator<<(std::ostream& stream,
|
||||
const std::vector<NodeAnchor>& anchor) {
|
||||
for (std::vector<NodeAnchor>::const_iterator i = anchor.begin();
|
||||
i != anchor.end(); ++i) {
|
||||
stream << *i;
|
||||
if (i + 1 != anchor.end()) {
|
||||
stream << "<-";
|
||||
}
|
||||
}
|
||||
|
||||
return stream;
|
||||
}
|
||||
} // namespace Gramambular
|
||||
} // namespace Formosa
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -25,88 +25,77 @@
|
|||
// OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
|
||||
#ifndef Span_h
|
||||
#define Span_h
|
||||
#ifndef SPAN_H_
|
||||
#define SPAN_H_
|
||||
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <sstream>
|
||||
|
||||
#include "Node.h"
|
||||
|
||||
namespace Formosa {
|
||||
namespace Gramambular {
|
||||
class Span {
|
||||
namespace Gramambular {
|
||||
class Span {
|
||||
public:
|
||||
Span();
|
||||
|
||||
void clear();
|
||||
void insertNodeOfLength(const Node& inNode, size_t inLength);
|
||||
void removeNodeOfLengthGreaterThan(size_t inLength);
|
||||
void insertNodeOfLength(const Node& node, size_t length);
|
||||
void removeNodeOfLengthGreaterThan(size_t length);
|
||||
|
||||
Node* nodeOfLength(size_t inLength);
|
||||
Node* nodeOfLength(size_t length);
|
||||
size_t maximumLength() const;
|
||||
|
||||
protected:
|
||||
map<size_t, Node> m_lengthNodeMap;
|
||||
size_t m_maximumLength;
|
||||
};
|
||||
std::map<size_t, Node> m_lengthNodeMap;
|
||||
size_t m_maximumLength = 0;
|
||||
};
|
||||
|
||||
inline Span::Span()
|
||||
: m_maximumLength(0)
|
||||
{
|
||||
}
|
||||
|
||||
inline void Span::clear()
|
||||
{
|
||||
inline void Span::clear() {
|
||||
m_lengthNodeMap.clear();
|
||||
m_maximumLength = 0;
|
||||
}
|
||||
}
|
||||
|
||||
inline void Span::insertNodeOfLength(const Node& inNode, size_t inLength)
|
||||
{
|
||||
m_lengthNodeMap[inLength] = inNode;
|
||||
if (inLength > m_maximumLength) {
|
||||
m_maximumLength = inLength;
|
||||
}
|
||||
inline void Span::insertNodeOfLength(const Node& node, size_t length) {
|
||||
m_lengthNodeMap[length] = node;
|
||||
if (length > m_maximumLength) {
|
||||
m_maximumLength = length;
|
||||
}
|
||||
}
|
||||
|
||||
inline void Span::removeNodeOfLengthGreaterThan(size_t inLength)
|
||||
{
|
||||
if (inLength > m_maximumLength) {
|
||||
inline void Span::removeNodeOfLengthGreaterThan(size_t length) {
|
||||
if (length > m_maximumLength) {
|
||||
return;
|
||||
}
|
||||
|
||||
size_t max = 0;
|
||||
set<size_t> removeSet;
|
||||
for (map<size_t, Node>::iterator i = m_lengthNodeMap.begin(), e = m_lengthNodeMap.end() ; i != e ; ++i) {
|
||||
if ((*i).first > inLength) {
|
||||
std::set<size_t> removeSet;
|
||||
for (std::map<size_t, Node>::iterator i = m_lengthNodeMap.begin(),
|
||||
e = m_lengthNodeMap.end();
|
||||
i != e; ++i) {
|
||||
if ((*i).first > length) {
|
||||
removeSet.insert((*i).first);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
if ((*i).first > max) {
|
||||
max = (*i).first;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (set<size_t>::iterator i = removeSet.begin(), e = removeSet.end(); i != e; ++i) {
|
||||
for (std::set<size_t>::iterator i = removeSet.begin(), e = removeSet.end();
|
||||
i != e; ++i) {
|
||||
m_lengthNodeMap.erase(*i);
|
||||
}
|
||||
|
||||
m_maximumLength = max;
|
||||
}
|
||||
|
||||
inline Node* Span::nodeOfLength(size_t inLength)
|
||||
{
|
||||
map<size_t, Node>::iterator f = m_lengthNodeMap.find(inLength);
|
||||
return f == m_lengthNodeMap.end() ? 0 : &(*f).second;
|
||||
}
|
||||
|
||||
inline size_t Span::maximumLength() const
|
||||
{
|
||||
return m_maximumLength;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inline Node* Span::nodeOfLength(size_t length) {
|
||||
std::map<size_t, Node>::iterator f = m_lengthNodeMap.find(length);
|
||||
return f == m_lengthNodeMap.end() ? 0 : &(*f).second;
|
||||
}
|
||||
|
||||
inline size_t Span::maximumLength() const { return m_maximumLength; }
|
||||
} // namespace Gramambular
|
||||
} // namespace Formosa
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -25,80 +25,75 @@
|
|||
// OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
|
||||
#ifndef Unigram_h
|
||||
#define Unigram_h
|
||||
#ifndef UNIGRAM_H_
|
||||
#define UNIGRAM_H_
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "KeyValuePair.h"
|
||||
|
||||
namespace Formosa {
|
||||
namespace Gramambular {
|
||||
class Unigram {
|
||||
namespace Gramambular {
|
||||
|
||||
class Unigram {
|
||||
public:
|
||||
Unigram();
|
||||
|
||||
KeyValuePair keyValue;
|
||||
double score;
|
||||
|
||||
bool operator==(const Unigram& inAnother) const;
|
||||
bool operator<(const Unigram& inAnother) const;
|
||||
bool operator==(const Unigram& another) const;
|
||||
bool operator<(const Unigram& another) const;
|
||||
|
||||
static bool ScoreCompare(const Unigram& a, const Unigram& b);
|
||||
};
|
||||
};
|
||||
|
||||
inline ostream& operator<<(ostream& inStream, const Unigram& inGram)
|
||||
{
|
||||
streamsize p = inStream.precision();
|
||||
inStream.precision(6);
|
||||
inStream << "(" << inGram.keyValue << "," << inGram.score << ")";
|
||||
inStream.precision(p);
|
||||
return inStream;
|
||||
}
|
||||
inline std::ostream& operator<<(std::ostream& stream, const Unigram& gram) {
|
||||
std::streamsize p = stream.precision();
|
||||
stream.precision(6);
|
||||
stream << "(" << gram.keyValue << "," << gram.score << ")";
|
||||
stream.precision(p);
|
||||
return stream;
|
||||
}
|
||||
|
||||
inline ostream& operator<<(ostream& inStream, const vector<Unigram>& inGrams)
|
||||
{
|
||||
inStream << "[" << inGrams.size() << "]=>{";
|
||||
inline std::ostream& operator<<(std::ostream& stream,
|
||||
const std::vector<Unigram>& grams) {
|
||||
stream << "[" << grams.size() << "]=>{";
|
||||
|
||||
size_t index = 0;
|
||||
|
||||
for (vector<Unigram>::const_iterator gi = inGrams.begin() ; gi != inGrams.end() ; ++gi, ++index) {
|
||||
inStream << index << "=>";
|
||||
inStream << *gi;
|
||||
if (gi + 1 != inGrams.end()) {
|
||||
inStream << ",";
|
||||
for (std::vector<Unigram>::const_iterator gi = grams.begin();
|
||||
gi != grams.end(); ++gi, ++index) {
|
||||
stream << index << "=>";
|
||||
stream << *gi;
|
||||
if (gi + 1 != grams.end()) {
|
||||
stream << ",";
|
||||
}
|
||||
}
|
||||
|
||||
inStream << "}";
|
||||
return inStream;
|
||||
}
|
||||
|
||||
inline Unigram::Unigram()
|
||||
: score(0.0)
|
||||
{
|
||||
}
|
||||
|
||||
inline bool Unigram::operator==(const Unigram& inAnother) const
|
||||
{
|
||||
return keyValue == inAnother.keyValue && score == inAnother.score;
|
||||
}
|
||||
|
||||
inline bool Unigram::operator<(const Unigram& inAnother) const
|
||||
{
|
||||
if (keyValue < inAnother.keyValue) {
|
||||
return true;
|
||||
}
|
||||
else if (keyValue == inAnother.keyValue) {
|
||||
return score < inAnother.score;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
inline bool Unigram::ScoreCompare(const Unigram& a, const Unigram& b)
|
||||
{
|
||||
return a.score > b.score;
|
||||
}
|
||||
}
|
||||
stream << "}";
|
||||
return stream;
|
||||
}
|
||||
|
||||
inline Unigram::Unigram() : score(0.0) {}
|
||||
|
||||
inline bool Unigram::operator==(const Unigram& another) const {
|
||||
return keyValue == another.keyValue && score == another.score;
|
||||
}
|
||||
|
||||
inline bool Unigram::operator<(const Unigram& another) const {
|
||||
if (keyValue < another.keyValue) {
|
||||
return true;
|
||||
} else if (keyValue == another.keyValue) {
|
||||
return score < another.score;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
inline bool Unigram::ScoreCompare(const Unigram& a, const Unigram& b) {
|
||||
return a.score > b.score;
|
||||
}
|
||||
} // namespace Gramambular
|
||||
} // namespace Formosa
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -25,67 +25,69 @@
|
|||
// OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
|
||||
#ifndef Walker_h
|
||||
#define Walker_h
|
||||
#ifndef WALKER_H_
|
||||
#define WALKER_H_
|
||||
|
||||
#include <algorithm>
|
||||
#include <vector>
|
||||
|
||||
#include "Grid.h"
|
||||
|
||||
namespace Formosa {
|
||||
namespace Gramambular {
|
||||
using namespace std;
|
||||
namespace Gramambular {
|
||||
|
||||
class Walker {
|
||||
class Walker {
|
||||
public:
|
||||
Walker(Grid* inGrid);
|
||||
const vector<NodeAnchor> reverseWalk(size_t inLocation, double inAccumulatedScore = 0.0);
|
||||
explicit Walker(Grid* inGrid);
|
||||
const std::vector<NodeAnchor> reverseWalk(size_t location,
|
||||
double accumulatedScore = 0.0);
|
||||
|
||||
protected:
|
||||
Grid* m_grid;
|
||||
};
|
||||
};
|
||||
|
||||
inline Walker::Walker(Grid* inGrid)
|
||||
: m_grid(inGrid)
|
||||
{
|
||||
inline Walker::Walker(Grid* inGrid) : m_grid(inGrid) {}
|
||||
|
||||
inline const std::vector<NodeAnchor> Walker::reverseWalk(
|
||||
size_t location, double accumulatedScore) {
|
||||
if (!location || location > m_grid->width()) {
|
||||
return std::vector<NodeAnchor>();
|
||||
}
|
||||
|
||||
inline const vector<NodeAnchor> Walker::reverseWalk(size_t inLocation, double inAccumulatedScore)
|
||||
{
|
||||
if (!inLocation || inLocation > m_grid->width()) {
|
||||
return vector<NodeAnchor>();
|
||||
}
|
||||
std::vector<std::vector<NodeAnchor> > paths;
|
||||
|
||||
vector<vector<NodeAnchor> > paths;
|
||||
std::vector<NodeAnchor> nodes = m_grid->nodesEndingAt(location);
|
||||
|
||||
vector<NodeAnchor> nodes = m_grid->nodesEndingAt(inLocation);
|
||||
|
||||
for (vector<NodeAnchor>::iterator ni = nodes.begin() ; ni != nodes.end() ; ++ni) {
|
||||
for (std::vector<NodeAnchor>::iterator ni = nodes.begin(); ni != nodes.end();
|
||||
++ni) {
|
||||
if (!(*ni).node) {
|
||||
continue;
|
||||
}
|
||||
|
||||
(*ni).accumulatedScore = inAccumulatedScore + (*ni).node->score();
|
||||
(*ni).accumulatedScore = accumulatedScore + (*ni).node->score();
|
||||
|
||||
vector<NodeAnchor> path = reverseWalk(inLocation - (*ni).spanningLength, (*ni).accumulatedScore);
|
||||
std::vector<NodeAnchor> path =
|
||||
reverseWalk(location - (*ni).spanningLength, (*ni).accumulatedScore);
|
||||
path.insert(path.begin(), *ni);
|
||||
|
||||
paths.push_back(path);
|
||||
}
|
||||
|
||||
if (!paths.size()) {
|
||||
return vector<NodeAnchor>();
|
||||
return std::vector<NodeAnchor>();
|
||||
}
|
||||
|
||||
vector<NodeAnchor>* result = &*(paths.begin());
|
||||
for (vector<vector<NodeAnchor> >::iterator pi = paths.begin() ; pi != paths.end() ; ++pi) {
|
||||
std::vector<NodeAnchor>* result = &*(paths.begin());
|
||||
for (std::vector<std::vector<NodeAnchor> >::iterator pi = paths.begin();
|
||||
pi != paths.end(); ++pi) {
|
||||
if ((*pi).back().accumulatedScore > result->back().accumulatedScore) {
|
||||
result = &*pi;
|
||||
}
|
||||
}
|
||||
|
||||
return *result;
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace Gramambular
|
||||
} // namespace Formosa
|
||||
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Reference in New Issue