floraachy
/
vChewing-macOS
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

IME // Rebrand FastLM to CoreLM.

This commit is contained in:
ShikiSuen 2022-02-05 10:21:40 +08:00
parent ca81d7874e
commit e164cfdc6c
5 changed files with 92 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
Source/Modules/LangModelRelated/SubLanguageModels/CoreLM.h Normal file
View File

@ -0,0 +1,64 @@
/*
* CoreLM.hh
*
* Copyright 2021-2022 vChewing Project (3-Clause BSD License).
* Derived from 2011-2022 OpenVanilla Project (MIT License).
* Some rights reserved. See "LICENSE.TXT" for details.
*/
#ifndef CoreLM_H
#define CoreLM_H
#include "LanguageModel.h"
#include <string>
#include <vector>
#include <map>
#include <iostream>
// this class relies on the fact that we have a space-separated data
// format, and we use mmap and zero-out the separators and line feeds
// to avoid creating new string objects; the parser is a simple DFA
using namespace std;
using namespace Taiyan::Gramambular;
namespace vChewing {
class CoreLM : public Taiyan::Gramambular::LanguageModel {
public:
CoreLM();
~CoreLM();
bool isLoaded();
bool open(const char *path);
void close();
void dump();
virtual const vector<Bigram> bigramsForKeys(const string& preceedingKey, const string& key);
virtual const vector<Unigram> unigramsForKey(const string& key);
virtual bool hasUnigramsForKey(const string& key);
protected:
struct CStringCmp
{
bool operator()(const char* s1, const char* s2) const
{
return strcmp(s1, s2) < 0;
}
};
struct Row {
const char *key;
const char *value;
const char *logProbability;
};
map<const char *, vector<Row>, CStringCmp> keyRowMap;
int fd;
void *data;
size_t length;
};
}; // namespace vChewing
#endif

28
Source/Modules/LangModelRelated/SubLanguageModels/FastLM.mm → Source/Modules/LangModelRelated/SubLanguageModels/CoreLM.mm
View File

@ -1,12 +1,12 @@
/*
* FastLM.mm
* CoreLM.mm
*
* Copyright 2021-2022 vChewing Project (3-Clause BSD License).
* Derived from 2011-2022 OpenVanilla Project (MIT License).
* Some rights reserved. See "LICENSE.TXT" for details.
*/
#include "FastLM.h"
#include "CoreLM.h"
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
@ -16,21 +16,29 @@
using namespace Taiyan::Gramambular;
FastLM::FastLM()
vChewing::CoreLM::CoreLM()
: fd(-1)
, data(0)
, length(0)
{
}
FastLM::~FastLM()
vChewing::CoreLM::~CoreLM()
{
if (data) {
close();
}
}
bool FastLM::open(const char *path)
bool vChewing::CoreLM::isLoaded()
{
if (data) {
return true;
}
return false;
}
bool vChewing::CoreLM::open(const char *path)
{
if (data) {
return false;
@ -238,7 +246,7 @@ end:
return true;
}
void FastLM::close()
void vChewing::CoreLM::close()
{
if (data) {
munmap(data, length);
@ -249,7 +257,7 @@ void FastLM::close()
keyRowMap.clear();
}
void FastLM::dump()
void vChewing::CoreLM::dump()
{
size_t rows = 0;
for (map<const char *, vector<Row> >::const_iterator i = keyRowMap.begin(), e = keyRowMap.end(); i != e; ++i) {
@ -262,12 +270,12 @@ void FastLM::dump()
}
}
const vector<Bigram> FastLM::bigramsForKeys(const string& preceedingKey, const string& key)
const vector<Bigram> vChewing::CoreLM::bigramsForKeys(const string& preceedingKey, const string& key)
{
return vector<Bigram>();
}
const vector<Unigram> FastLM::unigramsForKey(const string& key)
const vector<Unigram> vChewing::CoreLM::unigramsForKey(const string& key)
{
vector<Unigram> v;
map<const char *, vector<Row> >::const_iterator i = keyRowMap.find(key.c_str());
@ -286,7 +294,7 @@ const vector<Unigram> FastLM::unigramsForKey(const string& key)
return v;
}
bool FastLM::hasUnigramsForKey(const string& key)
bool vChewing::CoreLM::hasUnigramsForKey(const string& key)
{
return keyRowMap.find(key.c_str()) != keyRowMap.end();
}

61
Source/Modules/LangModelRelated/SubLanguageModels/FastLM.h
View File

@ -1,61 +0,0 @@
/*
* FastLM.h
*
* Copyright 2021-2022 vChewing Project (3-Clause BSD License).
* Derived from 2011-2022 OpenVanilla Project (MIT License).
* Some rights reserved. See "LICENSE.TXT" for details.
*/
#ifndef FASTLM_H
#define FASTLM_H
#include <string>
#include <map>
#include <iostream>
#include "LanguageModel.h"
// this class relies on the fact that we have a space-separated data
// format, and we use mmap and zero-out the separators and line feeds
// to avoid creating new string objects; the parser is a simple DFA
namespace Taiyan {
namespace Gramambular {
class FastLM : public LanguageModel
{
public:
FastLM();
~FastLM();
bool open(const char *path);
void close();
void dump();
virtual const vector<Bigram> bigramsForKeys(const string& preceedingKey, const string& key);
virtual const vector<Unigram> unigramsForKey(const string& key);
virtual bool hasUnigramsForKey(const string& key);
protected:
struct CStringCmp
{
bool operator()(const char* s1, const char* s2) const
{
return strcmp(s1, s2) < 0;
}
};
struct Row {
const char *key;
const char *value;
const char *logProbability;
};
map<const char *, vector<Row>, CStringCmp> keyRowMap;
int fd;
void *data;
size_t length;
};
}
}
#endif

4
Source/Modules/LangModelRelated/vChewingLM.h
View File

@ -10,7 +10,7 @@
#define VCHEWINGLM_H
#include <stdio.h>
#include "FastLM.h"
#include "CoreLM.h"
#include "CNSLM.h"
#include "UserPhrasesLM.h"
#include "PhraseReplacementMap.h"
@ -45,7 +45,7 @@ protected:
const std::unordered_set<string>& excludedValues,
std::unordered_set<string>& insertedValues);
FastLM m_languageModel;
CoreLM m_languageModel;
CNSLM m_cnsModel;
UserPhrasesLM m_userPhrases;
UserPhrasesLM m_excludedPhrases;

12
vChewing.xcodeproj/project.pbxproj
View File

@ -56,7 +56,7 @@
5BDF2D062791DFF200838ADB /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5BDF2D052791DA6700838ADB /* AppDelegate.swift */; };
5BE798A42792E58A00337FF9 /* TooltipController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5BE798A32792E58A00337FF9 /* TooltipController.swift */; };
5BE798A72793280C00337FF9 /* NotifierController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5BE798A62793280C00337FF9 /* NotifierController.swift */; };
6A0421A815FEF3F50061ED63 /* FastLM.mm in Sources */ = {isa = PBXBuildFile; fileRef = 6A0421A615FEF3F50061ED63 /* FastLM.mm */; };
6A0421A815FEF3F50061ED63 /* CoreLM.mm in Sources */ = {isa = PBXBuildFile; fileRef = 6A0421A615FEF3F50061ED63 /* CoreLM.mm */; };
6A0D4EA715FC0D2D00ABF4B3 /* Cocoa.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 6A0D4EA615FC0D2D00ABF4B3 /* Cocoa.framework */; };
6A0D4ED215FC0D6400ABF4B3 /* ctlInputMethod.mm in Sources */ = {isa = PBXBuildFile; fileRef = 6A0D4EC715FC0D6400ABF4B3 /* ctlInputMethod.mm */; };
6A0D4ED315FC0D6400ABF4B3 /* main.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6A0D4EC815FC0D6400ABF4B3 /* main.swift */; };
@ -180,8 +180,8 @@
5BDF2D052791DA6700838ADB /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
5BE798A32792E58A00337FF9 /* TooltipController.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TooltipController.swift; sourceTree = "<group>"; };
5BE798A62793280C00337FF9 /* NotifierController.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NotifierController.swift; sourceTree = "<group>"; };
6A0421A615FEF3F50061ED63 /* FastLM.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = FastLM.mm; sourceTree = "<group>"; };
6A0421A715FEF3F50061ED63 /* FastLM.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FastLM.h; sourceTree = "<group>"; };
6A0421A615FEF3F50061ED63 /* CoreLM.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = CoreLM.mm; sourceTree = "<group>"; };
6A0421A715FEF3F50061ED63 /* CoreLM.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CoreLM.h; sourceTree = "<group>"; };
6A0D4EA215FC0D2D00ABF4B3 /* vChewing.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = vChewing.app; sourceTree = BUILT_PRODUCTS_DIR; };
6A0D4EA615FC0D2D00ABF4B3 /* Cocoa.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Cocoa.framework; path = System/Library/Frameworks/Cocoa.framework; sourceTree = SDKROOT; };
6A0D4EA915FC0D2D00ABF4B3 /* AppKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = AppKit.framework; path = System/Library/Frameworks/AppKit.framework; sourceTree = SDKROOT; };
@ -380,8 +380,8 @@
children = (
5BDD25FB279D6D6200AA18F8 /* CNSLM.mm */,
5BDD25FC279D6D6300AA18F8 /* CNSLM.h */,
6A0421A615FEF3F50061ED63 /* FastLM.mm */,
6A0421A715FEF3F50061ED63 /* FastLM.h */,
6A0421A615FEF3F50061ED63 /* CoreLM.mm */,
6A0421A715FEF3F50061ED63 /* CoreLM.h */,
5B42B63E27876FDC00BB9B9F /* UserOverrideModel.mm */,
5B42B63F27876FDC00BB9B9F /* UserOverrideModel.h */,
5B5F4F962792A4EA00922DC2 /* UserPhrasesLM.mm */,
@ -805,7 +805,7 @@
D427A9C125ED28CC005D43E0 /* OpenCCBridge.swift in Sources */,
6A0D4F4515FC0EB100ABF4B3 /* Mandarin.mm in Sources */,
5B6797B52794822C004AC7CE /* PhraseReplacementMap.h in Sources */,
6A0421A815FEF3F50061ED63 /* FastLM.mm in Sources */,
6A0421A815FEF3F50061ED63 /* CoreLM.mm in Sources */,
5BDF2D012791C03B00838ADB /* ctlPrefWindow.swift in Sources */,
5BC2D28D2793B98F002C0BEC /* PreferencesModule.swift in Sources */,
5B217128279BB22700F91A2B /* ctlAboutWindow.swift in Sources */,