IME // Rebrand FastLM to CoreLM.
This commit is contained in:
parent
ca81d7874e
commit
e164cfdc6c
|
@ -0,0 +1,64 @@
|
||||||
|
/*
|
||||||
|
* CoreLM.hh
|
||||||
|
*
|
||||||
|
* Copyright 2021-2022 vChewing Project (3-Clause BSD License).
|
||||||
|
* Derived from 2011-2022 OpenVanilla Project (MIT License).
|
||||||
|
* Some rights reserved. See "LICENSE.TXT" for details.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef CoreLM_H
|
||||||
|
#define CoreLM_H
|
||||||
|
|
||||||
|
#include "LanguageModel.h"
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
#include <map>
|
||||||
|
#include <iostream>
|
||||||
|
|
||||||
|
// this class relies on the fact that we have a space-separated data
|
||||||
|
// format, and we use mmap and zero-out the separators and line feeds
|
||||||
|
// to avoid creating new string objects; the parser is a simple DFA
|
||||||
|
|
||||||
|
using namespace std;
|
||||||
|
using namespace Taiyan::Gramambular;
|
||||||
|
|
||||||
|
namespace vChewing {
|
||||||
|
|
||||||
|
class CoreLM : public Taiyan::Gramambular::LanguageModel {
|
||||||
|
public:
|
||||||
|
CoreLM();
|
||||||
|
~CoreLM();
|
||||||
|
|
||||||
|
bool isLoaded();
|
||||||
|
bool open(const char *path);
|
||||||
|
void close();
|
||||||
|
void dump();
|
||||||
|
|
||||||
|
virtual const vector<Bigram> bigramsForKeys(const string& preceedingKey, const string& key);
|
||||||
|
virtual const vector<Unigram> unigramsForKey(const string& key);
|
||||||
|
virtual bool hasUnigramsForKey(const string& key);
|
||||||
|
|
||||||
|
protected:
|
||||||
|
struct CStringCmp
|
||||||
|
{
|
||||||
|
bool operator()(const char* s1, const char* s2) const
|
||||||
|
{
|
||||||
|
return strcmp(s1, s2) < 0;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
struct Row {
|
||||||
|
const char *key;
|
||||||
|
const char *value;
|
||||||
|
const char *logProbability;
|
||||||
|
};
|
||||||
|
|
||||||
|
map<const char *, vector<Row>, CStringCmp> keyRowMap;
|
||||||
|
int fd;
|
||||||
|
void *data;
|
||||||
|
size_t length;
|
||||||
|
};
|
||||||
|
|
||||||
|
}; // namespace vChewing
|
||||||
|
|
||||||
|
#endif
|
|
@ -1,12 +1,12 @@
|
||||||
/*
|
/*
|
||||||
* FastLM.mm
|
* CoreLM.mm
|
||||||
*
|
*
|
||||||
* Copyright 2021-2022 vChewing Project (3-Clause BSD License).
|
* Copyright 2021-2022 vChewing Project (3-Clause BSD License).
|
||||||
* Derived from 2011-2022 OpenVanilla Project (MIT License).
|
* Derived from 2011-2022 OpenVanilla Project (MIT License).
|
||||||
* Some rights reserved. See "LICENSE.TXT" for details.
|
* Some rights reserved. See "LICENSE.TXT" for details.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include "FastLM.h"
|
#include "CoreLM.h"
|
||||||
#include <sys/mman.h>
|
#include <sys/mman.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
#include <fcntl.h>
|
#include <fcntl.h>
|
||||||
|
@ -16,21 +16,29 @@
|
||||||
|
|
||||||
using namespace Taiyan::Gramambular;
|
using namespace Taiyan::Gramambular;
|
||||||
|
|
||||||
FastLM::FastLM()
|
vChewing::CoreLM::CoreLM()
|
||||||
: fd(-1)
|
: fd(-1)
|
||||||
, data(0)
|
, data(0)
|
||||||
, length(0)
|
, length(0)
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
FastLM::~FastLM()
|
vChewing::CoreLM::~CoreLM()
|
||||||
{
|
{
|
||||||
if (data) {
|
if (data) {
|
||||||
close();
|
close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
bool FastLM::open(const char *path)
|
bool vChewing::CoreLM::isLoaded()
|
||||||
|
{
|
||||||
|
if (data) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool vChewing::CoreLM::open(const char *path)
|
||||||
{
|
{
|
||||||
if (data) {
|
if (data) {
|
||||||
return false;
|
return false;
|
||||||
|
@ -238,7 +246,7 @@ end:
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
void FastLM::close()
|
void vChewing::CoreLM::close()
|
||||||
{
|
{
|
||||||
if (data) {
|
if (data) {
|
||||||
munmap(data, length);
|
munmap(data, length);
|
||||||
|
@ -249,7 +257,7 @@ void FastLM::close()
|
||||||
keyRowMap.clear();
|
keyRowMap.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
void FastLM::dump()
|
void vChewing::CoreLM::dump()
|
||||||
{
|
{
|
||||||
size_t rows = 0;
|
size_t rows = 0;
|
||||||
for (map<const char *, vector<Row> >::const_iterator i = keyRowMap.begin(), e = keyRowMap.end(); i != e; ++i) {
|
for (map<const char *, vector<Row> >::const_iterator i = keyRowMap.begin(), e = keyRowMap.end(); i != e; ++i) {
|
||||||
|
@ -262,12 +270,12 @@ void FastLM::dump()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const vector<Bigram> FastLM::bigramsForKeys(const string& preceedingKey, const string& key)
|
const vector<Bigram> vChewing::CoreLM::bigramsForKeys(const string& preceedingKey, const string& key)
|
||||||
{
|
{
|
||||||
return vector<Bigram>();
|
return vector<Bigram>();
|
||||||
}
|
}
|
||||||
|
|
||||||
const vector<Unigram> FastLM::unigramsForKey(const string& key)
|
const vector<Unigram> vChewing::CoreLM::unigramsForKey(const string& key)
|
||||||
{
|
{
|
||||||
vector<Unigram> v;
|
vector<Unigram> v;
|
||||||
map<const char *, vector<Row> >::const_iterator i = keyRowMap.find(key.c_str());
|
map<const char *, vector<Row> >::const_iterator i = keyRowMap.find(key.c_str());
|
||||||
|
@ -286,7 +294,7 @@ const vector<Unigram> FastLM::unigramsForKey(const string& key)
|
||||||
return v;
|
return v;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool FastLM::hasUnigramsForKey(const string& key)
|
bool vChewing::CoreLM::hasUnigramsForKey(const string& key)
|
||||||
{
|
{
|
||||||
return keyRowMap.find(key.c_str()) != keyRowMap.end();
|
return keyRowMap.find(key.c_str()) != keyRowMap.end();
|
||||||
}
|
}
|
|
@ -1,61 +0,0 @@
|
||||||
/*
|
|
||||||
* FastLM.h
|
|
||||||
*
|
|
||||||
* Copyright 2021-2022 vChewing Project (3-Clause BSD License).
|
|
||||||
* Derived from 2011-2022 OpenVanilla Project (MIT License).
|
|
||||||
* Some rights reserved. See "LICENSE.TXT" for details.
|
|
||||||
*/
|
|
||||||
|
|
||||||
#ifndef FASTLM_H
|
|
||||||
#define FASTLM_H
|
|
||||||
|
|
||||||
#include <string>
|
|
||||||
#include <map>
|
|
||||||
#include <iostream>
|
|
||||||
#include "LanguageModel.h"
|
|
||||||
|
|
||||||
// this class relies on the fact that we have a space-separated data
|
|
||||||
// format, and we use mmap and zero-out the separators and line feeds
|
|
||||||
// to avoid creating new string objects; the parser is a simple DFA
|
|
||||||
|
|
||||||
namespace Taiyan {
|
|
||||||
namespace Gramambular {
|
|
||||||
class FastLM : public LanguageModel
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
FastLM();
|
|
||||||
~FastLM();
|
|
||||||
|
|
||||||
bool open(const char *path);
|
|
||||||
void close();
|
|
||||||
void dump();
|
|
||||||
|
|
||||||
virtual const vector<Bigram> bigramsForKeys(const string& preceedingKey, const string& key);
|
|
||||||
virtual const vector<Unigram> unigramsForKey(const string& key);
|
|
||||||
virtual bool hasUnigramsForKey(const string& key);
|
|
||||||
|
|
||||||
protected:
|
|
||||||
struct CStringCmp
|
|
||||||
{
|
|
||||||
bool operator()(const char* s1, const char* s2) const
|
|
||||||
{
|
|
||||||
return strcmp(s1, s2) < 0;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
struct Row {
|
|
||||||
const char *key;
|
|
||||||
const char *value;
|
|
||||||
const char *logProbability;
|
|
||||||
};
|
|
||||||
|
|
||||||
map<const char *, vector<Row>, CStringCmp> keyRowMap;
|
|
||||||
int fd;
|
|
||||||
void *data;
|
|
||||||
size_t length;
|
|
||||||
};
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#endif
|
|
|
@ -10,7 +10,7 @@
|
||||||
#define VCHEWINGLM_H
|
#define VCHEWINGLM_H
|
||||||
|
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include "FastLM.h"
|
#include "CoreLM.h"
|
||||||
#include "CNSLM.h"
|
#include "CNSLM.h"
|
||||||
#include "UserPhrasesLM.h"
|
#include "UserPhrasesLM.h"
|
||||||
#include "PhraseReplacementMap.h"
|
#include "PhraseReplacementMap.h"
|
||||||
|
@ -45,7 +45,7 @@ protected:
|
||||||
const std::unordered_set<string>& excludedValues,
|
const std::unordered_set<string>& excludedValues,
|
||||||
std::unordered_set<string>& insertedValues);
|
std::unordered_set<string>& insertedValues);
|
||||||
|
|
||||||
FastLM m_languageModel;
|
CoreLM m_languageModel;
|
||||||
CNSLM m_cnsModel;
|
CNSLM m_cnsModel;
|
||||||
UserPhrasesLM m_userPhrases;
|
UserPhrasesLM m_userPhrases;
|
||||||
UserPhrasesLM m_excludedPhrases;
|
UserPhrasesLM m_excludedPhrases;
|
||||||
|
|
|
@ -56,7 +56,7 @@
|
||||||
5BDF2D062791DFF200838ADB /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5BDF2D052791DA6700838ADB /* AppDelegate.swift */; };
|
5BDF2D062791DFF200838ADB /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5BDF2D052791DA6700838ADB /* AppDelegate.swift */; };
|
||||||
5BE798A42792E58A00337FF9 /* TooltipController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5BE798A32792E58A00337FF9 /* TooltipController.swift */; };
|
5BE798A42792E58A00337FF9 /* TooltipController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5BE798A32792E58A00337FF9 /* TooltipController.swift */; };
|
||||||
5BE798A72793280C00337FF9 /* NotifierController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5BE798A62793280C00337FF9 /* NotifierController.swift */; };
|
5BE798A72793280C00337FF9 /* NotifierController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5BE798A62793280C00337FF9 /* NotifierController.swift */; };
|
||||||
6A0421A815FEF3F50061ED63 /* FastLM.mm in Sources */ = {isa = PBXBuildFile; fileRef = 6A0421A615FEF3F50061ED63 /* FastLM.mm */; };
|
6A0421A815FEF3F50061ED63 /* CoreLM.mm in Sources */ = {isa = PBXBuildFile; fileRef = 6A0421A615FEF3F50061ED63 /* CoreLM.mm */; };
|
||||||
6A0D4EA715FC0D2D00ABF4B3 /* Cocoa.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 6A0D4EA615FC0D2D00ABF4B3 /* Cocoa.framework */; };
|
6A0D4EA715FC0D2D00ABF4B3 /* Cocoa.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 6A0D4EA615FC0D2D00ABF4B3 /* Cocoa.framework */; };
|
||||||
6A0D4ED215FC0D6400ABF4B3 /* ctlInputMethod.mm in Sources */ = {isa = PBXBuildFile; fileRef = 6A0D4EC715FC0D6400ABF4B3 /* ctlInputMethod.mm */; };
|
6A0D4ED215FC0D6400ABF4B3 /* ctlInputMethod.mm in Sources */ = {isa = PBXBuildFile; fileRef = 6A0D4EC715FC0D6400ABF4B3 /* ctlInputMethod.mm */; };
|
||||||
6A0D4ED315FC0D6400ABF4B3 /* main.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6A0D4EC815FC0D6400ABF4B3 /* main.swift */; };
|
6A0D4ED315FC0D6400ABF4B3 /* main.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6A0D4EC815FC0D6400ABF4B3 /* main.swift */; };
|
||||||
|
@ -180,8 +180,8 @@
|
||||||
5BDF2D052791DA6700838ADB /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
|
5BDF2D052791DA6700838ADB /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
|
||||||
5BE798A32792E58A00337FF9 /* TooltipController.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TooltipController.swift; sourceTree = "<group>"; };
|
5BE798A32792E58A00337FF9 /* TooltipController.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TooltipController.swift; sourceTree = "<group>"; };
|
||||||
5BE798A62793280C00337FF9 /* NotifierController.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NotifierController.swift; sourceTree = "<group>"; };
|
5BE798A62793280C00337FF9 /* NotifierController.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NotifierController.swift; sourceTree = "<group>"; };
|
||||||
6A0421A615FEF3F50061ED63 /* FastLM.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = FastLM.mm; sourceTree = "<group>"; };
|
6A0421A615FEF3F50061ED63 /* CoreLM.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = CoreLM.mm; sourceTree = "<group>"; };
|
||||||
6A0421A715FEF3F50061ED63 /* FastLM.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FastLM.h; sourceTree = "<group>"; };
|
6A0421A715FEF3F50061ED63 /* CoreLM.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CoreLM.h; sourceTree = "<group>"; };
|
||||||
6A0D4EA215FC0D2D00ABF4B3 /* vChewing.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = vChewing.app; sourceTree = BUILT_PRODUCTS_DIR; };
|
6A0D4EA215FC0D2D00ABF4B3 /* vChewing.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = vChewing.app; sourceTree = BUILT_PRODUCTS_DIR; };
|
||||||
6A0D4EA615FC0D2D00ABF4B3 /* Cocoa.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Cocoa.framework; path = System/Library/Frameworks/Cocoa.framework; sourceTree = SDKROOT; };
|
6A0D4EA615FC0D2D00ABF4B3 /* Cocoa.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Cocoa.framework; path = System/Library/Frameworks/Cocoa.framework; sourceTree = SDKROOT; };
|
||||||
6A0D4EA915FC0D2D00ABF4B3 /* AppKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = AppKit.framework; path = System/Library/Frameworks/AppKit.framework; sourceTree = SDKROOT; };
|
6A0D4EA915FC0D2D00ABF4B3 /* AppKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = AppKit.framework; path = System/Library/Frameworks/AppKit.framework; sourceTree = SDKROOT; };
|
||||||
|
@ -380,8 +380,8 @@
|
||||||
children = (
|
children = (
|
||||||
5BDD25FB279D6D6200AA18F8 /* CNSLM.mm */,
|
5BDD25FB279D6D6200AA18F8 /* CNSLM.mm */,
|
||||||
5BDD25FC279D6D6300AA18F8 /* CNSLM.h */,
|
5BDD25FC279D6D6300AA18F8 /* CNSLM.h */,
|
||||||
6A0421A615FEF3F50061ED63 /* FastLM.mm */,
|
6A0421A615FEF3F50061ED63 /* CoreLM.mm */,
|
||||||
6A0421A715FEF3F50061ED63 /* FastLM.h */,
|
6A0421A715FEF3F50061ED63 /* CoreLM.h */,
|
||||||
5B42B63E27876FDC00BB9B9F /* UserOverrideModel.mm */,
|
5B42B63E27876FDC00BB9B9F /* UserOverrideModel.mm */,
|
||||||
5B42B63F27876FDC00BB9B9F /* UserOverrideModel.h */,
|
5B42B63F27876FDC00BB9B9F /* UserOverrideModel.h */,
|
||||||
5B5F4F962792A4EA00922DC2 /* UserPhrasesLM.mm */,
|
5B5F4F962792A4EA00922DC2 /* UserPhrasesLM.mm */,
|
||||||
|
@ -805,7 +805,7 @@
|
||||||
D427A9C125ED28CC005D43E0 /* OpenCCBridge.swift in Sources */,
|
D427A9C125ED28CC005D43E0 /* OpenCCBridge.swift in Sources */,
|
||||||
6A0D4F4515FC0EB100ABF4B3 /* Mandarin.mm in Sources */,
|
6A0D4F4515FC0EB100ABF4B3 /* Mandarin.mm in Sources */,
|
||||||
5B6797B52794822C004AC7CE /* PhraseReplacementMap.h in Sources */,
|
5B6797B52794822C004AC7CE /* PhraseReplacementMap.h in Sources */,
|
||||||
6A0421A815FEF3F50061ED63 /* FastLM.mm in Sources */,
|
6A0421A815FEF3F50061ED63 /* CoreLM.mm in Sources */,
|
||||||
5BDF2D012791C03B00838ADB /* ctlPrefWindow.swift in Sources */,
|
5BDF2D012791C03B00838ADB /* ctlPrefWindow.swift in Sources */,
|
||||||
5BC2D28D2793B98F002C0BEC /* PreferencesModule.swift in Sources */,
|
5BC2D28D2793B98F002C0BEC /* PreferencesModule.swift in Sources */,
|
||||||
5B217128279BB22700F91A2B /* ctlAboutWindow.swift in Sources */,
|
5B217128279BB22700F91A2B /* ctlAboutWindow.swift in Sources */,
|
||||||
|
|
Loading…
Reference in New Issue