floraachy
/
vChewing-macOS
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

LMAssembly // Add replaceData() to certain submodels. 

- LMCoreEX // Add replaceData(), etc.
- LMReplacements // Add replaceData(), etc.
- LMAssociates // Add replaceData(), etc.
This commit is contained in:
ShikiSuen 2022-12-02 17:07:09 +08:00
parent 2913c22d40
commit cd85f821d2
3 changed files with 69 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmAssociates.swift
View File

@ -42,20 +42,8 @@ extension vChewingLM {
LMConsolidator.consolidate(path: path, pragma: true)
do {
strData = try String(contentsOfFile: path, encoding: .utf8).replacingOccurrences(of: "\t", with: " ")
strData = strData.replacingOccurrences(of: "\r", with: "\n")
strData.ranges(splitBy: "\n").filter { !$0.isEmpty }.forEach {
let neta = strData[$0].split(separator: " ")
if neta.count >= 2 {
let theKey = String(neta[0])
if !theKey.isEmpty, theKey.first != "#" {
for (i, _) in neta.filter({ $0.first != "#" && !$0.isEmpty }).enumerated() {
if i == 0 { continue }
rangeMap[cnvNGramKeyFromPinyinToPhona(target: theKey), default: []].append(($0, i))
}
}
}
}
let rawStrData = try String(contentsOfFile: path, encoding: .utf8)
replaceData(textData: rawStrData)
} catch {
vCLog("\(error)")
vCLog("↑ Exception happened when reading data at: \(path).")
@ -65,6 +53,26 @@ extension vChewingLM {
return true
}
/// 將資料從檔案讀入至資料庫辭典內。
/// - parameters:
/// - path: 給定路徑。
public mutating func replaceData(textData rawStrData: String) {
if strData == rawStrData { return }
strData = rawStrData
strData.ranges(splitBy: "\n").filter { !$0.isEmpty }.forEach {
let neta = strData[$0].split(separator: " ")
if neta.count >= 2 {
let theKey = String(neta[0])
if !theKey.isEmpty, theKey.first != "#" {
for (i, _) in neta.filter({ $0.first != "#" && !$0.isEmpty }).enumerated() {
if i == 0 { continue }
rangeMap[cnvNGramKeyFromPinyinToPhona(target: theKey), default: []].append(($0, i))
}
}
}
}
}
public mutating func clear() {
rangeMap.removeAll()
}

40
Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmCoreEX.swift
View File

@ -60,27 +60,21 @@ extension vChewingLM {
/// - path: 給定路徑。
@discardableResult public mutating func open(_ path: String) -> Bool {
if isLoaded { return false }
var consolidated = false
if allowConsolidation {
LMConsolidator.fixEOF(path: path)
LMConsolidator.consolidate(path: path, pragma: true)
consolidated = true
}
do {
strData = try String(contentsOfFile: path, encoding: .utf8).replacingOccurrences(of: "\t", with: " ")
strData = strData.replacingOccurrences(of: "\r", with: "\n")
strData.ranges(splitBy: "\n").filter { !$0.isEmpty }.forEach {
let neta = strData[$0].split(separator: " ")
if neta.count >= 2, String(neta[0]).first != "#" {
if !neta[0].isEmpty, !neta[1].isEmpty {
var theKey = shouldReverse ? String(neta[1]) : String(neta[0])
let theValue = $0
theKey.converToPhonabets()
rangeMap[theKey, default: []].append(theValue)
}
}
var rawStrData = try String(contentsOfFile: path, encoding: .utf8)
if !consolidated {
rawStrData = rawStrData.replacingOccurrences(of: "\t", with: " ")
rawStrData = rawStrData.replacingOccurrences(of: "\r", with: "\n")
}
temporaryMap.removeAll()
replaceData(textData: rawStrData)
} catch {
vCLog("\(error)")
vCLog("↑ Exception happened when reading data at: \(path).")
@ -90,6 +84,26 @@ extension vChewingLM {
return true
}
/// 將資料從檔案讀入至資料庫辭典內。
/// - parameters:
/// - path: 給定路徑。
public mutating func replaceData(textData rawStrData: String) {
if strData == rawStrData { return }
strData = rawStrData
strData.ranges(splitBy: "\n").filter { !$0.isEmpty }.forEach {
let neta = strData[$0].split(separator: " ")
if neta.count >= 2, String(neta[0]).first != "#" {
if !neta[0].isEmpty, !neta[1].isEmpty {
var theKey = shouldReverse ? String(neta[1]) : String(neta[0])
let theValue = $0
theKey.converToPhonabets()
rangeMap[theKey, default: []].append(theValue)
}
}
}
temporaryMap.removeAll()
}
/// 將當前語言模組的資料庫辭典自記憶體內卸除。
public mutating func clear() {
rangeMap.removeAll()

32
Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmReplacements.swift
View File

@ -29,18 +29,8 @@ extension vChewingLM {
LMConsolidator.consolidate(path: path, pragma: true)
do {
strData = try String(contentsOfFile: path, encoding: .utf8).replacingOccurrences(of: "\t", with: " ")
strData = strData.replacingOccurrences(of: "\r", with: "\n")
strData.ranges(splitBy: "\n").filter { !$0.isEmpty }.forEach {
let neta = strData[$0].split(separator: " ")
if neta.count >= 2 {
let theKey = String(neta[0])
if !neta[0].isEmpty, !neta[1].isEmpty, theKey.first != "#" {
let theValue = $0
rangeMap[theKey] = theValue
}
}
}
let rawStrData = try String(contentsOfFile: path, encoding: .utf8)
replaceData(textData: rawStrData)
} catch {
vCLog("\(error)")
vCLog("↑ Exception happened when reading data at: \(path).")
@ -50,6 +40,24 @@ extension vChewingLM {
return true
}
/// 將資料從檔案讀入至資料庫辭典內。
/// - parameters:
/// - path: 給定路徑。
public mutating func replaceData(textData rawStrData: String) {
if strData == rawStrData { return }
strData = rawStrData
strData.ranges(splitBy: "\n").filter { !$0.isEmpty }.forEach {
let neta = strData[$0].split(separator: " ")
if neta.count >= 2 {
let theKey = String(neta[0])
if !neta[0].isEmpty, !neta[1].isEmpty, theKey.first != "#" {
let theValue = $0
rangeMap[theKey] = theValue
}
}
}
}
public mutating func clear() {
rangeMap.removeAll()
}