LMs // Simplify data reading process.

This commit is contained in:
ShikiSuen 2022-05-04 11:18:57 +08:00
parent 05d9c252cf
commit 3bd8fa1ea7
4 changed files with 13 additions and 40 deletions

View File

@ -29,7 +29,6 @@ import Foundation
extension vChewing { extension vChewing {
@frozen public struct LMAssociates { @frozen public struct LMAssociates {
var keyValueMap: [String: [Megrez.KeyValuePair]] = [:] var keyValueMap: [String: [Megrez.KeyValuePair]] = [:]
var theData: String = ""
public var count: Int { public var count: Int {
keyValueMap.count keyValueMap.count
@ -37,7 +36,6 @@ extension vChewing {
public init() { public init() {
keyValueMap = [:] keyValueMap = [:]
theData = ""
} }
public func isLoaded() -> Bool { public func isLoaded() -> Bool {
@ -52,20 +50,16 @@ extension vChewing {
LMConsolidator.fixEOF(path: path) LMConsolidator.fixEOF(path: path)
LMConsolidator.consolidate(path: path, pragma: true) LMConsolidator.consolidate(path: path, pragma: true)
var arrData: [String] = []
do { do {
theData = try String(contentsOfFile: path, encoding: .utf8) arrData = try String(contentsOfFile: path, encoding: .utf8).components(separatedBy: "\n")
} catch { } catch {
IME.prtDebugIntel("\(error)") IME.prtDebugIntel("\(error)")
IME.prtDebugIntel("↑ Exception happened when reading Associated Phrases data.") IME.prtDebugIntel("↑ Exception happened when reading Associated Phrases data.")
return false return false
} }
let length = theData.count
guard length > 0 else {
return false
}
let arrData = theData.components(separatedBy: "\n")
for (lineID, lineContent) in arrData.enumerated() { for (lineID, lineContent) in arrData.enumerated() {
if !lineContent.hasPrefix("#") { if !lineContent.hasPrefix("#") {
if lineContent.components(separatedBy: " ").count < 2 { if lineContent.components(separatedBy: " ").count < 2 {
@ -88,7 +82,6 @@ extension vChewing {
} }
} }
IME.prtDebugIntel("\(count) entries of data loaded from: \(path)") IME.prtDebugIntel("\(count) entries of data loaded from: \(path)")
theData = ""
return true return true
} }

View File

@ -29,7 +29,6 @@ import Foundation
extension vChewing { extension vChewing {
@frozen public struct LMCore { @frozen public struct LMCore {
var keyValueScoreMap: [String: [Megrez.Unigram]] = [:] var keyValueScoreMap: [String: [Megrez.Unigram]] = [:]
var theData: String = ""
var shouldReverse: Bool = false var shouldReverse: Bool = false
var allowConsolidation: Bool = false var allowConsolidation: Bool = false
var defaultScore: Double = 0 var defaultScore: Double = 0
@ -44,7 +43,6 @@ extension vChewing {
forceDefaultScore: Bool = false forceDefaultScore: Bool = false
) { ) {
keyValueScoreMap = [:] keyValueScoreMap = [:]
theData = ""
allowConsolidation = consolidate allowConsolidation = consolidate
shouldReverse = reverse shouldReverse = reverse
defaultScore = scoreDefault defaultScore = scoreDefault
@ -65,20 +63,16 @@ extension vChewing {
LMConsolidator.consolidate(path: path, pragma: true) LMConsolidator.consolidate(path: path, pragma: true)
} }
var arrData: [String] = []
do { do {
theData = try String(contentsOfFile: path, encoding: .utf8) arrData = try String(contentsOfFile: path, encoding: .utf8).components(separatedBy: "\n")
} catch { } catch {
IME.prtDebugIntel("\(error)") IME.prtDebugIntel("\(error)")
IME.prtDebugIntel("↑ Exception happened when reading Associated Phrases data.") IME.prtDebugIntel("↑ Exception happened when reading Associated Phrases data.")
return false return false
} }
let length = theData.count
guard length > 0 else {
return false
}
let arrData = theData.components(separatedBy: "\n")
for (lineID, lineContent) in arrData.enumerated() { for (lineID, lineContent) in arrData.enumerated() {
if !lineContent.hasPrefix("#") { if !lineContent.hasPrefix("#") {
let lineContent = lineContent.replacingOccurrences(of: "\t", with: " ") let lineContent = lineContent.replacingOccurrences(of: "\t", with: " ")
@ -122,7 +116,6 @@ extension vChewing {
} }
} }
IME.prtDebugIntel("\(count) entries of data loaded from: \(path)") IME.prtDebugIntel("\(count) entries of data loaded from: \(path)")
theData = ""
return true return true
} }

View File

@ -29,7 +29,6 @@ import Foundation
extension vChewing { extension vChewing {
@frozen public struct LMLite { @frozen public struct LMLite {
var keyValueMap: [String: [Megrez.KeyValuePair]] = [:] var keyValueMap: [String: [Megrez.KeyValuePair]] = [:]
var theData: String = ""
var allowConsolidation = false var allowConsolidation = false
public var count: Int { public var count: Int {
@ -38,7 +37,6 @@ extension vChewing {
public init(consolidate: Bool = false) { public init(consolidate: Bool = false) {
keyValueMap = [:] keyValueMap = [:]
theData = ""
allowConsolidation = consolidate allowConsolidation = consolidate
} }
@ -56,20 +54,16 @@ extension vChewing {
LMConsolidator.consolidate(path: path, pragma: true) LMConsolidator.consolidate(path: path, pragma: true)
} }
var arrData: [String] = []
do { do {
theData = try String(contentsOfFile: path, encoding: .utf8) arrData = try String(contentsOfFile: path, encoding: .utf8).components(separatedBy: "\n")
} catch { } catch {
IME.prtDebugIntel("\(error)") IME.prtDebugIntel("\(error)")
IME.prtDebugIntel("↑ Exception happened when reading Associated Phrases data.") IME.prtDebugIntel("↑ Exception happened when reading Associated Phrases data.")
return false return false
} }
let length = theData.count
guard length > 0 else {
return false
}
let arrData = theData.components(separatedBy: "\n")
for (lineID, lineContent) in arrData.enumerated() { for (lineID, lineContent) in arrData.enumerated() {
if !lineContent.hasPrefix("#") { if !lineContent.hasPrefix("#") {
if lineContent.components(separatedBy: " ").count < 2 { if lineContent.components(separatedBy: " ").count < 2 {
@ -92,7 +86,6 @@ extension vChewing {
} }
} }
IME.prtDebugIntel("\(count) entries of data loaded from: \(path)") IME.prtDebugIntel("\(count) entries of data loaded from: \(path)")
theData = ""
if path.contains("vChewing/") { if path.contains("vChewing/") {
dump() dump()
} }

View File

@ -29,7 +29,6 @@ import Foundation
extension vChewing { extension vChewing {
@frozen public struct LMReplacments { @frozen public struct LMReplacments {
var keyValueMap: [String: String] = [:] var keyValueMap: [String: String] = [:]
var theData: String = ""
public var count: Int { public var count: Int {
keyValueMap.count keyValueMap.count
@ -37,7 +36,6 @@ extension vChewing {
public init() { public init() {
keyValueMap = [:] keyValueMap = [:]
theData = ""
} }
public func isLoaded() -> Bool { public func isLoaded() -> Bool {
@ -52,20 +50,17 @@ extension vChewing {
LMConsolidator.fixEOF(path: path) LMConsolidator.fixEOF(path: path)
LMConsolidator.consolidate(path: path, pragma: true) LMConsolidator.consolidate(path: path, pragma: true)
var arrData: [String] = []
do { do {
theData = try String(contentsOfFile: path, encoding: .utf8) arrData = try String(contentsOfFile: path, encoding: .utf8).components(separatedBy: "\n")
} catch { } catch {
IME.prtDebugIntel("\(error)") IME.prtDebugIntel("\(error)")
IME.prtDebugIntel("↑ Exception happened when reading Associated Phrases data.") IME.prtDebugIntel("↑ Exception happened when reading Associated Phrases data.")
return false return false
} }
let length = theData.count
guard length > 0 else {
return false
}
let arrData = theData.components(separatedBy: "\n")
for (lineID, lineContent) in arrData.enumerated() { for (lineID, lineContent) in arrData.enumerated() {
if !lineContent.hasPrefix("#") { if !lineContent.hasPrefix("#") {
if lineContent.components(separatedBy: " ").count < 2 { if lineContent.components(separatedBy: " ").count < 2 {
@ -88,7 +83,6 @@ extension vChewing {
} }
} }
IME.prtDebugIntel("\(count) entries of data loaded from: \(path)") IME.prtDebugIntel("\(count) entries of data loaded from: \(path)")
theData = ""
return true return true
} }