diff --git a/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmAssociates.swift b/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmAssociates.swift index c8ad6756..0c873596 100644 --- a/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmAssociates.swift +++ b/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmAssociates.swift @@ -42,20 +42,8 @@ extension vChewingLM { LMConsolidator.consolidate(path: path, pragma: true) do { - strData = try String(contentsOfFile: path, encoding: .utf8).replacingOccurrences(of: "\t", with: " ") - strData = strData.replacingOccurrences(of: "\r", with: "\n") - strData.ranges(splitBy: "\n").filter { !$0.isEmpty }.forEach { - let neta = strData[$0].split(separator: " ") - if neta.count >= 2 { - let theKey = String(neta[0]) - if !theKey.isEmpty, theKey.first != "#" { - for (i, _) in neta.filter({ $0.first != "#" && !$0.isEmpty }).enumerated() { - if i == 0 { continue } - rangeMap[cnvNGramKeyFromPinyinToPhona(target: theKey), default: []].append(($0, i)) - } - } - } - } + let rawStrData = try String(contentsOfFile: path, encoding: .utf8) + replaceData(textData: rawStrData) } catch { vCLog("\(error)") vCLog("↑ Exception happened when reading data at: \(path).") @@ -65,6 +53,26 @@ extension vChewingLM { return true } + /// 將資料從檔案讀入至資料庫辭典內。 + /// - parameters: + /// - path: 給定路徑。 + public mutating func replaceData(textData rawStrData: String) { + if strData == rawStrData { return } + strData = rawStrData + strData.ranges(splitBy: "\n").filter { !$0.isEmpty }.forEach { + let neta = strData[$0].split(separator: " ") + if neta.count >= 2 { + let theKey = String(neta[0]) + if !theKey.isEmpty, theKey.first != "#" { + for (i, _) in neta.filter({ $0.first != "#" && !$0.isEmpty }).enumerated() { + if i == 0 { continue } + rangeMap[cnvNGramKeyFromPinyinToPhona(target: theKey), default: []].append(($0, i)) + } + } + } + } + } + public mutating func clear() { rangeMap.removeAll() } diff --git a/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmCoreEX.swift b/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmCoreEX.swift index b68c0a6d..9bd0adc9 100644 --- a/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmCoreEX.swift +++ b/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmCoreEX.swift @@ -60,27 +60,21 @@ extension vChewingLM { /// - path: 給定路徑。 @discardableResult public mutating func open(_ path: String) -> Bool { if isLoaded { return false } + var consolidated = false if allowConsolidation { LMConsolidator.fixEOF(path: path) LMConsolidator.consolidate(path: path, pragma: true) + consolidated = true } do { - strData = try String(contentsOfFile: path, encoding: .utf8).replacingOccurrences(of: "\t", with: " ") - strData = strData.replacingOccurrences(of: "\r", with: "\n") - strData.ranges(splitBy: "\n").filter { !$0.isEmpty }.forEach { - let neta = strData[$0].split(separator: " ") - if neta.count >= 2, String(neta[0]).first != "#" { - if !neta[0].isEmpty, !neta[1].isEmpty { - var theKey = shouldReverse ? String(neta[1]) : String(neta[0]) - let theValue = $0 - theKey.converToPhonabets() - rangeMap[theKey, default: []].append(theValue) - } - } + var rawStrData = try String(contentsOfFile: path, encoding: .utf8) + if !consolidated { + rawStrData = rawStrData.replacingOccurrences(of: "\t", with: " ") + rawStrData = rawStrData.replacingOccurrences(of: "\r", with: "\n") } - temporaryMap.removeAll() + replaceData(textData: rawStrData) } catch { vCLog("\(error)") vCLog("↑ Exception happened when reading data at: \(path).") @@ -90,6 +84,26 @@ extension vChewingLM { return true } + /// 將資料從檔案讀入至資料庫辭典內。 + /// - parameters: + /// - path: 給定路徑。 + public mutating func replaceData(textData rawStrData: String) { + if strData == rawStrData { return } + strData = rawStrData + strData.ranges(splitBy: "\n").filter { !$0.isEmpty }.forEach { + let neta = strData[$0].split(separator: " ") + if neta.count >= 2, String(neta[0]).first != "#" { + if !neta[0].isEmpty, !neta[1].isEmpty { + var theKey = shouldReverse ? String(neta[1]) : String(neta[0]) + let theValue = $0 + theKey.converToPhonabets() + rangeMap[theKey, default: []].append(theValue) + } + } + } + temporaryMap.removeAll() + } + /// 將當前語言模組的資料庫辭典自記憶體內卸除。 public mutating func clear() { rangeMap.removeAll() diff --git a/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmReplacements.swift b/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmReplacements.swift index 4abf4a7a..86a7c250 100644 --- a/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmReplacements.swift +++ b/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmReplacements.swift @@ -29,18 +29,8 @@ extension vChewingLM { LMConsolidator.consolidate(path: path, pragma: true) do { - strData = try String(contentsOfFile: path, encoding: .utf8).replacingOccurrences(of: "\t", with: " ") - strData = strData.replacingOccurrences(of: "\r", with: "\n") - strData.ranges(splitBy: "\n").filter { !$0.isEmpty }.forEach { - let neta = strData[$0].split(separator: " ") - if neta.count >= 2 { - let theKey = String(neta[0]) - if !neta[0].isEmpty, !neta[1].isEmpty, theKey.first != "#" { - let theValue = $0 - rangeMap[theKey] = theValue - } - } - } + let rawStrData = try String(contentsOfFile: path, encoding: .utf8) + replaceData(textData: rawStrData) } catch { vCLog("\(error)") vCLog("↑ Exception happened when reading data at: \(path).") @@ -50,6 +40,24 @@ extension vChewingLM { return true } + /// 將資料從檔案讀入至資料庫辭典內。 + /// - parameters: + /// - path: 給定路徑。 + public mutating func replaceData(textData rawStrData: String) { + if strData == rawStrData { return } + strData = rawStrData + strData.ranges(splitBy: "\n").filter { !$0.isEmpty }.forEach { + let neta = strData[$0].split(separator: " ") + if neta.count >= 2 { + let theKey = String(neta[0]) + if !neta[0].isEmpty, !neta[1].isEmpty, theKey.first != "#" { + let theValue = $0 + rangeMap[theKey] = theValue + } + } + } + } + public mutating func clear() { rangeMap.removeAll() }