floraachy
/
vChewing-macOS
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

LMA // Trim unnecessary newlines.

This commit is contained in:
ShikiSuen 2022-10-26 10:36:37 +08:00
parent e35a5b246f
commit 7cbaad774d
3 changed files with 12 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmCassette.swift
View File

@ -96,11 +96,11 @@ extension vChewingLM {
let cells: [String.SubSequence] =
strLine.contains("\t") ? strLine.split(separator: "\t") : strLine.split(separator: " ")
guard cells.count >= 2 else { continue }
let strFirstCell = String(cells[0])
let strFirstCell = cells[0].trimmingCharacters(in: .newlines)
let strSecondCell = cells[1].trimmingCharacters(in: .newlines)
if loadingKeys, !cells[0].contains("%keyname") {
keyNameMap[strFirstCell] = String(cells[1])
keyNameMap[strFirstCell] = cells[1].trimmingCharacters(in: .newlines)
} else if loadingCharDefinitions, !strLine.contains("%chardef") {
let strSecondCell = String(cells[1])
theMaxKeyLength = max(theMaxKeyLength, cells[0].count)
charDefMap[strFirstCell, default: []].append(strSecondCell)
reverseLookupMap[strSecondCell, default: []].append(strFirstCell)
@ -113,7 +113,7 @@ extension vChewingLM {
guard let countValue = Int(cells[1]) else { continue }
switch cells.count {
case 2: octagramMap[strFirstCell] = countValue
case 3: octagramDividedMap[strFirstCell] = (countValue, String(cells[2]))
case 3: octagramDividedMap[strFirstCell] = (countValue, cells[2].trimmingCharacters(in: .newlines))
default: break
}
norm += Self.fscale ** (Double(cells[0].count) / 3.0 - 1.0) * Double(countValue)
@ -127,13 +127,13 @@ extension vChewingLM {
break
}
}
if nameENG.isEmpty { nameENG = String(cells[1]) }
if nameENG.isEmpty { nameENG = strSecondCell }
}
if nameIntl.isEmpty, strLine.contains("%intlname ") {
nameIntl = String(cells[1]).replacingOccurrences(of: "_", with: " ")
nameIntl = strSecondCell.replacingOccurrences(of: "_", with: " ")
}
if nameCJK.isEmpty, strLine.contains("%cname ") { nameCJK = String(cells[1]) }
if nameShort.isEmpty, strLine.contains("%sname ") { nameShort = String(cells[1]) }
if nameCJK.isEmpty, strLine.contains("%cname ") { nameCJK = strSecondCell }
if nameShort.isEmpty, strLine.contains("%sname ") { nameShort = strSecondCell }
if selectionKeys.isEmpty, strLine.contains("%selkey ") {
selectionKeys = cells[1].map { String($0) }.deduplicated
}

4
Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmCoreNS.swift
View File

@ -90,7 +90,7 @@ extension vChewingLM {
let theKey = entry.key
for netaSet in netaSets {
let strNetaSet = String(decoding: netaSet, as: UTF8.self)
let neta = Array(strNetaSet.components(separatedBy: " ").reversed())
let neta = Array(strNetaSet.trimmingCharacters(in: .newlines).components(separatedBy: " ").reversed())
let theValue = neta[0]
var theScore = defaultScore
if neta.count >= 2, !shouldForceDefaultScore {
@ -110,7 +110,7 @@ extension vChewingLM {
if let arrRangeRecords: [Data] = rangeMap[cnvPhonabetToASCII(key)] {
for netaSet in arrRangeRecords {
let strNetaSet = String(decoding: netaSet, as: UTF8.self)
let neta = Array(strNetaSet.split(separator: " ").reversed())
let neta = Array(strNetaSet.trimmingCharacters(in: .newlines).split(separator: " ").reversed())
let theValue: String = .init(neta[0])
var theScore = defaultScore
if neta.count >= 2, !shouldForceDefaultScore {

5
Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmPlainBopomofo.swift
View File

@ -45,11 +45,10 @@ extension vChewingLM {
public func valuesFor(key: String) -> [String] {
var pairs: [String] = []
if let arrRangeRecords: String = rangeMap[key] {
if let arrRangeRecords: String = rangeMap[key]?.trimmingCharacters(in: .newlines) {
pairs.append(contentsOf: arrRangeRecords.map { String($0) })
}
var set = Set<String>()
return pairs.filter { set.insert($0).inserted }
return pairs.deduplicated
}
public func hasValuesFor(key: String) -> Bool { rangeMap.keys.contains(key) }