From 7cbaad774d140862d8c650d87e7ac0f2319087dd Mon Sep 17 00:00:00 2001
From: ShikiSuen <shikisuen@outlook.com>
Date: Wed, 26 Oct 2022 10:36:37 +0800
Subject: [PATCH] LMA // Trim unnecessary newlines.

---
 .../LangModelAssembly/SubLMs/lmCassette.swift    | 16 ++++++++--------
 .../LangModelAssembly/SubLMs/lmCoreNS.swift      |  4 ++--
 .../SubLMs/lmPlainBopomofo.swift                 |  5 ++---
 3 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmCassette.swift b/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmCassette.swift
index 10beedc2..da3fee1a 100644
--- a/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmCassette.swift
+++ b/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmCassette.swift
@@ -96,11 +96,11 @@ extension vChewingLM {
             let cells: [String.SubSequence] =
               strLine.contains("\t") ? strLine.split(separator: "\t") : strLine.split(separator: " ")
             guard cells.count >= 2 else { continue }
-            let strFirstCell = String(cells[0])
+            let strFirstCell = cells[0].trimmingCharacters(in: .newlines)
+            let strSecondCell = cells[1].trimmingCharacters(in: .newlines)
             if loadingKeys, !cells[0].contains("%keyname") {
-              keyNameMap[strFirstCell] = String(cells[1])
+              keyNameMap[strFirstCell] = cells[1].trimmingCharacters(in: .newlines)
             } else if loadingCharDefinitions, !strLine.contains("%chardef") {
-              let strSecondCell = String(cells[1])
               theMaxKeyLength = max(theMaxKeyLength, cells[0].count)
               charDefMap[strFirstCell, default: []].append(strSecondCell)
               reverseLookupMap[strSecondCell, default: []].append(strFirstCell)
@@ -113,7 +113,7 @@ extension vChewingLM {
               guard let countValue = Int(cells[1]) else { continue }
               switch cells.count {
                 case 2: octagramMap[strFirstCell] = countValue
-                case 3: octagramDividedMap[strFirstCell] = (countValue, String(cells[2]))
+                case 3: octagramDividedMap[strFirstCell] = (countValue, cells[2].trimmingCharacters(in: .newlines))
                 default: break
               }
               norm += Self.fscale ** (Double(cells[0].count) / 3.0 - 1.0) * Double(countValue)
@@ -127,13 +127,13 @@ extension vChewingLM {
                   break
                 }
               }
-              if nameENG.isEmpty { nameENG = String(cells[1]) }
+              if nameENG.isEmpty { nameENG = strSecondCell }
             }
             if nameIntl.isEmpty, strLine.contains("%intlname ") {
-              nameIntl = String(cells[1]).replacingOccurrences(of: "_", with: " ")
+              nameIntl = strSecondCell.replacingOccurrences(of: "_", with: " ")
             }
-            if nameCJK.isEmpty, strLine.contains("%cname ") { nameCJK = String(cells[1]) }
-            if nameShort.isEmpty, strLine.contains("%sname ") { nameShort = String(cells[1]) }
+            if nameCJK.isEmpty, strLine.contains("%cname ") { nameCJK = strSecondCell }
+            if nameShort.isEmpty, strLine.contains("%sname ") { nameShort = strSecondCell }
             if selectionKeys.isEmpty, strLine.contains("%selkey ") {
               selectionKeys = cells[1].map { String($0) }.deduplicated
             }
diff --git a/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmCoreNS.swift b/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmCoreNS.swift
index 249e5c16..5c710880 100644
--- a/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmCoreNS.swift
+++ b/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmCoreNS.swift
@@ -90,7 +90,7 @@ extension vChewingLM {
         let theKey = entry.key
         for netaSet in netaSets {
           let strNetaSet = String(decoding: netaSet, as: UTF8.self)
-          let neta = Array(strNetaSet.components(separatedBy: " ").reversed())
+          let neta = Array(strNetaSet.trimmingCharacters(in: .newlines).components(separatedBy: " ").reversed())
           let theValue = neta[0]
           var theScore = defaultScore
           if neta.count >= 2, !shouldForceDefaultScore {
@@ -110,7 +110,7 @@ extension vChewingLM {
       if let arrRangeRecords: [Data] = rangeMap[cnvPhonabetToASCII(key)] {
         for netaSet in arrRangeRecords {
           let strNetaSet = String(decoding: netaSet, as: UTF8.self)
-          let neta = Array(strNetaSet.split(separator: " ").reversed())
+          let neta = Array(strNetaSet.trimmingCharacters(in: .newlines).split(separator: " ").reversed())
           let theValue: String = .init(neta[0])
           var theScore = defaultScore
           if neta.count >= 2, !shouldForceDefaultScore {
diff --git a/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmPlainBopomofo.swift b/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmPlainBopomofo.swift
index 44c267f1..049564f0 100644
--- a/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmPlainBopomofo.swift
+++ b/Packages/vChewing_LangModelAssembly/Sources/LangModelAssembly/SubLMs/lmPlainBopomofo.swift
@@ -45,11 +45,10 @@ extension vChewingLM {
 
     public func valuesFor(key: String) -> [String] {
       var pairs: [String] = []
-      if let arrRangeRecords: String = rangeMap[key] {
+      if let arrRangeRecords: String = rangeMap[key]?.trimmingCharacters(in: .newlines) {
         pairs.append(contentsOf: arrRangeRecords.map { String($0) })
       }
-      var set = Set<String>()
-      return pairs.filter { set.insert($0).inserted }
+      return pairs.deduplicated
     }
 
     public func hasValuesFor(key: String) -> Bool { rangeMap.keys.contains(key) }