Pre Merge pull request !60 from ShikiSuen/upd/1.8.0sp1

This commit is contained in:
ShikiSuen 2022-07-13 09:09:18 +00:00 committed by Gitee
commit 41ebe3dd54
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
4 changed files with 41 additions and 44 deletions

View File

@ -38,7 +38,12 @@ extension Megrez {
private var langModel: LangModelProtocol private var langModel: LangModelProtocol
/// 0 /// 0
private(set) var cursorRegionMap: [Int: Int] = .init() private(set) var cursorRegionMap: [Int: Int] = .init()
private(set) var walkedAnchors: [Megrez.NodeAnchor] = [] // ///
private(set) var walkedAnchors: [NodeAnchor] = []
///
public func updateWalkedAnchors(with nodes: [Node]) {
walkedAnchors = nodes.map { Megrez.NodeAnchor(node: $0) }
}
/// ///
public var joinSeparator: String = "-" public var joinSeparator: String = "-"
@ -88,7 +93,7 @@ extension Megrez {
/// - separator: /// - separator:
public init(lm: LangModelProtocol, length: Int = 10, separator: String = "-") { public init(lm: LangModelProtocol, length: Int = 10, separator: String = "-") {
langModel = lm langModel = lm
super.init(spanLength: abs(length)) // super.init(spanLengthLimit: abs(length)) //
joinSeparator = separator joinSeparator = separator
} }
@ -181,7 +186,7 @@ extension Megrez {
var paths = [[NodeAnchor]]() var paths = [[NodeAnchor]]()
let nodes = nodesEndingAt(location: location).stableSorted { let nodes = nodesEndingAt(location: location).stableSorted {
$0.scoreForSort > $1.scoreForSort $0.node.score > $1.node.score
} }
guard !nodes.isEmpty else { return .init() } // guard !nodes.isEmpty else { return .init() } //
@ -270,7 +275,7 @@ extension Megrez {
if hasMatchedNode(location: p, spanLength: q, key: combinedReading) { continue } if hasMatchedNode(location: p, spanLength: q, key: combinedReading) { continue }
let unigrams: [Unigram] = langModel.unigramsFor(key: combinedReading) let unigrams: [Unigram] = langModel.unigramsFor(key: combinedReading)
if unigrams.isEmpty { continue } if unigrams.isEmpty { continue }
let n = Node(key: combinedReading, unigrams: unigrams) let n: Node = .init(key: combinedReading, spanLength: q, unigrams: unigrams)
insertNode(node: n, location: p, spanLength: q) insertNode(node: n, location: p, spanLength: q)
} }
} }

View File

@ -41,8 +41,8 @@ extension Megrez {
public var isEmpty: Bool { spans.isEmpty } public var isEmpty: Bool { spans.isEmpty }
/// ///
public init(spanLength: Int = 10) { public init(spanLengthLimit: Int = 10) {
maxBuildSpanLength = spanLength maxBuildSpanLength = spanLengthLimit
spans = [Megrez.SpanUnit]() spans = [Megrez.SpanUnit]()
} }
@ -114,13 +114,7 @@ extension Megrez {
let span = spans[location] let span = spans[location]
for i in 1...maxBuildSpanLength { for i in 1...maxBuildSpanLength {
if let np = span.nodeOf(length: i) { if let np = span.nodeOf(length: i) {
results.append( results.append(.init(node: np))
.init(
node: np,
location: location,
spanLength: i
)
)
} }
} }
return results // return results //
@ -137,13 +131,7 @@ extension Megrez {
let span = spans[i] let span = spans[i]
if i + span.maxLength < location { continue } if i + span.maxLength < location { continue }
if let np = span.nodeOf(length: location - i) { if let np = span.nodeOf(length: location - i) {
results.append( results.append(.init(node: np))
.init(
node: np,
location: i,
spanLength: location - i
)
)
} }
} }
return results // return results //
@ -162,13 +150,7 @@ extension Megrez {
for j in 1...span.maxLength { for j in 1...span.maxLength {
if i + j < location { continue } if i + j < location { continue }
if let np = span.nodeOf(length: j) { if let np = span.nodeOf(length: j) {
results.append( results.append(.init(node: np))
.init(
node: np,
location: i,
spanLength: location - i
)
)
} }
} }
} }
@ -193,7 +175,7 @@ extension Megrez {
@discardableResult public func fixNodeWithCandidateLiteral(_ value: String, at location: Int) -> NodeAnchor { @discardableResult public func fixNodeWithCandidateLiteral(_ value: String, at location: Int) -> NodeAnchor {
let location = abs(location) // let location = abs(location) //
var node = NodeAnchor() var node = NodeAnchor()
for theAnchor in nodesOverlappedAt(location: location) { for theAnchor in nodesCrossingOrEndingAt(location: location) {
let candidates = theAnchor.node.candidates let candidates = theAnchor.node.candidates
// //
theAnchor.node.resetCandidate() theAnchor.node.resetCandidate()
@ -217,7 +199,7 @@ extension Megrez {
@discardableResult public func fixNodeWithCandidate(_ pair: KeyValuePaired, at location: Int) -> NodeAnchor { @discardableResult public func fixNodeWithCandidate(_ pair: KeyValuePaired, at location: Int) -> NodeAnchor {
let location = abs(location) // let location = abs(location) //
var node = NodeAnchor() var node = NodeAnchor()
for theAnchor in nodesOverlappedAt(location: location) { for theAnchor in nodesCrossingOrEndingAt(location: location) {
let candidates = theAnchor.node.candidates let candidates = theAnchor.node.candidates
// //
theAnchor.node.resetCandidate() theAnchor.node.resetCandidate()

View File

@ -30,28 +30,39 @@ extension Megrez {
public var isEmpty: Bool { node.key.isEmpty } public var isEmpty: Bool { node.key.isEmpty }
/// ///
public var node: Node = .init() public var node: Node = .init()
///
public var location: Int = 0
/// ///
public var spanLength: Int = 0 public var spanLength: Int { node.spanLength }
///
public var scoreForSort: Double { node.score }
/// ///
public var mass: Double = 0.0 public var mass: Double = 0.0
///
public var unigrams: [Unigram] { node.unigrams }
///
public var bigrams: [Bigram] { node.bigrams }
///
public var key: String { node.key }
/// ///
public var keyLength: Int { public var keyLength: Int {
isEmpty ? node.key.count : 0 isEmpty ? node.key.count : 0
} }
///
public init(node: Node = .init(), mass: Double? = nil) {
self.node = node
self.mass = mass ?? self.node.score
}
///
public func hash(into hasher: inout Hasher) { public func hash(into hasher: inout Hasher) {
hasher.combine(node) hasher.combine(node)
hasher.combine(location)
hasher.combine(spanLength)
hasher.combine(mass) hasher.combine(mass)
} }
/// ///
public var description: String { public var description: String {
var stream = "" var stream = ""
stream += "{@(" + String(location) + "," + String(spanLength) + ")," stream += "{@(" + String(spanLength) + "),"
if node.key.isEmpty { if node.key.isEmpty {
stream += node.description stream += node.description
} else { } else {
@ -60,11 +71,6 @@ extension Megrez {
stream += "}" stream += "}"
return stream return stream
} }
///
public var scoreForSort: Double {
isEmpty ? node.score : 0
}
} }
} }

View File

@ -30,7 +30,7 @@ extension Megrez {
lhs.key == rhs.key && lhs.score == rhs.score && lhs.unigrams == rhs.unigrams && lhs.bigrams == rhs.bigrams lhs.key == rhs.key && lhs.score == rhs.score && lhs.unigrams == rhs.unigrams && lhs.bigrams == rhs.bigrams
&& lhs.candidates == rhs.candidates && lhs.valueUnigramIndexMap == rhs.valueUnigramIndexMap && lhs.candidates == rhs.candidates && lhs.valueUnigramIndexMap == rhs.valueUnigramIndexMap
&& lhs.precedingBigramMap == rhs.precedingBigramMap && lhs.isCandidateFixed == rhs.isCandidateFixed && lhs.precedingBigramMap == rhs.precedingBigramMap && lhs.isCandidateFixed == rhs.isCandidateFixed
&& lhs.selectedUnigramIndex == rhs.selectedUnigramIndex && lhs.selectedUnigramIndex == rhs.selectedUnigramIndex && lhs.spanLength == rhs.spanLength
} }
public func hash(into hasher: inout Hasher) { public func hash(into hasher: inout Hasher) {
@ -38,6 +38,7 @@ extension Megrez {
hasher.combine(score) hasher.combine(score)
hasher.combine(unigrams) hasher.combine(unigrams)
hasher.combine(bigrams) hasher.combine(bigrams)
hasher.combine(spanLength)
hasher.combine(candidates) hasher.combine(candidates)
hasher.combine(valueUnigramIndexMap) hasher.combine(valueUnigramIndexMap)
hasher.combine(precedingBigramMap) hasher.combine(precedingBigramMap)
@ -50,9 +51,11 @@ extension Megrez {
/// ///
private(set) var score: Double = 0 private(set) var score: Double = 0
/// ///
private var unigrams: [Unigram] private(set) var unigrams: [Unigram]
/// ///
private var bigrams: [Bigram] private(set) var bigrams: [Bigram]
///
public var spanLength: Int = 0
/// ///
private(set) var candidates: [KeyValuePaired] = [] private(set) var candidates: [KeyValuePaired] = []
/// 調 /// 調
@ -83,10 +86,11 @@ extension Megrez {
/// - key: /// - key:
/// - unigrams: /// - unigrams:
/// - bigrams: /// - bigrams:
public init(key: String = "", unigrams: [Megrez.Unigram] = [], bigrams: [Megrez.Bigram] = []) { public init(key: String = "", spanLength: Int = 0, unigrams: [Megrez.Unigram] = [], bigrams: [Megrez.Bigram] = []) {
self.key = key self.key = key
self.unigrams = unigrams self.unigrams = unigrams
self.bigrams = bigrams self.bigrams = bigrams
self.spanLength = spanLength
self.unigrams.sort { self.unigrams.sort {
$0.score > $1.score $0.score > $1.score