Zonble: UserPhrases // UTF8 Surrogate Pair Fix - phase 1

- This already fixed the issue, but we need another commit to prevent disastrous results by similar issues.

Source/Engine/ControllerModules/InputState.swift

@@ -40,180 +40,267 @@ import Cocoa
 /// - Choosing Candidate: The candidate window is open to let the user to choose
 ///   one among the candidates.
 class InputState: NSObject {
+
+    /// Represents that the input controller is deactivated.
+    @objc (InputStateDeactivated)
+    class Deactivated: InputState {
+        override var description: String {
+            "<InputState.Deactivated>"
+        }
+    }
+
+    /// Represents that the composing buffer is empty.
+    @objc (InputStateEmpty)
+    class Empty: InputState {
+        @objc var composingBuffer: String {
+            ""
+        }
+
+        override var description: String {
+            "<InputState.Empty>"
+        }
+    }
+
+    /// Represents that the composing buffer is empty.
+    @objc (InputStateEmptyIgnoringPreviousState)
+    class EmptyIgnoringPreviousState: InputState {
+        @objc var composingBuffer: String {
+            ""
+        }
+        override var description: String {
+            "<InputState.EmptyIgnoringPreviousState>"
+        }
+    }
+
+    /// Represents that the input controller is committing text into client app.
+    @objc (InputStateCommitting)
+    class Committing: InputState {
+        @objc private(set) var poppedText: String = ""
+
+        @objc convenience init(poppedText: String) {
+            self.init()
+            self.poppedText = poppedText
+        }
+
+        override var description: String {
+            "<InputState.Committing poppedText:\(poppedText)>"
+        }
+    }
+    /// Represents that the composing buffer is not empty.
+    @objc (InputStateNotEmpty)
+    class NotEmpty: InputState {
+        @objc private(set) var composingBuffer: String
+        @objc private(set) var cursorIndex: UInt
+        @objc private(set) var phrases: [InputPhrase]
+
+        @objc init(composingBuffer: String, cursorIndex: UInt, phrases: [InputPhrase]) {
+            self.composingBuffer = composingBuffer
+            self.cursorIndex = cursorIndex
+            self.phrases = phrases
+        }
+
+        override var description: String {
+            "<InputState.NotEmpty, composingBuffer:\(composingBuffer), cursorIndex:\(cursorIndex), phrases:\(phrases)>"
+        }
+    }
+
+    /// Represents that the user is inputting text.
+    @objc (InputStateInputting)
+    class Inputting: NotEmpty {
+        @objc var poppedText: String = ""
+
+        @objc override init(composingBuffer: String, cursorIndex: UInt, phrases: [InputPhrase]) {
+            super.init(composingBuffer: composingBuffer, cursorIndex: cursorIndex, phrases: phrases)
+        }
+
+        @objc var attributedString: NSAttributedString {
+            let attributedSting = NSAttributedString(string: composingBuffer, attributes: [
+                .underlineStyle: NSUnderlineStyle.single.rawValue,
+                .markedClauseSegment: 0
+            ])
+            return attributedSting
+        }
+
+        override var description: String {
+            "<InputState.Inputting, composingBuffer:\(composingBuffer), cursorIndex:\(cursorIndex), phrases:\(phrases)>, poppedText:\(poppedText)>"
+        }
+    }
+
+    private let kMinMarkRangeLength = 2
+    private let kMaxMarkRangeLength = 6
+
+    /// Represents that the user is marking a range in the composing buffer.
+    @objc (InputStateMarking)
+    class Marking: NotEmpty {
+
+        @objc private(set) var markerIndex: UInt
+        @objc private(set) var markedRange: NSRange
+        @objc var tooltip: String {
+    
+            if Preferences.phraseReplacementEnabled {
+                return NSLocalizedString("⚠︎ Phrase replacement mode enabled, interfering user phrase entry.", comment: "")
+            }
+    
+            if markedRange.length == 0 {
+                return ""
+            }
+    
+            let text = (composingBuffer as NSString).substring(with: markedRange)
+            if markedRange.length < kMinMarkRangeLength {
+                return String(format: NSLocalizedString("\"%@\" length must ≥ 2 for a user phrase.", comment: ""), text)
+            } else if (markedRange.length > kMaxMarkRangeLength) {
+                return String(format: NSLocalizedString("\"%@\" length should ≤ %d for a user phrase.", comment: ""), text, kMaxMarkRangeLength)
+            }
+            return String(format: NSLocalizedString("\"%@\" selected. ENTER to add user phrase.", comment: ""), text)
+        }
+
+        @objc init(composingBuffer: String, cursorIndex: UInt, markerIndex: UInt, phrases: [InputPhrase]) {
+            self.markerIndex = markerIndex
+            let begin = min(cursorIndex, markerIndex)
+            let end = max(cursorIndex, markerIndex)
+            markedRange = NSMakeRange(Int(begin), Int(end - begin))
+            super.init(composingBuffer: composingBuffer, cursorIndex: cursorIndex, phrases: phrases)
+        }
+
+        @objc var attributedString: NSAttributedString {
+            let attributedSting = NSMutableAttributedString(string: composingBuffer)
+            let end = markedRange.location + markedRange.length
+
+            attributedSting.setAttributes([
+                .underlineStyle: NSUnderlineStyle.single.rawValue,
+                .markedClauseSegment: 0
+            ], range: NSRange(location: 0, length: markedRange.location))
+            attributedSting.setAttributes([
+                .underlineStyle: NSUnderlineStyle.thick.rawValue,
+                .markedClauseSegment: 1
+            ], range: markedRange)
+            attributedSting.setAttributes([
+                .underlineStyle: NSUnderlineStyle.single.rawValue,
+                .markedClauseSegment: 2
+            ], range: NSRange(location: end,
+                              length: (composingBuffer as NSString).length - end))
+            return attributedSting
+        }
+
+        override var description: String {
+            "<InputState.Marking, composingBuffer:\(composingBuffer), cursorIndex:\(cursorIndex), markedRange:\(markedRange), phrases:\(phrases)>"
+        }
+
+        @objc func convertToInputting() -> Inputting {
+            let state = Inputting(composingBuffer: composingBuffer, cursorIndex: cursorIndex, phrases: phrases)
+            return state
+        }
+
+        @objc var validToWrite: Bool {
+            markedRange.length >= kMinMarkRangeLength && markedRange.length <= kMaxMarkRangeLength
+        }
+
+        @objc var userPhrase: String {
+            let text = (composingBuffer as NSString).substring(with: markedRange)
+            let end = markedRange.location + markedRange.length
+            var selectedPhrases = [InputPhrase]()
+            var length = 0
+            for component in self.phrases {
+                if length >= end {
+                    break
+                }
+                if length >= markedRange.location {
+                    selectedPhrases.append(component)
+                }
+                length += (component.text as NSString).length
+            }
+
+            let readings = selectedPhrases.map { $0.reading }
+            let joined = readings.joined(separator: "-")
+            return "\(text) \(joined)"
+        }
+    }
+
+    /// Represents that the user is choosing in a candidates list.
+    @objc (InputStateChoosingCandidate)
+    class ChoosingCandidate: NotEmpty {
+        @objc private(set) var candidates: [String]
+        @objc private(set) var useVerticalMode: Bool
+
+        @objc init(composingBuffer: String, cursorIndex: UInt, candidates: [String], phrases: [InputPhrase], useVerticalMode: Bool) {
+            self.candidates = candidates
+            self.useVerticalMode = useVerticalMode
+            super.init(composingBuffer: composingBuffer, cursorIndex: cursorIndex, phrases: phrases)
+        }
+
+        @objc var attributedString: NSAttributedString {
+            let attributedSting = NSAttributedString(string: composingBuffer, attributes: [
+                .underlineStyle: NSUnderlineStyle.single.rawValue,
+                .markedClauseSegment: 0
+            ])
+            return attributedSting
+        }
+
+        override var description: String {
+            "<InputState.ChoosingCandidate, candidates:\(candidates), useVerticalMode:\(useVerticalMode), phrases:\(phrases),  composingBuffer:\(composingBuffer), cursorIndex:\(cursorIndex)>"
+        }
+    }
+
+    /// Represents that the user is choosing in a candidates list
+    /// in the associated phrases mode.
+    @objc (InputStateAssociatedPhrases)
+    class AssociatedPhrases: InputState {
+        @objc private(set) var candidates: [String] = []
+        @objc private(set) var useVerticalMode: Bool = false
+        @objc init(candidates: [String], useVerticalMode: Bool) {
+            self.candidates = candidates
+            self.useVerticalMode = useVerticalMode
+            super.init()
+        }
+
+        override var description: String {
+            "<InputState.AssociatedPhrases, candidates:\(candidates), useVerticalMode:\(useVerticalMode)>"
+        }
+    }
+
 }
 
-/// Represents that the input controller is deactivated.
+class InputPhrase: NSObject {
-class InputStateDeactivated: InputState {
+    @objc private (set) var text: String
-	override var description: String {
+    @objc private (set) var reading: String
-		"<InputStateDeactivated>"
+
-	}
+    @objc init(text: String, reading: String) {
+        self.text = text
+        self.reading = reading
+        super.init()
+    }
 }
 
-/// Represents that the composing buffer is empty.
+class StringUtils: NSObject {
-class InputStateEmpty: InputState {
+
-	@objc var composingBuffer: String {
+    static func convertToCharIndex(from utf16Index: Int, in string: String) -> Int {
-		""
+        var length = 0
-	}
+        for (i, c) in string.enumerated() {
-}
+            if length >= utf16Index {
-
+                return i
-/// Represents that the composing buffer is empty.
+            }
-class InputStateEmptyIgnoringPreviousState: InputState {
+            length += c.utf16.count
-	@objc var composingBuffer: String {
+        }
-		""
+        return string.count
-	}
+    }
-}
+
-
+    @objc (nextUtf16PositionForIndex:in:)
-/// Represents that the input controller is committing text into client app.
+    static func nextUtf16Position(for index: Int, in string: String) -> Int {
-class InputStateCommitting: InputState {
+        var index = convertToCharIndex(from: index, in: string)
-	@objc private(set) var poppedText: String = ""
+        if index < string.count {
-
+            index += 1
-	@objc convenience init(poppedText: String) {
+        }
-		self.init()
+        let count = string[..<string.index(string.startIndex, offsetBy: index)].utf16.count
-		self.poppedText = poppedText
+        return count
-	}
+    }
-
+
-	override var description: String {
+    @objc (previousUtf16PositionForIndex:in:)
-		"<InputStateCommitting poppedText:\(poppedText)>"
+    static func previousUtf16Position(for index: Int, in string: String) -> Int {
-	}
+        var index = convertToCharIndex(from: index, in: string)
-}
+        if index > 0 {
-
+            index -= 1
-/// Represents that the composing buffer is not empty.
+        }
-class InputStateNotEmpty: InputState {
+        let count = string[..<string.index(string.startIndex, offsetBy: index)].utf16.count
-	@objc private(set) var composingBuffer: String = ""
+        return count
-	@objc private(set) var cursorIndex: UInt = 0
+    }
-
-	@objc init(composingBuffer: String, cursorIndex: UInt) {
-		self.composingBuffer = composingBuffer
-		self.cursorIndex = cursorIndex
-	}
-
-	override var description: String {
-		"<InputStateNotEmpty, composingBuffer:\(composingBuffer), cursorIndex:\(cursorIndex)>"
-	}
-}
-
-/// Represents that the user is inputting text.
-class InputStateInputting: InputStateNotEmpty {
-	@objc var bpmfReading: String = ""
-	@objc var bpmfReadingCursorIndex: UInt8 = 0
-	@objc var poppedText: String = ""
-
-	@objc override init(composingBuffer: String, cursorIndex: UInt) {
-		super.init(composingBuffer: composingBuffer, cursorIndex: cursorIndex)
-	}
-
-	@objc var attributedString: NSAttributedString {
-		let attributedSting = NSAttributedString(string: composingBuffer, attributes: [
-			.underlineStyle: NSUnderlineStyle.single.rawValue,
-			.markedClauseSegment: 0
-		])
-		return attributedSting
-	}
-
-	override var description: String {
-		"<InputStateInputting, composingBuffer:\(composingBuffer), cursorIndex:\(cursorIndex), poppedText:\(poppedText)>"
-	}
-}
-
-private let kMinMarkRangeLength = 2
-private let kMaxMarkRangeLength = Preferences.maxCandidateLength
-
-/// Represents that the user is marking a range in the composing buffer.
-class InputStateMarking: InputStateNotEmpty {
-	@objc private(set) var markerIndex: UInt
-	@objc private(set) var markedRange: NSRange
-	@objc var tooltip: String {
-
-		if Preferences.phraseReplacementEnabled {
-			return NSLocalizedString("⚠︎ Phrase replacement mode enabled, interfering user phrase entry.", comment: "")
-		}
-
-		if markedRange.length == 0 {
-			return ""
-		}
-
-		let text = (composingBuffer as NSString).substring(with: markedRange)
-		if markedRange.length < kMinMarkRangeLength {
-			return String(format: NSLocalizedString("\"%@\" length must ≥ 2 for a user phrase.", comment: ""), text)
-		} else if (markedRange.length > kMaxMarkRangeLength) {
-			return String(format: NSLocalizedString("\"%@\" length should ≤ %d for a user phrase.", comment: ""), text, kMaxMarkRangeLength)
-		}
-		return String(format: NSLocalizedString("\"%@\" selected. ENTER to add user phrase.", comment: ""), text)
-	}
-
-	@objc private(set) var readings: [String] = []
-
-	@objc init(composingBuffer: String, cursorIndex: UInt, markerIndex: UInt, readings: [String]) {
-		self.markerIndex = markerIndex
-		let begin = min(cursorIndex, markerIndex)
-		let end = max(cursorIndex, markerIndex)
-		markedRange = NSMakeRange(Int(begin), Int(end - begin))
-		self.readings = readings
-		super.init(composingBuffer: composingBuffer, cursorIndex: cursorIndex)
-	}
-
-	@objc var attributedString: NSAttributedString {
-		let attributedSting = NSMutableAttributedString(string: composingBuffer)
-		let end = markedRange.location + markedRange.length
-
-		attributedSting.setAttributes([
-			.underlineStyle: NSUnderlineStyle.single.rawValue,
-			.markedClauseSegment: 0
-		], range: NSRange(location: 0, length: markedRange.location))
-		attributedSting.setAttributes([
-			.underlineStyle: NSUnderlineStyle.thick.rawValue,
-			.markedClauseSegment: 1
-		], range: markedRange)
-		attributedSting.setAttributes([
-			.underlineStyle: NSUnderlineStyle.single.rawValue,
-			.markedClauseSegment: 2
-		], range: NSRange(location: end,
-				length: composingBuffer.count - end))
-		return attributedSting
-	}
-
-	override var description: String {
-		"<InputStateMarking, composingBuffer:\(composingBuffer), cursorIndex:\(cursorIndex), markedRange:\(markedRange), readings:\(readings)>"
-	}
-
-	@objc func convertToInputting() -> InputStateInputting {
-		let state = InputStateInputting(composingBuffer: composingBuffer, cursorIndex: cursorIndex)
-		return state
-	}
-
-	@objc var validToWrite: Bool {
-		markedRange.length >= kMinMarkRangeLength && markedRange.length <= kMaxMarkRangeLength
-	}
-
-	@objc var userPhrase: String {
-		let text = (composingBuffer as NSString).substring(with: markedRange)
-		let end = markedRange.location + markedRange.length
-		let readings = readings[markedRange.location..<end]
-		let joined = readings.joined(separator: "-")
-		return "\(text) \(joined)"
-	}
-}
-
-/// Represents that the user is choosing in a candidates list.
-class InputStateChoosingCandidate: InputStateNotEmpty {
-	@objc private(set) var candidates: [String] = []
-	@objc private(set) var useVerticalMode: Bool = false
-
-	@objc init(composingBuffer: String, cursorIndex: UInt, candidates: [String], useVerticalMode: Bool) {
-		self.candidates = candidates
-		self.useVerticalMode = useVerticalMode
-		super.init(composingBuffer: composingBuffer, cursorIndex: cursorIndex)
-	}
-
-	@objc var attributedString: NSAttributedString {
-		let attributedSting = NSAttributedString(string: composingBuffer, attributes: [
-			.underlineStyle: NSUnderlineStyle.single.rawValue,
-			.markedClauseSegment: 0
-		])
-		return attributedSting
-	}
-
-	override var description: String {
-		"<InputStateChoosingCandidate, candidates:\(candidates), useVerticalMode:\(useVerticalMode), composingBuffer:\(composingBuffer), cursorIndex:\(cursorIndex)>"
-	}
 }

Source/Engine/ControllerModules/KeyHandler.mm

@@ -141,13 +141,13 @@ static NSString *const kGraphVizOutputfile = @"/tmp/vChewing-visualization.dot";
     if (self) {
         _bpmfReadingBuffer = new BopomofoReadingBuffer(BopomofoKeyboardLayout::StandardLayout());
 
-		// create the lattice builder
+        // create the lattice builder
-		_languageModel = [LanguageModelManager languageModelCoreCHT];
+        _languageModel = [LanguageModelManager languageModelCoreCHT];
-		_languageModel->setPhraseReplacementEnabled(Preferences.phraseReplacementEnabled);
+        _languageModel->setPhraseReplacementEnabled(Preferences.phraseReplacementEnabled);
-		_languageModel->setCNSEnabled(Preferences.cns11643Enabled);
+        _languageModel->setCNSEnabled(Preferences.cns11643Enabled);
-		_userOverrideModel = [LanguageModelManager userOverrideModelCHT];
+        _userOverrideModel = [LanguageModelManager userOverrideModelCHT];
-		
+        
-		_builder = new BlockReadingBuilder(_languageModel);
+        _builder = new BlockReadingBuilder(_languageModel);
 
         // each Mandarin syllable is separated by a hyphen
         _builder->setJoinSeparator("-");
@@ -220,7 +220,7 @@ static NSString *const kGraphVizOutputfile = @"/tmp/vChewing-visualization.dot";
     }
 
     // if the composing buffer is empty and there's no reading, and there is some function key combination, we ignore it
-	BOOL isFunctionKey = ([input isCommandHold] || [input isOptionHold] || [input isNumericPad]) || [input isControlHotKey];
+    BOOL isFunctionKey = ([input isCommandHold] || [input isOptionHold] || [input isNumericPad]) || [input isControlHotKey];
     if (![state isKindOfClass:[InputStateNotEmpty class]] && isFunctionKey) {
         return NO;
     }
@@ -282,7 +282,7 @@ static NSString *const kGraphVizOutputfile = @"/tmp/vChewing-visualization.dot";
 
     // MARK: Handle BPMF Keys
     // see if it's valid BPMF reading
-	if (![input isControlHold] && _bpmfReadingBuffer->isValidKey((char) charCode)) {
+    if (![input isControlHold] && _bpmfReadingBuffer->isValidKey((char) charCode)) {
         _bpmfReadingBuffer->combineKey((char) charCode);
 
         // if we have a tone marker, we have to insert the reading to the
@@ -364,14 +364,14 @@ static NSString *const kGraphVizOutputfile = @"/tmp/vChewing-visualization.dot";
             // if the spacebar is NOT set to be a selection key
             if ([input isShiftHold] || !Preferences.chooseCandidateUsingSpace) {
                 if (_builder->cursorIndex() >= _builder->length()) {
-					if ([state isKindOfClass:[InputStateNotEmpty class]]) {
+                    if ([state isKindOfClass:[InputStateNotEmpty class]]) {
-						NSString *composingBuffer = [(InputStateNotEmpty *)state composingBuffer];
+                        NSString *composingBuffer = [(InputStateNotEmpty *)state composingBuffer];
-						if ([composingBuffer length]) {
+                        if ([composingBuffer length]) {
-							InputStateCommitting *committing = [[InputStateCommitting alloc] initWithPoppedText:composingBuffer];
+                            InputStateCommitting *committing = [[InputStateCommitting alloc] initWithPoppedText:composingBuffer];
-							stateCallback(committing);
+                            stateCallback(committing);
-						}
+                        }
-					}
+                    }
-					[self clear];
+                    [self clear];
                     InputStateCommitting *committing = [[InputStateCommitting alloc] initWithPoppedText:@" "];
                     stateCallback(committing);
                     InputStateEmpty *empty = [[InputStateEmpty alloc] init];
@@ -457,15 +457,15 @@ static NSString *const kGraphVizOutputfile = @"/tmp/vChewing-visualization.dot";
 
     // MARK: Punctuation
     // if nothing is matched, see if it's a punctuation key for current layout.
-	string punctuationNamePrefix;
+    string punctuationNamePrefix;
-	if ([input isControlHold]) {
+    if ([input isControlHold]) {
-		punctuationNamePrefix = string("_ctrl_punctuation_");
+        punctuationNamePrefix = string("_ctrl_punctuation_");
-	} else if (Preferences.halfWidthPunctuationEnabled) {
+    } else if (Preferences.halfWidthPunctuationEnabled) {
-		punctuationNamePrefix = string("_half_punctuation_");
+        punctuationNamePrefix = string("_half_punctuation_");
-	} else {
+    } else {
-		punctuationNamePrefix = string("_punctuation_");
+        punctuationNamePrefix = string("_punctuation_");
-	}
+    }
-	string layout = [self _currentLayout];
+    string layout = [self _currentLayout];
     string customPunctuation = punctuationNamePrefix + layout + string(1, (char) charCode);
     if ([self _handlePunctuation:customPunctuation state:state usingVerticalMode:input.useVerticalMode stateCallback:stateCallback errorCallback:errorCallback]) {
         return YES;
@@ -543,8 +543,9 @@ static NSString *const kGraphVizOutputfile = @"/tmp/vChewing-visualization.dot";
 
     if ([input isShiftHold]) {
         // Shift + left
-        if (_builder->cursorIndex() > 0) {
+        if (currentState.cursorIndex > 0) {
-            InputStateMarking *marking = [[InputStateMarking alloc] initWithComposingBuffer:currentState.composingBuffer cursorIndex:currentState.cursorIndex markerIndex:currentState.cursorIndex - 1 readings: [self _currentReadings]];
+            NSInteger previousPosition = [StringUtils previousUtf16PositionForIndex:currentState.cursorIndex in:currentState.composingBuffer];
+            InputStateMarking *marking = [[InputStateMarking alloc] initWithComposingBuffer:currentState.composingBuffer cursorIndex:currentState.cursorIndex markerIndex:previousPosition phrases:currentState.phrases];
             stateCallback(marking);
         } else {
             errorCallback();
@@ -579,8 +580,9 @@ static NSString *const kGraphVizOutputfile = @"/tmp/vChewing-visualization.dot";
 
     if ([input isShiftHold]) {
         // Shift + Right
-        if (_builder->cursorIndex() < _builder->length()) {
+        if (currentState.cursorIndex < currentState.composingBuffer.length) {
-            InputStateMarking *marking = [[InputStateMarking alloc] initWithComposingBuffer:currentState.composingBuffer cursorIndex:currentState.cursorIndex markerIndex:currentState.cursorIndex + 1 readings: [self _currentReadings]];
+            NSInteger nextPosition = [StringUtils nextUtf16PositionForIndex:currentState.cursorIndex in:currentState.composingBuffer];
+            InputStateMarking *marking = [[InputStateMarking alloc] initWithComposingBuffer:currentState.composingBuffer cursorIndex:currentState.cursorIndex markerIndex:nextPosition phrases:currentState.phrases];
             stateCallback(marking);
         } else {
             errorCallback();
@@ -806,8 +808,8 @@ static NSString *const kGraphVizOutputfile = @"/tmp/vChewing-visualization.dot";
             && ([input isShiftHold])) {
         NSUInteger index = state.markerIndex;
         if (index > 0) {
-            index -= 1;
+            index = [StringUtils previousUtf16PositionForIndex:index in:state.composingBuffer];
-            InputStateMarking *marking = [[InputStateMarking alloc] initWithComposingBuffer:state.composingBuffer cursorIndex:state.cursorIndex markerIndex:index readings:state.readings];
+            InputStateMarking *marking = [[InputStateMarking alloc] initWithComposingBuffer:state.composingBuffer cursorIndex:state.cursorIndex markerIndex:index phrases:state.phrases];
             stateCallback(marking);
         } else {
             errorCallback();
@@ -821,8 +823,8 @@ static NSString *const kGraphVizOutputfile = @"/tmp/vChewing-visualization.dot";
             && ([input isShiftHold])) {
         NSUInteger index = state.markerIndex;
         if (index < state.composingBuffer.length) {
-            index += 1;
+            index = [StringUtils nextUtf16PositionForIndex:index in:state.composingBuffer];
-            InputStateMarking *marking = [[InputStateMarking alloc] initWithComposingBuffer:state.composingBuffer cursorIndex:state.cursorIndex markerIndex:index readings:state.readings];
+                        InputStateMarking *marking = [[InputStateMarking alloc] initWithComposingBuffer:state.composingBuffer cursorIndex:state.cursorIndex markerIndex:index phrases:state.phrases];
             stateCallback(marking);
         } else {
             errorCallback();
@@ -844,7 +846,7 @@ static NSString *const kGraphVizOutputfile = @"/tmp/vChewing-visualization.dot";
     UniChar charCode = input.charCode;
     VTCandidateController *gCurrentCandidateController = [self.delegate candidateControllerForKeyHandler:self];
 
-    BOOL cancelCandidateKey = (charCode == 27) || (charCode == 8) || [input isDelete]; 
+    BOOL cancelCandidateKey = (charCode == 27) || (charCode == 8) || [input isDelete];
 
     if (cancelCandidateKey) {
         if (Preferences.useWinNT351BPMF) {
@@ -1005,14 +1007,14 @@ static NSString *const kGraphVizOutputfile = @"/tmp/vChewing-visualization.dot";
 
     if (Preferences.useWinNT351BPMF) {
         string layout = [self _currentLayout];
-		string punctuationNamePrefix;
+        string punctuationNamePrefix;
-		if ([input isControlHold]) {
+        if ([input isControlHold]) {
-			punctuationNamePrefix = string("_ctrl_punctuation_");
+            punctuationNamePrefix = string("_ctrl_punctuation_");
-		} else if (Preferences.halfWidthPunctuationEnabled) {
+        } else if (Preferences.halfWidthPunctuationEnabled) {
-			punctuationNamePrefix = string("_half_punctuation_");
+            punctuationNamePrefix = string("_half_punctuation_");
-		} else {
+        } else {
-			punctuationNamePrefix = string("_punctuation_");
+            punctuationNamePrefix = string("_punctuation_");
-		}
+        }
         string customPunctuation = punctuationNamePrefix + layout + string(1, (char) charCode);
         string punctuation = punctuationNamePrefix + string(1, (char) charCode);
 
@@ -1056,6 +1058,8 @@ static NSString *const kGraphVizOutputfile = @"/tmp/vChewing-visualization.dot";
     size_t readingCursorIndex = 0;
     size_t builderCursorIndex = _builder->cursorIndex();
 
+    NSMutableArray <InputPhrase *> *phrases = [[NSMutableArray alloc] init];
+
     // we must do some Unicode codepoint counting to find the actual cursor location for the client
     // i.e. we need to take UTF-16 into consideration, for which a surrogate pair takes 2 UniChars
     // locations
@@ -1068,6 +1072,10 @@ static NSString *const kGraphVizOutputfile = @"/tmp/vChewing-visualization.dot";
             NSString *valueString = [NSString stringWithUTF8String:nodeStr.c_str()];
             [composingBuffer appendString:valueString];
 
+            NSString *readingString = [NSString stringWithUTF8String:(*wi).node->currentKeyValue().key.c_str()];
+            InputPhrase *phrase = [[InputPhrase alloc] initWithText:valueString reading:readingString];
+            [phrases addObject:phrase];
+
             // this re-aligns the cursor index in the composed string
             // (the actual cursor on the screen) with the builder's logical
             // cursor (reading) cursor; each built node has a "spanning length"
@@ -1096,7 +1104,7 @@ static NSString *const kGraphVizOutputfile = @"/tmp/vChewing-visualization.dot";
     NSString *composedText = [head stringByAppendingString:[reading stringByAppendingString:tail]];
     NSInteger cursorIndex = composedStringCursorIndex + [reading length];
 
-    InputStateInputting *newState = [[InputStateInputting alloc] initWithComposingBuffer:composedText cursorIndex:cursorIndex];
+    InputStateInputting *newState = [[InputStateInputting alloc] initWithComposingBuffer:composedText cursorIndex:cursorIndex phrases:phrases];
     return newState;
 }
 
@@ -1166,7 +1174,7 @@ static NSString *const kGraphVizOutputfile = @"/tmp/vChewing-visualization.dot";
         }
     }
 
-    InputStateChoosingCandidate *state = [[InputStateChoosingCandidate alloc] initWithComposingBuffer:currentState.composingBuffer cursorIndex:currentState.cursorIndex candidates:candidatesArray useVerticalMode:useVerticalMode];
+    InputStateChoosingCandidate *state = [[InputStateChoosingCandidate alloc] initWithComposingBuffer:currentState.composingBuffer cursorIndex:currentState.cursorIndex candidates:candidatesArray phrases:currentState.phrases useVerticalMode:useVerticalMode];
     return state;
 }
 

Source/InputMethodController.mm

@@ -443,6 +443,10 @@ static inline NSString *LocalizationNotNeeded(NSString *s) {
 	// some apps (e.g. Twitter for Mac's search bar) handle this call incorrectly, hence the try-catch
 	@try {
 		[client attributesForCharacterIndex:cursor lineHeightRectangle:&lineHeightRect];
+		if ((lineHeightRect.origin.x == 0) && (lineHeightRect.origin.y == 0) && (cursor > 0)) {
+			cursor -= 1; // Zonble's UPR fix: "Corrects the selection range while using Shift + Arrow keys to add new phrases."
+			[client attributesForCharacterIndex:cursor lineHeightRectangle:&lineHeightRect];
+		}
 	}
 	@catch (NSException *exception) {
 		NSLog(@"lineHeightRectangle %@", exception);
@@ -688,6 +692,10 @@ static inline NSString *LocalizationNotNeeded(NSString *s) {
 	// some apps (e.g. Twitter for Mac's search bar) handle this call incorrectly, hence the try-catch
 	@try {
 		[client attributesForCharacterIndex:cursor lineHeightRectangle:&lineHeightRect];
+		if ((lineHeightRect.origin.x == 0) && (lineHeightRect.origin.y == 0) && (cursor > 0)) {
+			cursor -= 1; // Zonble's UPR fix: "Corrects the selection range while using Shift + Arrow keys to add new phrases."
+			[client attributesForCharacterIndex:cursor lineHeightRectangle:&lineHeightRect];
+		}
 	}
 	@catch (NSException *exception) {
 		NSLog(@"%@", exception);