From 872e1aa34b86677cf60b9537c66c81d6bbbae0db Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Wed, 15 May 2024 09:22:23 -0600 Subject: [PATCH 01/13] Modern keyboard: define an overlay class for Windows 11 emoji panel navigation menu item. Define a dedicated overlay class for Windows 11 emoji panel navigation menu items (deriving from UIA list item class) to handle elemenet selected event and others if needed. This keeps the element selectd event handler defined in the app module simple. --- ...omposableshell_experiences_textinput_inputapp.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py index b2a913cff88..9d0de4de890 100644 --- a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py +++ b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py @@ -20,7 +20,7 @@ import config import winVersion import controlTypes -from NVDAObjects.UIA import UIA, XamlEditableText +from NVDAObjects.UIA import UIA, XamlEditableText, ListItem from NVDAObjects.behaviors import CandidateItem as CandidateItemBehavior, EditableTextWithAutoSelectDetection from NVDAObjects import NVDAObject @@ -128,6 +128,17 @@ def event_UIA_elementSelected(self): self.reportFocus() +class NavigationMenuItem(ListItem): + """ + A Windows 11 emoji panel navigation menu item. + In Windows 10 Version 1903 and later, emoji panel can be used to insert emojis, kaomojis, and symbols. + System focus cannot move to these choices in Windows 10 but can do so in Windows 11. + In addition to the choices above, Windows 11 adds GIF and clipboard history to navigation menu. + """ + + pass + + class AppModule(appModuleHandler.AppModule): # Cache the most recently selected item. From 86e16e89c7dc6bbd966f020b4d816eb4e8f0b7ae Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Wed, 15 May 2024 10:26:24 -0600 Subject: [PATCH 02/13] Emoji panel/choose overlay classes: recognize list items with Automation Id string staring with 'navigation-menu-item' as navigation menu item class. Rework list item recognition: if encountering a list item, differentiate between IME candidate items and Windows 11 emoji panel navigation menu items. --- ...bleshell_experiences_textinput_inputapp.py | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py index 9d0de4de890..09bd5a5fc54 100644 --- a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py +++ b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py @@ -364,21 +364,24 @@ def event_gainFocus(self, obj: NVDAObject, nextHandler: Callable[[], None]): def chooseNVDAObjectOverlayClasses(self, obj, clsList): if isinstance(obj, UIA): - if obj.role == controlTypes.Role.LISTITEM and ( - ( - obj.parent.UIAAutomationId in ( - "ExpandedCandidateList", - "TEMPLATE_PART_AdaptiveSuggestionList", + if obj.role == controlTypes.Role.LISTITEM: + if ( + ( + obj.parent.UIAAutomationId in ( + "ExpandedCandidateList", + "TEMPLATE_PART_AdaptiveSuggestionList", + ) + and obj.parent.parent.UIAAutomationId == "IME_Candidate_Window" ) - and obj.parent.parent.UIAAutomationId == "IME_Candidate_Window" - ) - or obj.parent.UIAAutomationId in ( - "IME_Candidate_Window", - "IME_Prediction_Window", - "TEMPLATE_PART_CandidatePanel", - ) - ): - clsList.insert(0, ImeCandidateItem) + or obj.parent.UIAAutomationId in ( + "IME_Candidate_Window", + "IME_Prediction_Window", + "TEMPLATE_PART_CandidatePanel", + ) + ): + clsList.insert(0, ImeCandidateItem) + elif obj.UIAAutomationId.startswith("navigation-menu-item") + clsList.insert(0, NavigationMenuItem) elif ( obj.role in (controlTypes.Role.PANE, controlTypes.Role.LIST, controlTypes.Role.POPUPMENU) and obj.UIAAutomationId in ( From 88ac0922a73631771fc95ed2fdb8f91a1f7d8f3c Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Wed, 15 May 2024 10:29:42 -0600 Subject: [PATCH 03/13] Emoji panel/UIA element selected: let navigation menu item overlay class handle the event. Allow Windows 11 emoji panel navigation menu item overlay class to handle UIA element selected evnt in situations such as element selected event being fired when emoji panel closes in Windows 11. --- ...nal_composableshell_experiences_textinput_inputapp.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py index 09bd5a5fc54..1663abbac4d 100644 --- a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py +++ b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py @@ -149,9 +149,14 @@ class AppModule(appModuleHandler.AppModule): disableBrowseModeByDefault: bool = True def event_UIA_elementSelected(self, obj, nextHandler): - # Logic for IME candidate items is handled all within its own object + # Logic for the following items is handled by overlay classes # Therefore pass these events straight on. - if isinstance(obj, ImeCandidateItem): + if isinstance( + obj, ( + ImeCandidateItem, # IME candidate items + NavigationMenuItem # Windows 11 emoji panel navigation menu items + ) + ): return nextHandler() # #7273: When this is fired on categories, # the first emoji from the new category is selected but not announced. From 2843b364529a4f71b33313aee6e25a26e2061706 Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Wed, 15 May 2024 18:07:46 -0600 Subject: [PATCH 04/13] Windows 11 emoji panel/navigation menu item: ignore element selected event if the event fires when the emoji panel is closed. Re #16346. Define UIA element selected event for navigation menu item overlay class, designed to ignore the event when conditions arise. The first condition is event being fired when closing the emoji panel - check the app modules and ignore the event if the focused app module is not the emoji panel app module, utilizing Python 3.8's assignment expression (:=) syntax (focus object is neeeded in other conditions). --- ...nal_composableshell_experiences_textinput_inputapp.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py index 1663abbac4d..1d50bcd006f 100644 --- a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py +++ b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py @@ -136,7 +136,14 @@ class NavigationMenuItem(ListItem): In addition to the choices above, Windows 11 adds GIF and clipboard history to navigation menu. """ - pass + def event_UIA_elementSelected(self): + # Workarounds for Windows 11 emoji panel category items. + # Ignore the event altogether. + if ( + # #16346: system focus restored. + (focus := api.getFocusObject()).appModule != self.appModule + ): + return class AppModule(appModuleHandler.AppModule): From 813faab90f3086e0f5f2e26b228d70e205612e5f Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Wed, 15 May 2024 18:11:48 -0600 Subject: [PATCH 05/13] Windows 11 emoji panel/navigation menu item: report the selected menu item when apropriate. Re #16346. Report the selected menu item when ignore conditions are dealt with. --- ...wsinternal_composableshell_experiences_textinput_inputapp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py index 1d50bcd006f..e1ec3951a29 100644 --- a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py +++ b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py @@ -144,6 +144,8 @@ def event_UIA_elementSelected(self): (focus := api.getFocusObject()).appModule != self.appModule ): return + # Report the selected navigation menu item. + super().event_UIA_elementSelected() class AppModule(appModuleHandler.AppModule): From 3917efeda0cc5ba340c948d7b71257d466776320 Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Wed, 15 May 2024 18:16:13 -0600 Subject: [PATCH 06/13] Windows 11 emoji panel/navigation menu item: manipulate focus when emoji panel is closed and emoji search field is offscreen. Re #16346. Unlike the event ignore condition, NVDA's focus object stays on emoji edit field when the panel is closed, leaving it offscren. In this case, NVDA does not know where to move the 'focus object' to. Therefore, tell NVDA to queue gain focus event on the system focus control returned by obj.objectWithFocus() method. --- ...ernal_composableshell_experiences_textinput_inputapp.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py index e1ec3951a29..5e155a3c71a 100644 --- a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py +++ b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py @@ -144,6 +144,13 @@ def event_UIA_elementSelected(self): (focus := api.getFocusObject()).appModule != self.appModule ): return + # Manipulate NVDA's focus object. + if ( + # #16346: NVDA is stuck in a nonexistent edit field (location is None). + not any(focus.location) + ): + eventHandler.queueEvent("gainFocus", self.objectWithFocus()) + return # Report the selected navigation menu item. super().event_UIA_elementSelected() From a4b9bc1009e1ec823a2141eadac354da2ef8fd1a Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Wed, 15 May 2024 18:20:25 -0600 Subject: [PATCH 07/13] Winodws 11 emoji panel/navigation menu item: ignore element selected event if menu items are gaining focus. Re #16532. Gain focus event is fired by navigation menu itmes if the panel is open to clipboard history, resulting in repetitions (both gain focus and element selected events are fired for clipboard history item). Ignore the element selected event so gain focus event can announce the selected menu item. --- ...wsinternal_composableshell_experiences_textinput_inputapp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py index 5e155a3c71a..aafc337520a 100644 --- a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py +++ b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py @@ -142,6 +142,8 @@ def event_UIA_elementSelected(self): if ( # #16346: system focus restored. (focus := api.getFocusObject()).appModule != self.appModule + # #16532: repeat announcement due to pending gain focus event on category entries. + or eventHandler.isPendingEvents("gainFocus") ): return # Manipulate NVDA's focus object. From c2031650bd0dda827dfc0c93f2ad78c84e3d3348 Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Wed, 15 May 2024 18:23:39 -0600 Subject: [PATCH 08/13] Windows 11 emoji panel/navigation menu item: ignore the event when reviewing kaomojis and symbols in fullscreen. Re #16533. Gain focus event is fired when moving through kaomoji and symbol entries. When Tab is pressed, both gain focus and elemenet selected events are fired. Therefore, trust the gain focus event. --- ...wsinternal_composableshell_experiences_textinput_inputapp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py index aafc337520a..989a7d9b0f0 100644 --- a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py +++ b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py @@ -144,6 +144,8 @@ def event_UIA_elementSelected(self): (focus := api.getFocusObject()).appModule != self.appModule # #16532: repeat announcement due to pending gain focus event on category entries. or eventHandler.isPendingEvents("gainFocus") + # #16533: system focus is located in GIF/kaomoji/symbol entry. + or focus.UIAAutomationId.startswith("item-") ): return # Manipulate NVDA's focus object. From fc7d87c7562924858d7d51fe0c0ab9c4a5320f79 Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Wed, 15 May 2024 18:28:42 -0600 Subject: [PATCH 09/13] Windows 11 emoji panel/navigation menu item: move NVDA focus to system focus when emoji panel main window unexpected gains focus accompanied by elemenet selected event. Re #16347. A variation of issue 16347: sometimes when closing emoji panel/clipboard history, input experience main window grabs NVDA's focus accompanied with UIA element selected event. The gain focus event handler defined at the app module level only manipulates focus if no events are pending, but inthis case, events were pending. Therefore, catch this and manipulate NVDA focus if NVDA finds itself stuck in emoji panel main window element while handling element selected event. --- ...sinternal_composableshell_experiences_textinput_inputapp.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py index 989a7d9b0f0..9e9513e12a6 100644 --- a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py +++ b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py @@ -152,6 +152,9 @@ def event_UIA_elementSelected(self): if ( # #16346: NVDA is stuck in a nonexistent edit field (location is None). not any(focus.location) + # #16347: focus is once again stuck in top-level modern keyboard window + # after switching to clipboard history from other emoji panel screens. + or focus.firstChild and focus.firstChild.UIAAutomationId == "Windows.Shell.InputApp.FloatingSuggestionUI" ): eventHandler.queueEvent("gainFocus", self.objectWithFocus()) return From 87d09e5c395a59abce74d60f4b511e265c9287b3 Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Wed, 15 May 2024 18:29:24 -0600 Subject: [PATCH 10/13] Windows 11 emoji panel/overlay class chooser (Flake8): add missing colon. --- ...wsinternal_composableshell_experiences_textinput_inputapp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py index 9e9513e12a6..25a8bd10a65 100644 --- a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py +++ b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py @@ -408,7 +408,7 @@ def chooseNVDAObjectOverlayClasses(self, obj, clsList): ) ): clsList.insert(0, ImeCandidateItem) - elif obj.UIAAutomationId.startswith("navigation-menu-item") + elif obj.UIAAutomationId.startswith("navigation-menu-item"): clsList.insert(0, NavigationMenuItem) elif ( obj.role in (controlTypes.Role.PANE, controlTypes.Role.LIST, controlTypes.Role.POPUPMENU) From 201ca947755003b84d4290acdf8c138abdcdba14 Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Wed, 15 May 2024 19:05:03 -0600 Subject: [PATCH 11/13] Changelog: add Windows 11 emoji panel navigation menu entries --- user_docs/en/changes.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/user_docs/en/changes.md b/user_docs/en/changes.md index c1d504990cc..8fdbd319237 100644 --- a/user_docs/en/changes.md +++ b/user_docs/en/changes.md @@ -24,8 +24,10 @@ * Improved reliability of automatic text readout, particularly in terminal applications. (#15850, #16027, @Danstiv) * NVDA will correctly announce selection changes when editing a cell's text in Microsoft Excel. (#15843) * Windows 11 fixes: - * In emoji panel, NVDA will no longer appear to get stuck when closing clipboard history. (#16347, @josephsl) + * In emoji panel, NVDA will no longer appear to get stuck when closing clipboard history and emoji panel. (#16346, #16347, @josephsl) * NVDA will once again announce visible candidates when opening Windows 11 IME interface. (#14023, @josephsl) + * NVDA will no longer announce "clipboard history" twice when reviewing emoji panel navigation menu items. (#16532, @josephsl) + * NVDA will no longer cut off speech and braille while reviewing kaomojis and symbols in emoji panel. (#16533, @josephsl) ### Changes for Developers From ab2108021b0b6b539d91b7368b21ea7cbb4f3b73 Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Wed, 15 May 2024 21:59:40 -0600 Subject: [PATCH 12/13] Changelog: clarity (suggested by Code Rabbit AI) --- user_docs/en/changes.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/user_docs/en/changes.md b/user_docs/en/changes.md index 8fdbd319237..42227d93b4f 100644 --- a/user_docs/en/changes.md +++ b/user_docs/en/changes.md @@ -24,10 +24,10 @@ * Improved reliability of automatic text readout, particularly in terminal applications. (#15850, #16027, @Danstiv) * NVDA will correctly announce selection changes when editing a cell's text in Microsoft Excel. (#15843) * Windows 11 fixes: - * In emoji panel, NVDA will no longer appear to get stuck when closing clipboard history and emoji panel. (#16346, #16347, @josephsl) - * NVDA will once again announce visible candidates when opening Windows 11 IME interface. (#14023, @josephsl) - * NVDA will no longer announce "clipboard history" twice when reviewing emoji panel navigation menu items. (#16532, @josephsl) - * NVDA will no longer cut off speech and braille while reviewing kaomojis and symbols in emoji panel. (#16533, @josephsl) + * NVDA will no longer appear to get stuck when closing the clipboard history and emoji panel. (#16346, #16347, @josephsl) + * NVDA will announce visible candidates again when opening the Windows 11 IME interface. (#14023, @josephsl) + * NVDA will no longer announce "clipboard history" twice when navigating through the emoji panel menu items. (#16532, @josephsl) + * NVDA will no longer cut off speech and braille when reviewing kaomojis and symbols in the emoji panel. (#16533, @josephsl) ### Changes for Developers From 7994efe91cb5e8bd44799c6a3c56a4b6018c52a6 Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Thu, 16 May 2024 15:34:11 -0600 Subject: [PATCH 13/13] Windows 11 emoji panel/navigation menu item: add type hint for element selected event handler --- ...wsinternal_composableshell_experiences_textinput_inputapp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py index 25a8bd10a65..5b64d8c12b8 100644 --- a/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py +++ b/source/appModules/windowsinternal_composableshell_experiences_textinput_inputapp.py @@ -136,7 +136,7 @@ class NavigationMenuItem(ListItem): In addition to the choices above, Windows 11 adds GIF and clipboard history to navigation menu. """ - def event_UIA_elementSelected(self): + def event_UIA_elementSelected(self) -> None: # Workarounds for Windows 11 emoji panel category items. # Ignore the event altogether. if (