[UI Tests] Add complete OCR UI test coverage (#41947)

## Summary of the Pull Request * Enable Text Extractor. Then: - [x] Press the activation shortcut and verify the overlay appears. - [x] Press Escape and verify the overlay disappears. - [x] Press the activation shortcut and verify the overlay appears. - [x] Right-click and select Cancel. Verify the overlay disappears. - [x] Disable Text Extractor and verify that the activation shortuct no longer activates the utility. * With Text Extractor enabled and activated: - [x] Try to select text and verify it is copied to the clipboard. - [x] Try to select a different OCR language by right-clicking and verify the change is applied. * Test the different settings and verify they are applied: - [x] Activation shortcut - [x] OCR Language  ## PR Checklist - [ ] Closes: #xxx - [ ] **Communication:** I've discussed this with core contributors already. If the work hasn't been agreed, this work might be rejected - [ ] **Tests:** Added/updated and all pass - [ ] **Localization:** All end-user-facing strings can be localized - [ ] **Dev docs:** Added/updated - [ ] **New binaries:** Added on the required places - [ ] [JSON for signing](https://github.com/microsoft/PowerToys/blob/main/.pipelines/ESRPSigning_core.json) for new binaries - [ ] [WXS for installer](https://github.com/microsoft/PowerToys/blob/main/installer/PowerToysSetup/Product.wxs) for new binaries and localization folder - [ ] [YML for CI pipeline](https://github.com/microsoft/PowerToys/blob/main/.pipelines/ci/templates/build-powertoys-steps.yml) for new test projects - [ ] [YML for signed pipeline](https://github.com/microsoft/PowerToys/blob/main/.pipelines/release.yml) - [ ] **Documentation updated:** If checked, please file a pull request on [our docs repo](https://github.com/MicrosoftDocs/windows-uwp/tree/docs/hub/powertoys) and link it here: #xxx  ## Detailed Description of the Pull Request / Additional comments  ## Validation Steps Performed
2025-12-15 19:27:56 +01:00 · 2025-11-04 11:59:10 +08:00
parent 70e1177a6a
commit 229bedd09f
4 changed files with 281 additions and 21 deletions
--- a/src/modules/PowerOCR/PowerOCR-UITests/PowerOCRTests.cs
+++ b/src/modules/PowerOCR/PowerOCR-UITests/PowerOCRTests.cs
@@ -2,8 +2,10 @@
 // The Microsoft Corporation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.

+using System;
 using Microsoft.PowerToys.UITest;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
+using OpenQA.Selenium.Interactions;
 using static Microsoft.PowerToys.UITest.UITestBase;

 namespace PowerOCR.UITests;
@@ -19,41 +21,274 @@ public class PowerOCRTests : UITestBase
    [TestInitialize]
    public void TestInitialize()
    {
-        if (FindAll<NavigationViewItem>("Text Extractor").Count == 0)
+        if (FindAll<NavigationViewItem>(By.AccessibilityId("TextExtractorNavItem")).Count == 0)
        {
-            // Expand Advanced list-group if needed
-            Find<NavigationViewItem>("System Tools").Click();
+            // Expand System Tools list-group if needed
+            Find<NavigationViewItem>(By.AccessibilityId("SystemToolsNavItem")).Click();
        }

-        Find<NavigationViewItem>("Text Extractor").Click();
+        Find<NavigationViewItem>(By.AccessibilityId("TextExtractorNavItem")).Click();

-        Find<ToggleSwitch>("Enable Text Extractor").Toggle(true);
+        Find<ToggleSwitch>(By.AccessibilityId("EnableTextExtractorToggleSwitch")).Toggle(true);

-        SendKeys(Key.Win, Key.D);
+        // Reset activation shortcut to default (Win+Shift+T)
+        var shortcutControl = Find<Element>(By.AccessibilityId("TextExtractorActivationShortcut"), 5000);
+        if (shortcutControl != null)
+        {
+            shortcutControl.Click();
+            Thread.Sleep(500);
+
+            // Set default shortcut Win+Shift+T
+            SendKeys(Key.Win, Key.Shift, Key.T);
+            Thread.Sleep(1000);
+
+            // Click Save to confirm
+            var saveButton = Find<Button>(By.Name("Save"), 3000);
+            if (saveButton != null)
+            {
+                saveButton.Click();
+                Thread.Sleep(1000);
+            }
+        }
    }

    [TestMethod("PowerOCR.DetectTextExtractor")]
    [TestCategory("PowerOCR Detection")]
    public void DetectTextExtractorTest()
    {
-        try
+        // Step 1: Press the activation shortcut and verify the overlay appears
+        SendKeys(Key.Win, Key.Shift, Key.T);
+        var textExtractorWindow = Find<Element>(By.AccessibilityId("TextExtractorWindow"), 10000, true);
+        Assert.IsNotNull(textExtractorWindow, "TextExtractor window should be found after hotkey activation");
+
+        // Step 2: Press Escape and verify the overlay disappears
+        SendKeys(Key.Esc);
+        Thread.Sleep(3000);
+
+        var windowsAfterEscape = FindAll<Element>(By.AccessibilityId("TextExtractorWindow"), 2000, true);
+        Assert.AreEqual(0, windowsAfterEscape.Count, "TextExtractor window should be dismissed after pressing Escape");
+
+        // Step 3: Press the activation shortcut again and verify the overlay appears
+        SendKeys(Key.Win, Key.Shift, Key.T);
+
+        textExtractorWindow = Find<Element>(By.AccessibilityId("TextExtractorWindow"), 10000, true);
+        Assert.IsNotNull(textExtractorWindow, "TextExtractor window should appear again after hotkey activation");
+
+        // Step 4: Right-click and select Cancel. Verify the overlay disappears
+        textExtractorWindow.Click(rightClick: true);
+        Thread.Sleep(500);
+
+        // Look for Cancel menu item using its AutomationId
+        var cancelMenuItem = Find<Element>(By.AccessibilityId("CancelMenuItem"), 3000, true);
+        Assert.IsNotNull(cancelMenuItem, "Cancel menu item should be available in context menu");
+
+        cancelMenuItem.Click();
+        Thread.Sleep(3000);
+
+        var windowsAfterCancel = FindAll<Element>(By.AccessibilityId("TextExtractorWindow"), 2000, true);
+        Assert.AreEqual(0, windowsAfterCancel.Count, "TextExtractor window should be dismissed after clicking Cancel");
+    }
+
+    [TestMethod("PowerOCR.DisableTextExtractorTest")]
+    [TestCategory("PowerOCR Settings")]
+    public void DisableTextExtractorTest()
+    {
+        Find<ToggleSwitch>(By.AccessibilityId("EnableTextExtractorToggleSwitch")).Toggle(false);
+
+        SendKeys(Key.Win, Key.Shift, Key.T);
+
+        // Verify that no TextExtractor window appears
+        var windowsWhenDisabled = FindAll<Element>(By.AccessibilityId("TextExtractorWindow"), 10000, true);
+        Assert.AreEqual(0, windowsWhenDisabled.Count, "TextExtractor window should not appear when the utility is disabled");
+    }
+
+    [TestMethod("PowerOCR.ActivationShortcutSettingsTest")]
+    [TestCategory("PowerOCR Settings")]
+    public void ActivationShortcutSettingsTest()
+    {
+        // Find the activation shortcut control
+        var shortcutControl = Find<Element>(By.AccessibilityId("TextExtractorActivationShortcut"), 5000);
+        Assert.IsNotNull(shortcutControl, "Activation shortcut control should be found");
+
+        // Click to focus the shortcut control
+        shortcutControl.Click();
+        Thread.Sleep(500);
+
+        // Test changing the shortcut to Ctrl+Shift+T
+        SendKeys(Key.Ctrl, Key.Shift, Key.T);
+        Thread.Sleep(1000);
+
+        // Click the Save button to confirm the shortcut change
+        var saveButton = Find<Button>(By.Name("Save"), 3000);
+        Assert.IsNotNull(saveButton, "Save button should be found in the shortcut dialog");
+        saveButton.Click();
+        Thread.Sleep(1000);
+
+        // Test the new shortcut
+        SendKeys(Key.Ctrl, Key.Shift, Key.T);
+        Thread.Sleep(3000);
+
+        var textExtractorWindow = FindAll<Element>(By.AccessibilityId("TextExtractorWindow"), 3000, true);
+        Assert.IsTrue(textExtractorWindow.Count > 0, "TextExtractor should activate with new shortcut Ctrl+Shift+T");
+    }
+
+    [TestMethod("PowerOCR.OCRLanguageSettingsTest")]
+    [TestCategory("PowerOCR Settings")]
+    public void OCRLanguageSettingsTest()
+    {
+        // Find the language combo box
+        var languageComboBox = Find<ComboBox>(By.AccessibilityId("TextExtractorLanguageComboBox"), 5000);
+        Assert.IsNotNull(languageComboBox, "Language combo box should be found");
+
+        // Click to open the dropdown
+        languageComboBox.Click();
+
+        // Verify dropdown is opened by checking if dropdown items are available
+        var dropdownItems = FindAll<Element>(By.ClassName("ComboBoxItem"), 2000);
+        Assert.IsTrue(dropdownItems.Count > 0, "Dropdown should contain language options");
+
+        // Close dropdown by pressing Escape
+        SendKeys(Key.Esc);
+    }
+
+    [TestMethod("PowerOCR.OCRLanguageSelectionTest")]
+    [TestCategory("PowerOCR Language")]
+    public void OCRLanguageSelectionTest()
+    {
+        // Activate Text Extractor overlay
+        SendKeys(Key.Win, Key.Shift, Key.T);
+        Thread.Sleep(3000);
+
+        var textExtractorWindow = Find<Element>(By.AccessibilityId("TextExtractorWindow"), 10000, true);
+        Assert.IsNotNull(textExtractorWindow, "TextExtractor window should be found after hotkey activation");
+
+        // Right-click on the canvas to open context menu
+        textExtractorWindow.Click(rightClick: true);
+
+        // Look for language options that should appear after Cancel menu item
+        var allMenuItems = FindAll<Element>(By.ClassName("MenuItem"), 2000, true);
+        if (allMenuItems.Count > 4)
        {
-            SendKeys(Key.Win, Key.Shift, Key.T);
+            // Find the Cancel menu item first
+            Element? cancelItem = null;
+            int cancelIndex = -1;
+            for (int i = 0; i < allMenuItems.Count; i++)
+            {
+                if (allMenuItems[i].GetAttribute("AutomationId") == "CancelMenuItem")
+                {
+                    cancelItem = allMenuItems[i];
+                    cancelIndex = i;
+                    break;
+                }
+            }

-            Thread.Sleep(5000);
+            Assert.IsNotNull(cancelItem, "Cancel menu item should be found");

-            var textExtractorWindow = Find("TextExtractor", 10000, true);
+            // Look for language options after Cancel menu item
+            if (cancelIndex >= 0 && cancelIndex < allMenuItems.Count - 1)
+            {
+                // Select the first language option after Cancel
+                var languageOption = allMenuItems[cancelIndex + 1];
+                languageOption.Click();
+                Thread.Sleep(1000);

-            Assert.IsNotNull(textExtractorWindow, "TextExtractor window should be found after hotkey activation");
-
-            Console.WriteLine("✓ TextExtractor window detected successfully after hotkey activation");
-
-            SendKeys(Key.Esc);
+                Assert.IsTrue(true, "Language selection changed successfully through right-click menu");
+            }
        }
-        catch (Exception ex)
+
+        // Close the TextExtractor overlay
+        SendKeys(Key.Esc);
+        Thread.Sleep(1000);
+    }
+
+    [TestMethod("PowerOCR.TextSelectionAndClipboardTest")]
+    [TestCategory("PowerOCR Selection")]
+    public void TextSelectionAndClipboardTest()
+    {
+        // Clear clipboard first using STA thread
+        ClearClipboardSafely();
+        Thread.Sleep(500);
+
+        // Activate Text Extractor overlay
+        SendKeys(Key.Win, Key.Shift, Key.T);
+        Thread.Sleep(3000);
+
+        var textExtractorWindow = Find<Element>(By.AccessibilityId("TextExtractorWindow"), 10000, true);
+        Assert.IsNotNull(textExtractorWindow, "TextExtractor window should be found after hotkey activation");
+
+        // Click on the TextExtractor window to position cursor
+        textExtractorWindow.Click();
+        Thread.Sleep(500);
+
+        // Get screen dimensions for full screen selection
+        var primaryScreen = System.Windows.Forms.Screen.PrimaryScreen;
+        Assert.IsNotNull(primaryScreen, "Primary screen should be available");
+
+        var screenWidth = primaryScreen.Bounds.Width;
+        var screenHeight = primaryScreen.Bounds.Height;
+
+        // Define full screen selection area
+        var startX = 0;
+        var startY = 0;
+        var endX = screenWidth;
+        var endY = screenHeight;
+
+        // Perform continuous mouse drag to select entire screen
+        PerformSeleniumDrag(startX, startY, endX, endY);
+        Thread.Sleep(3000); // Wait longer for full screen OCR processing
+
+        // Verify text was copied to clipboard using STA thread
+        var clipboardText = GetClipboardTextSafely();
+
+        Assert.IsFalse(string.IsNullOrWhiteSpace(clipboardText), "Clipboard should contain extracted text after selection");
+
+        // Close the TextExtractor overlay
+        SendKeys(Key.Esc);
+        Thread.Sleep(1000);
+    }
+
+    private static void ClearClipboardSafely()
+    {
+        var thread = new System.Threading.Thread(() =>
        {
-            Console.WriteLine($"Failed to detect TextExtractor window: {ex.Message}");
-            Assert.Fail("TextExtractor window was not found after hotkey activation");
-        }
+            System.Windows.Forms.Clipboard.Clear();
+        });
+        thread.SetApartmentState(System.Threading.ApartmentState.STA);
+        thread.Start();
+        thread.Join();
+    }
+
+    private static string GetClipboardTextSafely()
+    {
+        string result = string.Empty;
+        var thread = new System.Threading.Thread(() =>
+        {
+            try
+            {
+                result = System.Windows.Forms.Clipboard.GetText();
+            }
+            catch (Exception)
+            {
+                result = string.Empty;
+            }
+        });
+        thread.SetApartmentState(System.Threading.ApartmentState.STA);
+        thread.Start();
+        thread.Join();
+        return result;
+    }
+
+    private void PerformSeleniumDrag(int startX, int startY, int endX, int endY)
+    {
+        // Use Selenium Actions for proper drag and drop operation
+        var actions = new Actions(Session.Root);
+
+        // Move to start position, click and hold, drag to end position, then release
+        actions.MoveByOffset(startX, startY)
+               .ClickAndHold()
+               .MoveByOffset(endX - startX, endY - startY)
+               .Release()
+               .Build()
+               .Perform();
    }
 }
--- a/src/modules/PowerOCR/PowerOCR-UITests/tests-checklist-text-extractor.md
+++ b/src/modules/PowerOCR/PowerOCR-UITests/tests-checklist-text-extractor.md
@@ -0,0 +1,15 @@
+## Text Extractor
+ * Enable Text Extractor. Then:
+   - [x] Press the activation shortcut and verify the overlay appears.
+   - [x] Press Escape and verify the overlay disappears.
+   - [x] Press the activation shortcut and verify the overlay appears.
+   - [x] Right-click and select Cancel. Verify the overlay disappears.
+   - [x] Disable Text Extractor and verify that the activation shortcut no longer activates the utility.
+ * With Text Extractor enabled and activated:
+   - [x] Try to select text and verify it is copied to the clipboard.
+   - [x] Try to select a different OCR language by right-clicking and verify the change is applied.
+ * In a multi-monitor setup with different DPIs on each monitor:
+   - [ ] Verify text is correctly captured on all monitors.
+ * Test the different settings and verify they are applied:
+   - [x] Activation shortcut
+   - [x] OCR Language
--- a/src/modules/PowerOCR/PowerOCR/OCROverlay.xaml
+++ b/src/modules/PowerOCR/PowerOCR/OCROverlay.xaml
@@ -6,6 +6,7 @@
    xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
    xmlns:p="clr-namespace:PowerOCR.Properties"
    xmlns:ui="http://schemas.lepo.co/wpfui/2022/xaml"
+    x:Name="TextExtractorWindow"
    Title="TextExtractor"
    ui:Design.Background="Transparent"
    AllowsTransparency="True"
@@ -87,6 +88,7 @@
                    <Separator />
                    <MenuItem
                        Name="CancelMenuItem"
+                        AutomationProperties.AutomationId="CancelMenuItem"
                        Click="CancelMenuItem_Click"
                        Header="{x:Static p:Resources.Cancel}" />
                </ContextMenu>
@@ -117,6 +119,7 @@
                <ComboBox
                    x:Name="LanguagesComboBox"
                    Margin="4,0"
+                    AutomationProperties.AutomationId="OCROverlayLanguagesComboBox"
                    AutomationProperties.Name="{x:Static p:Resources.SelectedLang}"
                    SelectionChanged="LanguagesComboBox_SelectionChanged">
                    <ComboBox.ItemTemplate>
--- a/src/settings-ui/Settings.UI/SettingsXAML/Views/PowerOcrPage.xaml
+++ b/src/settings-ui/Settings.UI/SettingsXAML/Views/PowerOcrPage.xaml
@@ -29,7 +29,10 @@
                        Name="TextExtractorEnableToggleControlHeaderText"
                        x:Uid="TextExtractor_EnableToggleControl_HeaderText"
                        HeaderIcon="{ui:BitmapIcon Source=/Assets/Settings/Icons/TextExtractor.png}">
-                        <ToggleSwitch x:Uid="ToggleSwitch" IsOn="{x:Bind ViewModel.IsEnabled, Mode=TwoWay}" />
+                        <ToggleSwitch
+                            x:Uid="ToggleSwitch"
+                            AutomationProperties.AutomationId="EnableTextExtractorToggleSwitch"
+                            IsOn="{x:Bind ViewModel.IsEnabled, Mode=TwoWay}" />
                    </tkcontrols:SettingsCard>
                </controls:GPOInfoControl>
                <InfoBar
@@ -48,12 +51,16 @@
                        Name="ActivationShortcut"
                        x:Uid="Activation_Shortcut"
                        HeaderIcon="{ui:FontIcon Glyph=&#xEDA7;}">
-                        <controls:ShortcutControl MinWidth="{StaticResource SettingActionControlMinWidth}" HotkeySettings="{x:Bind Path=ViewModel.ActivationShortcut, Mode=TwoWay}" />
+                        <controls:ShortcutControl
+                            MinWidth="{StaticResource SettingActionControlMinWidth}"
+                            AutomationProperties.AutomationId="TextExtractorActivationShortcut"
+                            HotkeySettings="{x:Bind Path=ViewModel.ActivationShortcut, Mode=TwoWay}" />
                    </tkcontrols:SettingsCard>
                    <tkcontrols:SettingsCard Name="TextExtractorLanguages" x:Uid="TextExtractor_Languages">
                        <ComboBox
                            x:Name="TextExtractor_ComboBox"
                            MinWidth="{StaticResource SettingActionControlMinWidth}"
+                            AutomationProperties.AutomationId="TextExtractorLanguageComboBox"
                            DropDownOpened="TextExtractor_ComboBox_DropDownOpened"
                            ItemsSource="{x:Bind Path=ViewModel.AvailableLanguages, Mode=OneWay}"
                            Loaded="TextExtractor_ComboBox_Loaded"