UI Test Automation (#39777)

### Summary

This pull request includes the following updates:

1. Improvements and stabilization of the UI automation framework  
2. Setup of the UI automation pipeline  
3. Add UI test cases for FancyZones  
4. Add UI test cases for MouseUtils  
5. Improvements of Hosts Editor UI tests

---

### Related Links

- **Current Release checklist coverage**:  
 
https://github.com/microsoft/PowerToys/blob/feature/UITestAutomation/src/common/UITestAutomation/Doc/ui-automation-cover-list.md

- **UI Automation pipeline**:  
 
https://microsoft.visualstudio.com/Dart/_build?definitionId=161438&_a=summary

---------

Signed-off-by: Shawn Yuan <shuai.yuan.zju@gmail.com>
Signed-off-by: Shawn Yuan <shuaiyuan@microsoft.com>
Co-authored-by: Jerry Xu <n.xu@outlook.com>
Co-authored-by: Zhaopeng Wang <zhaopengwang@microsoft.com>
Co-authored-by: Xiaofeng Wang (from Dev Box) <xiaofengwang@microsoft.com>
Co-authored-by: Mengyuan <162882040+chenmy77@users.noreply.github.com>
Co-authored-by: yaqingmi <miyaqing01@gmail.com>
Co-authored-by: Clint Rutkas <clint@rutkas.com>
Co-authored-by: Yaqing Mi (from Dev Box) <yaqingmi@microsoft.com>
Co-authored-by: Kai Tao <69313318+vanzue@users.noreply.github.com>
Co-authored-by: zhaopeng wang <33367956+wang563681252@users.noreply.github.com>
Co-authored-by: Laszlo Nemeth <57342539+donlaci@users.noreply.github.com>
Co-authored-by: RokyZevon <12629919+RokyZevon@users.noreply.github.com>
Co-authored-by: Yu Leng <42196638+moooyo@users.noreply.github.com>
Co-authored-by: Yu Leng (from Dev Box) <yuleng@microsoft.com>
Co-authored-by: Davide Giacometti <25966642+davidegiacometti@users.noreply.github.com>
Co-authored-by: Gordon Lam <73506701+yeelam-gordon@users.noreply.github.com>
Co-authored-by: ruslanlap <106077551+ruslanlap@users.noreply.github.com>
Co-authored-by: Muhammad Danish <mdanishkhdev@gmail.com>
Co-authored-by: Bennett Blodinger <benwa@users.noreply.github.com>
Co-authored-by: Jaime Bernardo <jaime@janeasystems.com>
Co-authored-by: Ionuț Manța <ionut@janeasystems.com>
Co-authored-by: Hao Liu <liuhaobupt@163.com>
Co-authored-by: OlegHarchevkin <40352094+OlegKharchevkin@users.noreply.github.com>
Co-authored-by: dcog989 <89043002+dcog989@users.noreply.github.com>
Co-authored-by: PesBandi <127593627+PesBandi@users.noreply.github.com>
Co-authored-by: Stefan Markovic <57057282+stefansjfw@users.noreply.github.com>
Co-authored-by: vanzue <vanzue@outlook.com>
Co-authored-by: Typpi <20943337+Nick2bad4u@users.noreply.github.com>
Co-authored-by: Mike Griese <migrie@microsoft.com>
Co-authored-by: Carlos Zamora <carlos.zamora@microsoft.com>
Co-authored-by: Abhyudit <64366765+bitmap4@users.noreply.github.com>
Co-authored-by: Heiko <61519853+htcfreek@users.noreply.github.com>
Co-authored-by: Ved Nig <vednig12@outlook.com>
Co-authored-by: Niels Laute <niels.laute@live.nl>
Co-authored-by: Aung Khaing Khant <aungkhaingkhant.dev@gmail.com>
Co-authored-by: Aung Khaing Khant <aungkhaingkhant@advent-soft.com>
Co-authored-by: Dustin L. Howett <duhowett@microsoft.com>
Co-authored-by: leileizhang <leilzh@microsoft.com>
Co-authored-by: Dustin L. Howett <dustin@howett.net>
Co-authored-by: Shawn Yuan <128874481+shuaiyuanxx@users.noreply.github.com>
Co-authored-by: Shawn Yuan <shuai.yuan.zju@gmail.com>
Co-authored-by: cryolithic <cryolithic@gmail.com>
Co-authored-by: Lemonyte <49930425+lemonyte@users.noreply.github.com>
Co-authored-by: Gordon Lam (SH) <yeelam@microsoft.com>
Co-authored-by: Corey Hayward <72159232+CoreyHayward@users.noreply.github.com>
Co-authored-by: Jerry Xu <nxu@microsoft.com>
Co-authored-by: Shawn Yuan <shuaiyuan@microsoft.com>
Co-authored-by: Kayla Cinnamon <cinnamon@microsoft.com>
Co-authored-by: Jeremy Sinclair <4016293+snickler@users.noreply.github.com>

src/modules/Hosts/Hosts.UITests/HostModuleTests.cs

@@ -13,7 +13,7 @@ namespace Hosts.UITests
     public class HostModuleTests : UITestBase
     {
         public HostModuleTests()
-            : base(PowerToysModule.Hosts)
+            : base(PowerToysModule.Hosts, WindowSize.Small_Vertical)
         {
         }
 
@@ -31,14 +31,17 @@ namespace Hosts.UITests
         /// </item>
         /// </list>
         /// </summary>
-        [TestMethod]
+        [TestMethod("Hosts.Basic.EmptyViewShouldWork")]
+        [TestCategory("Hosts File Editor #4")]
         public void TestEmptyView()
         {
             this.CloseWarningDialog();
             this.RemoveAllEntries();
 
             // 'Add an entry' button (only show-up when list is empty) should be visible
-            Assert.IsTrue(this.FindAll<HyperlinkButton>("Add an entry").Count == 1, "'Add an entry' button should be visible in the empty view");
+            Assert.IsTrue(this.HasOne<HyperlinkButton>("Add an entry"), "'Add an entry' button should be visible in the empty view");
+
+            VisualAssert.AreEqual(this.TestContext, this.Find("Entries"), "EmptyView");
 
             // Click 'Add an entry' from empty-view for adding Host override rule
             this.Find<HyperlinkButton>("Add an entry").Click();
@@ -46,8 +49,10 @@ namespace Hosts.UITests
             this.AddEntry("192.168.0.1", "localhost", false, false);
 
             // Should have one row now and not more empty view
-            Assert.IsTrue(this.FindAll<Button>("Delete").Count == 1, "Should have one row now");
-            Assert.IsTrue(this.FindAll<HyperlinkButton>("Add an entry").Count == 0, "'Add an entry' button should be invisible if not empty view");
+            Assert.IsTrue(this.Has<Button>("Delete"), "Should have one row now");
+            Assert.IsFalse(this.Has<HyperlinkButton>("Add an entry"), "'Add an entry' button should be invisible if not empty view");
+
+            VisualAssert.AreEqual(this.TestContext, this.Find("Entries"), "NonEmptyView");
         }
 
         /// <summary>
@@ -58,17 +63,20 @@ namespace Hosts.UITests
         /// </item>
         /// </list>
         /// </summary>
-        [TestMethod]
+        [TestMethod("Hosts.Basic.AddEntryButtonShouldWork")]
+        [TestCategory("Hosts File Editor #4")]
         public void TestAddingEntry()
         {
             this.CloseWarningDialog();
             this.RemoveAllEntries();
 
-            Assert.IsTrue(this.FindAll<Button>("Delete").Count == 0, "Should have no row after removing all");
+            Assert.IsFalse(this.Has<Button>("Delete"), "Should have no row after removing all");
 
             this.AddEntry("192.168.0.1", "localhost", true);
 
-            Assert.IsTrue(this.FindAll<Button>("Delete").Count == 1, "Should have one row now");
+            Assert.IsTrue(this.Has<Button>("Delete"), "Should have one row now");
+
+            VisualAssert.AreEqual(this.TestContext, this.Find("Entries"));
         }
 
         /// <summary>
@@ -82,7 +90,8 @@ namespace Hosts.UITests
         /// </item>
         /// </list>
         /// </summary>
-        [TestMethod]
+        [TestMethod("Hosts.Basic.CanNotAddMoreThenNighHosts")]
+        [TestCategory("Hosts File Editor #5")]
         public void TestTooManyHosts()
         {
             this.CloseWarningDialog();
@@ -116,18 +125,48 @@ namespace Hosts.UITests
         /// </item>
         /// </list>
         /// </summary>
-        [TestMethod]
+        [TestMethod("Hosts.Basic.ErrorMessageShowupIfNotRunAsAdmin")]
+        [TestCategory("Hosts File Editor #8")]
         public void TestErrorMessageWithNonAdminPermission()
         {
-            this.CloseWarningDialog();
-            this.RemoveAllEntries();
+            if (this.Session.IsElevated == false)
+            {
+                this.CloseWarningDialog();
+                this.RemoveAllEntries();
 
-            // Add new URL override and a warning tip should be shown
-            this.AddEntry("192.168.0.1", "localhost", true);
+                // Add new URL override and a warning tip should be shown
+                this.AddEntry("192.168.0.1", "localhost", true);
 
-            Assert.IsTrue(
-                this.FindAll<TextBlock>("The hosts file cannot be saved because the program isn't running as administrator.").Count == 1,
-                "Should display host-file saving error if not run as administrator");
+                Assert.IsTrue(
+                    this.FindAll<TextBlock>("The hosts file cannot be saved because the program isn't running as administrator.").Count == 1,
+                    "Should display host-file saving error if not run as administrator");
+            }
+        }
+
+        /// <summary>
+        /// Test No Error-message in the Hosts-File-Editor
+        /// <list type="bullet">
+        /// <item>
+        /// <description>Validating error message should be shown if not run as admin.</description>
+        /// </item>
+        /// </list>
+        /// </summary>
+        [TestMethod("Hosts.Basic.NoErrorMessageShowupIfRunAsAdmin")]
+        [TestCategory("Hosts File Editor #8")]
+        public void TestNoErrorMessageWithNonAdminPermission()
+        {
+            if (this.Session.IsElevated == true)
+            {
+                this.CloseWarningDialog();
+                this.RemoveAllEntries();
+
+                // Add new URL override and a warning tip should be shown
+                this.AddEntry("192.168.0.1", "localhost", true);
+
+                Assert.IsFalse(
+                    this.FindAll<TextBlock>("The hosts file cannot be saved because the program isn't running as administrator.").Count > 0,
+                    "Should display host-file saving error if not run as administrator");
+            }
         }
 
         /// <summary>
@@ -144,7 +183,8 @@ namespace Hosts.UITests
         /// </item>
         /// </list>
         /// </summary>
-        [TestMethod]
+        [TestMethod("Hosts.Basic.FiltersControlShouldWork")]
+        [TestCategory("Hosts File Editor #6")]
         public void TestFilterControl()
         {
             this.CloseWarningDialog();
@@ -212,7 +252,7 @@ namespace Hosts.UITests
 
             // Close-filter-panel
             this.Find<Button>("Filters").Click();
-            Assert.IsFalse(this.FindAll<Button>("Clear filters").Count == 1, "Filter panel should be closed afer click Filter Button");
+            Assert.IsFalse(this.FindAll<Button>("Clear filters").Count == 1, "Filter panel should be closed after clicking Filter Button");
         }
 
         private void AddEntry(string ip, string host, bool active = true, bool clickAddEntryButton = true)
@@ -243,25 +283,25 @@ namespace Hosts.UITests
         private void CloseWarningDialog()
         {
             // Find 'Accept' button which come in 'Warning' dialog
-            if (this.FindAll("Warning").Count > 0 &&
-                this.FindAll<Button>("Accept").Count > 0)
+            if (this.FindAll("Warning", 1000).Count > 0 &&
+                this.FindAll<Button>("Accept", 1000).Count > 0)
             {
                 // Hide Warning dialog if any
-                this.Find<Button>("Accept").Click();
+                this.Find<Button>("Accept", 1000).Click();
             }
         }
 
         private void RemoveAllEntries()
         {
             // Delete all existing host-override rules
-            foreach (var deleteBtn in this.FindAll<Button>("Delete"))
+            foreach (var deleteBtn in this.FindAll<Button>("Delete", 1000))
             {
                 deleteBtn.Click();
-                this.Find<Button>("Yes").Click();
+                this.Find<Button>("Yes", 1000).Click();
             }
 
             // Should have no row left, and no more delete button
-            Assert.IsTrue(this.FindAll<Button>("Delete").Count == 0);
+            Assert.IsTrue(this.FindAll<Button>("Delete", 1000).Count == 0);
         }
     }
 }

src/modules/Hosts/Hosts.UITests/Hosts.UITests.csproj

@@ -17,6 +17,17 @@
     <PropertyGroup>
       <OutputPath>$(SolutionDir)$(Platform)\$(Configuration)\tests\Hosts.UITests\</OutputPath>
     </PropertyGroup>
+    <ItemGroup>
+      <EmbeddedResource Include="Baseline\HostModuleTests_TestAddingEntry_arm64.png" />
+      <EmbeddedResource Include="Baseline\HostModuleTests_TestEmptyView_EmptyView_arm64.png" />
+      <EmbeddedResource Include="Baseline\HostModuleTests_TestEmptyView_NonEmptyView_arm64.png" />
+      <EmbeddedResource Include="Baseline\HostModuleTests_TestAddingEntry_x64Win10.png" />
+      <EmbeddedResource Include="Baseline\HostModuleTests_TestEmptyView_EmptyView_x64Win10.png" />
+      <EmbeddedResource Include="Baseline\HostModuleTests_TestEmptyView_NonEmptyView_x64Win10.png" />
+      <EmbeddedResource Include="Baseline\HostModuleTests_TestAddingEntry_x64Win11.png" />
+      <EmbeddedResource Include="Baseline\HostModuleTests_TestEmptyView_EmptyView_x64Win11.png" />
+      <EmbeddedResource Include="Baseline\HostModuleTests_TestEmptyView_NonEmptyView_x64Win11.png" />
+    </ItemGroup>
 
     <ItemGroup>
       <PackageReference Include="MSTest" />

src/modules/Hosts/Hosts.UITests/HostsSettingTests.cs

@@ -12,6 +12,11 @@ namespace Hosts.UITests
     [TestClass]
     public class HostsSettingTests : UITestBase
     {
+        public HostsSettingTests()
+        : base(PowerToysModule.PowerToysSettings, WindowSize.Medium)
+        {
+        }
+
         /// <summary>
         /// Test Warning Dialog at startup
         /// <list type="bullet">
@@ -29,7 +34,9 @@ namespace Hosts.UITests
         /// </item>
         /// </list>
         /// </summary>
-        [TestMethod]
+        [TestMethod("Hosts.Settings.ShowWarningDialogIfRunAsAdmin")]
+        [TestCategory("Hosts File Editor #1")]
+        [TestCategory("Hosts File Editor #9")]
         public void TestWarningDialog()
         {
             this.LaunchFromSetting(showWarning: true);
@@ -51,10 +58,7 @@ namespace Hosts.UITests
 
             this.Find<Button>("Launch Hosts File Editor").Click();
 
-            // wait for 500 ms to make sure Hosts File Editor is launched
-            Task.Delay(500).Wait();
-
-            this.Session.Attach(PowerToysModule.Hosts);
+            this.Session.Attach(PowerToysModule.Hosts, WindowSize.Small_Vertical);
 
             // Should show warning dialog
             Assert.IsTrue(this.FindAll("Warning").Count > 0, "Should show warning dialog");
@@ -68,7 +72,7 @@ namespace Hosts.UITests
             Assert.IsFalse(this.IsHostsFileEditorClosed(), "Hosts File Editor should NOT be closed after click Accept button in Warning Dialog");
 
             // Close Hosts File Editor window
-            this.Session.Find<Window>("Hosts File Editor").Close();
+            this.Session.CloseMainWindow();
 
             // Restore back to PowerToysSettings Session
             this.Session.Attach(PowerToysModule.PowerToysSettings);
@@ -82,7 +86,7 @@ namespace Hosts.UITests
             Assert.IsFalse(this.IsHostsFileEditorClosed(), "Hosts File Editor should NOT be closed");
 
             // Close Hosts File Editor window
-            this.Session.Find<Window>("Hosts File Editor").Close();
+            this.Session.CloseMainWindow();
 
             // Restore back to PowerToysSettings Session
             this.Session.Attach(PowerToysModule.PowerToysSettings);
@@ -90,14 +94,9 @@ namespace Hosts.UITests
 
         private bool IsHostsFileEditorClosed()
         {
-            try
+            if (this.Session.FindAll<Window>("Hosts File Editor").Count == 0 && this.Session.FindAll<Window>("Administrator: Hosts File Editor").Count == 0)
             {
-                this.Session.FindAll<Window>("Hosts File Editor");
-            }
-            catch (Exception ex)
-            {
-                // Validate if editor window closed by checking exception.Message
-                return ex.Message.Contains("Currently selected window has been closed");
+                return true;
             }
 
             return false;
@@ -121,7 +120,7 @@ namespace Hosts.UITests
             // launch Hosts File Editor
             this.Find<Button>("Launch Hosts File Editor").Click();
 
-            Task.Delay(500).Wait();
+            Task.Delay(2000).Wait();
 
             this.Session.Attach(PowerToysModule.Hosts);
         }