From af8d5402c88c206a8ff9dcdb59f0e804e6ae6854 Mon Sep 17 00:00:00 2001
From: Mengyuan <162882040+chenmy77@users.noreply.github.com>
Date: Thu, 17 Apr 2025 16:27:03 +0800
Subject: [PATCH 1/4] [UITestAutomation > KeyBoardHelper] Add PressKey,
ReleaseKey, and KeyDownAndDrag combinationschen/UI automationtools (#38921)
* Add PressKey and ReleaseKey Seperately Add KeyDownAndDrag combination operator in element
---
.../UITestAutomation/Element/Element.cs | 37 +++++++++++++++
src/common/UITestAutomation/KeyboardHelper.cs | 46 +++++++++++++++++++
src/common/UITestAutomation/Session.cs | 24 ++++++++++
src/common/UITestAutomation/UITestBase.cs | 2 +-
4 files changed, 108 insertions(+), 1 deletion(-)
diff --git a/src/common/UITestAutomation/Element/Element.cs b/src/common/UITestAutomation/Element/Element.cs
index a1deac2eeb..865ec86c63 100644
--- a/src/common/UITestAutomation/Element/Element.cs
+++ b/src/common/UITestAutomation/Element/Element.cs
@@ -7,6 +7,7 @@ using System.Drawing;
using System.Runtime.CompilerServices;
using System.Xml.Linq;
using ABI.Windows.Foundation;
+using Microsoft.PowerToys.UITest;
using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using OpenQA.Selenium;
@@ -199,6 +200,42 @@ namespace Microsoft.PowerToys.UITest
});
}
+ ///
+ /// Simulates holding a key, clicking and dragging a UI element to the specified screen coordinates.
+ ///
+ /// The keyboard key to press and hold during the drag operation.
+ /// The target X-coordinate to drag the element to.
+ /// The target Y-coordinate to drag the element to.
+ public void KeyDownAndDrag(Key key, int targetX, int targetY)
+ {
+ PerformAction((actions, windowElement) =>
+ {
+ KeyboardHelper.PressKey(key);
+
+ actions.MoveToElement(windowsElement)
+ .ClickAndHold()
+ .Perform();
+
+ int dx = targetX - windowElement.Rect.X;
+ int dy = targetY - windowElement.Rect.Y;
+
+ int stepCount = 10;
+ int stepX = dx / stepCount;
+ int stepY = dy / stepCount;
+
+ for (int i = 0; i < stepCount; i++)
+ {
+ var stepAction = new Actions(driver);
+ stepAction.MoveByOffset(stepX, stepY).Perform();
+ }
+
+ var releaseAction = new Actions(driver);
+ releaseAction.Release().Perform();
+
+ KeyboardHelper.ReleaseKey(key);
+ });
+ }
+
///
/// Gets the attribute value of the UI element.
///
diff --git a/src/common/UITestAutomation/KeyboardHelper.cs b/src/common/UITestAutomation/KeyboardHelper.cs
index aedaac5caf..1892ab9dd3 100644
--- a/src/common/UITestAutomation/KeyboardHelper.cs
+++ b/src/common/UITestAutomation/KeyboardHelper.cs
@@ -112,6 +112,16 @@ namespace Microsoft.PowerToys.UITest
SendWinKeyCombination(keysToSend);
}
+ public static void PressKey(Key key)
+ {
+ PressVirtualKey(TranslateKeyHex(key));
+ }
+
+ public static void ReleaseKey(Key key)
+ {
+ ReleaseVirtualKey(TranslateKeyHex(key));
+ }
+
///
/// Translates a key to its corresponding SendKeys representation.
///
@@ -260,6 +270,26 @@ namespace Microsoft.PowerToys.UITest
}
}
+ ///
+ /// map the virtual key codes to the corresponding keys.
+ ///
+ private static byte TranslateKeyHex(Key key)
+ {
+ switch (key)
+ {
+ case Key.Win:
+ return 0x5B; // Windows Key - 0x5B in hex
+ case Key.Ctrl:
+ return 0x11; // Ctrl Key - 0x11 in hex
+ case Key.Alt:
+ return 0x12; // Alt Key - 0x12 in hex
+ case Key.Shift:
+ return 0x10; // Shift Key - 0x10 in hex
+ default:
+ throw new ArgumentException($"Key {key} is not supported, Please add your key at TranslateKeyHex for translation to hex.");
+ }
+ }
+
///
/// Sends a combination of keys, including the Windows key, to the system.
///
@@ -283,5 +313,21 @@ namespace Microsoft.PowerToys.UITest
keybd_event(VK_LWIN, 0, KEYEVENTF_KEYUP, UIntPtr.Zero);
}
}
+
+ ///
+ /// Just press the key.(no release)
+ ///
+ private static void PressVirtualKey(byte key)
+ {
+ keybd_event(key, 0, KEYEVENTF_KEYDOWN, UIntPtr.Zero);
+ }
+
+ ///
+ /// Release only the button (if pressed first)
+ ///
+ private static void ReleaseVirtualKey(byte key)
+ {
+ keybd_event(key, 0, KEYEVENTF_KEYUP, UIntPtr.Zero);
+ }
}
}
diff --git a/src/common/UITestAutomation/Session.cs b/src/common/UITestAutomation/Session.cs
index 08fde7160a..e4ee6017e4 100644
--- a/src/common/UITestAutomation/Session.cs
+++ b/src/common/UITestAutomation/Session.cs
@@ -427,6 +427,30 @@ namespace Microsoft.PowerToys.UITest
});
}
+ ///
+ /// release the key (after the hold key and drag is completed.)
+ ///
+ /// The key release.
+ public void PressKey(Key key)
+ {
+ PerformAction(() =>
+ {
+ KeyboardHelper.PressKey(key);
+ });
+ }
+
+ ///
+ /// press and hold the specified key.
+ ///
+ /// The key to press and hold .
+ public void ReleaseKey(Key key)
+ {
+ PerformAction(() =>
+ {
+ KeyboardHelper.ReleaseKey(key);
+ });
+ }
+
///
/// Sends a sequence of keys.
///
diff --git a/src/common/UITestAutomation/UITestBase.cs b/src/common/UITestAutomation/UITestBase.cs
index 610af00d6b..7a0370344d 100644
--- a/src/common/UITestAutomation/UITestBase.cs
+++ b/src/common/UITestAutomation/UITestBase.cs
@@ -538,7 +538,7 @@ namespace Microsoft.PowerToys.UITest
}
else
{
- Console.WriteLine("virtual monitor create faild");
+ Console.WriteLine("virtual monitor create failed");
}
}
From 7f5aaf26dcec1ebc143e143fedda78266cd72273 Mon Sep 17 00:00:00 2001
From: "Xiaofeng Wang (from Dev Box)"
Date: Thu, 17 Apr 2025 16:37:27 +0800
Subject: [PATCH 2/4] Test visual in pipeline
---
src/common/UITestAutomation/VisualAssert.cs | 25 ++++++++++++-------
.../Hosts/Hosts.UITests/HostModuleTests.cs | 6 ++---
2 files changed, 19 insertions(+), 12 deletions(-)
diff --git a/src/common/UITestAutomation/VisualAssert.cs b/src/common/UITestAutomation/VisualAssert.cs
index 280740daca..0f17428d39 100644
--- a/src/common/UITestAutomation/VisualAssert.cs
+++ b/src/common/UITestAutomation/VisualAssert.cs
@@ -5,6 +5,7 @@
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Drawing;
+using System.IO;
using Microsoft.VisualStudio.TestTools.UnitTesting;
namespace Microsoft.PowerToys.UITest
@@ -20,7 +21,7 @@ namespace Microsoft.PowerToys.UITest
/// Element object
/// additional scenario name if two or more scenarios in one test
[RequiresUnreferencedCode("This method uses reflection which may not be compatible with trimming.")]
- public static void AreEqual(TestContext? testContext, Element element, System.Reflection.Assembly callerAssembly, string scenarioSubname = "")
+ public static void AreEqual(TestContext? testContext, Element element, string scenarioSubname = "")
{
if (element == null)
{
@@ -65,21 +66,27 @@ namespace Microsoft.PowerToys.UITest
bool isSame = false;
-#pragma warning disable CS8604 // Possible null reference argument.
- using (var baselineImage = new Bitmap(callerMethod!.DeclaringType!.Assembly.GetManifestResourceStream(baselineImageResourceName)))
+ using (var stream = callerMethod!.DeclaringType!.Assembly.GetManifestResourceStream(baselineImageResourceName))
{
- using (var testImage = new Bitmap(tempTestImagePath))
+ if (stream == null)
{
- isSame = VisualAssert.AreEqual(baselineImage, testImage);
+ Assert.Fail($"Resource stream '{baselineImageResourceName}' is null.");
+ }
- if (!isSame)
+ using (var baselineImage = new Bitmap(stream))
+ {
+ using (var testImage = new Bitmap(tempTestImagePath))
{
- // Copy baseline image to temp folder as well
- baselineImage.Save(tempBaselineImagePath);
+ isSame = VisualAssert.AreEqual(baselineImage, testImage);
+
+ if (!isSame)
+ {
+ // Copy baseline image to temp folder as well
+ baselineImage.Save(tempBaselineImagePath);
+ }
}
}
}
-#pragma warning restore CS8604 // Possible null reference argument.
if (!isSame)
{
diff --git a/src/modules/Hosts/Hosts.UITests/HostModuleTests.cs b/src/modules/Hosts/Hosts.UITests/HostModuleTests.cs
index fd38283581..300f7c236a 100644
--- a/src/modules/Hosts/Hosts.UITests/HostModuleTests.cs
+++ b/src/modules/Hosts/Hosts.UITests/HostModuleTests.cs
@@ -40,7 +40,7 @@ namespace Hosts.UITests
// 'Add an entry' button (only show-up when list is empty) should be visible
Assert.IsTrue(this.HasOne("Add an entry"), "'Add an entry' button should be visible in the empty view");
- // VisualAssert.AreEqual(this.Find("Entries"), "EmptyView");
+ VisualAssert.AreEqual(this.TestContext, this.Find("Entries"), "EmptyView");
// Click 'Add an entry' from empty-view for adding Host override rule
this.Find("Add an entry").Click();
@@ -51,7 +51,7 @@ namespace Hosts.UITests
Assert.IsTrue(this.Has