From 0e9b5f3acb5200836bcf7a93ef4f7e02b43578d6 Mon Sep 17 00:00:00 2001 From: Gleb Khmyznikov Date: Mon, 1 Dec 2025 15:27:24 -0800 Subject: [PATCH] Try to add screen recording for test results --- .../UITestAutomation/ScreenRecording.cs | 377 ++++++++++++++++++ src/common/UITestAutomation/UITestBase.cs | 92 ++++- 2 files changed, 467 insertions(+), 2 deletions(-) create mode 100644 src/common/UITestAutomation/ScreenRecording.cs diff --git a/src/common/UITestAutomation/ScreenRecording.cs b/src/common/UITestAutomation/ScreenRecording.cs new file mode 100644 index 0000000000..f01895314b --- /dev/null +++ b/src/common/UITestAutomation/ScreenRecording.cs @@ -0,0 +1,377 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Drawing; +using System.Drawing.Imaging; +using System.IO; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.PowerToys.UITest +{ + /// + /// Provides methods for recording the screen during UI tests. + /// Requires FFmpeg to be installed and available in PATH. + /// + internal class ScreenRecording : IDisposable + { + private readonly string outputDirectory; + private readonly string framesDirectory; + private readonly string outputFilePath; + private readonly List capturedFrames; + private readonly SemaphoreSlim recordingLock = new(1, 1); + private readonly Stopwatch recordingStopwatch = new(); + private readonly string? ffmpegPath; + private CancellationTokenSource? recordingCancellation; + private Task? recordingTask; + private bool isRecording; + private int frameCount; + + [DllImport("user32.dll")] + private static extern IntPtr GetDC(IntPtr hWnd); + + [DllImport("gdi32.dll")] + private static extern int GetDeviceCaps(IntPtr hdc, int nIndex); + + [DllImport("user32.dll")] + private static extern IntPtr ReleaseDC(IntPtr hWnd, IntPtr hDC); + + [DllImport("user32.dll")] + private static extern bool GetCursorInfo(out ScreenCapture.CURSORINFO pci); + + [DllImport("user32.dll")] + private static extern bool DrawIconEx(IntPtr hdc, int x, int y, IntPtr hIcon, int cx, int cy, int istepIfAniCur, IntPtr hbrFlickerFreeDraw, int diFlags); + + private const int CURSORSHOWING = 0x00000001; + private const int DESKTOPHORZRES = 118; + private const int DESKTOPVERTRES = 117; + private const int DINORMAL = 0x0003; + private const int TargetFps = 15; // 15 FPS for good balance of quality and size + + /// + /// Initializes a new instance of the class. + /// + /// Directory where the recording will be saved. + public ScreenRecording(string outputDirectory) + { + this.outputDirectory = outputDirectory; + string timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss"); + framesDirectory = Path.Combine(outputDirectory, $"frames_{timestamp}"); + outputFilePath = Path.Combine(outputDirectory, $"recording_{timestamp}.mp4"); + capturedFrames = new List(); + frameCount = 0; + + // Check if FFmpeg is available + ffmpegPath = FindFfmpeg(); + if (ffmpegPath == null) + { + Console.WriteLine("FFmpeg not found. Screen recording will be disabled."); + Console.WriteLine("To enable video recording, install FFmpeg: https://ffmpeg.org/download.html"); + } + } + + /// + /// Gets a value indicating whether screen recording is available (FFmpeg found). + /// + public bool IsAvailable => ffmpegPath != null; + + /// + /// Starts recording the screen. + /// + /// A task representing the asynchronous operation. + public async Task StartRecordingAsync() + { + await recordingLock.WaitAsync(); + try + { + if (isRecording || !IsAvailable) + { + return; + } + + // Create frames directory + Directory.CreateDirectory(framesDirectory); + + recordingCancellation = new CancellationTokenSource(); + isRecording = true; + recordingStopwatch.Start(); + + // Start the recording task + recordingTask = Task.Run(() => RecordFrames(recordingCancellation.Token)); + + Console.WriteLine($"Started screen recording at {TargetFps} FPS"); + } + catch (Exception ex) + { + Console.WriteLine($"Failed to start recording: {ex.Message}"); + isRecording = false; + } + finally + { + recordingLock.Release(); + } + } + + /// + /// Stops recording and encodes video. + /// + /// A task representing the asynchronous operation. + public async Task StopRecordingAsync() + { + await recordingLock.WaitAsync(); + try + { + if (!isRecording || recordingCancellation == null) + { + return; + } + + // Signal cancellation + recordingCancellation.Cancel(); + + // Wait for recording task to complete + if (recordingTask != null) + { + await recordingTask; + } + + recordingStopwatch.Stop(); + isRecording = false; + + double duration = recordingStopwatch.Elapsed.TotalSeconds; + Console.WriteLine($"Recording stopped. Captured {capturedFrames.Count} frames in {duration:F2} seconds"); + + // Encode to video + await EncodeToVideoAsync(); + } + catch (Exception ex) + { + Console.WriteLine($"Error stopping recording: {ex.Message}"); + } + finally + { + Cleanup(); + recordingLock.Release(); + } + } + + /// + /// Records frames from the screen. + /// + private void RecordFrames(CancellationToken cancellationToken) + { + try + { + int frameInterval = 1000 / TargetFps; + var frameTimer = Stopwatch.StartNew(); + + while (!cancellationToken.IsCancellationRequested) + { + var frameStart = frameTimer.ElapsedMilliseconds; + + try + { + CaptureFrame(); + } + catch (Exception ex) + { + Console.WriteLine($"Error capturing frame: {ex.Message}"); + } + + // Sleep for remaining time to maintain target FPS + var frameTime = frameTimer.ElapsedMilliseconds - frameStart; + var sleepTime = Math.Max(0, frameInterval - (int)frameTime); + + if (sleepTime > 0) + { + Thread.Sleep(sleepTime); + } + } + } + catch (OperationCanceledException) + { + // Expected when stopping + } + catch (Exception ex) + { + Console.WriteLine($"Error during recording: {ex.Message}"); + } + } + + /// + /// Captures a single frame. + /// + private void CaptureFrame() + { + IntPtr hdc = GetDC(IntPtr.Zero); + int screenWidth = GetDeviceCaps(hdc, DESKTOPHORZRES); + int screenHeight = GetDeviceCaps(hdc, DESKTOPVERTRES); + ReleaseDC(IntPtr.Zero, hdc); + + Rectangle bounds = new Rectangle(0, 0, screenWidth, screenHeight); + using (Bitmap bitmap = new Bitmap(bounds.Width, bounds.Height, PixelFormat.Format24bppRgb)) + { + using (Graphics g = Graphics.FromImage(bitmap)) + { + g.CopyFromScreen(bounds.Location, Point.Empty, bounds.Size); + + ScreenCapture.CURSORINFO cursorInfo; + cursorInfo.CbSize = Marshal.SizeOf(); + if (GetCursorInfo(out cursorInfo) && cursorInfo.Flags == CURSORSHOWING) + { + IntPtr hdcDest = g.GetHdc(); + DrawIconEx(hdcDest, cursorInfo.PTScreenPos.X, cursorInfo.PTScreenPos.Y, cursorInfo.HCursor, 0, 0, 0, IntPtr.Zero, DINORMAL); + g.ReleaseHdc(hdcDest); + } + } + + string framePath = Path.Combine(framesDirectory, $"frame_{frameCount:D6}.jpg"); + bitmap.Save(framePath, ImageFormat.Jpeg); + capturedFrames.Add(framePath); + frameCount++; + } + } + + /// + /// Encodes captured frames to video using ffmpeg. + /// + private async Task EncodeToVideoAsync() + { + if (capturedFrames.Count == 0) + { + Console.WriteLine("No frames captured"); + return; + } + + try + { + // Build ffmpeg command + string inputPattern = Path.Combine(framesDirectory, "frame_%06d.jpg"); + string args = $"-y -framerate {TargetFps} -i \"{inputPattern}\" -c:v libx264 -pix_fmt yuv420p -crf 23 \"{outputFilePath}\""; + + Console.WriteLine($"Encoding {capturedFrames.Count} frames to video..."); + + var startInfo = new ProcessStartInfo + { + FileName = ffmpegPath!, + Arguments = args, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + + using var process = Process.Start(startInfo); + if (process != null) + { + await process.WaitForExitAsync(); + + if (process.ExitCode == 0 && File.Exists(outputFilePath)) + { + var fileInfo = new FileInfo(outputFilePath); + Console.WriteLine($"Video created: {outputFilePath} ({fileInfo.Length / 1024 / 1024:F1} MB)"); + } + else + { + Console.WriteLine($"FFmpeg encoding failed with exit code {process.ExitCode}"); + } + } + } + catch (Exception ex) + { + Console.WriteLine($"Error encoding video: {ex.Message}"); + } + } + + /// + /// Finds ffmpeg executable. + /// + private static string? FindFfmpeg() + { + // Check if ffmpeg is in PATH + var pathDirs = Environment.GetEnvironmentVariable("PATH")?.Split(Path.PathSeparator) ?? Array.Empty(); + + foreach (var dir in pathDirs) + { + var ffmpegPath = Path.Combine(dir, "ffmpeg.exe"); + if (File.Exists(ffmpegPath)) + { + return ffmpegPath; + } + } + + // Check common installation locations + var commonPaths = new[] + { + @"C:\.tools\ffmpeg\bin\ffmpeg.exe", + @"C:\ffmpeg\bin\ffmpeg.exe", + @"C:\Program Files\ffmpeg\bin\ffmpeg.exe", + @"C:\Program Files (x86)\ffmpeg\bin\ffmpeg.exe", + @$"{Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData)}\Microsoft\WinGet\Links\ffmpeg.exe", + }; + + foreach (var path in commonPaths) + { + if (File.Exists(path)) + { + return path; + } + } + + return null; + } + + /// + /// Gets the path to the recorded video file. + /// + public string OutputFilePath => outputFilePath; + + /// + /// Gets the directory containing recordings. + /// + public string OutputDirectory => outputDirectory; + + /// + /// Cleans up resources. + /// + private void Cleanup() + { + recordingCancellation?.Dispose(); + recordingCancellation = null; + recordingTask = null; + + // Clean up frames directory if it exists + try + { + if (Directory.Exists(framesDirectory)) + { + Directory.Delete(framesDirectory, true); + } + } + catch (Exception ex) + { + Console.WriteLine($"Failed to cleanup frames directory: {ex.Message}"); + } + } + + /// + /// Disposes resources. + /// + public void Dispose() + { + if (isRecording) + { + StopRecordingAsync().GetAwaiter().GetResult(); + } + + Cleanup(); + recordingLock.Dispose(); + GC.SuppressFinalize(this); + } + } +} diff --git a/src/common/UITestAutomation/UITestBase.cs b/src/common/UITestAutomation/UITestBase.cs index 1c72be05f4..2ae270c870 100644 --- a/src/common/UITestAutomation/UITestBase.cs +++ b/src/common/UITestAutomation/UITestBase.cs @@ -29,6 +29,8 @@ namespace Microsoft.PowerToys.UITest public string? ScreenshotDirectory { get; set; } + public string? RecordingDirectory { get; set; } + public static MonitorInfoData.ParamsWrapper MonitorInfoData { get; set; } = new MonitorInfoData.ParamsWrapper() { Monitors = new List() }; private readonly PowerToysModule scope; @@ -36,6 +38,7 @@ namespace Microsoft.PowerToys.UITest private readonly string[]? commandLineArgs; private SessionHelper? sessionHelper; private System.Threading.Timer? screenshotTimer; + private ScreenRecording? screenRecording; public UITestBase(PowerToysModule scope = PowerToysModule.PowerToysSettings, WindowSize size = WindowSize.UnSpecified, string[]? commandLineArgs = null) { @@ -63,14 +66,37 @@ namespace Microsoft.PowerToys.UITest { KeyboardHelper.SendKeys(Key.Win, Key.M); CloseOtherApplications(); - if (IsInPipeline) + if (!IsInPipeline) { - ScreenshotDirectory = Path.Combine(this.TestContext.TestResultsDirectory ?? string.Empty, "UITestScreenshots_" + Guid.NewGuid().ToString()); + string baseDirectory = this.TestContext.TestResultsDirectory ?? string.Empty; + ScreenshotDirectory = Path.Combine(baseDirectory, "UITestScreenshots_" + Guid.NewGuid().ToString()); Directory.CreateDirectory(ScreenshotDirectory); + RecordingDirectory = Path.Combine(baseDirectory, "UITestRecordings_" + Guid.NewGuid().ToString()); + Directory.CreateDirectory(RecordingDirectory); + // Take screenshot every 1 second screenshotTimer = new System.Threading.Timer(ScreenCapture.TimerCallback, ScreenshotDirectory, TimeSpan.Zero, TimeSpan.FromMilliseconds(1000)); + // Start screen recording (requires FFmpeg) + try + { + screenRecording = new ScreenRecording(RecordingDirectory); + if (screenRecording.IsAvailable) + { + _ = screenRecording.StartRecordingAsync(); + } + else + { + screenRecording = null; + } + } + catch (Exception ex) + { + Console.WriteLine($"Failed to start screen recording: {ex.Message}"); + screenRecording = null; + } + // Escape Popups before starting System.Windows.Forms.SendKeys.SendWait("{ESC}"); } @@ -88,6 +114,20 @@ namespace Microsoft.PowerToys.UITest if (IsInPipeline) { screenshotTimer?.Change(Timeout.Infinite, Timeout.Infinite); + + // Stop screen recording + if (screenRecording != null) + { + try + { + screenRecording.StopRecordingAsync().GetAwaiter().GetResult(); + } + catch (Exception ex) + { + Console.WriteLine($"Failed to stop screen recording: {ex.Message}"); + } + } + Dispose(); if (TestContext.CurrentTestOutcome is UnitTestOutcome.Failed or UnitTestOutcome.Error @@ -95,8 +135,14 @@ namespace Microsoft.PowerToys.UITest { Task.Delay(1000).Wait(); AddScreenShotsToTestResultsDirectory(); + AddRecordingsToTestResultsDirectory(); AddLogFilesToTestResultsDirectory(); } + else + { + // Clean up recording if test passed + CleanupRecordingDirectory(); + } } this.Session.Cleanup(); @@ -106,6 +152,7 @@ namespace Microsoft.PowerToys.UITest public void Dispose() { screenshotTimer?.Dispose(); + screenRecording?.Dispose(); GC.SuppressFinalize(this); } @@ -600,6 +647,47 @@ namespace Microsoft.PowerToys.UITest } } + /// + /// Adds screen recordings to test results directory when test fails. + /// + protected void AddRecordingsToTestResultsDirectory() + { + if (RecordingDirectory != null && Directory.Exists(RecordingDirectory)) + { + // Add video files (MP4) + var videoFiles = Directory.GetFiles(RecordingDirectory, "*.mp4"); + foreach (string file in videoFiles) + { + this.TestContext.AddResultFile(file); + var fileInfo = new FileInfo(file); + Console.WriteLine($"Added video recording: {Path.GetFileName(file)} ({fileInfo.Length / 1024 / 1024:F1} MB)"); + } + + if (videoFiles.Length == 0) + { + Console.WriteLine("No video recording available (FFmpeg not found). Screenshots are still captured."); + } + } + } + + /// + /// Cleans up recording directory when test passes. + /// + private void CleanupRecordingDirectory() + { + if (RecordingDirectory != null && Directory.Exists(RecordingDirectory)) + { + try + { + Directory.Delete(RecordingDirectory, true); + } + catch (Exception ex) + { + Console.WriteLine($"Failed to cleanup recording directory: {ex.Message}"); + } + } + } + /// /// Copies PowerToys log files to test results directory when test fails. /// Renames files to include the directory structure after \PowerToys.