From 6d29c3a2c90c66b6a3f1d1cc4083d5d177999c9b Mon Sep 17 00:00:00 2001 From: leileizhang Date: Thu, 10 Jul 2025 09:26:26 +0800 Subject: [PATCH] [pipeline] feat: Implement flexible UI test pipeline with configurable build and execution modes (#40490) ## Summary of the Pull Request **Root Cause:** The current pipeline builds the entire solution and runs all UI tests every time, which takes more than 2 hours to complete. **Fix** Make the PowerToys UI test pipeline provides flexible options for building and testing: ### Pipeline Options - **useLatestOfficialBuild**: When checked, downloads the latest official PowerToys build and installs it for testing. This skips the full solution build and only builds UI test projects. - **useCurrentBranchBuild**: When checked along with `useLatestOfficialBuild`, downloads the official build from the current branch instead of main. - **uiTestModules**: Specify which UI test modules to build and run. Examples: - `UITests-FancyZones` - Only FancyZones UI tests - `MouseUtils.UITests` - Only MouseUtils UI tests - `['UITests-FancyZones', 'MouseUtils.UITests']` - Multiple specific modules - Leave empty to build and run all UI test modules ### Build Modes 1. **Official Build + Selective Testing** (`useLatestOfficialBuild = true`) - Downloads and installs official PowerToys build - Builds only specified UI test projects - Runs specified UI tests against installed PowerToys - Controlled by `uiTestModules` parameter 2. **Full Build + Testing** (`useLatestOfficialBuild = false`) - Builds entire PowerToys solution - Builds UI test projects (all or specific based on `uiTestModules`) - Runs UI tests (all or specific based on `uiTestModules`) - Uses freshly built PowerToys for testing ## PR Checklist - [ ] **Closes:** #xxx - [ ] **Communication:** I've discussed this with core contributors already. If the work hasn't been agreed, this work might be rejected - [x] **Tests:** Added/updated and all pass - [ ] **Localization:** All end-user-facing strings can be localized - [x] **Dev docs:** Added/updated - [ ] **New binaries:** Added on the required places - [ ] [JSON for signing](https://github.com/microsoft/PowerToys/blob/main/.pipelines/ESRPSigning_core.json) for new binaries - [ ] [WXS for installer](https://github.com/microsoft/PowerToys/blob/main/installer/PowerToysSetup/Product.wxs) for new binaries and localization folder - [ ] [YML for CI pipeline](https://github.com/microsoft/PowerToys/blob/main/.pipelines/ci/templates/build-powertoys-steps.yml) for new test projects - [ ] [YML for signed pipeline](https://github.com/microsoft/PowerToys/blob/main/.pipelines/release.yml) - [ ] **Documentation updated:** If checked, please file a pull request on [our docs repo](https://github.com/MicrosoftDocs/windows-uwp/tree/docs/hub/powertoys) and link it here: #xxx ## Detailed Description of the Pull Request / Additional comments ## Validation Steps Performed --- .github/actions/spell-check/expect.txt | 4 + .pipelines/installPowertoys.ps1 | 46 ++++++ .../v2/templates/job-build-ui-tests.yml | 132 ++++++++++++++++ .pipelines/v2/templates/job-test-project.yml | 112 +++++++++++--- .../pipeline-ui-tests-automation.yml | 145 ++++++++++++++---- doc/devdocs/UITests.md | 50 ++++++ .../UITestAutomation/ModuleConfigData.cs | 81 +++++++--- src/common/UITestAutomation/ModuleInfo.cs | 54 +++++++ src/common/UITestAutomation/SessionHelper.cs | 9 +- .../Hosts/Hosts.UITests/Hosts.UITests.csproj | 2 +- 10 files changed, 562 insertions(+), 73 deletions(-) create mode 100644 .pipelines/installPowertoys.ps1 create mode 100644 .pipelines/v2/templates/job-build-ui-tests.yml create mode 100644 src/common/UITestAutomation/ModuleInfo.cs diff --git a/.github/actions/spell-check/expect.txt b/.github/actions/spell-check/expect.txt index 1917f2b4fb..f8ebc8636b 100644 --- a/.github/actions/spell-check/expect.txt +++ b/.github/actions/spell-check/expect.txt @@ -76,6 +76,7 @@ ARPINSTALLLOCATION ARPPRODUCTICON ARRAYSIZE ARROWKEYS +ARTIFACTSTAGINGDIRECTORY asf Ashcraft AShortcut @@ -1722,6 +1723,8 @@ UFlags UHash UIA UIEx +uild +uitests ULONGLONG ums uncompilable @@ -1748,6 +1751,7 @@ Uptool urld Usb USEDEFAULT +USEINSTALLERFORTEST USEFILEATTRIBUTES USESHOWWINDOW USESTDHANDLES diff --git a/.pipelines/installPowertoys.ps1 b/.pipelines/installPowertoys.ps1 new file mode 100644 index 0000000000..26abc5271f --- /dev/null +++ b/.pipelines/installPowertoys.ps1 @@ -0,0 +1,46 @@ +param( + [Parameter()] + [ValidateSet("Machine", "PerUser")] + [string]$InstallMode = "Machine" +) + +$ProgressPreference = 'SilentlyContinue' + +# Get artifact path +$ArtifactPath = $ENV:BUILD_ARTIFACTSTAGINGDIRECTORY +if (-not $ArtifactPath) { + throw "BUILD_ARTIFACTSTAGINGDIRECTORY environment variable not set" +} + +# Since we only download PowerToysSetup-*.exe files, we can directly find it +$Installer = Get-ChildItem -Path $ArtifactPath -Filter 'PowerToys*.exe' | Select-Object -First 1 + +if (-not $Installer) { + throw "PowerToys installer not found" +} + +Write-Host "Installing PowerToys: $($Installer.Name)" + +# Install PowerToys +$Process = Start-Process -Wait -FilePath $Installer.FullName -ArgumentList "/passive", "/norestart" -PassThru -NoNewWindow + +if ($Process.ExitCode -eq 0 -or $Process.ExitCode -eq 3010) { + Write-Host "✅ PowerToys installation completed successfully" +} else { + throw "PowerToys installation failed with exit code: $($Process.ExitCode)" +} + +# Verify installation +if ($InstallMode -eq "PerUser") { + if (Test-Path "${env:LOCALAPPDATA}\PowerToys\PowerToys.exe") { + Write-Host "✅ PowerToys verified at: ${env:LOCALAPPDATA}\PowerToys\PowerToys.exe" + } else { + throw "PowerToys installation verification failed" + } +} else { + if (Test-Path "${env:ProgramFiles}\PowerToys\PowerToys.exe") { + Write-Host "✅ PowerToys verified at: ${env:ProgramFiles}\PowerToys\PowerToys.exe" + } else { + throw "PowerToys installation verification failed" + } +} diff --git a/.pipelines/v2/templates/job-build-ui-tests.yml b/.pipelines/v2/templates/job-build-ui-tests.yml new file mode 100644 index 0000000000..ca99c00932 --- /dev/null +++ b/.pipelines/v2/templates/job-build-ui-tests.yml @@ -0,0 +1,132 @@ +# Minimal UI Tests Build Template +# This template only builds UI test projects and stages their test DLLs for consumption by test pipelines + +parameters: + - name: buildConfigurations + type: object + default: + - Release + - name: buildPlatforms + type: object + default: + - x64 + - name: condition + type: string + default: '' + - name: dependsOn + type: object + default: [] + - name: pool + type: object + default: [] + - name: variables + type: object + default: {} + - name: uiTestModules + type: object + default: [] + +jobs: +- job: BuildUITests + ${{ if ne(length(parameters.pool), 0) }}: + pool: ${{ parameters.pool }} + dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} + strategy: + matrix: + ${{ each config in parameters.buildConfigurations }}: + ${{ each platform in parameters.buildPlatforms }}: + ${{ config }}_${{ platform }}: + BuildConfiguration: ${{ config }} + BuildPlatform: ${{ platform }} + variables: + JobOutputDirectory: $(Build.ArtifactStagingDirectory) + LogOutputDirectory: $(Build.ArtifactStagingDirectory)\logs + JobOutputArtifactName: build-$(BuildPlatform)-$(BuildConfiguration) + NUGET_RESTORE_MSBUILD_ARGS: /p:Platform=$(BuildPlatform) + ${{ insert }}: ${{ parameters.variables }} + displayName: Build UI Tests Only + timeoutInMinutes: 60 + cancelTimeoutInMinutes: 1 + templateContext: + outputs: + - output: pipelineArtifact + artifactName: $(JobOutputArtifactName) + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - checkout: self + clean: true + submodules: true + persistCredentials: True + fetchTags: false + fetchDepth: 1 + + - template: steps-ensure-dotnet-version.yml + parameters: + sdk: true + version: '9.0' + + - template: .\steps-restore-nuget.yml + + - task: NuGetCommand@2 + displayName: Restore solution-level NuGet packages + inputs: + command: restore + feedsToUse: config + configPath: nuget.config + restoreSolution: PowerToys.sln + restoreDirectory: '$(Build.SourcesDirectory)\packages' + + # Build all UI test projects if no specific modules are specified + - ${{ if eq(length(parameters.uiTestModules), 0) }}: + - task: VSBuild@1 + displayName: Build UI Test Projects + inputs: + solution: '**/*UITest*.csproj' + vsVersion: 17.0 + msbuildArgs: >- + -restore + -graph + /p:RestorePackagesConfig=true + /p:BuildProjectReferences=true + /p:CIBuild=true + /bl:$(LogOutputDirectory)\build-all-uitests.binlog + $(NUGET_RESTORE_MSBUILD_ARGS) + platform: $(BuildPlatform) + configuration: $(BuildConfiguration) + msbuildArchitecture: x64 + maximumCpuCount: true + + # Build specific UI test modules + - ${{ if ne(length(parameters.uiTestModules), 0) }}: + - ${{ each module in parameters.uiTestModules }}: + - task: VSBuild@1 + displayName: 'Build UI Test Module: ${{ module }}' + inputs: + solution: '**/*${{ module }}*.csproj' + vsVersion: 17.0 + msbuildArgs: >- + -restore + -graph + /p:RestorePackagesConfig=true + /p:BuildProjectReferences=true + /p:CIBuild=true + /bl:$(LogOutputDirectory)\build-${{ module }}.binlog + $(NUGET_RESTORE_MSBUILD_ARGS) + platform: $(BuildPlatform) + configuration: $(BuildConfiguration) + msbuildArchitecture: x64 + maximumCpuCount: true + + # Stage test project outputs with directory structure + - task: CopyFiles@2 + displayName: Stage UI Test Build Outputs + inputs: + sourceFolder: '$(Build.SourcesDirectory)' + contents: '$(BuildPlatform)/$(BuildConfiguration)/**/*' + targetFolder: '$(JobOutputDirectory)\$(BuildPlatform)\$(BuildConfiguration)' + + - publish: $(JobOutputDirectory) + artifact: $(JobOutputArtifactName) + displayName: Publish UI Test artifacts + condition: always() \ No newline at end of file diff --git a/.pipelines/v2/templates/job-test-project.yml b/.pipelines/v2/templates/job-test-project.yml index ab682cd5a5..ec850b6dfe 100644 --- a/.pipelines/v2/templates/job-test-project.yml +++ b/.pipelines/v2/templates/job-test-project.yml @@ -11,10 +11,28 @@ parameters: - name: useLatestWebView2 type: boolean default: false + - name: useLatestOfficialBuild + type: boolean + default: true + - name: useCurrentBranchBuild + type: boolean + default: false + - name: uiTestModules + type: object + default: [] + - name: installMode + type: string + default: 'machine' + values: + - 'machine' + - 'peruser' + - name: jobSuffix + type: string + default: '' jobs: -- job: Test${{ parameters.platform }}${{ parameters.configuration }} - displayName: Test ${{ parameters.platform }} ${{ parameters.configuration }} +- job: Test${{ parameters.platform }}${{ parameters.configuration }}${{ parameters.jobSuffix }} + displayName: Test ${{ parameters.platform }} ${{ parameters.configuration }}${{ parameters.jobSuffix }} timeoutInMinutes: 300 variables: ${{ if or(eq(parameters.platform, 'x64Win10'), eq(parameters.platform, 'x64Win11')) }}: @@ -95,28 +113,80 @@ jobs: & '$(build.sourcesdirectory)\.pipelines\InstallWinAppDriver.ps1' displayName: Download and install WinAppDriver + - ${{ if eq(parameters.useLatestOfficialBuild, true) }}: + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'specific' + project: 'Dart' + definition: '76541' + buildVersionToDownload: 'latestFromBranch' + ${{ if eq(parameters.useCurrentBranchBuild, true) }}: + branchName: '$(Build.SourceBranch)' + ${{ else }}: + branchName: 'refs/heads/main' + artifactName: 'build-$(BuildPlatform)-Release' + targetPath: '$(Build.ArtifactStagingDirectory)' + ${{ if eq(parameters.installMode, 'peruser') }}: + patterns: | + **/PowerToysUserSetup*.exe + ${{ else }}: + patterns: | + **/PowerToysSetup*.exe + + - ${{ if eq(parameters.useLatestOfficialBuild, true) }}: + - ${{ if eq(parameters.installMode, 'peruser') }}: + - pwsh: |- + & "$(build.sourcesdirectory)\.pipelines\installPowerToys.ps1" -InstallMode "PerUser" + displayName: Install PowerToys (Per-User) + + - ${{ if eq(parameters.installMode, 'machine') }}: + - pwsh: |- + & "$(build.sourcesdirectory)\.pipelines\installPowerToys.ps1" -InstallMode "Machine" + displayName: Install PowerToys (Machine-Level) + - ${{ if ne(parameters.platform, 'arm64') }}: - task: ScreenResolutionUtility@1 inputs: displaySettings: 'optimal' - - task: VSTest@3 - displayName: Run UI Tests - inputs: - platform: '$(BuildPlatform)' - configuration: '$(BuildConfiguration)' - testSelector: 'testAssemblies' - searchFolder: '$(Pipeline.Workspace)\$(TestArtifactsName)' - vsTestVersion: 'toolsInstaller' - uiTests: true - rerunFailedTests: true - # Since UITests-FancyZonesEditor.dll is generated in both UITests-FancyZonesEditor and UITests-FancyZones, removed one to avoid duplicate test runs - testAssemblyVer2: | - **\*UITest*.dll - !**\obj\** - !**\ref\** - !**\UITests-FancyZones\**\UITests-FancyZonesEditor.dll + - ${{ if eq(length(parameters.uiTestModules), 0) }}: + - task: VSTest@3 + displayName: Run UI Tests + inputs: + platform: '$(BuildPlatform)' + configuration: '$(BuildConfiguration)' + testSelector: 'testAssemblies' + searchFolder: '$(Pipeline.Workspace)\$(TestArtifactsName)' + vsTestVersion: 'toolsInstaller' + uiTests: true + rerunFailedTests: true + # Since UITests-FancyZonesEditor.dll is generated in both UITests-FancyZonesEditor and UITests-FancyZones, removed one to avoid duplicate test runs + testAssemblyVer2: | + **\*UITest*.dll + !**\obj\** + !**\ref\** + !**\UITests-FancyZones\**\UITests-FancyZonesEditor.dll + env: + platform: '$(TestPlatform)' + useInstallerForTest: ${{ parameters.useLatestOfficialBuild }} - - env: - platform: '$(TestPlatform)' + - ${{ if ne(length(parameters.uiTestModules), 0) }}: + - ${{ each module in parameters.uiTestModules }}: + - task: VSTest@3 + displayName: Run UI Test - ${{ module }} + inputs: + platform: '$(BuildPlatform)' + configuration: '$(BuildConfiguration)' + testSelector: 'testAssemblies' + searchFolder: '$(Pipeline.Workspace)\$(TestArtifactsName)' + vsTestVersion: 'toolsInstaller' + uiTests: true + rerunFailedTests: true + testAssemblyVer2: | + **\*${{ module }}*.dll + !**\obj\** + !**\ref\** + !**\UITests-FancyZones\**\UITests-FancyZonesEditor.dll + env: + platform: '$(TestPlatform)' + useInstallerForTest: ${{ parameters.useLatestOfficialBuild }} diff --git a/.pipelines/v2/templates/pipeline-ui-tests-automation.yml b/.pipelines/v2/templates/pipeline-ui-tests-automation.yml index 4db1005f3b..628a691fed 100644 --- a/.pipelines/v2/templates/pipeline-ui-tests-automation.yml +++ b/.pipelines/v2/templates/pipeline-ui-tests-automation.yml @@ -22,63 +22,154 @@ parameters: - name: useLatestWebView2 type: boolean default: false + - name: useLatestOfficialBuild + type: boolean + default: true + - name: testBothInstallModes + type: boolean + default: true + - name: useCurrentBranchBuild + type: boolean + default: false + - name: uiTestModules + type: object + default: [] stages: - ${{ each platform in parameters.buildPlatforms }}: - - stage: Build_${{ platform }} - displayName: Build ${{ platform }} - dependsOn: [] - jobs: - - template: job-build-project.yml - parameters: - pool: - ${{ if eq(variables['System.CollectionId'], 'cb55739e-4afe-46a3-970f-1b49d8ee7564') }}: - name: SHINE-INT-L - ${{ else }}: - name: SHINE-OSS-L - ${{ if eq(parameters.useVSPreview, true) }}: - demands: ImageOverride -equals SHINE-VS17-Preview - buildPlatforms: - - ${{ platform }} - buildConfigurations: [Release] - enablePackageCaching: true - enableMsBuildCaching: ${{ parameters.enableMsBuildCaching }} - runTests: false - buildTests: true - useVSPreview: ${{ parameters.useVSPreview }} + - ${{ if eq(parameters.useLatestOfficialBuild, false) }}: + - stage: Build_${{ platform }} + displayName: Build ${{ platform }} + dependsOn: [] + jobs: + - template: job-build-project.yml + parameters: + pool: + ${{ if eq(variables['System.CollectionId'], 'cb55739e-4afe-46a3-970f-1b49d8ee7564') }}: + name: SHINE-INT-L + ${{ else }}: + name: SHINE-OSS-L + ${{ if eq(parameters.useVSPreview, true) }}: + demands: ImageOverride -equals SHINE-VS17-Preview + buildPlatforms: + - ${{ platform }} + buildConfigurations: [Release] + enablePackageCaching: true + enableMsBuildCaching: ${{ parameters.enableMsBuildCaching }} + runTests: false + buildTests: true + useVSPreview: ${{ parameters.useVSPreview }} + + - ${{ if eq(parameters.useLatestOfficialBuild, true) }}: + - stage: BuildUITests_${{ platform }} + displayName: Build UI Tests Only + dependsOn: [] + jobs: + - template: job-build-ui-tests.yml + parameters: + pool: + ${{ if eq(variables['System.CollectionId'], 'cb55739e-4afe-46a3-970f-1b49d8ee7564') }}: + name: SHINE-INT-L + ${{ else }}: + name: SHINE-OSS-L + ${{ if eq(parameters.useVSPreview, true) }}: + demands: ImageOverride -equals SHINE-VS17-Preview + buildPlatforms: + - ${{ platform }} + uiTestModules: ${{ parameters.uiTestModules }} - ${{ if eq(platform, 'x64') }}: - stage: Test_x64Win10 displayName: Test x64Win10 - dependsOn: - - Build_${{platform}} + ${{ if eq(parameters.useLatestOfficialBuild, true) }}: + dependsOn: + - BuildUITests_${{ platform }} + ${{ else }}: + dependsOn: + - Build_${{ platform }} jobs: - template: job-test-project.yml parameters: platform: x64Win10 configuration: Release useLatestWebView2: ${{ parameters.useLatestWebView2 }} + useLatestOfficialBuild: ${{ parameters.useLatestOfficialBuild }} + useCurrentBranchBuild: ${{ parameters.useCurrentBranchBuild }} + uiTestModules: ${{ parameters.uiTestModules }} + + # Additional per-user installation test (when both modes are enabled) + - ${{ if and(eq(parameters.useLatestOfficialBuild, true), eq(parameters.testBothInstallModes, true)) }}: + - template: job-test-project.yml + parameters: + platform: x64Win10 + configuration: Release + useLatestWebView2: ${{ parameters.useLatestWebView2 }} + useLatestOfficialBuild: ${{ parameters.useLatestOfficialBuild }} + useCurrentBranchBuild: ${{ parameters.useCurrentBranchBuild }} + uiTestModules: ${{ parameters.uiTestModules }} + installMode: 'peruser' + jobSuffix: '_PerUser' - ${{ if eq(platform, 'x64') }}: - stage: Test_x64Win11 displayName: Test x64Win11 - dependsOn: - - Build_${{platform}} + ${{ if eq(parameters.useLatestOfficialBuild, true) }}: + dependsOn: + - BuildUITests_${{ platform }} + ${{ else }}: + dependsOn: + - Build_${{ platform }} jobs: - template: job-test-project.yml parameters: platform: x64Win11 configuration: Release useLatestWebView2: ${{ parameters.useLatestWebView2 }} + useLatestOfficialBuild: ${{ parameters.useLatestOfficialBuild }} + useCurrentBranchBuild: ${{ parameters.useCurrentBranchBuild }} + uiTestModules: ${{ parameters.uiTestModules }} + + # Additional per-user installation test (when both modes are enabled) + - ${{ if and(eq(parameters.useLatestOfficialBuild, true), eq(parameters.testBothInstallModes, true)) }}: + - template: job-test-project.yml + parameters: + platform: x64Win11 + configuration: Release + useLatestWebView2: ${{ parameters.useLatestWebView2 }} + useLatestOfficialBuild: ${{ parameters.useLatestOfficialBuild }} + useCurrentBranchBuild: ${{ parameters.useCurrentBranchBuild }} + uiTestModules: ${{ parameters.uiTestModules }} + installMode: 'peruser' + jobSuffix: '_PerUser' - ${{ if ne(platform, 'x64') }}: - stage: Test_${{ platform }} displayName: Test ${{ platform }} - dependsOn: - - Build_${{platform}} + ${{ if eq(parameters.useLatestOfficialBuild, true) }}: + dependsOn: + - BuildUITests_${{ platform }} + ${{ else }}: + dependsOn: + - Build_${{ platform }} jobs: - template: job-test-project.yml parameters: platform: ${{ platform }} configuration: Release useLatestWebView2: ${{ parameters.useLatestWebView2 }} + useLatestOfficialBuild: ${{ parameters.useLatestOfficialBuild }} + useCurrentBranchBuild: ${{ parameters.useCurrentBranchBuild }} + uiTestModules: ${{ parameters.uiTestModules }} + + # Additional per-user installation test (when both modes are enabled) + - ${{ if and(eq(parameters.useLatestOfficialBuild, true), eq(parameters.testBothInstallModes, true)) }}: + - template: job-test-project.yml + parameters: + platform: ${{ platform }} + configuration: Release + useLatestWebView2: ${{ parameters.useLatestWebView2 }} + useLatestOfficialBuild: ${{ parameters.useLatestOfficialBuild }} + useCurrentBranchBuild: ${{ parameters.useCurrentBranchBuild }} + uiTestModules: ${{ parameters.uiTestModules }} + installMode: 'peruser' + jobSuffix: '_PerUser' \ No newline at end of file diff --git a/doc/devdocs/UITests.md b/doc/devdocs/UITests.md index d02eeb6993..5e2e39c75b 100644 --- a/doc/devdocs/UITests.md +++ b/doc/devdocs/UITests.md @@ -16,6 +16,56 @@ - Run tests in the Test Explorer (`Test > Test Explorer` or `Ctrl+E, T`). +## Running tests in pipeline + +The PowerToys UI test pipeline provides flexible options for building and testing: + +### Pipeline Options + +- **useLatestOfficialBuild**: When checked, downloads the latest official PowerToys build and installs it for testing. This skips the full solution build and only builds UI test projects. + +- **useCurrentBranchBuild**: When checked along with `useLatestOfficialBuild`, downloads the official build from the current branch instead of main. + + **Default value**: `false` (downloads from main branch) + + **When to use this**: + - **Default scenario**: The pipeline tests against the latest signed PowerToys build from the `main` branch, regardless of which branch your test code changes are from + - **Custom branch testing**: Only specify `true` when: + - Your branch has produced its own signed PowerToys build via the official build pipeline + - You want to test against that specific branch's PowerToys build instead of main + - You are testing PowerToys functionality changes that are only available in your branch's build + + **Important notes**: + - The test pipeline itself runs from your specified branch, but by default tests against the main branch's PowerToys build + - Not all branches have signed builds available - only use this if you're certain your branch has a signed build + - If enabled but no build exists for your branch, the pipeline may fail or fall back to main + +- **uiTestModules**: Specify which UI test modules to build and run. This parameter controls both the `.csproj` projects to build and the `.dll` test assemblies to execute. Examples: + - `['UITests-FancyZones']` - Only FancyZones UI tests + - `['MouseUtils.UITests']` - Only MouseUtils UI tests + - `['UITests-FancyZones', 'MouseUtils.UITests']` - Multiple specific modules + - Leave empty to build and run all UI test modules + + **Important**: The `uiTestModules` parameter values must match both the test project names (for `.csproj` selection during build) and the test assembly names (for `.dll` execution during testing). + +### Build Modes + +1. **Official Build + Selective Testing** (`useLatestOfficialBuild = true`) + - Downloads and installs official PowerToys build + - Builds only specified UI test projects + - Runs specified UI tests against installed PowerToys + - Controlled by `uiTestModules` parameter + +2. **Full Build + Testing** (`useLatestOfficialBuild = false`) + - Builds entire PowerToys solution + - Builds UI test projects (all or specific based on `uiTestModules`) + - Runs UI tests (all or specific based on `uiTestModules`) + - Uses freshly built PowerToys for testing + +> **Note**: Both modes support the `uiTestModules` parameter to control which specific UI test modules to build and run. + +### Pipeline Access +- Pipeline: https://microsoft.visualstudio.com/Dart/_build?definitionId=161438&_a=summary ## How to add the first UI tests for your modules diff --git a/src/common/UITestAutomation/ModuleConfigData.cs b/src/common/UITestAutomation/ModuleConfigData.cs index 3445be19cd..56b8789251 100644 --- a/src/common/UITestAutomation/ModuleConfigData.cs +++ b/src/common/UITestAutomation/ModuleConfigData.cs @@ -77,7 +77,7 @@ namespace Microsoft.PowerToys.UITest internal class ModuleConfigData { - private Dictionary ModulePath { get; } + private Dictionary ModuleInfo { get; } // Singleton instance of ModuleConfigData. private static readonly Lazy SingletonInstance = new Lazy(() => new ModuleConfigData()); @@ -86,37 +86,74 @@ namespace Microsoft.PowerToys.UITest public const string WindowsApplicationDriverUrl = "http://127.0.0.1:4723"; - public Dictionary ModuleWindowName { get; } + private bool UseInstallerForTest { get; } private ModuleConfigData() { - // The exe window name for each module. - ModuleWindowName = new Dictionary - { - [PowerToysModule.PowerToysSettings] = "PowerToys Settings", - [PowerToysModule.FancyZone] = "FancyZones Layout", - [PowerToysModule.Hosts] = "Hosts File Editor", - [PowerToysModule.Runner] = "PowerToys", - [PowerToysModule.Workspaces] = "Workspaces Editor", - [PowerToysModule.PowerRename] = "PowerRename", - }; + // Check if we should use installer paths from environment variable + string? useInstallerForTestEnv = + Environment.GetEnvironmentVariable("useInstallerForTest") ?? Environment.GetEnvironmentVariable("USEINSTALLERFORTEST"); + UseInstallerForTest = !string.IsNullOrEmpty(useInstallerForTestEnv) && bool.TryParse(useInstallerForTestEnv, out bool result) && result; - // Exe start path for the module if it exists. - ModulePath = new Dictionary + // Module information including executable name, window name, and optional subdirectory + ModuleInfo = new Dictionary { - [PowerToysModule.PowerToysSettings] = @"\..\..\..\WinUI3Apps\PowerToys.Settings.exe", - [PowerToysModule.FancyZone] = @"\..\..\..\PowerToys.FancyZonesEditor.exe", - [PowerToysModule.Hosts] = @"\..\..\..\WinUI3Apps\PowerToys.Hosts.exe", - [PowerToysModule.Runner] = @"\..\..\..\PowerToys.exe", - [PowerToysModule.Workspaces] = @"\..\..\..\PowerToys.WorkspacesEditor.exe", - [PowerToysModule.PowerRename] = @"\..\..\..\WinUI3Apps\PowerToys.PowerRename.exe", + [PowerToysModule.PowerToysSettings] = new ModuleInfo("PowerToys.Settings.exe", "PowerToys Settings", "WinUI3Apps"), + [PowerToysModule.FancyZone] = new ModuleInfo("PowerToys.FancyZonesEditor.exe", "FancyZones Layout"), + [PowerToysModule.Hosts] = new ModuleInfo("PowerToys.Hosts.exe", "Hosts File Editor", "WinUI3Apps"), + [PowerToysModule.Runner] = new ModuleInfo("PowerToys.exe", "PowerToys"), + [PowerToysModule.Workspaces] = new ModuleInfo("PowerToys.WorkspacesEditor.exe", "Workspaces Editor"), + [PowerToysModule.PowerRename] = new ModuleInfo("PowerToys.PowerRename.exe", "PowerRename", "WinUI3Apps"), }; } - public string GetModulePath(PowerToysModule scope) => ModulePath[scope]; + private string GetPowerToysInstallPath() + { + // Try common installation paths + string[] possiblePaths = + { + @"C:\Program Files\PowerToys", + @"C:\Program Files (x86)\PowerToys", + Environment.ExpandEnvironmentVariables(@"%LocalAppData%\PowerToys"), + Environment.ExpandEnvironmentVariables(@"%ProgramFiles%\PowerToys"), + }; + + foreach (string path in possiblePaths) + { + if (Directory.Exists(path) && File.Exists(Path.Combine(path, "PowerToys.exe"))) + { + return path; + } + } + + // Fallback to Program Files if not found + return @"C:\Program Files\PowerToys"; + } + + public string GetModulePath(PowerToysModule scope) + { + var moduleInfo = ModuleInfo[scope]; + + if (UseInstallerForTest) + { + string powerToysInstallPath = GetPowerToysInstallPath(); + string installedPath = moduleInfo.GetInstalledPath(powerToysInstallPath); + + if (File.Exists(installedPath)) + { + return installedPath; + } + else + { + Console.WriteLine($"Warning: Installed module not found at {installedPath}, using development path"); + } + } + + return moduleInfo.GetDevelopmentPath(); + } public string GetWindowsApplicationDriverUrl() => WindowsApplicationDriverUrl; - public string GetModuleWindowName(PowerToysModule scope) => ModuleWindowName[scope]; + public string GetModuleWindowName(PowerToysModule scope) => ModuleInfo[scope].WindowName; } } diff --git a/src/common/UITestAutomation/ModuleInfo.cs b/src/common/UITestAutomation/ModuleInfo.cs new file mode 100644 index 0000000000..35add0e0d2 --- /dev/null +++ b/src/common/UITestAutomation/ModuleInfo.cs @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.PowerToys.UITest +{ + internal class ModuleInfo + { + public string ExecutableName { get; } + + public string? SubDirectory { get; } + + public string WindowName { get; } + + public ModuleInfo(string executableName, string windowName, string? subDirectory = null) + { + ExecutableName = executableName; + WindowName = windowName; + SubDirectory = subDirectory; + } + + /// + /// Gets the relative development path for this module + /// + public string GetDevelopmentPath() + { + if (string.IsNullOrEmpty(SubDirectory)) + { + return $@"\..\..\..\{ExecutableName}"; + } + + return $@"\..\..\..\{SubDirectory}\{ExecutableName}"; + } + + /// + /// Gets the installed path for this module based on the PowerToys install directory + /// + public string GetInstalledPath(string powerToysInstallPath) + { + if (string.IsNullOrEmpty(SubDirectory)) + { + return Path.Combine(powerToysInstallPath, ExecutableName); + } + + return Path.Combine(powerToysInstallPath, SubDirectory, ExecutableName); + } + } +} diff --git a/src/common/UITestAutomation/SessionHelper.cs b/src/common/UITestAutomation/SessionHelper.cs index 75c8c96581..b12b1f831b 100644 --- a/src/common/UITestAutomation/SessionHelper.cs +++ b/src/common/UITestAutomation/SessionHelper.cs @@ -37,13 +37,18 @@ namespace Microsoft.PowerToys.UITest private PowerToysModule scope; private string[]? commandLineArgs; + private bool UseInstallerForTest { get; } + [UnconditionalSuppressMessage("SingleFile", "IL3000:Avoid accessing Assembly file path when publishing as a single file", Justification = "")] public SessionHelper(PowerToysModule scope, string[]? commandLineArgs = null) { this.scope = scope; this.commandLineArgs = commandLineArgs; this.sessionPath = ModuleConfigData.Instance.GetModulePath(scope); - this.locationPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + string? useInstallerForTestEnv = + Environment.GetEnvironmentVariable("useInstallerForTest") ?? Environment.GetEnvironmentVariable("USEINSTALLERFORTEST"); + UseInstallerForTest = !string.IsNullOrEmpty(useInstallerForTestEnv) && bool.TryParse(useInstallerForTestEnv, out bool result) && result; + this.locationPath = UseInstallerForTest ? string.Empty : Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); CheckWinAppDriverAndRoot(); } @@ -156,7 +161,7 @@ namespace Microsoft.PowerToys.UITest if (root != null) { - const int maxRetries = 3; + const int maxRetries = 5; const int delayMs = 5000; var windowName = "PowerToys Settings"; diff --git a/src/modules/Hosts/Hosts.UITests/Hosts.UITests.csproj b/src/modules/Hosts/Hosts.UITests/Hosts.UITests.csproj index 0fb64ee396..c8a45ea3aa 100644 --- a/src/modules/Hosts/Hosts.UITests/Hosts.UITests.csproj +++ b/src/modules/Hosts/Hosts.UITests/Hosts.UITests.csproj @@ -15,7 +15,7 @@ false - $(SolutionDir)$(Platform)\$(Configuration)\tests\Hosts.UITests\ + ..\..\..\..\$(Platform)\$(Configuration)\tests\Hosts.UITests\