Opt into Microsoft.Build.RunVSTest to run tests within MSBuild (#32166)

.github/actions/spell-check/expect.txt

@@ -1613,6 +1613,7 @@ TRAYMOUSEMESSAGE
 triaging
 TRK
 trl
+trx
 Tsd
 TServer
 TStr
@@ -1714,6 +1715,7 @@ vscdb
 vsconfig
 VSCROLL
 vsetq
+VSINSTALLDIR
 VSM
 vso
 vsonline

.pipelines/ci/templates/build-powertoys-steps.yml

@@ -91,16 +91,16 @@ steps:
     displayName: 'nuget restore packages.config'
 
 - task: VSBuild@1
-  displayName: 'Build PowerToys.sln'
+  displayName: 'Build and Test PowerToys.sln'
   inputs:
     solution: '**\PowerToys.sln'
     vsVersion: 17.0
     platform: '$(BuildPlatform)'
     configuration: '$(BuildConfiguration)'
     ${{ if eq(parameters.enableCaching, true) }}:
-      msbuildArgs: -restore ${{ parameters.additionalBuildArguments }} -graph -reportfileaccesses -p:MSBuildCacheEnabled=true -p:MSBuildCacheLogDirectory=$(Build.ArtifactStagingDirectory)\logs\MSBuildCache -bl:$(Build.ArtifactStagingDirectory)\logs\PowerToys.binlog -ds:false
+      msbuildArgs: -restore ${{ parameters.additionalBuildArguments }} -t:Build;Test -graph -reportfileaccesses -p:MSBuildCacheEnabled=true -p:MSBuildCacheLogDirectory=$(Build.ArtifactStagingDirectory)\logs\MSBuildCache -bl:$(Build.ArtifactStagingDirectory)\logs\PowerToys.binlog -ds:false
     ${{ else }}:
-      msbuildArgs: -restore ${{ parameters.additionalBuildArguments }} -graph -bl:$(Build.ArtifactStagingDirectory)\logs\PowerToys.binlog -ds:false
+      msbuildArgs: -restore ${{ parameters.additionalBuildArguments }} -t:Build;Test -graph -bl:$(Build.ArtifactStagingDirectory)\logs\PowerToys.binlog -ds:false
     msbuildArchitecture: x64
     maximumCpuCount: true
   ${{ if eq(parameters.enableCaching, true) }}:
@@ -222,47 +222,13 @@ steps:
     arguments: -targetDir '$(build.sourcesdirectory)\$(BuildPlatform)\$(BuildConfiguration)\WinUI3Apps'
     pwsh: true
 
-# directly not doing WinAppDriver testing
+# Publish test results which ran in MSBuild
-- task: VSTest@2
+- task: PublishTestResults@2
-  displayName: 'MS Tests'
+  displayName: 'Publish Test Results'
-  condition: ne(variables['BuildPlatform'], 'arm64') # No arm64 agents to run the tests.
   inputs:
-    platform: '$(BuildPlatform)'
+    testResultsFormat: VSTest
-    configuration: '$(BuildConfiguration)'
+    testResultsFiles: '**/*.trx'
-    testSelector: 'testAssemblies'
+  condition: always()
-    testAssemblyVer2: |
-      **\UnitTests-GcodeThumbnailProvider.dll
-      **\UnitTests-StlThumbnailProvider.dll
-      **\UnitTests-PdfThumbnailProvider.dll
-      **\UnitTests-QoiThumbnailProvider.dll
-      **\Settings.UI.UnitTests.dll
-      **\UnitTests-GcodePreviewHandler.dll
-      **\UnitTests-QoiPreviewHandler.dll
-      **\UnitTests-FancyZonesEditor.dll
-      **\UnitTests-PdfPreviewHandler.dll
-      **\UnitTests-PreviewHandlerCommon.dll
-      **\Microsoft.PowerToys.Run.Plugin.Registry.UnitTests.dll
-      **\UnitTest-ColorPickerUI.dll
-      **\Microsoft.Interop.Tests.dll
-      **\ImageResizer.Test.dll
-      **\Community.PowerToys.Run.Plugin.UnitConverter.UnitTest.dll
-      **\Community.PowerToys.Run.Plugin.ValueGenerator.UnitTests.dll
-      **\Microsoft.Plugin.Folder.UnitTests.dll
-      **\Microsoft.Plugin.Program.UnitTests.dll
-      **\Microsoft.PowerToys.Run.Plugin.Calculator.UnitTest.dll
-      **\Microsoft.Plugin.Uri.UnitTests.dll
-      **\Wox.Test.dll
-      **\Microsoft.PowerToys.Run.Plugin.System.UnitTests.dll
-      **\Microsoft.PowerToys.Run.Plugin.TimeDate.UnitTests.dll
-      **\Microsoft.Plugin.WindowsTerminal.UnitTests.dll
-      **\Microsoft.Plugin.WindowWalker.UnitTests.dll
-      **\PreviewPaneUnitTests.dll
-      **\UnitTests-SvgThumbnailProvider.dll
-      **\UnitTests-SvgPreviewHandler.dll
-      **\PowerToys.Hosts.Tests.dll
-      **\MouseJumpUI.UnitTests.dll
-      !**\obj\**
-      !**\ref\**
 
 # Native dlls
 - task: VSTest@2
@@ -277,7 +243,6 @@ steps:
       **\KeyboardManagerEditorTest.dll
       **\UnitTests-CommonLib.dll
       **\PowerRenameUnitTests.dll
-      **\powerpreviewTest.dll
       **\UnitTests-FancyZones.dll
       !**\obj\**
 

Directory.Build.props

@@ -44,6 +44,20 @@
     </PackageReference>
   </ItemGroup>
 
+  <!-- Add ability to run tests via "msbuild /t:Test" -->
+  <Sdk Name="Microsoft.Build.RunVSTest" Version="1.0.319" />
+  <PropertyGroup>
+    <VSTestLogger>trx</VSTestLogger>
+    <!--
+      RunVSTest by default uses %VSINSTALLDIR%\Common7\IDE\CommonExtensions\Microsoft\TestWindow\vstest.console.exe,
+      but some of the CI scenarios don't define %VSINSTALLDIR%, so be explicit about where to look for vstest.
+      Note: $(VsInstallRoot) is a built-in MSBuild property, so should always be defined.
+    -->
+    <VSTestToolPath>$(VsInstallRoot)\Common7\IDE\CommonExtensions\Microsoft\TestWindow</VSTestToolPath>
+    <!-- No arm64 agents to run the tests. -->
+    <RunVSTest Condition="'$(Platform)' == 'ARM64'">false</RunVSTest>
+  </PropertyGroup>
+
   <!-- MSBuildCache -->
   <PropertyGroup>
     <!-- Off by default -->
@@ -74,6 +88,12 @@
       E:\;
     </MSBuildCacheAllowFileAccessAfterProjectFinishFilePatterns>
 
+    <!-- Unit tests of low-priv processes, eg the preview handler tests, may log to this location. -->
+    <MSBuildCacheAllowFileAccessAfterProjectFinishFilePatterns>
+      $(MSBuildCacheAllowFileAccessAfterProjectFinishFilePatterns);
+      $(USERPROFILE)\AppData\LocalLow\Microsoft\PowerToys\**;
+    </MSBuildCacheAllowFileAccessAfterProjectFinishFilePatterns>
+
     <!-- 
       This repo uses a common output directory with many projects writing duplicate outputs. Allow everything, but note this costs some performance in the form of requiring
       the cache to use copies instead of hardlinks when pulling from cache.

packages.config

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Microsoft.MSBuildCache.AzurePipelines" version="0.1.258-preview" />
+  <package id="Microsoft.MSBuildCache.AzurePipelines" version="0.1.271-preview" />
-  <package id="Microsoft.MSBuildCache.Local" version="0.1.258-preview" />
+  <package id="Microsoft.MSBuildCache.Local" version="0.1.271-preview" />
-  <package id="Microsoft.MSBuildCache.SharedCompilation" version="0.1.258-preview" />
+  <package id="Microsoft.MSBuildCache.SharedCompilation" version="0.1.271-preview" />
 </packages>

src/modules/fancyzones/UITests-FancyZones/UITests-FancyZones.csproj

@@ -11,6 +11,8 @@
     <Nullable>enable</Nullable>
     <OutputType>Library</OutputType>
     <Version>$(Version).0</Version>
+    <!-- This is a UI test, so don't run as part of MSBuild -->
+    <RunVSTest>false</RunVSTest>
   </PropertyGroup>
 
   <PropertyGroup>

src/modules/fancyzones/UITests-FancyZonesEditor/UITests-FancyZonesEditor.csproj

@@ -11,6 +11,8 @@
     <Nullable>enable</Nullable>
     <OutputType>Library</OutputType>
     <Version>$(Version).0</Version>
+    <!-- This is a UI test, so don't run as part of MSBuild -->
+    <RunVSTest>false</RunVSTest>
   </PropertyGroup>
 
   <PropertyGroup>