diff --git a/src/settings-ui/Settings.UI/PowerToys.Settings.csproj b/src/settings-ui/Settings.UI/PowerToys.Settings.csproj
index 45650ecb26..49ecf138f1 100644
--- a/src/settings-ui/Settings.UI/PowerToys.Settings.csproj
+++ b/src/settings-ui/Settings.UI/PowerToys.Settings.csproj
@@ -227,4 +227,23 @@
 		<Message Importance="high" Text="[Settings] Building XamlIndexBuilder prior to compile. Views='$(MSBuildProjectDirectory)\SettingsXAML\Views' Out='$(GeneratedJsonFile)'" />
 		<MSBuild Projects="..\Settings.UI.XamlIndexBuilder\Settings.UI.XamlIndexBuilder.csproj" Targets="Build" Properties="Configuration=$(Configuration);Platform=Any CPU;TargetFramework=net9.0;XamlViewsDir=$(MSBuildProjectDirectory)\SettingsXAML\Views;GeneratedJsonFile=$(GeneratedJsonFile)" />
 	</Target>
+
+	<!--
+	When Settings is activated via sparse package identity, package resource lookup can resolve from
+	the external location root (..\$(Platform)\$(Configuration)). Mirror Settings assets there so
+	ms-appx:///Assets/Settings/... resolves consistently in both identity and non-identity launches.
+	-->
+	<Target Name="MirrorSettingsAssetsToSparseRoot" AfterTargets="Build" Condition="'$(DesignTimeBuild)' != 'true' and '$(UseSparseIdentity)'=='true'">
+		<PropertyGroup>
+			<_SparseRootDir>$([System.IO.Path]::GetFullPath('$(OutputPath)..\'))</_SparseRootDir>
+			<_SparseSettingsAssetsDir>$(_SparseRootDir)Assets\Settings\</_SparseSettingsAssetsDir>
+		</PropertyGroup>
+		<ItemGroup>
+			<_SettingsAssetsFiles Include="$(MSBuildProjectDirectory)\Assets\Settings\**\*" />
+		</ItemGroup>
+		<Copy
+			SourceFiles="@(_SettingsAssetsFiles)"
+			DestinationFiles="@(_SettingsAssetsFiles->'$(_SparseSettingsAssetsDir)%(RecursiveDir)%(Filename)%(Extension)')"
+			SkipUnchangedFiles="true" />
+	</Target>
 </Project>
diff --git a/tools/SettingsSearchEvaluation.Tests/EvaluatorTests.cs b/tools/SettingsSearchEvaluation.Tests/EvaluatorTests.cs
index 463a15009e..ce4e0c13ed 100644
--- a/tools/SettingsSearchEvaluation.Tests/EvaluatorTests.cs
+++ b/tools/SettingsSearchEvaluation.Tests/EvaluatorTests.cs
@@ -66,4 +66,64 @@ public class EvaluatorTests
         Assert.IsTrue(engine.CaseResults[0].HitAtK);
         Assert.AreEqual(1, engine.CaseResults[0].BestRank);
     }
+
+    [TestMethod]
+    public async Task RunAsync_SemanticEngine_ReturnsReportWithoutThrowing()
+    {
+        const string json = """
+[
+  {
+    "type": 0,
+    "header": "Fancy Zones",
+    "pageTypeName": "FancyZonesPage",
+    "elementName": "",
+    "elementUid": "FancyZones",
+    "parentElementName": "",
+    "description": "",
+    "icon": null
+  }
+]
+""";
+
+        var (entries, diagnostics) = EvaluationDataLoader.LoadEntriesFromJson(json);
+        var cases = new[]
+        {
+            new EvaluationCase
+            {
+                Query = "Fancy Zones",
+                ExpectedIds = new[] { "FancyZones" },
+                Notes = "Semantic smoke test.",
+            },
+        };
+
+        var options = new RunnerOptions
+        {
+            IndexJsonPath = "test-index.json",
+            CasesJsonPath = null,
+            Engines = new[] { SearchEngineKind.Semantic },
+            MaxResults = 5,
+            TopK = 5,
+            Iterations = 1,
+            WarmupIterations = 0,
+            SemanticIndexTimeout = TimeSpan.FromSeconds(3),
+            OutputJsonPath = null,
+        };
+
+        var report = await Evaluator.RunAsync(options, entries, diagnostics, cases);
+
+        Assert.AreEqual(1, report.Engines.Count);
+        var engine = report.Engines[0];
+        Assert.AreEqual(SearchEngineKind.Semantic, engine.Engine);
+
+        if (!engine.IsAvailable)
+        {
+            Assert.IsFalse(string.IsNullOrWhiteSpace(engine.AvailabilityError));
+            Assert.AreEqual(0, engine.QueryCount);
+        }
+        else
+        {
+            Assert.AreEqual(1, engine.QueryCount);
+            Assert.AreEqual(1, engine.CaseResults.Count);
+        }
+    }
 }
diff --git a/tools/SettingsSearchEvaluation/SettingsSearchEvaluation.csproj b/tools/SettingsSearchEvaluation/SettingsSearchEvaluation.csproj
index 789903a625..ac2618528c 100644
--- a/tools/SettingsSearchEvaluation/SettingsSearchEvaluation.csproj
+++ b/tools/SettingsSearchEvaluation/SettingsSearchEvaluation.csproj
@@ -6,6 +6,7 @@
     <OutputType>Exe</OutputType>
     <RootNamespace>SettingsSearchEvaluation</RootNamespace>
     <AssemblyName>SettingsSearchEvaluation</AssemblyName>
+    <WindowsPackageType>None</WindowsPackageType>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
     <DefaultItemExcludes>$(DefaultItemExcludes);artifacts\**\*;bin\**\*;obj\**\*</DefaultItemExcludes>