diff --git a/PowerToys.slnx b/PowerToys.slnx index 506545a754..aab1e7fa3b 100644 --- a/PowerToys.slnx +++ b/PowerToys.slnx @@ -1059,6 +1059,16 @@ + + + + + + + + + + diff --git a/tools/SettingsSearchEvaluation.Tests/EvaluationDataLoaderTests.cs b/tools/SettingsSearchEvaluation.Tests/EvaluationDataLoaderTests.cs new file mode 100644 index 0000000000..4a4fb8c04b --- /dev/null +++ b/tools/SettingsSearchEvaluation.Tests/EvaluationDataLoaderTests.cs @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.IO; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace SettingsSearchEvaluation.Tests; + +[TestClass] +public class EvaluationDataLoaderTests +{ + [TestMethod] + public void LoadEntriesFromJson_NormalizesHeaderAndDetectsDuplicates() + { + const string json = """ +[ + { + "type": 0, + "header": null, + "pageTypeName": "ColorPickerPage", + "elementName": "", + "elementUid": "Activation_Shortcut", + "parentElementName": "", + "description": null, + "icon": null + }, + { + "type": 0, + "header": null, + "pageTypeName": "FancyZonesPage", + "elementName": "", + "elementUid": "Activation_Shortcut", + "parentElementName": "", + "description": null, + "icon": null + } +] +"""; + + var (entries, diagnostics) = EvaluationDataLoader.LoadEntriesFromJson(json); + + Assert.AreEqual(2, entries.Count); + Assert.AreEqual("Activation Shortcut", entries[0].Header); + Assert.AreEqual(1, diagnostics.DuplicateIdBucketCount); + Assert.IsTrue(diagnostics.DuplicateIdCounts.TryGetValue("Activation_Shortcut", out var count)); + Assert.AreEqual(2, count); + } + + [TestMethod] + public void LoadCases_GeneratesFallbackCases_WhenNoCasesFileSpecified() + { + const string json = """ +[ + { + "type": 0, + "header": "Fancy Zones", + "pageTypeName": "FancyZonesPage", + "elementName": "", + "elementUid": "FancyZones", + "parentElementName": "", + "description": "", + "icon": null + } +] +"""; + + var (entries, _) = EvaluationDataLoader.LoadEntriesFromJson(json); + var cases = EvaluationDataLoader.LoadCases(null, entries); + + Assert.AreEqual(1, cases.Count); + Assert.AreEqual("Fancy Zones", cases[0].Query); + Assert.AreEqual("FancyZones", cases[0].ExpectedIds[0]); + } + + [TestMethod] + public void LoadCases_LoadsAndNormalizesCasesFile() + { + const string entriesJson = """ +[ + { + "type": 0, + "header": "Fancy Zones", + "pageTypeName": "FancyZonesPage", + "elementName": "", + "elementUid": "FancyZones", + "parentElementName": "", + "description": "", + "icon": null + } +] +"""; + + const string casesJson = """ +[ + { + "query": " fancy zones ", + "expectedIds": [ "FancyZones", " fancyzones ", "" ], + "notes": "normalization test" + }, + { + "query": "", + "expectedIds": [ "FancyZones" ] + }, + { + "query": "missing expected", + "expectedIds": [ "" ] + } +] +"""; + + var (entries, _) = EvaluationDataLoader.LoadEntriesFromJson(entriesJson); + var casesFile = Path.GetTempFileName(); + try + { + File.WriteAllText(casesFile, casesJson); + var cases = EvaluationDataLoader.LoadCases(casesFile, entries); + + Assert.AreEqual(1, cases.Count); + Assert.AreEqual("fancy zones", cases[0].Query); + Assert.AreEqual(1, cases[0].ExpectedIds.Count); + Assert.AreEqual("FancyZones", cases[0].ExpectedIds[0]); + Assert.AreEqual("normalization test", cases[0].Notes); + } + finally + { + File.Delete(casesFile); + } + } +} diff --git a/tools/SettingsSearchEvaluation.Tests/EvaluationMathTests.cs b/tools/SettingsSearchEvaluation.Tests/EvaluationMathTests.cs new file mode 100644 index 0000000000..e5171bebde --- /dev/null +++ b/tools/SettingsSearchEvaluation.Tests/EvaluationMathTests.cs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace SettingsSearchEvaluation.Tests; + +[TestClass] +public class EvaluationMathTests +{ + private static readonly double[] LatencySamples = { 10.0, 20.0, 30.0, 40.0, 50.0 }; + + [TestMethod] + public void FindBestRank_ReturnsExpectedRank() + { + var ranked = new[] { "a", "b", "c", "d" }; + var expected = new HashSet(StringComparer.OrdinalIgnoreCase) { "c" }; + + var rank = EvaluationMath.FindBestRank(ranked, expected); + + Assert.AreEqual(3, rank); + } + + [TestMethod] + public void FindBestRank_ReturnsZero_WhenMissing() + { + var ranked = new[] { "a", "b", "c", "d" }; + var expected = new HashSet(StringComparer.OrdinalIgnoreCase) { "x", "y" }; + + var rank = EvaluationMath.FindBestRank(ranked, expected); + + Assert.AreEqual(0, rank); + } + + [TestMethod] + public void ComputeLatencySummary_ComputesQuantiles() + { + var summary = EvaluationMath.ComputeLatencySummary(LatencySamples); + + Assert.AreEqual(5, summary.Samples); + Assert.AreEqual(10.0, summary.MinMs); + Assert.AreEqual(30.0, summary.P50Ms); + Assert.AreEqual(50.0, summary.P95Ms); + Assert.AreEqual(50.0, summary.MaxMs); + Assert.AreEqual(30.0, summary.AverageMs, 0.0001); + } +} diff --git a/tools/SettingsSearchEvaluation.Tests/EvaluatorTests.cs b/tools/SettingsSearchEvaluation.Tests/EvaluatorTests.cs new file mode 100644 index 0000000000..463a15009e --- /dev/null +++ b/tools/SettingsSearchEvaluation.Tests/EvaluatorTests.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace SettingsSearchEvaluation.Tests; + +[TestClass] +public class EvaluatorTests +{ + [TestMethod] + public async Task RunAsync_BasicEngine_ReturnsExpectedMetricsForExactSingleEntry() + { + const string json = """ +[ + { + "type": 0, + "header": "Fancy Zones", + "pageTypeName": "FancyZonesPage", + "elementName": "", + "elementUid": "FancyZones", + "parentElementName": "", + "description": "", + "icon": null + } +] +"""; + + var (entries, diagnostics) = EvaluationDataLoader.LoadEntriesFromJson(json); + var cases = new[] + { + new EvaluationCase + { + Query = "Fancy Zones", + ExpectedIds = new[] { "FancyZones" }, + Notes = "Exact query should be rank 1.", + }, + }; + + var options = new RunnerOptions + { + IndexJsonPath = "test-index.json", + CasesJsonPath = null, + Engines = new[] { SearchEngineKind.Basic }, + MaxResults = 5, + TopK = 5, + Iterations = 1, + WarmupIterations = 0, + SemanticIndexTimeout = TimeSpan.FromSeconds(1), + OutputJsonPath = null, + }; + + var report = await Evaluator.RunAsync(options, entries, diagnostics, cases); + + Assert.AreEqual(1, report.Engines.Count); + var engine = report.Engines[0]; + Assert.AreEqual(SearchEngineKind.Basic, engine.Engine); + Assert.IsTrue(engine.IsAvailable); + Assert.AreEqual(1, engine.QueryCount); + Assert.AreEqual(1.0, engine.RecallAtK, 0.0001); + Assert.AreEqual(1.0, engine.Mrr, 0.0001); + Assert.AreEqual(1, engine.CaseResults.Count); + Assert.IsTrue(engine.CaseResults[0].HitAtK); + Assert.AreEqual(1, engine.CaseResults[0].BestRank); + } +} diff --git a/tools/SettingsSearchEvaluation.Tests/SettingsSearchEvaluation.Tests.csproj b/tools/SettingsSearchEvaluation.Tests/SettingsSearchEvaluation.Tests.csproj new file mode 100644 index 0000000000..f6cf65ba5b --- /dev/null +++ b/tools/SettingsSearchEvaluation.Tests/SettingsSearchEvaluation.Tests.csproj @@ -0,0 +1,17 @@ + + + + + + false + ..\..\$(Configuration)\$(Platform)\tests\SettingsSearchEvaluationTests\ + + + + + + + + + + diff --git a/tools/SettingsSearchEvaluation/DatasetDiagnostics.cs b/tools/SettingsSearchEvaluation/DatasetDiagnostics.cs new file mode 100644 index 0000000000..848ee9226e --- /dev/null +++ b/tools/SettingsSearchEvaluation/DatasetDiagnostics.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.ObjectModel; + +namespace SettingsSearchEvaluation; + +internal sealed class DatasetDiagnostics +{ + public required int TotalEntries { get; init; } + + public required int DistinctIds { get; init; } + + public required int DuplicateIdBucketCount { get; init; } + + public required IReadOnlyDictionary DuplicateIdCounts { get; init; } + + public static DatasetDiagnostics Empty { get; } = new() + { + TotalEntries = 0, + DistinctIds = 0, + DuplicateIdBucketCount = 0, + DuplicateIdCounts = new ReadOnlyDictionary(new Dictionary()), + }; +} diff --git a/tools/SettingsSearchEvaluation/EngineEvaluationReport.cs b/tools/SettingsSearchEvaluation/EngineEvaluationReport.cs new file mode 100644 index 0000000000..f855406240 --- /dev/null +++ b/tools/SettingsSearchEvaluation/EngineEvaluationReport.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace SettingsSearchEvaluation; + +internal sealed class EngineEvaluationReport +{ + public required SearchEngineKind Engine { get; init; } + + public required bool IsAvailable { get; init; } + + public string? AvailabilityError { get; init; } + + public string? CapabilitiesSummary { get; init; } + + public int IndexedEntries { get; init; } + + public int QueryCount { get; init; } + + public double IndexingTimeMs { get; init; } + + public double RecallAtK { get; init; } + + public double Mrr { get; init; } + + public required LatencySummary SearchLatencyMs { get; init; } + + public required IReadOnlyList CaseResults { get; init; } +} diff --git a/tools/SettingsSearchEvaluation/EvaluationCase.cs b/tools/SettingsSearchEvaluation/EvaluationCase.cs new file mode 100644 index 0000000000..0709c935ec --- /dev/null +++ b/tools/SettingsSearchEvaluation/EvaluationCase.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace SettingsSearchEvaluation; + +internal sealed class EvaluationCase +{ + public required string Query { get; init; } + + public required IReadOnlyList ExpectedIds { get; init; } + + public string? Notes { get; init; } +} diff --git a/tools/SettingsSearchEvaluation/EvaluationDataLoader.cs b/tools/SettingsSearchEvaluation/EvaluationDataLoader.cs new file mode 100644 index 0000000000..aa43ea31f2 --- /dev/null +++ b/tools/SettingsSearchEvaluation/EvaluationDataLoader.cs @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.ObjectModel; +using System.Text.Json; +using System.Text.RegularExpressions; +using Settings.UI.Library; + +namespace SettingsSearchEvaluation; + +internal static partial class EvaluationDataLoader +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + }; + + public static (IReadOnlyList Entries, DatasetDiagnostics Diagnostics) LoadEntriesFromFile(string path) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + var json = File.ReadAllText(path); + return LoadEntriesFromJson(json); + } + + public static (IReadOnlyList Entries, DatasetDiagnostics Diagnostics) LoadEntriesFromJson(string json) + { + if (string.IsNullOrWhiteSpace(json)) + { + return (Array.Empty(), DatasetDiagnostics.Empty); + } + + var rawEntries = JsonSerializer.Deserialize>(json, JsonOptions) ?? new List(); + var normalized = new List(rawEntries.Count); + + foreach (var raw in rawEntries) + { + var pageType = raw.PageTypeName?.Trim() ?? string.Empty; + var elementName = raw.ElementName?.Trim() ?? string.Empty; + var elementUid = raw.ElementUid?.Trim() ?? string.Empty; + + if (string.IsNullOrEmpty(elementUid)) + { + elementUid = $"{pageType}|{elementName}"; + } + + var header = raw.Header?.Trim(); + if (string.IsNullOrEmpty(header)) + { + header = BuildFallbackHeader(elementUid, elementName, pageType); + } + + var description = raw.Description?.Trim() ?? string.Empty; + var parent = raw.ParentElementName?.Trim() ?? string.Empty; + var icon = raw.Icon?.Trim() ?? string.Empty; + + normalized.Add(new SettingEntry( + raw.Type, + header, + pageType, + elementName, + elementUid, + parent, + description, + icon)); + } + + return (normalized, BuildDiagnostics(normalized)); + } + + public static IReadOnlyList LoadCases(string? casesPath, IReadOnlyList entries) + { + if (!string.IsNullOrWhiteSpace(casesPath)) + { + var json = File.ReadAllText(casesPath); + var parsed = JsonSerializer.Deserialize>(json, JsonOptions) ?? new List(); + var normalized = parsed + .Where(c => !string.IsNullOrWhiteSpace(c.Query)) + .Select(c => new EvaluationCase + { + Query = c.Query!.Trim(), + ExpectedIds = c.ExpectedIds? + .Where(id => !string.IsNullOrWhiteSpace(id)) + .Select(id => id.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() ?? Array.Empty(), + Notes = c.Notes, + }) + .Where(c => c.ExpectedIds.Count > 0) + .ToList(); + + if (normalized.Count > 0) + { + return normalized; + } + } + + return GenerateFallbackCases(entries); + } + + private static DatasetDiagnostics BuildDiagnostics(IReadOnlyList entries) + { + var duplicateBuckets = entries + .GroupBy(x => x.Id, StringComparer.OrdinalIgnoreCase) + .Where(group => group.Count() > 1) + .OrderByDescending(group => group.Count()) + .ToDictionary(group => group.Key, group => group.Count(), StringComparer.OrdinalIgnoreCase); + + return new DatasetDiagnostics + { + TotalEntries = entries.Count, + DistinctIds = entries.Select(x => x.Id).Distinct(StringComparer.OrdinalIgnoreCase).Count(), + DuplicateIdBucketCount = duplicateBuckets.Count, + DuplicateIdCounts = new ReadOnlyDictionary(duplicateBuckets), + }; + } + + private static IReadOnlyList GenerateFallbackCases(IReadOnlyList entries) + { + return entries + .Where(entry => !string.IsNullOrWhiteSpace(entry.Header) && !string.IsNullOrWhiteSpace(entry.Id)) + .GroupBy(entry => entry.Id, StringComparer.OrdinalIgnoreCase) + .Select(group => group.First()) + .Take(40) + .Select(entry => new EvaluationCase + { + Query = entry.Header, + ExpectedIds = new[] { entry.Id }, + Notes = "Autogenerated case from index entry header.", + }) + .ToArray(); + } + + private static string BuildFallbackHeader(string elementUid, string elementName, string pageTypeName) + { + var candidate = !string.IsNullOrWhiteSpace(elementUid) + ? elementUid + : (!string.IsNullOrWhiteSpace(elementName) ? elementName : pageTypeName); + + candidate = candidate.Replace('_', ' ').Trim(); + candidate = ConsecutiveWhitespaceRegex().Replace(candidate, " "); + candidate = CamelBoundaryRegex().Replace(candidate, "$1 $2"); + return candidate; + } + + [GeneratedRegex(@"\s+")] + private static partial Regex ConsecutiveWhitespaceRegex(); + + [GeneratedRegex("([a-z0-9])([A-Z])")] + private static partial Regex CamelBoundaryRegex(); + + private sealed class RawSettingEntry + { + public EntryType Type { get; init; } + + public string? Header { get; init; } + + public string? PageTypeName { get; init; } + + public string? ElementName { get; init; } + + public string? ElementUid { get; init; } + + public string? ParentElementName { get; init; } + + public string? Description { get; init; } + + public string? Icon { get; init; } + } + + private sealed class RawEvaluationCase + { + public string? Query { get; init; } + + public List? ExpectedIds { get; init; } + + public string? Notes { get; init; } + } +} diff --git a/tools/SettingsSearchEvaluation/EvaluationMath.cs b/tools/SettingsSearchEvaluation/EvaluationMath.cs new file mode 100644 index 0000000000..de2383b64f --- /dev/null +++ b/tools/SettingsSearchEvaluation/EvaluationMath.cs @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace SettingsSearchEvaluation; + +internal static class EvaluationMath +{ + public static int FindBestRank(IReadOnlyList rankedResultIds, IReadOnlySet expectedIds) + { + ArgumentNullException.ThrowIfNull(rankedResultIds); + ArgumentNullException.ThrowIfNull(expectedIds); + + if (expectedIds.Count == 0 || rankedResultIds.Count == 0) + { + return 0; + } + + for (int index = 0; index < rankedResultIds.Count; index++) + { + if (expectedIds.Contains(rankedResultIds[index])) + { + return index + 1; + } + } + + return 0; + } + + public static LatencySummary ComputeLatencySummary(IReadOnlyList samplesMs) + { + ArgumentNullException.ThrowIfNull(samplesMs); + + if (samplesMs.Count == 0) + { + return LatencySummary.Empty; + } + + var sorted = samplesMs.OrderBy(x => x).ToArray(); + var total = samplesMs.Sum(); + + return new LatencySummary + { + Samples = sorted.Length, + MinMs = sorted[0], + P50Ms = Percentile(sorted, 0.50), + P95Ms = Percentile(sorted, 0.95), + MaxMs = sorted[^1], + AverageMs = total / sorted.Length, + }; + } + + private static double Percentile(IReadOnlyList sortedSamples, double percentile) + { + if (sortedSamples.Count == 0) + { + return 0; + } + + var clamped = Math.Clamp(percentile, 0, 1); + var rank = (int)Math.Ceiling(clamped * sortedSamples.Count) - 1; + rank = Math.Clamp(rank, 0, sortedSamples.Count - 1); + return sortedSamples[rank]; + } +} diff --git a/tools/SettingsSearchEvaluation/EvaluationReport.cs b/tools/SettingsSearchEvaluation/EvaluationReport.cs new file mode 100644 index 0000000000..e38c81132a --- /dev/null +++ b/tools/SettingsSearchEvaluation/EvaluationReport.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace SettingsSearchEvaluation; + +internal sealed class EvaluationReport +{ + public required DateTimeOffset GeneratedAtUtc { get; init; } + + public required string IndexJsonPath { get; init; } + + public required DatasetDiagnostics Dataset { get; init; } + + public required int CaseCount { get; init; } + + public required IReadOnlyList Engines { get; init; } +} diff --git a/tools/SettingsSearchEvaluation/Evaluator.cs b/tools/SettingsSearchEvaluation/Evaluator.cs new file mode 100644 index 0000000000..74fd2e2ba6 --- /dev/null +++ b/tools/SettingsSearchEvaluation/Evaluator.cs @@ -0,0 +1,292 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics; +using Common.Search; +using Common.Search.FuzzSearch; +using Common.Search.SemanticSearch; +using Settings.UI.Library; + +namespace SettingsSearchEvaluation; + +internal static class Evaluator +{ + public static async Task RunAsync( + RunnerOptions options, + IReadOnlyList entries, + DatasetDiagnostics dataset, + IReadOnlyList cases, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(entries); + ArgumentNullException.ThrowIfNull(dataset); + ArgumentNullException.ThrowIfNull(cases); + + var reports = new List(options.Engines.Count); + foreach (var engine in options.Engines) + { + cancellationToken.ThrowIfCancellationRequested(); + reports.Add(engine switch + { + SearchEngineKind.Basic => await EvaluateBasicAsync(options, entries, cases, cancellationToken), + SearchEngineKind.Semantic => await EvaluateSemanticAsync(options, entries, cases, cancellationToken), + _ => throw new InvalidOperationException($"Unsupported engine '{engine}'."), + }); + } + + return new EvaluationReport + { + GeneratedAtUtc = DateTimeOffset.UtcNow, + IndexJsonPath = options.IndexJsonPath, + Dataset = dataset, + CaseCount = cases.Count, + Engines = reports, + }; + } + + private static async Task EvaluateBasicAsync( + RunnerOptions options, + IReadOnlyList entries, + IReadOnlyList cases, + CancellationToken cancellationToken) + { + using var engine = new FuzzSearchEngine(); + + var indexingStopwatch = Stopwatch.StartNew(); + await engine.InitializeAsync(cancellationToken); + await engine.IndexBatchAsync(entries, cancellationToken); + indexingStopwatch.Stop(); + + var metrics = await EvaluateQueryLoopAsync( + cases, + options, + (query, searchOptions, token) => engine.SearchAsync(query, searchOptions, token), + cancellationToken); + + return new EngineEvaluationReport + { + Engine = SearchEngineKind.Basic, + IsAvailable = true, + AvailabilityError = null, + CapabilitiesSummary = "Fuzzy text search engine", + IndexedEntries = entries.Count, + QueryCount = cases.Count, + IndexingTimeMs = indexingStopwatch.Elapsed.TotalMilliseconds, + RecallAtK = metrics.RecallAtK, + Mrr = metrics.Mrr, + SearchLatencyMs = metrics.Latency, + CaseResults = metrics.CaseResults, + }; + } + + private static async Task EvaluateSemanticAsync( + RunnerOptions options, + IReadOnlyList entries, + IReadOnlyList cases, + CancellationToken cancellationToken) + { + var indexName = $"PowerToys.Settings.Eval.{Environment.ProcessId}.{Guid.NewGuid():N}"; + using var engine = new SemanticSearchEngine(indexName); + + var initResult = await engine.InitializeWithResultAsync(cancellationToken); + if (initResult.IsFailure || !engine.IsReady) + { + return new EngineEvaluationReport + { + Engine = SearchEngineKind.Semantic, + IsAvailable = false, + AvailabilityError = FormatError(initResult.Error) ?? "Semantic engine is not ready.", + CapabilitiesSummary = null, + IndexedEntries = 0, + QueryCount = 0, + IndexingTimeMs = 0, + RecallAtK = 0, + Mrr = 0, + SearchLatencyMs = LatencySummary.Empty, + CaseResults = Array.Empty(), + }; + } + + var indexingStopwatch = Stopwatch.StartNew(); + var indexResult = await engine.IndexBatchWithResultAsync(entries, cancellationToken); + if (indexResult.IsFailure) + { + return new EngineEvaluationReport + { + Engine = SearchEngineKind.Semantic, + IsAvailable = false, + AvailabilityError = FormatError(indexResult.Error) ?? "Semantic indexing failed.", + CapabilitiesSummary = BuildCapabilitiesSummary(engine.SemanticCapabilities), + IndexedEntries = 0, + QueryCount = 0, + IndexingTimeMs = indexingStopwatch.Elapsed.TotalMilliseconds, + RecallAtK = 0, + Mrr = 0, + SearchLatencyMs = LatencySummary.Empty, + CaseResults = Array.Empty(), + }; + } + + try + { + await engine.WaitForIndexingCompleteAsync(options.SemanticIndexTimeout); + } + catch (Exception ex) + { + return new EngineEvaluationReport + { + Engine = SearchEngineKind.Semantic, + IsAvailable = false, + AvailabilityError = $"Semantic indexing did not become idle: {ex.Message}", + CapabilitiesSummary = BuildCapabilitiesSummary(engine.SemanticCapabilities), + IndexedEntries = 0, + QueryCount = 0, + IndexingTimeMs = indexingStopwatch.Elapsed.TotalMilliseconds, + RecallAtK = 0, + Mrr = 0, + SearchLatencyMs = LatencySummary.Empty, + CaseResults = Array.Empty(), + }; + } + + indexingStopwatch.Stop(); + var metrics = await EvaluateQueryLoopAsync( + cases, + options, + async (query, searchOptions, token) => + { + var result = await engine.SearchWithResultAsync(query, searchOptions, token); + return result.Value ?? Array.Empty>(); + }, + cancellationToken); + + return new EngineEvaluationReport + { + Engine = SearchEngineKind.Semantic, + IsAvailable = true, + AvailabilityError = null, + CapabilitiesSummary = BuildCapabilitiesSummary(engine.SemanticCapabilities), + IndexedEntries = entries.Count, + QueryCount = cases.Count, + IndexingTimeMs = indexingStopwatch.Elapsed.TotalMilliseconds, + RecallAtK = metrics.RecallAtK, + Mrr = metrics.Mrr, + SearchLatencyMs = metrics.Latency, + CaseResults = metrics.CaseResults, + }; + } + + private static async Task EvaluateQueryLoopAsync( + IReadOnlyList cases, + RunnerOptions options, + Func>>> searchAsync, + CancellationToken cancellationToken) + { + var caseResults = new List(cases.Count); + var latencySamples = new List(Math.Max(1, cases.Count * options.Iterations)); + + var hits = 0; + var reciprocalRankSum = 0.0; + var searchOptions = new SearchOptions + { + MaxResults = options.MaxResults, + IncludeMatchSpans = false, + }; + + foreach (var queryCase in cases) + { + cancellationToken.ThrowIfCancellationRequested(); + + for (int warmup = 0; warmup < options.WarmupIterations; warmup++) + { + _ = await searchAsync(queryCase.Query, searchOptions, cancellationToken); + } + + IReadOnlyList> firstMeasuredResult = Array.Empty>(); + for (int iteration = 0; iteration < options.Iterations; iteration++) + { + var sw = Stopwatch.StartNew(); + var queryResult = await searchAsync(queryCase.Query, searchOptions, cancellationToken); + sw.Stop(); + latencySamples.Add(sw.Elapsed.TotalMilliseconds); + + if (iteration == 0) + { + firstMeasuredResult = queryResult; + } + } + + var rankedIds = firstMeasuredResult.Select(result => result.Item.Id).ToArray(); + var expected = new HashSet(queryCase.ExpectedIds, StringComparer.OrdinalIgnoreCase); + var bestRank = EvaluationMath.FindBestRank(rankedIds, expected); + var hit = bestRank > 0 && bestRank <= options.TopK; + + if (hit) + { + hits++; + } + + if (bestRank > 0) + { + reciprocalRankSum += 1.0 / bestRank; + } + + caseResults.Add(new QueryEvaluationResult + { + Query = queryCase.Query, + ExpectedIds = queryCase.ExpectedIds, + TopResultIds = rankedIds.Take(options.TopK).ToArray(), + BestRank = bestRank, + HitAtK = hit, + Notes = queryCase.Notes, + }); + } + + var totalCases = Math.Max(1, cases.Count); + return new QueryRunMetrics + { + CaseResults = caseResults, + RecallAtK = hits / (double)totalCases, + Mrr = reciprocalRankSum / totalCases, + Latency = EvaluationMath.ComputeLatencySummary(latencySamples), + }; + } + + private static string? FormatError(SearchError? error) + { + if (error == null) + { + return null; + } + + if (!string.IsNullOrWhiteSpace(error.Details)) + { + return $"{error.Message} ({error.Details})"; + } + + return error.Message; + } + + private static string BuildCapabilitiesSummary(SemanticSearchCapabilities? capabilities) + { + if (capabilities == null) + { + return "Capabilities unavailable"; + } + + return $"TextLexical={capabilities.TextLexicalAvailable}, TextSemantic={capabilities.TextSemanticAvailable}, ImageSemantic={capabilities.ImageSemanticAvailable}, ImageOcr={capabilities.ImageOcrAvailable}"; + } + + private sealed class QueryRunMetrics + { + public required IReadOnlyList CaseResults { get; init; } + + public required double RecallAtK { get; init; } + + public required double Mrr { get; init; } + + public required LatencySummary Latency { get; init; } + } +} diff --git a/tools/SettingsSearchEvaluation/LatencySummary.cs b/tools/SettingsSearchEvaluation/LatencySummary.cs new file mode 100644 index 0000000000..fe93fc0921 --- /dev/null +++ b/tools/SettingsSearchEvaluation/LatencySummary.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace SettingsSearchEvaluation; + +internal sealed class LatencySummary +{ + public int Samples { get; init; } + + public double MinMs { get; init; } + + public double P50Ms { get; init; } + + public double P95Ms { get; init; } + + public double MaxMs { get; init; } + + public double AverageMs { get; init; } + + public static LatencySummary Empty { get; } = new(); +} diff --git a/tools/SettingsSearchEvaluation/Program.cs b/tools/SettingsSearchEvaluation/Program.cs new file mode 100644 index 0000000000..b62ae8b678 --- /dev/null +++ b/tools/SettingsSearchEvaluation/Program.cs @@ -0,0 +1,361 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text.Json; + +namespace SettingsSearchEvaluation; + +internal static class Program +{ + private static readonly JsonSerializerOptions OutputJsonOptions = new() + { + WriteIndented = true, + }; + + private static int Main(string[] args) + { + try + { + return MainAsync(args).GetAwaiter().GetResult(); + } + catch (Exception ex) + { + Console.Error.WriteLine($"Unhandled error: {ex.Message}"); + return 99; + } + } + + private static async Task MainAsync(string[] args) + { + if (args.Any(arg => string.Equals(arg, "--help", StringComparison.OrdinalIgnoreCase) || string.Equals(arg, "-h", StringComparison.OrdinalIgnoreCase))) + { + PrintUsage(); + return 0; + } + + if (!TryParseArgs(args, out var options, out var parseError)) + { + Console.Error.WriteLine(parseError); + Console.Error.WriteLine(); + PrintUsage(); + return 2; + } + + if (!File.Exists(options.IndexJsonPath)) + { + Console.Error.WriteLine($"Index file not found: {options.IndexJsonPath}"); + return 3; + } + + if (!string.IsNullOrWhiteSpace(options.CasesJsonPath) && !File.Exists(options.CasesJsonPath)) + { + Console.Error.WriteLine($"Cases file not found: {options.CasesJsonPath}"); + return 3; + } + + var (entries, dataset) = EvaluationDataLoader.LoadEntriesFromFile(options.IndexJsonPath); + var cases = EvaluationDataLoader.LoadCases(options.CasesJsonPath, entries); + if (cases.Count == 0) + { + Console.Error.WriteLine("No valid evaluation cases were found."); + return 3; + } + + Console.WriteLine($"Loaded {entries.Count} entries from '{options.IndexJsonPath}'."); + Console.WriteLine($"Cases: {cases.Count}"); + Console.WriteLine($"Duplicate id buckets: {dataset.DuplicateIdBucketCount}"); + if (dataset.DuplicateIdBucketCount > 0) + { + var largest = dataset.DuplicateIdCounts + .OrderByDescending(x => x.Value) + .Take(5) + .Select(x => $"{x.Key} x{x.Value}"); + Console.WriteLine($"Top duplicate ids: {string.Join(", ", largest)}"); + } + + var report = await Evaluator.RunAsync(options, entries, dataset, cases); + PrintSummary(report, options.TopK); + + if (!string.IsNullOrWhiteSpace(options.OutputJsonPath)) + { + var outputDirectory = Path.GetDirectoryName(options.OutputJsonPath); + if (!string.IsNullOrWhiteSpace(outputDirectory)) + { + Directory.CreateDirectory(outputDirectory); + } + + var json = JsonSerializer.Serialize(report, OutputJsonOptions); + File.WriteAllText(options.OutputJsonPath, json); + Console.WriteLine($"Wrote report to '{options.OutputJsonPath}'."); + } + + return report.Engines.Any(engine => engine.IsAvailable) ? 0 : 4; + } + + private static bool TryParseArgs(string[] args, out RunnerOptions options, out string error) + { + string defaultIndex = GetDefaultIndexPath(); + string? indexPath = null; + string? casesPath = null; + string? outputPath = null; + var maxResults = 10; + var topK = 5; + var iterations = 5; + var warmup = 1; + var semanticTimeoutMs = 15000; + IReadOnlyList engines = new[] { SearchEngineKind.Basic, SearchEngineKind.Semantic }; + + for (int i = 0; i < args.Length; i++) + { + var arg = args[i]; + switch (arg.ToLowerInvariant()) + { + case "--index-json": + if (!TryReadValue(args, ref i, out indexPath)) + { + options = null!; + error = "Missing value for --index-json"; + return false; + } + + break; + case "--cases-json": + if (!TryReadValue(args, ref i, out casesPath)) + { + options = null!; + error = "Missing value for --cases-json"; + return false; + } + + break; + case "--output-json": + if (!TryReadValue(args, ref i, out outputPath)) + { + options = null!; + error = "Missing value for --output-json"; + return false; + } + + break; + case "--engine": + if (!TryReadValue(args, ref i, out var engineText)) + { + options = null!; + error = "Missing value for --engine"; + return false; + } + + if (!TryParseEngines(engineText!, out engines)) + { + options = null!; + error = "Invalid --engine value. Allowed values: basic, semantic, both."; + return false; + } + + break; + case "--max-results": + if (!TryReadInt(args, ref i, out maxResults) || maxResults <= 0) + { + options = null!; + error = "Invalid --max-results value. Must be a positive integer."; + return false; + } + + break; + case "--top-k": + if (!TryReadInt(args, ref i, out topK) || topK <= 0) + { + options = null!; + error = "Invalid --top-k value. Must be a positive integer."; + return false; + } + + break; + case "--iterations": + if (!TryReadInt(args, ref i, out iterations) || iterations <= 0) + { + options = null!; + error = "Invalid --iterations value. Must be a positive integer."; + return false; + } + + break; + case "--warmup": + if (!TryReadInt(args, ref i, out warmup) || warmup < 0) + { + options = null!; + error = "Invalid --warmup value. Must be a non-negative integer."; + return false; + } + + break; + case "--semantic-timeout-ms": + if (!TryReadInt(args, ref i, out semanticTimeoutMs) || semanticTimeoutMs <= 0) + { + options = null!; + error = "Invalid --semantic-timeout-ms value. Must be a positive integer."; + return false; + } + + break; + default: + options = null!; + error = $"Unknown argument: {arg}"; + return false; + } + } + + options = new RunnerOptions + { + IndexJsonPath = Path.GetFullPath(indexPath ?? defaultIndex), + CasesJsonPath = string.IsNullOrWhiteSpace(casesPath) ? null : Path.GetFullPath(casesPath), + Engines = engines, + MaxResults = maxResults, + TopK = topK, + Iterations = iterations, + WarmupIterations = warmup, + SemanticIndexTimeout = TimeSpan.FromMilliseconds(semanticTimeoutMs), + OutputJsonPath = string.IsNullOrWhiteSpace(outputPath) ? null : Path.GetFullPath(outputPath), + }; + error = string.Empty; + return true; + } + + private static string GetDefaultIndexPath() + { + var repoRoot = FindRepoRoot(AppContext.BaseDirectory) ?? Environment.CurrentDirectory; + return Path.GetFullPath(Path.Combine(repoRoot, "src", "settings-ui", "Settings.UI", "Assets", "Settings", "search.index.json")); + } + + private static string? FindRepoRoot(string startingDirectory) + { + var current = new DirectoryInfo(startingDirectory); + while (current != null) + { + var markerPath = Path.Combine(current.FullName, "PowerToys.slnx"); + if (File.Exists(markerPath)) + { + return current.FullName; + } + + current = current.Parent; + } + + return null; + } + + private static bool TryParseEngines(string value, out IReadOnlyList engines) + { + if (string.Equals(value, "both", StringComparison.OrdinalIgnoreCase)) + { + engines = new[] { SearchEngineKind.Basic, SearchEngineKind.Semantic }; + return true; + } + + if (string.Equals(value, "basic", StringComparison.OrdinalIgnoreCase)) + { + engines = new[] { SearchEngineKind.Basic }; + return true; + } + + if (string.Equals(value, "semantic", StringComparison.OrdinalIgnoreCase)) + { + engines = new[] { SearchEngineKind.Semantic }; + return true; + } + + engines = Array.Empty(); + return false; + } + + private static bool TryReadValue(string[] args, ref int index, out string? value) + { + if (index + 1 >= args.Length) + { + value = null; + return false; + } + + index++; + value = args[index]; + return true; + } + + private static bool TryReadInt(string[] args, ref int index, out int value) + { + value = 0; + if (!TryReadValue(args, ref index, out var text)) + { + return false; + } + + return int.TryParse(text, out value); + } + + private static void PrintSummary(EvaluationReport report, int topK) + { + Console.WriteLine(); + Console.WriteLine("=== Evaluation Summary ==="); + Console.WriteLine($"Generated: {report.GeneratedAtUtc:O}"); + Console.WriteLine($"Dataset entries: {report.Dataset.TotalEntries} ({report.Dataset.DistinctIds} distinct ids)"); + Console.WriteLine($"Cases: {report.CaseCount}"); + Console.WriteLine(); + + foreach (var engine in report.Engines) + { + Console.WriteLine($"[{engine.Engine}]"); + if (!engine.IsAvailable) + { + Console.WriteLine($" Unavailable: {engine.AvailabilityError}"); + Console.WriteLine(); + continue; + } + + Console.WriteLine($" Capabilities: {engine.CapabilitiesSummary}"); + Console.WriteLine($" Indexed entries: {engine.IndexedEntries}"); + Console.WriteLine($" Indexing time (ms): {engine.IndexingTimeMs:F2}"); + Console.WriteLine($" Recall@{topK}: {engine.RecallAtK:F4}"); + Console.WriteLine($" MRR: {engine.Mrr:F4}"); + Console.WriteLine($" Search latency ms (avg/p50/p95/max): {engine.SearchLatencyMs.AverageMs:F2}/{engine.SearchLatencyMs.P50Ms:F2}/{engine.SearchLatencyMs.P95Ms:F2}/{engine.SearchLatencyMs.MaxMs:F2}"); + + var misses = engine.CaseResults + .Where(result => !result.HitAtK) + .Take(3) + .ToList(); + + if (misses.Count > 0) + { + Console.WriteLine(" Sample misses:"); + foreach (var miss in misses) + { + var top = miss.TopResultIds.Count == 0 ? "(none)" : string.Join(", ", miss.TopResultIds); + Console.WriteLine($" Query='{miss.Query}', expected='{string.Join("|", miss.ExpectedIds)}', top='{top}'"); + } + } + + Console.WriteLine(); + } + } + + private static void PrintUsage() + { + Console.WriteLine("SettingsSearchEvaluation"); + Console.WriteLine("Evaluates basic and semantic settings search for recall and performance."); + Console.WriteLine(); + Console.WriteLine("Usage:"); + Console.WriteLine(" SettingsSearchEvaluation [options]"); + Console.WriteLine(); + Console.WriteLine("Options:"); + Console.WriteLine(" --index-json Path to settings search index JSON."); + Console.WriteLine(" --cases-json Optional path to evaluation cases JSON."); + Console.WriteLine(" --engine Engine selection. Default: both."); + Console.WriteLine(" --max-results Maximum returned results per query. Default: 10."); + Console.WriteLine(" --top-k Recall cut-off K. Default: 5."); + Console.WriteLine(" --iterations Measured runs per query. Default: 5."); + Console.WriteLine(" --warmup Warmup runs per query. Default: 1."); + Console.WriteLine(" --semantic-timeout-ms Semantic index idle wait timeout in ms. Default: 15000."); + Console.WriteLine(" --output-json Optional output report file."); + Console.WriteLine(" --help Show this help."); + } +} diff --git a/tools/SettingsSearchEvaluation/Properties/AssemblyInfo.cs b/tools/SettingsSearchEvaluation/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..82ce01bccc --- /dev/null +++ b/tools/SettingsSearchEvaluation/Properties/AssemblyInfo.cs @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("SettingsSearchEvaluation.Tests")] diff --git a/tools/SettingsSearchEvaluation/QueryEvaluationResult.cs b/tools/SettingsSearchEvaluation/QueryEvaluationResult.cs new file mode 100644 index 0000000000..266e55a4c9 --- /dev/null +++ b/tools/SettingsSearchEvaluation/QueryEvaluationResult.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace SettingsSearchEvaluation; + +internal sealed class QueryEvaluationResult +{ + public required string Query { get; init; } + + public required IReadOnlyList ExpectedIds { get; init; } + + public required IReadOnlyList TopResultIds { get; init; } + + public required int BestRank { get; init; } + + public required bool HitAtK { get; init; } + + public string? Notes { get; init; } +} diff --git a/tools/SettingsSearchEvaluation/README.md b/tools/SettingsSearchEvaluation/README.md new file mode 100644 index 0000000000..54bc36a54c --- /dev/null +++ b/tools/SettingsSearchEvaluation/README.md @@ -0,0 +1,51 @@ +# Settings Search Evaluation + +This tool evaluates Settings search quality and latency for: + +- `basic` search (`FuzzSearchEngine`) +- `semantic` search (`SemanticSearchEngine`) + +It reports: + +- `Recall@K` +- `MRR` (mean reciprocal rank) +- Search latency (`avg`, `p50`, `p95`, `max`) +- Dataset diagnostics including duplicate `SettingEntry.Id` buckets + +## Run + +Build with Visual Studio `MSBuild.exe` (the project references native components): + +```powershell +$vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" +$msbuild = & $vswhere -latest -products * -requires Microsoft.Component.MSBuild -find MSBuild\**\Bin\MSBuild.exe +& $msbuild tools\SettingsSearchEvaluation\SettingsSearchEvaluation.csproj ` + /t:Build /p:Configuration=Debug /p:Platform=arm64 /m:1 /nologo +``` + +Run the built executable: + +```powershell +.\tools\SettingsSearchEvaluation\bin\arm64\Debug\net9.0-windows10.0.26100.0\SettingsSearchEvaluation.exe ` + --index-json src/settings-ui/Settings.UI/Assets/Settings/search.index.json ` + --cases-json tools/SettingsSearchEvaluation/cases/settings-search-cases.sample.json ` + --engine both ` + --top-k 5 ` + --iterations 5 ` + --warmup 1 ` + --output-json tools/SettingsSearchEvaluation/artifacts/report.json +``` + +## Case file format + +```json +[ + { + "query": "color picker", + "expectedIds": ["ColorPicker"], + "notes": "Module entry" + } +] +``` + +If `--cases-json` is not provided, fallback cases are auto-generated from the index headers. diff --git a/tools/SettingsSearchEvaluation/RunnerOptions.cs b/tools/SettingsSearchEvaluation/RunnerOptions.cs new file mode 100644 index 0000000000..793cf61d8a --- /dev/null +++ b/tools/SettingsSearchEvaluation/RunnerOptions.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace SettingsSearchEvaluation; + +internal sealed class RunnerOptions +{ + public required string IndexJsonPath { get; init; } + + public string? CasesJsonPath { get; init; } + + public required IReadOnlyList Engines { get; init; } + + public int MaxResults { get; init; } = 10; + + public int TopK { get; init; } = 5; + + public int Iterations { get; init; } = 5; + + public int WarmupIterations { get; init; } = 1; + + public TimeSpan SemanticIndexTimeout { get; init; } = TimeSpan.FromSeconds(15); + + public string? OutputJsonPath { get; init; } +} diff --git a/tools/SettingsSearchEvaluation/SearchEngineKind.cs b/tools/SettingsSearchEvaluation/SearchEngineKind.cs new file mode 100644 index 0000000000..016135a609 --- /dev/null +++ b/tools/SettingsSearchEvaluation/SearchEngineKind.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace SettingsSearchEvaluation; + +internal enum SearchEngineKind +{ + Basic, + Semantic, +} diff --git a/tools/SettingsSearchEvaluation/SettingsSearchEvaluation.csproj b/tools/SettingsSearchEvaluation/SettingsSearchEvaluation.csproj new file mode 100644 index 0000000000..789903a625 --- /dev/null +++ b/tools/SettingsSearchEvaluation/SettingsSearchEvaluation.csproj @@ -0,0 +1,18 @@ + + + + + + Exe + SettingsSearchEvaluation + SettingsSearchEvaluation + enable + enable + $(DefaultItemExcludes);artifacts\**\*;bin\**\*;obj\**\* + + + + + + + diff --git a/tools/SettingsSearchEvaluation/cases/settings-search-cases.sample.json b/tools/SettingsSearchEvaluation/cases/settings-search-cases.sample.json new file mode 100644 index 0000000000..02ec90c9a2 --- /dev/null +++ b/tools/SettingsSearchEvaluation/cases/settings-search-cases.sample.json @@ -0,0 +1,72 @@ +[ + { + "query": "always on top", + "expectedIds": [ "AlwaysOnTop" ], + "notes": "Module entry" + }, + { + "query": "awake mode", + "expectedIds": [ "Awake_ModeSettingsCard" ], + "notes": "Feature setting" + }, + { + "query": "color picker", + "expectedIds": [ "ColorPicker" ], + "notes": "Module entry" + }, + { + "query": "fancy zones", + "expectedIds": [ "FancyZones" ], + "notes": "Module entry" + }, + { + "query": "launch cmd pal", + "expectedIds": [ "CmdPal_Launch" ], + "notes": "CmdPal launch setting" + }, + { + "query": "image resizer", + "expectedIds": [ "ImageResizer" ], + "notes": "Module entry" + }, + { + "query": "keyboard manager", + "expectedIds": [ "KeyboardManager" ], + "notes": "Module entry" + }, + { + "query": "mouse jump", + "expectedIds": [ "MouseUtils_Enable_MouseJump" ], + "notes": "Module entry" + }, + { + "query": "mouse without borders", + "expectedIds": [ "MouseWithoutBorders" ], + "notes": "Module setting" + }, + { + "query": "peek", + "expectedIds": [ "Peek" ], + "notes": "Module entry" + }, + { + "query": "power rename", + "expectedIds": [ "PowerRename" ], + "notes": "Module entry" + }, + { + "query": "power toys run", + "expectedIds": [ "PowerLauncher" ], + "notes": "Module entry" + }, + { + "query": "registry preview", + "expectedIds": [ "RegistryPreview" ], + "notes": "Module entry" + }, + { + "query": "workspaces", + "expectedIds": [ "Workspaces_EnableToggleControl_HeaderText" ], + "notes": "Module setting" + } +]