From 888a1427244056ee23c2f04749a650a2683942d7 Mon Sep 17 00:00:00 2001 From: Michael Jolley Date: Wed, 12 Aug 2026 11:18:10 -0500 Subject: [PATCH] [CmdPal] Consolidate search ranking changes (#49832) ## What's going on The stacked pull requests are blocked by GitHub's stack merge flow. This gives the full remaining search ranking change set one PR against `main`. ## The plan - Consolidates the open work from #49190, #49191, #49194, #49195, #49197, #49246, #49247, and #49249. - Keeps the existing stack unchanged while this PR provides an alternate merge path. --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 5459b847-afeb-4163-a803-977759dd92df Copilot-Session: 49185697-186e-406f-b081-8c985c134274 Copilot-Session: 92905c83-2de6-449c-b4a1-a08003fe2576 Copilot-Session: efe987e5-9297-47fc-af99-2f49b057285a Copilot-Session: f42917e2-d298-4bde-9056-79a9e6e17dfa --- DATA_AND_PRIVACY.md | 2 + .../Helpers/InternalListHelpers.cs | 78 +++ .../Commands/MainListPage.cs | 507 ++++++++++++++---- .../Commands/MainListPageSearchTelemetry.cs | 248 +++++++++ .../Commands/MainListRanker.cs | 17 + .../HistoryItem.cs | 11 +- .../TelemetrySearchResultSelectedMessage.cs | 15 + .../Messages/TelemetrySearchResultsMessage.cs | 12 + .../ProviderSettings.cs | 25 + .../ProviderSettingsViewModel.cs | 35 ++ .../RecentCommandsManager.cs | 171 ++++-- .../TopLevelViewModel.cs | 10 +- .../Events/CmdPalSearchResultSelected.cs | 50 ++ .../Events/CmdPalSearchResults.cs | 55 ++ .../Helpers/TelemetryForwarder.cs | 23 +- .../Settings/ExtensionPage.xaml | 11 + .../Strings/en-us/Resources.resw | 18 + .../EarlyFrameRelevanceTests.cs | 315 +++++++++++ .../FastFirstPaintTests.cs | 215 ++++++++ .../MainListRankerTests.cs | 141 +++++ .../ProviderWeightingTests.cs | 214 ++++++++ .../RecentCommandsTests.cs | 268 +++++---- .../RelevanceHarnessTests.cs | 284 ++++++++++ .../ScoringParallelEquivalenceTests.cs | 252 +++++++++ .../ScoringTestCatalog.cs | 92 ++++ .../ScoringThroughputHarnessTests.cs | 319 +++++++++++ .../SearchTelemetryTests.cs | 198 +++++++ 27 files changed, 3343 insertions(+), 243 deletions(-) create mode 100644 src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/Commands/MainListPageSearchTelemetry.cs create mode 100644 src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/Messages/TelemetrySearchResultSelectedMessage.cs create mode 100644 src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/Messages/TelemetrySearchResultsMessage.cs create mode 100644 src/modules/cmdpal/Microsoft.CmdPal.UI/Events/CmdPalSearchResultSelected.cs create mode 100644 src/modules/cmdpal/Microsoft.CmdPal.UI/Events/CmdPalSearchResults.cs create mode 100644 src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/EarlyFrameRelevanceTests.cs create mode 100644 src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/FastFirstPaintTests.cs create mode 100644 src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/MainListRankerTests.cs create mode 100644 src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/ProviderWeightingTests.cs create mode 100644 src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/RelevanceHarnessTests.cs create mode 100644 src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/ScoringParallelEquivalenceTests.cs create mode 100644 src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/ScoringTestCatalog.cs create mode 100644 src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/ScoringThroughputHarnessTests.cs create mode 100644 src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/SearchTelemetryTests.cs diff --git a/DATA_AND_PRIVACY.md b/DATA_AND_PRIVACY.md index e7afe68017..f92231615d 100644 --- a/DATA_AND_PRIVACY.md +++ b/DATA_AND_PRIVACY.md @@ -145,6 +145,8 @@ Thank you for using PowerToys! | Microsoft.PowerToys.CmdPalHotkeySummoned | Logs when the Command Palette is summoned via hotkey, distinguishing between global and context-specific hotkeys. | | Microsoft.PowerToys.CmdPalInvokeResult | Records the result type of a Command Palette invocation. | | Microsoft.PowerToys.CmdPalProcessStarted | Triggered when the Command Palette process is started. | +| Microsoft.PowerToys.CmdPal_SearchResults | Triggered when a search is triggered on the main list, including query length, result count, and duration. | +| Microsoft.PowerToys.CmdPal_SearchResultSelected | Triggered when a search result is selected, including query length, selection index, and result tier. | ### Crop and lock diff --git a/src/modules/cmdpal/Microsoft.CmdPal.Common/Helpers/InternalListHelpers.cs b/src/modules/cmdpal/Microsoft.CmdPal.Common/Helpers/InternalListHelpers.cs index eb2ad98aef..bf0e4479af 100644 --- a/src/modules/cmdpal/Microsoft.CmdPal.Common/Helpers/InternalListHelpers.cs +++ b/src/modules/cmdpal/Microsoft.CmdPal.Common/Helpers/InternalListHelpers.cs @@ -60,6 +60,84 @@ public static partial class InternalListHelpers } } + // Minimum item count before the parallel path earns back its partitioning and merge overhead, + // which keeps commands and small app sets on the serial path. + private const int ParallelScoringThreshold = 512; + + /// + /// Order-preserving parallel variant of that scores + /// contiguous index ranges on separate threads and concatenates them back in partition order, + /// producing a pre-sort buffer identical to the serial path. The scoring function has to be + /// pure per item, since each item is scored by exactly one thread. + /// + public static RoScored[] FilterListWithScoresParallel( + IReadOnlyList? items, + in FuzzyQuery query, + in ScoringFunction scoreFunction) + { + if (items is null || items.Count == 0) + { + return []; + } + + var count = items.Count; + var partitions = Math.Min(Environment.ProcessorCount, Math.Max(1, count / ParallelScoringThreshold)); + + if (count < ParallelScoringThreshold || partitions <= 1) + { + return FilterListWithScores(items, query, scoreFunction); + } + + // Copy the by-ref parameters into locals so the parallel body can capture them. FuzzyQuery + // is immutable, so one shared copy is safe to read from every thread. + var q = query; + var fn = scoreFunction; + var source = items; + + var partitionResults = new List>[partitions]; + + System.Threading.Tasks.Parallel.For(0, partitions, p => + { + var start = (int)((long)p * count / partitions); + var end = (int)((long)(p + 1) * count / partitions); + + var local = new List>(end - start); + for (var i = start; i < end; i++) + { + var item = source[i]; + var score = fn(in q, item); + if (score > 0) + { + local.Add(new RoScored(item, score)); + } + } + + partitionResults[p] = local; + }); + + var total = 0; + for (var p = 0; p < partitions; p++) + { + total += partitionResults[p].Count; + } + + // Contiguous ranges merged in partition order reproduce the serial buffer's exact + // enumeration order. + var buffer = GC.AllocateUninitializedArray>(total); + var pos = 0; + for (var p = 0; p < partitions; p++) + { + var list = partitionResults[p]; + for (var j = 0; j < list.Count; j++) + { + buffer[pos++] = list[j]; + } + } + + Array.Sort(buffer, 0, total, default(RoScoredDescendingComparer)); + return buffer; + } + private static void GrowBuffer(ref RoScored[] buffer, int count) { var newBuffer = ArrayPool>.Shared.Rent(buffer.Length * 2); diff --git a/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/Commands/MainListPage.cs b/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/Commands/MainListPage.cs index d2c7be8dd3..a3b35cf712 100644 --- a/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/Commands/MainListPage.cs +++ b/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/Commands/MainListPage.cs @@ -6,6 +6,7 @@ #define CMDPAL_FF_MAINPAGE_TIME_RAISE_ITEMS */ +using System.Collections.Immutable; using System.Collections.Specialized; using System.Diagnostics; using CommunityToolkit.Mvvm.Messaging; @@ -49,6 +50,10 @@ public sealed partial class MainListPage : DynamicListPage, private readonly ScoringFunction _fallbackScoringFunction; private readonly IFuzzyMatcherProvider _fuzzyMatcherProvider; + // All main-page search telemetry state and emission is owned by this dedicated type, keeping + // MainListPage responsible for producing results rather than for tracking telemetry bookkeeping. + private readonly MainListPageSearchTelemetry _searchTelemetry = new(); + // Stable separator instances so that the VM cache and InPlaceUpdateList // recognise them across successive GetItems() calls private readonly Separator _pinnedSeparator = new(Resources.home_sections_pinned_title); @@ -63,16 +68,34 @@ public sealed partial class MainListPage : DynamicListPage, private RoScored[]? _filteredItems; private RoScored[]? _filteredApps; - // Keep as IEnumerable for deferred execution. Fallback item titles are updated - // asynchronously, so scoring must happen lazily when GetItems is called. - private IEnumerable>? _scoredFallbackItems; + // Published with _filteredApps so filtering uses the query that produced the scores. + private int _filteredAppsQueryLength; + + // Global/special fallbacks are scored on the render path, not at keystroke time, because + // their titles resolve asynchronously. We snapshot the source list and query together so a + // superseding keystroke replaces both atomically. + private IReadOnlyList? _globalFallbackSources; + private FuzzyQuery _globalFallbackQuery; + + // Common fallbacks use query-independent scores, so freezing them is safe; only their live + // titles decide whether they render. private IEnumerable>? _fallbackItems; private bool _includeApps; private bool _filteredItemsIncludesApps; + // Last per-provider settings we reacted to, so a settings reload can tell whether any + // provider's search weight actually changed and only then re-rank the active query. + private ImmutableDictionary? _lastProviderSettingsSnapshot; + private int AppResultLimit => AllAppsCommandProvider.TopLevelResultLimit; + // Longest query to filter fuzzy app matches on. This prevents weak app matches on short queries. + private const int ShortQueryAppFilterMaxLength = 2; + + // Minimum tier an app must reach to appear for a short query. + private const RankTier ShortQueryAppFilterMinTier = RankTier.AcronymWordBoundary; + private InterlockedBoolean _fullRefreshRequested; private InterlockedBoolean _refreshRunning; private InterlockedBoolean _refreshRequested; @@ -100,7 +123,7 @@ public sealed partial class MainListPage : DynamicListPage, _appStateService = appStateService; _tlcManager = topLevelCommandManager; _fuzzyMatcherProvider = fuzzyMatcherProvider; - _scoringFunction = (in query, item) => ScoreTopLevelItem(in query, item, _appStateService.State.RecentCommands, _fuzzyMatcherProvider.Current); + _scoringFunction = (in query, item) => ScoreTopLevelItem(in query, item, _appStateService.State.RecentCommands, _fuzzyMatcherProvider.Current, ResolveProviderSearchWeight); _fallbackScoringFunction = (in _, item) => ScoreFallbackItem(item, _settingsService.Settings.FallbackRanks); _tlcManager.PropertyChanged += TlcManager_PropertyChanged; @@ -259,22 +282,125 @@ public sealed partial class MainListPage : DynamicListPage, private IListItem[] GetSearchViewItems() { - var validScoredFallbacks = _scoredFallbackItems? - .Where(s => !string.IsNullOrWhiteSpace(s.Item.Title)) - .ToList(); + // Score global fallbacks against their current titles so a fallback whose title + // resolved after first paint gets the right score. Cheap: only a handful are configured. + var validScoredFallbacks = ScoreDeferredFallbacks(_globalFallbackSources, _globalFallbackQuery, _scoringFunction); var validFallbacks = _fallbackItems? .Where(s => !string.IsNullOrWhiteSpace(s.Item.Title)) .ToList(); - return MainListPageResultFactory.Create( + // Remove fuzzy-only app matches for short queries so frecency-boosted weak matches don't + // appear while typing. + var filteredApps = FilterAppsForShortQueries(_filteredApps, _filteredAppsQueryLength); + + var result = MainListPageResultFactory.Create( _filteredItems, validScoredFallbacks, - _filteredApps, + filteredApps, validFallbacks, _resultsSeparator, _fallbacksSeparator, AppResultLimit); + + // Snapshot the rendered order plus every scored input and the query length together, so + // selection telemetry resolves an invoked item's rank, tier, and query length from this one + // generation off the hot path. These are plain reference assignments - no extra allocation. + _searchTelemetry.CaptureSearchView( + result, + _filteredItems, + _filteredApps, + validScoredFallbacks, + _fallbackItems, + SearchText?.Length ?? 0); + + return result; + } + + // Scores the current global-fallback snapshot against its query, dropping any whose title is + // still empty. Static so it can be unit tested with a fake slow source. + internal static List>? ScoreDeferredFallbacks( + IReadOnlyList? sources, + in FuzzyQuery query, + ScoringFunction scoringFunction) + { + if (sources is null || sources.Count == 0) + { + return null; + } + + var scored = InternalListHelpers.FilterListWithScores(sources, query, scoringFunction); + if (scored.Length == 0) + { + return null; + } + + List>? valid = null; + foreach (var s in scored) + { + if (string.IsNullOrWhiteSpace(s.Item.Title)) + { + continue; + } + + valid ??= new List>(scored.Length); + valid.Add(s); + } + + return valid; + } + + // Returns a filtered view of the scored array without reordering it. + internal static IList>? FilterAppsForShortQueries( + RoScored[]? scoredApps, + int queryLength) + { + if (scoredApps is null || scoredApps.Length == 0) + { + return scoredApps; + } + + if (queryLength <= 0 || queryLength > ShortQueryAppFilterMaxLength) + { + return scoredApps; + } + + var keep = GetHighConfidenceAppsCount(scoredApps, ShortQueryAppFilterMinTier); + return keep == scoredApps.Length + ? scoredApps + : new ArraySegment>(scoredApps, 0, keep); + } + + // Qualifying apps form a contiguous prefix because the array is already sorted by score. + internal static int GetHighConfidenceAppsCount(IReadOnlyList> scored, RankTier minTier) + { + var min = (int)minTier; + for (var i = 0; i < scored.Count; i++) + { + if ((int)MainListRanker.TierOf(scored[i].Score) < min) + { + return i; + } + } + + return scored.Count; + } + + // Applies the short-query filter and result limit to the telemetry count. + internal static int GetVisibleAppCount(RoScored[]? scoredApps, int queryLength, int appResultLimit) + { + if (scoredApps is null || scoredApps.Length == 0) + { + return 0; + } + + var count = scoredApps.Length; + if (queryLength > 0 && queryLength <= ShortQueryAppFilterMaxLength) + { + count = GetHighConfidenceAppsCount(scoredApps, ShortQueryAppFilterMinTier); + } + + return Math.Min(count, appResultLimit); } private IListItem[] GetDefaultViewItems() @@ -361,8 +487,12 @@ public sealed partial class MainListPage : DynamicListPage, { _filteredItems = null; _filteredApps = null; + _filteredAppsQueryLength = 0; _fallbackItems = null; - _scoredFallbackItems = null; + _globalFallbackSources = null; + + // Clear the paired query too, so both are reset together. + _globalFallbackQuery = default; } public override void UpdateSearchText(string oldSearch, string newSearch) @@ -403,6 +533,10 @@ public sealed partial class MainListPage : DynamicListPage, if (aliases.CheckAlias(newSearch)) { + // An alias query supersedes any normal query whose settled-search telemetry is + // still pending in the debounce; drop it so the superseded query never emits. + _searchTelemetry.CancelPendingResults(); + if (_filteredItemsIncludesApps != _includeApps) { lock (_tlcManager.TopLevelCommands) @@ -422,6 +556,17 @@ public sealed partial class MainListPage : DynamicListPage, } var commands = _tlcManager.TopLevelCommands; + + // Inputs captured under the lock so the heavy scoring below can run off it. GetItems() + // takes the same lock, so it now only contends with the short snapshot and publish sections. + IReadOnlyList itemsSource; + IReadOnlyList appsSource; + IReadOnlyList fallbackSource; + IListItem[] globalFallbackSources; + bool includeAppsSnapshot; + bool tookFullCatalog = false; + + // ===== SNAPSHOT PHASE (under lock) ===== lock (commands) { if (token.IsCancellationRequested) @@ -463,58 +608,37 @@ public sealed partial class MainListPage : DynamicListPage, { _filteredItemsIncludesApps = _includeApps; ClearResults(); + + // Drop any pending settled-search telemetry so a cleared query never emits. + _searchTelemetry.ClearSearchView(); + var wasAlreadyEmpty = string.IsNullOrWhiteSpace(oldSearch); RequestRefresh(fullRefresh: true, interval: wasAlreadyEmpty ? null : TimeSpan.Zero); return; } - // If the new string doesn't start with the old string, then we can't - // re-use previous results. Reset _filteredItems, and keep er moving. - if (!newSearch.StartsWith(oldSearch, StringComparison.CurrentCultureIgnoreCase)) - { - ClearResults(); - } + includeAppsSnapshot = _includeApps; - // If the internal state has changed, reset _filteredItems to reset the list. - if (_filteredItemsIncludesApps != _includeApps) - { - ClearResults(); - } + // A query that doesn't extend the old one, or a change in app inclusion, means we + // can't re-use the previous results and have to rebuild from the full catalog. On an + // extend we re-score only the previously matched subset. + var reset = !newSearch.StartsWith(oldSearch, StringComparison.CurrentCultureIgnoreCase) + || _filteredItemsIncludesApps != includeAppsSnapshot; - if (token.IsCancellationRequested) - { - return; - } + var prevFilteredItems = reset ? null : _filteredItems; + var prevApps = reset ? null : _filteredApps; + var prevFallbacks = reset ? null : _fallbackItems; - var newFilteredItems = Enumerable.Empty(); - var newFallbacks = Enumerable.Empty(); - var newApps = Enumerable.Empty(); - - if (_filteredItems is not null) - { - newFilteredItems = _filteredItems.Select(s => s.Item); - } - - if (token.IsCancellationRequested) - { - return; - } - - if (_filteredApps is not null) - { - newApps = _filteredApps.Select(s => s.Item); - } - - if (token.IsCancellationRequested) - { - return; - } - - if (_fallbackItems is not null) - { - newFallbacks = _fallbackItems.Select(s => s.Item); - } + IEnumerable newFilteredItems = prevFilteredItems is not null + ? prevFilteredItems.Select(s => s.Item) + : Enumerable.Empty(); + IEnumerable newApps = prevApps is not null + ? prevApps.Select(s => s.Item) + : Enumerable.Empty(); + IEnumerable newFallbacks = prevFallbacks is not null + ? prevFallbacks.Select(s => s.Item) + : Enumerable.Empty(); if (token.IsCancellationRequested) { @@ -525,6 +649,7 @@ public sealed partial class MainListPage : DynamicListPage, // with a list of all our commands & apps. if (!newFilteredItems.Any() && !newApps.Any()) { + tookFullCatalog = true; newFilteredItems = commands.Where(s => !s.IsFallback); // Fallbacks are always included in the list, even if they @@ -537,9 +662,7 @@ public sealed partial class MainListPage : DynamicListPage, return; } - _filteredItemsIncludesApps = _includeApps; - - if (_includeApps) + if (includeAppsSnapshot) { var allNewApps = AllAppsCommandProvider.Page.GetItems().Cast().ToList(); @@ -562,71 +685,151 @@ public sealed partial class MainListPage : DynamicListPage, } } - var searchQuery = _fuzzyMatcherProvider.Current.PrecomputeQuery(SearchText); + // Materialize every source while still under the lock, so the scoring passes never + // touch the live TopLevelCommands collection or the app provider. + itemsSource = MaterializeSource(newFilteredItems); + appsSource = MaterializeSource(newApps); + fallbackSource = MaterializeSource(newFallbacks); + globalFallbackSources = [.. specialFallbacks]; + } - // Produce a list of everything that matches the current filter. - _filteredItems = InternalListHelpers.FilterListWithScores(newFilteredItems, searchQuery, _scoringFunction); + if (token.IsCancellationRequested) + { + return; + } + + // ===== SCORING PHASE (off the lock) ===== + // The dominant apps pass is parallelized, commands and fallbacks stay serial, and none of + // it holds the TopLevelCommands lock any more, so it no longer blocks GetItems()/render. + // + // Snapshot every scoring input once, up front: the live fields can be swapped mid-pass when + // a selection calls WithHistoryItem on another thread, which would mix two frecency + // snapshots into one pass or race a parallel thread against an unwarmed history index. + var recent = _appStateService.State.RecentCommands; + recent.PrewarmIndex(); + var matcher = _fuzzyMatcherProvider.Current; + var settings = _settingsService.Settings; + var scoringNow = DateTimeOffset.UtcNow; + + // Precompute from the snapshotted newSearch, not the live SearchText, which a newer + // keystroke may already have advanced past. + var searchQuery = matcher.PrecomputeQuery(newSearch); + + // Every installed app belongs to the well-known AllApps provider, so its weight is constant + // for the whole pass and we resolve it once instead of once per app. + var appsProviderWeight = ResolveProviderSearchWeight(settings, AllAppsCommandProvider.WellKnownId); + Func commandsProviderLookup = item => ResolveProviderSearchWeight(settings, item); + Func appsProviderLookup = _ => appsProviderWeight; + + ScoringFunction commandsScorer = (in FuzzyQuery q, IListItem item) => + ScoreTopLevelItem(in q, item, recent, matcher, commandsProviderLookup, scoringNow); + ScoringFunction appsScorer = (in FuzzyQuery q, IListItem item) => + ScoreTopLevelItem(in q, item, recent, matcher, appsProviderLookup, scoringNow); + + var scoredFilteredItems = InternalListHelpers.FilterListWithScores(itemsSource, searchQuery, commandsScorer); + + if (token.IsCancellationRequested) + { + return; + } + + var scoredFallbackItems = InternalListHelpers.FilterListWithScores(fallbackSource, searchQuery, _fallbackScoringFunction); + + if (token.IsCancellationRequested) + { + return; + } + + RoScored[]? scoredApps = null; + if (appsSource.Count > 0) + { + scoredApps = InternalListHelpers.FilterListWithScoresParallel(appsSource, searchQuery, appsScorer); if (token.IsCancellationRequested) { return; } - - IEnumerable newFallbacksForScoring = commands.Where(s => s.IsFallback && configuredGlobalFallbackIds.Contains(s.Id)); - _scoredFallbackItems = InternalListHelpers.FilterListWithScores(newFallbacksForScoring, searchQuery, _scoringFunction); - - if (token.IsCancellationRequested) - { - return; - } - - _fallbackItems = InternalListHelpers.FilterListWithScores(newFallbacks ?? [], searchQuery, _fallbackScoringFunction); - - if (token.IsCancellationRequested) - { - return; - } - - // Produce a list of filtered apps with the appropriate limit - if (newApps.Any()) - { - _filteredApps = InternalListHelpers.FilterListWithScores(newApps, searchQuery, _scoringFunction); - - if (token.IsCancellationRequested) - { - return; - } - } + } #if CMDPAL_FF_MAINPAGE_TIME_RAISE_ITEMS - var filterDoneTimestamp = stopwatch.ElapsedMilliseconds; + var filterDoneTimestamp = stopwatch.ElapsedMilliseconds; #endif + + // ===== PUBLISH PHASE (under lock) ===== + // The critical section is the field swaps only. Telemetry debounce and refresh throttling + // happen after the lock so _searchTelemetryLock never nests under the commands lock. + var deterministicResultCount = 0; + lock (commands) + { + // A newer keystroke cancels this token before doing its own work, so a stale snapshot + // can never overwrite a newer query's results. + if (token.IsCancellationRequested) + { + return; + } + + if (tookFullCatalog) + { + _filteredItemsIncludesApps = includeAppsSnapshot; + } + + _filteredItems = scoredFilteredItems; + _fallbackItems = scoredFallbackItems; + + // Snapshot the global fallbacks and query, but score them later on the render path, + // since their titles are still resolving asynchronously (BeginUpdate, above). + _globalFallbackSources = globalFallbackSources; + _globalFallbackQuery = searchQuery; + + // With no apps source, publish null so a rebuild clears any stale set, matching the old + // ClearResults behavior. + _filteredApps = appsSource.Count > 0 ? scoredApps : null; + + // Publish the length with the array so filtering and telemetry use the same query. + _filteredAppsQueryLength = _filteredApps is null ? 0 : newSearch.Length; + if (isUserInput) { - // Make sure that the throttle delay is consistent from the user's perspective, even if filtering - // takes a long time. If we always use the full throttle duration, then a slow filter could make the UI feel sluggish. - var adjustedInterval = RaiseItemsChangedThrottleForUserInput - stopwatch.Elapsed; - if (adjustedInterval < TimeSpan.Zero) - { - adjustedInterval = TimeSpan.Zero; - } - - RequestRefresh(fullRefresh: true, adjustedInterval); - } - else - { - RequestRefresh(fullRefresh: true); + deterministicResultCount = (_filteredItems?.Length ?? 0) + + GetVisibleAppCount(_filteredApps, _filteredAppsQueryLength, AppResultLimit); } #if CMDPAL_FF_MAINPAGE_TIME_RAISE_ITEMS var listPageUpdatedTimestamp = stopwatch.ElapsedMilliseconds; Logger.LogDebug($"Render items with '{newSearch}' in {listPageUpdatedTimestamp}ms /d {listPageUpdatedTimestamp - filterDoneTimestamp}ms"); #endif + } - stopwatch.Stop(); + // Getting here means the swap happened, since the superseded path returns inside the lock. + stopwatch.Stop(); + + if (isUserInput) + { + // Queue a settled-search telemetry event. It's debounced so it only fires once the + // query settles, and it carries the query LENGTH only, never the text. + _searchTelemetry.QueueSearchResults(newSearch.Length, deterministicResultCount, stopwatch.ElapsedMilliseconds); + + // Make sure that the throttle delay is consistent from the user's perspective, even if filtering + // takes a long time. If we always use the full throttle duration, then a slow filter could make the UI feel sluggish. + var adjustedInterval = RaiseItemsChangedThrottleForUserInput - stopwatch.Elapsed; + if (adjustedInterval < TimeSpan.Zero) + { + adjustedInterval = TimeSpan.Zero; + } + + RequestRefresh(fullRefresh: true, adjustedInterval); + } + else + { + RequestRefresh(fullRefresh: true); } } + // Materializes a source into a stable, indexable snapshot so scoring can run off the lock. + // Anything already an IReadOnlyList passes through; lazy LINQ over live data gets copied. + private static IReadOnlyList MaterializeSource(IEnumerable items) + => items as IReadOnlyList ?? items.ToArray(); + private bool ActuallyLoading() { var allApps = AllAppsCommandProvider.Page; @@ -640,7 +843,9 @@ public sealed partial class MainListPage : DynamicListPage, in FuzzyQuery query, IListItem topLevelOrAppItem, IRecentCommandsManager history, - IPrecomputedFuzzyMatcher precomputedFuzzyMatcher) + IPrecomputedFuzzyMatcher precomputedFuzzyMatcher, + Func? providerWeightLookup = null, + DateTimeOffset? now = null) { var title = topLevelOrAppItem.Title; if (string.IsNullOrWhiteSpace(title)) @@ -705,14 +910,19 @@ public sealed partial class MainListPage : DynamicListPage, return 0; } - var frecencyWeight = history.GetCommandHistoryWeight(id); + var frecencyWeight = history.GetCommandHistoryWeight(id, now ?? DateTimeOffset.UtcNow); var aliasSubstringBonus = isAliasSubstringMatch && !isAliasMatch ? MainListRanker.AliasSubstringBonus : 0.0; + // Per-provider weight is a within-tier nudge only. Resolving it here (rather than in + // the tier classifier) guarantees it can never promote an item across a tier boundary. + var providerWeight = providerWeightLookup?.Invoke(topLevelOrAppItem) ?? ProviderSearchWeight.Normal; + var providerBonus = MainListRanker.ProviderBonus(providerWeight); + var withinTier = MainListRanker.WithinTierScore( lexicalQuality, frecencyWeight, aliasSubstringBonus, - providerBonus: 0.0); + providerBonus: providerBonus); return MainListRanker.Pack(tier, withinTier); } @@ -752,6 +962,8 @@ public sealed partial class MainListPage : DynamicListPage, { RecentCommands = state.RecentCommands.WithHistoryItem(id), }); + + _searchTelemetry.ReportSelection(topLevelOrAppItem, _resultsSeparator, _fallbacksSeparator); } private static string IdForTopLevelOrAppItem(IListItem topLevelOrAppItem) @@ -767,6 +979,34 @@ public sealed partial class MainListPage : DynamicListPage, } } + // Resolves the user-configured per-provider search weight for an item. Top-level commands + // carry their own provider id; installed apps all belong to the well-known "AllApps" + // provider, so app items are weighted by that provider's setting. The static overloads take a + // settings snapshot so the hot path resolves against one captured SettingsModel. + private ProviderSearchWeight ResolveProviderSearchWeight(IListItem topLevelOrAppItem) + => ResolveProviderSearchWeight(_settingsService.Settings, topLevelOrAppItem); + + private static ProviderSearchWeight ResolveProviderSearchWeight(SettingsModel settings, IListItem topLevelOrAppItem) + { + var providerId = topLevelOrAppItem is TopLevelViewModel topLevel + ? topLevel.CommandProviderId + : AllAppsCommandProvider.WellKnownId; + + return ResolveProviderSearchWeight(settings, providerId); + } + + private static ProviderSearchWeight ResolveProviderSearchWeight(SettingsModel settings, string providerId) + { + if (string.IsNullOrEmpty(providerId)) + { + return ProviderSearchWeight.Normal; + } + + return settings.ProviderSettings.TryGetValue(providerId, out var providerSettings) + ? providerSettings.SearchWeight + : ProviderSearchWeight.Normal; + } + public void Receive(ClearSearchMessage message) => SearchText = string.Empty; public void Receive(UpdateFallbackItemsMessage message) @@ -778,13 +1018,72 @@ public sealed partial class MainListPage : DynamicListPage, private void SettingsChangedHandler(ISettingsService sender, SettingsModel args) => HotReloadSettings(args); - private void HotReloadSettings(SettingsModel settings) => ShowDetails = settings.ShowAppDetails; + private void HotReloadSettings(SettingsModel settings) + { + ShowDetails = settings.ShowAppDetails; + + // A per-provider search-weight change has to reorder the query that is already on screen. + // Scoring reads the weight live, but scored results are cached, so without an explicit + // re-score the active query keeps its old order until the next keystroke. Detect a weight + // change and re-rank the current search in place. + var providerSettings = settings.ProviderSettings; + var weightsChanged = ProviderWeightsChanged(_lastProviderSettingsSnapshot, providerSettings); + _lastProviderSettingsSnapshot = providerSettings; + + if (weightsChanged && !string.IsNullOrEmpty(SearchText)) + { + RerankActiveSearch(); + } + } + + // Re-scores the current query off the UI thread so a settings change (e.g. a per-provider + // search-weight change) reorders the results already shown. This reuses the same non-reset + // re-score path as an app-inclusion refresh: the retained matches are re-scored with the new + // weights, which is sufficient because provider weight only nudges order within a tier and + // never changes which items match. + private void RerankActiveSearch() + { + var current = SearchText; + if (!string.IsNullOrEmpty(current)) + { + _ = Task.Run(() => UpdateSearchTextCore(current, current, isUserInput: false)); + } + } + + // True when the effective per-provider search weight differs between two snapshots. A provider + // absent from a snapshot is treated as Normal, so adding or removing an entry whose weight is + // Normal does not count as a change. + private static bool ProviderWeightsChanged( + ImmutableDictionary? previous, + ImmutableDictionary current) + { + previous ??= ImmutableDictionary.Empty; + if (ReferenceEquals(previous, current)) + { + return false; + } + + var keys = new HashSet(previous.Keys, StringComparer.Ordinal); + keys.UnionWith(current.Keys); + foreach (var key in keys) + { + var previousWeight = previous.TryGetValue(key, out var p) ? p.SearchWeight : ProviderSearchWeight.Normal; + var currentWeight = current.TryGetValue(key, out var c) ? c.SearchWeight : ProviderSearchWeight.Normal; + if (previousWeight != currentWeight) + { + return true; + } + } + + return false; + } public void Dispose() { _cancellationTokenSource?.Cancel(); _cancellationTokenSource?.Dispose(); _fallbackUpdateManager.Dispose(); + _searchTelemetry.Dispose(); _tlcManager.PropertyChanged -= TlcManager_PropertyChanged; _tlcManager.TopLevelCommands.CollectionChanged -= Commands_CollectionChanged; diff --git a/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/Commands/MainListPageSearchTelemetry.cs b/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/Commands/MainListPageSearchTelemetry.cs new file mode 100644 index 0000000000..050fdb0b90 --- /dev/null +++ b/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/Commands/MainListPageSearchTelemetry.cs @@ -0,0 +1,248 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using CommunityToolkit.Mvvm.Messaging; +using Microsoft.CmdPal.Common.Helpers; +using Microsoft.CmdPal.Core.Common.Helpers; +using Microsoft.CmdPal.UI.ViewModels.Messages; +using Microsoft.CommandPalette.Extensions; +using Microsoft.CommandPalette.Extensions.Toolkit; + +namespace Microsoft.CmdPal.UI.ViewModels.MainPage; + +/// +/// Owns all of the main-page search telemetry state and emission, so +/// stays responsible for producing results and this type is the single, clearly-identifiable owner +/// of the (privacy-safe) telemetry. Everything here is opt-in and non-identifying: search events +/// carry only query LENGTH, result count and latency; selection events carry only query LENGTH, the +/// invoked item's visible rank and its ranker tier - never the raw query text or item content. +/// +/// Search events are emitted only when a query settles (trailing-edge debounce) so we never send an +/// event on every keystroke; selection events are emitted only when the user invokes a result. All +/// emission is measured at boundaries, never inside the per-item scoring loop. +/// +internal sealed partial class MainListPageSearchTelemetry : IDisposable +{ + private static readonly TimeSpan SettleDelay = TimeSpan.FromMilliseconds(600); + + private readonly ThrottledDebouncedAction _resultsDebounce; + private readonly Lock _pendingLock = new(); + private (int QueryLength, int ResultCount, long LatencyMs) _pendingResults; + + // Snapshots of the most recent rendered search results, read off the hot path (only when the + // user invokes a result) to resolve the invoked item's visible rank and ranker tier. The scored + // inputs and query length are captured together with the rendered items at render time, so + // selection telemetry resolves rank, tier and query length from one coherent generation even + // after a newer query has already published fresh scored fields. + private IReadOnlyList? _lastViewItems; + private IReadOnlyList>? _lastScoredGlobalFallbacks; + private RoScored[]? _lastViewFilteredItems; + private RoScored[]? _lastViewFilteredApps; + private IEnumerable>? _lastViewFallbackItems; + private int _lastViewQueryLength; + + public MainListPageSearchTelemetry() + { + _resultsDebounce = new ThrottledDebouncedAction(EmitPendingResults, SettleDelay); + } + + // Stores the latest settled-search metrics and (re)arms the debounce. Only the query LENGTH is + // retained - the query text is never stored for telemetry. + public void QueueSearchResults(int queryLength, int resultCount, long latencyMs) + { + lock (_pendingLock) + { + _pendingResults = (queryLength, resultCount, latencyMs); + } + + _resultsDebounce.Invoke(); + } + + // Snapshots the rendered order plus every scored input and the query length together, so a later + // selection resolves an invoked item's rank, tier and query length from this one generation off + // the hot path. These are plain reference assignments - no extra allocation. + public void CaptureSearchView( + IReadOnlyList renderedItems, + RoScored[]? filteredItems, + RoScored[]? filteredApps, + IReadOnlyList>? scoredGlobalFallbacks, + IEnumerable>? fallbackItems, + int queryLength) + { + _lastViewItems = renderedItems; + _lastScoredGlobalFallbacks = scoredGlobalFallbacks; + _lastViewFilteredItems = filteredItems; + _lastViewFilteredApps = filteredApps; + _lastViewFallbackItems = fallbackItems; + _lastViewQueryLength = queryLength; + } + + // Drops any pending settled-search event without emitting it. Used when an alias query supersedes + // a normal query whose telemetry is still pending in the debounce. + public void CancelPendingResults() => _resultsDebounce.Cancel(); + + // Drops any pending settled-search event and forgets the last rendered search view, so a cleared + // query never emits and a subsequent selection resolves to nothing. + public void ClearSearchView() + { + _resultsDebounce.Cancel(); + _lastViewItems = null; + _lastScoredGlobalFallbacks = null; + _lastViewFilteredItems = null; + _lastViewFilteredApps = null; + _lastViewFallbackItems = null; + _lastViewQueryLength = 0; + } + + // Emits selection telemetry when the user invokes a result during an active search. Runs only on + // invoke (a deliberate, infrequent user action - never on the typing/scoring path) and captures + // only non-identifying aggregates: the query LENGTH, the invoked item's visible rank, and its + // ranker tier. Nothing is emitted for the default (no-search) view, or when the invoked item is + // not among the last rendered search results. + public void ReportSelection(IListItem invoked, Separator resultsSeparator, Separator fallbacksSeparator) + { + // Resolve everything from the last rendered search-view snapshot so the invoked item's rank, + // tier, and the reported query length all come from one generation. If the last render was + // the default (no-search) view, _lastViewItems is null and nothing is emitted. + var lastView = _lastViewItems; + if (lastView is null || _lastViewQueryLength <= 0) + { + return; + } + + var index = ResolveVisibleIndex(lastView, invoked, resultsSeparator, fallbacksSeparator); + if (index < 0) + { + return; + } + + var packed = (_lastViewFilteredItems ?? Enumerable.Empty>()) + .Concat(_lastViewFilteredApps ?? Enumerable.Empty>()) + .Concat(_lastScoredGlobalFallbacks ?? Enumerable.Empty>()); + + var tier = ResolveSelectedTier(invoked, packed, _lastViewFallbackItems); + if (tier == RankTier.None) + { + return; + } + + WeakReferenceMessenger.Default.Send(BuildSearchSelectedMessage(_lastViewQueryLength, index, tier)); + } + + private void EmitPendingResults() + { + (int QueryLength, int ResultCount, long LatencyMs) snapshot; + lock (_pendingLock) + { + snapshot = _pendingResults; + } + + if (snapshot.QueryLength <= 0) + { + return; + } + + WeakReferenceMessenger.Default.Send( + BuildSearchResultsMessage(snapshot.QueryLength, snapshot.ResultCount, snapshot.LatencyMs)); + } + + // Builds the settled-search telemetry payload from a query string, capturing only its LENGTH. + // Exposed for tests to prove the raw query text is never carried. + internal static TelemetrySearchResultsMessage BuildSearchResultsMessage(string query, int resultCount, long latencyMs) + => BuildSearchResultsMessage(query?.Length ?? 0, resultCount, latencyMs); + + internal static TelemetrySearchResultsMessage BuildSearchResultsMessage(int queryLength, int resultCount, long latencyMs) + { + var length = Math.Max(queryLength, 0); + var count = Math.Max(resultCount, 0); + var latency = latencyMs < 0 ? 0UL : (ulong)latencyMs; + return new TelemetrySearchResultsMessage(length, count, count == 0, latency); + } + + // Builds the selection telemetry payload, capturing only the query LENGTH, the selected rank, + // and the ranker tier. Exposed for tests to prove the raw query text is never carried. + internal static TelemetrySearchResultSelectedMessage BuildSearchSelectedMessage(string query, int selectedIndex, RankTier selectedTier) + => BuildSearchSelectedMessage(query?.Length ?? 0, selectedIndex, selectedTier); + + internal static TelemetrySearchResultSelectedMessage BuildSearchSelectedMessage(int queryLength, int selectedIndex, RankTier selectedTier) + => new(Math.Max(queryLength, 0), selectedIndex, selectedTier); + + // Zero-based visible rank of an invoked item within the rendered results, skipping the section + // separators. Returns -1 when the item is not present (e.g. it was invoked from a different view). + internal static int ResolveVisibleIndex(IReadOnlyList? renderedResults, IListItem invoked, params IListItem[] separators) + { + if (renderedResults is null) + { + return -1; + } + + var visible = 0; + foreach (var item in renderedResults) + { + var isSeparator = false; + foreach (var separator in separators) + { + if (ReferenceEquals(item, separator)) + { + isSeparator = true; + break; + } + } + + if (isSeparator) + { + continue; + } + + if (ReferenceEquals(item, invoked)) + { + return visible; + } + + visible++; + } + + return -1; + } + + // Resolves the ranker tier of an invoked item. Packed sources (commands, apps, global + // fallbacks) decode their tier via MainListRanker.TierOf; common fallbacks carry rank-based + // (non-packed) scores, so they are reported at the fallback floor. Returns None when the item + // is not found in any source. + internal static RankTier ResolveSelectedTier( + IListItem invoked, + IEnumerable>? packedResults, + IEnumerable>? fallbackResults) + { + if (packedResults is not null) + { + foreach (var scored in packedResults) + { + if (ReferenceEquals(scored.Item, invoked)) + { + return MainListRanker.TierOf(scored.Score); + } + } + } + + if (fallbackResults is not null) + { + foreach (var scored in fallbackResults) + { + if (ReferenceEquals(scored.Item, invoked)) + { + return RankTier.FallbackFloor; + } + } + } + + return RankTier.None; + } + + public void Dispose() => _resultsDebounce.Dispose(); +} diff --git a/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/Commands/MainListRanker.cs b/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/Commands/MainListRanker.cs index 79bbeb3f1d..563121ea5e 100644 --- a/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/Commands/MainListRanker.cs +++ b/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/Commands/MainListRanker.cs @@ -31,6 +31,23 @@ internal static class MainListRanker // exact alias, which gets its own top tier). Mirrors the previous +1-before-x10 boost. internal const double AliasSubstringBonus = 10.0; + // Magnitude of the per-provider within-tier nudge. Deliberately small - half a point of + // lexical quality (LexicalScale = 10) - so a Higher/Lower provider only breaks near-ties + // and reorders items that already share a tier. It can NEVER move an item across a tier + // boundary because the packed within-tier score is clamped to a single tier's band. + internal const double ProviderWeightBonus = 5.0; + + /// + /// Maps a per-provider to an additive within-tier + /// bonus. Lower subtracts, Normal is neutral, Higher adds. The enum's underlying value is + /// the sign of the nudge, so the result is simply the weight times + /// . Note the nudge is slightly asymmetric at the tier + /// floor: clamps the within-tier score to a non-negative band, so a + /// Lower nudge on an item already scoring near 0 (weak match, no history) can clamp to 0 + /// and read the same as Normal, whereas Higher always applies. + /// + public static double ProviderBonus(ProviderSearchWeight weight) => (int)weight * ProviderWeightBonus; + /// /// Packs a tier and within-tier score into a single descending-sortable integer. /// Returns 0 for so non-matches are filtered by the diff --git a/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/HistoryItem.cs b/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/HistoryItem.cs index 0f8d90acea..554202f9ac 100644 --- a/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/HistoryItem.cs +++ b/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/HistoryItem.cs @@ -2,7 +2,7 @@ // The Microsoft Corporation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using CommunityToolkit.Mvvm.ComponentModel; +using System; namespace Microsoft.CmdPal.UI.ViewModels; @@ -11,4 +11,13 @@ public record HistoryItem public required string CommandId { get; init; } public required int Uses { get; init; } + + /// + /// Gets the moment this command was last invoked. Persisted so ranking can apply real + /// time-decay instead of the previous list-position heuristic. History written before + /// this field existed deserializes to default(DateTimeOffset); + /// treats that sentinel as a mild backdate so + /// day-one ordering degrades gracefully to Uses-ordering rather than going all-equal. + /// + public DateTimeOffset LastUsed { get; init; } } diff --git a/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/Messages/TelemetrySearchResultSelectedMessage.cs b/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/Messages/TelemetrySearchResultSelectedMessage.cs new file mode 100644 index 0000000000..cb79b3f562 --- /dev/null +++ b/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/Messages/TelemetrySearchResultSelectedMessage.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.CmdPal.UI.ViewModels.MainPage; + +namespace Microsoft.CmdPal.UI.ViewModels.Messages; + +/// +/// Telemetry message sent when the user selects (invokes) a main-page search result. +/// Carries only non-identifying aggregates - never the selected item's title/id or the raw +/// query text. is the query's character count, +/// is the zero-based rank of the selected result, and is its ranker tier. +/// +public record TelemetrySearchResultSelectedMessage(int QueryLength, int SelectedIndex, RankTier SelectedTier); diff --git a/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/Messages/TelemetrySearchResultsMessage.cs b/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/Messages/TelemetrySearchResultsMessage.cs new file mode 100644 index 0000000000..1d970c33de --- /dev/null +++ b/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/Messages/TelemetrySearchResultsMessage.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.CmdPal.UI.ViewModels.Messages; + +/// +/// Telemetry message sent when a main-page search query settles. +/// Carries only non-identifying aggregates - never the raw query text. +/// is the query's character count. +/// +public record TelemetrySearchResultsMessage(int QueryLength, int ResultCount, bool NoResults, ulong LatencyMs); diff --git a/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/ProviderSettings.cs b/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/ProviderSettings.cs index c1b6a48b0a..3e8d64bef8 100644 --- a/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/ProviderSettings.cs +++ b/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/ProviderSettings.cs @@ -7,6 +7,25 @@ using System.Text.Json.Serialization; namespace Microsoft.CmdPal.UI.ViewModels; +/// +/// Per-provider relevance nudge applied to main/root page search results. The value is a +/// within-tier bonus only: it can reorder items that already share a relevance tier but can +/// never move an item across a tier boundary. The underlying integer values are the sign of +/// the bonus (Lower subtracts, Higher adds), and is 0 so a missing or +/// legacy setting deserializes to the neutral default. +/// +public enum ProviderSearchWeight +{ + /// De-prioritize this provider's results within their tier. + Lower = -1, + + /// Default. No provider nudge. + Normal = 0, + + /// Prioritize this provider's results within their tier. + Higher = 1, +} + public record ProviderSettings { // List of built-in fallbacks that should not have global results enabled by default @@ -18,6 +37,12 @@ public record ProviderSettings public bool IsEnabled { get; init; } = true; + /// + /// Per-provider within-tier ranking nudge for main-page search. Defaults to + /// ; missing/legacy values deserialize to Normal. + /// + public ProviderSearchWeight SearchWeight { get; init; } = ProviderSearchWeight.Normal; + private ImmutableDictionary? _fallbackCommands = ImmutableDictionary.Empty; diff --git a/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/ProviderSettingsViewModel.cs b/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/ProviderSettingsViewModel.cs index e6338b5646..831e63b054 100644 --- a/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/ProviderSettingsViewModel.cs +++ b/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/ProviderSettingsViewModel.cs @@ -134,6 +134,41 @@ public partial class ProviderSettingsViewModel : ObservableObject } } + /// + /// Per-provider search weight surfaced as a 0/1/2 ComboBox index (Lower / Normal / + /// Higher) so it can bind to ComboBox.SelectedIndex in the same style as the + /// other per-provider options. Persists to . + /// + public int SearchWeightIndex + { + get => _providerSettings.SearchWeight switch + { + ProviderSearchWeight.Lower => 0, + ProviderSearchWeight.Higher => 2, + _ => 1, + }; + set + { + var newWeight = value switch + { + 0 => ProviderSearchWeight.Lower, + 2 => ProviderSearchWeight.Higher, + _ => ProviderSearchWeight.Normal, + }; + + if (newWeight != _providerSettings.SearchWeight) + { + var newSettings = _providerSettings with { SearchWeight = newWeight }; + _settingsService.UpdateSettings(s => s with + { + ProviderSettings = s.ProviderSettings.SetItem(_provider.ProviderId, newSettings), + }); + _providerSettings = newSettings; + OnPropertyChanged(nameof(SearchWeightIndex)); + } + } + } + /// /// Gets a value indicating whether returns true if we have a settings page /// that's initialized, or we are still working on initializing that diff --git a/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/RecentCommandsManager.cs b/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/RecentCommandsManager.cs index 51e714a526..dd7a89ec93 100644 --- a/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/RecentCommandsManager.cs +++ b/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/RecentCommandsManager.cs @@ -3,61 +3,156 @@ // See the LICENSE file in the project root for more information. using System.Collections.Immutable; +using System.Text.Json.Serialization; namespace Microsoft.CmdPal.UI.ViewModels; public record RecentCommandsManager : IRecentCommandsManager { + // Recency half-life: a command's recency contribution halves every this many days. Three + // days balances an interactive launcher's usage rhythm - commands touched in the last few + // days still feel "recent", while month-old one-offs decay toward zero and stop crowding + // out newer habits. + internal const double HalfLifeDays = 3.0; + + // Baseline points for a just-used command (recency == 1) before the frequency term. Kept + // near the previous top recency bucket so the frecency signal keeps roughly the same + // influence in the ranker's within-tier math (MainListRanker.FrecencyScale == 1.0). + internal const double BaseWeight = 10.0; + + // Points added per unit of log2(uses + 1), scaled by recency. log() keeps heavy usage + // helpful without letting it dominate recency (uses 1 -> +10, 7 -> +30, 31 -> +50). + internal const double FrequencyWeight = 10.0; + + // Upper bound on the returned weight. Holds the signal in the ~0..70 range the previous + // implementation produced, so tier ordering is unaffected (the tier always dominates and + // frecency only reorders within a tier). + internal const double MaxWeight = 70.0; + + // Retain a few hundred commands. Large enough to remember habitual commands across weeks + // of use, small enough that the persisted JSON stays tiny (tens of KB). The old cap of 50 + // evicted still-useful history within a single busy session on active machines. + internal const int MaxHistoryEntries = 500; + + // Defensive fallback for any history item that carries a default (missing) LastUsed - for + // example a value constructed without a timestamp, or state written by a build that predates + // the LastUsed field. Such an item is treated as used one day ago: recent enough that it + // still ranks, mild enough that a single fresh use outranks it, and uniform so a group of + // them falls back to Uses (frequency) ordering instead of collapsing to all-equal or zero. + // In practice this rarely fires: earlier builds never actually persisted history (the + // internal History property was dropped by the serializer, see [JsonInclude] below), so + // upgrading users start from an empty store rather than one full of timestamp-less items. + internal static readonly TimeSpan LegacyBackdate = TimeSpan.FromDays(1); + private ImmutableList? _history = ImmutableList.Empty; + // Cached commandId -> entry lookup over History, rebuilt lazily whenever History is + // (re)assigned and never persisted. ScoreTopLevelItem calls GetCommandHistoryWeight for + // every candidate item on every keystroke, and History can hold up to MaxHistoryEntries + // (500) entries, so a plain linear scan would be O(items x history) per keystroke. The + // dictionary keeps each lookup O(1) so the hot path stays cheap as the store grows. + [JsonIgnore] + private Dictionary? _index; + + // Persisted so recent-command frecency (including the LastUsed timestamps) survives a + // restart. [JsonInclude] is required because the property is internal; without it the + // source-generated serializer emits an empty object and history is silently dropped. + // Old persisted state (an empty object) still deserializes fine - History stays empty. + [JsonInclude] internal ImmutableList History { get => _history ?? ImmutableList.Empty; - init => _history = value; + init + { + _history = value; + + // Invalidate the cached lookup so it is rebuilt from the new list on next use. + // A record 'with' copy carries over the old field, so this reset is what keeps + // the index from going stale after History changes. + _index = null; + } + } + + private Dictionary Index + { + get + { + if (_index is null) + { + // Ordinal to match the string '==' comparison the previous linear scan used. + var map = new Dictionary(StringComparer.Ordinal); + foreach (var item in History) + { + // History is most-recent-first and command ids are unique in it, but keep + // the first (most recent) occurrence if a duplicate ever slips in so the + // lookup matches the old FirstOrDefault behavior. + map.TryAdd(item.CommandId, item); + } + + _index = map; + } + + return _index; + } } public RecentCommandsManager() { } + /// + /// Builds the lazy command-id lookup now, on the calling thread. The build isn't thread-safe, + /// so call this once before scoring items in parallel; the reads afterward are safe. + /// + public void PrewarmIndex() => _ = Index; + public int GetCommandHistoryWeight(string commandId) + => GetCommandHistoryWeight(commandId, DateTimeOffset.UtcNow); + + /// + /// Computes the time-decayed frecency weight for a command relative to . + /// Recency uses an exponential half-life decay and frequency uses log(uses); the two are + /// combined so recency leads while frequency amplifies. The parameterless overload uses the + /// current time; this one lets a batch pin a single snapshot. + /// + public int GetCommandHistoryWeight(string commandId, DateTimeOffset now) { - var entry = History - .Index() - .Where(item => item.Item.CommandId == commandId) - .FirstOrDefault(); - - // These numbers are vaguely scaled so that "VS" will make "Visual Studio" the - // match after one use. - // Usually it has a weight of 84, compared to 109 for the VS cmd prompt - if (entry.Item is not null) + if (!Index.TryGetValue(commandId, out var entry)) { - var index = entry.Index; - - // First, add some weight based on how early in the list this appears - var bucket = index switch - { - _ when index <= 2 => 35, - _ when index <= 10 => 25, - _ when index <= 15 => 15, - _ when index <= 35 => 10, - _ => 5, - }; - - // Then, add weight for how often this is used, but cap the weight from usage. - var uses = Math.Min(entry.Item.Uses * 5, 35); - - return bucket + uses; + return 0; } - return 0; + // Migrate items with a default (missing) timestamp to a mild backdate so they degrade + // to Uses-ordering instead of appearing brand-new or invisible. See LegacyBackdate. + var lastUsed = entry.LastUsed == default ? now - LegacyBackdate : entry.LastUsed; + + // Clamp age at zero so a slightly-future timestamp (e.g. clock skew) can't amplify. + var ageDays = Math.Max(0.0, (now - lastUsed).TotalDays); + + // Exponential time decay: recency in (0, 1], halving every HalfLifeDays. + var recency = Math.Pow(2.0, -ageDays / HalfLifeDays); + + // Frequency via log2 so heavy usage helps but can't outrun recency. + var frequency = Math.Log(entry.Uses + 1, 2); + + var weight = recency * (BaseWeight + (FrequencyWeight * frequency)); + + return (int)Math.Round(Math.Clamp(weight, 0.0, MaxWeight)); } /// /// Returns a new RecentCommandsManager with the given command added/promoted in history. - /// Pure function — does not mutate this instance. + /// Pure function - does not mutate this instance. /// public RecentCommandsManager WithHistoryItem(string commandId) + => WithHistoryItem(commandId, DateTimeOffset.UtcNow); + + /// + /// Records a use of at the explicit time . + /// The public overload uses the current time; this overload lets tests inject + /// strictly-increasing timestamps so time-decay behavior is deterministic. + /// + internal RecentCommandsManager WithHistoryItem(string commandId, DateTimeOffset now) { var existing = History.FirstOrDefault(item => item.CommandId == commandId); ImmutableList newHistory; @@ -65,18 +160,18 @@ public record RecentCommandsManager : IRecentCommandsManager if (existing is not null) { newHistory = History.Remove(existing); - var updated = existing with { Uses = existing.Uses + 1 }; + var updated = existing with { Uses = existing.Uses + 1, LastUsed = now }; newHistory = newHistory.Insert(0, updated); } else { - var newItem = new HistoryItem { CommandId = commandId, Uses = 1 }; + var newItem = new HistoryItem { CommandId = commandId, Uses = 1, LastUsed = now }; newHistory = History.Insert(0, newItem); } - if (newHistory.Count > 50) + if (newHistory.Count > MaxHistoryEntries) { - newHistory = newHistory.RemoveRange(50, newHistory.Count - 50); + newHistory = newHistory.RemoveRange(MaxHistoryEntries, newHistory.Count - MaxHistoryEntries); } return this with { History = newHistory }; @@ -87,5 +182,17 @@ public interface IRecentCommandsManager { int GetCommandHistoryWeight(string commandId); + /// + /// Frecency weight for a command at an explicit evaluation time, so a whole batch can be + /// scored against one snapshot instead of a moving clock. + /// + int GetCommandHistoryWeight(string commandId, DateTimeOffset now); + RecentCommandsManager WithHistoryItem(string commandId); + + /// + /// Builds any lazy internal state on the calling thread. Call it once before scoring items in + /// parallel, since the build itself isn't thread-safe. + /// + void PrewarmIndex(); } diff --git a/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/TopLevelViewModel.cs b/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/TopLevelViewModel.cs index 877d01aa34..bc498e8554 100644 --- a/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/TopLevelViewModel.cs +++ b/src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/TopLevelViewModel.cs @@ -433,12 +433,16 @@ public sealed partial class TopLevelViewModel : ObservableObject, IListItem, IEx // RPC to check type if (model is IFallbackCommandItem fallback) { - var wasEmpty = string.IsNullOrEmpty(Title); + var oldTitle = Title; // RPC for method fallback.FallbackHandler.UpdateQuery(newQuery); - var isEmpty = string.IsNullOrEmpty(Title); - return wasEmpty != isEmpty; + var newTitle = Title; + + // Report any title change, not just an empty <-> non-empty flip: the render path + // re-scores fallbacks off this signal, so a change like "server01" -> "server02" + // must still trigger a refresh or the fallback keeps its stale score and position. + return !string.Equals(oldTitle, newTitle, StringComparison.Ordinal); } return false; diff --git a/src/modules/cmdpal/Microsoft.CmdPal.UI/Events/CmdPalSearchResultSelected.cs b/src/modules/cmdpal/Microsoft.CmdPal.UI/Events/CmdPalSearchResultSelected.cs new file mode 100644 index 0000000000..62bc909dce --- /dev/null +++ b/src/modules/cmdpal/Microsoft.CmdPal.UI/Events/CmdPalSearchResultSelected.cs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Tracing; +using Microsoft.PowerToys.Telemetry; +using Microsoft.PowerToys.Telemetry.Events; + +namespace Microsoft.CmdPal.UI.Events; + +/// +/// Tracks which main-page search result the user selected (invoked). +/// Purpose: measure whether the ranking overhaul surfaces the wanted result near the top. +/// Privacy: only non-identifying aggregates are captured. The selected item's title, +/// subtitle, id, and the raw query text are never logged - only the query character +/// length, the zero-based rank (index) of the selected result, and its ranker tier name. +/// Emission goes through , which respects the existing +/// PowerToys data-diagnostics consent gate. +/// +[EventData] +[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties)] +public class CmdPalSearchResultSelected : EventBase, IEvent +{ + /// + /// Gets or sets the character length of the query at selection (never the query text). + /// + public int QueryLength { get; set; } + + /// + /// Gets or sets the zero-based rank of the selected result within the visible results. + /// + public int SelectedIndex { get; set; } + + /// + /// Gets or sets the ranker tier name of the selected result (e.g. "ExactTitle"). + /// This is a fixed, non-identifying enum name, not user content. + /// + public string SelectedTier { get; set; } + + public CmdPalSearchResultSelected(int queryLength, int selectedIndex, string selectedTier) + { + EventName = "CmdPal_SearchResultSelected"; + QueryLength = queryLength; + SelectedIndex = selectedIndex; + SelectedTier = selectedTier; + } + + public PartA_PrivTags PartA_PrivTags => PartA_PrivTags.ProductAndServiceUsage; +} diff --git a/src/modules/cmdpal/Microsoft.CmdPal.UI/Events/CmdPalSearchResults.cs b/src/modules/cmdpal/Microsoft.CmdPal.UI/Events/CmdPalSearchResults.cs new file mode 100644 index 0000000000..5b0e1cbf26 --- /dev/null +++ b/src/modules/cmdpal/Microsoft.CmdPal.UI/Events/CmdPalSearchResults.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Tracing; +using Microsoft.PowerToys.Telemetry; +using Microsoft.PowerToys.Telemetry.Events; + +namespace Microsoft.CmdPal.UI.Events; + +/// +/// Tracks the outcome of a settled main-page search query. +/// Purpose: measure search relevance and perceived speed for the ranking overhaul. +/// Privacy: only non-identifying aggregates are captured. The raw query text is never +/// logged - only its character length. No titles, subtitles, paths, or app names are +/// captured. Emission goes through , which respects the +/// existing PowerToys data-diagnostics consent gate. +/// +[EventData] +[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties)] +public class CmdPalSearchResults : EventBase, IEvent +{ + /// + /// Gets or sets the character length of the query (never the query text itself). + /// + public int QueryLength { get; set; } + + /// + /// Gets or sets the number of deterministic first-paint results (commands and apps) + /// produced for the query. + /// + public int ResultCount { get; set; } + + /// + /// Gets or sets a value indicating whether the query produced no deterministic results. + /// + public bool NoResults { get; set; } + + /// + /// Gets or sets the time in milliseconds to produce the deterministic first-paint results. + /// + public ulong LatencyMs { get; set; } + + public CmdPalSearchResults(int queryLength, int resultCount, bool noResults, ulong latencyMs) + { + EventName = "CmdPal_SearchResults"; + QueryLength = queryLength; + ResultCount = resultCount; + NoResults = noResults; + LatencyMs = latencyMs; + } + + public PartA_PrivTags PartA_PrivTags => PartA_PrivTags.ProductAndServiceUsage; +} diff --git a/src/modules/cmdpal/Microsoft.CmdPal.UI/Helpers/TelemetryForwarder.cs b/src/modules/cmdpal/Microsoft.CmdPal.UI/Helpers/TelemetryForwarder.cs index 84fab2734b..267b8d2a33 100644 --- a/src/modules/cmdpal/Microsoft.CmdPal.UI/Helpers/TelemetryForwarder.cs +++ b/src/modules/cmdpal/Microsoft.CmdPal.UI/Helpers/TelemetryForwarder.cs @@ -21,7 +21,9 @@ internal sealed class TelemetryForwarder : IRecipient, IRecipient, IRecipient, - IRecipient + IRecipient, + IRecipient, + IRecipient { public TelemetryForwarder() { @@ -29,6 +31,8 @@ internal sealed class TelemetryForwarder : WeakReferenceMessenger.Default.Register(this); WeakReferenceMessenger.Default.Register(this); WeakReferenceMessenger.Default.Register(this); + WeakReferenceMessenger.Default.Register(this); + WeakReferenceMessenger.Default.Register(this); } // Message handlers for telemetry events from core layer @@ -68,6 +72,23 @@ internal sealed class TelemetryForwarder : message.EndBands)); } + public void Receive(TelemetrySearchResultsMessage message) + { + PowerToysTelemetry.Log.WriteEvent(new CmdPalSearchResults( + message.QueryLength, + message.ResultCount, + message.NoResults, + message.LatencyMs)); + } + + public void Receive(TelemetrySearchResultSelectedMessage message) + { + PowerToysTelemetry.Log.WriteEvent(new CmdPalSearchResultSelected( + message.QueryLength, + message.SelectedIndex, + message.SelectedTier.ToString())); + } + // Static method for logging session duration from UI layer public static void LogSessionDuration( ulong durationMs, diff --git a/src/modules/cmdpal/Microsoft.CmdPal.UI/Settings/ExtensionPage.xaml b/src/modules/cmdpal/Microsoft.CmdPal.UI/Settings/ExtensionPage.xaml index 785abaa86e..15cbb4beb1 100644 --- a/src/modules/cmdpal/Microsoft.CmdPal.UI/Settings/ExtensionPage.xaml +++ b/src/modules/cmdpal/Microsoft.CmdPal.UI/Settings/ExtensionPage.xaml @@ -79,6 +79,17 @@ Value="{x:Bind ViewModel.IsEnabled, Mode=OneWay}"> + + + + + + + diff --git a/src/modules/cmdpal/Microsoft.CmdPal.UI/Strings/en-us/Resources.resw b/src/modules/cmdpal/Microsoft.CmdPal.UI/Strings/en-us/Resources.resw index 80755b0045..e6b7d5a2fa 100644 --- a/src/modules/cmdpal/Microsoft.CmdPal.UI/Strings/en-us/Resources.resw +++ b/src/modules/cmdpal/Microsoft.CmdPal.UI/Strings/en-us/Resources.resw @@ -348,6 +348,24 @@ Right-click to remove the key combination, thereby deactivating the shortcut. Choose when the alias runs. Direct runs as soon as you type the alias. Indirect runs after a trailing space. + + Search weight + + + Nudge this provider's results up or down among equally-relevant matches. + + + Search weight + + + Lower + + + Normal + + + Higher + Built-in diff --git a/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/EarlyFrameRelevanceTests.cs b/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/EarlyFrameRelevanceTests.cs new file mode 100644 index 0000000000..ee74b637b4 --- /dev/null +++ b/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/EarlyFrameRelevanceTests.cs @@ -0,0 +1,315 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.CmdPal.Common.Helpers; +using Microsoft.CmdPal.Common.Text; +using Microsoft.CmdPal.UI.ViewModels.Commands; +using Microsoft.CmdPal.UI.ViewModels.MainPage; +using Microsoft.CommandPalette.Extensions; +using Microsoft.CommandPalette.Extensions.Toolkit; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.CmdPal.UI.ViewModels.UnitTests; + +[TestClass] +public sealed partial class EarlyFrameRelevanceTests +{ + public TestContext TestContext { get; set; } = null!; + + private sealed partial class CatalogItem : ListItem, IPrecomputedListItem + { + private FuzzyTargetCache _titleCache; + private FuzzyTargetCache _subtitleCache; + + public CatalogItem(string title, string subtitle, string id) + : base(new NoOpCommand() { Id = id }) + { + Title = title; + Subtitle = subtitle; + Id = id; + } + + public string Id { get; } + + public FuzzyTarget GetTitleTarget(IPrecomputedFuzzyMatcher matcher) => _titleCache.GetOrUpdate(matcher, Title); + + public FuzzyTarget GetSubtitleTarget(IPrecomputedFuzzyMatcher matcher) => _subtitleCache.GetOrUpdate(matcher, Subtitle); + } + + private static IPrecomputedFuzzyMatcher CreateMatcher() => new PrecomputedFuzzyMatcher(new PrecomputedFuzzyMatcherOptions()); + + private static ScoringFunction BuildScoringFunction(IRecentCommandsManager history, IPrecomputedFuzzyMatcher matcher) + => (in FuzzyQuery query, IListItem item) => MainListPage.ScoreTopLevelItem(query, item, history, matcher, null); + + private static RoScored[] Score(IReadOnlyList apps, string rawQuery, IRecentCommandsManager history, IPrecomputedFuzzyMatcher matcher) + { + var query = matcher.PrecomputeQuery(rawQuery); + var fn = BuildScoringFunction(history, matcher); + return InternalListHelpers.FilterListWithScores(apps.Cast().ToArray(), query, fn); + } + + private static RecentCommandsManager SeedUses(RecentCommandsManager history, string commandId, int uses) + { + for (var i = 0; i < uses; i++) + { + history = history.WithHistoryItem(commandId); + } + + return history; + } + + // Every app matches "x" only at the Fuzzy tier. + private static CatalogItem[] BuildFuzzyOnlyCatalogForX() => + [ + new CatalogItem("Galaxy Store", "Shop for apps", "app.galaxy"), + new CatalogItem("Nexus Mods", "Manage game mods", "app.nexus"), + new CatalogItem("Toolbox Companion", "Developer tools", "app.toolbox"), + new CatalogItem("Max Cleaner", "Free up disk space", "app.max"), + new CatalogItem("Voxel Editor", "Edit voxel art", "app.voxel"), + ]; + + // Apps spanning multiple tiers for "c": some titles start with it, some have a non-leading word + // that does, and some only contain it mid-word. + private static CatalogItem[] BuildMixedCatalogForC() => + [ + new CatalogItem("Calculator", "Perform calculations", "app.calc"), + new CatalogItem("Calendar", "View your schedule", "app.cal"), + new CatalogItem("Visual Studio Code", "Code editor", "app.vscode"), + new CatalogItem("Windows Camera", "Take photos", "app.camera"), + new CatalogItem("Microsoft Edge", "Browse the web", "app.edge"), + new CatalogItem("Office Hub", "Productivity apps", "app.office"), + new CatalogItem("Discord", "Chat with friends", "app.discord"), + ]; + + /// + /// The mechanism, locked as a test: when a 1-char query only matches mid-word, every result is + /// Fuzzy, so seeding frecency on any one of them floats it straight to rank 1. + /// + [TestMethod] + public void ShortQuery_FrecencyFloatsWeakFuzzyMatchToTop() + { + var matcher = CreateMatcher(); + var apps = BuildFuzzyOnlyCatalogForX(); + + // With no history, some app is at rank 1 purely on lexical quality. + var noHistory = Score(apps, "x", new RecentCommandsManager(), matcher); + Assert.IsTrue(noHistory.Length > 0, "The fuzzy-only catalog must still match 'x'."); + foreach (var s in noHistory) + { + Assert.AreEqual( + RankTier.Fuzzy, + MainListRanker.TierOf(s.Score), + $"Every 'x' match must be Fuzzy tier; '{s.Item.Title}' was {MainListRanker.TierOf(s.Score)}."); + } + + // Seed heavy frecency on an app that was NOT already at the top, and confirm it floats up. + var seededId = noHistory[^1].Item is CatalogItem last ? last.Id : throw new InvalidOperationException(); + var seededTitle = noHistory[^1].Item.Title; + var history = SeedUses(new RecentCommandsManager(), seededId, 40); + + var withHistory = Score(apps, "x", history, matcher); + + TestContext.WriteLine($"no-history rank1='{noHistory[0].Item.Title}', seeded '{seededTitle}' -> rank1='{withHistory[0].Item.Title}'."); + + Assert.AreEqual(RankTier.Fuzzy, MainListRanker.TierOf(withHistory[0].Score), "The floated rank-1 item is still only a Fuzzy match."); + Assert.AreEqual(seededTitle, withHistory[0].Item.Title, "Frecency should float the seeded weak match to rank 1 within the Fuzzy tier."); + } + + /// + /// A short query filters every fuzzy-only app match. + /// + [TestMethod] + public void ShortQuery_Filter_RemovesEveryFuzzyMatch() + { + var matcher = CreateMatcher(); + var apps = BuildFuzzyOnlyCatalogForX(); + + var seededId = apps[0].Id; + var history = SeedUses(new RecentCommandsManager(), seededId, 40); + + var scored = Score(apps, "x", history, matcher); + Assert.IsTrue(scored.Length > 0, "Precondition: the ungated result surfaces weak fuzzy matches."); + + var filtered = MainListPage.FilterAppsForShortQueries(scored, queryLength: 1); + + Assert.IsNotNull(filtered); + Assert.AreEqual(0, filtered!.Count, "A one-character query should filter fuzzy-only app matches."); + } + + /// + /// The filter keeps high-confidence matches and removes fuzzy matches. + /// + [TestMethod] + public void ShortQuery_Filter_KeepsHighConfidenceMatches() + { + var matcher = CreateMatcher(); + var apps = BuildMixedCatalogForC(); + + var scored = Score(apps, "c", new RecentCommandsManager(), matcher); + + var fuzzyCount = scored.Count(s => MainListRanker.TierOf(s.Score) == RankTier.Fuzzy); + var confidentCount = scored.Count(s => (int)MainListRanker.TierOf(s.Score) >= (int)RankTier.AcronymWordBoundary); + + Assert.IsTrue(fuzzyCount > 0, "Precondition: the mixed catalog produces some fuzzy-tail matches for 'c'."); + Assert.IsTrue(confidentCount > 0, "Precondition: the mixed catalog produces some confident matches for 'c'."); + + var filtered = MainListPage.FilterAppsForShortQueries(scored, queryLength: 1); + Assert.IsNotNull(filtered); + + Assert.AreEqual(confidentCount, filtered!.Count, "Only word-boundary or stronger matches should remain."); + foreach (var s in filtered) + { + Assert.IsTrue( + (int)MainListRanker.TierOf(s.Score) >= (int)RankTier.AcronymWordBoundary, + $"Filtered item '{s.Item.Title}' must be word-boundary tier or higher, was {MainListRanker.TierOf(s.Score)}."); + } + + // Filtering preserves the scored order. + for (var i = 0; i < filtered.Count; i++) + { + Assert.AreSame(scored[i].Item, filtered[i].Item, $"Filtered item at index {i} must keep its scored position."); + } + } + + /// + /// Queries longer than two characters return the original array. + /// + [TestMethod] + public void LongerQuery_Filter_ReturnsInputUnchanged() + { + var matcher = CreateMatcher(); + var apps = BuildMixedCatalogForC(); + + foreach (var raw in new[] { "cal", "calc", "code" }) + { + var scored = Score(apps, raw, new RecentCommandsManager(), matcher); + var filtered = MainListPage.FilterAppsForShortQueries(scored, raw.Length); + + Assert.AreSame(scored, filtered, $"A {raw.Length}-character query ('{raw}') should return the original array."); + } + } + + /// + /// The filter applies to query lengths one and two. + /// + [TestMethod] + public void Filter_LengthBoundary_AppliesToOneAndTwo_NotThree() + { + var matcher = CreateMatcher(); + var apps = BuildFuzzyOnlyCatalogForX(); + var scored = Score(apps, "x", new RecentCommandsManager(), matcher); + Assert.IsTrue(scored.Length > 0, "Precondition: 'x' matches fuzzily."); + + Assert.AreEqual(0, MainListPage.FilterAppsForShortQueries(scored, 1)!.Count, "Length 1 should be filtered."); + Assert.AreEqual(0, MainListPage.FilterAppsForShortQueries(scored, 2)!.Count, "Length 2 should be filtered."); + Assert.AreSame(scored, MainListPage.FilterAppsForShortQueries(scored, 3), "Length 3 should not be filtered."); + } + + /// + /// Null, empty, and default-view inputs are unchanged. + /// + [TestMethod] + public void Filter_NullEmptyAndZeroLength_AreNoOps() + { + Assert.IsNull(MainListPage.FilterAppsForShortQueries(null, 1)); + + var empty = Array.Empty>(); + Assert.AreSame(empty, MainListPage.FilterAppsForShortQueries(empty, 1)); + + var matcher = CreateMatcher(); + var scored = Score(BuildFuzzyOnlyCatalogForX(), "x", new RecentCommandsManager(), matcher); + Assert.AreSame(scored, MainListPage.FilterAppsForShortQueries(scored, 0), "A zero-length query should return the original array."); + } + + /// + /// Counts the leading entries at or above the requested tier. + /// + [TestMethod] + public void GetHighConfidenceAppsCount_CountsLeadingHighTierEntries() + { + RoScored Make(RankTier tier, int within) + => new(new CatalogItem($"{tier}", string.Empty, $"{tier}.{within}"), MainListRanker.Pack(tier, within)); + + // Sorted descending by packed score: Exact(5) > Prefix(4) > WordBoundary(3) > Fuzzy(2) > Fallback(1). + var scored = new[] + { + Make(RankTier.ExactTitle, 100), + Make(RankTier.Prefix, 50), + Make(RankTier.AcronymWordBoundary, 20), + Make(RankTier.Fuzzy, 9000), + Make(RankTier.FallbackFloor, 5), + }; + + Assert.AreEqual(5, MainListPage.GetHighConfidenceAppsCount(scored, RankTier.FallbackFloor)); + Assert.AreEqual(4, MainListPage.GetHighConfidenceAppsCount(scored, RankTier.Fuzzy)); + Assert.AreEqual(3, MainListPage.GetHighConfidenceAppsCount(scored, RankTier.AcronymWordBoundary)); + Assert.AreEqual(2, MainListPage.GetHighConfidenceAppsCount(scored, RankTier.Prefix)); + Assert.AreEqual(1, MainListPage.GetHighConfidenceAppsCount(scored, RankTier.ExactTitle)); + Assert.AreEqual(0, MainListPage.GetHighConfidenceAppsCount(scored, RankTier.AliasExact)); + } + + /// + /// Uses the query length published with the scored array. + /// + [TestMethod] + public void Filter_UsesSuppliedPublishedLength() + { + RoScored Fuzzy(int within) + => new(new CatalogItem($"fuzzy.{within}", string.Empty, $"fuzzy.{within}"), MainListRanker.Pack(RankTier.Fuzzy, within)); + + var scored = new[] { Fuzzy(30), Fuzzy(20), Fuzzy(10) }; + + var filteredByPublished = MainListPage.FilterAppsForShortQueries(scored, queryLength: 2); + Assert.IsNotNull(filteredByPublished); + Assert.AreEqual(0, filteredByPublished!.Count, "The published short length should filter fuzzy matches."); + + Assert.AreSame(scored, MainListPage.FilterAppsForShortQueries(scored, queryLength: 5), "A longer published length should return the original array."); + } + + /// + /// Telemetry counts only visible apps after filtering. + /// + [TestMethod] + public void GetVisibleAppCount_ShortQuery_CountsOnlyFilteredApps() + { + var matcher = CreateMatcher(); + var apps = BuildMixedCatalogForC(); + var scored = Score(apps, "c", new RecentCommandsManager(), matcher); + + var full = scored.Length; + var confident = scored.Count(s => (int)MainListRanker.TierOf(s.Score) >= (int)RankTier.AcronymWordBoundary); + Assert.IsTrue(confident < full, "Precondition: fuzzy matches make the filtered count smaller."); + + const int NoCap = 1000; + + Assert.AreEqual(confident, MainListPage.GetVisibleAppCount(scored, queryLength: 1, appResultLimit: NoCap)); + Assert.AreEqual(confident, MainListPage.GetVisibleAppCount(scored, queryLength: 2, appResultLimit: NoCap)); + + Assert.AreEqual(full, MainListPage.GetVisibleAppCount(scored, queryLength: 3, appResultLimit: NoCap)); + Assert.AreEqual(full, MainListPage.GetVisibleAppCount(scored, queryLength: 0, appResultLimit: NoCap), "A zero-length query should count the full set."); + } + + /// + /// The count respects the app limit and handles empty input. + /// + [TestMethod] + public void GetVisibleAppCount_RespectsCap_AndEmptyInput() + { + var matcher = CreateMatcher(); + + var fuzzyOnly = Score(BuildFuzzyOnlyCatalogForX(), "x", new RecentCommandsManager(), matcher); + Assert.IsTrue(fuzzyOnly.Length > 0, "Precondition: 'x' matches fuzzily."); + Assert.AreEqual(0, MainListPage.GetVisibleAppCount(fuzzyOnly, queryLength: 1, appResultLimit: 1000)); + + var mixed = Score(BuildMixedCatalogForC(), "c", new RecentCommandsManager(), matcher); + Assert.IsTrue(mixed.Length > 2, "Precondition: the mixed catalog has more than two matches so the cap bites."); + Assert.AreEqual(2, MainListPage.GetVisibleAppCount(mixed, queryLength: 3, appResultLimit: 2), "The app result limit should cap the count."); + + Assert.AreEqual(0, MainListPage.GetVisibleAppCount(null, 1, 1000)); + Assert.AreEqual(0, MainListPage.GetVisibleAppCount(Array.Empty>(), 1, 1000)); + } +} diff --git a/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/FastFirstPaintTests.cs b/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/FastFirstPaintTests.cs new file mode 100644 index 0000000000..4be0ea0a8c --- /dev/null +++ b/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/FastFirstPaintTests.cs @@ -0,0 +1,215 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using Microsoft.CmdPal.Common.Helpers; +using Microsoft.CmdPal.Common.Text; +using Microsoft.CmdPal.UI.ViewModels.Commands; +using Microsoft.CmdPal.UI.ViewModels.MainPage; +using Microsoft.CommandPalette.Extensions; +using Microsoft.CommandPalette.Extensions.Toolkit; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Windows.Foundation; + +namespace Microsoft.CmdPal.UI.ViewModels.UnitTests; + +/// +/// Guardrail tests for the "fast first paint" behavior of the main/root page. +/// +/// The typing path renders deterministic in-proc results (top-level commands + installed +/// apps) immediately and folds in slow, out-of-proc fallback contributions asynchronously +/// as they resolve. These tests pin the two invariants that make that safe: +/// 1. Deterministic results are produced without any fallback contribution present, so +/// first paint never waits on a slow source. +/// 2. Late-arriving fallbacks are merged with a fresh score from the same ranker, always +/// at the FallbackFloor tier, so they can never leapfrog deterministic matches, and a +/// superseding query's snapshot replaces any stale one. +/// +[TestClass] +public sealed partial class FastFirstPaintTests +{ + private static readonly Separator _resultsSeparator = new("Results"); + private static readonly Separator _fallbacksSeparator = new("Fallbacks"); + + // A list item whose Title can be mutated to simulate an extension resolving a dynamic + // fallback title asynchronously, after first paint. + private sealed partial class MutableListItem : IListItem + { + public string Title { get; set; } = string.Empty; + + public string Subtitle { get; set; } = string.Empty; + + public ICommand Command => new NoOpCommand(); + + public IDetails? Details => null; + + public IIconInfo? Icon => null; + + public string Section => string.Empty; + + public ITag[] Tags => []; + + public string TextToSuggest => string.Empty; + + public IContextItem[] MoreCommands => []; + +#pragma warning disable CS0067 // The event is never used + public event TypedEventHandler? PropChanged; +#pragma warning restore CS0067 // The event is never used + + public override string ToString() => Title; + } + + // A stand-in ranker that mirrors the real contract for fallbacks: an item with no + // (resolved) title scores 0 and is dropped; otherwise it lands at the FallbackFloor tier + // with a stronger within-floor score when its current title overlaps the query text. + private static ScoringFunction FloorScorerFor(string queryText) + { + return (in FuzzyQuery _, IListItem item) => + { + var title = item.Title; + if (string.IsNullOrWhiteSpace(title)) + { + return 0; + } + + var within = title.Contains(queryText, StringComparison.OrdinalIgnoreCase) ? 100 : 1; + return MainListRanker.Pack(RankTier.FallbackFloor, within); + }; + } + + private static RoScored ScoredFuzzy(string title, int within) + => new(score: MainListRanker.Pack(RankTier.Fuzzy, within), item: new MutableListItem { Title = title }); + + private static RoScored ScoredFloorMax(string title) + => new(score: MainListRanker.Pack(RankTier.FallbackFloor, MainListRanker.TierStride - 1), item: new MutableListItem { Title = title }); + + [TestMethod] + public void FirstPaint_ProducesDeterministicResults_WithNoFallbackContribution() + { + var command = ScoredFuzzy("Notepad", within: 50); + var app = ScoredFuzzy("Notepad++", within: 40); + var filtered = new List> { command }; + var apps = new List> { app }; + + // No scored fallbacks and no fallback items: this models first paint before any slow + // out-of-proc source has responded. + var result = MainListPageResultFactory.Create( + filtered, + scoredFallbackItems: null, + apps, + fallbackItems: null, + _resultsSeparator, + _fallbacksSeparator, + appResultLimit: 10); + + CollectionAssert.AreEqual( + new IListItem[] { _resultsSeparator, command.Item, app.Item }, + result, + "First paint must render the deterministic command/app results without any fallback contribution."); + } + + [TestMethod] + public void UnresolvedFallback_IsAbsentAtFirstPaint() + { + var fallback = new MutableListItem { Title = string.Empty }; + IReadOnlyList sources = new IListItem[] { fallback }; + + var scored = MainListPage.ScoreDeferredFallbacks(sources, default, FloorScorerFor("remote")); + + Assert.IsNull(scored, "A fallback whose dynamic title has not resolved yet must not appear."); + } + + [TestMethod] + public void SlowFallback_FoldsIn_WhenItsTitleResolves() + { + var fallback = new MutableListItem { Title = string.Empty }; + IReadOnlyList sources = new IListItem[] { fallback }; + var scorer = FloorScorerFor("remote"); + + // First paint: unresolved title -> not present. + Assert.IsNull(MainListPage.ScoreDeferredFallbacks(sources, default, scorer)); + + // The extension resolves the dynamic title asynchronously (off the typing path). + fallback.Title = "Remote Desktop: server01"; + + // A later refresh re-scores the same snapshot and folds the fallback in. + var after = MainListPage.ScoreDeferredFallbacks(sources, default, scorer); + + Assert.IsNotNull(after); + Assert.AreEqual(1, after!.Count); + Assert.AreSame(fallback, after[0].Item); + Assert.AreEqual(RankTier.FallbackFloor, MainListRanker.TierOf(after[0].Score)); + } + + [TestMethod] + public void ReScore_ReflectsCurrentTitle_NotAFrozenValue() + { + var fallback = new MutableListItem { Title = "no match here" }; + IReadOnlyList sources = new IListItem[] { fallback }; + var scorer = FloorScorerFor("remote"); + + var weak = MainListPage.ScoreDeferredFallbacks(sources, default, scorer); + Assert.IsNotNull(weak); + var weakScore = weak![0].Score; + + // The title resolves to a strong match. If scoring were frozen at keystroke time, the + // weak score would persist; deferred re-scoring must reflect the fresh title. + fallback.Title = "Remote host"; + var strong = MainListPage.ScoreDeferredFallbacks(sources, default, scorer); + + Assert.IsNotNull(strong); + Assert.IsTrue(strong![0].Score > weakScore, "Re-scoring must reflect the freshly resolved title, not a frozen value."); + } + + [TestMethod] + public void ReScoreUsesLatestSnapshot_StaleStrongMatchNotApplied() + { + // This exercises the render-path re-score directly: it always scores whatever snapshot is + // current, so a superseding keystroke's query wins and the prior query's strong score is + // gone. In production the field pair (_globalFallbackSources + _globalFallbackQuery) is + // written under lock (commands) in UpdateSearchTextCore and read under the same lock in + // GetItems, so the swap is atomic; this test stands in for that behavior at the helper level. + var fallback = new MutableListItem { Title = "Remote Desktop" }; + IReadOnlyList sources = new IListItem[] { fallback }; + + // Query A ("remote") is a strong match for the current title. + var a = MainListPage.ScoreDeferredFallbacks(sources, default, FloorScorerFor("remote")); + Assert.IsNotNull(a); + var strongScore = a![0].Score; + + // A newer keystroke installs query B ("zzz"), which does not overlap the title. The + // render path always scores the latest snapshot, so query A's strong score is gone. + var b = MainListPage.ScoreDeferredFallbacks(sources, default, FloorScorerFor("zzz")); + + Assert.IsNotNull(b); + Assert.IsTrue(b![0].Score < strongScore, "A superseding query must not inherit the prior query's stale score."); + } + + [TestMethod] + public void LateFallback_MergesBelowDeterministicMatches_NoLeapfrog() + { + // A minimal deterministic Fuzzy match versus a fallback with the maximum possible + // within-floor score. The tier ladder must still keep the fallback below the command. + var command = ScoredFuzzy("Notepad", within: 1); + var fallback = ScoredFloorMax("Search the web for notepad"); + + var filtered = new List> { command }; + var scoredFallbacks = new List> { fallback }; + + var result = MainListPageResultFactory.Create( + filtered, + scoredFallbacks, + filteredApps: null, + fallbackItems: null, + _resultsSeparator, + _fallbacksSeparator, + appResultLimit: 10); + + Assert.AreEqual(_resultsSeparator, result[0]); + Assert.AreSame(command.Item, result[1], "Deterministic matches must sort above floor-tier fallbacks."); + Assert.AreSame(fallback.Item, result[2], "A late floor-tier fallback merges after deterministic results, never leapfrogging them."); + } +} diff --git a/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/MainListRankerTests.cs b/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/MainListRankerTests.cs new file mode 100644 index 0000000000..91db2372c7 --- /dev/null +++ b/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/MainListRankerTests.cs @@ -0,0 +1,141 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using Microsoft.CmdPal.UI.ViewModels; +using Microsoft.CmdPal.UI.ViewModels.MainPage; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.CmdPal.UI.ViewModels.UnitTests; + +/// +/// Focused, per-tier unit tests for the primitives - tier +/// classification, the tier/within-tier packing invariants, and the within-tier score inputs. +/// These cover the pieces the end-to-end cannot easily drive +/// through app/command mocks (alias-exact and fallback-floor classification, and the packing +/// guarantee that a higher tier always outranks a lower one regardless of within-tier score). +/// +[TestClass] +public class MainListRankerTests +{ + [DataTestMethod] + [DataRow("gh", "GitHub", true, true, false, false, RankTier.AliasExact, DisplayName = "Alias-exact is the strongest, most explicit signal and beats even a fallback flag")] + [DataRow("anything", "Some Fallback", true, false, false, true, RankTier.FallbackFloor, DisplayName = "A fallback that is not alias-exact lands on the floor regardless of a lexical match")] + [DataRow("calculator", "Calculator", false, false, false, true, RankTier.ExactTitle, DisplayName = "Exact title match")] + [DataRow("cal", "Calculator", false, false, false, true, RankTier.Prefix, DisplayName = "Title prefix match")] + [DataRow("code", "Visual Studio Code", false, false, false, true, RankTier.AcronymWordBoundary, DisplayName = "Word-boundary match")] + [DataRow("vs", "Visual Studio Code", false, false, false, true, RankTier.AcronymWordBoundary, DisplayName = "Acronym match")] + [DataRow("cmd", "Command Prompt", false, false, false, true, RankTier.Fuzzy, DisplayName = "A lexical match that is not exact/prefix/word-boundary/acronym is fuzzy")] + [DataRow("zzz", "Command Prompt", false, false, false, false, RankTier.None, DisplayName = "Nothing matched")] + [DataRow("zz", "Some Command", false, false, true, false, RankTier.Fuzzy, DisplayName = "An alias-substring match keeps the item at the fuzzy floor even with no lexical match")] + public void ClassifyTier_ClassifiesEachSignalIntoItsTier( + string query, + string title, + bool isFallback, + bool isAliasExact, + bool isAliasSubstringMatch, + bool matchedLexically, + RankTier expected) + { + Assert.AreEqual( + expected, + MainListRanker.ClassifyTier(query, title, isFallback, isAliasExact, isAliasSubstringMatch, matchedLexically)); + } + + [TestMethod] + public void Pack_HigherTierAlwaysOutranksLowerTier() + { + // The core invariant: a higher tier with the WORST possible within-tier score still + // outranks a lower tier with the BEST possible within-tier score. This is what makes + // "an exact match always beats a fuzzy one" true no matter how much frecency piles up. + RankTier[] ascending = + { + RankTier.FallbackFloor, + RankTier.Fuzzy, + RankTier.AcronymWordBoundary, + RankTier.Prefix, + RankTier.ExactTitle, + RankTier.AliasExact, + }; + + for (var i = 0; i < ascending.Length - 1; i++) + { + var lower = MainListRanker.Pack(ascending[i], MainListRanker.TierStride - 1); + var higher = MainListRanker.Pack(ascending[i + 1], 0); + Assert.IsTrue( + higher > lower, + $"{ascending[i + 1]} (min within-tier) must outrank {ascending[i]} (max within-tier)"); + } + } + + [TestMethod] + public void Pack_NoneIsZeroAndFiltered() + { + Assert.AreEqual(0, MainListRanker.Pack(RankTier.None, 999_999)); + } + + [TestMethod] + public void Pack_WithinTierScoreIsClampedToItsBand() + { + // An absurd within-tier score must never spill into the next tier's band. + var packed = MainListRanker.Pack(RankTier.Fuzzy, double.MaxValue); + Assert.AreEqual(RankTier.Fuzzy, MainListRanker.TierOf(packed)); + + var nextTierFloor = MainListRanker.Pack(RankTier.AcronymWordBoundary, 0); + Assert.IsTrue(packed < nextTierFloor, "A clamped within-tier score must stay below the next tier"); + } + + [TestMethod] + public void Pack_WithinTierScoreOrdersItemsInTheSameTier() + { + var low = MainListRanker.Pack(RankTier.Prefix, 10); + var high = MainListRanker.Pack(RankTier.Prefix, 20); + Assert.IsTrue(high > low, "Within the same tier, a higher within-tier score sorts higher"); + Assert.AreEqual(MainListRanker.TierOf(low), MainListRanker.TierOf(high), "Both remain in the same tier"); + } + + [TestMethod] + public void TierOf_RoundTripsEveryTier() + { + foreach (RankTier tier in Enum.GetValues(typeof(RankTier))) + { + if (tier == RankTier.None) + { + continue; + } + + var packed = MainListRanker.Pack(tier, 42); + Assert.AreEqual(tier, MainListRanker.TierOf(packed), $"Packing then unpacking {tier} should round-trip"); + } + } + + [TestMethod] + public void WithinTierScore_LexicalQualityLeads() + { + // More lexical quality raises the within-tier score, all else equal. + var lowLexical = MainListRanker.WithinTierScore(lexicalQuality: 5, frecencyWeight: 0, aliasSubstringBonus: 0, providerBonus: 0); + var highLexical = MainListRanker.WithinTierScore(lexicalQuality: 6, frecencyWeight: 0, aliasSubstringBonus: 0, providerBonus: 0); + Assert.IsTrue(highLexical > lowLexical, "Higher lexical quality should raise the within-tier score"); + } + + [TestMethod] + public void WithinTierScore_FrecencyBreaksTies() + { + var noFrecency = MainListRanker.WithinTierScore(lexicalQuality: 5, frecencyWeight: 0, aliasSubstringBonus: 0, providerBonus: 0); + var withFrecency = MainListRanker.WithinTierScore(lexicalQuality: 5, frecencyWeight: 3, aliasSubstringBonus: 0, providerBonus: 0); + Assert.IsTrue(withFrecency > noFrecency, "Frecency should raise the within-tier score for otherwise-equal items"); + } + + [TestMethod] + public void ProviderBonus_LowerIsBelowNormalIsBelowHigher() + { + Assert.IsTrue( + MainListRanker.ProviderBonus(ProviderSearchWeight.Lower) < MainListRanker.ProviderBonus(ProviderSearchWeight.Normal), + "Lower should subtract relative to Normal"); + Assert.IsTrue( + MainListRanker.ProviderBonus(ProviderSearchWeight.Normal) < MainListRanker.ProviderBonus(ProviderSearchWeight.Higher), + "Higher should add relative to Normal"); + Assert.AreEqual(0.0, MainListRanker.ProviderBonus(ProviderSearchWeight.Normal), "Normal is the neutral default"); + } +} diff --git a/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/ProviderWeightingTests.cs b/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/ProviderWeightingTests.cs new file mode 100644 index 0000000000..bf75ddb7b4 --- /dev/null +++ b/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/ProviderWeightingTests.cs @@ -0,0 +1,214 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Text.Json; +using Microsoft.CmdPal.Common.Text; +using Microsoft.CmdPal.Ext.UnitTestBase; +using Microsoft.CmdPal.UI.ViewModels.MainPage; +using Microsoft.CommandPalette.Extensions; +using Microsoft.CommandPalette.Extensions.Toolkit; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Windows.Foundation; +using WyHash; + +namespace Microsoft.CmdPal.UI.ViewModels.UnitTests; + +[TestClass] +public partial class ProviderWeightingTests : CommandPaletteUnitTestBase +{ + private static IPrecomputedFuzzyMatcher CreateMatcher() + => new PrecomputedFuzzyMatcher(new PrecomputedFuzzyMatcherOptions()); + + private static RecentCommandsManager EmptyHistory() => new(); + + // Maps each item to the weight configured for its provider id. + private static Func LookupBy(IReadOnlyDictionary byProvider) + => item => + { + var providerId = item is WeightItemMock mock ? mock.ProviderId ?? string.Empty : string.Empty; + return byProvider.TryGetValue(providerId, out var weight) ? weight : ProviderSearchWeight.Normal; + }; + + [TestMethod] + public void ProviderBonus_MapsSignedMagnitude() + { + Assert.AreEqual(-MainListRanker.ProviderWeightBonus, MainListRanker.ProviderBonus(ProviderSearchWeight.Lower)); + Assert.AreEqual(0.0, MainListRanker.ProviderBonus(ProviderSearchWeight.Normal)); + Assert.AreEqual(MainListRanker.ProviderWeightBonus, MainListRanker.ProviderBonus(ProviderSearchWeight.Higher)); + } + + [TestMethod] + public void ProviderWeight_ReordersWithinTier() + { + // Two items that land in the SAME tier (exact-title) with identical lexical quality + // and no history. The only differentiator is the per-provider weight. + var fuzzyMatcher = CreateMatcher(); + var q = fuzzyMatcher.PrecomputeQuery("git"); + + var itemA = new WeightItemMock("git", ProviderId: "providerA"); + var itemB = new WeightItemMock("git", ProviderId: "providerB"); + + // Baseline: both Normal -> equal scores. + var neutral = LookupBy(new Dictionary()); + var baseA = MainListPage.ScoreTopLevelItem(q, itemA, EmptyHistory(), fuzzyMatcher, neutral); + var baseB = MainListPage.ScoreTopLevelItem(q, itemB, EmptyHistory(), fuzzyMatcher, neutral); + Assert.AreEqual(baseA, baseB, "With both providers Normal, tied items should score equally"); + + // A Higher, B Lower -> A must now sort above B. + var lookup = LookupBy(new Dictionary + { + ["providerA"] = ProviderSearchWeight.Higher, + ["providerB"] = ProviderSearchWeight.Lower, + }); + var higherA = MainListPage.ScoreTopLevelItem(q, itemA, EmptyHistory(), fuzzyMatcher, lookup); + var lowerB = MainListPage.ScoreTopLevelItem(q, itemB, EmptyHistory(), fuzzyMatcher, lookup); + + Assert.IsTrue(higherA > baseA, "Higher weight should raise the score"); + Assert.IsTrue(lowerB < baseB, "Lower weight should reduce the score"); + Assert.IsTrue(higherA > lowerB, "Higher-weighted provider should outrank the lower-weighted one within the tier"); + + // Same tier the whole time - the nudge only reordered within it. + Assert.AreEqual(MainListRanker.TierOf(baseA), MainListRanker.TierOf(higherA)); + Assert.AreEqual(MainListRanker.TierOf(baseB), MainListRanker.TierOf(lowerB)); + } + + [TestMethod] + public void ProviderWeight_NeverCrossesTierBoundary() + { + var fuzzyMatcher = CreateMatcher(); + var q = fuzzyMatcher.PrecomputeQuery("code"); + + // "code" exactly matches -> ExactTitle tier. Give it the WORST weight. + var exact = new WeightItemMock("code", ProviderId: "weak"); + + // "Visual Studio Code" only matches "code" at a word boundary -> lower tier. Give it + // the BEST weight, plus a pile of history, to try to jump the tier boundary. + var lowerTier = new WeightItemMock("Visual Studio Code", ProviderId: "strong"); + + var lookup = LookupBy(new Dictionary + { + ["weak"] = ProviderSearchWeight.Lower, + ["strong"] = ProviderSearchWeight.Higher, + }); + + var history = EmptyHistory(); + for (var i = 0; i < 50; i++) + { + history = history.WithHistoryItem(lowerTier.Id); + } + + var exactScore = MainListPage.ScoreTopLevelItem(q, exact, EmptyHistory(), fuzzyMatcher, lookup); + var lowerScore = MainListPage.ScoreTopLevelItem(q, lowerTier, history, fuzzyMatcher, lookup); + + Assert.IsTrue( + MainListRanker.TierOf(exactScore) > MainListRanker.TierOf(lowerScore), + "The exact match must live in a strictly higher tier"); + Assert.IsTrue( + exactScore > lowerScore, + "A within-tier nudge (even Higher + heavy history) must never promote an item across a tier boundary"); + } + + [TestMethod] + public void ProviderWeight_DefaultNormalMatchesNoLookup() + { + var fuzzyMatcher = CreateMatcher(); + var q = fuzzyMatcher.PrecomputeQuery("term"); + var item = new WeightItemMock("Terminal", ProviderId: "providerA"); + + // No lookup at all should behave exactly like an all-Normal lookup, which should + // behave exactly like the previous (provider-unaware) scoring. + var noLookup = MainListPage.ScoreTopLevelItem(q, item, EmptyHistory(), fuzzyMatcher); + var normalLookup = MainListPage.ScoreTopLevelItem( + q, + item, + EmptyHistory(), + fuzzyMatcher, + LookupBy(new Dictionary { ["providerA"] = ProviderSearchWeight.Normal })); + + Assert.AreEqual(noLookup, normalLookup, "Normal weight must be a no-op relative to the default path"); + } + + [TestMethod] + public void ProviderWeight_AppliesToAnyProviderItem() + { + // App items are plain IListItems (not TopLevelViewModel). This asserts the scorer + // honors the provider weight for ANY item, which is how installed apps (the "AllApps" + // provider) get nudged. + var fuzzyMatcher = CreateMatcher(); + var q = fuzzyMatcher.PrecomputeQuery("note"); + var appItem = new WeightItemMock("Notepad", ProviderId: "AllApps"); + + var normal = MainListPage.ScoreTopLevelItem(q, appItem, EmptyHistory(), fuzzyMatcher); + var higher = MainListPage.ScoreTopLevelItem( + q, + appItem, + EmptyHistory(), + fuzzyMatcher, + LookupBy(new Dictionary { ["AllApps"] = ProviderSearchWeight.Higher })); + + Assert.IsTrue(higher > normal, "An app-style item should also respond to its provider's Higher weight"); + } + + [TestMethod] + public void SearchWeight_SerializationRoundTrips() + { + var dict = ImmutableDictionary.Empty + .SetItem("p", new ProviderSettings { SearchWeight = ProviderSearchWeight.Higher }); + + var json = JsonSerializer.Serialize(dict, JsonSerializationContext.Default.ImmutableProviderSettingsDictionary); + var restored = JsonSerializer.Deserialize(json, JsonSerializationContext.Default.ImmutableProviderSettingsDictionary); + + Assert.IsNotNull(restored); + Assert.AreEqual(ProviderSearchWeight.Higher, restored!["p"].SearchWeight); + } + + [TestMethod] + public void SearchWeight_LegacyJsonDeserializesToNormal() + { + // Legacy persisted settings predate SearchWeight; the missing property must default + // to Normal rather than throwing or landing on Lower. + const string legacyJson = "{\"p\":{\"IsEnabled\":true}}"; + + var restored = JsonSerializer.Deserialize(legacyJson, JsonSerializationContext.Default.ImmutableProviderSettingsDictionary); + + Assert.IsNotNull(restored); + Assert.AreEqual(ProviderSearchWeight.Normal, restored!["p"].SearchWeight); + } + + private sealed partial record WeightItemMock( + string Title, + string? Subtitle = "", + string? GivenId = "", + string? ProviderId = "") : IListItem + { + public string Id => string.IsNullOrEmpty(GivenId) ? GenerateId() : GivenId; + + public IDetails Details => throw new NotImplementedException(); + + public string Section => throw new NotImplementedException(); + + public ITag[] Tags => throw new NotImplementedException(); + + public string TextToSuggest => throw new NotImplementedException(); + + public ICommand Command => new NoOpCommand() { Id = Id }; + + public IIconInfo Icon => throw new NotImplementedException(); + + public IContextItem[] MoreCommands => throw new NotImplementedException(); + +#pragma warning disable CS0067 + public event TypedEventHandler? PropChanged; +#pragma warning restore CS0067 + + private string GenerateId() + { + var result = WyHash64.ComputeHash64(ProviderId + Title + Subtitle, seed: 0); + return $"{ProviderId}{result}"; + } + } +} diff --git a/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/RecentCommandsTests.cs b/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/RecentCommandsTests.cs index c51d8fa12e..d9378f7d58 100644 --- a/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/RecentCommandsTests.cs +++ b/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/RecentCommandsTests.cs @@ -4,7 +4,9 @@ using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.Linq; +using System.Text.Json; using Microsoft.CmdPal.Common.Text; using Microsoft.CmdPal.Ext.UnitTestBase; using Microsoft.CmdPal.UI.ViewModels.MainPage; @@ -33,20 +35,6 @@ public partial class RecentCommandsTests : CommandPaletteUnitTestBase return history; } - private static RecentCommandsManager CreateBasicHistoryService() - { - var commonCommands = new List - { - "com.microsoft.cmdpal.shell", - "com.microsoft.cmdpal.windowwalker", - "Visual Studio 2022 Preview_6533433915015224980", - "com.microsoft.cmdpal.reload", - "com.microsoft.cmdpal.shell", - }; - - return CreateHistory(commonCommands); - } - [TestMethod] public void ValidateHistoryFunctionality() { @@ -63,21 +51,31 @@ public partial class RecentCommandsTests : CommandPaletteUnitTestBase [TestMethod] public void ValidateHistoryWeighting() { - // Setup - var history = CreateBasicHistoryService(); + // Build history with explicit, strictly-increasing timestamps so time-decay is + // deterministic. "shell" is used twice (most uses) and most recently; the others are + // each used once, progressively more recently. + var t0 = new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero); + var history = new RecentCommandsManager(); + history = history.WithHistoryItem("com.microsoft.cmdpal.shell", t0); + history = history.WithHistoryItem("com.microsoft.cmdpal.windowwalker", t0.AddDays(1)); + history = history.WithHistoryItem("Visual Studio 2022 Preview_6533433915015224980", t0.AddDays(2)); + history = history.WithHistoryItem("com.microsoft.cmdpal.reload", t0.AddDays(3)); + history = history.WithHistoryItem("com.microsoft.cmdpal.shell", t0.AddDays(4)); + + var now = t0.AddDays(4); // Act - var shellWeight = history.GetCommandHistoryWeight("com.microsoft.cmdpal.shell"); - var windowWalkerWeight = history.GetCommandHistoryWeight("com.microsoft.cmdpal.windowwalker"); - var vsWeight = history.GetCommandHistoryWeight("Visual Studio 2022 Preview_6533433915015224980"); - var reloadWeight = history.GetCommandHistoryWeight("com.microsoft.cmdpal.reload"); - var nonExistentWeight = history.GetCommandHistoryWeight("non.existent.command"); + var shellWeight = history.GetCommandHistoryWeight("com.microsoft.cmdpal.shell", now); + var windowWalkerWeight = history.GetCommandHistoryWeight("com.microsoft.cmdpal.windowwalker", now); + var vsWeight = history.GetCommandHistoryWeight("Visual Studio 2022 Preview_6533433915015224980", now); + var reloadWeight = history.GetCommandHistoryWeight("com.microsoft.cmdpal.reload", now); + var nonExistentWeight = history.GetCommandHistoryWeight("non.existent.command", now); // Assert - Assert.IsTrue(shellWeight > windowWalkerWeight, "Shell should be weighted higher than Window Walker, more uses"); - Assert.IsTrue(vsWeight > windowWalkerWeight, "Visual Studio should be weighted higher than Window Walker, because recency"); - Assert.AreEqual(reloadWeight, vsWeight, "both reload and VS were used in the last three commands, same weight"); - Assert.IsTrue(shellWeight > vsWeight, "VS and run were both used in the last 3, but shell has 2 more frequency"); + Assert.IsTrue(shellWeight > windowWalkerWeight, "Shell is both the most-used and most-recent command"); + Assert.IsTrue(vsWeight > windowWalkerWeight, "Visual Studio was used more recently than Window Walker"); + Assert.IsTrue(reloadWeight > vsWeight, "Reload was used more recently than Visual Studio"); + Assert.IsTrue(shellWeight > vsWeight, "Shell is both more recent and more frequently used than Visual Studio"); Assert.AreEqual(0, nonExistentWeight, "Nonexistent command should have zero weight"); } @@ -159,97 +157,163 @@ public partial class RecentCommandsTests : CommandPaletteUnitTestBase } [TestMethod] - public void ValidateHistoryBuckets() + public void ValidateRecencyDecay() { - // Setup - // (these will be checked in reverse order, so that A is the most recent) - var items = new List - { - new("Command A", "Subtitle A", GivenId: "idA"), // #0 -> bucket 0 - new("Command B", "Subtitle B", GivenId: "idB"), // #1 -> bucket 0 - new("Command C", "Subtitle C", GivenId: "idC"), // #2 -> bucket 0 - new("Command D", "Subtitle D", GivenId: "idD"), // #3 -> bucket 1 - new("Command E", "Subtitle E", GivenId: "idE"), // #4 -> bucket 1 - new("Command F", "Subtitle F", GivenId: "idF"), // #5 -> bucket 1 - new("Command G", "Subtitle G", GivenId: "idG"), // #6 -> bucket 1 - new("Command H", "Subtitle H", GivenId: "idH"), // #7 -> bucket 1 - new("Command I", "Subtitle I", GivenId: "idI"), // #8 -> bucket 1 - new("Command J", "Subtitle J", GivenId: "idJ"), // #9 -> bucket 1 - new("Command K", "Subtitle K", GivenId: "idK"), // #10 -> bucket 1 - new("Command L", "Subtitle L", GivenId: "idL"), // #11 -> bucket 2 - new("Command M", "Subtitle M", GivenId: "idM"), // #12 -> bucket 2 - new("Command N", "Subtitle N", GivenId: "idN"), // #13 -> bucket 2 - new("Command O", "Subtitle O", GivenId: "idO"), // #14 -> bucket 2 - }; + // Each command is used exactly once, at progressively older times. Weight must decay + // monotonically with age, and a 3-day-old use (one half-life) should weigh about half + // of a use that just happened. + var now = new DateTimeOffset(2025, 2, 1, 0, 0, 0, TimeSpan.Zero); + var history = new RecentCommandsManager() + .WithHistoryItem("today", now) + .WithHistoryItem("three-days", now.AddDays(-3)) + .WithHistoryItem("ten-days", now.AddDays(-10)) + .WithHistoryItem("thirty-days", now.AddDays(-30)); - for (var i = items.Count; i <= 50; i++) + var today = history.GetCommandHistoryWeight("today", now); + var threeDays = history.GetCommandHistoryWeight("three-days", now); + var tenDays = history.GetCommandHistoryWeight("ten-days", now); + var thirtyDays = history.GetCommandHistoryWeight("thirty-days", now); + + Assert.IsTrue(today > threeDays, "A more recent use must weigh more than an older one"); + Assert.IsTrue(threeDays > tenDays, "Decay must be monotonic with age"); + Assert.IsTrue(tenDays > thirtyDays, "Older uses keep decaying toward zero"); + Assert.IsTrue(thirtyDays >= 0, "Weight must never go negative"); + + // The half-life is 3 days, so a 3-day-old single use is about half of a fresh one. + Assert.AreEqual(today / 2.0, threeDays, 1.0, "Three days should be a single half-life"); + } + + [TestMethod] + public void ValidateFrequencyWeighting() + { + // Hold recency constant (same timestamp) and vary only the use count. More uses must + // weigh more, but the log-scaled frequency term stays within the documented cap. + var now = new DateTimeOffset(2025, 2, 1, 0, 0, 0, TimeSpan.Zero); + var history = new RecentCommandsManager().WithHistoryItem("once", now); + for (var i = 0; i < 7; i++) { - items.Add(new ListItemMock($"Command #{i}", GivenId: $"id{i}")); + history = history.WithHistoryItem("many", now); } - // Act - var history = CreateHistory(items.Reverse().ToList()); + var once = history.GetCommandHistoryWeight("once", now); + var many = history.GetCommandHistoryWeight("many", now); - // Assert - // First three items should be in the top bucket - var weightA = history.GetCommandHistoryWeight("idA"); - var weightB = history.GetCommandHistoryWeight("idB"); - var weightC = history.GetCommandHistoryWeight("idC"); + Assert.IsTrue(many > once, "At equal recency, a more frequently used command weighs more"); + Assert.IsTrue(many <= RecentCommandsManager.MaxWeight, "Weight stays within the documented cap"); + } - Assert.AreEqual(weightA, weightB, "Items A and B were used in the last 3 commands"); - Assert.AreEqual(weightB, weightC, "Items B and C were used in the last 3 commands"); + [TestMethod] + public void ValidateLookupIndexStaysConsistentAcrossChanges() + { + // The commandId lookup is backed by a cached index that must be invalidated whenever + // history changes (WithHistoryItem returns a new record, and a 'with' copy carries the + // old cached field over). Force the index to build on one instance, then mutate, and + // confirm each instance reports exactly its own history. + var now = new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero); + var first = new RecentCommandsManager().WithHistoryItem("alpha", now); - // Next eight items (3-10 inclusive) should be in the second bucket - var weightD = history.GetCommandHistoryWeight("idD"); - var weightE = history.GetCommandHistoryWeight("idE"); - var weightF = history.GetCommandHistoryWeight("idF"); - var weightG = history.GetCommandHistoryWeight("idG"); - var weightH = history.GetCommandHistoryWeight("idH"); - var weightI = history.GetCommandHistoryWeight("idI"); - var weightJ = history.GetCommandHistoryWeight("idJ"); - var weightK = history.GetCommandHistoryWeight("idK"); + // Read once to build the cached index on 'first'. + var alphaFirst = first.GetCommandHistoryWeight("alpha", now); + Assert.IsTrue(alphaFirst > 0, "alpha should have weight on the first instance"); + Assert.AreEqual(0, first.GetCommandHistoryWeight("beta", now), "beta is not in the first instance"); - Assert.AreEqual(weightD, weightE, "Items D and E were used in the last 10 commands"); - Assert.AreEqual(weightE, weightF, "Items E and F were used in the last 10 commands"); - Assert.AreEqual(weightF, weightG, "Items F and G were used in the last 10 commands"); - Assert.AreEqual(weightG, weightH, "Items G and H were used in the last 10 commands"); - Assert.AreEqual(weightH, weightI, "Items H and I were used in the last 10 commands"); - Assert.AreEqual(weightI, weightJ, "Items I and J were used in the last 10 commands"); - Assert.AreEqual(weightJ, weightK, "Items J and K were used in the last 10 commands"); + // Add a use of alpha and a brand-new beta, producing a new instance. + var second = first + .WithHistoryItem("alpha", now.AddDays(1)) + .WithHistoryItem("beta", now.AddDays(1)); - // Items up to the 15th should be in the third bucket - var weightL = history.GetCommandHistoryWeight("idL"); - var weightM = history.GetCommandHistoryWeight("idM"); - var weightN = history.GetCommandHistoryWeight("idN"); - var weightO = history.GetCommandHistoryWeight("idO"); - var weight15 = history.GetCommandHistoryWeight("id15"); - Assert.AreEqual(weightL, weightM, "Items L and M were used in the last 15 commands"); - Assert.AreEqual(weightM, weightN, "Items M and N were used in the last 15 commands"); - Assert.AreEqual(weightN, weightO, "Items N and O were used in the last 15 commands"); - Assert.AreEqual(weightO, weight15, "Items O and 15 were used in the last 15 commands"); + // The new instance sees the newer alpha (more recent + higher use) and the new beta. + Assert.IsTrue( + second.GetCommandHistoryWeight("alpha", now.AddDays(1)) > 0, + "alpha should still resolve on the updated instance"); + Assert.IsTrue( + second.GetCommandHistoryWeight("beta", now.AddDays(1)) > 0, + "beta should resolve on the updated instance after its index rebuilds"); - // Items after that should be in the lowest buckets - var weight0 = history.GetCommandHistoryWeight(items[0].Id); - var weight3 = history.GetCommandHistoryWeight(items[3].Id); - var weight11 = history.GetCommandHistoryWeight(items[11].Id); - var weight16 = history.GetCommandHistoryWeight("id16"); - var weight20 = history.GetCommandHistoryWeight("id20"); - var weight30 = history.GetCommandHistoryWeight("id30"); - var weight40 = history.GetCommandHistoryWeight("id40"); - var weight49 = history.GetCommandHistoryWeight("id49"); + // The original instance is unchanged - its index never gained beta. + Assert.AreEqual(0, first.GetCommandHistoryWeight("beta", now), "the original instance must not see beta"); + } - Assert.IsTrue(weight0 > weight3); - Assert.IsTrue(weight3 > weight11); - Assert.IsTrue(weight11 > weight16); + [TestMethod] + public void ValidateHistoryCap() + { + // Insert well beyond the cap, each newer than the last. The store keeps the most-recent + // entries and evicts the oldest, replacing the previous 50-entry limit. + var now = new DateTimeOffset(2025, 2, 1, 0, 0, 0, TimeSpan.Zero); + var history = new RecentCommandsManager(); - Assert.AreEqual(weight16, weight20); - Assert.AreEqual(weight20, weight30); - Assert.IsTrue(weight30 > weight40); - Assert.AreEqual(weight40, weight49); + var total = RecentCommandsManager.MaxHistoryEntries + 50; + for (var i = 0; i < total; i++) + { + history = history.WithHistoryItem($"cmd-{i}", now.AddMinutes(i)); + } - // The 50th item has fallen out of the list now - var weight50 = history.GetCommandHistoryWeight("id50"); - Assert.AreEqual(0, weight50, "Item 50 should have fallen out of the history list"); + var evaluatedAt = now.AddMinutes(total); + + Assert.IsTrue( + history.History.Count <= RecentCommandsManager.MaxHistoryEntries, + "History should be capped at MaxHistoryEntries"); + + // The earliest (now-evicted) entries have fallen out of the store. + Assert.AreEqual(0, history.GetCommandHistoryWeight("cmd-0", evaluatedAt), "The oldest entry should have been evicted"); + + // The most-recent entries survive and still carry weight. + Assert.IsTrue( + history.GetCommandHistoryWeight($"cmd-{total - 1}", evaluatedAt) > 0, + "The most recent entry should be retained"); + } + + [TestMethod] + public void ValidateLegacyHistoryMigration() + { + // Legacy items persisted before LastUsed existed deserialize with a default timestamp. + // They should be mildly backdated so ordering falls back to Uses (frequency) rather + // than collapsing to all-equal or zero. + var now = new DateTimeOffset(2025, 5, 1, 0, 0, 0, TimeSpan.Zero); + var legacy = new RecentCommandsManager + { + History = ImmutableList.Create( + new HistoryItem { CommandId = "rare", Uses = 1 }, + new HistoryItem { CommandId = "common", Uses = 8 }), + }; + + var rare = legacy.GetCommandHistoryWeight("rare", now); + var common = legacy.GetCommandHistoryWeight("common", now); + + Assert.IsTrue(rare > 0, "Legacy items should be mildly backdated, not zeroed out"); + Assert.IsTrue(common > rare, "Legacy ordering should fall back to Uses (frequency)"); + + // A brand-new, single real use should outrank a backdated single-use legacy item. + var withFresh = legacy.WithHistoryItem("brand-new", now); + var fresh = withFresh.GetCommandHistoryWeight("brand-new", now); + Assert.IsTrue(fresh > rare, "A just-used command should outrank a backdated single-use legacy item"); + } + + [TestMethod] + public void ValidateHistorySerializationRoundTrips() + { + // The persisted history (see SettingsModel's JsonSerializable context) must round-trip, + // including the new LastUsed timestamp, so decay survives a save/load cycle. + var now = new DateTimeOffset(2025, 3, 15, 12, 30, 0, TimeSpan.Zero); + var original = new RecentCommandsManager() + .WithHistoryItem("alpha", now) + .WithHistoryItem("beta", now.AddMinutes(5)); + + var json = JsonSerializer.Serialize(original, JsonSerializationContext.Default.RecentCommandsManager); + var restored = JsonSerializer.Deserialize(json, JsonSerializationContext.Default.RecentCommandsManager); + + Assert.IsNotNull(restored, "Round-tripped history should not be null"); + Assert.AreEqual(2, restored!.History.Count, "All history entries should round-trip"); + + var beta = restored.History.First(h => h.CommandId == "beta"); + Assert.AreEqual(now.AddMinutes(5), beta.LastUsed, "The LastUsed timestamp should round-trip"); + Assert.AreEqual(1, beta.Uses, "The use count should round-trip"); + + // Weights computed from the restored state should match the original. + Assert.AreEqual( + original.GetCommandHistoryWeight("beta", now.AddMinutes(5)), + restored.GetCommandHistoryWeight("beta", now.AddMinutes(5)), + "Restored history should produce the same weight as the original"); } [TestMethod] diff --git a/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/RelevanceHarnessTests.cs b/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/RelevanceHarnessTests.cs new file mode 100644 index 0000000000..a5c21d2ac7 --- /dev/null +++ b/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/RelevanceHarnessTests.cs @@ -0,0 +1,284 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.CmdPal.Common.Text; +using Microsoft.CmdPal.Ext.UnitTestBase; +using Microsoft.CmdPal.UI.ViewModels; +using Microsoft.CmdPal.UI.ViewModels.MainPage; +using Microsoft.CommandPalette.Extensions; +using Microsoft.CommandPalette.Extensions.Toolkit; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using WyHash; + +namespace Microsoft.CmdPal.UI.ViewModels.UnitTests; + +/// +/// End-to-end relevance harness for the main/root page ranker. Every case is expressed as a +/// realistic query paired with an ordering constraint (rank-1, or "X must rank above Y") and +/// asserted against the REAL scoring path, sorted +/// exactly the way the product sorts (positive scores, descending). The intent is to lock in +/// "results seem logical and relevant" as an objective, extendable yardstick: to add a new +/// scenario, drop another entry in the fixture and another constraint in a test. +/// +/// Focused, per-tier unit tests for the primitives live alongside +/// this harness in . +/// +[TestClass] +public partial class RelevanceHarnessTests : CommandPaletteUnitTestBase +{ + // A lightweight top-level-command / installed-app stand-in built on the real ListItem + // toolkit type, so the harness drives the same scoring path as the product (as the sibling + // RecentCommandsTests does) rather than a reimplementation. ProviderId lets a test key a + // per-provider weight lookup; Id is derived deterministically so frecency history can target it. + private sealed partial class ListItemMock : ListItem + { + public ListItemMock(string title, string? subtitle = "", string? givenId = "", string? providerId = "") + { + Title = title; + Subtitle = subtitle ?? string.Empty; + ProviderId = providerId ?? string.Empty; + Id = string.IsNullOrEmpty(givenId) ? GenerateId() : givenId; + Command = new NoOpCommand() { Id = Id }; + } + + public string Id { get; } + + public string ProviderId { get; } + + private string GenerateId() + { + var result = WyHash64.ComputeHash64(ProviderId + Title + Subtitle, seed: 0); + return $"{ProviderId}{result}"; + } + } + + private static IPrecomputedFuzzyMatcher CreateMatcher() => + new PrecomputedFuzzyMatcher(new PrecomputedFuzzyMatcherOptions()); + + private static RecentCommandsManager EmptyHistory() => new(); + + // A representative slice of the main page: installed apps + top-level commands with + // realistic titles, subtitles and shared prefixes/acronyms. Deliberately includes the + // "confusable" clusters users complain about (Calc*, Visual Studio *, Command Prompt vs + // Control Panel) so the ordering constraints below have real competition to beat. + private static List Fixture() => new() + { + new("Command Prompt", "Run the classic command interpreter"), + new("Control Panel", "Adjust your computer's settings"), + new("Calculator", "Perform calculations"), + new("Calendar", "View your schedule"), + new("Visual Studio Code", "Code editing. Redefined."), + new("Visual Studio 2022", "Full-featured IDE"), + new("Windows Settings", "Change PC settings"), + new("Windows Terminal", "Modern terminal for command-line tools"), + new("Task Manager", "Monitor apps and processes"), + new("Notepad", "A simple text editor"), + new("Microsoft Edge", "Browse the web"), + new("Paint", "Draw and edit images"), + new("Paint 3D", "Create in three dimensions"), + }; + + // Scores every fixture item for a query through the real product scorer and returns the + // matched titles in the exact order the product would render them: positive scores only, + // sorted descending. Mirrors InternalListHelpers.FilterListWithScores and the existing + // RecentCommandsTests.GetMatches helper. + private static List Rank( + string query, + IEnumerable items, + IRecentCommandsManager? history = null, + Func? providerWeightLookup = null) + { + var matcher = CreateMatcher(); + var q = matcher.PrecomputeQuery(query); + history ??= EmptyHistory(); + + return items + .Select(item => (item.Title, Score: MainListPage.ScoreTopLevelItem(q, item, history, matcher, providerWeightLookup))) + .Where(x => x.Score > 0) + .OrderByDescending(x => x.Score) + .Select(x => x.Title) + .ToList(); + } + + private static void AssertRank1( + string query, + string expectedTitle, + IRecentCommandsManager? history = null, + Func? providerWeightLookup = null) + { + var ranked = Rank(query, Fixture(), history, providerWeightLookup); + Assert.IsTrue(ranked.Count > 0, $"Query '{query}' should return at least one match"); + Assert.AreEqual( + expectedTitle, + ranked[0], + $"Query '{query}' should surface '{expectedTitle}' at rank 1. Actual order: [{string.Join(", ", ranked)}]"); + } + + private static void AssertRanksAbove( + string query, + string higher, + string lower, + IRecentCommandsManager? history = null, + Func? providerWeightLookup = null) + { + var ranked = Rank(query, Fixture(), history, providerWeightLookup); + var higherIndex = ranked.IndexOf(higher); + var lowerIndex = ranked.IndexOf(lower); + + Assert.IsTrue(higherIndex >= 0, $"Query '{query}' should match '{higher}'. Actual order: [{string.Join(", ", ranked)}]"); + Assert.IsTrue(lowerIndex >= 0, $"Query '{query}' should match '{lower}'. Actual order: [{string.Join(", ", ranked)}]"); + Assert.IsTrue( + higherIndex < lowerIndex, + $"Query '{query}' should rank '{higher}' above '{lower}'. Actual order: [{string.Join(", ", ranked)}]"); + } + + // End-to-end cases: the tier ladder, exercised through the real scorer. + [TestMethod] + public void EndToEnd_ExactTitleBeatsPrefix() + { + // "Paint" is an exact title; "Paint 3D" only has it as a prefix. Exact must win. + AssertRank1("paint", "Paint"); + AssertRanksAbove("paint", "Paint", "Paint 3D"); + } + + [TestMethod] + public void EndToEnd_PrefixBeatsWordBoundary() + { + // "co" is a title prefix of Command Prompt and Control Panel, but only a word-boundary + // match for "Code" inside Visual Studio Code. Prefix outranks word-boundary. + AssertRanksAbove("co", "Command Prompt", "Visual Studio Code"); + AssertRanksAbove("co", "Control Panel", "Visual Studio Code"); + } + + [TestMethod] + public void EndToEnd_WordBoundaryBeatsFuzzy() + { + // "man" starts the word "Manager" in Task Manager (word-boundary), but is only a loose + // subsequence (m..a..n) of "Command Prompt" (fuzzy). Word-boundary must win. + AssertRank1("man", "Task Manager"); + AssertRanksAbove("man", "Task Manager", "Command Prompt"); + } + + [TestMethod] + public void EndToEnd_AcronymSurfacesTheRightApp() + { + // "vsc" is the acronym of Visual Studio Code (V-S-C); Visual Studio 2022 (V-S-2) is not + // a match. The acronym should surface the obviously-right app at rank 1. + AssertRank1("vsc", "Visual Studio Code"); + } + + [TestMethod] + public void EndToEnd_ComplaintCase_SingleLetterSurfacesFrecentApp() + { + // "c" prefixes several apps (Calculator, Calendar, Command Prompt, Control Panel). With + // no signal they tie; a user who keeps opening Calculator should see it at rank 1. This + // is the canonical "the thing I want is buried" complaint, fixed by within-tier frecency. + var history = EmptyHistory(); + var calculatorId = Fixture().First(i => i.Title == "Calculator").Id; + for (var i = 0; i < 5; i++) + { + history = history.WithHistoryItem(calculatorId); + } + + AssertRank1("c", "Calculator", history); + } + + [TestMethod] + public void EndToEnd_ComplaintCase_CodeSurfacesVsCode() + { + // Typing "code" should put Visual Studio Code first (word-boundary on "Code"). + AssertRank1("code", "Visual Studio Code"); + } + + [TestMethod] + public void EndToEnd_ComplaintCase_SetSurfacesSettings() + { + // Typing "set" should put Windows Settings first (word-boundary on "Settings"). + AssertRank1("set", "Windows Settings"); + } + + [TestMethod] + public void EndToEnd_FrecencyReordersWithinTierOnly() + { + // Heavy use of Visual Studio Code (a word-boundary match for "co") must NOT lift it over + // Command Prompt / Control Panel, which are prefix matches a whole tier above it. + // Frecency reorders within a tier; it can never cross a tier boundary. + var history = EmptyHistory(); + var vsCodeId = Fixture().First(i => i.Title == "Visual Studio Code").Id; + for (var i = 0; i < 50; i++) + { + history = history.WithHistoryItem(vsCodeId); + } + + AssertRanksAbove("co", "Command Prompt", "Visual Studio Code", history); + AssertRanksAbove("co", "Control Panel", "Visual Studio Code", history); + } + + [TestMethod] + public void EndToEnd_FrecencyBreaksTieWithinTier() + { + // "vs" is an acronym match for both Visual Studio Code and Visual Studio 2022 (same + // tier). With no history they tie; the recently/repeatedly used one should climb to the + // top of the tier. + var fixture = Fixture(); + var vs2022Id = fixture.First(i => i.Title == "Visual Studio 2022").Id; + + var history = EmptyHistory(); + for (var i = 0; i < 5; i++) + { + history = history.WithHistoryItem(vs2022Id); + } + + AssertRanksAbove("vs", "Visual Studio 2022", "Visual Studio Code", history); + } + + [TestMethod] + public void EndToEnd_ProviderHigherBreaksAnExactTie() + { + // Two providers surface an identically-titled "Settings" command. Everything else being + // equal (same tier, same lexical quality, no frecency), a provider marked Higher should + // sort above the Normal one. Provider weight is a within-tier nudge for near-ties only. + var alpha = new ListItemMock("Settings", "From provider Alpha", providerId: "alpha"); + var bravo = new ListItemMock("Settings", "From provider Bravo", providerId: "bravo"); + var items = new List { alpha, bravo }; + + var matcher = CreateMatcher(); + var q = matcher.PrecomputeQuery("Settings"); + var history = EmptyHistory(); + + // Baseline: with no provider weighting the two tie exactly. + var baseAlpha = MainListPage.ScoreTopLevelItem(q, alpha, history, matcher); + var baseBravo = MainListPage.ScoreTopLevelItem(q, bravo, history, matcher); + Assert.AreEqual(baseAlpha, baseBravo, "The two identically-titled items should tie before provider weighting"); + + Func lookup = item => + item is ListItemMock m && m.ProviderId == "bravo" + ? ProviderSearchWeight.Higher + : ProviderSearchWeight.Normal; + + var ranked = items + .Select(item => (item.ProviderId, Score: MainListPage.ScoreTopLevelItem(q, item, history, matcher, lookup))) + .OrderByDescending(x => x.Score) + .Select(x => x.ProviderId) + .ToList(); + + Assert.AreEqual("bravo", ranked[0], "The Higher-weighted provider should win an otherwise exact tie"); + } + + [TestMethod] + public void EndToEnd_ProviderWeightCannotCrossTierBoundary() + { + // Even marked Higher, a word-boundary match (Visual Studio Code for "co") must stay + // below a prefix match (Command Prompt). Provider weight is clamped within a tier. + Func boostVsCode = item => + item is ListItemMock m && m.Title == "Visual Studio Code" + ? ProviderSearchWeight.Higher + : ProviderSearchWeight.Normal; + + AssertRanksAbove("co", "Command Prompt", "Visual Studio Code", providerWeightLookup: boostVsCode); + } +} diff --git a/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/ScoringParallelEquivalenceTests.cs b/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/ScoringParallelEquivalenceTests.cs new file mode 100644 index 0000000000..aa6a3c732f --- /dev/null +++ b/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/ScoringParallelEquivalenceTests.cs @@ -0,0 +1,252 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Linq; +using Microsoft.CmdPal.Common.Helpers; +using Microsoft.CmdPal.Common.Text; +using Microsoft.CmdPal.UI.ViewModels.Commands; +using Microsoft.CmdPal.UI.ViewModels.MainPage; +using Microsoft.CommandPalette.Extensions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using static Microsoft.CmdPal.UI.ViewModels.UnitTests.ScoringTestCatalog; + +namespace Microsoft.CmdPal.UI.ViewModels.UnitTests; + +/// +/// Guardrail for the throughput work: moving scoring off the TopLevelCommands lock and +/// parallelizing the apps pass may change how fast and where the settled list is computed, never +/// what it is. Across a synthetic catalog and several queries (the 1-char pathological case, the +/// extend chain, and the retype rebuild) the parallel scorer has to match the sequential one item +/// for item and score for score. +/// +[TestClass] +public sealed partial class ScoringParallelEquivalenceTests +{ + // Big enough that the parallel path is actually taken across multiple partitions, small enough + // to stay fast on CI. + private const int AppCount = 4000; + private const int CommandCount = 300; + private const int HistorySeedCount = 250; + + // "c" is the pathological 1-char case, the "ca"/"cal"/"calc" chain is the extend path, and the + // acronym and multi-word cases stress the tier classifier. + private static readonly string[] Queries = + ["c", "ca", "cal", "calc", "vs", "vsc", "vs code", "term", "set", "e"]; + + public TestContext TestContext { get; set; } = null!; + + private static ScoringFunction BuildScoringFunction( + IRecentCommandsManager history, + IPrecomputedFuzzyMatcher matcher) + => (in FuzzyQuery query, IListItem item) => + MainListPage.ScoreTopLevelItem(query, item, history, matcher, null); + + private static void AssertOrderedResultsIdentical( + string context, + RoScored[] reference, + RoScored[] candidate) + { + Assert.AreEqual(reference.Length, candidate.Length, $"[{context}] result count must match the sequential reference."); + + for (var i = 0; i < reference.Length; i++) + { + // Same packed score at the same index. + Assert.AreEqual( + reference[i].Score, + candidate[i].Score, + $"[{context}] score at index {i} must match the sequential reference."); + + // Same item reference at the same index, which proves the order matches including + // tie-breaks, not just that the same scores turn up. + Assert.AreSame( + reference[i].Item, + candidate[i].Item, + $"[{context}] item at index {i} must be the exact same instance as the sequential reference."); + } + } + + /// + /// The rebuild path: a fresh query scored against the whole catalog, where the parallel apps + /// pass has to match the sequential reference exactly. + /// + [TestMethod] + public void ParallelScoring_FullCatalog_MatchesSequentialForEveryQuery() + { + var apps = BuildCatalog(AppCount, "app"); + var matcher = CreateMatcher(); + var history = SeedHistory(apps, HistorySeedCount); + var scoringFn = BuildScoringFunction(history, matcher); + var source = apps.Cast().ToArray(); + + // Mirror the product: build the frecency index once before the parallel pass reads it. + history.PrewarmIndex(); + + foreach (var raw in Queries) + { + var query = matcher.PrecomputeQuery(raw); + + var sequential = InternalListHelpers.FilterListWithScores(source, query, scoringFn); + var parallel = InternalListHelpers.FilterListWithScoresParallel(source, query, scoringFn); + + TestContext.WriteLine($"query '{raw}': {sequential.Length} matches (sequential) vs {parallel.Length} (parallel)."); + AssertOrderedResultsIdentical($"full '{raw}'", sequential, parallel); + } + } + + /// + /// The extend path: score the whole catalog for a 1-char query, keep the matched subset in the + /// order it came back, then re-score that subset for the extending query. + /// + [TestMethod] + public void ParallelScoring_ExtendPath_MatchesSequentialOverRetainedSubset() + { + var apps = BuildCatalog(AppCount, "app"); + var matcher = CreateMatcher(); + var history = SeedHistory(apps, HistorySeedCount); + var scoringFn = BuildScoringFunction(history, matcher); + var source = apps.Cast().ToArray(); + + history.PrewarmIndex(); + + // Each step narrows the previous result. + var chain = new[] { "c", "ca", "cal", "calc" }; + + var retained = source; + for (var step = 1; step < chain.Length; step++) + { + // This is exactly what the product feeds the next keystroke: the previous result's + // items, in the previous result's order. + var prevQuery = matcher.PrecomputeQuery(chain[step - 1]); + var prev = InternalListHelpers.FilterListWithScores(retained, prevQuery, scoringFn); + retained = prev.Select(s => s.Item).ToArray(); + + var query = matcher.PrecomputeQuery(chain[step]); + var sequential = InternalListHelpers.FilterListWithScores(retained, query, scoringFn); + var parallel = InternalListHelpers.FilterListWithScoresParallel(retained, query, scoringFn); + + TestContext.WriteLine($"extend '{chain[step - 1]}' -> '{chain[step]}': retained {retained.Length}, kept {sequential.Length}."); + AssertOrderedResultsIdentical($"extend '{chain[step - 1]}'->'{chain[step]}'", sequential, parallel); + + Assert.IsTrue(sequential.Length <= retained.Length, "Extending a query cannot add matches beyond the retained set."); + } + } + + /// + /// The retype path: a query that doesn't extend the previous one forces a full rebuild, so + /// check a run of unrelated queries scored fresh each time. + /// + [TestMethod] + public void ParallelScoring_RetypeRebuild_MatchesSequential() + { + var apps = BuildCatalog(AppCount, "app"); + var matcher = CreateMatcher(); + var history = SeedHistory(apps, HistorySeedCount); + var scoringFn = BuildScoringFunction(history, matcher); + var source = apps.Cast().ToArray(); + + history.PrewarmIndex(); + + // Unrelated queries (each a fresh rebuild, never an extend of the last). + foreach (var raw in new[] { "calc", "term", "vs code", "settings", "e" }) + { + var query = matcher.PrecomputeQuery(raw); + var sequential = InternalListHelpers.FilterListWithScores(source, query, scoringFn); + var parallel = InternalListHelpers.FilterListWithScoresParallel(source, query, scoringFn); + + AssertOrderedResultsIdentical($"retype '{raw}'", sequential, parallel); + } + } + + /// + /// Commands (hundreds) stay serial, so the parallel entry point has to fall back below its + /// threshold and still match the sequential result. + /// + [TestMethod] + public void ParallelScoring_Commands_MatchesSequential() + { + var commands = BuildCatalog(CommandCount, "cmd"); + var matcher = CreateMatcher(); + var history = SeedHistory(commands, HistorySeedCount); + var scoringFn = BuildScoringFunction(history, matcher); + var source = commands.Cast().ToArray(); + + history.PrewarmIndex(); + + foreach (var raw in Queries) + { + var query = matcher.PrecomputeQuery(raw); + var sequential = InternalListHelpers.FilterListWithScores(source, query, scoringFn); + var parallel = InternalListHelpers.FilterListWithScoresParallel(source, query, scoringFn); + + AssertOrderedResultsIdentical($"commands '{raw}'", sequential, parallel); + } + } + + /// + /// Running the parallel scorer over the same catalog and query repeatedly gives the same + /// ordered result every time, whatever the thread scheduling does. + /// + [TestMethod] + public void ParallelScoring_IsDeterministicAcrossRuns() + { + var apps = BuildCatalog(AppCount, "app"); + var matcher = CreateMatcher(); + var history = SeedHistory(apps, HistorySeedCount); + var scoringFn = BuildScoringFunction(history, matcher); + var source = apps.Cast().ToArray(); + + history.PrewarmIndex(); + + var query = matcher.PrecomputeQuery("c"); + var first = InternalListHelpers.FilterListWithScoresParallel(source, query, scoringFn); + + for (var run = 0; run < 8; run++) + { + var again = InternalListHelpers.FilterListWithScoresParallel(source, query, scoringFn); + AssertOrderedResultsIdentical($"determinism run {run}", first, again); + } + } + + /// + /// The hot path snapshots the frecency manager, matcher, settings and one evaluation time per + /// query, and feeds the apps pass a single constant provider weight. This proves that captured + /// context lands on the same ordered result as the old per-item live reads. + /// + [TestMethod] + public void CapturedContext_ConstantWeightAndFixedNow_MatchesPerItemLiveRead() + { + var apps = BuildCatalog(AppCount, "app"); + var matcher = CreateMatcher(); + var history = SeedHistory(apps, HistorySeedCount); + var source = apps.Cast().ToArray(); + + history.PrewarmIndex(); + + // A non-default weight, so the value has to actually flow through to the packed score. + const ProviderSearchWeight weight = ProviderSearchWeight.Higher; + Func perItemLookup = _ => weight; + Func constantLookup = _ => weight; + + // Captured once before the loop, exactly as the product captures scoringNow. The reference + // path below omits it, so it reads the current time per call. + var capturedNow = DateTimeOffset.UtcNow; + + ScoringFunction liveReadScorer = (in FuzzyQuery query, IListItem item) => + MainListPage.ScoreTopLevelItem(query, item, history, matcher, perItemLookup); + ScoringFunction capturedContextScorer = (in FuzzyQuery query, IListItem item) => + MainListPage.ScoreTopLevelItem(query, item, history, matcher, constantLookup, capturedNow); + + foreach (var raw in Queries) + { + var query = matcher.PrecomputeQuery(raw); + + var reference = InternalListHelpers.FilterListWithScores(source, query, liveReadScorer); + var candidate = InternalListHelpers.FilterListWithScoresParallel(source, query, capturedContextScorer); + + TestContext.WriteLine($"captured-context '{raw}': {reference.Length} matches (per-item live read) vs {candidate.Length} (constant weight + fixed now)."); + AssertOrderedResultsIdentical($"captured '{raw}'", reference, candidate); + } + } +} diff --git a/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/ScoringTestCatalog.cs b/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/ScoringTestCatalog.cs new file mode 100644 index 0000000000..a8f7250c28 --- /dev/null +++ b/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/ScoringTestCatalog.cs @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using Microsoft.CmdPal.Common.Helpers; +using Microsoft.CmdPal.Common.Text; +using Microsoft.CmdPal.UI.ViewModels.Commands; +using Microsoft.CommandPalette.Extensions.Toolkit; + +namespace Microsoft.CmdPal.UI.ViewModels.UnitTests; + +internal static partial class ScoringTestCatalog +{ + internal sealed partial class CatalogItem : ListItem, IPrecomputedListItem + { + private FuzzyTargetCache _titleCache; + private FuzzyTargetCache _subtitleCache; + + internal CatalogItem(string title, string subtitle, string id) + : base(new NoOpCommand() { Id = id }) + { + Title = title; + Subtitle = subtitle; + Id = id; + } + + internal string Id { get; } + + public FuzzyTarget GetTitleTarget(IPrecomputedFuzzyMatcher matcher) => _titleCache.GetOrUpdate(matcher, Title); + + public FuzzyTarget GetSubtitleTarget(IPrecomputedFuzzyMatcher matcher) => _subtitleCache.GetOrUpdate(matcher, Subtitle); + } + + private static readonly string[] Nouns = + [ + "Calculator", "Calendar", "Camera", "Canvas", "Command", "Control", "Cloud", "Cast", + "Visual", "Studio", "Code", "Terminal", "Task", "Notepad", "Paint", "Photos", "Player", + "Panel", "Prompt", "Settings", "Store", "System", "Manager", "Monitor", "Editor", "Browser", + "Mail", "Maps", "Music", "Movies", "Network", "Office", "Onenote", "Outlook", "People", + ]; + + private static readonly string[] Qualifiers = + [ + string.Empty, "Pro", "2022", "3D", "Preview", "X", "Lite", "Plus", "Home", "Enterprise", + "for Windows", "Insider", "Legacy", "New", "Classic", + ]; + + private static readonly string[] SubtitleWords = + [ + "Perform calculations and conversions", "View and manage your schedule", "Edit and refine images", + "Modern terminal for command-line tools", "Full-featured integrated development environment", + "Browse the web quickly and securely", "Adjust your computer settings", "Monitor apps and processes", + "A simple and fast text editor", "Play and organize your media library", + ]; + + internal static IPrecomputedFuzzyMatcher CreateMatcher() => new PrecomputedFuzzyMatcher(new PrecomputedFuzzyMatcherOptions()); + + internal static CatalogItem[] BuildCatalog(int count, string idPrefix) + { + var items = new CatalogItem[count]; + for (var i = 0; i < count; i++) + { + var noun = Nouns[i % Nouns.Length]; + var qualifier = Qualifiers[(i / Nouns.Length) % Qualifiers.Length]; + var title = string.IsNullOrEmpty(qualifier) ? noun : $"{noun} {qualifier}"; + + if (i >= Nouns.Length * Qualifiers.Length) + { + title = $"{title} {i}"; + } + + var subtitle = SubtitleWords[i % SubtitleWords.Length]; + items[i] = new CatalogItem(title, subtitle, $"{idPrefix}.{i}"); + } + + return items; + } + + internal static RecentCommandsManager SeedHistory(CatalogItem[] apps, int seedCount) + { + var history = new RecentCommandsManager(); + var n = Math.Min(seedCount, apps.Length); + for (var i = 0; i < n; i++) + { + var idx = (i * 7) % apps.Length; + history = history.WithHistoryItem(apps[idx].Id); + } + + return history; + } +} diff --git a/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/ScoringThroughputHarnessTests.cs b/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/ScoringThroughputHarnessTests.cs new file mode 100644 index 0000000000..2241445d82 --- /dev/null +++ b/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/ScoringThroughputHarnessTests.cs @@ -0,0 +1,319 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Microsoft.CmdPal.Common.Helpers; +using Microsoft.CmdPal.Common.Text; +using Microsoft.CmdPal.UI.ViewModels.Commands; +using Microsoft.CmdPal.UI.ViewModels.MainPage; +using Microsoft.CommandPalette.Extensions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using static Microsoft.CmdPal.UI.ViewModels.UnitTests.ScoringTestCatalog; + +namespace Microsoft.CmdPal.UI.ViewModels.UnitTests; + +/// +/// Times the real per-keystroke scoring passes over a synthetic catalog and attributes the cost +/// across apps-enumeration, command scoring, app scoring (the dominant pass), and fallback +/// fold-in, plus a per-item split of into fuzzy DP +/// vs tier classification vs frecency. Timings go to and the assertions +/// only lock structural facts, so nothing here flakes on a wall clock. +/// +[TestClass] +public sealed partial class ScoringThroughputHarnessTests +{ + // Sized to mirror a heavy-but-realistic machine, and small enough that the whole harness runs + // in a couple of seconds on CI. + private const int AppCount = 3000; + private const int CommandCount = 300; + private const int GlobalFallbackCount = 5; + private const int PinnedAppCount = 20; + + // Seed enough history that frecency lookups actually hit, so we measure the hit path too. + private const int HistorySeedCount = 200; + + // Report-only counts: warmups prime the JIT and target caches, measured runs get averaged. + private const int WarmupIterations = 3; + private const int MeasuredIterations = 10; + + // "c" is the pathological 1-char case that matches nearly every app, the rest narrow a real + // prefix, and "vsc"/"vs code" hit the acronym and word-boundary paths. + private static readonly string[] Queries = ["c", "ca", "cal", "calc", "vsc", "vs code"]; + + public TestContext TestContext { get; set; } = null!; + + private static ScoringFunction BuildScoringFunction( + IRecentCommandsManager history, + IPrecomputedFuzzyMatcher matcher, + Func? providerWeightLookup = null) + => (in FuzzyQuery query, IListItem item) => + MainListPage.ScoreTopLevelItem(query, item, history, matcher, providerWeightLookup); + + // Averaged elapsed milliseconds after a warmup, report-only and never asserted against a + // threshold. + private static double TimeAverageMs(Action action) + { + for (var i = 0; i < WarmupIterations; i++) + { + action(); + } + + var sw = Stopwatch.StartNew(); + for (var i = 0; i < MeasuredIterations; i++) + { + action(); + } + + sw.Stop(); + return sw.Elapsed.TotalMilliseconds / MeasuredIterations; + } + + /// + /// Attributes a full rebuild keystroke (empty to query, the worst case that scores the whole + /// catalog) across the four passes. Asserts only that the app pass outweighs the command pass + /// and that scoring is deterministic. + /// + [TestMethod] + public void FullKeystroke_AttributesCostAcrossBuckets() + { + var apps = BuildCatalog(AppCount, "app"); + var commands = BuildCatalog(CommandCount, "cmd"); + var globalFallbacks = BuildCatalog(GlobalFallbackCount, "gfb").Cast().ToList(); + var matcher = CreateMatcher(); + var history = SeedHistory(apps, HistorySeedCount); + var scoringFn = BuildScoringFunction(history, matcher); + + // Simulate the pinned-app removal the product does on the rebuild path. + var pinnedIds = new HashSet(apps.Take(PinnedAppCount).Select(a => a.Id)); + + TestContext.WriteLine($"Catalog: {AppCount} apps, {CommandCount} commands, {GlobalFallbackCount} global fallbacks, {HistorySeedCount} history seeds."); + TestContext.WriteLine($"Iterations: {WarmupIterations} warmup + {MeasuredIterations} measured (averaged)."); + TestContext.WriteLine("query | appsEnum ms | cmdScore ms | appScore ms | fallback ms | TOTAL ms | cmdMatches | appMatches"); + + foreach (var raw in Queries) + { + var query = matcher.PrecomputeQuery(raw); + + var enumMs = TimeAverageMs(() => + { + // Mirrors the product's GetItems().Cast().ToList() plus the pinned filter. + var materialized = apps.ToList(); + _ = materialized.Where(a => !pinnedIds.Contains(a.Id)).ToList(); + }); + + RoScored[] cmdScored = []; + var cmdMs = TimeAverageMs(() => + { + cmdScored = InternalListHelpers.FilterListWithScores(commands.Cast(), query, scoringFn); + }); + + RoScored[] appScored = []; + var appMs = TimeAverageMs(() => + { + appScored = InternalListHelpers.FilterListWithScores(apps.Cast(), query, scoringFn); + }); + + var fallbackMs = TimeAverageMs(() => + { + _ = MainListPage.ScoreDeferredFallbacks(globalFallbacks, query, scoringFn); + }); + + var total = enumMs + cmdMs + appMs + fallbackMs; + TestContext.WriteLine( + $"{raw,-8}| {enumMs,10:F3} | {cmdMs,10:F3} | {appMs,10:F3} | {fallbackMs,10:F3} | {total,8:F3} | {cmdScored.Length,10} | {appScored.Length,10}"); + + // The app pass weighs thousands of items against the command pass's hundreds, which is + // why it dominates the keystroke cost. + Assert.IsTrue(apps.Length > commands.Length, "Apps must outnumber commands in the catalog."); + + // Re-scoring the same catalog with the same query yields the same result set. + var appScoredAgain = InternalListHelpers.FilterListWithScores(apps.Cast(), query, scoringFn); + Assert.AreEqual(appScored.Length, appScoredAgain.Length, $"App scoring must be deterministic for query '{raw}'."); + for (var i = 0; i < Math.Min(10, appScored.Length); i++) + { + Assert.AreEqual(appScored[i].Score, appScoredAgain[i].Score, $"Top-10 scores must be stable for query '{raw}'."); + } + } + } + + /// + /// Characterizes why the settle time spikes: a 1-char query retains a large slice of the + /// catalog, so the next few keystrokes still re-score a big set before it narrows. + /// + [TestMethod] + public void OneCharQuery_RetainsLargeMatchSet_DrivesIncrementalCost() + { + var apps = BuildCatalog(AppCount, "app"); + var matcher = CreateMatcher(); + var history = SeedHistory(apps, HistorySeedCount); + var scoringFn = BuildScoringFunction(history, matcher); + + var oneChar = matcher.PrecomputeQuery("c"); + var firstMatches = InternalListHelpers.FilterListWithScores(apps.Cast(), oneChar, scoringFn); + + // Extending to "ca" re-scores only the retained subset, but that subset is still large, + // which is why the spike carries across frames. + var retained = firstMatches.Select(s => s.Item).ToList(); + var twoChar = matcher.PrecomputeQuery("ca"); + var secondMatches = InternalListHelpers.FilterListWithScores(retained, twoChar, scoringFn); + + var firstFraction = (double)firstMatches.Length / apps.Length; + TestContext.WriteLine($"1-char 'c' matches {firstMatches.Length}/{apps.Length} apps ({firstFraction:P1}); extending to 'ca' re-scores {retained.Count} and keeps {secondMatches.Length}."); + + // A narrower query can only keep a subset of the wider query's matches. + Assert.IsTrue(secondMatches.Length <= retained.Count, "Extending a query cannot add matches beyond the retained set."); + Assert.IsTrue(firstMatches.Length > 0, "The 1-char query should match a non-trivial set."); + } + + /// + /// Splits a single into fuzzy DP scoring, tier + /// classification, and frecency lookup so the overhaul's added cost is visible. Asserts only + /// the direction of the extension-score delta, which holds regardless of the machine. + /// + [TestMethod] + public void PerItemScore_SubAttribution_DpVsTierVsFrecency() + { + var apps = BuildCatalog(AppCount, "app"); + var matcher = CreateMatcher(); + var history = SeedHistory(apps, HistorySeedCount); + var scoringFn = BuildScoringFunction(history, matcher); + + // Precompute targets once, like the live cached path, so this measures matcher.Score and + // not target construction. + var titleTargets = apps.Select(a => a.GetTitleTarget(matcher)).ToArray(); + var subtitleTargets = apps.Select(a => a.GetSubtitleTarget(matcher)).ToArray(); + var extensionTargets = apps.Select(a => matcher.PrecomputeTarget($"{a.Title} Extension")).ToArray(); + var ids = apps.Select(a => a.Id).ToArray(); + + TestContext.WriteLine("Per-item sub-attribution (nanoseconds/item, averaged over the app catalog):"); + TestContext.WriteLine("query | full score | 2 DP | 3 DP | extDelta | classifyTier | wordBoundary | frecency"); + + foreach (var raw in Queries) + { + var query = matcher.PrecomputeQuery(raw); + + var fullNs = PerItemNs(() => + { + for (var i = 0; i < apps.Length; i++) + { + _ = MainListPage.ScoreTopLevelItem(query, apps[i], history, matcher, null); + } + }); + + var twoDpNs = PerItemNs(() => + { + for (var i = 0; i < apps.Length; i++) + { + _ = matcher.Score(query, titleTargets[i]) + matcher.Score(query, subtitleTargets[i]); + } + }); + + var threeDpNs = PerItemNs(() => + { + for (var i = 0; i < apps.Length; i++) + { + _ = matcher.Score(query, titleTargets[i]) + matcher.Score(query, subtitleTargets[i]) + matcher.Score(query, extensionTargets[i]); + } + }); + + var classifyNs = PerItemNs(() => + { + for (var i = 0; i < apps.Length; i++) + { + _ = MainListRanker.ClassifyTier(query.Original, apps[i].Title, false, false, false, true); + } + }); + + var wordBoundaryNs = PerItemNs(() => + { + for (var i = 0; i < apps.Length; i++) + { + _ = MainListRanker.MatchesWordBoundaryOrAcronym(apps[i].Title, query.Original.AsSpan()); + } + }); + + var frecencyNs = PerItemNs(() => + { + for (var i = 0; i < apps.Length; i++) + { + _ = history.GetCommandHistoryWeight(ids[i]); + } + }); + + var extDelta = threeDpNs - twoDpNs; + TestContext.WriteLine( + $"{raw,-8}| {fullNs,10:F1} | {twoDpNs,6:F1} | {threeDpNs,6:F1} | {extDelta,8:F1} | {classifyNs,12:F1} | {wordBoundaryNs,12:F1} | {frecencyNs,8:F1}"); + + // Adding a third DP score can't make the measurement cheaper. + Assert.IsTrue(threeDpNs >= twoDpNs * 0.5, "Three DP scores should not be dramatically cheaper than two; extension scoring is real added work."); + } + } + + /// + /// Times the dominant apps pass serial versus parallel and reports the speedup per query. + /// Report-only: it asserts the two paths return the same match count, never a wall-clock + /// threshold. + /// + [TestMethod] + public void AppScoring_BeforeAfter_SerialVsParallelThroughput() + { + var apps = BuildCatalog(AppCount, "app"); + var matcher = CreateMatcher(); + var history = SeedHistory(apps, HistorySeedCount); + var scoringFn = BuildScoringFunction(history, matcher); + var source = apps.Cast().ToArray(); + + // Build the frecency index once, single-threaded, before the parallel pass reads it. + history.PrewarmIndex(); + + TestContext.WriteLine($"CPU count: {Environment.ProcessorCount}. Catalog: {AppCount} apps."); + TestContext.WriteLine("query | serial ms (before) | parallel ms (after) | speedup | matches"); + + foreach (var raw in Queries) + { + var query = matcher.PrecomputeQuery(raw); + + RoScored[] serialResult = []; + var serialMs = TimeAverageMs(() => + { + serialResult = InternalListHelpers.FilterListWithScores(source, query, scoringFn); + }); + + RoScored[] parallelResult = []; + var parallelMs = TimeAverageMs(() => + { + parallelResult = InternalListHelpers.FilterListWithScoresParallel(source, query, scoringFn); + }); + + var speedup = parallelMs > 0 ? serialMs / parallelMs : 0.0; + TestContext.WriteLine( + $"{raw,-8}| {serialMs,17:F3} | {parallelMs,18:F3} | {speedup,6:F2}x | {serialResult.Length,7}"); + + // The parallel path returns the same match count, on any machine. + Assert.AreEqual(serialResult.Length, parallelResult.Length, $"Match count must match for query '{raw}'."); + } + } + + // Averaged per-item nanoseconds for a loop that internally iterates the whole app catalog once. + private static double PerItemNs(Action loopOverCatalog) + { + for (var i = 0; i < WarmupIterations; i++) + { + loopOverCatalog(); + } + + var sw = Stopwatch.StartNew(); + for (var i = 0; i < MeasuredIterations; i++) + { + loopOverCatalog(); + } + + sw.Stop(); + var totalItems = (double)MeasuredIterations * AppCount; + return sw.Elapsed.TotalMilliseconds * 1_000_000.0 / totalItems; + } +} diff --git a/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/SearchTelemetryTests.cs b/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/SearchTelemetryTests.cs new file mode 100644 index 0000000000..abec49d279 --- /dev/null +++ b/src/modules/cmdpal/Tests/Microsoft.CmdPal.UI.ViewModels.UnitTests/SearchTelemetryTests.cs @@ -0,0 +1,198 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.CmdPal.Common.Helpers; +using Microsoft.CmdPal.UI.ViewModels.MainPage; +using Microsoft.CmdPal.UI.ViewModels.Messages; +using Microsoft.CommandPalette.Extensions; +using Microsoft.CommandPalette.Extensions.Toolkit; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Windows.Foundation; + +namespace Microsoft.CmdPal.UI.ViewModels.UnitTests; + +/// +/// Tests for the opt-in, privacy-safe main-page search telemetry payload builders. These assert +/// that only non-identifying aggregates are captured (query LENGTH, result count, selected rank, +/// ranker tier) and never the raw query text or item content. The actual telemetry sink is not +/// exercised - only the payload-building logic. +/// +[TestClass] +public partial class SearchTelemetryTests +{ + private sealed partial class MockListItem : IListItem + { + public string Title { get; set; } = string.Empty; + + public string Subtitle { get; set; } = string.Empty; + + public ICommand Command => new NoOpCommand(); + + public IDetails? Details => null; + + public IIconInfo? Icon => null; + + public string Section => throw new NotImplementedException(); + + public ITag[] Tags => throw new NotImplementedException(); + + public string TextToSuggest => throw new NotImplementedException(); + + public IContextItem[] MoreCommands => throw new NotImplementedException(); + +#pragma warning disable CS0067 // The event is never used + public event TypedEventHandler? PropChanged; +#pragma warning restore CS0067 // The event is never used + + public override string ToString() => Title; + } + + private static RoScored Scored(IListItem item, int score) => new(item, score); + + [TestMethod] + public void SearchResultsMessage_CapturesQueryLengthNotText() + { + const string query = "hello world"; + var message = MainListPageSearchTelemetry.BuildSearchResultsMessage(query, resultCount: 4, latencyMs: 12); + + Assert.AreEqual(query.Length, message.QueryLength); + Assert.AreEqual(4, message.ResultCount); + Assert.IsFalse(message.NoResults); + Assert.AreEqual(12UL, message.LatencyMs); + } + + [TestMethod] + public void SearchResultsMessage_SetsNoResultsFlagWhenCountIsZero() + { + var noResults = MainListPageSearchTelemetry.BuildSearchResultsMessage("abc", resultCount: 0, latencyMs: 5); + Assert.IsTrue(noResults.NoResults); + Assert.AreEqual(0, noResults.ResultCount); + + var hasResults = MainListPageSearchTelemetry.BuildSearchResultsMessage("abc", resultCount: 3, latencyMs: 5); + Assert.IsFalse(hasResults.NoResults); + } + + [TestMethod] + public void SearchResultsMessage_ClampsNegativeInputs() + { + var message = MainListPageSearchTelemetry.BuildSearchResultsMessage(queryLength: -3, resultCount: -1, latencyMs: -100); + + Assert.AreEqual(0, message.QueryLength); + Assert.AreEqual(0, message.ResultCount); + Assert.IsTrue(message.NoResults); + Assert.AreEqual(0UL, message.LatencyMs); + } + + [TestMethod] + public void SearchResultsMessage_HasNoStringFields() + { + // A raw query string can only be captured through a string member. Assert there is none, + // so the payload provably cannot carry the query text. + var stringProperties = typeof(TelemetrySearchResultsMessage) + .GetProperties() + .Where(p => p.PropertyType == typeof(string)) + .ToList(); + + Assert.AreEqual(0, stringProperties.Count, "Search results telemetry must not carry any string (potential query text)."); + } + + [TestMethod] + public void SelectedMessage_CapturesQueryLengthIndexAndTier() + { + const string query = "code"; + var message = MainListPageSearchTelemetry.BuildSearchSelectedMessage(query, selectedIndex: 2, selectedTier: RankTier.Prefix); + + Assert.AreEqual(query.Length, message.QueryLength); + Assert.AreEqual(2, message.SelectedIndex); + Assert.AreEqual(RankTier.Prefix, message.SelectedTier); + } + + [TestMethod] + public void SelectedMessage_HasNoStringFields() + { + var stringProperties = typeof(TelemetrySearchResultSelectedMessage) + .GetProperties() + .Where(p => p.PropertyType == typeof(string)) + .ToList(); + + Assert.AreEqual(0, stringProperties.Count, "Selection telemetry must not carry any string (potential query text or item title)."); + } + + [TestMethod] + public void ResolveSelectedTier_DerivesTierFromPackedScore() + { + var exact = new MockListItem { Title = "Visual Studio" }; + var fuzzy = new MockListItem { Title = "Notepad" }; + + var packed = new List> + { + Scored(exact, MainListRanker.Pack(RankTier.ExactTitle, withinTierScore: 500)), + Scored(fuzzy, MainListRanker.Pack(RankTier.Fuzzy, withinTierScore: 10)), + }; + + Assert.AreEqual(RankTier.ExactTitle, MainListPageSearchTelemetry.ResolveSelectedTier(exact, packed, fallbackResults: null)); + Assert.AreEqual(RankTier.Fuzzy, MainListPageSearchTelemetry.ResolveSelectedTier(fuzzy, packed, fallbackResults: null)); + } + + [TestMethod] + public void ResolveSelectedTier_ReportsFallbackFloorForCommonFallbacks() + { + var fallback = new MockListItem { Title = "Search the web" }; + + // Common fallbacks carry small rank-based (non-packed) scores. They must be reported at the + // fallback floor rather than being decoded as a packed tier. + var fallbacks = new List> { Scored(fallback, 3) }; + + Assert.AreEqual(RankTier.FallbackFloor, MainListPageSearchTelemetry.ResolveSelectedTier(fallback, packedResults: null, fallbacks)); + } + + [TestMethod] + public void ResolveSelectedTier_ReturnsNoneWhenItemNotFound() + { + var known = new MockListItem { Title = "Known" }; + var unknown = new MockListItem { Title = "Unknown" }; + + var packed = new List> { Scored(known, MainListRanker.Pack(RankTier.Prefix, 1)) }; + + Assert.AreEqual(RankTier.None, MainListPageSearchTelemetry.ResolveSelectedTier(unknown, packed, fallbackResults: null)); + } + + [TestMethod] + public void ResolveVisibleIndex_SkipsSeparatorsAndReturnsVisibleRank() + { + var resultsSeparator = new Separator("Results"); + var fallbacksSeparator = new Separator("Fallbacks"); + + var a = new MockListItem { Title = "A" }; + var b = new MockListItem { Title = "B" }; + var c = new MockListItem { Title = "C" }; + var missing = new MockListItem { Title = "Missing" }; + + var rendered = new IListItem[] { resultsSeparator, a, b, fallbacksSeparator, c }; + + Assert.AreEqual(0, MainListPageSearchTelemetry.ResolveVisibleIndex(rendered, a, resultsSeparator, fallbacksSeparator)); + Assert.AreEqual(1, MainListPageSearchTelemetry.ResolveVisibleIndex(rendered, b, resultsSeparator, fallbacksSeparator)); + Assert.AreEqual(2, MainListPageSearchTelemetry.ResolveVisibleIndex(rendered, c, resultsSeparator, fallbacksSeparator)); + Assert.AreEqual(-1, MainListPageSearchTelemetry.ResolveVisibleIndex(rendered, missing, resultsSeparator, fallbacksSeparator)); + Assert.AreEqual(-1, MainListPageSearchTelemetry.ResolveVisibleIndex(null, a, resultsSeparator, fallbacksSeparator)); + } + + [TestMethod] + public void TierOf_RoundTripsEveryPackedTier() + { + foreach (RankTier tier in Enum.GetValues()) + { + if (tier == RankTier.None) + { + continue; + } + + var packed = MainListRanker.Pack(tier, withinTierScore: 42); + Assert.AreEqual(tier, MainListRanker.TierOf(packed), $"TierOf should round-trip {tier}."); + } + } +}