CmdPal: Sync access to TopLevelCommandManager from UpdateCommandsForProvider (#40752)

## Summary of the Pull Request

Fixes unsynchronized access to `LoadTopLevelCommands` in
`TopLevelCommandManager.UpdateCommandsForProvider`, which previously led
to `InvalidOperationException: Collection was modified`.

Addressing this also uncovered another issue: overlapping invocations of
`ReloadAllCommandsAsync` were causing duplication of items in the main
list -- so I'm fixing that as well.

## PR Checklist

- [x] Closes
    - Fixes #38194 
    - Partially solves #40776
- [ ] **Communication:** I've discussed this with core contributors
already. If the work hasn't been agreed, this work might be rejected
- [ ] **Tests:**
- [x] **Localization:** none
- [x] **Dev docs:** none
- [x] **New binaries:** nope
- [x] **Documentation updated:** no need

## Detailed Description of the Pull Request / Additional comments

## Validation Steps Performed
Tested with bookmarks.

src/modules/cmdpal/Microsoft.CmdPal.Common/Helpers/SupersedingAsyncGate.cs

@@ -0,0 +1,139 @@
+// Copyright (c) Microsoft Corporation
+// The Microsoft Corporation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.CmdPal.Common.Helpers;
+
+/// <summary>
+/// An async gate that ensures only one operation runs at a time.
+/// If ExecuteAsync is called while already executing, it cancels the current execution
+/// and starts the operation again (superseding behavior).
+/// </summary>
+public class SupersedingAsyncGate : IDisposable
+{
+    private readonly Func<CancellationToken, Task> _action;
+    private readonly Lock _lock = new();
+    private int _callId;
+    private TaskCompletionSource<bool>? _currentTcs;
+    private CancellationTokenSource? _currentCancellationSource;
+    private Task? _executingTask;
+
+    public SupersedingAsyncGate(Func<CancellationToken, Task> action)
+    {
+        ArgumentNullException.ThrowIfNull(action);
+        _action = action;
+    }
+
+    /// <summary>
+    /// Executes the configured action. If another execution is running, this call will
+    /// cancel the current execution and restart the operation.
+    /// </summary>
+    /// <param name="cancellationToken">Optional external cancellation token</param>
+    public async Task ExecuteAsync(CancellationToken cancellationToken = default)
+    {
+        TaskCompletionSource<bool> tcs;
+
+        lock (_lock)
+        {
+            _currentCancellationSource?.Cancel();
+            _currentTcs?.TrySetException(new OperationCanceledException("Superseded by newer call"));
+
+            tcs = new();
+            _currentTcs = tcs;
+            _callId++;
+
+            var shouldStartExecution = _executingTask is null;
+            if (shouldStartExecution)
+            {
+                _executingTask = Task.Run(ExecuteLoop, CancellationToken.None);
+            }
+        }
+
+        await using var ctr = cancellationToken.Register(() => tcs.TrySetCanceled(cancellationToken));
+        await tcs.Task;
+    }
+
+    private async Task ExecuteLoop()
+    {
+        try
+        {
+            while (true)
+            {
+                TaskCompletionSource<bool>? currentTcs;
+                CancellationTokenSource? currentCts;
+                int currentCallId;
+
+                lock (_lock)
+                {
+                    currentTcs = _currentTcs;
+                    currentCallId = _callId;
+
+                    if (currentTcs is null)
+                    {
+                        break;
+                    }
+
+                    _currentCancellationSource?.Dispose();
+                    _currentCancellationSource = new();
+                    currentCts = _currentCancellationSource;
+                }
+
+                try
+                {
+                    await _action(currentCts.Token);
+                    CompleteIfCurrent(currentTcs, currentCallId, static t => t.TrySetResult(true));
+                }
+                catch (OperationCanceledException)
+                {
+                    CompleteIfCurrent(currentTcs, currentCallId, tcs => tcs.SetCanceled(currentCts.Token));
+                }
+                catch (Exception ex)
+                {
+                    CompleteIfCurrent(currentTcs, currentCallId, tcs => tcs.TrySetException(ex));
+                }
+            }
+        }
+        finally
+        {
+            lock (_lock)
+            {
+                _currentTcs = null;
+                _currentCancellationSource?.Dispose();
+                _currentCancellationSource = null;
+                _executingTask = null;
+            }
+        }
+    }
+
+    private void CompleteIfCurrent(
+        TaskCompletionSource<bool> candidate,
+        int id,
+        Action<TaskCompletionSource<bool>> complete)
+    {
+        lock (_lock)
+        {
+            if (_currentTcs == candidate && _callId == id)
+            {
+                complete(candidate);
+                _currentTcs = null;
+            }
+        }
+    }
+
+    public void Dispose()
+    {
+        lock (_lock)
+        {
+            _currentCancellationSource?.Cancel();
+            _currentCancellationSource?.Dispose();
+            _currentTcs?.TrySetException(new ObjectDisposedException(nameof(SupersedingAsyncGate)));
+            _currentTcs = null;
+        }
+
+        GC.SuppressFinalize(this);
+    }
+}

src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/TopLevelCommandManager.cs

@@ -9,6 +9,7 @@ using CommunityToolkit.Mvvm.ComponentModel;
 using CommunityToolkit.Mvvm.Input;
 using CommunityToolkit.Mvvm.Messaging;
 using ManagedCommon;
+using Microsoft.CmdPal.Common.Helpers;
 using Microsoft.CmdPal.Common.Services;
 using Microsoft.CmdPal.Core.ViewModels;
 using Microsoft.CmdPal.Core.ViewModels.Messages;
@@ -20,7 +21,8 @@ namespace Microsoft.CmdPal.UI.ViewModels;
 
 public partial class TopLevelCommandManager : ObservableObject,
     IRecipient<ReloadCommandsMessage>,
-    IPageContext
+    IPageContext,
+    IDisposable
 {
     private readonly IServiceProvider _serviceProvider;
     private readonly TaskScheduler _taskScheduler;
@@ -28,6 +30,7 @@ public partial class TopLevelCommandManager : ObservableObject,
     private readonly List<CommandProviderWrapper> _builtInCommands = [];
     private readonly List<CommandProviderWrapper> _extensionCommandProviders = [];
     private readonly Lock _commandProvidersLock = new();
+    private readonly SupersedingAsyncGate _reloadCommandsGate;
 
     TaskScheduler IPageContext.Scheduler => _taskScheduler;
 
@@ -36,6 +39,7 @@ public partial class TopLevelCommandManager : ObservableObject,
         _serviceProvider = serviceProvider;
         _taskScheduler = _serviceProvider.GetService<TaskScheduler>()!;
         WeakReferenceMessenger.Default.Register<ReloadCommandsMessage>(this);
+        _reloadCommandsGate = new(ReloadAllCommandsAsyncCore);
     }
 
     public ObservableCollection<TopLevelViewModel> TopLevelCommands { get; set; } = [];
@@ -143,14 +147,30 @@ public partial class TopLevelCommandManager : ObservableObject,
     /// <param name="args">the ItemsChangedEvent the provider raised</param>
     /// <returns>an awaitable task</returns>
     private async Task UpdateCommandsForProvider(CommandProviderWrapper sender, IItemsChangedEventArgs args)
+    {
+        WeakReference<IPageContext> weakSelf = new(this);
+        await sender.LoadTopLevelCommands(_serviceProvider, weakSelf);
+
+        List<TopLevelViewModel> newItems = [..sender.TopLevelItems];
+        foreach (var i in sender.FallbackItems)
+        {
+            if (i.IsEnabled)
+            {
+                newItems.Add(i);
+            }
+        }
+
+        // modify the TopLevelCommands under shared lock; event if we clone it, we don't want
+        // TopLevelCommands to get modified while we're working on it. Otherwise, we might
+        // out clone would be stale at the end of this method.
+        lock (TopLevelCommands)
         {
             // Work on a clone of the list, so that we can just do one atomic
             // update to the actual observable list at the end
+            // TODO: just added a lock around all of this anyway, but keeping the clone
+            // while looking on some other ways to improve this; can be removed later.
             List<TopLevelViewModel> clone = [.. TopLevelCommands];
-        List<TopLevelViewModel> newItems = [];
             var startIndex = -1;
-        var firstCommand = sender.TopLevelItems[0];
-        var commandsToRemove = sender.TopLevelItems.Length + sender.FallbackItems.Length;
 
             // Tricky: all Commands from a single provider get added to the
             // top-level list all together, in a row. So if we find just the first
@@ -160,8 +180,7 @@ public partial class TopLevelCommandManager : ObservableObject,
                 var wrapper = clone[i];
                 try
                 {
-                var isTheSame = wrapper == firstCommand;
+                    if (sender.ProviderId == wrapper.CommandProviderId)
-                if (isTheSame)
                     {
                         startIndex = i;
                         break;
@@ -172,45 +191,21 @@ public partial class TopLevelCommandManager : ObservableObject,
                 }
             }
 
-        WeakReference<IPageContext> weakSelf = new(this);
+            clone.RemoveAll(item => item.CommandProviderId == sender.ProviderId);
-
-        // Fetch the new items
-        await sender.LoadTopLevelCommands(_serviceProvider, weakSelf);
-
-        var settings = _serviceProvider.GetService<SettingsModel>()!;
-
-        foreach (var i in sender.TopLevelItems)
-        {
-            newItems.Add(i);
-        }
-
-        foreach (var i in sender.FallbackItems)
-        {
-            if (i.IsEnabled)
-            {
-                newItems.Add(i);
-            }
-        }
-
-        // Slice out the old commands
-        if (startIndex != -1)
-        {
-            clone.RemoveRange(startIndex, commandsToRemove);
-        }
-        else
-        {
-            // ... or, just stick them at the end (this is unexpected)
-            startIndex = clone.Count;
-        }
-
-        // add the new commands into the list at the place we found the old ones
             clone.InsertRange(startIndex, newItems);
-
-        // now update the actual observable list with the new contents
             ListHelpers.InPlaceUpdateList(TopLevelCommands, clone);
         }
+    }
 
     public async Task ReloadAllCommandsAsync()
+    {
+        // gate ensures that the reload is serialized and if multiple calls
+        // request a reload, only the first and the last one will be executed.
+        // this should be superseded with a cancellable version.
+        await _reloadCommandsGate.ExecuteAsync(CancellationToken.None);
+    }
+
+    private async Task ReloadAllCommandsAsyncCore(CancellationToken cancellationToken)
     {
         IsLoading = true;
         var extensionService = _serviceProvider.GetService<IExtensionService>()!;
@@ -419,4 +414,10 @@ public partial class TopLevelCommandManager : ObservableObject,
                    || _extensionCommandProviders.Any(wrapper => wrapper.Id == id && wrapper.IsActive);
         }
     }
+
+    public void Dispose()
+    {
+        _reloadCommandsGate.Dispose();
+        GC.SuppressFinalize(this);
+    }
 }

src/modules/cmdpal/Microsoft.CmdPal.UI.ViewModels/TopLevelViewModel.cs

@@ -47,6 +47,8 @@ public sealed partial class TopLevelViewModel : ObservableObject, IListItem
 
     public CommandItemViewModel ItemViewModel => _commandItemViewModel;
 
+    public string CommandProviderId => _commandProviderId;
+
     ////// ICommandItem
     public string Title => _commandItemViewModel.Title;
 
@@ -351,4 +353,9 @@ public sealed partial class TopLevelViewModel : ObservableObject, IListItem
     {
         return new PerformCommandMessage(this.CommandViewModel.Model, new Core.ViewModels.Models.ExtensionObject<IListItem>(this));
     }
+
+    public override string ToString()
+    {
+        return $"{nameof(TopLevelViewModel)}: {Id} ({Title}) - display: {DisplayTitle} - fallback: {IsFallback} - enabled: {IsEnabled}";
+    }
 }