CmdPal: Make Command Palette's main and toast windows tool windows (#41835)

## Summary of the Pull Request
Replaces the `SetVisibilityInSwitchers` method with `WS_EX_TOOLWINDOW`
style, removing reliance on the fragile `IsShownInSwitchers` property.
Explicitly overrides window corner preference so tool windows keep the
same rounded corners as regular app windows (tool windows otherwise
default to a smaller corner radius, which would change the look).

Benefits:
- Avoids use of the unreliable `IsShownInSwitchers` property, which has
multiple known failure cases.
- Allows window managers to correctly treat Command Palette as a popup
window rather than a normal app window, preventing unwanted tiling, grid
placement, or similar behaviors. This better matches the intended design
of CmdPal as an overlay UI.


<!-- Please review the items on the PR checklist before submitting-->
## PR Checklist

- [x] Closes: #41772
- [x] Closes: #41593
- [ ] **Communication:** I've discussed this with core contributors
already. If the work hasn't been agreed, this work might be rejected
- [ ] **Tests:** Added/updated and all pass
- [ ] **Localization:** All end-user-facing strings can be localized
- [ ] **Dev docs:** Added/updated
- [ ] **New binaries:** Added on the required places
- [ ] [JSON for
signing](https://github.com/microsoft/PowerToys/blob/main/.pipelines/ESRPSigning_core.json)
for new binaries
- [ ] [WXS for
installer](https://github.com/microsoft/PowerToys/blob/main/installer/PowerToysSetup/Product.wxs)
for new binaries and localization folder
- [ ] [YML for CI
pipeline](https://github.com/microsoft/PowerToys/blob/main/.pipelines/ci/templates/build-powertoys-steps.yml)
for new test projects
- [ ] [YML for signed
pipeline](https://github.com/microsoft/PowerToys/blob/main/.pipelines/release.yml)
- [ ] **Documentation updated:** If checked, please file a pull request
on [our docs
repo](https://github.com/MicrosoftDocs/windows-uwp/tree/docs/hub/powertoys)
and link it here: #xxx

<!-- Provide a more detailed description of the PR, other things fixed,
or any additional comments/features here -->
## Detailed Description of the Pull Request / Additional comments

<!-- Describe how you validated the behavior. Add automated tests
wherever possible, but list manual validation steps taken as well -->
## Validation Steps Performed

src/modules/cmdpal/Microsoft.CmdPal.UI/Helpers/WindowExtensions.cs

@@ -5,10 +5,14 @@
 using Microsoft.UI;
 using Microsoft.UI.Windowing;
 using Microsoft.UI.Xaml;
+using Windows.Win32;
+using Windows.Win32.Foundation;
+using Windows.Win32.Graphics.Dwm;
+using Windows.Win32.UI.WindowsAndMessaging;
 
 namespace Microsoft.CmdPal.UI.Helpers;
 
-public static class WindowExtensions
+internal static class WindowExtensions
 {
     public static void SetIcon(this Window window)
     {
@@ -18,18 +22,59 @@ public static class WindowExtensions
         appWindow.SetIcon(@"Assets\icon.ico");
     }
 
-    public static void SetVisibilityInSwitchers(this Window window, bool showInSwitchers)
+    private static HWND GetWindowHwnd(this Window window)
     {
-        try
+        return window is null
+            ? throw new ArgumentNullException(nameof(window))
+            : new HWND(WinRT.Interop.WindowNative.GetWindowHandle(window));
+    }
+
+    /// <summary>
+    /// Toggles the specified extended window style on or off for the supplied <see cref="Window"/>.
+    /// </summary>
+    /// <param name="window">The <see cref="Window"/> whose extended window styles will be modified. Cannot be null.</param>
+    /// <param name="style">The <see cref="WINDOW_EX_STYLE"/> flag(s) to set or clear.</param>
+    /// <param name="isStyleSet">When true, the specified <paramref name="style"/> bit(s) will be set (added). When false, the bit(s) will be cleared (removed).</param>
+    /// <returns>True if the call to SetWindowLong succeeded and the style was applied; otherwise false.</returns>
+    /// <exception cref="ArgumentNullException">Thrown if <paramref name="window"/> is null.</exception>
+    internal static bool ToggleExtendedWindowStyle(this Window window, WINDOW_EX_STYLE style, bool isStyleSet)
+    {
+        var hWnd = GetWindowHwnd(window);
+        var currentStyle = PInvoke.GetWindowLong(hWnd, WINDOW_LONG_PTR_INDEX.GWL_EXSTYLE);
+
+        if (isStyleSet)
         {
-            // IsShownInSwitchers needs to change the value to apply the effect, but its state might be out-of-sync with
+            currentStyle |= (int)style;
-            // the actual state of the switchers, so we need to toggle it.
-            window.AppWindow.IsShownInSwitchers = !showInSwitchers;
-            window.AppWindow.IsShownInSwitchers = showInSwitchers;
         }
-        catch (NotImplementedException)
+        else
         {
-            // Setting IsShownInSwitchers failed. This can happen if the Explorer is not running.
+            currentStyle &= ~(int)style;
+        }
+
+        return PInvoke.SetWindowLong(hWnd, WINDOW_LONG_PTR_INDEX.GWL_EXSTYLE, currentStyle) != 0;
+    }
+
+    /// <summary>
+    /// Sets the window corner preference
+    /// </summary>
+    /// <param name="window">The window</param>
+    /// <param name="cornerPreference">The desired corner preference</param>
+    /// <returns>True if the operation succeeded</returns>
+    public static bool SetCornerPreference(this Window window, DWM_WINDOW_CORNER_PREFERENCE cornerPreference)
+    {
+        return window.GetWindowHwnd().SetDwmWindowAttribute(DWMWINDOWATTRIBUTE.DWMWA_WINDOW_CORNER_PREFERENCE, cornerPreference);
+    }
+
+    /// <summary>
+    /// Unified wrapper for DwmSetWindowAttribute calls with enum values
+    /// </summary>
+    private static bool SetDwmWindowAttribute<T>(this HWND hwnd, DWMWINDOWATTRIBUTE attribute, T value)
+        where T : unmanaged, Enum
+    {
+        unsafe
+        {
+            var result = PInvoke.DwmSetWindowAttribute(hwnd, attribute, &value, (uint)sizeof(T));
+            return result.Succeeded;
         }
     }
 }

src/modules/cmdpal/Microsoft.CmdPal.UI/MainWindow.xaml.cs

@@ -126,6 +126,16 @@ public sealed partial class MainWindow : WindowEx,
 
         // Force window to be created, and then cloaked. This will offset initial animation when the window is shown.
         HideWindow();
+
+        ApplyWindowStyle();
+    }
+
+    private void ApplyWindowStyle()
+    {
+        // Tool windows don't show up in ALT+TAB, and don't show up in the taskbar
+        // Since tool windows have smaller corner radii, we need to force the normal ones
+        this.ToggleExtendedWindowStyle(WINDOW_EX_STYLE.WS_EX_TOOLWINDOW, !Debugger.IsAttached);
+        this.SetCornerPreference(DWM_WINDOW_CORNER_PREFERENCE.DWMWCP_ROUND);
     }
 
     private static void LocalKeyboardListener_OnKeyPressed(object? sender, LocalKeyboardListenerKeyPressedEventArgs e)
@@ -173,8 +183,6 @@ public sealed partial class MainWindow : WindowEx,
         App.Current.Services.GetService<TrayIconService>()!.SetupTrayIcon(settings.ShowSystemTrayIcon);
 
         _ignoreHotKeyWhenFullScreen = settings.IgnoreShortcutWhenFullscreen;
-
-        this.SetVisibilityInSwitchers(Debugger.IsAttached);
     }
 
     // We want to use DesktopAcrylicKind.Thin and custom colors as this is the default material
@@ -252,6 +260,13 @@ public sealed partial class MainWindow : WindowEx,
         var display = GetScreen(hwnd, target);
         PositionCentered(display);
 
+        // Check if the debugger is attached. If it is, we don't want to apply the tool window style,
+        // because that would make it hard to debug the app
+        if (Debugger.IsAttached)
+        {
+            ApplyWindowStyle();
+        }
+
         // Just to be sure, SHOW our hwnd.
         PInvoke.ShowWindow(hwnd, SHOW_WINDOW_CMD.SW_SHOW);
 

src/modules/cmdpal/Microsoft.CmdPal.UI/NativeMethods.txt

@@ -44,6 +44,7 @@ MessageBox
 DwmGetWindowAttribute
 DwmSetWindowAttribute
 DWM_CLOAKED_APP
+DWM_WINDOW_CORNER_PREFERENCE
 
 CoWaitForMultipleObjects
 INFINITE
@@ -54,3 +55,7 @@ SetWindowsHookEx
 UnhookWindowsHookEx
 CallNextHookEx
 GetModuleHandle
+
+GetWindowLong
+SetWindowLong
+WINDOW_EX_STYLE

src/modules/cmdpal/Microsoft.CmdPal.UI/ToastWindow.xaml.cs

@@ -12,6 +12,7 @@ using Microsoft.UI.Dispatching;
 using Microsoft.UI.Windowing;
 using Windows.Win32;
 using Windows.Win32.Foundation;
+using Windows.Win32.Graphics.Dwm;
 using Windows.Win32.Graphics.Gdi;
 using Windows.Win32.UI.HiDpi;
 using Windows.Win32.UI.WindowsAndMessaging;
@@ -33,13 +34,18 @@ public sealed partial class ToastWindow : WindowEx,
     {
         this.InitializeComponent();
         AppWindow.Hide();
-        this.SetVisibilityInSwitchers(false);
         ExtendsContentIntoTitleBar = true;
         AppWindow.SetPresenter(AppWindowPresenterKind.CompactOverlay);
         this.SetIcon();
         AppWindow.Title = RS_.GetString("ToastWindowTitle");
         AppWindow.TitleBar.PreferredHeightOption = TitleBarHeightOption.Collapsed;
 
+        // Tool windows don't show up in ALT+TAB, and don't show up in the taskbar
+        // Since tool windows have smaller corner radii, we need to force the normal ones
+        // to visually match system toasts.
+        this.ToggleExtendedWindowStyle(WINDOW_EX_STYLE.WS_EX_TOOLWINDOW, true);
+        this.SetCornerPreference(DWM_WINDOW_CORNER_PREFERENCE.DWMWCP_ROUND);
+
         _hwnd = new HWND(WinRT.Interop.WindowNative.GetWindowHandle(this).ToInt32());
         PInvoke.EnableWindow(_hwnd, false);