Advanced Paste: AI pasting enhancement (#42374)

<!-- Enter a brief description/summary of your PR here. What does it
fix/what does it change/how was it tested (even manually, if necessary)?
-->
## Summary of the Pull Request
* Add multiple endpoint support for paste with AI
* Add Local AI support for paste AI
* Advanced AI implementation

<!-- Please review the items on the PR checklist before submitting-->
## PR Checklist

- [x] Closes: #32960
- [x] **Communication:** I've discussed this with core contributors
already. If the work hasn't been agreed, this work might be rejected
- [x] **Tests:** Added/updated and all pass
- [x] **Localization:** All end-user-facing strings can be localized
- [x] **Dev docs:** Added/updated
- [x] **New binaries:** Added on the required places
- [x] [JSON for
signing](https://github.com/microsoft/PowerToys/blob/main/.pipelines/ESRPSigning_core.json)
for new binaries
- [x] [WXS for
installer](https://github.com/microsoft/PowerToys/blob/main/installer/PowerToysSetup/Product.wxs)
for new binaries and localization folder
- [x] [YML for CI
pipeline](https://github.com/microsoft/PowerToys/blob/main/.pipelines/ci/templates/build-powertoys-steps.yml)
for new test projects
- [x] [YML for signed
pipeline](https://github.com/microsoft/PowerToys/blob/main/.pipelines/release.yml)
- [ ] **Documentation updated:** If checked, please file a pull request
on [our docs
repo](https://github.com/MicrosoftDocs/windows-uwp/tree/docs/hub/powertoys)
and link it here: #xxx

<!-- Provide a more detailed description of the PR, other things fixed,
or any additional comments/features here -->
## Detailed Description of the Pull Request / Additional comments

<!-- Describe how you validated the behavior. Add automated tests
wherever possible, but list manual validation steps taken as well -->
## Validation Steps Performed

### GPO
- [x] Paste with AI should not be available if the original GPO for
paste AI is set to false
   - [x] Paste with AI should be controlled within endpoint granularity
- [x] Advanced Paste UI should disable AI ability if GPO is set to
disable for any llm
### Paste AI
   - [x] Every AI endpoint should work as expected
   - [x] Default prompt should be able to give a reasonable result
   - [x] Local AI should work as expected
### Advanced AI
- [x] Open AI and Azure OPENAI should be able to configure as advanced
AI endpoint
- [x] Advanced AI should be able to pick up functions correctly to do
the transformation and give reasonable result

---------

Signed-off-by: Shawn Yuan <shuaiyuan@microsoft.com>
Signed-off-by: Shuai Yuan <shuai.yuan.zju@gmail.com>
Signed-off-by: Shawn Yuan (from Dev Box) <shuaiyuan@microsoft.com>
Co-authored-by: Leilei Zhang <leilzh@microsoft.com>
Co-authored-by: Niels Laute <niels.laute@live.nl>
Co-authored-by: Kai Tao <kaitao@microsoft.com>
Co-authored-by: Kai Tao <69313318+vanzue@users.noreply.github.com>
Co-authored-by: vanzue <vanzue@outlook.com>
Co-authored-by: Gordon Lam (SH) <yeelam@microsoft.com>
This commit is contained in:
Shawn Yuan
2025-11-05 16:13:55 +08:00
committed by GitHub
parent c364aa7c70
commit a3b8dc6cb8
119 changed files with 8441 additions and 958 deletions

View File

@@ -0,0 +1,192 @@
<?xml version="1.0" encoding="utf-8" ?>
<UserControl
x:Class="Microsoft.PowerToys.Settings.UI.Controls.FoundryLocalModelPicker"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:local="using:Microsoft.PowerToys.Settings.UI.Controls"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:models="using:LanguageModelProvider"
xmlns:tkconverters="using:CommunityToolkit.WinUI.Converters"
xmlns:toolkit="using:CommunityToolkit.WinUI.Controls"
x:Name="Root"
mc:Ignorable="d">
<UserControl.Resources>
<tkconverters:BoolToVisibilityConverter x:Key="BoolToVisibilityConverter" />
<Style x:Key="TagBorderStyle" TargetType="Border">
<Setter Property="Background" Value="{ThemeResource LayerFillColorDefaultBrush}" />
<Setter Property="BorderBrush" Value="{ThemeResource ControlStrongStrokeColorDefaultBrush}" />
<Setter Property="BorderThickness" Value="1" />
<Setter Property="CornerRadius" Value="8" />
<Setter Property="Padding" Value="8,2" />
<Setter Property="VerticalAlignment" Value="Center" />
</Style>
<Style x:Key="TagTextStyle" TargetType="TextBlock">
<Setter Property="FontSize" Value="12" />
<Setter Property="Foreground" Value="{ThemeResource TextFillColorPrimaryBrush}" />
<Setter Property="TextWrapping" Value="NoWrap" />
</Style>
</UserControl.Resources>
<Grid>
<StackPanel
x:Name="LoadingPanel"
HorizontalAlignment="Center"
VerticalAlignment="Center"
Spacing="12">
<ProgressRing
x:Name="LoadingIndicator"
Width="36"
Height="36"
HorizontalAlignment="Center" />
<TextBlock
x:Name="LoadingStatusTextBlock"
Foreground="{ThemeResource TextFillColorSecondaryBrush}"
Text="Loading Foundry Local status..."
TextAlignment="Center"
TextWrapping="Wrap" />
</StackPanel>
<ScrollViewer x:Name="ModelsView" Visibility="Collapsed">
<Grid Padding="0,12,0,16">
<Grid.RowDefinitions>
<RowDefinition Height="Auto" />
<RowDefinition Height="Auto" />
<RowDefinition Height="Auto" />
<RowDefinition Height="Auto" />
</Grid.RowDefinitions>
<StackPanel
x:Name="NoModelsPanel"
Grid.Row="0"
Margin="0,0,0,16"
HorizontalAlignment="Center"
Orientation="Vertical"
Spacing="4">
<FontIcon FontSize="24" Glyph="&#xE74E;" />
<TextBlock
HorizontalAlignment="Center"
Style="{StaticResource BodyStrongTextBlockStyle}"
Text="No models downloaded"
TextAlignment="Center" />
<TextBlock
HorizontalAlignment="Center"
FontSize="12"
Foreground="{ThemeResource TextFillColorSecondaryBrush}"
Text="Run Foundry Local to download or add a local model below."
TextAlignment="Center"
TextWrapping="Wrap" />
<Button
x:Name="LaunchFoundryModelListButton"
HorizontalAlignment="Center"
Click="LaunchFoundryModelListButton_Click"
Content="Open Foundry model list"
Style="{StaticResource AccentButtonStyle}" />
</StackPanel>
<StackPanel Grid.Row="1" Spacing="12">
<Grid ColumnSpacing="4">
<Grid.ColumnDefinitions>
<ColumnDefinition Width="*" />
<ColumnDefinition Width="Auto" />
</Grid.ColumnDefinitions>
<ComboBox
x:Name="CachedModelsComboBox"
Grid.Column="0"
HorizontalAlignment="Stretch"
DisplayMemberPath="Name"
ItemsSource="{x:Bind CachedModels, Mode=OneWay}"
SelectedItem="{x:Bind SelectedModel, Mode=TwoWay}"
SelectionChanged="CachedModelsComboBox_SelectionChanged">
<ComboBox.Header>
<TextBlock>
<Run Text="Foundry Local model" /><LineBreak /><Run
FontSize="12"
Foreground="{ThemeResource TextFillColorSecondaryBrush}"
Text="Use the Foundry Local CLI to download models that run locally on-device. They'll appear here." />
</TextBlock>
</ComboBox.Header>
</ComboBox>
<Button
x:Name="RefreshModelsButton"
Grid.Column="1"
MinHeight="32"
VerticalAlignment="Bottom"
Click="RefreshModelsButton_Click"
Style="{StaticResource SubtleButtonStyle}"
ToolTipService.ToolTip="Refresh model list">
<FontIcon FontSize="16" Glyph="&#xE72C;" />
</Button>
</Grid>
<StackPanel
x:Name="SelectedModelDetailsPanel"
Spacing="8"
Visibility="Collapsed">
<TextBlock
x:Name="SelectedModelDescriptionText"
Foreground="{ThemeResource TextFillColorSecondaryBrush}"
TextWrapping="Wrap" />
<toolkit:WrapPanel
x:Name="SelectedModelTagsPanel"
HorizontalSpacing="4"
VerticalSpacing="4"
Visibility="Collapsed" />
</StackPanel>
</StackPanel>
</Grid>
</ScrollViewer>
<Grid x:Name="NotAvailableGrid" Visibility="Collapsed">
<StackPanel
Margin="48,0,48,48"
HorizontalAlignment="Center"
VerticalAlignment="Center"
Orientation="Vertical"
Spacing="8">
<Image Width="36" Source="ms-appx:///Assets/Settings/Icons/Models/FoundryLocal.svg" />
<TextBlock
FontWeight="SemiBold"
Text="Foundry Local is not available on this device yet."
TextAlignment="Center"
TextWrapping="Wrap" />
<TextBlock
FontSize="12"
Foreground="{ThemeResource TextFillColorSecondaryBrush}"
IsTextSelectionEnabled="True"
TextAlignment="Center"
TextWrapping="Wrap">
<Run Text="Start the Foundry Local service before returning to PowerToys." />
</TextBlock>
<HyperlinkButton Content="Follow the Foundry Local CLI guide" NavigateUri="https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-local/get-started" />
<TextBlock
x:Uid="FoundryLocal_RestartRequiredNote"
FontSize="12"
Foreground="{ThemeResource TextFillColorSecondaryBrush}"
Text="Note: After installing the Foundry Local CLI, restart PowerToys to use it."
TextAlignment="Center"
TextWrapping="Wrap" />
</StackPanel>
</Grid>
<VisualStateManager.VisualStateGroups>
<VisualStateGroup x:Name="StateGroup">
<VisualState x:Name="ShowLoading" />
<VisualState x:Name="ShowModels">
<VisualState.Setters>
<Setter Target="LoadingPanel.Visibility" Value="Collapsed" />
<Setter Target="NotAvailableGrid.Visibility" Value="Collapsed" />
<Setter Target="ModelsView.Visibility" Value="Visible" />
</VisualState.Setters>
</VisualState>
<VisualState x:Name="ShowNotAvailable">
<VisualState.Setters>
<Setter Target="LoadingPanel.Visibility" Value="Collapsed" />
<Setter Target="NotAvailableGrid.Visibility" Value="Visible" />
<Setter Target="ModelsView.Visibility" Value="Collapsed" />
</VisualState.Setters>
</VisualState>
</VisualStateGroup>
</VisualStateManager.VisualStateGroups>
</Grid>
</UserControl>

View File

@@ -0,0 +1,457 @@
// Copyright (c) Microsoft Corporation
// The Microsoft Corporation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Diagnostics;
using System.Linq;
using LanguageModelProvider;
using Microsoft.UI.Xaml;
using Microsoft.UI.Xaml.Controls;
namespace Microsoft.PowerToys.Settings.UI.Controls;
public sealed partial class FoundryLocalModelPicker : UserControl
{
private INotifyCollectionChanged _cachedModelsSubscription;
private INotifyCollectionChanged _downloadableModelsSubscription;
private bool _suppressSelection;
public FoundryLocalModelPicker()
{
InitializeComponent();
Loaded += (_, _) => UpdateVisualStates();
}
public delegate void ModelSelectionChangedEventHandler(object sender, ModelDetails model);
public delegate void DownloadRequestedEventHandler(object sender, object payload);
public delegate void LoadRequestedEventHandler(object sender, FoundryLoadRequestedEventArgs args);
public event ModelSelectionChangedEventHandler SelectionChanged;
public event LoadRequestedEventHandler LoadRequested;
public IEnumerable<ModelDetails> CachedModels
{
get => (IEnumerable<ModelDetails>)GetValue(CachedModelsProperty);
set => SetValue(CachedModelsProperty, value);
}
public static readonly DependencyProperty CachedModelsProperty =
DependencyProperty.Register(nameof(CachedModels), typeof(IEnumerable<ModelDetails>), typeof(FoundryLocalModelPicker), new PropertyMetadata(null, OnCachedModelsChanged));
public IEnumerable DownloadableModels
{
get => (IEnumerable)GetValue(DownloadableModelsProperty);
set => SetValue(DownloadableModelsProperty, value);
}
public static readonly DependencyProperty DownloadableModelsProperty =
DependencyProperty.Register(nameof(DownloadableModels), typeof(IEnumerable), typeof(FoundryLocalModelPicker), new PropertyMetadata(null, OnDownloadableModelsChanged));
public ModelDetails SelectedModel
{
get => (ModelDetails)GetValue(SelectedModelProperty);
set => SetValue(SelectedModelProperty, value);
}
public static readonly DependencyProperty SelectedModelProperty =
DependencyProperty.Register(nameof(SelectedModel), typeof(ModelDetails), typeof(FoundryLocalModelPicker), new PropertyMetadata(null, OnSelectedModelChanged));
public bool IsLoading
{
get => (bool)GetValue(IsLoadingProperty);
set => SetValue(IsLoadingProperty, value);
}
public static readonly DependencyProperty IsLoadingProperty =
DependencyProperty.Register(nameof(IsLoading), typeof(bool), typeof(FoundryLocalModelPicker), new PropertyMetadata(false, OnStatePropertyChanged));
public bool IsAvailable
{
get => (bool)GetValue(IsAvailableProperty);
set => SetValue(IsAvailableProperty, value);
}
public static readonly DependencyProperty IsAvailableProperty =
DependencyProperty.Register(nameof(IsAvailable), typeof(bool), typeof(FoundryLocalModelPicker), new PropertyMetadata(false, OnStatePropertyChanged));
public string StatusText
{
get => (string)GetValue(StatusTextProperty);
set => SetValue(StatusTextProperty, value);
}
public static readonly DependencyProperty StatusTextProperty =
DependencyProperty.Register(nameof(StatusText), typeof(string), typeof(FoundryLocalModelPicker), new PropertyMetadata(string.Empty, OnStatePropertyChanged));
public bool HasCachedModels => CachedModels?.Any() ?? false;
public bool HasDownloadableModels => DownloadableModels?.Cast<object>().Any() ?? false;
public void RequestLoad(bool refresh)
{
if (IsLoading)
{
// Allow refresh requests to continue even if already loading by cancelling via host.
}
else
{
IsLoading = true;
}
IsAvailable = false;
StatusText = "Loading Foundry Local status...";
LoadRequested?.Invoke(this, new FoundryLoadRequestedEventArgs(refresh));
}
private static void OnCachedModelsChanged(DependencyObject d, DependencyPropertyChangedEventArgs e)
{
var control = (FoundryLocalModelPicker)d;
control.SubscribeToCachedModels(e.OldValue as IEnumerable<ModelDetails>, e.NewValue as IEnumerable<ModelDetails>);
control.UpdateVisualStates();
}
private static void OnDownloadableModelsChanged(DependencyObject d, DependencyPropertyChangedEventArgs e)
{
var control = (FoundryLocalModelPicker)d;
control.SubscribeToDownloadableModels(e.OldValue as IEnumerable, e.NewValue as IEnumerable);
control.UpdateVisualStates();
}
private static void OnSelectedModelChanged(DependencyObject d, DependencyPropertyChangedEventArgs e)
{
var control = (FoundryLocalModelPicker)d;
if (control._suppressSelection)
{
return;
}
try
{
control._suppressSelection = true;
if (control.CachedModelsComboBox is not null)
{
control.CachedModelsComboBox.SelectedItem = e.NewValue;
}
}
finally
{
control._suppressSelection = false;
}
control.UpdateSelectedModelDetails();
}
private static void OnStatePropertyChanged(DependencyObject d, DependencyPropertyChangedEventArgs e)
{
var control = (FoundryLocalModelPicker)d;
control.UpdateVisualStates();
}
private void SubscribeToCachedModels(IEnumerable<ModelDetails> oldValue, IEnumerable<ModelDetails> newValue)
{
if (_cachedModelsSubscription is not null)
{
_cachedModelsSubscription.CollectionChanged -= CachedModels_CollectionChanged;
_cachedModelsSubscription = null;
}
if (newValue is INotifyCollectionChanged observable)
{
observable.CollectionChanged += CachedModels_CollectionChanged;
_cachedModelsSubscription = observable;
}
}
private void SubscribeToDownloadableModels(IEnumerable oldValue, IEnumerable newValue)
{
if (_downloadableModelsSubscription is not null)
{
_downloadableModelsSubscription.CollectionChanged -= DownloadableModels_CollectionChanged;
_downloadableModelsSubscription = null;
}
if (newValue is INotifyCollectionChanged observable)
{
observable.CollectionChanged += DownloadableModels_CollectionChanged;
_downloadableModelsSubscription = observable;
}
}
private void CachedModels_CollectionChanged(object sender, NotifyCollectionChangedEventArgs e)
{
UpdateVisualStates();
}
private void DownloadableModels_CollectionChanged(object sender, NotifyCollectionChangedEventArgs e)
{
UpdateVisualStates();
}
private void CachedModelsComboBox_SelectionChanged(object sender, SelectionChangedEventArgs e)
{
if (_suppressSelection)
{
return;
}
try
{
_suppressSelection = true;
var selected = CachedModelsComboBox.SelectedItem as ModelDetails;
SetValue(SelectedModelProperty, selected);
SelectionChanged?.Invoke(this, selected);
}
finally
{
_suppressSelection = false;
}
UpdateSelectedModelDetails();
}
private void UpdateSelectedModelDetails()
{
if (SelectedModelDetailsPanel is null || SelectedModelDescriptionText is null || SelectedModelTagsPanel is null)
{
return;
}
if (!HasCachedModels || SelectedModel is not ModelDetails model)
{
SelectedModelDetailsPanel.Visibility = Visibility.Collapsed;
SelectedModelDescriptionText.Text = string.Empty;
SelectedModelTagsPanel.Children.Clear();
SelectedModelTagsPanel.Visibility = Visibility.Collapsed;
return;
}
SelectedModelDetailsPanel.Visibility = Visibility.Visible;
SelectedModelDescriptionText.Text = string.IsNullOrWhiteSpace(model.Description)
? "No description provided."
: model.Description;
SelectedModelTagsPanel.Children.Clear();
AddTag(GetModelSizeText(model.Size));
AddTag(GetLicenseShortText(model.License), model.License);
foreach (var deviceTag in GetDeviceTags(model.HardwareAccelerators))
{
AddTag(deviceTag);
}
SelectedModelTagsPanel.Visibility = SelectedModelTagsPanel.Children.Count > 0 ? Visibility.Visible : Visibility.Collapsed;
void AddTag(string text, string tooltip = null)
{
if (string.IsNullOrWhiteSpace(text) || SelectedModelTagsPanel is null)
{
return;
}
Border tag = new();
if (Resources.TryGetValue("TagBorderStyle", out var borderStyleObj) && borderStyleObj is Style borderStyle)
{
tag.Style = borderStyle;
}
TextBlock label = new()
{
Text = text,
};
if (Resources.TryGetValue("TagTextStyle", out var textStyleObj) && textStyleObj is Style textStyle)
{
label.Style = textStyle;
}
tag.Child = label;
if (!string.IsNullOrWhiteSpace(tooltip))
{
ToolTipService.SetToolTip(tag, new TextBlock
{
Text = tooltip,
TextWrapping = TextWrapping.Wrap,
});
}
SelectedModelTagsPanel.Children.Add(tag);
}
}
private void LaunchFoundryModelListButton_Click(object sender, RoutedEventArgs e)
{
try
{
ProcessStartInfo processInfo = new()
{
FileName = "powershell.exe",
Arguments = "-NoExit -Command \"foundry model list\"",
UseShellExecute = true,
};
Process.Start(processInfo);
StatusText = "Opening PowerShell and running 'foundry model list'...";
}
catch (Exception ex)
{
StatusText = $"Unable to start PowerShell. {ex.Message}";
Debug.WriteLine($"[FoundryLocalModelPicker] Failed to run 'foundry model list': {ex}");
}
}
private void RefreshModelsButton_Click(object sender, RoutedEventArgs e)
{
RequestLoad(refresh: true);
}
private void UpdateVisualStates()
{
LoadingIndicator.IsActive = IsLoading;
if (IsLoading)
{
VisualStateManager.GoToState(this, "ShowLoading", true);
}
else if (!IsAvailable)
{
VisualStateManager.GoToState(this, "ShowNotAvailable", true);
}
else
{
VisualStateManager.GoToState(this, "ShowModels", true);
}
if (LoadingStatusTextBlock is not null)
{
LoadingStatusTextBlock.Text = string.IsNullOrWhiteSpace(StatusText)
? "Loading Foundry Local status..."
: StatusText;
}
NoModelsPanel.Visibility = HasCachedModels ? Visibility.Collapsed : Visibility.Visible;
if (CachedModelsComboBox is not null)
{
CachedModelsComboBox.Visibility = HasCachedModels ? Visibility.Visible : Visibility.Collapsed;
CachedModelsComboBox.IsEnabled = HasCachedModels;
}
UpdateSelectedModelDetails();
Bindings.Update();
}
public static string GetModelSizeText(long size)
{
if (size <= 0)
{
return string.Empty;
}
const long kiloByte = 1024;
const long megaByte = kiloByte * 1024;
const long gigaByte = megaByte * 1024;
if (size >= gigaByte)
{
return $"{size / (double)gigaByte:0.##} GB";
}
if (size >= megaByte)
{
return $"{size / (double)megaByte:0.##} MB";
}
if (size >= kiloByte)
{
return $"{size / (double)kiloByte:0.##} KB";
}
return $"{size} B";
}
public static Visibility GetModelSizeVisibility(long size)
{
return size > 0 ? Visibility.Visible : Visibility.Collapsed;
}
public static IEnumerable<string> GetDeviceTags(IReadOnlyCollection<HardwareAccelerator> accelerators)
{
if (accelerators is null || accelerators.Count == 0)
{
return Array.Empty<string>();
}
HashSet<string> tags = new(StringComparer.OrdinalIgnoreCase);
foreach (var accelerator in accelerators)
{
switch (accelerator)
{
case HardwareAccelerator.CPU:
tags.Add("CPU");
break;
case HardwareAccelerator.GPU:
case HardwareAccelerator.DML:
tags.Add("GPU");
break;
case HardwareAccelerator.NPU:
case HardwareAccelerator.QNN:
tags.Add("NPU");
break;
}
}
return tags.Count > 0 ? tags.ToArray() : Array.Empty<string>();
}
public static Visibility GetDeviceVisibility(IReadOnlyCollection<HardwareAccelerator> accelerators)
{
return GetDeviceTags(accelerators).Any() ? Visibility.Visible : Visibility.Collapsed;
}
public static string GetLicenseShortText(string license)
{
if (string.IsNullOrWhiteSpace(license))
{
return string.Empty;
}
var trimmed = license.Trim();
int separatorIndex = trimmed.IndexOfAny(['(', '[', ':']);
if (separatorIndex > 0)
{
trimmed = trimmed[..separatorIndex].Trim();
}
if (trimmed.Length > 24)
{
trimmed = $"{trimmed[..24].TrimEnd()}…";
}
return trimmed;
}
public static Visibility GetLicenseVisibility(string license)
{
return string.IsNullOrWhiteSpace(license) ? Visibility.Collapsed : Visibility.Visible;
}
public sealed class FoundryLoadRequestedEventArgs : EventArgs
{
public FoundryLoadRequestedEventArgs(bool refresh)
{
Refresh = refresh;
}
public bool Refresh { get; }
}
}