mirror of
https://github.com/microsoft/PowerToys.git
synced 2025-12-15 03:07:56 +01:00
Advanced paste: Tweak Foundry Local Displayed Model and start server if server is turned on when using AP (#43529)
<!-- Enter a brief description/summary of your PR here. What does it fix/what does it change/how was it tested (even manually, if necessary)? --> ## Summary of the Pull Request 1. Foundry local model name should not prefixed by fl:// 2. If foundry service is shutdown, we should not just fail it, we should start it then call FL to make availability better. <!-- Please review the items on the PR checklist before submitting--> ## PR Checklist - [ ] Closes: #xxx <!-- - [ ] Closes: #yyy (add separate lines for additional resolved issues) --> - [ ] **Communication:** I've discussed this with core contributors already. If the work hasn't been agreed, this work might be rejected - [ ] **Tests:** Added/updated and all pass - [ ] **Localization:** All end-user-facing strings can be localized - [ ] **Dev docs:** Added/updated - [ ] **New binaries:** Added on the required places - [ ] [JSON for signing](https://github.com/microsoft/PowerToys/blob/main/.pipelines/ESRPSigning_core.json) for new binaries - [ ] [WXS for installer](https://github.com/microsoft/PowerToys/blob/main/installer/PowerToysSetup/Product.wxs) for new binaries and localization folder - [ ] [YML for CI pipeline](https://github.com/microsoft/PowerToys/blob/main/.pipelines/ci/templates/build-powertoys-steps.yml) for new test projects - [ ] [YML for signed pipeline](https://github.com/microsoft/PowerToys/blob/main/.pipelines/release.yml) - [ ] **Documentation updated:** If checked, please file a pull request on [our docs repo](https://github.com/MicrosoftDocs/windows-uwp/tree/docs/hub/powertoys) and link it here: #xxx <!-- Provide a more detailed description of the PR, other things fixed, or any additional comments/features here --> ## Detailed Description of the Pull Request / Additional comments <!-- Describe how you validated the behavior. Add automated tests wherever possible, but list manual validation steps taken as well --> ## Validation Steps Performed Verified locally: 1. Manually disable foundry local service, then run AP with foundry local, it can return result instead of direct failure. 2. <img width="659" height="294" alt="image" src="https://github.com/user-attachments/assets/113da451-7131-4ce7-ae82-0ccf772ad8aa" /> <img width="988" height="192" alt="image" src="https://github.com/user-attachments/assets/aa3650ba-668a-40c4-ad8a-303e09000dd4" /> ![Uploading image.png…]()
This commit is contained in:
Kai Tao
@@ -205,4 +205,12 @@ internal sealed class FoundryClient
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task EnsureRunning()
|
||||
{
|
||||
if (!_foundryManager.IsServiceRunning)
|
||||
{
|
||||
await _foundryManager.StartServiceAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ namespace LanguageModelProvider;
|
||||
public sealed class FoundryLocalModelProvider : ILanguageModelProvider
|
||||
{
|
||||
private IEnumerable<ModelDetails>? _downloadedModels;
|
||||
private FoundryClient? _foundryManager;
|
||||
private FoundryClient? _foundryClient;
|
||||
private string? _serviceUrl;
|
||||
|
||||
public static FoundryLocalModelProvider Instance { get; } = new();
|
||||
@@ -22,13 +22,11 @@ public sealed class FoundryLocalModelProvider : ILanguageModelProvider
|
||||
|
||||
public string ProviderDescription => "The model will run locally via Foundry Local";
|
||||
|
||||
public string UrlPrefix => "fl://";
|
||||
|
||||
public IChatClient? GetIChatClient(string url)
|
||||
public IChatClient? GetIChatClient(string modelId)
|
||||
{
|
||||
try
|
||||
{
|
||||
Logger.LogInfo($"[FoundryLocal] GetIChatClient called with url: {url}");
|
||||
Logger.LogInfo($"[FoundryLocal] GetIChatClient called with url: {modelId}");
|
||||
InitializeAsync().GetAwaiter().GetResult();
|
||||
}
|
||||
catch (Exception ex)
|
||||
@@ -37,26 +35,22 @@ public sealed class FoundryLocalModelProvider : ILanguageModelProvider
|
||||
return null;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(_serviceUrl) || _foundryManager == null)
|
||||
if (string.IsNullOrWhiteSpace(_serviceUrl) || _foundryClient == null)
|
||||
{
|
||||
Logger.LogError("[FoundryLocal] Service URL or manager is null");
|
||||
return null;
|
||||
}
|
||||
|
||||
// Extract model ID from URL (format: fl://modelname)
|
||||
var modelId = url.Replace(UrlPrefix, string.Empty).Trim('/');
|
||||
if (string.IsNullOrWhiteSpace(modelId))
|
||||
{
|
||||
Logger.LogError("[FoundryLocal] Model ID is empty after extraction");
|
||||
return null;
|
||||
}
|
||||
|
||||
Logger.LogInfo($"[FoundryLocal] Extracted model ID: {modelId}");
|
||||
|
||||
// Ensure the model is loaded before returning chat client
|
||||
try
|
||||
{
|
||||
var isLoaded = _foundryManager.EnsureModelLoaded(modelId).GetAwaiter().GetResult();
|
||||
var isLoaded = _foundryClient.EnsureModelLoaded(modelId).GetAwaiter().GetResult();
|
||||
if (!isLoaded)
|
||||
{
|
||||
Logger.LogError($"[FoundryLocal] Failed to load model: {modelId}");
|
||||
@@ -72,7 +66,7 @@ public sealed class FoundryLocalModelProvider : ILanguageModelProvider
|
||||
}
|
||||
|
||||
// Use ServiceUri instead of Endpoint since Endpoint already includes /v1
|
||||
var baseUri = _foundryManager.GetServiceUri();
|
||||
var baseUri = _foundryClient.GetServiceUri();
|
||||
if (baseUri == null)
|
||||
{
|
||||
Logger.LogError("[FoundryLocal] Service URI is null");
|
||||
@@ -133,24 +127,25 @@ public sealed class FoundryLocalModelProvider : ILanguageModelProvider
|
||||
|
||||
private async Task InitializeAsync(CancellationToken cancelationToken = default)
|
||||
{
|
||||
if (_foundryManager != null && _downloadedModels != null && _downloadedModels.Any())
|
||||
if (_foundryClient != null && _downloadedModels != null && _downloadedModels.Any())
|
||||
{
|
||||
await _foundryClient.EnsureRunning().ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
Logger.LogInfo("[FoundryLocal] Initializing provider");
|
||||
_foundryManager ??= await FoundryClient.CreateAsync();
|
||||
_foundryClient ??= await FoundryClient.CreateAsync();
|
||||
|
||||
if (_foundryManager == null)
|
||||
if (_foundryClient == null)
|
||||
{
|
||||
Logger.LogError("[FoundryLocal] Failed to create Foundry client");
|
||||
return;
|
||||
}
|
||||
|
||||
_serviceUrl ??= await _foundryManager.GetServiceUrl();
|
||||
_serviceUrl ??= await _foundryClient.GetServiceUrl();
|
||||
Logger.LogInfo($"[FoundryLocal] Service URL: {_serviceUrl}");
|
||||
|
||||
var cachedModels = await _foundryManager.ListCachedModels();
|
||||
var cachedModels = await _foundryClient.ListCachedModels();
|
||||
Logger.LogInfo($"[FoundryLocal] Found {cachedModels.Count} cached models");
|
||||
|
||||
List<ModelDetails> downloadedModels = [];
|
||||
@@ -162,7 +157,7 @@ public sealed class FoundryLocalModelProvider : ILanguageModelProvider
|
||||
{
|
||||
Id = $"fl-{model.Name}",
|
||||
Name = model.Name,
|
||||
Url = $"{UrlPrefix}{model.Name}",
|
||||
Url = $"fl://{model.Name}",
|
||||
Description = $"{model.Name} running locally with Foundry Local",
|
||||
HardwareAccelerators = [HardwareAccelerator.FOUNDRYLOCAL],
|
||||
SupportedOnQualcomm = true,
|
||||
@@ -178,7 +173,7 @@ public sealed class FoundryLocalModelProvider : ILanguageModelProvider
|
||||
{
|
||||
Logger.LogInfo("[FoundryLocal] Checking availability");
|
||||
await InitializeAsync();
|
||||
var available = _foundryManager != null;
|
||||
var available = _foundryClient != null;
|
||||
Logger.LogInfo($"[FoundryLocal] Available: {available}");
|
||||
return available;
|
||||
}
|
||||
|
||||
@@ -10,13 +10,11 @@ public interface ILanguageModelProvider
|
||||
{
|
||||
string Name { get; }
|
||||
|
||||
string UrlPrefix { get; }
|
||||
|
||||
string ProviderDescription { get; }
|
||||
|
||||
Task<IEnumerable<ModelDetails>> GetModelsAsync(bool ignoreCached = false, CancellationToken cancelationToken = default);
|
||||
|
||||
IChatClient? GetIChatClient(string url);
|
||||
IChatClient? GetIChatClient(string modelId);
|
||||
|
||||
string GetIChatClientString(string url);
|
||||
}
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
// Copyright (c) Microsoft Corporation
|
||||
// The Microsoft Corporation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.AI;
|
||||
|
||||
namespace LanguageModelProvider;
|
||||
|
||||
public sealed class LanguageModelService
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, ILanguageModelProvider> _providersByPrefix;
|
||||
|
||||
public LanguageModelService(IEnumerable<ILanguageModelProvider> providers)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(providers);
|
||||
|
||||
_providersByPrefix = new ConcurrentDictionary<string, ILanguageModelProvider>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var provider in providers)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(provider.UrlPrefix))
|
||||
{
|
||||
_providersByPrefix[provider.UrlPrefix] = provider;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static LanguageModelService CreateDefault()
|
||||
{
|
||||
return new LanguageModelService(new[]
|
||||
{
|
||||
FoundryLocalModelProvider.Instance,
|
||||
});
|
||||
}
|
||||
|
||||
public IReadOnlyCollection<ILanguageModelProvider> Providers => _providersByPrefix.Values.ToArray();
|
||||
|
||||
public bool RegisterProvider(ILanguageModelProvider provider)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(provider);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(provider.UrlPrefix))
|
||||
{
|
||||
throw new ArgumentException("Provider must supply a URL prefix.", nameof(provider));
|
||||
}
|
||||
|
||||
_providersByPrefix[provider.UrlPrefix] = provider;
|
||||
return true;
|
||||
}
|
||||
|
||||
public ILanguageModelProvider? GetProviderFor(string? modelReference)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(modelReference))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
foreach (var provider in _providersByPrefix.Values)
|
||||
{
|
||||
if (modelReference.StartsWith(provider.UrlPrefix, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return provider;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<ModelDetails>> GetModelsAsync(bool refresh = false, CancellationToken cancellationToken = default)
|
||||
{
|
||||
List<ModelDetails> models = [];
|
||||
|
||||
foreach (var provider in _providersByPrefix.Values)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var providerModels = await provider.GetModelsAsync(refresh, cancellationToken).ConfigureAwait(false);
|
||||
models.AddRange(providerModels);
|
||||
}
|
||||
|
||||
return models;
|
||||
}
|
||||
|
||||
public IChatClient? GetClient(ModelDetails model)
|
||||
{
|
||||
if (model is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var reference = !string.IsNullOrWhiteSpace(model.Url) ? model.Url : model.Id;
|
||||
return GetClient(reference);
|
||||
}
|
||||
|
||||
public IChatClient? GetClient(string? modelReference)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(modelReference))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var provider = GetProviderFor(modelReference);
|
||||
|
||||
return provider?.GetIChatClient(modelReference);
|
||||
}
|
||||
}
|
||||
@@ -23,7 +23,7 @@ public sealed class FoundryLocalPasteProvider : IPasteAIProvider
|
||||
|
||||
public static PasteAIProviderRegistration Registration { get; } = new(SupportedTypes, config => new FoundryLocalPasteProvider(config));
|
||||
|
||||
private static readonly LanguageModelService LanguageModels = LanguageModelService.CreateDefault();
|
||||
private static readonly FoundryLocalModelProvider _modelProvider = FoundryLocalModelProvider.Instance;
|
||||
|
||||
private readonly PasteAIConfig _config;
|
||||
|
||||
@@ -72,11 +72,11 @@ public sealed class FoundryLocalPasteProvider : IPasteAIProvider
|
||||
throw new PasteActionException(
|
||||
"No Foundry Local model selected",
|
||||
new InvalidOperationException("Model identifier is required"),
|
||||
aiServiceMessage: "Please select a model in the AI provider settings. Model identifier should be in the format 'fl://model-name'.");
|
||||
aiServiceMessage: "Please select a model in the AI provider settings.");
|
||||
}
|
||||
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var chatClient = LanguageModels.GetClient(modelReference);
|
||||
var chatClient = _modelProvider.GetIChatClient(modelReference);
|
||||
if (chatClient is null)
|
||||
{
|
||||
throw new PasteActionException(
|
||||
@@ -85,9 +85,6 @@ public sealed class FoundryLocalPasteProvider : IPasteAIProvider
|
||||
aiServiceMessage: "The model may not be downloaded or the Foundry Local service may not be running. Please check the model status in settings.");
|
||||
}
|
||||
|
||||
// Extract actual model ID from the URL (format: fl://modelId)
|
||||
var actualModelId = modelReference.Replace("fl://", string.Empty).Trim('/');
|
||||
|
||||
var userMessageContent = $"""
|
||||
User instructions:
|
||||
{prompt}
|
||||
@@ -104,7 +101,7 @@ public sealed class FoundryLocalPasteProvider : IPasteAIProvider
|
||||
new(ChatRole.User, userMessageContent),
|
||||
};
|
||||
|
||||
var chatOptions = CreateChatOptions(_config?.SystemPrompt, actualModelId);
|
||||
var chatOptions = CreateChatOptions(_config?.SystemPrompt, modelReference);
|
||||
|
||||
progress?.Report(0.1);
|
||||
|
||||
|
||||
@@ -27,7 +27,6 @@ namespace Microsoft.PowerToys.Settings.UI.Views
|
||||
public sealed partial class AdvancedPastePage : NavigablePage, IRefreshablePage, IDisposable
|
||||
{
|
||||
private readonly ObservableCollection<ModelDetails> _foundryCachedModels = new();
|
||||
private readonly ObservableCollection<FoundryDownloadableModel> _foundryDownloadableModels = new();
|
||||
private CancellationTokenSource _foundryModelLoadCts;
|
||||
private bool _suppressFoundrySelectionChanged;
|
||||
private bool _isFoundryLocalAvailable;
|
||||
@@ -57,7 +56,6 @@ namespace Microsoft.PowerToys.Settings.UI.Views
|
||||
if (FoundryLocalPicker is not null)
|
||||
{
|
||||
FoundryLocalPicker.CachedModels = _foundryCachedModels;
|
||||
FoundryLocalPicker.DownloadableModels = _foundryDownloadableModels;
|
||||
FoundryLocalPicker.SelectionChanged += FoundryLocalPicker_SelectionChanged;
|
||||
FoundryLocalPicker.LoadRequested += FoundryLocalPicker_LoadRequested;
|
||||
}
|
||||
@@ -469,7 +467,7 @@ namespace Microsoft.PowerToys.Settings.UI.Views
|
||||
|
||||
var cachedModels = cachedModelsEnumerable?.ToList() ?? new List<ModelDetails>();
|
||||
|
||||
UpdateFoundryCollections(cachedModels, []);
|
||||
UpdateFoundryCollections(cachedModels);
|
||||
ShowFoundryAvailableState();
|
||||
RestoreFoundrySelection(cachedModels);
|
||||
}
|
||||
@@ -538,7 +536,7 @@ namespace Microsoft.PowerToys.Settings.UI.Views
|
||||
UpdateFoundrySaveButtonState();
|
||||
}
|
||||
|
||||
private void UpdateFoundryCollections(IReadOnlyCollection<ModelDetails> cachedModels, IReadOnlyCollection<ModelDetails> catalogModels)
|
||||
private void UpdateFoundryCollections(IReadOnlyCollection<ModelDetails> cachedModels)
|
||||
{
|
||||
_foundryCachedModels.Clear();
|
||||
|
||||
@@ -547,20 +545,7 @@ namespace Microsoft.PowerToys.Settings.UI.Views
|
||||
_foundryCachedModels.Add(model);
|
||||
}
|
||||
|
||||
var cachedReferences = new HashSet<string>(_foundryCachedModels.Select(m => NormalizeFoundryModelReference(m.Url ?? m.Name)), StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
_foundryDownloadableModels.Clear();
|
||||
|
||||
foreach (var model in catalogModels.OrderBy(m => m.Name, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
var reference = NormalizeFoundryModelReference(model.Url ?? model.Name);
|
||||
if (cachedReferences.Contains(reference))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
_foundryDownloadableModels.Add(new FoundryDownloadableModel(model));
|
||||
}
|
||||
var cachedReferences = new HashSet<string>(_foundryCachedModels.Select(m => m.Name), StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private void RestoreFoundrySelection(IReadOnlyCollection<ModelDetails> cachedModels)
|
||||
@@ -576,9 +561,8 @@ namespace Microsoft.PowerToys.Settings.UI.Views
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(currentModelReference))
|
||||
{
|
||||
var normalizedReference = NormalizeFoundryModelReference(currentModelReference);
|
||||
matchingModel = cachedModels.FirstOrDefault(model =>
|
||||
string.Equals(NormalizeFoundryModelReference(model.Url ?? model.Name), normalizedReference, StringComparison.OrdinalIgnoreCase));
|
||||
string.Equals(model.Name, currentModelReference, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
if (FoundryLocalPicker is null)
|
||||
@@ -608,7 +592,7 @@ namespace Microsoft.PowerToys.Settings.UI.Views
|
||||
{
|
||||
if (ViewModel?.PasteAIProviderDraft is not null)
|
||||
{
|
||||
ViewModel.PasteAIProviderDraft.ModelName = NormalizeFoundryModelReference(matchingModel.Url ?? matchingModel.Name);
|
||||
ViewModel.PasteAIProviderDraft.ModelName = matchingModel.Name;
|
||||
}
|
||||
|
||||
if (FoundryLocalPicker is not null)
|
||||
@@ -620,19 +604,6 @@ namespace Microsoft.PowerToys.Settings.UI.Views
|
||||
UpdateFoundrySaveButtonState();
|
||||
}
|
||||
|
||||
private static string NormalizeFoundryModelReference(string modelReference)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(modelReference))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var prefix = FoundryLocalModelProvider.Instance.UrlPrefix;
|
||||
return modelReference.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)
|
||||
? modelReference
|
||||
: $"{prefix}{modelReference}";
|
||||
}
|
||||
|
||||
private void UpdateFoundrySaveButtonState()
|
||||
{
|
||||
if (PasteAIProviderConfigurationDialog is null)
|
||||
@@ -656,7 +627,7 @@ namespace Microsoft.PowerToys.Settings.UI.Views
|
||||
return;
|
||||
}
|
||||
|
||||
if (!_isFoundryLocalAvailable || _foundryDownloadableModels.Any(model => model.IsDownloading))
|
||||
if (!_isFoundryLocalAvailable)
|
||||
{
|
||||
PasteAIProviderConfigurationDialog.IsPrimaryButtonEnabled = false;
|
||||
return;
|
||||
@@ -677,7 +648,7 @@ namespace Microsoft.PowerToys.Settings.UI.Views
|
||||
{
|
||||
if (ViewModel?.PasteAIProviderDraft is not null)
|
||||
{
|
||||
ViewModel.PasteAIProviderDraft.ModelName = NormalizeFoundryModelReference(selectedModel.Url ?? selectedModel.Name);
|
||||
ViewModel.PasteAIProviderDraft.ModelName = selectedModel.Name;
|
||||
}
|
||||
|
||||
if (FoundryLocalPicker is not null)
|
||||
|
||||
Reference in New Issue
Block a user