diff --git a/Directory.Packages.props b/Directory.Packages.props index f52a62239a..297bcfcd4a 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,4 +1,4 @@ - + true true @@ -39,9 +39,11 @@ + + - - + + @@ -71,7 +73,7 @@ - + diff --git a/PowerToys.sln b/PowerToys.sln index 50063816ea..5999cac651 100644 --- a/PowerToys.sln +++ b/PowerToys.sln @@ -825,6 +825,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LightSwitch.UITests", "src\ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.CmdPal.Ext.ClipboardHistory.UnitTests", "src\modules\cmdpal\Tests\Microsoft.CmdPal.Ext.ClipboardHistory.UnitTests\Microsoft.CmdPal.Ext.ClipboardHistory.UnitTests.csproj", "{4E0FCF69-B06B-D272-76BF-ED3A559B4EDA}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LanguageModelProvider", "src\common\LanguageModelProvider\LanguageModelProvider.csproj", "{45354F4F-1414-45CE-B600-51CD1209FD19}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|ARM64 = Debug|ARM64 @@ -2995,6 +2997,14 @@ Global {4E0FCF69-B06B-D272-76BF-ED3A559B4EDA}.Release|ARM64.Build.0 = Release|ARM64 {4E0FCF69-B06B-D272-76BF-ED3A559B4EDA}.Release|x64.ActiveCfg = Release|x64 {4E0FCF69-B06B-D272-76BF-ED3A559B4EDA}.Release|x64.Build.0 = Release|x64 + {45354F4F-1414-45CE-B600-51CD1209FD19}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {45354F4F-1414-45CE-B600-51CD1209FD19}.Debug|ARM64.Build.0 = Debug|ARM64 + {45354F4F-1414-45CE-B600-51CD1209FD19}.Debug|x64.ActiveCfg = Debug|x64 + {45354F4F-1414-45CE-B600-51CD1209FD19}.Debug|x64.Build.0 = Debug|x64 + {45354F4F-1414-45CE-B600-51CD1209FD19}.Release|ARM64.ActiveCfg = Release|ARM64 + {45354F4F-1414-45CE-B600-51CD1209FD19}.Release|ARM64.Build.0 = Release|ARM64 + {45354F4F-1414-45CE-B600-51CD1209FD19}.Release|x64.ActiveCfg = Release|x64 + {45354F4F-1414-45CE-B600-51CD1209FD19}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -3323,6 +3333,7 @@ Global {3DCCD936-D085-4869-A1DE-CA6A64152C94} = {5B201255-53C8-490B-A34F-01F05D48A477} {F5333ED7-06D8-4AB3-953A-36D63F08CB6F} = {3DCCD936-D085-4869-A1DE-CA6A64152C94} {4E0FCF69-B06B-D272-76BF-ED3A559B4EDA} = {8EF25507-2575-4ADE-BF7E-D23376903AB8} + {45354F4F-1414-45CE-B600-51CD1209FD19} = {1AFB6476-670D-4E80-A464-657E01DFF482} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {C3A2F9D1-7930-4EF4-A6FC-7EE0A99821D0} diff --git a/src/common/LanguageModelProvider/AppUtils.cs b/src/common/LanguageModelProvider/AppUtils.cs new file mode 100644 index 0000000000..ce48283337 --- /dev/null +++ b/src/common/LanguageModelProvider/AppUtils.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace LanguageModelProvider; + +internal static class AppUtils +{ + public static string GetThemeAssetSuffix() + { + // Default suffix for assets that are theme-agnostic today. + return string.Empty; + } +} diff --git a/src/common/LanguageModelProvider/FoundryLocal/FoundryCachedModel.cs b/src/common/LanguageModelProvider/FoundryLocal/FoundryCachedModel.cs new file mode 100644 index 0000000000..489a779179 --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocal/FoundryCachedModel.cs @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace LanguageModelProvider.FoundryLocal; + +internal sealed record FoundryCachedModel(string Name, string? Id); diff --git a/src/common/LanguageModelProvider/FoundryLocal/FoundryCatalogModel.cs b/src/common/LanguageModelProvider/FoundryLocal/FoundryCatalogModel.cs new file mode 100644 index 0000000000..413bb47316 --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocal/FoundryCatalogModel.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text.Json.Serialization; + +namespace LanguageModelProvider.FoundryLocal; + +internal sealed record FoundryCatalogModel +{ + [JsonPropertyName("name")] + public string Name { get; init; } = string.Empty; + + [JsonPropertyName("displayName")] + public string DisplayName { get; init; } = string.Empty; + + [JsonPropertyName("providerType")] + public string ProviderType { get; init; } = string.Empty; + + [JsonPropertyName("uri")] + public string Uri { get; init; } = string.Empty; + + [JsonPropertyName("version")] + public string Version { get; init; } = string.Empty; + + [JsonPropertyName("modelType")] + public string ModelType { get; init; } = string.Empty; + + [JsonPropertyName("promptTemplate")] + public PromptTemplate PromptTemplate { get; init; } = default!; + + [JsonPropertyName("publisher")] + public string Publisher { get; init; } = string.Empty; + + [JsonPropertyName("task")] + public string Task { get; init; } = string.Empty; + + [JsonPropertyName("runtime")] + public Runtime Runtime { get; init; } = default!; + + [JsonPropertyName("fileSizeMb")] + public long FileSizeMb { get; init; } + + [JsonPropertyName("modelSettings")] + public ModelSettings ModelSettings { get; init; } = default!; + + [JsonPropertyName("alias")] + public string Alias { get; init; } = string.Empty; + + [JsonPropertyName("supportsToolCalling")] + public bool SupportsToolCalling { get; init; } + + [JsonPropertyName("license")] + public string License { get; init; } = string.Empty; + + [JsonPropertyName("licenseDescription")] + public string LicenseDescription { get; init; } = string.Empty; + + [JsonPropertyName("parentModelUri")] + public string ParentModelUri { get; init; } = string.Empty; +} diff --git a/src/common/LanguageModelProvider/FoundryLocal/FoundryClient.cs b/src/common/LanguageModelProvider/FoundryLocal/FoundryClient.cs new file mode 100644 index 0000000000..a84bdec687 --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocal/FoundryClient.cs @@ -0,0 +1,214 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace LanguageModelProvider.FoundryLocal; + +internal sealed class FoundryClient +{ + public static async Task CreateAsync() + { + var serviceManager = FoundryServiceManager.TryCreate(); + if (serviceManager is null) + { + return null; + } + + if (!await serviceManager.IsRunning().ConfigureAwait(false)) + { + if (!await serviceManager.StartService().ConfigureAwait(false)) + { + return null; + } + } + + var serviceUrl = await serviceManager.GetServiceUrl().ConfigureAwait(false); + + if (string.IsNullOrEmpty(serviceUrl)) + { + return null; + } + + return new FoundryClient(serviceUrl, serviceManager, new HttpClient()); + } + + public FoundryServiceManager ServiceManager { get; } + + private readonly HttpClient _httpClient; + private readonly string _baseUrl; + private readonly List _catalogModels = []; + + private FoundryClient(string baseUrl, FoundryServiceManager serviceManager, HttpClient httpClient) + { + ServiceManager = serviceManager; + _baseUrl = baseUrl; + _httpClient = httpClient; + } + + public async Task> ListCatalogModels() + { + if (_catalogModels.Count > 0) + { + return _catalogModels; + } + + try + { + var response = await _httpClient.GetAsync($"{_baseUrl}/foundry/list").ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + var models = await JsonSerializer.DeserializeAsync( + response.Content.ReadAsStream(), + FoundryJsonContext.Default.ListFoundryCatalogModel).ConfigureAwait(false); + + if (models is { Count: > 0 }) + { + models.ForEach(_catalogModels.Add); + } + } + catch + { + // Surfacing errors here prevents listing other providers; swallow and return cached list instead. + } + + return _catalogModels; + } + + public async Task> ListCachedModels() + { + var response = await _httpClient.GetAsync($"{_baseUrl}/openai/models").ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + var catalogModels = await ListCatalogModels().ConfigureAwait(false); + + var content = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + var modelIds = content + .Trim('[', ']') + .Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries) + .Select(id => id.Trim('"')); + + List models = []; + + foreach (var id in modelIds) + { + var model = catalogModels.FirstOrDefault(m => m.Name == id); + models.Add(model != null ? new FoundryCachedModel(id, model.Alias) : new FoundryCachedModel(id, null)); + } + + return models; + } + + public async Task DownloadModel(FoundryCatalogModel model, IProgress? progress, CancellationToken cancellationToken = default) + { + var models = await ListCachedModels().ConfigureAwait(false); + + if (models.Any(m => m.Name == model.Name)) + { + return new(true, "Model already downloaded"); + } + + return await Task.Run( + async () => + { + try + { + var modelDownload = new FoundryModelDownload( + Name: model.Name, + Uri: model.Uri, + Path: await GetModelPath(model.Uri).ConfigureAwait(false), // temporary + ProviderType: model.ProviderType, + PromptTemplate: model.PromptTemplate); + + var uploadBody = new FoundryDownloadBody(modelDownload, IgnorePipeReport: true); + + var downloadBodyContext = FoundryJsonContext.Default.FoundryDownloadBody; + string body = JsonSerializer.Serialize(uploadBody, downloadBodyContext); + + using var request = new HttpRequestMessage(HttpMethod.Post, $"{_baseUrl}/openai/download") + { + Content = new StringContent(body, Encoding.UTF8, "application/json"), + }; + + using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + response.EnsureSuccessStatusCode(); + + using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var reader = new StreamReader(stream); + + string? finalJson = null; + + while (!reader.EndOfStream && !cancellationToken.IsCancellationRequested) + { + cancellationToken.ThrowIfCancellationRequested(); + var line = await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + line = line.Trim(); + + if (finalJson != null || line.StartsWith('{')) + { + finalJson += line; + continue; + } + + var match = Regex.Match(line, @"\d+(\.\d+)?%"); + if (match.Success) + { + var percentage = match.Value; + if (float.TryParse(percentage.TrimEnd('%'), out float progressValue)) + { + progress?.Report(progressValue / 100); + } + } + } + + var downloadResultContext = FoundryJsonContext.Default.FoundryDownloadResult; + var result = finalJson is not null + ? JsonSerializer.Deserialize(finalJson, downloadResultContext)! + : new FoundryDownloadResult(false, "Missing final result from server."); + + return result; + } + catch (Exception e) + { + return new FoundryDownloadResult(false, e.Message); + } + }, + cancellationToken).ConfigureAwait(false); + } + + // this is a temporary function to get the model path from the blob storage + // it will be removed once the tag is available in the list response + private async Task GetModelPath(string assetId) + { + var registryUri = + $"https://eastus.api.azureml.ms/modelregistry/v1.0/registry/models/nonazureaccount?assetId={Uri.EscapeDataString(assetId)}"; + + using var resp = await _httpClient.GetAsync(registryUri).ConfigureAwait(false); + resp.EnsureSuccessStatusCode(); + + await using var jsonStream = await resp.Content.ReadAsStreamAsync().ConfigureAwait(false); + var jsonRoot = await JsonDocument.ParseAsync(jsonStream).ConfigureAwait(false); + var blobSasUri = jsonRoot.RootElement.GetProperty("blobSasUri").GetString()!; + + var uriBuilder = new UriBuilder(blobSasUri); + var existingQuery = string.IsNullOrWhiteSpace(uriBuilder.Query) + ? string.Empty + : uriBuilder.Query.TrimStart('?') + "&"; + + uriBuilder.Query = existingQuery + "restype=container&comp=list&delimiter=/"; + + var listXml = await _httpClient.GetStringAsync(uriBuilder.Uri).ConfigureAwait(false); + + var match = Regex.Match(listXml, @"(.*?)\/<\/Name>"); + return match.Success ? match.Groups[1].Value : string.Empty; + } +} diff --git a/src/common/LanguageModelProvider/FoundryLocal/FoundryDownloadBody.cs b/src/common/LanguageModelProvider/FoundryLocal/FoundryDownloadBody.cs new file mode 100644 index 0000000000..6aed43c779 --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocal/FoundryDownloadBody.cs @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace LanguageModelProvider.FoundryLocal; + +internal sealed record FoundryDownloadBody(FoundryModelDownload Model, bool IgnorePipeReport); diff --git a/src/common/LanguageModelProvider/FoundryLocal/FoundryDownloadResult.cs b/src/common/LanguageModelProvider/FoundryLocal/FoundryDownloadResult.cs new file mode 100644 index 0000000000..56fcb51626 --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocal/FoundryDownloadResult.cs @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace LanguageModelProvider.FoundryLocal; + +internal sealed record FoundryDownloadResult(bool Success, string? ErrorMessage); diff --git a/src/common/LanguageModelProvider/FoundryLocal/FoundryJsonContext.cs b/src/common/LanguageModelProvider/FoundryLocal/FoundryJsonContext.cs new file mode 100644 index 0000000000..084f84fb8d --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocal/FoundryJsonContext.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace LanguageModelProvider.FoundryLocal; + +[JsonSourceGenerationOptions( + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + WriteIndented = false)] +[JsonSerializable(typeof(FoundryCatalogModel))] +[JsonSerializable(typeof(List))] +[JsonSerializable(typeof(FoundryDownloadResult))] +[JsonSerializable(typeof(FoundryDownloadBody))] +internal sealed partial class FoundryJsonContext : JsonSerializerContext +{ +} diff --git a/src/common/LanguageModelProvider/FoundryLocal/FoundryModelDownload.cs b/src/common/LanguageModelProvider/FoundryLocal/FoundryModelDownload.cs new file mode 100644 index 0000000000..5fbc28dbfb --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocal/FoundryModelDownload.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace LanguageModelProvider.FoundryLocal; + +internal sealed record FoundryModelDownload( + string Name, + string Uri, + string Path, + string ProviderType, + PromptTemplate PromptTemplate); diff --git a/src/common/LanguageModelProvider/FoundryLocal/FoundryServiceManager.cs b/src/common/LanguageModelProvider/FoundryLocal/FoundryServiceManager.cs new file mode 100644 index 0000000000..95574c697a --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocal/FoundryServiceManager.cs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics; +using System.Text.RegularExpressions; + +namespace LanguageModelProvider.FoundryLocal; + +internal sealed class FoundryServiceManager +{ + public static FoundryServiceManager? TryCreate() + { + return IsAvailable() ? new FoundryServiceManager() : null; + } + + private static bool IsAvailable() + { + using var process = new Process(); + process.StartInfo.FileName = "where"; + process.StartInfo.Arguments = "foundry"; + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.UseShellExecute = false; + process.StartInfo.CreateNoWindow = true; + process.Start(); + process.WaitForExit(); + return process.ExitCode == 0; + } + + private static string? GetUrl(string output) + { + var match = Regex.Match(output, @"https?:\/\/[^\/]+:\d+"); + return match.Success ? match.Value : null; + } + + public async Task GetServiceUrl() + { + var status = await Utils.RunFoundryWithArguments("service status").ConfigureAwait(false); + + if (status.ExitCode != 0 || string.IsNullOrWhiteSpace(status.Output)) + { + return null; + } + + return GetUrl(status.Output); + } + + public async Task IsRunning() + { + var url = await GetServiceUrl().ConfigureAwait(false); + return url is not null; + } + + public async Task StartService() + { + if (await IsRunning().ConfigureAwait(false)) + { + return true; + } + + var status = await Utils.RunFoundryWithArguments("service start").ConfigureAwait(false); + if (status.ExitCode != 0 || string.IsNullOrWhiteSpace(status.Output)) + { + return false; + } + + return GetUrl(status.Output) is not null; + } +} diff --git a/src/common/LanguageModelProvider/FoundryLocal/ModelSettings.cs b/src/common/LanguageModelProvider/FoundryLocal/ModelSettings.cs new file mode 100644 index 0000000000..fda91217eb --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocal/ModelSettings.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace LanguageModelProvider.FoundryLocal; + +internal sealed record ModelSettings +{ + // The sample shows an empty array; keep it open-ended. + [JsonPropertyName("parameters")] + public List Parameters { get; init; } = []; +} diff --git a/src/common/LanguageModelProvider/FoundryLocal/PromptTemplate.cs b/src/common/LanguageModelProvider/FoundryLocal/PromptTemplate.cs new file mode 100644 index 0000000000..a2cbb9fe45 --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocal/PromptTemplate.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text.Json.Serialization; + +namespace LanguageModelProvider.FoundryLocal; + +internal sealed record PromptTemplate +{ + [JsonPropertyName("assistant")] + public string Assistant { get; init; } = string.Empty; + + [JsonPropertyName("prompt")] + public string Prompt { get; init; } = string.Empty; +} diff --git a/src/common/LanguageModelProvider/FoundryLocal/Runtime.cs b/src/common/LanguageModelProvider/FoundryLocal/Runtime.cs new file mode 100644 index 0000000000..e2019c8f87 --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocal/Runtime.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text.Json.Serialization; + +namespace LanguageModelProvider.FoundryLocal; + +internal sealed record Runtime +{ + [JsonPropertyName("deviceType")] + public string DeviceType { get; init; } = string.Empty; + + [JsonPropertyName("executionProvider")] + public string ExecutionProvider { get; init; } = string.Empty; +} diff --git a/src/common/LanguageModelProvider/FoundryLocal/Utils.cs b/src/common/LanguageModelProvider/FoundryLocal/Utils.cs new file mode 100644 index 0000000000..ffcb9c1687 --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocal/Utils.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics; + +namespace LanguageModelProvider.FoundryLocal; + +internal static class Utils +{ + public static async Task<(string? Output, string? Error, int ExitCode)> RunFoundryWithArguments(string arguments) + { + try + { + using var process = new Process(); + process.StartInfo.FileName = "foundry"; + process.StartInfo.Arguments = arguments; + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.UseShellExecute = false; + process.StartInfo.CreateNoWindow = true; + + process.Start(); + + string? output = await process.StandardOutput.ReadToEndAsync().ConfigureAwait(false); + string? error = await process.StandardError.ReadToEndAsync().ConfigureAwait(false); + + await process.WaitForExitAsync().ConfigureAwait(false); + + return (output, error, process.ExitCode); + } + catch + { + return (null, null, -1); + } + } +} diff --git a/src/common/LanguageModelProvider/FoundryLocalModelProvider.cs b/src/common/LanguageModelProvider/FoundryLocalModelProvider.cs new file mode 100644 index 0000000000..c51959ad7e --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocalModelProvider.cs @@ -0,0 +1,210 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.ClientModel; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using LanguageModelProvider.FoundryLocal; +using Microsoft.Extensions.AI; +using OpenAI; + +namespace LanguageModelProvider; + +public sealed class FoundryLocalModelProvider : ILanguageModelProvider +{ + private IEnumerable? _downloadedModels; + private IEnumerable? _catalogModels; + private FoundryClient? _foundryManager; + private string? _serviceUrl; + + public static FoundryLocalModelProvider Instance { get; } = new(); + + public string Name => "FoundryLocal"; + + public HardwareAccelerator ModelHardwareAccelerator => HardwareAccelerator.FOUNDRYLOCAL; + + public List NugetPackageReferences { get; } = ["Microsoft.Extensions.AI.OpenAI"]; + + public string ProviderDescription => "The model will run locally via Foundry Local"; + + public string UrlPrefix => "fl://"; + + public string Icon => $"fl{AppUtils.GetThemeAssetSuffix()}.svg"; + + public string Url => _serviceUrl ?? string.Empty; + + public string IChatClientImplementationNamespace { get; } = "OpenAI"; + + public string GetDetailsUrl(ModelDetails details) + { + throw new NotImplementedException(); + } + + public IChatClient? GetIChatClient(string url) + { + try + { + InitializeAsync().GetAwaiter().GetResult(); + } + catch + { + return null; + } + + if (string.IsNullOrWhiteSpace(_serviceUrl)) + { + return null; + } + + var modelId = url.Split('/').LastOrDefault(); + if (string.IsNullOrWhiteSpace(modelId)) + { + return null; + } + + return new OpenAIClient( + new ApiKeyCredential("none"), + new OpenAIClientOptions { Endpoint = new Uri($"{_serviceUrl}/v1") }) + .GetChatClient(modelId) + .AsIChatClient(); + } + + public string GetIChatClientString(string url) + { + try + { + InitializeAsync().GetAwaiter().GetResult(); + } + catch + { + return string.Empty; + } + + var modelId = url.Split('/').LastOrDefault(); + + if (string.IsNullOrWhiteSpace(_serviceUrl) || string.IsNullOrWhiteSpace(modelId)) + { + return string.Empty; + } + + return $"new OpenAIClient(new ApiKeyCredential(\"none\"), new OpenAIClientOptions{{ Endpoint = new Uri(\"{_serviceUrl}/v1\") }}).GetChatClient(\"{modelId}\").AsIChatClient()"; + } + + public async Task> GetModelsAsync(bool ignoreCached = false, CancellationToken cancelationToken = default) + { + if (ignoreCached) + { + Reset(); + } + + await InitializeAsync(cancelationToken); + + return _downloadedModels ?? []; + } + + public IEnumerable GetAllModelsInCatalog() + { + return _catalogModels ?? []; + } + + public async Task DownloadModel(ModelDetails modelDetails, IProgress? progress, CancellationToken cancellationToken = default) + { + if (_foundryManager == null) + { + return false; + } + + if (modelDetails.ProviderModelDetails is not FoundryCatalogModel model) + { + return false; + } + + return (await _foundryManager.DownloadModel(model, progress, cancellationToken)).Success; + } + + private void Reset() + { + _downloadedModels = null; + _ = InitializeAsync(); + } + + private async Task InitializeAsync(CancellationToken cancelationToken = default) + { + if (_foundryManager != null && _downloadedModels != null && _downloadedModels.Any()) + { + return; + } + + _foundryManager ??= await FoundryClient.CreateAsync(); + + if (_foundryManager == null) + { + return; + } + + _serviceUrl ??= await _foundryManager.ServiceManager.GetServiceUrl(); + + if (_catalogModels == null || !_catalogModels.Any()) + { + _catalogModels = (await _foundryManager.ListCatalogModels()).Select(ToModelDetails).ToArray(); + } + + var cachedModels = await _foundryManager.ListCachedModels(); + + List downloadedModels = []; + + foreach (var model in _catalogModels) + { + var cachedModel = cachedModels.FirstOrDefault(m => m.Name == model.Name); + + if (cachedModel != default) + { + model.Id = $"{UrlPrefix}{cachedModel.Id}"; + downloadedModels.Add(model); + cachedModels.Remove(cachedModel); + } + } + + foreach (var model in cachedModels) + { + downloadedModels.Add(new ModelDetails + { + Id = $"fl-{model.Name}", + Name = model.Name, + Url = $"{UrlPrefix}{model.Name}", + Description = $"{model.Name} running locally with Foundry Local", + HardwareAccelerators = [HardwareAccelerator.FOUNDRYLOCAL], + SupportedOnQualcomm = true, + ProviderModelDetails = model, + }); + } + + _downloadedModels = downloadedModels; + } + + private ModelDetails ToModelDetails(FoundryCatalogModel model) + { + return new ModelDetails + { + Id = $"fl-{model.Name}", + Name = model.Name, + Url = $"{UrlPrefix}{model.Name}", + Description = $"{model.Alias} running locally with Foundry Local", + HardwareAccelerators = [HardwareAccelerator.FOUNDRYLOCAL], + Size = model.FileSizeMb * 1024 * 1024, + SupportedOnQualcomm = true, + License = model.License?.ToLowerInvariant() ?? string.Empty, + ProviderModelDetails = model, + }; + } + + public async Task IsAvailable() + { + await InitializeAsync(); + return _foundryManager != null; + } +} diff --git a/src/common/LanguageModelProvider/HardwareAccelerator.cs b/src/common/LanguageModelProvider/HardwareAccelerator.cs new file mode 100644 index 0000000000..d2c94b8155 --- /dev/null +++ b/src/common/LanguageModelProvider/HardwareAccelerator.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace LanguageModelProvider; + +public enum HardwareAccelerator +{ + CPU, + DML, + QNN, + WCRAPI, + OLLAMA, + OPENAI, + FOUNDRYLOCAL, + LEMONADE, + NPU, + GPU, + VitisAI, + OpenVINO, + NvTensorRT, +} diff --git a/src/common/LanguageModelProvider/ILanguageModelProvider.cs b/src/common/LanguageModelProvider/ILanguageModelProvider.cs new file mode 100644 index 0000000000..a79a959ccf --- /dev/null +++ b/src/common/LanguageModelProvider/ILanguageModelProvider.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Extensions.AI; + +namespace LanguageModelProvider; + +public interface ILanguageModelProvider +{ + string Name { get; } + + string UrlPrefix { get; } + + string Icon { get; } + + HardwareAccelerator ModelHardwareAccelerator { get; } + + List NugetPackageReferences { get; } + + string ProviderDescription { get; } + + Task> GetModelsAsync(bool ignoreCached = false, CancellationToken cancelationToken = default); + + IChatClient? GetIChatClient(string url); + + string IChatClientImplementationNamespace { get; } + + string GetIChatClientString(string url); + + string GetDetailsUrl(ModelDetails details); + + string Url { get; } +} diff --git a/src/common/LanguageModelProvider/LanguageModelProvider.csproj b/src/common/LanguageModelProvider/LanguageModelProvider.csproj new file mode 100644 index 0000000000..cf4bde1cd1 --- /dev/null +++ b/src/common/LanguageModelProvider/LanguageModelProvider.csproj @@ -0,0 +1,15 @@ + + + + + + enable + enable + + + + + + + + diff --git a/src/common/LanguageModelProvider/LanguageModelService.cs b/src/common/LanguageModelProvider/LanguageModelService.cs new file mode 100644 index 0000000000..91b62caf87 --- /dev/null +++ b/src/common/LanguageModelProvider/LanguageModelService.cs @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Concurrent; +using System.Linq; +using Microsoft.Extensions.AI; + +namespace LanguageModelProvider; + +public sealed class LanguageModelService +{ + private readonly ConcurrentDictionary _providersByPrefix; + + public LanguageModelService(IEnumerable providers) + { + ArgumentNullException.ThrowIfNull(providers); + + _providersByPrefix = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var provider in providers) + { + if (!string.IsNullOrWhiteSpace(provider.UrlPrefix)) + { + _providersByPrefix[provider.UrlPrefix] = provider; + } + } + } + + public static LanguageModelService CreateDefault() + { + return new LanguageModelService(new[] + { + FoundryLocalModelProvider.Instance, + }); + } + + public IReadOnlyCollection Providers => _providersByPrefix.Values.ToArray(); + + public bool RegisterProvider(ILanguageModelProvider provider) + { + ArgumentNullException.ThrowIfNull(provider); + + if (string.IsNullOrWhiteSpace(provider.UrlPrefix)) + { + throw new ArgumentException("Provider must supply a URL prefix.", nameof(provider)); + } + + _providersByPrefix[provider.UrlPrefix] = provider; + return true; + } + + public ILanguageModelProvider? GetProviderFor(string? modelReference) + { + if (string.IsNullOrWhiteSpace(modelReference)) + { + return null; + } + + foreach (var provider in _providersByPrefix.Values) + { + if (modelReference.StartsWith(provider.UrlPrefix, StringComparison.OrdinalIgnoreCase)) + { + return provider; + } + } + + return null; + } + + public async Task> GetModelsAsync(bool refresh = false, CancellationToken cancellationToken = default) + { + List models = []; + + foreach (var provider in _providersByPrefix.Values) + { + cancellationToken.ThrowIfCancellationRequested(); + var providerModels = await provider.GetModelsAsync(refresh, cancellationToken).ConfigureAwait(false); + models.AddRange(providerModels); + } + + return models; + } + + public IChatClient? GetClient(ModelDetails model) + { + if (model is null) + { + return null; + } + + var reference = !string.IsNullOrWhiteSpace(model.Url) ? model.Url : model.Id; + return GetClient(reference); + } + + public IChatClient? GetClient(string? modelReference) + { + if (string.IsNullOrWhiteSpace(modelReference)) + { + return null; + } + + var provider = GetProviderFor(modelReference); + + return provider?.GetIChatClient(modelReference); + } +} diff --git a/src/common/LanguageModelProvider/ModelDetails.cs b/src/common/LanguageModelProvider/ModelDetails.cs new file mode 100644 index 0000000000..2e68ca6feb --- /dev/null +++ b/src/common/LanguageModelProvider/ModelDetails.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; + +namespace LanguageModelProvider; + +public class ModelDetails +{ + public string Id { get; set; } = string.Empty; + + public string Name { get; set; } = string.Empty; + + public string Url { get; set; } = string.Empty; + + public string Description { get; set; } = string.Empty; + + public long Size { get; set; } + + public bool IsUserAdded { get; set; } + + public string Icon { get; set; } = string.Empty; + + public List HardwareAccelerators { get; set; } = []; + + public bool SupportedOnQualcomm { get; set; } + + public string License { get; set; } = string.Empty; + + public object? ProviderModelDetails { get; set; } +}