diff --git a/.github/actions/spell-check/expect.txt b/.github/actions/spell-check/expect.txt index 63469ee8ee..c50b54b1d9 100644 --- a/.github/actions/spell-check/expect.txt +++ b/.github/actions/spell-check/expect.txt @@ -34,6 +34,7 @@ AFX AGGREGATABLE AHK AHybrid +AIUI akv ALarger ALIGNRIGHT @@ -45,6 +46,7 @@ Allmodule ALLOWUNDO ALLVIEW ALPHATYPE +amazonbedrock AModifier amr ANDSCANS @@ -112,6 +114,9 @@ AValid AWAYMODE azcliversion azman +azureaiinference +azureinference +azureopenai bbwe BCIE bck @@ -204,6 +209,7 @@ CIBUILD cidl CIELCh cim +claude CImage cla CLASSDC @@ -236,6 +242,7 @@ CODENAME codereview Codespaces Coen +cognitiveservices COINIT colid colorconv @@ -420,6 +427,7 @@ DROPFILES DSTINVERT DString DSVG +dto DTo DUMMYUNIONNAME dutil @@ -551,6 +559,7 @@ FIXEDSYS flac flyouts FMask +foundrylocal fmtid FOF FOFX @@ -615,6 +624,8 @@ GValue gwl GWLP GWLSTYLE +googleai +googlegemini hangeul Hanzi Hardlines @@ -677,6 +688,7 @@ hmonitor homies homljgmgpmcbpjbnjpfijnhipfkiclkd HOOKPROC +huggingface HORZRES HORZSIZE Hostbackdropbrush @@ -1140,6 +1152,7 @@ nullrefs NOOWNERZORDER NOPARENTNOTIFY NOPREFIX +NPU NOREDIRECTIONBITMAP NOREDRAW NOREMOVE @@ -1199,6 +1212,9 @@ opencode OPENFILENAME opensource openxmlformats +ollama +Olllama +onnx OPTIMIZEFORINVOKE ORPHANEDDIALOGTITLE ORSCANS @@ -1388,6 +1404,8 @@ pwsz pwtd QDC qit +QNN +Qualcomm QITAB QITABENT qoi @@ -1936,6 +1954,7 @@ wcsicmp wcsncpy wcsnicmp WCT +WCRAPI WDA wdm wdp @@ -1979,6 +1998,8 @@ WINL winlogon winmd WINNT +windowsml +winml winres winrt winsdk diff --git a/.pipelines/ESRPSigning_core.json b/.pipelines/ESRPSigning_core.json index d99fabf0eb..5b6dd50bb5 100644 --- a/.pipelines/ESRPSigning_core.json +++ b/.pipelines/ESRPSigning_core.json @@ -5,7 +5,6 @@ { "MatchedPath": [ "*.resources.dll", - "WinUI3Apps\\Assets\\Settings\\Scripts\\*.ps1", "PowerToys.ActionRunner.exe", @@ -27,6 +26,7 @@ "PowerToys.GPOWrapper.dll", "PowerToys.GPOWrapperProjection.dll", "PowerToys.AllExperiments.dll", + "LanguageModelProvider.dll", "Common.Search.dll", @@ -346,6 +346,8 @@ "Testably.Abstractions.FileSystem.Interface.dll", "WinUI3Apps\\Testably.Abstractions.FileSystem.Interface.dll", "ColorCode.Core.dll", + "Microsoft.SemanticKernel.Connectors.Ollama.dll", + "OllamaSharp.dll", "UnitsNet.dll", "UtfUnknown.dll", diff --git a/.pipelines/v2/ci.yml b/.pipelines/v2/ci.yml index 6b0105a38a..297c268757 100644 --- a/.pipelines/v2/ci.yml +++ b/.pipelines/v2/ci.yml @@ -32,7 +32,7 @@ parameters: - name: enableMsBuildCaching type: boolean displayName: "Enable MSBuild Caching" - default: true + default: false - name: runTests type: boolean displayName: "Run Tests" diff --git a/.pipelines/v2/templates/job-build-project.yml b/.pipelines/v2/templates/job-build-project.yml index 5c2f74e9e0..34e4ce4340 100644 --- a/.pipelines/v2/templates/job-build-project.yml +++ b/.pipelines/v2/templates/job-build-project.yml @@ -266,6 +266,26 @@ jobs: env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) + - task: VSBuild@1 + displayName: Generate DSC artifacts for ARM64 + condition: and(succeeded(), eq(variables['BuildPlatform'], 'arm64')) + inputs: + solution: PowerToys.sln + vsVersion: 17.0 + msbuildArgs: >- + -restore + /p:Configuration=$(BuildConfiguration) + /p:Platform=x64 + /t:DSC\PowerToys_Settings_DSC_Schema_Generator + /bl:$(LogOutputDirectory)\build-dsc-generator.binlog + ${{ parameters.additionalBuildOptions }} + $(MSBuildCacheParameters) + $(RestoreAdditionalProjectSourcesArg) + platform: x64 + configuration: $(BuildConfiguration) + msbuildArchitecture: x64 + maximumCpuCount: true + # Build PowerToys.DSC.exe for ARM64 (x64 uses existing binary from previous build) - task: VSBuild@1 displayName: Build PowerToys.DSC.exe (x64 for generating manifests) diff --git a/.vscode/launch.json b/.vscode/launch.json index b2d2bca9ac..940ff302de 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -38,6 +38,17 @@ "env": {}, "console": "internalConsole", "stopAtEntry": false - } + }, + { + "name": "Run AdvancedPaste (managed, no build, ARCH configurable)", + "type": "coreclr", + "request": "launch", + "program": "${workspaceFolder}\\${input:arch}\\Debug\\WinUI3Apps\\PowerToys.AdvancedPaste.exe", + "args": [], + "cwd": "${workspaceFolder}", + "env": {}, + "console": "internalConsole", + "stopAtEntry": false + }, ] } \ No newline at end of file diff --git a/Directory.Packages.props b/Directory.Packages.props index 129abadb00..38d587dfee 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,4 +1,4 @@ - + true true @@ -9,7 +9,6 @@ - @@ -40,12 +39,24 @@ - - - + + + + + + + - + + + + + + + + + @@ -74,7 +85,7 @@ - + @@ -95,6 +106,7 @@ + @@ -127,4 +139,4 @@ - + \ No newline at end of file diff --git a/NOTICE.md b/NOTICE.md index 1998ea805a..a0d87d429c 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -1495,7 +1495,6 @@ SOFTWARE. - AdaptiveCards.Rendering.WinUI3 - AdaptiveCards.Templating - Appium.WebDriver -- Azure.AI.OpenAI - CoenM.ImageSharp.ImageHash - CommunityToolkit.Common - CommunityToolkit.Labs.WinUI.Controls.MarkdownTextBlock diff --git a/PowerToys.sln b/PowerToys.sln index f4f2e1bd0f..b343993b68 100644 --- a/PowerToys.sln +++ b/PowerToys.sln @@ -828,6 +828,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LightSwitch.UITests", "src\ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.CmdPal.Ext.ClipboardHistory.UnitTests", "src\modules\cmdpal\Tests\Microsoft.CmdPal.Ext.ClipboardHistory.UnitTests\Microsoft.CmdPal.Ext.ClipboardHistory.UnitTests.csproj", "{4E0FCF69-B06B-D272-76BF-ED3A559B4EDA}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LanguageModelProvider", "src\common\LanguageModelProvider\LanguageModelProvider.csproj", "{45354F4F-1414-45CE-B600-51CD1209FD19}" +EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.CmdPal.UI.ViewModels.UnitTests", "src\modules\cmdpal\Tests\Microsoft.CmdPal.UI.ViewModels.UnitTests\Microsoft.CmdPal.UI.ViewModels.UnitTests.csproj", "{A66E9270-5D93-EC9C-F06E-CE7295BB9A6C}" EndProject Global @@ -3008,6 +3010,14 @@ Global {4E0FCF69-B06B-D272-76BF-ED3A559B4EDA}.Release|ARM64.Build.0 = Release|ARM64 {4E0FCF69-B06B-D272-76BF-ED3A559B4EDA}.Release|x64.ActiveCfg = Release|x64 {4E0FCF69-B06B-D272-76BF-ED3A559B4EDA}.Release|x64.Build.0 = Release|x64 + {45354F4F-1414-45CE-B600-51CD1209FD19}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {45354F4F-1414-45CE-B600-51CD1209FD19}.Debug|ARM64.Build.0 = Debug|ARM64 + {45354F4F-1414-45CE-B600-51CD1209FD19}.Debug|x64.ActiveCfg = Debug|x64 + {45354F4F-1414-45CE-B600-51CD1209FD19}.Debug|x64.Build.0 = Debug|x64 + {45354F4F-1414-45CE-B600-51CD1209FD19}.Release|ARM64.ActiveCfg = Release|ARM64 + {45354F4F-1414-45CE-B600-51CD1209FD19}.Release|ARM64.Build.0 = Release|ARM64 + {45354F4F-1414-45CE-B600-51CD1209FD19}.Release|x64.ActiveCfg = Release|x64 + {45354F4F-1414-45CE-B600-51CD1209FD19}.Release|x64.Build.0 = Release|x64 {A66E9270-5D93-EC9C-F06E-CE7295BB9A6C}.Debug|ARM64.ActiveCfg = Debug|ARM64 {A66E9270-5D93-EC9C-F06E-CE7295BB9A6C}.Debug|ARM64.Build.0 = Debug|ARM64 {A66E9270-5D93-EC9C-F06E-CE7295BB9A6C}.Debug|x64.ActiveCfg = Debug|x64 @@ -3344,6 +3354,7 @@ Global {3DCCD936-D085-4869-A1DE-CA6A64152C94} = {5B201255-53C8-490B-A34F-01F05D48A477} {F5333ED7-06D8-4AB3-953A-36D63F08CB6F} = {3DCCD936-D085-4869-A1DE-CA6A64152C94} {4E0FCF69-B06B-D272-76BF-ED3A559B4EDA} = {8EF25507-2575-4ADE-BF7E-D23376903AB8} + {45354F4F-1414-45CE-B600-51CD1209FD19} = {1AFB6476-670D-4E80-A464-657E01DFF482} {A66E9270-5D93-EC9C-F06E-CE7295BB9A6C} = {8EF25507-2575-4ADE-BF7E-D23376903AB8} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution diff --git a/installer/PowerToysSetupVNext/CmdPal.wxs b/installer/PowerToysSetupVNext/CmdPal.wxs index 8304551c12..f05a1f2f35 100644 --- a/installer/PowerToysSetupVNext/CmdPal.wxs +++ b/installer/PowerToysSetupVNext/CmdPal.wxs @@ -4,6 +4,13 @@ + + + + + + + @@ -18,14 +25,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/installer/PowerToysSetupVNext/Settings.wxs b/installer/PowerToysSetupVNext/Settings.wxs index f9e5312ea7..cf7cf7f727 100644 --- a/installer/PowerToysSetupVNext/Settings.wxs +++ b/installer/PowerToysSetupVNext/Settings.wxs @@ -14,11 +14,16 @@ + + + - + + + @@ -45,6 +50,11 @@ + + + + + @@ -67,6 +77,7 @@ + diff --git a/installer/PowerToysSetupVNext/generateAllFileComponents.ps1 b/installer/PowerToysSetupVNext/generateAllFileComponents.ps1 index 39020b4f04..6724d95170 100644 --- a/installer/PowerToysSetupVNext/generateAllFileComponents.ps1 +++ b/installer/PowerToysSetupVNext/generateAllFileComponents.ps1 @@ -307,10 +307,12 @@ Generate-FileList -fileDepsJson "" -fileListName SettingsV2AssetsFiles -wxsFileP Generate-FileList -fileDepsJson "" -fileListName SettingsV2AssetsModulesFiles -wxsFilePath $PSScriptRoot\Settings.wxs -depsPath "$PSScriptRoot..\..\..\$platform\Release\WinUI3Apps\Assets\Settings\Modules\" Generate-FileList -fileDepsJson "" -fileListName SettingsV2OOBEAssetsModulesFiles -wxsFilePath $PSScriptRoot\Settings.wxs -depsPath "$PSScriptRoot..\..\..\$platform\Release\WinUI3Apps\Assets\Settings\Modules\OOBE\" Generate-FileList -fileDepsJson "" -fileListName SettingsV2OOBEAssetsFluentIconsFiles -wxsFilePath $PSScriptRoot\Settings.wxs -depsPath "$PSScriptRoot..\..\..\$platform\Release\WinUI3Apps\Assets\Settings\Icons\" +Generate-FileList -fileDepsJson "" -fileListName SettingsV2IconsModelsFiles -wxsFilePath $PSScriptRoot\Settings.wxs -depsPath "$PSScriptRoot..\..\..\$platform\Release\WinUI3Apps\Assets\Settings\Icons\Models\" Generate-FileComponents -fileListName "SettingsV2AssetsFiles" -wxsFilePath $PSScriptRoot\Settings.wxs Generate-FileComponents -fileListName "SettingsV2AssetsModulesFiles" -wxsFilePath $PSScriptRoot\Settings.wxs Generate-FileComponents -fileListName "SettingsV2OOBEAssetsModulesFiles" -wxsFilePath $PSScriptRoot\Settings.wxs Generate-FileComponents -fileListName "SettingsV2OOBEAssetsFluentIconsFiles" -wxsFilePath $PSScriptRoot\Settings.wxs +Generate-FileComponents -fileListName "SettingsV2IconsModelsFiles" -wxsFilePath $PSScriptRoot\Settings.wxs #Workspaces Generate-FileList -fileDepsJson "" -fileListName WorkspacesImagesComponentFiles -wxsFilePath $PSScriptRoot\Workspaces.wxs -depsPath "$PSScriptRoot..\..\..\$platform\Release\Assets\Workspaces\" diff --git a/src/common/GPOWrapper/GPOWrapper.cpp b/src/common/GPOWrapper/GPOWrapper.cpp index 361255f66f..b8df132fe4 100644 --- a/src/common/GPOWrapper/GPOWrapper.cpp +++ b/src/common/GPOWrapper/GPOWrapper.cpp @@ -192,6 +192,38 @@ namespace winrt::PowerToys::GPOWrapper::implementation { return static_cast(powertoys_gpo::getAllowedAdvancedPasteOnlineAIModelsValue()); } + GpoRuleConfigured GPOWrapper::GetAllowedAdvancedPasteOpenAIValue() + { + return static_cast(powertoys_gpo::getAllowedAdvancedPasteOpenAIValue()); + } + GpoRuleConfigured GPOWrapper::GetAllowedAdvancedPasteAzureOpenAIValue() + { + return static_cast(powertoys_gpo::getAllowedAdvancedPasteAzureOpenAIValue()); + } + GpoRuleConfigured GPOWrapper::GetAllowedAdvancedPasteAzureAIInferenceValue() + { + return static_cast(powertoys_gpo::getAllowedAdvancedPasteAzureAIInferenceValue()); + } + GpoRuleConfigured GPOWrapper::GetAllowedAdvancedPasteMistralValue() + { + return static_cast(powertoys_gpo::getAllowedAdvancedPasteMistralValue()); + } + GpoRuleConfigured GPOWrapper::GetAllowedAdvancedPasteGoogleValue() + { + return static_cast(powertoys_gpo::getAllowedAdvancedPasteGoogleValue()); + } + GpoRuleConfigured GPOWrapper::GetAllowedAdvancedPasteAnthropicValue() + { + return static_cast(powertoys_gpo::getAllowedAdvancedPasteAnthropicValue()); + } + GpoRuleConfigured GPOWrapper::GetAllowedAdvancedPasteOllamaValue() + { + return static_cast(powertoys_gpo::getAllowedAdvancedPasteOllamaValue()); + } + GpoRuleConfigured GPOWrapper::GetAllowedAdvancedPasteFoundryLocalValue() + { + return static_cast(powertoys_gpo::getAllowedAdvancedPasteFoundryLocalValue()); + } GpoRuleConfigured GPOWrapper::GetConfiguredNewPlusEnabledValue() { return static_cast(powertoys_gpo::getConfiguredNewPlusEnabledValue()); diff --git a/src/common/GPOWrapper/GPOWrapper.h b/src/common/GPOWrapper/GPOWrapper.h index c0fff9f542..4f60a989db 100644 --- a/src/common/GPOWrapper/GPOWrapper.h +++ b/src/common/GPOWrapper/GPOWrapper.h @@ -54,6 +54,14 @@ namespace winrt::PowerToys::GPOWrapper::implementation static GpoRuleConfigured GetConfiguredQoiPreviewEnabledValue(); static GpoRuleConfigured GetConfiguredQoiThumbnailsEnabledValue(); static GpoRuleConfigured GetAllowedAdvancedPasteOnlineAIModelsValue(); + static GpoRuleConfigured GetAllowedAdvancedPasteOpenAIValue(); + static GpoRuleConfigured GetAllowedAdvancedPasteAzureOpenAIValue(); + static GpoRuleConfigured GetAllowedAdvancedPasteAzureAIInferenceValue(); + static GpoRuleConfigured GetAllowedAdvancedPasteMistralValue(); + static GpoRuleConfigured GetAllowedAdvancedPasteGoogleValue(); + static GpoRuleConfigured GetAllowedAdvancedPasteAnthropicValue(); + static GpoRuleConfigured GetAllowedAdvancedPasteOllamaValue(); + static GpoRuleConfigured GetAllowedAdvancedPasteFoundryLocalValue(); static GpoRuleConfigured GetConfiguredNewPlusEnabledValue(); static GpoRuleConfigured GetConfiguredWorkspacesEnabledValue(); static GpoRuleConfigured GetConfiguredMwbClipboardSharingEnabledValue(); diff --git a/src/common/GPOWrapper/GPOWrapper.idl b/src/common/GPOWrapper/GPOWrapper.idl index 630beab9c9..d1af719998 100644 --- a/src/common/GPOWrapper/GPOWrapper.idl +++ b/src/common/GPOWrapper/GPOWrapper.idl @@ -58,6 +58,14 @@ namespace PowerToys static GpoRuleConfigured GetConfiguredQoiPreviewEnabledValue(); static GpoRuleConfigured GetConfiguredQoiThumbnailsEnabledValue(); static GpoRuleConfigured GetAllowedAdvancedPasteOnlineAIModelsValue(); + static GpoRuleConfigured GetAllowedAdvancedPasteOpenAIValue(); + static GpoRuleConfigured GetAllowedAdvancedPasteAzureOpenAIValue(); + static GpoRuleConfigured GetAllowedAdvancedPasteAzureAIInferenceValue(); + static GpoRuleConfigured GetAllowedAdvancedPasteMistralValue(); + static GpoRuleConfigured GetAllowedAdvancedPasteGoogleValue(); + static GpoRuleConfigured GetAllowedAdvancedPasteAnthropicValue(); + static GpoRuleConfigured GetAllowedAdvancedPasteOllamaValue(); + static GpoRuleConfigured GetAllowedAdvancedPasteFoundryLocalValue(); static GpoRuleConfigured GetConfiguredNewPlusEnabledValue(); static GpoRuleConfigured GetConfiguredWorkspacesEnabledValue(); static GpoRuleConfigured GetConfiguredMwbClipboardSharingEnabledValue(); diff --git a/src/common/LanguageModelProvider/FoundryLocal/FoundryCachedModel.cs b/src/common/LanguageModelProvider/FoundryLocal/FoundryCachedModel.cs new file mode 100644 index 0000000000..489a779179 --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocal/FoundryCachedModel.cs @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace LanguageModelProvider.FoundryLocal; + +internal sealed record FoundryCachedModel(string Name, string? Id); diff --git a/src/common/LanguageModelProvider/FoundryLocal/FoundryCatalogModel.cs b/src/common/LanguageModelProvider/FoundryLocal/FoundryCatalogModel.cs new file mode 100644 index 0000000000..413bb47316 --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocal/FoundryCatalogModel.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text.Json.Serialization; + +namespace LanguageModelProvider.FoundryLocal; + +internal sealed record FoundryCatalogModel +{ + [JsonPropertyName("name")] + public string Name { get; init; } = string.Empty; + + [JsonPropertyName("displayName")] + public string DisplayName { get; init; } = string.Empty; + + [JsonPropertyName("providerType")] + public string ProviderType { get; init; } = string.Empty; + + [JsonPropertyName("uri")] + public string Uri { get; init; } = string.Empty; + + [JsonPropertyName("version")] + public string Version { get; init; } = string.Empty; + + [JsonPropertyName("modelType")] + public string ModelType { get; init; } = string.Empty; + + [JsonPropertyName("promptTemplate")] + public PromptTemplate PromptTemplate { get; init; } = default!; + + [JsonPropertyName("publisher")] + public string Publisher { get; init; } = string.Empty; + + [JsonPropertyName("task")] + public string Task { get; init; } = string.Empty; + + [JsonPropertyName("runtime")] + public Runtime Runtime { get; init; } = default!; + + [JsonPropertyName("fileSizeMb")] + public long FileSizeMb { get; init; } + + [JsonPropertyName("modelSettings")] + public ModelSettings ModelSettings { get; init; } = default!; + + [JsonPropertyName("alias")] + public string Alias { get; init; } = string.Empty; + + [JsonPropertyName("supportsToolCalling")] + public bool SupportsToolCalling { get; init; } + + [JsonPropertyName("license")] + public string License { get; init; } = string.Empty; + + [JsonPropertyName("licenseDescription")] + public string LicenseDescription { get; init; } = string.Empty; + + [JsonPropertyName("parentModelUri")] + public string ParentModelUri { get; init; } = string.Empty; +} diff --git a/src/common/LanguageModelProvider/FoundryLocal/FoundryClient.cs b/src/common/LanguageModelProvider/FoundryLocal/FoundryClient.cs new file mode 100644 index 0000000000..84c38c49f2 --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocal/FoundryClient.cs @@ -0,0 +1,208 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using ManagedCommon; +using Microsoft.AI.Foundry.Local; + +namespace LanguageModelProvider.FoundryLocal; + +internal sealed class FoundryClient +{ + public static async Task CreateAsync() + { + try + { + Logger.LogInfo("[FoundryClient] Creating Foundry Local client"); + + var manager = new FoundryLocalManager(); + + // Check if service is already running + if (manager.IsServiceRunning) + { + Logger.LogInfo("[FoundryClient] Foundry service is already running"); + return new FoundryClient(manager); + } + + // Start the service using SDK's method + Logger.LogInfo("[FoundryClient] Starting Foundry service using manager.StartServiceAsync()"); + await manager.StartServiceAsync().ConfigureAwait(false); + + Logger.LogInfo("[FoundryClient] Foundry service started successfully"); + return new FoundryClient(manager); + } + catch (Exception ex) + { + Logger.LogError($"[FoundryClient] Error creating client: {ex.Message}"); + if (ex.InnerException != null) + { + Logger.LogError($"[FoundryClient] Inner exception: {ex.InnerException.Message}"); + } + + return null; + } + } + + private readonly FoundryLocalManager _foundryManager; + private readonly List _catalogModels = []; + + private FoundryClient(FoundryLocalManager foundryManager) + { + _foundryManager = foundryManager; + } + + public Task GetServiceUrl() + { + try + { + return Task.FromResult(_foundryManager.Endpoint?.ToString()); + } + catch + { + return Task.FromResult(null); + } + } + + public Uri? GetServiceUri() + { + try + { + return _foundryManager.ServiceUri; + } + catch + { + return null; + } + } + + public async Task> ListCatalogModels() + { + if (_catalogModels.Count > 0) + { + return _catalogModels; + } + + try + { + Logger.LogInfo("[FoundryClient] Listing catalog models"); + var models = await _foundryManager.ListCatalogModelsAsync().ConfigureAwait(false); + + if (models != null) + { + foreach (var model in models) + { + _catalogModels.Add(new FoundryCatalogModel + { + Name = model.ModelId ?? string.Empty, + DisplayName = model.DisplayName ?? string.Empty, + ProviderType = model.ProviderType ?? string.Empty, + Uri = model.Uri ?? string.Empty, + Version = model.Version ?? string.Empty, + ModelType = model.ModelType ?? string.Empty, + Publisher = model.Publisher ?? string.Empty, + Task = model.Task ?? string.Empty, + FileSizeMb = model.FileSizeMb, + Alias = model.Alias ?? string.Empty, + License = model.License ?? string.Empty, + LicenseDescription = model.LicenseDescription ?? string.Empty, + ParentModelUri = model.ParentModelUri ?? string.Empty, + SupportsToolCalling = model.SupportsToolCalling, + }); + } + + Logger.LogInfo($"[FoundryClient] Found {_catalogModels.Count} catalog models"); + } + } + catch (Exception ex) + { + Logger.LogError($"[FoundryClient] Error listing catalog models: {ex.Message}"); + + // Surfacing errors here prevents listing other providers; swallow and return cached list instead. + } + + return _catalogModels; + } + + public async Task> ListCachedModels() + { + try + { + Logger.LogInfo("[FoundryClient] Listing cached models"); + var cachedModels = await _foundryManager.ListCachedModelsAsync().ConfigureAwait(false); + var catalogModels = await ListCatalogModels().ConfigureAwait(false); + + List models = []; + + foreach (var model in cachedModels) + { + var catalogModel = catalogModels.FirstOrDefault(m => m.Name == model.ModelId); + var alias = catalogModel?.Alias ?? model.Alias; + models.Add(new FoundryCachedModel(model.ModelId ?? string.Empty, alias)); + } + + Logger.LogInfo($"[FoundryClient] Found {models.Count} cached models"); + return models; + } + catch (Exception ex) + { + Logger.LogError($"[FoundryClient] Error listing cached models: {ex.Message}"); + return []; + } + } + + public async Task IsModelLoaded(string modelId) + { + try + { + var loadedModels = await _foundryManager.ListLoadedModelsAsync().ConfigureAwait(false); + var isLoaded = loadedModels.Any(m => m.ModelId == modelId); + Logger.LogInfo($"[FoundryClient] IsModelLoaded({modelId}): {isLoaded}"); + Logger.LogInfo($"[FoundryClient] Loaded models: {string.Join(", ", loadedModels.Select(m => m.ModelId))}"); + return isLoaded; + } + catch (Exception ex) + { + Logger.LogError($"[FoundryClient] IsModelLoaded exception: {ex.Message}"); + return false; + } + } + + public async Task EnsureModelLoaded(string modelId) + { + try + { + Logger.LogInfo($"[FoundryClient] EnsureModelLoaded called with: {modelId}"); + + // Check if already loaded + if (await IsModelLoaded(modelId).ConfigureAwait(false)) + { + Logger.LogInfo($"[FoundryClient] Model already loaded: {modelId}"); + return true; + } + + // Check if model exists in cache + var cachedModels = await ListCachedModels().ConfigureAwait(false); + Logger.LogInfo($"[FoundryClient] Cached models: {string.Join(", ", cachedModels.Select(m => m.Name))}"); + + if (!cachedModels.Any(m => m.Name == modelId)) + { + Logger.LogWarning($"[FoundryClient] Model not found in cache: {modelId}"); + return false; + } + + // Load the model + Logger.LogInfo($"[FoundryClient] Loading model: {modelId}"); + await _foundryManager.LoadModelAsync(modelId).ConfigureAwait(false); + + // Verify it's loaded + var loaded = await IsModelLoaded(modelId).ConfigureAwait(false); + Logger.LogInfo($"[FoundryClient] Model load result: {loaded}"); + return loaded; + } + catch (Exception ex) + { + Logger.LogError($"[FoundryClient] EnsureModelLoaded exception: {ex.Message}"); + return false; + } + } +} diff --git a/src/common/LanguageModelProvider/FoundryLocal/FoundryJsonContext.cs b/src/common/LanguageModelProvider/FoundryLocal/FoundryJsonContext.cs new file mode 100644 index 0000000000..5dcb4076ed --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocal/FoundryJsonContext.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace LanguageModelProvider.FoundryLocal; + +[JsonSourceGenerationOptions( + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + WriteIndented = false)] +[JsonSerializable(typeof(FoundryCatalogModel))] +[JsonSerializable(typeof(List))] +internal sealed partial class FoundryJsonContext : JsonSerializerContext +{ +} diff --git a/src/common/LanguageModelProvider/FoundryLocal/ModelSettings.cs b/src/common/LanguageModelProvider/FoundryLocal/ModelSettings.cs new file mode 100644 index 0000000000..fda91217eb --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocal/ModelSettings.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace LanguageModelProvider.FoundryLocal; + +internal sealed record ModelSettings +{ + // The sample shows an empty array; keep it open-ended. + [JsonPropertyName("parameters")] + public List Parameters { get; init; } = []; +} diff --git a/src/common/LanguageModelProvider/FoundryLocal/PromptTemplate.cs b/src/common/LanguageModelProvider/FoundryLocal/PromptTemplate.cs new file mode 100644 index 0000000000..a2cbb9fe45 --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocal/PromptTemplate.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text.Json.Serialization; + +namespace LanguageModelProvider.FoundryLocal; + +internal sealed record PromptTemplate +{ + [JsonPropertyName("assistant")] + public string Assistant { get; init; } = string.Empty; + + [JsonPropertyName("prompt")] + public string Prompt { get; init; } = string.Empty; +} diff --git a/src/common/LanguageModelProvider/FoundryLocal/Runtime.cs b/src/common/LanguageModelProvider/FoundryLocal/Runtime.cs new file mode 100644 index 0000000000..e2019c8f87 --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocal/Runtime.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text.Json.Serialization; + +namespace LanguageModelProvider.FoundryLocal; + +internal sealed record Runtime +{ + [JsonPropertyName("deviceType")] + public string DeviceType { get; init; } = string.Empty; + + [JsonPropertyName("executionProvider")] + public string ExecutionProvider { get; init; } = string.Empty; +} diff --git a/src/common/LanguageModelProvider/FoundryLocalModelProvider.cs b/src/common/LanguageModelProvider/FoundryLocalModelProvider.cs new file mode 100644 index 0000000000..b866ab05d9 --- /dev/null +++ b/src/common/LanguageModelProvider/FoundryLocalModelProvider.cs @@ -0,0 +1,185 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.ClientModel; +using LanguageModelProvider.FoundryLocal; +using ManagedCommon; +using Microsoft.Extensions.AI; +using OpenAI; + +namespace LanguageModelProvider; + +public sealed class FoundryLocalModelProvider : ILanguageModelProvider +{ + private IEnumerable? _downloadedModels; + private FoundryClient? _foundryManager; + private string? _serviceUrl; + + public static FoundryLocalModelProvider Instance { get; } = new(); + + public string Name => "FoundryLocal"; + + public string ProviderDescription => "The model will run locally via Foundry Local"; + + public string UrlPrefix => "fl://"; + + public IChatClient? GetIChatClient(string url) + { + try + { + Logger.LogInfo($"[FoundryLocal] GetIChatClient called with url: {url}"); + InitializeAsync().GetAwaiter().GetResult(); + } + catch (Exception ex) + { + Logger.LogError($"[FoundryLocal] Failed to initialize: {ex.Message}"); + return null; + } + + if (string.IsNullOrWhiteSpace(_serviceUrl) || _foundryManager == null) + { + Logger.LogError("[FoundryLocal] Service URL or manager is null"); + return null; + } + + // Extract model ID from URL (format: fl://modelname) + var modelId = url.Replace(UrlPrefix, string.Empty).Trim('/'); + if (string.IsNullOrWhiteSpace(modelId)) + { + Logger.LogError("[FoundryLocal] Model ID is empty after extraction"); + return null; + } + + Logger.LogInfo($"[FoundryLocal] Extracted model ID: {modelId}"); + + // Ensure the model is loaded before returning chat client + try + { + var isLoaded = _foundryManager.EnsureModelLoaded(modelId).GetAwaiter().GetResult(); + if (!isLoaded) + { + Logger.LogError($"[FoundryLocal] Failed to load model: {modelId}"); + return null; + } + + Logger.LogInfo($"[FoundryLocal] Model is loaded: {modelId}"); + } + catch (Exception ex) + { + Logger.LogError($"[FoundryLocal] Exception ensuring model loaded: {ex.Message}"); + return null; + } + + // Use ServiceUri instead of Endpoint since Endpoint already includes /v1 + var baseUri = _foundryManager.GetServiceUri(); + if (baseUri == null) + { + Logger.LogError("[FoundryLocal] Service URI is null"); + return null; + } + + var endpointUri = new Uri($"{baseUri.ToString().TrimEnd('/')}/v1"); + Logger.LogInfo($"[FoundryLocal] Creating OpenAI client with endpoint: {endpointUri}"); + Logger.LogInfo($"[FoundryLocal] Model ID for chat client: {modelId}"); + + return new OpenAIClient( + new ApiKeyCredential("none"), + new OpenAIClientOptions { Endpoint = endpointUri }) + .GetChatClient(modelId) + .AsIChatClient(); + } + + public string GetIChatClientString(string url) + { + try + { + InitializeAsync().GetAwaiter().GetResult(); + } + catch + { + return string.Empty; + } + + var modelId = url.Split('/').LastOrDefault(); + + if (string.IsNullOrWhiteSpace(_serviceUrl) || string.IsNullOrWhiteSpace(modelId)) + { + return string.Empty; + } + + return $"new OpenAIClient(new ApiKeyCredential(\"none\"), new OpenAIClientOptions{{ Endpoint = new Uri(\"{_serviceUrl}/v1\") }}).GetChatClient(\"{modelId}\").AsIChatClient()"; + } + + public async Task> GetModelsAsync(bool ignoreCached = false, CancellationToken cancelationToken = default) + { + if (ignoreCached) + { + Logger.LogInfo("[FoundryLocal] Ignoring cached models, resetting"); + Reset(); + } + + await InitializeAsync(cancelationToken); + + Logger.LogInfo($"[FoundryLocal] Returning {_downloadedModels?.Count() ?? 0} downloaded models"); + return _downloadedModels ?? []; + } + + private void Reset() + { + _downloadedModels = null; + _ = InitializeAsync(); + } + + private async Task InitializeAsync(CancellationToken cancelationToken = default) + { + if (_foundryManager != null && _downloadedModels != null && _downloadedModels.Any()) + { + return; + } + + Logger.LogInfo("[FoundryLocal] Initializing provider"); + _foundryManager ??= await FoundryClient.CreateAsync(); + + if (_foundryManager == null) + { + Logger.LogError("[FoundryLocal] Failed to create Foundry client"); + return; + } + + _serviceUrl ??= await _foundryManager.GetServiceUrl(); + Logger.LogInfo($"[FoundryLocal] Service URL: {_serviceUrl}"); + + var cachedModels = await _foundryManager.ListCachedModels(); + Logger.LogInfo($"[FoundryLocal] Found {cachedModels.Count} cached models"); + + List downloadedModels = []; + + foreach (var model in cachedModels) + { + Logger.LogInfo($"[FoundryLocal] Adding unmatched cached model: {model.Name}"); + downloadedModels.Add(new ModelDetails + { + Id = $"fl-{model.Name}", + Name = model.Name, + Url = $"{UrlPrefix}{model.Name}", + Description = $"{model.Name} running locally with Foundry Local", + HardwareAccelerators = [HardwareAccelerator.FOUNDRYLOCAL], + SupportedOnQualcomm = true, + ProviderModelDetails = model, + }); + } + + _downloadedModels = downloadedModels; + Logger.LogInfo($"[FoundryLocal] Initialization complete. Total downloaded models: {downloadedModels.Count}"); + } + + public async Task IsAvailable() + { + Logger.LogInfo("[FoundryLocal] Checking availability"); + await InitializeAsync(); + var available = _foundryManager != null; + Logger.LogInfo($"[FoundryLocal] Available: {available}"); + return available; + } +} diff --git a/src/common/LanguageModelProvider/HardwareAccelerator.cs b/src/common/LanguageModelProvider/HardwareAccelerator.cs new file mode 100644 index 0000000000..d2c94b8155 --- /dev/null +++ b/src/common/LanguageModelProvider/HardwareAccelerator.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace LanguageModelProvider; + +public enum HardwareAccelerator +{ + CPU, + DML, + QNN, + WCRAPI, + OLLAMA, + OPENAI, + FOUNDRYLOCAL, + LEMONADE, + NPU, + GPU, + VitisAI, + OpenVINO, + NvTensorRT, +} diff --git a/src/common/LanguageModelProvider/ILanguageModelProvider.cs b/src/common/LanguageModelProvider/ILanguageModelProvider.cs new file mode 100644 index 0000000000..60b3d99386 --- /dev/null +++ b/src/common/LanguageModelProvider/ILanguageModelProvider.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Extensions.AI; + +namespace LanguageModelProvider; + +public interface ILanguageModelProvider +{ + string Name { get; } + + string UrlPrefix { get; } + + string ProviderDescription { get; } + + Task> GetModelsAsync(bool ignoreCached = false, CancellationToken cancelationToken = default); + + IChatClient? GetIChatClient(string url); + + string GetIChatClientString(string url); +} diff --git a/src/common/LanguageModelProvider/LanguageModelProvider.csproj b/src/common/LanguageModelProvider/LanguageModelProvider.csproj new file mode 100644 index 0000000000..4dba9247a3 --- /dev/null +++ b/src/common/LanguageModelProvider/LanguageModelProvider.csproj @@ -0,0 +1,20 @@ + + + + + + enable + enable + + + + + + + + + + + + + diff --git a/src/common/LanguageModelProvider/LanguageModelService.cs b/src/common/LanguageModelProvider/LanguageModelService.cs new file mode 100644 index 0000000000..1cfb3b3c49 --- /dev/null +++ b/src/common/LanguageModelProvider/LanguageModelService.cs @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Concurrent; +using Microsoft.Extensions.AI; + +namespace LanguageModelProvider; + +public sealed class LanguageModelService +{ + private readonly ConcurrentDictionary _providersByPrefix; + + public LanguageModelService(IEnumerable providers) + { + ArgumentNullException.ThrowIfNull(providers); + + _providersByPrefix = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var provider in providers) + { + if (!string.IsNullOrWhiteSpace(provider.UrlPrefix)) + { + _providersByPrefix[provider.UrlPrefix] = provider; + } + } + } + + public static LanguageModelService CreateDefault() + { + return new LanguageModelService(new[] + { + FoundryLocalModelProvider.Instance, + }); + } + + public IReadOnlyCollection Providers => _providersByPrefix.Values.ToArray(); + + public bool RegisterProvider(ILanguageModelProvider provider) + { + ArgumentNullException.ThrowIfNull(provider); + + if (string.IsNullOrWhiteSpace(provider.UrlPrefix)) + { + throw new ArgumentException("Provider must supply a URL prefix.", nameof(provider)); + } + + _providersByPrefix[provider.UrlPrefix] = provider; + return true; + } + + public ILanguageModelProvider? GetProviderFor(string? modelReference) + { + if (string.IsNullOrWhiteSpace(modelReference)) + { + return null; + } + + foreach (var provider in _providersByPrefix.Values) + { + if (modelReference.StartsWith(provider.UrlPrefix, StringComparison.OrdinalIgnoreCase)) + { + return provider; + } + } + + return null; + } + + public async Task> GetModelsAsync(bool refresh = false, CancellationToken cancellationToken = default) + { + List models = []; + + foreach (var provider in _providersByPrefix.Values) + { + cancellationToken.ThrowIfCancellationRequested(); + var providerModels = await provider.GetModelsAsync(refresh, cancellationToken).ConfigureAwait(false); + models.AddRange(providerModels); + } + + return models; + } + + public IChatClient? GetClient(ModelDetails model) + { + if (model is null) + { + return null; + } + + var reference = !string.IsNullOrWhiteSpace(model.Url) ? model.Url : model.Id; + return GetClient(reference); + } + + public IChatClient? GetClient(string? modelReference) + { + if (string.IsNullOrWhiteSpace(modelReference)) + { + return null; + } + + var provider = GetProviderFor(modelReference); + + return provider?.GetIChatClient(modelReference); + } +} diff --git a/src/common/LanguageModelProvider/ModelDetails.cs b/src/common/LanguageModelProvider/ModelDetails.cs new file mode 100644 index 0000000000..2e68ca6feb --- /dev/null +++ b/src/common/LanguageModelProvider/ModelDetails.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; + +namespace LanguageModelProvider; + +public class ModelDetails +{ + public string Id { get; set; } = string.Empty; + + public string Name { get; set; } = string.Empty; + + public string Url { get; set; } = string.Empty; + + public string Description { get; set; } = string.Empty; + + public long Size { get; set; } + + public bool IsUserAdded { get; set; } + + public string Icon { get; set; } = string.Empty; + + public List HardwareAccelerators { get; set; } = []; + + public bool SupportedOnQualcomm { get; set; } + + public string License { get; set; } = string.Empty; + + public object? ProviderModelDetails { get; set; } +} diff --git a/src/common/utils/gpo.h b/src/common/utils/gpo.h index 471cefe480..db1e0f72e8 100644 --- a/src/common/utils/gpo.h +++ b/src/common/utils/gpo.h @@ -82,6 +82,14 @@ namespace powertoys_gpo const std::wstring POLICY_CONFIGURE_RUN_AT_STARTUP = L"ConfigureRunAtStartup"; const std::wstring POLICY_CONFIGURE_ENABLED_POWER_LAUNCHER_ALL_PLUGINS = L"PowerLauncherAllPluginsEnabledState"; const std::wstring POLICY_ALLOW_ADVANCED_PASTE_ONLINE_AI_MODELS = L"AllowPowerToysAdvancedPasteOnlineAIModels"; + const std::wstring POLICY_ALLOW_ADVANCED_PASTE_OPENAI = L"AllowAdvancedPasteOpenAI"; + const std::wstring POLICY_ALLOW_ADVANCED_PASTE_AZURE_OPENAI = L"AllowAdvancedPasteAzureOpenAI"; + const std::wstring POLICY_ALLOW_ADVANCED_PASTE_AZURE_AI_INFERENCE = L"AllowAdvancedPasteAzureAIInference"; + const std::wstring POLICY_ALLOW_ADVANCED_PASTE_MISTRAL = L"AllowAdvancedPasteMistral"; + const std::wstring POLICY_ALLOW_ADVANCED_PASTE_GOOGLE = L"AllowAdvancedPasteGoogle"; + const std::wstring POLICY_ALLOW_ADVANCED_PASTE_ANTHROPIC = L"AllowAdvancedPasteAnthropic"; + const std::wstring POLICY_ALLOW_ADVANCED_PASTE_OLLAMA = L"AllowAdvancedPasteOllama"; + const std::wstring POLICY_ALLOW_ADVANCED_PASTE_FOUNDRY_LOCAL = L"AllowAdvancedPasteFoundryLocal"; const std::wstring POLICY_MWB_CLIPBOARD_SHARING_ENABLED = L"MwbClipboardSharingEnabled"; const std::wstring POLICY_MWB_FILE_TRANSFER_ENABLED = L"MwbFileTransferEnabled"; const std::wstring POLICY_MWB_USE_ORIGINAL_USER_INTERFACE = L"MwbUseOriginalUserInterface"; @@ -575,6 +583,46 @@ namespace powertoys_gpo return getConfiguredValue(POLICY_ALLOW_ADVANCED_PASTE_ONLINE_AI_MODELS); } + inline gpo_rule_configured_t getAllowedAdvancedPasteOpenAIValue() + { + return getConfiguredValue(POLICY_ALLOW_ADVANCED_PASTE_OPENAI); + } + + inline gpo_rule_configured_t getAllowedAdvancedPasteAzureOpenAIValue() + { + return getConfiguredValue(POLICY_ALLOW_ADVANCED_PASTE_AZURE_OPENAI); + } + + inline gpo_rule_configured_t getAllowedAdvancedPasteAzureAIInferenceValue() + { + return getConfiguredValue(POLICY_ALLOW_ADVANCED_PASTE_AZURE_AI_INFERENCE); + } + + inline gpo_rule_configured_t getAllowedAdvancedPasteMistralValue() + { + return getConfiguredValue(POLICY_ALLOW_ADVANCED_PASTE_MISTRAL); + } + + inline gpo_rule_configured_t getAllowedAdvancedPasteGoogleValue() + { + return getConfiguredValue(POLICY_ALLOW_ADVANCED_PASTE_GOOGLE); + } + + inline gpo_rule_configured_t getAllowedAdvancedPasteAnthropicValue() + { + return getConfiguredValue(POLICY_ALLOW_ADVANCED_PASTE_ANTHROPIC); + } + + inline gpo_rule_configured_t getAllowedAdvancedPasteOllamaValue() + { + return getConfiguredValue(POLICY_ALLOW_ADVANCED_PASTE_OLLAMA); + } + + inline gpo_rule_configured_t getAllowedAdvancedPasteFoundryLocalValue() + { + return getConfiguredValue(POLICY_ALLOW_ADVANCED_PASTE_FOUNDRY_LOCAL); + } + inline gpo_rule_configured_t getConfiguredMwbClipboardSharingEnabledValue() { return getConfiguredValue(POLICY_MWB_CLIPBOARD_SHARING_ENABLED); diff --git a/src/dsc/PowerToys.Settings.DSC.Schema.Generator/PowerToys.Settings.DSC.Schema.Generator.csproj b/src/dsc/PowerToys.Settings.DSC.Schema.Generator/PowerToys.Settings.DSC.Schema.Generator.csproj index 3186a01d43..b36e602d25 100644 --- a/src/dsc/PowerToys.Settings.DSC.Schema.Generator/PowerToys.Settings.DSC.Schema.Generator.csproj +++ b/src/dsc/PowerToys.Settings.DSC.Schema.Generator/PowerToys.Settings.DSC.Schema.Generator.csproj @@ -33,9 +33,4 @@ - - - - - diff --git a/src/dsc/v3/PowerToys.DSC.UnitTests/SettingsResourceTests/SettingsResourceAdvancedPasteModuleTest.cs b/src/dsc/v3/PowerToys.DSC.UnitTests/SettingsResourceTests/SettingsResourceAdvancedPasteModuleTest.cs index deae2eb832..45fd36d10e 100644 --- a/src/dsc/v3/PowerToys.DSC.UnitTests/SettingsResourceTests/SettingsResourceAdvancedPasteModuleTest.cs +++ b/src/dsc/v3/PowerToys.DSC.UnitTests/SettingsResourceTests/SettingsResourceAdvancedPasteModuleTest.cs @@ -23,7 +23,8 @@ public sealed class SettingsResourceAdvancedPasteModuleTest : SettingsResourceMo { s.Properties.ShowCustomPreview = !s.Properties.ShowCustomPreview; s.Properties.CloseAfterLosingFocus = !s.Properties.CloseAfterLosingFocus; - s.Properties.IsAdvancedAIEnabled = !s.Properties.IsAdvancedAIEnabled; + + // s.Properties.IsAdvancedAIEnabled = !s.Properties.IsAdvancedAIEnabled; s.Properties.AdvancedPasteUIShortcut = new HotkeySettings { Key = "mock", diff --git a/src/gpo/assets/PowerToys.admx b/src/gpo/assets/PowerToys.admx index 07d4f44bde..4b77a6783f 100644 --- a/src/gpo/assets/PowerToys.admx +++ b/src/gpo/assets/PowerToys.admx @@ -1,11 +1,11 @@ - + - + @@ -26,6 +26,7 @@ + @@ -614,6 +615,86 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/gpo/assets/en-US/PowerToys.adml b/src/gpo/assets/en-US/PowerToys.adml index 2703358bb0..1bfa55866d 100644 --- a/src/gpo/assets/en-US/PowerToys.adml +++ b/src/gpo/assets/en-US/PowerToys.adml @@ -1,7 +1,7 @@ - + PowerToys PowerToys @@ -33,6 +33,7 @@ PowerToys version 0.88.0 or later PowerToys version 0.89.0 or later PowerToys version 0.90.0 or later + PowerToys version 0.96.0 or later From PowerToys version 0.64.0 until PowerToys version 0.87.1 This policy configures the enabled state for all PowerToys utilities. @@ -291,6 +292,54 @@ If you don't configure this policy, the user will be able to control the setting QOI file preview: Configure enabled state QOI file thumbnail: Configure enabled state Allow using online AI models + Advanced Paste: Allow OpenAI endpoint + This policy controls whether users can use the OpenAI endpoint in Advanced Paste. + +If you enable or don't configure this policy, users can configure and use OpenAI as their AI provider. + +If you disable this policy, users will not be able to select or use OpenAI endpoint in Advanced Paste settings. + Advanced Paste: Allow Azure OpenAI endpoint + This policy controls whether users can use the Azure OpenAI endpoint in Advanced Paste. + +If you enable or don't configure this policy, users can configure and use Azure OpenAI as their AI provider. + +If you disable this policy, users will not be able to select or use Azure OpenAI endpoint in Advanced Paste settings. + Advanced Paste: Allow Azure AI Inference endpoint + This policy controls whether users can use the Azure AI Inference endpoint in Advanced Paste. + +If you enable or don't configure this policy, users can configure and use Azure AI Inference as their AI provider. + +If you disable this policy, users will not be able to select or use Azure AI Inference endpoint in Advanced Paste settings. + Advanced Paste: Allow Mistral endpoint + This policy controls whether users can use the Mistral AI endpoint in Advanced Paste. + +If you enable or don't configure this policy, users can configure and use Mistral as their AI provider. + +If you disable this policy, users will not be able to select or use Mistral endpoint in Advanced Paste settings. + Advanced Paste: Allow Google endpoint + This policy controls whether users can use the Google (Gemini) endpoint in Advanced Paste. + +If you enable or don't configure this policy, users can configure and use Google as their AI provider. + +If you disable this policy, users will not be able to select or use Google endpoint in Advanced Paste settings. + Advanced Paste: Allow Anthropic endpoint + This policy controls whether users can use the Anthropic (Claude) endpoint in Advanced Paste. + +If you enable or don't configure this policy, users can configure and use Anthropic as their AI provider. + +If you disable this policy, users will not be able to select or use Anthropic endpoint in Advanced Paste settings. + Advanced Paste: Allow Ollama endpoint + This policy controls whether users can use the Ollama local model endpoint in Advanced Paste. + +If you enable or don't configure this policy, users can configure and use Ollama as their AI provider. + +If you disable this policy, users will not be able to select or use Ollama endpoint in Advanced Paste settings. + Advanced Paste: Allow Foundry Local endpoint + This policy controls whether users can use the Foundry Local model endpoint in Advanced Paste. + +If you enable or don't configure this policy, users can configure and use Foundry Local as their AI provider. + +If you disable this policy, users will not be able to select or use Foundry Local endpoint in Advanced Paste settings. Clipboard sharing enabled File transfer enabled Original user interface is available diff --git a/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Mocks/IntegrationTestUserSettings.cs b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Mocks/IntegrationTestUserSettings.cs new file mode 100644 index 0000000000..f6c2f5098d --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Mocks/IntegrationTestUserSettings.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Threading.Tasks; +using AdvancedPaste.Models; +using AdvancedPaste.Settings; +using Microsoft.PowerToys.Settings.UI.Library; + +namespace AdvancedPaste.UnitTests.Mocks; + +/// +/// Minimal implementation used by integration tests that +/// need to construct the runtime Advanced Paste services. +/// +internal sealed class IntegrationTestUserSettings : IUserSettings +{ + private readonly PasteAIConfiguration _configuration; + private readonly IReadOnlyList _customActions; + private readonly IReadOnlyList _additionalActions; + + public IntegrationTestUserSettings() + { + var provider = new PasteAIProviderDefinition + { + Id = "integration-openai", + EnableAdvancedAI = true, + ServiceTypeKind = AIServiceType.OpenAI, + ModelName = "gpt-4o", + ModerationEnabled = true, + }; + + _configuration = new PasteAIConfiguration + { + ActiveProviderId = provider.Id, + Providers = new ObservableCollection { provider }, + }; + + _customActions = Array.Empty(); + _additionalActions = Array.Empty(); + } + + public bool IsAIEnabled => true; + + public bool ShowCustomPreview => false; + + public bool CloseAfterLosingFocus => false; + + public IReadOnlyList CustomActions => _customActions; + + public IReadOnlyList AdditionalActions => _additionalActions; + + public PasteAIConfiguration PasteAIConfiguration => _configuration; + + public event EventHandler Changed; + + public Task SetActiveAIProviderAsync(string providerId) + { + _configuration.ActiveProviderId = providerId ?? string.Empty; + Changed?.Invoke(this, EventArgs.Empty); + return Task.CompletedTask; + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/AIServiceBatchIntegrationTests.cs b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/AIServiceBatchIntegrationTests.cs index 3782b057f1..17b8139bad 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/AIServiceBatchIntegrationTests.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/AIServiceBatchIntegrationTests.cs @@ -13,6 +13,8 @@ using System.Threading.Tasks; using AdvancedPaste.Helpers; using AdvancedPaste.Models; +using AdvancedPaste.Services; +using AdvancedPaste.Services.CustomActions; using AdvancedPaste.Services.OpenAI; using AdvancedPaste.UnitTests.Mocks; using ManagedCommon; @@ -79,7 +81,9 @@ public sealed class AIServiceBatchIntegrationTests Assert.IsTrue(results.Count <= inputs.Count); CollectionAssert.AreEqual(results.Select(result => result.ToInput()).ToList(), inputs.Take(results.Count).ToList()); + #pragma warning disable IL2026, IL3050 // The tests rely on runtime JSON serialization for ad-hoc data files. async Task WriteResultsAsync() => await File.WriteAllTextAsync(resultsFile, JsonSerializer.Serialize(results, SerializerOptions)); + #pragma warning restore IL2026, IL3050 Logger.LogInfo($"Starting {nameof(TestGenerateBatchResults)}; Count={inputs.Count}, InCache={results.Count}"); @@ -101,8 +105,12 @@ public sealed class AIServiceBatchIntegrationTests await WriteResultsAsync(); } - private static async Task> GetDataListAsync(string filePath) => - File.Exists(filePath) ? JsonSerializer.Deserialize>(await File.ReadAllTextAsync(filePath)) : []; + private static async Task> GetDataListAsync(string filePath) + { + #pragma warning disable IL2026, IL3050 // Tests only run locally and can depend on runtime JSON serialization. + return File.Exists(filePath) ? JsonSerializer.Deserialize>(await File.ReadAllTextAsync(filePath)) : []; + #pragma warning restore IL2026, IL3050 + } private static async Task GetTextOutputAsync(BatchTestInput input, PasteFormats format) { @@ -130,23 +138,35 @@ public sealed class AIServiceBatchIntegrationTests private static async Task GetOutputDataPackageAsync(BatchTestInput batchTestInput, PasteFormats format) { - VaultCredentialsProvider credentialsProvider = new(); - PromptModerationService promptModerationService = new(credentialsProvider); + var services = CreateServices(); NoOpProgress progress = new(); - CustomTextTransformService customTextTransformService = new(credentialsProvider, promptModerationService); switch (format) { case PasteFormats.CustomTextTransformation: - return DataPackageHelpers.CreateFromText(await customTextTransformService.TransformTextAsync(batchTestInput.Prompt, batchTestInput.Clipboard, CancellationToken.None, progress)); + var transformResult = await services.CustomActionTransformService.TransformTextAsync(batchTestInput.Prompt, batchTestInput.Clipboard, CancellationToken.None, progress); + return DataPackageHelpers.CreateFromText(transformResult.Content ?? string.Empty); case PasteFormats.KernelQuery: var clipboardData = DataPackageHelpers.CreateFromText(batchTestInput.Clipboard).GetView(); - KernelService kernelService = new(new NoOpKernelQueryCacheService(), credentialsProvider, promptModerationService, customTextTransformService); - return await kernelService.TransformClipboardAsync(batchTestInput.Prompt, clipboardData, isSavedQuery: false, CancellationToken.None, progress); + return await services.KernelService.TransformClipboardAsync(batchTestInput.Prompt, clipboardData, isSavedQuery: false, CancellationToken.None, progress); default: throw new InvalidOperationException($"Unexpected format {format}"); } } + + private static IntegrationTestServices CreateServices() + { + IntegrationTestUserSettings userSettings = new(); + EnhancedVaultCredentialsProvider credentialsProvider = new(userSettings); + PromptModerationService promptModerationService = new(credentialsProvider); + PasteAIProviderFactory providerFactory = new(); + ICustomActionTransformService customActionTransformService = new CustomActionTransformService(promptModerationService, providerFactory, credentialsProvider, userSettings); + IKernelService kernelService = new AdvancedAIKernelService(credentialsProvider, new NoOpKernelQueryCacheService(), promptModerationService, userSettings, customActionTransformService); + + return new IntegrationTestServices(customActionTransformService, kernelService); + } + + private readonly record struct IntegrationTestServices(ICustomActionTransformService CustomActionTransformService, IKernelService KernelService); } diff --git a/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/KernelServiceIntegrationTests.cs b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/KernelServiceIntegrationTests.cs index 998534cf5e..7c16089cd5 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/KernelServiceIntegrationTests.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/KernelServiceIntegrationTests.cs @@ -11,6 +11,8 @@ using System.Threading.Tasks; using AdvancedPaste.Helpers; using AdvancedPaste.Models; +using AdvancedPaste.Services; +using AdvancedPaste.Services.CustomActions; using AdvancedPaste.Services.OpenAI; using AdvancedPaste.Telemetry; using AdvancedPaste.UnitTests.Mocks; @@ -27,16 +29,19 @@ namespace AdvancedPaste.UnitTests.ServicesTests; public sealed class KernelServiceIntegrationTests : IDisposable { private const string StandardImageFile = "image_with_text_example.png"; - private KernelService _kernelService; + private IKernelService _kernelService; private AdvancedPasteEventListener _eventListener; [TestInitialize] public void TestInitialize() { - VaultCredentialsProvider credentialsProvider = new(); + IntegrationTestUserSettings userSettings = new(); + EnhancedVaultCredentialsProvider credentialsProvider = new(userSettings); PromptModerationService promptModerationService = new(credentialsProvider); + PasteAIProviderFactory providerFactory = new(); + CustomActionTransformService customActionTransformService = new(promptModerationService, providerFactory, credentialsProvider, userSettings); - _kernelService = new KernelService(new NoOpKernelQueryCacheService(), credentialsProvider, promptModerationService, new CustomTextTransformService(credentialsProvider, promptModerationService)); + _kernelService = new AdvancedAIKernelService(credentialsProvider, new NoOpKernelQueryCacheService(), promptModerationService, userSettings, customActionTransformService); _eventListener = new(); } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.csproj b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.csproj index fba18de07c..54c1343c93 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.csproj +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.csproj @@ -33,11 +33,14 @@ + + + @@ -49,7 +52,6 @@ - @@ -57,10 +59,17 @@ + + + + + + + @@ -102,6 +111,7 @@ + @@ -114,9 +124,38 @@ true + + + MSBuild:Compile + + MSBuild:Compile + + + MSBuild:Compile + + + + + + + Assets\Settings\Icons\Models\%(Filename)%(Extension) + PreserveNewest + + + + PreserveNewest + + + PreserveNewest + + + + PreserveNewest + + diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/App.xaml b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/App.xaml index da79c36a11..df6ed811ac 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/App.xaml +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/App.xaml @@ -9,6 +9,7 @@ + diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/App.xaml.cs b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/App.xaml.cs index 3ac3baa9d0..2737fb44c2 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/App.xaml.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/App.xaml.cs @@ -10,10 +10,10 @@ using System.Linq; using System.Reflection; using System.Threading; using System.Threading.Tasks; - using AdvancedPaste.Helpers; using AdvancedPaste.Models; using AdvancedPaste.Services; +using AdvancedPaste.Services.CustomActions; using AdvancedPaste.Settings; using AdvancedPaste.ViewModels; using ManagedCommon; @@ -77,11 +77,12 @@ namespace AdvancedPaste { services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); }).Build(); @@ -111,7 +112,11 @@ namespace AdvancedPaste /// Invoked when the application is launched. /// /// Details about the launch request and process. +#if DEBUG + protected async override void OnLaunched(LaunchActivatedEventArgs args) +#else protected override void OnLaunched(LaunchActivatedEventArgs args) +#endif { var cmdArgs = Environment.GetCommandLineArgs(); if (cmdArgs?.Length > 1) @@ -133,6 +138,10 @@ namespace AdvancedPaste { ProcessNamedPipe(cmdArgs[2]); } + +#if DEBUG + await ShowWindow(); // This allows for direct access without using PowerToys Runner, not all functionality might work +#endif } private void ProcessNamedPipe(string pipeName) diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/AnimatedContentControl/AnimatedContentControl.xaml b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/AnimatedContentControl/AnimatedContentControl.xaml index f03a579821..a250fdffdc 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/AnimatedContentControl/AnimatedContentControl.xaml +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/AnimatedContentControl/AnimatedContentControl.xaml @@ -11,7 +11,7 @@ - + diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/ClipboardHistoryItemPreviewControl.xaml b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/ClipboardHistoryItemPreviewControl.xaml new file mode 100644 index 0000000000..e008d35a9a --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/ClipboardHistoryItemPreviewControl.xaml @@ -0,0 +1,60 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/ClipboardHistoryItemPreviewControl.xaml.cs b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/ClipboardHistoryItemPreviewControl.xaml.cs new file mode 100644 index 0000000000..a28c87ac61 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/ClipboardHistoryItemPreviewControl.xaml.cs @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using AdvancedPaste.Helpers; +using AdvancedPaste.Models; +using Microsoft.UI.Xaml; +using Microsoft.UI.Xaml.Controls; +using Microsoft.UI.Xaml.Media; + +namespace AdvancedPaste.Controls +{ + public sealed partial class ClipboardHistoryItemPreviewControl : UserControl + { + public static readonly DependencyProperty ClipboardItemProperty = DependencyProperty.Register( + nameof(ClipboardItem), + typeof(ClipboardItem), + typeof(ClipboardHistoryItemPreviewControl), + new PropertyMetadata(defaultValue: null, OnClipboardItemChanged)); + + public ClipboardItem ClipboardItem + { + get => (ClipboardItem)GetValue(ClipboardItemProperty); + set => SetValue(ClipboardItemProperty, value); + } + + // Computed properties for display + public string Header => ClipboardItem != null ? GetHeaderFromFormat(ClipboardItem.Format) : string.Empty; + + public string IconGlyph => ClipboardItem != null ? GetGlyphFromFormat(ClipboardItem.Format) : string.Empty; + + public string ContentText => ClipboardItem?.Content ?? string.Empty; + + public ImageSource ContentImage => ClipboardItem?.Image; + + public DateTimeOffset? Timestamp => ClipboardItem?.Timestamp ?? ClipboardItem?.Item?.Timestamp; + + public bool HasImage => ContentImage is not null; + + public bool HasText => !string.IsNullOrEmpty(ContentText) && !HasImage; + + public bool HasGlyph => !HasImage && !HasText && !string.IsNullOrEmpty(IconGlyph); + + public ClipboardHistoryItemPreviewControl() + { + InitializeComponent(); + } + + private static void OnClipboardItemChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) + { + if (d is ClipboardHistoryItemPreviewControl control) + { + // Notify bindings that all computed properties may have changed + control.Bindings.Update(); + } + } + + private static string GetHeaderFromFormat(ClipboardFormat format) + { + // Check flags in priority order (most specific first) + if (format.HasFlag(ClipboardFormat.Image)) + { + return GetStringOrFallback("ClipboardPreviewCategoryImage", "Image"); + } + + if (format.HasFlag(ClipboardFormat.Video)) + { + return GetStringOrFallback("ClipboardPreviewCategoryVideo", "Video"); + } + + if (format.HasFlag(ClipboardFormat.Audio)) + { + return GetStringOrFallback("ClipboardPreviewCategoryAudio", "Audio"); + } + + if (format.HasFlag(ClipboardFormat.File)) + { + return GetStringOrFallback("ClipboardPreviewCategoryFile", "File"); + } + + if (format.HasFlag(ClipboardFormat.Text) || format.HasFlag(ClipboardFormat.Html)) + { + return GetStringOrFallback("ClipboardPreviewCategoryText", "Text"); + } + + return GetStringOrFallback("ClipboardPreviewCategoryUnknown", "Clipboard"); + } + + private static string GetGlyphFromFormat(ClipboardFormat format) + { + // Check flags in priority order (most specific first) + if (format.HasFlag(ClipboardFormat.Image)) + { + return "\uEB9F"; // Image icon + } + + if (format.HasFlag(ClipboardFormat.Video)) + { + return "\uE714"; // Video icon + } + + if (format.HasFlag(ClipboardFormat.Audio)) + { + return "\uE189"; // Audio icon + } + + if (format.HasFlag(ClipboardFormat.File)) + { + return "\uE8A5"; // File icon + } + + if (format.HasFlag(ClipboardFormat.Text) || format.HasFlag(ClipboardFormat.Html)) + { + return "\uE8D2"; // Text icon + } + + return "\uE77B"; // Generic clipboard icon + } + + private static string GetStringOrFallback(string resourceKey, string fallback) + { + var value = ResourceLoaderInstance.ResourceLoader.GetString(resourceKey); + return string.IsNullOrEmpty(value) ? fallback : value; + } + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml index dd09c717b0..4450f7fdb1 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml @@ -7,8 +7,10 @@ xmlns:d="http://schemas.microsoft.com/expression/blend/2008" xmlns:local="using:AdvancedPaste.Controls" xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" + xmlns:settings="using:Microsoft.PowerToys.Settings.UI.Library" xmlns:tkconverters="using:CommunityToolkit.WinUI.Converters" xmlns:ui="using:CommunityToolkit.WinUI" + x:Name="PromptBoxControl" mc:Ignorable="d"> @@ -34,7 +36,7 @@ - + 44 + diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Assets/AdvancedPaste/Gradient.png b/src/modules/AdvancedPaste/AdvancedPaste/Assets/AdvancedPaste/Gradient.png index 73621edfc0..78a9a18606 100644 Binary files a/src/modules/AdvancedPaste/AdvancedPaste/Assets/AdvancedPaste/Gradient.png and b/src/modules/AdvancedPaste/AdvancedPaste/Assets/AdvancedPaste/Gradient.png differ diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AIServiceUsageHelper.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AIServiceUsageHelper.cs new file mode 100644 index 0000000000..ba7d33f4fb --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AIServiceUsageHelper.cs @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using AdvancedPaste.Models; +using Microsoft.SemanticKernel; + +namespace AdvancedPaste.Helpers; + +/// +/// Helper class for extracting AI service usage information from chat messages. +/// +public static class AIServiceUsageHelper +{ + /// + /// Extracts AI service usage information from OpenAI chat message metadata. + /// + /// The chat message containing usage metadata. + /// AI service usage information or AIServiceUsage.None if extraction fails. + public static AIServiceUsage GetOpenAIServiceUsage(ChatMessageContent chatMessage) + { + // Try to get usage information from metadata + if (chatMessage.Metadata?.TryGetValue("Usage", out var usageObj) == true) + { + // Handle different possible usage types through reflection to be version-agnostic + var usageType = usageObj.GetType(); + + try + { + // Try common property names for prompt tokens + var promptTokensProp = usageType.GetProperty("PromptTokens") ?? + usageType.GetProperty("InputTokens") ?? + usageType.GetProperty("InputTokenCount"); + + var completionTokensProp = usageType.GetProperty("CompletionTokens") ?? + usageType.GetProperty("OutputTokens") ?? + usageType.GetProperty("OutputTokenCount"); + + if (promptTokensProp != null && completionTokensProp != null) + { + var promptTokens = (int)(promptTokensProp.GetValue(usageObj) ?? 0); + var completionTokens = (int)(completionTokensProp.GetValue(usageObj) ?? 0); + return new AIServiceUsage(promptTokens, completionTokens); + } + } + catch + { + // If reflection fails, fall back to no usage + } + } + + return AIServiceUsage.None; + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/ClipboardItemHelper.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/ClipboardItemHelper.cs new file mode 100644 index 0000000000..9f824d3399 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/ClipboardItemHelper.cs @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Threading.Tasks; +using AdvancedPaste.Models; +using Microsoft.UI.Xaml.Media.Imaging; +using Windows.ApplicationModel.DataTransfer; + +namespace AdvancedPaste.Helpers +{ + internal static class ClipboardItemHelper + { + /// + /// Creates a ClipboardItem from current clipboard data. + /// + public static async Task CreateFromCurrentClipboardAsync( + DataPackageView clipboardData, + ClipboardFormat availableFormats, + DateTimeOffset? timestamp = null, + BitmapImage existingImage = null) + { + if (clipboardData == null || availableFormats == ClipboardFormat.None) + { + return null; + } + + var clipboardItem = new ClipboardItem + { + Format = availableFormats, + Timestamp = timestamp, + }; + + // Text or HTML content + if (availableFormats.HasFlag(ClipboardFormat.Text) || availableFormats.HasFlag(ClipboardFormat.Html)) + { + clipboardItem.Content = await clipboardData.GetTextOrEmptyAsync(); + } + + // Image content + else if (availableFormats.HasFlag(ClipboardFormat.Image)) + { + // Reuse existing image if provided + if (existingImage != null) + { + clipboardItem.Image = existingImage; + } + else + { + clipboardItem.Image = await TryCreateBitmapImageAsync(clipboardData); + } + } + + return clipboardItem; + } + + /// + /// Creates a BitmapImage from clipboard data. + /// + private static async Task TryCreateBitmapImageAsync(DataPackageView clipboardData) + { + try + { + var imageReference = await clipboardData.GetBitmapAsync(); + if (imageReference != null) + { + using (var imageStream = await imageReference.OpenReadAsync()) + { + var bitmapImage = new BitmapImage(); + await bitmapImage.SetSourceAsync(imageStream); + return bitmapImage; + } + } + } + catch + { + // Silently fail - caller can check for null + } + + return null; + } + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/DataPackageHelpers.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/DataPackageHelpers.cs index 529773f9a6..2cd7554a50 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/DataPackageHelpers.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/DataPackageHelpers.cs @@ -6,11 +6,13 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.InteropServices; using System.Text; +using System.Threading; using System.Threading.Tasks; - using AdvancedPaste.Models; using ManagedCommon; +using Microsoft.UI.Xaml.Media.Imaging; using Microsoft.Win32; using Windows.ApplicationModel.DataTransfer; using Windows.Data.Html; @@ -180,6 +182,46 @@ internal static class DataPackageHelpers } } + internal static async Task GetClipboardTextOrThrowAsync(this DataPackageView dataPackageView, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(dataPackageView); + + try + { + if (dataPackageView.Contains(StandardDataFormats.Text)) + { + return await dataPackageView.GetTextAsync(); + } + + if (dataPackageView.Contains(StandardDataFormats.Html)) + { + var html = await dataPackageView.GetHtmlFormatAsync(); + return HtmlUtilities.ConvertToText(html); + } + + if (dataPackageView.Contains(StandardDataFormats.Bitmap)) + { + var bitmap = await dataPackageView.GetImageContentAsync(); + if (bitmap != null) + { + return await OcrHelpers.ExtractTextAsync(bitmap, cancellationToken); + } + } + } + catch (Exception ex) when (ex is COMException or InvalidOperationException) + { + throw CreateClipboardTextMissingException(ex); + } + + throw CreateClipboardTextMissingException(); + } + + private static PasteActionException CreateClipboardTextMissingException(Exception innerException = null) + { + var message = ResourceLoaderInstance.ResourceLoader.GetString("ClipboardEmptyWarning"); + return new PasteActionException(message, innerException ?? new InvalidOperationException("Clipboard does not contain text content.")); + } + internal static async Task GetHtmlContentAsync(this DataPackageView dataPackageView) => dataPackageView.Contains(StandardDataFormats.Html) ? await dataPackageView.GetHtmlFormatAsync() : string.Empty; @@ -195,6 +237,22 @@ internal static class DataPackageHelpers return null; } + internal static async Task GetPreviewBitmapAsync(this DataPackageView dataPackageView) + { + var stream = await dataPackageView.GetImageStreamAsync(); + if (stream == null) + { + return null; + } + + using (stream) + { + var bitmapImage = new BitmapImage(); + bitmapImage.SetSource(stream); + return bitmapImage; + } + } + private static async Task GetImageStreamAsync(this DataPackageView dataPackageView) { if (dataPackageView.Contains(StandardDataFormats.StorageItems)) diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/IUserSettings.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/IUserSettings.cs index 105fe2c0d8..e32cf61af4 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/IUserSettings.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/IUserSettings.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using AdvancedPaste.Models; using Microsoft.PowerToys.Settings.UI.Library; @@ -12,7 +13,7 @@ namespace AdvancedPaste.Settings { public interface IUserSettings { - public bool IsAdvancedAIEnabled { get; } + public bool IsAIEnabled { get; } public bool ShowCustomPreview { get; } @@ -22,6 +23,10 @@ namespace AdvancedPaste.Settings public IReadOnlyList AdditionalActions { get; } + public PasteAIConfiguration PasteAIConfiguration { get; } + public event EventHandler Changed; + + Task SetActiveAIProviderAsync(string providerId); } } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/UserSettings.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/UserSettings.cs index 8a25b70f07..b6b6c19734 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/UserSettings.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/UserSettings.cs @@ -13,6 +13,7 @@ using AdvancedPaste.Models; using ManagedCommon; using Microsoft.PowerToys.Settings.UI.Library; using Microsoft.PowerToys.Settings.UI.Library.Utilities; +using Windows.Security.Credentials; namespace AdvancedPaste.Settings { @@ -33,7 +34,7 @@ namespace AdvancedPaste.Settings public event EventHandler Changed; - public bool IsAdvancedAIEnabled { get; private set; } + public bool IsAIEnabled { get; private set; } public bool ShowCustomPreview { get; private set; } @@ -43,13 +44,16 @@ namespace AdvancedPaste.Settings public IReadOnlyList CustomActions => _customActions; + public PasteAIConfiguration PasteAIConfiguration { get; private set; } + public UserSettings(IFileSystem fileSystem) { _settingsUtils = new SettingsUtils(fileSystem); - IsAdvancedAIEnabled = false; + IsAIEnabled = false; ShowCustomPreview = true; CloseAfterLosingFocus = false; + PasteAIConfiguration = new PasteAIConfiguration(); _additionalActions = []; _customActions = []; _taskScheduler = TaskScheduler.FromCurrentSynchronizationContext(); @@ -94,13 +98,16 @@ namespace AdvancedPaste.Settings var settings = _settingsUtils.GetSettingsOrDefault(AdvancedPasteModuleName); if (settings != null) { + bool migratedLegacyEnablement = TryMigrateLegacyAIEnablement(settings); + void UpdateSettings() { var properties = settings.Properties; - IsAdvancedAIEnabled = properties.IsAdvancedAIEnabled; + IsAIEnabled = properties.IsAIEnabled; ShowCustomPreview = properties.ShowCustomPreview; CloseAfterLosingFocus = properties.CloseAfterLosingFocus; + PasteAIConfiguration = properties.PasteAIConfiguration ?? new PasteAIConfiguration(); var sourceAdditionalActions = properties.AdditionalActions; (PasteFormats Format, IAdvancedPasteAction[] Actions)[] additionalActionFormats = @@ -126,6 +133,11 @@ namespace AdvancedPaste.Settings Task.Factory .StartNew(UpdateSettings, CancellationToken.None, TaskCreationOptions.None, _taskScheduler) .Wait(); + + if (migratedLegacyEnablement) + { + settings.Save(_settingsUtils); + } } retry = false; @@ -144,6 +156,114 @@ namespace AdvancedPaste.Settings } } + private static bool TryMigrateLegacyAIEnablement(AdvancedPasteSettings settings) + { + if (settings?.Properties is null) + { + return false; + } + + if (settings.Properties.IsAIEnabled || !LegacyOpenAIKeyExists()) + { + return false; + } + + settings.Properties.IsAIEnabled = true; + return true; + } + + private static bool LegacyOpenAIKeyExists() + { + try + { + PasswordVault vault = new(); + return vault.Retrieve("https://platform.openai.com/api-keys", "PowerToys_AdvancedPaste_OpenAIKey") is not null; + } + catch (Exception) + { + return false; + } + } + + public async Task SetActiveAIProviderAsync(string providerId) + { + if (string.IsNullOrWhiteSpace(providerId)) + { + return; + } + + await Task.Run(() => + { + lock (_loadingSettingsLock) + { + var settings = _settingsUtils.GetSettingsOrDefault(AdvancedPasteModuleName); + var configuration = settings?.Properties?.PasteAIConfiguration; + var providers = configuration?.Providers; + + if (configuration == null || providers == null || providers.Count == 0) + { + return; + } + + var target = providers.FirstOrDefault(provider => string.Equals(provider.Id, providerId, StringComparison.OrdinalIgnoreCase)); + if (target == null) + { + return; + } + + if (string.Equals(configuration.ActiveProvider?.Id, providerId, StringComparison.OrdinalIgnoreCase)) + { + return; + } + + configuration.ActiveProviderId = providerId; + + foreach (var provider in providers) + { + provider.IsActive = string.Equals(provider.Id, providerId, StringComparison.OrdinalIgnoreCase); + } + + try + { + settings.Save(_settingsUtils); + } + catch (Exception ex) + { + Logger.LogError("Failed to set active AI provider", ex); + return; + } + + try + { + Task.Factory + .StartNew( + () => + { + PasteAIConfiguration.ActiveProviderId = providerId; + + if (PasteAIConfiguration.Providers is not null) + { + foreach (var provider in PasteAIConfiguration.Providers) + { + provider.IsActive = string.Equals(provider.Id, providerId, StringComparison.OrdinalIgnoreCase); + } + } + + Changed?.Invoke(this, EventArgs.Empty); + }, + CancellationToken.None, + TaskCreationOptions.None, + _taskScheduler) + .Wait(); + } + catch (Exception ex) + { + Logger.LogError("Failed to dispatch active AI provider change", ex); + } + } + }); + } + public void Dispose() { Dispose(true); diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/ClipboardItem.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/ClipboardItem.cs index 1013108bc9..16814e7001 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Models/ClipboardItem.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Models/ClipboardItem.cs @@ -2,6 +2,7 @@ // The Microsoft Corporation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using AdvancedPaste.Helpers; using Microsoft.UI.Xaml.Media.Imaging; using Windows.ApplicationModel.DataTransfer; @@ -12,10 +13,15 @@ public class ClipboardItem { public string Content { get; set; } - public ClipboardHistoryItem Item { get; set; } - public BitmapImage Image { get; set; } + public ClipboardFormat Format { get; set; } + + public DateTimeOffset? Timestamp { get; set; } + + // Only used for clipboard history items that have a ClipboardHistoryItem + public ClipboardHistoryItem Item { get; set; } + public string Description => !string.IsNullOrEmpty(Content) ? Content : Image is not null ? ResourceLoaderInstance.ResourceLoader.GetString("ClipboardHistoryImage") : string.Empty; diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/AdvancedAIKernelService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/AdvancedAIKernelService.cs new file mode 100644 index 0000000000..b9566ed481 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/AdvancedAIKernelService.cs @@ -0,0 +1,227 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Linq; +using AdvancedPaste.Helpers; +using AdvancedPaste.Models; +using AdvancedPaste.Services.CustomActions; +using AdvancedPaste.Settings; +using Microsoft.PowerToys.Settings.UI.Library; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.Amazon; +using Microsoft.SemanticKernel.Connectors.AzureAIInference; +using Microsoft.SemanticKernel.Connectors.Google; +using Microsoft.SemanticKernel.Connectors.HuggingFace; +using Microsoft.SemanticKernel.Connectors.MistralAI; +using Microsoft.SemanticKernel.Connectors.Ollama; +using Microsoft.SemanticKernel.Connectors.OpenAI; + +namespace AdvancedPaste.Services; + +public sealed class AdvancedAIKernelService : KernelServiceBase +{ + private sealed record RuntimeConfiguration( + AIServiceType ServiceType, + string ModelName, + string Endpoint, + string DeploymentName, + string ModelPath, + string SystemPrompt, + bool ModerationEnabled) : IKernelRuntimeConfiguration; + + private readonly IAICredentialsProvider credentialsProvider; + + public AdvancedAIKernelService( + IAICredentialsProvider credentialsProvider, + IKernelQueryCacheService queryCacheService, + IPromptModerationService promptModerationService, + IUserSettings userSettings, + ICustomActionTransformService customActionTransformService) + : base(queryCacheService, promptModerationService, userSettings, customActionTransformService) + { + ArgumentNullException.ThrowIfNull(credentialsProvider); + + this.credentialsProvider = credentialsProvider; + } + + protected override string AdvancedAIModelName => GetRuntimeConfiguration().ModelName; + + protected override PromptExecutionSettings PromptExecutionSettings => CreatePromptExecutionSettings(); + + protected override void AddChatCompletionService(IKernelBuilder kernelBuilder) + { + ArgumentNullException.ThrowIfNull(kernelBuilder); + + var runtimeConfig = GetRuntimeConfiguration(); + var serviceType = runtimeConfig.ServiceType; + var modelName = runtimeConfig.ModelName; + var requiresApiKey = RequiresApiKey(serviceType); + var apiKey = string.Empty; + if (requiresApiKey) + { + this.credentialsProvider.Refresh(); + apiKey = (this.credentialsProvider.GetKey() ?? string.Empty).Trim(); + if (string.IsNullOrWhiteSpace(apiKey)) + { + throw new InvalidOperationException($"An API key is required for {serviceType} but none was found in the credential vault."); + } + } + + var endpoint = string.IsNullOrWhiteSpace(runtimeConfig.Endpoint) ? null : runtimeConfig.Endpoint.Trim(); + var deployment = string.IsNullOrWhiteSpace(runtimeConfig.DeploymentName) ? modelName : runtimeConfig.DeploymentName; + + switch (serviceType) + { + case AIServiceType.OpenAI: + kernelBuilder.AddOpenAIChatCompletion(modelName, apiKey, serviceId: modelName); + break; + case AIServiceType.AzureOpenAI: + kernelBuilder.AddAzureOpenAIChatCompletion(deployment, RequireEndpoint(endpoint, serviceType), apiKey, serviceId: modelName); + break; + default: + throw new NotSupportedException($"Service type '{runtimeConfig.ServiceType}' is not supported"); + } + } + + protected override AIServiceUsage GetAIServiceUsage(ChatMessageContent chatMessage) + { + return AIServiceUsageHelper.GetOpenAIServiceUsage(chatMessage); + } + + protected override bool ShouldModerateAdvancedAI() + { + if (!TryGetRuntimeConfiguration(out var runtimeConfig)) + { + return false; + } + + return runtimeConfig.ModerationEnabled && (runtimeConfig.ServiceType == AIServiceType.OpenAI || runtimeConfig.ServiceType == AIServiceType.AzureOpenAI); + } + + private static string GetModelName(PasteAIProviderDefinition config) + { + if (!string.IsNullOrWhiteSpace(config?.ModelName)) + { + return config.ModelName; + } + + return "gpt-4o"; + } + + protected override IKernelRuntimeConfiguration GetRuntimeConfiguration() + { + if (TryGetRuntimeConfiguration(out var runtimeConfig)) + { + return runtimeConfig; + } + + throw new InvalidOperationException("No Advanced AI provider is configured."); + } + + private bool TryGetRuntimeConfiguration(out IKernelRuntimeConfiguration runtimeConfig) + { + runtimeConfig = null; + + if (!TryResolveAdvancedProvider(out var provider)) + { + return false; + } + + var serviceType = NormalizeServiceType(provider.ServiceTypeKind); + if (!IsServiceTypeSupported(serviceType)) + { + return false; + } + + runtimeConfig = new RuntimeConfiguration( + serviceType, + GetModelName(provider), + provider.EndpointUrl, + provider.DeploymentName, + provider.ModelPath, + provider.SystemPrompt, + provider.ModerationEnabled); + return true; + } + + private bool TryResolveAdvancedProvider(out PasteAIProviderDefinition provider) + { + provider = null; + + var configuration = this.UserSettings?.PasteAIConfiguration; + if (configuration is null) + { + return false; + } + + var activeProvider = configuration.ActiveProvider; + if (IsAdvancedProvider(activeProvider)) + { + provider = activeProvider; + return true; + } + + if (activeProvider is not null) + { + return false; + } + + var fallback = configuration.Providers?.FirstOrDefault(IsAdvancedProvider); + if (fallback is not null) + { + provider = fallback; + return true; + } + + return false; + } + + private static bool IsAdvancedProvider(PasteAIProviderDefinition provider) + { + if (provider is null || !provider.EnableAdvancedAI) + { + return false; + } + + var serviceType = NormalizeServiceType(provider.ServiceTypeKind); + return IsServiceTypeSupported(serviceType); + } + + private static bool IsServiceTypeSupported(AIServiceType serviceType) + { + return serviceType is AIServiceType.OpenAI or AIServiceType.AzureOpenAI; + } + + private static AIServiceType NormalizeServiceType(AIServiceType serviceType) + { + return serviceType == AIServiceType.Unknown ? AIServiceType.OpenAI : serviceType; + } + + private static bool RequiresApiKey(AIServiceType serviceType) + { + return true; + } + + private static string RequireEndpoint(string endpoint, AIServiceType serviceType) + { + if (!string.IsNullOrWhiteSpace(endpoint)) + { + return endpoint; + } + + throw new InvalidOperationException($"Endpoint is required for {serviceType} configuration but was not provided."); + } + + private PromptExecutionSettings CreatePromptExecutionSettings() + { + var serviceType = GetRuntimeConfiguration().ServiceType; + return new OpenAIPromptExecutionSettings + { + FunctionChoiceBehavior = FunctionChoiceBehavior.Required(), + Temperature = 0.01, + }; + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/CustomActionTransformResult.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/CustomActionTransformResult.cs new file mode 100644 index 0000000000..562ea3976c --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/CustomActionTransformResult.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using AdvancedPaste.Models; + +namespace AdvancedPaste.Services.CustomActions +{ + public sealed class CustomActionTransformResult + { + public CustomActionTransformResult(string content, AIServiceUsage usage) + { + Content = content; + Usage = usage; + } + + public string Content { get; } + + public AIServiceUsage Usage { get; } + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/CustomActionTransformService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/CustomActionTransformService.cs new file mode 100644 index 0000000000..721a96070d --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/CustomActionTransformService.cs @@ -0,0 +1,200 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using AdvancedPaste.Helpers; +using AdvancedPaste.Models; +using AdvancedPaste.Settings; +using AdvancedPaste.Telemetry; +using ManagedCommon; +using Microsoft.PowerToys.Settings.UI.Library; +using Microsoft.PowerToys.Telemetry; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.OpenAI; + +namespace AdvancedPaste.Services.CustomActions +{ + public sealed class CustomActionTransformService : ICustomActionTransformService + { + private const string DefaultSystemPrompt = """ + You are tasked with reformatting user's clipboard data. Use the user's instructions, and the content of their clipboard below to edit their clipboard content as they have requested it. + Do not output anything else besides the reformatted clipboard content. + """; + + private readonly IPromptModerationService promptModerationService; + private readonly IPasteAIProviderFactory providerFactory; + private readonly IAICredentialsProvider credentialsProvider; + private readonly IUserSettings userSettings; + + public CustomActionTransformService(IPromptModerationService promptModerationService, IPasteAIProviderFactory providerFactory, IAICredentialsProvider credentialsProvider, IUserSettings userSettings) + { + this.promptModerationService = promptModerationService; + this.providerFactory = providerFactory; + this.credentialsProvider = credentialsProvider; + this.userSettings = userSettings; + } + + public async Task TransformTextAsync(string prompt, string inputText, CancellationToken cancellationToken, IProgress progress) + { + var pasteConfig = userSettings?.PasteAIConfiguration; + var providerConfig = BuildProviderConfig(pasteConfig); + + return await TransformAsync(prompt, inputText, providerConfig, cancellationToken, progress); + } + + private async Task TransformAsync(string prompt, string inputText, PasteAIConfig providerConfig, CancellationToken cancellationToken, IProgress progress) + { + ArgumentNullException.ThrowIfNull(providerConfig); + + if (string.IsNullOrWhiteSpace(prompt)) + { + return new CustomActionTransformResult(string.Empty, AIServiceUsage.None); + } + + if (string.IsNullOrWhiteSpace(inputText)) + { + Logger.LogWarning("Clipboard has no usable text data"); + return new CustomActionTransformResult(string.Empty, AIServiceUsage.None); + } + + var systemPrompt = providerConfig.SystemPrompt ?? DefaultSystemPrompt; + + var fullPrompt = (systemPrompt ?? string.Empty) + "\n\n" + (inputText ?? string.Empty); + + if (ShouldModerate(providerConfig)) + { + await promptModerationService.ValidateAsync(fullPrompt, cancellationToken); + } + + try + { + var provider = providerFactory.CreateProvider(providerConfig); + + var request = new PasteAIRequest + { + Prompt = prompt, + InputText = inputText, + SystemPrompt = systemPrompt, + }; + + var providerContent = await provider.ProcessPasteAsync( + request, + cancellationToken, + progress); + + var usage = request.Usage; + var content = providerContent ?? string.Empty; + + // Log endpoint usage + var endpointEvent = new AdvancedPasteEndpointUsageEvent(providerConfig.ProviderType); + PowerToysTelemetry.Log.WriteEvent(endpointEvent); + + Logger.LogDebug($"{nameof(CustomActionTransformService)}.{nameof(TransformAsync)} complete; ModelName={providerConfig.Model ?? string.Empty}, PromptTokens={usage.PromptTokens}, CompletionTokens={usage.CompletionTokens}"); + + return new CustomActionTransformResult(content, usage); + } + catch (Exception ex) + { + Logger.LogError($"{nameof(CustomActionTransformService)}.{nameof(TransformAsync)} failed", ex); + + if (ex is PasteActionException or OperationCanceledException) + { + throw; + } + + var statusCode = ExtractStatusCode(ex); + var failureMessage = providerConfig.ProviderType switch + { + AIServiceType.OpenAI or AIServiceType.AzureOpenAI => ErrorHelpers.TranslateErrorText(statusCode), + _ => ResourceLoaderInstance.ResourceLoader.GetString("PasteError"), + }; + + throw new PasteActionException(failureMessage, ex); + } + } + + private static int ExtractStatusCode(Exception exception) + { + if (exception is HttpOperationException httpOperationException) + { + return (int?)httpOperationException.StatusCode ?? -1; + } + + if (exception is HttpRequestException httpRequestException && httpRequestException.StatusCode is HttpStatusCode statusCode) + { + return (int)statusCode; + } + + return -1; + } + + private static AIServiceType NormalizeServiceType(AIServiceType serviceType) + { + return serviceType == AIServiceType.Unknown ? AIServiceType.OpenAI : serviceType; + } + + private PasteAIConfig BuildProviderConfig(PasteAIConfiguration config) + { + config ??= new PasteAIConfiguration(); + var provider = config.ActiveProvider ?? config.Providers?.FirstOrDefault() ?? new PasteAIProviderDefinition(); + var serviceType = NormalizeServiceType(provider.ServiceTypeKind); + var systemPrompt = string.IsNullOrWhiteSpace(provider.SystemPrompt) ? DefaultSystemPrompt : provider.SystemPrompt; + var apiKey = AcquireApiKey(serviceType); + var modelName = provider.ModelName; + + var providerConfig = new PasteAIConfig + { + ProviderType = serviceType, + ApiKey = apiKey, + Model = modelName, + Endpoint = provider.EndpointUrl, + DeploymentName = provider.DeploymentName, + LocalModelPath = provider.ModelPath, + ModelPath = provider.ModelPath, + SystemPrompt = systemPrompt, + ModerationEnabled = provider.ModerationEnabled, + }; + + return providerConfig; + } + + private string AcquireApiKey(AIServiceType serviceType) + { + if (!RequiresApiKey(serviceType)) + { + return string.Empty; + } + + credentialsProvider.Refresh(); + return credentialsProvider.GetKey() ?? string.Empty; + } + + private static bool RequiresApiKey(AIServiceType serviceType) + { + return serviceType switch + { + AIServiceType.Onnx => false, + AIServiceType.Ollama => false, + AIServiceType.Anthropic => false, + AIServiceType.AmazonBedrock => false, + _ => true, + }; + } + + private static bool ShouldModerate(PasteAIConfig providerConfig) + { + if (providerConfig is null || !providerConfig.ModerationEnabled) + { + return false; + } + + return providerConfig.ProviderType == AIServiceType.OpenAI || providerConfig.ProviderType == AIServiceType.AzureOpenAI; + } + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/FoundryLocalPasteProvider.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/FoundryLocalPasteProvider.cs new file mode 100644 index 0000000000..4b4148f995 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/FoundryLocalPasteProvider.cs @@ -0,0 +1,194 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using AdvancedPaste.Models; +using LanguageModelProvider; +using Microsoft.Extensions.AI; +using Microsoft.PowerToys.Settings.UI.Library; + +namespace AdvancedPaste.Services.CustomActions; + +public sealed class FoundryLocalPasteProvider : IPasteAIProvider +{ + private static readonly IReadOnlyCollection SupportedTypes = new[] + { + AIServiceType.FoundryLocal, + }; + + public static PasteAIProviderRegistration Registration { get; } = new(SupportedTypes, config => new FoundryLocalPasteProvider(config)); + + private static readonly LanguageModelService LanguageModels = LanguageModelService.CreateDefault(); + + private readonly PasteAIConfig _config; + + public FoundryLocalPasteProvider(PasteAIConfig config) + { + ArgumentNullException.ThrowIfNull(config); + _config = config; + } + + public string ProviderName => AIServiceType.FoundryLocal.ToNormalizedKey(); + + public string DisplayName => string.IsNullOrWhiteSpace(_config?.Model) ? "Foundry Local" : _config.Model; + + public async Task IsAvailableAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return await FoundryLocalModelProvider.Instance.IsAvailable().ConfigureAwait(false); + } + + public async Task ProcessPasteAsync(PasteAIRequest request, CancellationToken cancellationToken, IProgress progress) + { + ArgumentNullException.ThrowIfNull(request); + + try + { + var systemPrompt = request.SystemPrompt; + if (string.IsNullOrWhiteSpace(systemPrompt)) + { + throw new PasteActionException( + "System prompt is required for Foundry Local", + new ArgumentException("System prompt must be provided", nameof(request))); + } + + var prompt = request.Prompt; + var inputText = request.InputText; + if (string.IsNullOrWhiteSpace(prompt) || string.IsNullOrWhiteSpace(inputText)) + { + throw new PasteActionException( + "Prompt and input text are required", + new ArgumentException("Prompt and input text must be provided", nameof(request))); + } + + var modelReference = _config?.Model; + if (string.IsNullOrWhiteSpace(modelReference)) + { + throw new PasteActionException( + "No Foundry Local model selected", + new InvalidOperationException("Model identifier is required"), + aiServiceMessage: "Please select a model in the AI provider settings. Model identifier should be in the format 'fl://model-name'."); + } + + cancellationToken.ThrowIfCancellationRequested(); + var chatClient = LanguageModels.GetClient(modelReference); + if (chatClient is null) + { + throw new PasteActionException( + $"Unable to load Foundry Local model: {modelReference}", + new InvalidOperationException("Chat client resolution failed"), + aiServiceMessage: "The model may not be downloaded or the Foundry Local service may not be running. Please check the model status in settings."); + } + + // Extract actual model ID from the URL (format: fl://modelId) + var actualModelId = modelReference.Replace("fl://", string.Empty).Trim('/'); + + var userMessageContent = $""" + User instructions: + {prompt} + + Text: + {inputText} + + Output: + """; + + var chatMessages = new List + { + new(ChatRole.System, systemPrompt), + new(ChatRole.User, userMessageContent), + }; + + var chatOptions = CreateChatOptions(_config?.SystemPrompt, actualModelId); + + progress?.Report(0.1); + + var response = await chatClient.GetResponseAsync(chatMessages, chatOptions, cancellationToken).ConfigureAwait(false); + + progress?.Report(0.8); + + var responseText = GetResponseText(response); + request.Usage = ToUsage(response.Usage); + + progress?.Report(1.0); + + return responseText ?? string.Empty; + } + catch (OperationCanceledException) + { + // Let cancellation exceptions pass through unchanged + throw; + } + catch (PasteActionException) + { + // Let our custom exceptions pass through unchanged + throw; + } + catch (Exception ex) + { + // Wrap any other exceptions with context + var modelInfo = !string.IsNullOrWhiteSpace(_config?.Model) ? $" (Model: {_config.Model})" : string.Empty; + throw new PasteActionException( + $"Failed to generate response using Foundry Local{modelInfo}", + ex, + aiServiceMessage: $"Error details: {ex.Message}"); + } + } + + private static ChatOptions CreateChatOptions(string systemPrompt, string modelReference) + { + var options = new ChatOptions + { + ModelId = modelReference, + }; + + if (!string.IsNullOrWhiteSpace(systemPrompt)) + { + options.Instructions = systemPrompt; + } + + return options; + } + + private static string GetResponseText(ChatResponse response) + { + if (!string.IsNullOrWhiteSpace(response.Text)) + { + return response.Text; + } + + if (response.Messages is { Count: > 0 }) + { + var lastMessage = response.Messages.LastOrDefault(m => !string.IsNullOrWhiteSpace(m.Text)); + if (!string.IsNullOrWhiteSpace(lastMessage?.Text)) + { + return lastMessage.Text; + } + } + + return string.Empty; + } + + private static AIServiceUsage ToUsage(UsageDetails usageDetails) + { + if (usageDetails is null) + { + return AIServiceUsage.None; + } + + int promptTokens = (int)(usageDetails.InputTokenCount ?? 0); + int completionTokens = (int)(usageDetails.OutputTokenCount ?? 0); + + if (promptTokens == 0 && completionTokens == 0) + { + return AIServiceUsage.None; + } + + return new AIServiceUsage(promptTokens, completionTokens); + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/ICustomActionTransformService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/ICustomActionTransformService.cs new file mode 100644 index 0000000000..1c3ecb980c --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/ICustomActionTransformService.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Threading; +using System.Threading.Tasks; + +using AdvancedPaste.Settings; + +namespace AdvancedPaste.Services.CustomActions +{ + public interface ICustomActionTransformService + { + Task TransformTextAsync(string prompt, string inputText, CancellationToken cancellationToken, IProgress progress); + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/IPasteAIProvider.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/IPasteAIProvider.cs new file mode 100644 index 0000000000..764d99f942 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/IPasteAIProvider.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.PowerToys.Settings.UI.Library; + +namespace AdvancedPaste.Services.CustomActions +{ + public interface IPasteAIProvider + { + Task IsAvailableAsync(CancellationToken cancellationToken); + + Task ProcessPasteAsync(PasteAIRequest request, CancellationToken cancellationToken, IProgress progress); + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/IPasteAIProviderFactory.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/IPasteAIProviderFactory.cs new file mode 100644 index 0000000000..aacc61bec9 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/IPasteAIProviderFactory.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace AdvancedPaste.Services.CustomActions +{ + public interface IPasteAIProviderFactory + { + IPasteAIProvider CreateProvider(PasteAIConfig config); + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/LocalModelPasteProvider.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/LocalModelPasteProvider.cs new file mode 100644 index 0000000000..f4d45ccd74 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/LocalModelPasteProvider.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using AdvancedPaste.Models; +using Microsoft.PowerToys.Settings.UI.Library; + +namespace AdvancedPaste.Services.CustomActions +{ + public sealed class LocalModelPasteProvider : IPasteAIProvider + { + private static readonly IReadOnlyCollection SupportedTypes = new[] + { + AIServiceType.Onnx, + AIServiceType.ML, + }; + + public static PasteAIProviderRegistration Registration { get; } = new(SupportedTypes, config => new LocalModelPasteProvider(config)); + + private readonly PasteAIConfig _config; + + public LocalModelPasteProvider(PasteAIConfig config) + { + _config = config ?? throw new ArgumentNullException(nameof(config)); + } + + public Task IsAvailableAsync(CancellationToken cancellationToken) => Task.FromResult(true); + + public Task ProcessPasteAsync(PasteAIRequest request, CancellationToken cancellationToken, IProgress progress) + { + ArgumentNullException.ThrowIfNull(request); + + // TODO: Implement local model inference logic using _config.LocalModelPath/_config.ModelPath + var content = request.InputText ?? string.Empty; + request.Usage = AIServiceUsage.None; + return Task.FromResult(content); + } + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/PasteAIConfig.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/PasteAIConfig.cs new file mode 100644 index 0000000000..1d8a60f041 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/PasteAIConfig.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using AdvancedPaste.Models; +using Microsoft.PowerToys.Settings.UI.Library; +using Microsoft.SemanticKernel.ChatCompletion; + +namespace AdvancedPaste.Services.CustomActions +{ + public class PasteAIConfig + { + public AIServiceType ProviderType { get; set; } + + public string Model { get; set; } + + public string ApiKey { get; set; } + + public string Endpoint { get; set; } + + public string DeploymentName { get; set; } + + public string LocalModelPath { get; set; } + + public string ModelPath { get; set; } + + public string SystemPrompt { get; set; } + + public bool ModerationEnabled { get; set; } + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/PasteAIProviderFactory.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/PasteAIProviderFactory.cs new file mode 100644 index 0000000000..7339b4e4e3 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/PasteAIProviderFactory.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using Microsoft.PowerToys.Settings.UI.Library; + +namespace AdvancedPaste.Services.CustomActions +{ + public sealed class PasteAIProviderFactory : IPasteAIProviderFactory + { + private static readonly IReadOnlyList ProviderRegistrations = new[] + { + SemanticKernelPasteProvider.Registration, + LocalModelPasteProvider.Registration, + FoundryLocalPasteProvider.Registration, + }; + + private static readonly IReadOnlyDictionary> ProviderFactories = CreateProviderFactories(); + + public IPasteAIProvider CreateProvider(PasteAIConfig config) + { + ArgumentNullException.ThrowIfNull(config); + + var serviceType = config.ProviderType; + if (serviceType == AIServiceType.Unknown) + { + serviceType = AIServiceType.OpenAI; + config.ProviderType = serviceType; + } + + if (!ProviderFactories.TryGetValue(serviceType, out var factory)) + { + throw new NotSupportedException($"Provider {config.ProviderType} not supported"); + } + + return factory(config); + } + + private static IReadOnlyDictionary> CreateProviderFactories() + { + var map = new Dictionary>(); + + foreach (var registration in ProviderRegistrations) + { + Register(map, registration.SupportedTypes, registration.Factory); + } + + return map; + } + + private static void Register(Dictionary> map, IReadOnlyCollection types, Func factory) + { + foreach (var type in types) + { + map[type] = factory; + } + } + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/PasteAIProviderRegistration.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/PasteAIProviderRegistration.cs new file mode 100644 index 0000000000..6bd78450e8 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/PasteAIProviderRegistration.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; + +namespace AdvancedPaste.Services.CustomActions +{ + public sealed class PasteAIProviderRegistration + { + public PasteAIProviderRegistration(IReadOnlyCollection supportedTypes, Func factory) + { + SupportedTypes = supportedTypes ?? throw new ArgumentNullException(nameof(supportedTypes)); + Factory = factory ?? throw new ArgumentNullException(nameof(factory)); + } + + public IReadOnlyCollection SupportedTypes { get; } + + public Func Factory { get; } + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/PasteAIRequest.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/PasteAIRequest.cs new file mode 100644 index 0000000000..0e15c93e05 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/PasteAIRequest.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using AdvancedPaste.Models; + +namespace AdvancedPaste.Services.CustomActions +{ + public sealed class PasteAIRequest + { + public string Prompt { get; init; } + + public string InputText { get; init; } + + public string SystemPrompt { get; init; } + + public AIServiceUsage Usage { get; set; } = AIServiceUsage.None; + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/SemanticKernelPasteProvider.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/SemanticKernelPasteProvider.cs new file mode 100644 index 0000000000..00517e96d8 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/SemanticKernelPasteProvider.cs @@ -0,0 +1,203 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using AdvancedPaste.Helpers; +using AdvancedPaste.Models; +using Microsoft.PowerToys.Settings.UI.Library; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.Amazon; +using Microsoft.SemanticKernel.Connectors.AzureAIInference; +using Microsoft.SemanticKernel.Connectors.Google; +using Microsoft.SemanticKernel.Connectors.HuggingFace; +using Microsoft.SemanticKernel.Connectors.MistralAI; +using Microsoft.SemanticKernel.Connectors.Ollama; +using Microsoft.SemanticKernel.Connectors.OpenAI; + +namespace AdvancedPaste.Services.CustomActions +{ + public sealed class SemanticKernelPasteProvider : IPasteAIProvider + { + private static readonly IReadOnlyCollection SupportedTypes = new[] + { + AIServiceType.OpenAI, + AIServiceType.AzureOpenAI, + AIServiceType.Mistral, + AIServiceType.Google, + AIServiceType.HuggingFace, + AIServiceType.AzureAIInference, + AIServiceType.Ollama, + AIServiceType.Anthropic, + AIServiceType.AmazonBedrock, + }; + + public static PasteAIProviderRegistration Registration { get; } = new(SupportedTypes, config => new SemanticKernelPasteProvider(config)); + + private readonly PasteAIConfig _config; + private readonly AIServiceType _serviceType; + + public SemanticKernelPasteProvider(PasteAIConfig config) + { + ArgumentNullException.ThrowIfNull(config); + _config = config; + _serviceType = config.ProviderType; + if (_serviceType == AIServiceType.Unknown) + { + _serviceType = AIServiceType.OpenAI; + _config.ProviderType = _serviceType; + } + } + + public IReadOnlyCollection SupportedServiceTypes => SupportedTypes; + + public Task IsAvailableAsync(CancellationToken cancellationToken) => Task.FromResult(true); + + public async Task ProcessPasteAsync(PasteAIRequest request, CancellationToken cancellationToken, IProgress progress) + { + ArgumentNullException.ThrowIfNull(request); + + var systemPrompt = request.SystemPrompt; + if (string.IsNullOrWhiteSpace(systemPrompt)) + { + throw new ArgumentException("System prompt must be provided", nameof(request)); + } + + var prompt = request.Prompt; + var inputText = request.InputText; + if (string.IsNullOrWhiteSpace(prompt) || string.IsNullOrWhiteSpace(inputText)) + { + throw new ArgumentException("Prompt and input text must be provided", nameof(request)); + } + + var userMessageContent = $""" + User instructions: + {prompt} + + Clipboard Content: + {inputText} + + Output: + """; + + var executionSettings = CreateExecutionSettings(); + var kernel = CreateKernel(); + var modelId = _config.Model; + + IChatCompletionService chatService; + if (!string.IsNullOrWhiteSpace(modelId)) + { + try + { + chatService = kernel.GetRequiredService(modelId); + } + catch (Exception) + { + chatService = kernel.GetRequiredService(); + } + } + else + { + chatService = kernel.GetRequiredService(); + } + + var chatHistory = new ChatHistory(); + chatHistory.AddSystemMessage(systemPrompt); + chatHistory.AddUserMessage(userMessageContent); + + var response = await chatService.GetChatMessageContentAsync(chatHistory, executionSettings, kernel, cancellationToken); + chatHistory.Add(response); + + request.Usage = AIServiceUsageHelper.GetOpenAIServiceUsage(response); + return response.Content; + } + + private Kernel CreateKernel() + { + var kernelBuilder = Kernel.CreateBuilder(); + var endpoint = string.IsNullOrWhiteSpace(_config.Endpoint) ? null : _config.Endpoint.Trim(); + var apiKey = _config.ApiKey?.Trim() ?? string.Empty; + + if (RequiresApiKey(_serviceType) && string.IsNullOrWhiteSpace(apiKey)) + { + throw new InvalidOperationException($"API key is required for {_serviceType} but was not provided."); + } + + switch (_serviceType) + { + case AIServiceType.OpenAI: + kernelBuilder.AddOpenAIChatCompletion(_config.Model, apiKey, serviceId: _config.Model); + break; + case AIServiceType.AzureOpenAI: + var deploymentName = string.IsNullOrWhiteSpace(_config.DeploymentName) ? _config.Model : _config.DeploymentName; + kernelBuilder.AddAzureOpenAIChatCompletion(deploymentName, RequireEndpoint(endpoint, _serviceType), apiKey, serviceId: _config.Model); + break; + case AIServiceType.Mistral: + kernelBuilder.AddMistralChatCompletion(_config.Model, apiKey: apiKey); + break; + case AIServiceType.Google: + kernelBuilder.AddGoogleAIGeminiChatCompletion(_config.Model, apiKey: apiKey); + break; + case AIServiceType.HuggingFace: + kernelBuilder.AddHuggingFaceChatCompletion(_config.Model, apiKey: apiKey); + break; + case AIServiceType.AzureAIInference: + kernelBuilder.AddAzureAIInferenceChatCompletion(_config.Model, apiKey: apiKey, endpoint: new Uri(endpoint)); + break; + case AIServiceType.Ollama: + kernelBuilder.AddOllamaChatCompletion(_config.Model, endpoint: new Uri(endpoint)); + break; + case AIServiceType.Anthropic: + kernelBuilder.AddBedrockChatCompletionService(_config.Model); + break; + case AIServiceType.AmazonBedrock: + kernelBuilder.AddBedrockChatCompletionService(_config.Model); + break; + + default: + throw new NotSupportedException($"Provider '{_config.ProviderType}' is not supported by {nameof(SemanticKernelPasteProvider)}"); + } + + return kernelBuilder.Build(); + } + + private PromptExecutionSettings CreateExecutionSettings() + { + return _serviceType switch + { + AIServiceType.OpenAI or AIServiceType.AzureOpenAI => new OpenAIPromptExecutionSettings + { + Temperature = 0.01, + MaxTokens = 2000, + FunctionChoiceBehavior = null, + }, + _ => new PromptExecutionSettings(), + }; + } + + private static bool RequiresApiKey(AIServiceType serviceType) + { + return serviceType switch + { + AIServiceType.Ollama => false, + AIServiceType.Anthropic => false, + AIServiceType.AmazonBedrock => false, + _ => true, + }; + } + + private static string RequireEndpoint(string endpoint, AIServiceType serviceType) + { + if (!string.IsNullOrWhiteSpace(endpoint)) + { + return endpoint; + } + + throw new InvalidOperationException($"Endpoint is required for {serviceType} but was not provided."); + } + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/EnhancedVaultCredentialsProvider.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/EnhancedVaultCredentialsProvider.cs new file mode 100644 index 0000000000..2542f7310e --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/EnhancedVaultCredentialsProvider.cs @@ -0,0 +1,188 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Linq; +using System.Threading; +using AdvancedPaste.Settings; +using Microsoft.PowerToys.Settings.UI.Library; +using Windows.Security.Credentials; + +namespace AdvancedPaste.Services; + +/// +/// Enhanced credentials provider that supports different AI service types +/// Keys are stored in Windows Credential Vault with service-specific identifiers +/// +public sealed class EnhancedVaultCredentialsProvider : IAICredentialsProvider +{ + private sealed class CredentialSlot + { + public AIServiceType ServiceType { get; set; } = AIServiceType.Unknown; + + public string ProviderId { get; set; } = string.Empty; + + public (string Resource, string Username)? Entry { get; set; } + + public string Key { get; set; } = string.Empty; + } + + private readonly IUserSettings _userSettings; + private readonly CredentialSlot _slot; + private readonly Lock _syncRoot = new(); + + public EnhancedVaultCredentialsProvider(IUserSettings userSettings) + { + _userSettings = userSettings ?? throw new ArgumentNullException(nameof(userSettings)); + + _slot = new CredentialSlot(); + + Refresh(); + } + + public string GetKey() + { + using (_syncRoot.EnterScope()) + { + UpdateSlot(forceRefresh: false); + return _slot.Key; + } + } + + public bool IsConfigured() + { + return !string.IsNullOrEmpty(GetKey()); + } + + public bool Refresh() + { + using (_syncRoot.EnterScope()) + { + return UpdateSlot(forceRefresh: true); + } + } + + private bool UpdateSlot(bool forceRefresh) + { + var (serviceType, providerId) = ResolveCredentialTarget(); + var desiredServiceType = NormalizeServiceType(serviceType); + providerId ??= string.Empty; + + var hasChanged = false; + + if (_slot.ServiceType != desiredServiceType || !string.Equals(_slot.ProviderId, providerId, StringComparison.Ordinal)) + { + _slot.ServiceType = desiredServiceType; + _slot.ProviderId = providerId; + _slot.Entry = BuildCredentialEntry(desiredServiceType, providerId); + forceRefresh = true; + hasChanged = true; + } + + if (!forceRefresh) + { + return hasChanged; + } + + var newKey = LoadKey(_slot.Entry); + if (!string.Equals(_slot.Key, newKey, StringComparison.Ordinal)) + { + _slot.Key = newKey; + hasChanged = true; + } + + return hasChanged; + } + + private (AIServiceType ServiceType, string ProviderId) ResolveCredentialTarget() + { + var provider = _userSettings.PasteAIConfiguration?.ActiveProvider; + if (provider is null) + { + return (AIServiceType.OpenAI, string.Empty); + } + + return (provider.ServiceTypeKind, provider.Id ?? string.Empty); + } + + private static AIServiceType NormalizeServiceType(AIServiceType serviceType) + { + return serviceType == AIServiceType.Unknown ? AIServiceType.OpenAI : serviceType; + } + + private static string LoadKey((string Resource, string Username)? entry) + { + if (entry is null) + { + return string.Empty; + } + + try + { + var credential = new PasswordVault().Retrieve(entry.Value.Resource, entry.Value.Username); + return credential?.Password ?? string.Empty; + } + catch (Exception) + { + return string.Empty; + } + } + + private static (string Resource, string Username)? BuildCredentialEntry(AIServiceType serviceType, string providerId) + { + string resource; + string serviceKey; + + switch (serviceType) + { + case AIServiceType.OpenAI: + resource = "https://platform.openai.com/api-keys"; + serviceKey = "openai"; + break; + case AIServiceType.AzureOpenAI: + resource = "https://azure.microsoft.com/products/ai-services/openai-service"; + serviceKey = "azureopenai"; + break; + case AIServiceType.AzureAIInference: + resource = "https://azure.microsoft.com/products/ai-services/ai-inference"; + serviceKey = "azureaiinference"; + break; + case AIServiceType.Mistral: + resource = "https://console.mistral.ai/account/api-keys"; + serviceKey = "mistral"; + break; + case AIServiceType.Google: + resource = "https://ai.google.dev/"; + serviceKey = "google"; + break; + case AIServiceType.HuggingFace: + resource = "https://huggingface.co/settings/tokens"; + serviceKey = "huggingface"; + break; + case AIServiceType.FoundryLocal: + case AIServiceType.ML: + case AIServiceType.Onnx: + case AIServiceType.Ollama: + case AIServiceType.Anthropic: + case AIServiceType.AmazonBedrock: + return null; + default: + return null; + } + + string username = $"PowerToys_AdvancedPaste_PasteAI_{serviceKey}_{NormalizeProviderIdentifier(providerId)}"; + return (resource, username); + } + + private static string NormalizeProviderIdentifier(string providerId) + { + if (string.IsNullOrWhiteSpace(providerId)) + { + return "default"; + } + + var filtered = new string(providerId.Where(char.IsLetterOrDigit).ToArray()); + return string.IsNullOrWhiteSpace(filtered) ? "default" : filtered.ToLowerInvariant(); + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/IAICredentialsProvider.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/IAICredentialsProvider.cs index 54759b7dc8..7aa6f63b19 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/IAICredentialsProvider.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/IAICredentialsProvider.cs @@ -4,11 +4,26 @@ namespace AdvancedPaste.Services; +/// +/// Provides access to AI credentials stored for Advanced Paste scenarios. +/// public interface IAICredentialsProvider { - bool IsConfigured { get; } + /// + /// Gets a value indicating whether any credential is configured. + /// + /// when a non-empty credential exists for the active AI provider. + bool IsConfigured(); - string Key { get; } + /// + /// Retrieves the credential for the active AI provider. + /// + /// Credential string or when missing. + string GetKey(); + /// + /// Refreshes the cached credential for the active AI provider. + /// + /// when the credential changed. bool Refresh(); } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/ICustomTextTransformService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/ICustomTextTransformService.cs deleted file mode 100644 index 75f1df259e..0000000000 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/ICustomTextTransformService.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) Microsoft Corporation -// The Microsoft Corporation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Threading; -using System.Threading.Tasks; - -namespace AdvancedPaste.Services; - -public interface ICustomTextTransformService -{ - Task TransformTextAsync(string prompt, string inputText, CancellationToken cancellationToken, IProgress progress); -} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelRuntimeConfiguration.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelRuntimeConfiguration.cs new file mode 100644 index 0000000000..d634c13e30 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelRuntimeConfiguration.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.PowerToys.Settings.UI.Library; + +namespace AdvancedPaste.Services; + +/// +/// Represents runtime information required to configure an AI kernel service. +/// +public interface IKernelRuntimeConfiguration +{ + AIServiceType ServiceType { get; } + + string ModelName { get; } + + string Endpoint { get; } + + string DeploymentName { get; } + + string ModelPath { get; } + + string SystemPrompt { get; } + + bool ModerationEnabled { get; } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs index e921b21e54..0ea9ef40bc 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs @@ -5,15 +5,16 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text.Json; using System.Threading; using System.Threading.Tasks; - using AdvancedPaste.Helpers; using AdvancedPaste.Models; using AdvancedPaste.Models.KernelQueryCache; +using AdvancedPaste.Services.CustomActions; +using AdvancedPaste.Settings; using AdvancedPaste.Telemetry; using ManagedCommon; +using Microsoft.PowerToys.Settings.UI.Library; using Microsoft.PowerToys.Telemetry; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.ChatCompletion; @@ -21,15 +22,20 @@ using Windows.ApplicationModel.DataTransfer; namespace AdvancedPaste.Services; -public abstract class KernelServiceBase(IKernelQueryCacheService queryCacheService, IPromptModerationService promptModerationService, ICustomTextTransformService customTextTransformService) : IKernelService +public abstract class KernelServiceBase( + IKernelQueryCacheService queryCacheService, + IPromptModerationService promptModerationService, + IUserSettings userSettings, + ICustomActionTransformService customActionTransformService) : IKernelService { private const string PromptParameterName = "prompt"; private readonly IKernelQueryCacheService _queryCacheService = queryCacheService; private readonly IPromptModerationService _promptModerationService = promptModerationService; - private readonly ICustomTextTransformService _customTextTransformService = customTextTransformService; + private readonly IUserSettings _userSettings = userSettings; + private readonly ICustomActionTransformService _customActionTransformService = customActionTransformService; - protected abstract string ModelName { get; } + protected abstract string AdvancedAIModelName { get; } protected abstract PromptExecutionSettings PromptExecutionSettings { get; } @@ -37,6 +43,8 @@ public abstract class KernelServiceBase(IKernelQueryCacheService queryCacheServi protected abstract AIServiceUsage GetAIServiceUsage(ChatMessageContent chatMessage); + protected abstract IKernelRuntimeConfiguration GetRuntimeConfiguration(); + public async Task TransformClipboardAsync(string prompt, DataPackageView clipboardData, bool isSavedQuery, CancellationToken cancellationToken, IProgress progress) { Logger.LogTrace(); @@ -132,21 +140,20 @@ public abstract class KernelServiceBase(IKernelQueryCacheService queryCacheServi private async Task<(ChatHistory ChatHistory, AIServiceUsage Usage)> ExecuteAICompletion(Kernel kernel, string prompt, CancellationToken cancellationToken) { + var runtimeConfig = GetRuntimeConfiguration(); + ChatHistory chatHistory = []; - chatHistory.AddSystemMessage(""" - You are an agent who is tasked with helping users paste their clipboard data. You have functions available to help you with this task. - You never need to ask permission, always try to do as the user asks. The user will only input one message and will not be available for further questions, so try your best. - The user will put in a request to format their clipboard data and you will fulfill it. - You will not directly see the output clipboard content, and do not need to provide it in the chat. You just need to do the transform operations as needed. - If you are unable to fulfill the request, end with an error message in the language of the user's request. - """); + chatHistory.AddSystemMessage(runtimeConfig.SystemPrompt); chatHistory.AddSystemMessage($"Available clipboard formats: {await kernel.GetDataFormatsAsync()}"); chatHistory.AddUserMessage(prompt); - await _promptModerationService.ValidateAsync(GetFullPrompt(chatHistory), cancellationToken); + if (ShouldModerateAdvancedAI()) + { + await _promptModerationService.ValidateAsync(GetFullPrompt(chatHistory), cancellationToken); + } - var chatResult = await kernel.GetRequiredService() + var chatResult = await kernel.GetRequiredService(AdvancedAIModelName) .GetChatMessageContentAsync(chatHistory, PromptExecutionSettings, kernel, cancellationToken); chatHistory.Add(chatResult); @@ -175,10 +182,18 @@ public abstract class KernelServiceBase(IKernelQueryCacheService queryCacheServi return ([], AIServiceUsage.None); } + protected IUserSettings UserSettings => _userSettings; + private void LogResult(bool cacheUsed, bool isSavedQuery, IEnumerable actionChain, AIServiceUsage usage) { - AdvancedPasteSemanticKernelFormatEvent telemetryEvent = new(cacheUsed, isSavedQuery, usage.PromptTokens, usage.CompletionTokens, ModelName, AdvancedPasteSemanticKernelFormatEvent.FormatActionChain(actionChain)); + AdvancedPasteSemanticKernelFormatEvent telemetryEvent = new(cacheUsed, isSavedQuery, usage.PromptTokens, usage.CompletionTokens, AdvancedAIModelName, AdvancedPasteSemanticKernelFormatEvent.FormatActionChain(actionChain)); PowerToysTelemetry.Log.WriteEvent(telemetryEvent); + + // Log endpoint usage + var runtimeConfig = GetRuntimeConfiguration(); + var endpointEvent = new AdvancedPasteEndpointUsageEvent(runtimeConfig.ServiceType); + PowerToysTelemetry.Log.WriteEvent(endpointEvent); + var logEvent = new AIServiceFormatEvent(telemetryEvent); Logger.LogDebug($"{nameof(TransformClipboardAsync)} complete; {logEvent.ToJsonString()}"); } @@ -191,20 +206,96 @@ public abstract class KernelServiceBase(IKernelQueryCacheService queryCacheServi return kernelBuilder.Build(); } - private IEnumerable GetKernelFunctions() => - from format in Enum.GetValues() - let metadata = PasteFormat.MetadataDict[format] - let coreDescription = metadata.KernelFunctionDescription - where !string.IsNullOrEmpty(coreDescription) - let requiresPrompt = metadata.RequiresPrompt - orderby requiresPrompt descending - select KernelFunctionFactory.CreateFromMethod( - method: requiresPrompt ? async (Kernel kernel, string prompt) => await ExecutePromptTransformAsync(kernel, format, prompt) - : async (Kernel kernel) => await ExecuteStandardTransformAsync(kernel, format), - functionName: format.ToString(), - description: requiresPrompt ? coreDescription : $"{coreDescription} Puts the result back on the clipboard.", - parameters: requiresPrompt ? [new(PromptParameterName) { Description = "Input instructions to AI", ParameterType = typeof(string) }] : null, - returnParameter: new() { Description = "Array of available clipboard formats after operation" }); + private IEnumerable GetKernelFunctions() + { + // Get standard format functions + var standardFunctions = + from format in Enum.GetValues() + let metadata = PasteFormat.MetadataDict[format] + let coreDescription = metadata.KernelFunctionDescription + where !string.IsNullOrEmpty(coreDescription) + let requiresPrompt = metadata.RequiresPrompt + orderby requiresPrompt descending + select KernelFunctionFactory.CreateFromMethod( + method: requiresPrompt ? async (Kernel kernel, string prompt) => await ExecutePromptTransformAsync(kernel, format, prompt) + : async (Kernel kernel) => await ExecuteStandardTransformAsync(kernel, format), + functionName: format.ToString(), + description: requiresPrompt ? coreDescription : $"{coreDescription} Puts the result back on the clipboard.", + parameters: requiresPrompt ? [new(PromptParameterName) { Description = "Input instructions to AI", ParameterType = typeof(string) }] : null, + returnParameter: new() { Description = "Array of available clipboard formats after operation" }); + + HashSet usedFunctionNames = new(Enum.GetNames(), StringComparer.OrdinalIgnoreCase); + + // Get custom action functions + var customActionFunctions = _userSettings.CustomActions + .Where(customAction => !string.IsNullOrWhiteSpace(customAction.Name) && !string.IsNullOrWhiteSpace(customAction.Prompt)) + .Select(customAction => + { + var sanitizedBaseName = SanitizeFunctionName(customAction.Name); + var functionName = GetUniqueFunctionName(sanitizedBaseName, usedFunctionNames, customAction.Id); + var description = string.IsNullOrWhiteSpace(customAction.Description) + ? $"Runs the \"{customAction.Name}\" custom action." + : customAction.Description; + return KernelFunctionFactory.CreateFromMethod( + method: async (Kernel kernel) => await ExecuteCustomActionAsync(kernel, customAction.Prompt), + functionName: functionName, + description: description, + parameters: null, + returnParameter: new() { Description = "Array of available clipboard formats after operation" }); + }); + + return standardFunctions.Concat(customActionFunctions); + } + + private static string GetUniqueFunctionName(string baseName, HashSet usedFunctionNames, int customActionId) + { + ArgumentNullException.ThrowIfNull(usedFunctionNames); + + var candidate = string.IsNullOrEmpty(baseName) ? "_CustomAction" : baseName; + + if (usedFunctionNames.Add(candidate)) + { + return candidate; + } + + int suffix = 1; + while (true) + { + var nextCandidate = $"{candidate}_{customActionId}_{suffix}"; + if (usedFunctionNames.Add(nextCandidate)) + { + return nextCandidate; + } + + suffix++; + } + } + + private static string SanitizeFunctionName(string name) + { + // Remove invalid characters and ensure the function name is valid for kernel + var sanitized = new string(name.Where(c => char.IsLetterOrDigit(c) || c == '_').ToArray()); + + // Ensure it starts with a letter or underscore + if (sanitized.Length > 0 && !char.IsLetter(sanitized[0]) && sanitized[0] != '_') + { + sanitized = "_" + sanitized; + } + + // Ensure it's not empty + return string.IsNullOrEmpty(sanitized) ? "_CustomAction" : sanitized; + } + + private Task ExecuteCustomActionAsync(Kernel kernel, string fixedPrompt) => + ExecuteTransformAsync( + kernel, + new ActionChainItem(PasteFormats.CustomTextTransformation, Arguments: new() { { PromptParameterName, fixedPrompt } }), + async dataPackageView => + { + var input = await dataPackageView.GetClipboardTextOrThrowAsync(kernel.GetCancellationToken()); + var result = await _customActionTransformService.TransformTextAsync(fixedPrompt, input, kernel.GetCancellationToken(), kernel.GetProgress()); + return DataPackageHelpers.CreateFromText(result?.Content ?? string.Empty); + }); private Task ExecutePromptTransformAsync(Kernel kernel, PasteFormats format, string prompt) => ExecuteTransformAsync( @@ -212,7 +303,7 @@ public abstract class KernelServiceBase(IKernelQueryCacheService queryCacheServi new ActionChainItem(format, Arguments: new() { { PromptParameterName, prompt } }), async dataPackageView => { - var input = await dataPackageView.GetTextAsync(); + var input = await dataPackageView.GetClipboardTextOrThrowAsync(kernel.GetCancellationToken()); string output = await GetPromptBasedOutput(format, prompt, input, kernel.GetCancellationToken(), kernel.GetProgress()); return DataPackageHelpers.CreateFromText(output); }); @@ -220,7 +311,7 @@ public abstract class KernelServiceBase(IKernelQueryCacheService queryCacheServi private async Task GetPromptBasedOutput(PasteFormats format, string prompt, string input, CancellationToken cancellationToken, IProgress progress) => format switch { - PasteFormats.CustomTextTransformation => await _customTextTransformService.TransformTextAsync(prompt, input, cancellationToken, progress), + PasteFormats.CustomTextTransformation => (await _customActionTransformService.TransformTextAsync(prompt, input, cancellationToken, progress))?.Content ?? string.Empty, _ => throw new ArgumentException($"Unsupported format {format} for prompt transform", nameof(format)), }; @@ -281,4 +372,9 @@ public abstract class KernelServiceBase(IKernelQueryCacheService queryCacheServi var usageString = usage.HasUsage ? $" [{usage}]" : string.Empty; return $"-> {role}: {redactedContent}{usageString}"; } + + protected virtual bool ShouldModerateAdvancedAI() + { + return false; + } } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/CustomTextTransformService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/CustomTextTransformService.cs deleted file mode 100644 index b6aa156b9d..0000000000 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/CustomTextTransformService.cs +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) Microsoft Corporation -// The Microsoft Corporation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; - -using AdvancedPaste.Helpers; -using AdvancedPaste.Models; -using AdvancedPaste.Telemetry; -using Azure; -using Azure.AI.OpenAI; -using ManagedCommon; -using Microsoft.PowerToys.Telemetry; - -namespace AdvancedPaste.Services.OpenAI; - -public sealed class CustomTextTransformService(IAICredentialsProvider aiCredentialsProvider, IPromptModerationService promptModerationService) : ICustomTextTransformService -{ - private const string ModelName = "gpt-3.5-turbo-instruct"; - - private readonly IAICredentialsProvider _aiCredentialsProvider = aiCredentialsProvider; - private readonly IPromptModerationService _promptModerationService = promptModerationService; - - private async Task GetAICompletionAsync(string systemInstructions, string userMessage, CancellationToken cancellationToken) - { - var fullPrompt = systemInstructions + "\n\n" + userMessage; - - await _promptModerationService.ValidateAsync(fullPrompt, cancellationToken); - - OpenAIClient azureAIClient = new(_aiCredentialsProvider.Key); - - var response = await azureAIClient.GetCompletionsAsync( - new() - { - DeploymentName = ModelName, - Prompts = - { - fullPrompt, - }, - Temperature = 0.01F, - MaxTokens = 2000, - }, - cancellationToken); - - if (response.Value.Choices[0].FinishReason == "length") - { - Logger.LogDebug("Cut off due to length constraints"); - } - - return response; - } - - public async Task TransformTextAsync(string prompt, string inputText, CancellationToken cancellationToken, IProgress progress) - { - if (string.IsNullOrWhiteSpace(prompt)) - { - return string.Empty; - } - - if (string.IsNullOrWhiteSpace(inputText)) - { - Logger.LogWarning("Clipboard has no usable text data"); - return string.Empty; - } - - string systemInstructions = -$@"You are tasked with reformatting user's clipboard data. Use the user's instructions, and the content of their clipboard below to edit their clipboard content as they have requested it. -Do not output anything else besides the reformatted clipboard content."; - - string userMessage = -$@"User instructions: -{prompt} - -Clipboard Content: -{inputText} - -Output: -"; - - try - { - var response = await GetAICompletionAsync(systemInstructions, userMessage, cancellationToken); - - var usage = response.Usage; - AdvancedPasteGenerateCustomFormatEvent telemetryEvent = new(usage.PromptTokens, usage.CompletionTokens, ModelName); - PowerToysTelemetry.Log.WriteEvent(telemetryEvent); - var logEvent = new AIServiceFormatEvent(telemetryEvent); - - Logger.LogDebug($"{nameof(TransformTextAsync)} complete; {logEvent.ToJsonString()}"); - - return response.Choices[0].Text; - } - catch (Exception ex) - { - Logger.LogError($"{nameof(TransformTextAsync)} failed", ex); - - AdvancedPasteGenerateCustomErrorEvent errorEvent = new(ex is PasteActionModeratedException ? PasteActionModeratedException.ErrorDescription : ex.Message); - PowerToysTelemetry.Log.WriteEvent(errorEvent); - - if (ex is PasteActionException or OperationCanceledException) - { - throw; - } - else - { - throw new PasteActionException(ErrorHelpers.TranslateErrorText((ex as RequestFailedException)?.Status ?? -1), ex); - } - } - } -} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/KernelService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/KernelService.cs deleted file mode 100644 index b19a6d51cb..0000000000 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/KernelService.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) Microsoft Corporation -// The Microsoft Corporation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Collections.Generic; - -using AdvancedPaste.Models; -using Azure.AI.OpenAI; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Connectors.OpenAI; - -namespace AdvancedPaste.Services.OpenAI; - -public sealed class KernelService(IKernelQueryCacheService queryCacheService, IAICredentialsProvider aiCredentialsProvider, IPromptModerationService promptModerationService, ICustomTextTransformService customTextTransformService) : - KernelServiceBase(queryCacheService, promptModerationService, customTextTransformService) -{ - private readonly IAICredentialsProvider _aiCredentialsProvider = aiCredentialsProvider; - - protected override string ModelName => "gpt-4o"; - - protected override PromptExecutionSettings PromptExecutionSettings => - new OpenAIPromptExecutionSettings() - { - ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions, - Temperature = 0.01, - }; - - protected override void AddChatCompletionService(IKernelBuilder kernelBuilder) => kernelBuilder.AddOpenAIChatCompletion(ModelName, _aiCredentialsProvider.Key); - - protected override AIServiceUsage GetAIServiceUsage(ChatMessageContent chatMessage) => - chatMessage.Metadata?.GetValueOrDefault("Usage") is CompletionsUsage completionsUsage - ? new(PromptTokens: completionsUsage.PromptTokens, CompletionTokens: completionsUsage.CompletionTokens) - : AIServiceUsage.None; -} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/PromptModerationService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/PromptModerationService.cs index 0ca15e4161..2668300526 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/PromptModerationService.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/PromptModerationService.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using AdvancedPaste.Helpers; using AdvancedPaste.Models; +using AdvancedPaste.Services; using ManagedCommon; using OpenAI.Moderations; @@ -23,7 +24,16 @@ public sealed class PromptModerationService(IAICredentialsProvider aiCredentials { try { - ModerationClient moderationClient = new(ModelName, _aiCredentialsProvider.Key); + _aiCredentialsProvider.Refresh(); + var apiKey = _aiCredentialsProvider.GetKey()?.Trim() ?? string.Empty; + + if (string.IsNullOrEmpty(apiKey)) + { + Logger.LogWarning("Skipping OpenAI moderation because no credential is configured."); + return; + } + + ModerationClient moderationClient = new(ModelName, apiKey); var moderationClientResult = await moderationClient.ClassifyTextAsync(fullPrompt, cancellationToken); var moderationResult = moderationClientResult.Value; diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/VaultCredentialsProvider.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/VaultCredentialsProvider.cs deleted file mode 100644 index 169c1c2422..0000000000 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/VaultCredentialsProvider.cs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) Microsoft Corporation -// The Microsoft Corporation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; - -using Windows.Security.Credentials; - -namespace AdvancedPaste.Services.OpenAI; - -public sealed class VaultCredentialsProvider : IAICredentialsProvider -{ - public VaultCredentialsProvider() => Refresh(); - - public string Key { get; private set; } - - public bool IsConfigured => !string.IsNullOrEmpty(Key); - - public bool Refresh() - { - var oldKey = Key; - Key = LoadKey(); - return oldKey != Key; - } - - private static string LoadKey() - { - try - { - return new PasswordVault().Retrieve("https://platform.openai.com/api-keys", "PowerToys_AdvancedPaste_OpenAIKey")?.Password ?? string.Empty; - } - catch (Exception) - { - return string.Empty; - } - } -} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/PasteFormatExecutor.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/PasteFormatExecutor.cs index 5d6740977b..aef9e39bb9 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/PasteFormatExecutor.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/PasteFormatExecutor.cs @@ -8,15 +8,16 @@ using System.Threading.Tasks; using AdvancedPaste.Helpers; using AdvancedPaste.Models; +using AdvancedPaste.Services.CustomActions; using Microsoft.PowerToys.Telemetry; using Windows.ApplicationModel.DataTransfer; namespace AdvancedPaste.Services; -public sealed class PasteFormatExecutor(IKernelService kernelService, ICustomTextTransformService customTextTransformService) : IPasteFormatExecutor +public sealed class PasteFormatExecutor(IKernelService kernelService, ICustomActionTransformService customActionTransformService) : IPasteFormatExecutor { private readonly IKernelService _kernelService = kernelService; - private readonly ICustomTextTransformService _customTextTransformService = customTextTransformService; + private readonly ICustomActionTransformService _customActionTransformService = customActionTransformService; public async Task ExecutePasteFormatAsync(PasteFormat pasteFormat, PasteActionSource source, CancellationToken cancellationToken, IProgress progress) { @@ -36,7 +37,7 @@ public sealed class PasteFormatExecutor(IKernelService kernelService, ICustomTex pasteFormat.Format switch { PasteFormats.KernelQuery => await _kernelService.TransformClipboardAsync(pasteFormat.Prompt, clipboardData, pasteFormat.IsSavedQuery, cancellationToken, progress), - PasteFormats.CustomTextTransformation => DataPackageHelpers.CreateFromText(await _customTextTransformService.TransformTextAsync(pasteFormat.Prompt, await clipboardData.GetTextAsync(), cancellationToken, progress)), + PasteFormats.CustomTextTransformation => DataPackageHelpers.CreateFromText((await _customActionTransformService.TransformTextAsync(pasteFormat.Prompt, await clipboardData.GetClipboardTextOrThrowAsync(cancellationToken), cancellationToken, progress))?.Content ?? string.Empty), _ => await TransformHelpers.TransformAsync(format, clipboardData, cancellationToken, progress), }); } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Strings/en-us/Resources.resw b/src/modules/AdvancedPaste/AdvancedPaste/Strings/en-us/Resources.resw index 604cbf403b..f6c66154af 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Strings/en-us/Resources.resw +++ b/src/modules/AdvancedPaste/AdvancedPaste/Strings/en-us/Resources.resw @@ -144,16 +144,67 @@ The paste operation was moderated due to sensitive content. Please try another query. - + Clipboard history Clipboard history + + AI provider selector + + + Select an AI provider + + + Active provider: {0} + + + AI providers + + + No AI providers configured + + + Configure models in Settings + Image data Label used to represent an image in the clipboard history + + Text + + + Image + + + Audio + + + Video + + + File + + + Clipboard + + + Copied just now + + + Copied {0} sec ago + + + Copied {0} min ago + + + Copied {0} hr ago + + + Copied {0} day ago + More options @@ -196,7 +247,7 @@ Transcode to .mp3 Option to transcode audio files to MP3 format - + Transcode to .mp4 (H.264/AAC) Option to transcode video files to MP4 format with H.264 video codec and AAC audio codec @@ -272,11 +323,11 @@ Next result - - OpenAI Privacy + + Privacy Policy - - OpenAI Terms + + Terms To custom with AI is disabled by your organization @@ -287,4 +338,27 @@ PowerToys_Paste_ + + Just now + + + 1 minute ago + + + {0} minutes ago + + + Today, {0} + + + Yesterday, {0} + + + {0}, {1} + (e.g., “Wednesday, 17:05”) + + + {0}, {1} + (e.g., “10/20/2025, 17:05” in the user’s locale) + \ No newline at end of file diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteEndpointUsageEvent.cs b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteEndpointUsageEvent.cs new file mode 100644 index 0000000000..04777eff79 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteEndpointUsageEvent.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Tracing; +using Microsoft.PowerToys.Settings.UI.Library; +using Microsoft.PowerToys.Telemetry; +using Microsoft.PowerToys.Telemetry.Events; + +namespace AdvancedPaste.Telemetry; + +[EventData] +[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties)] +public class AdvancedPasteEndpointUsageEvent : EventBase, IEvent +{ + /// + /// Gets or sets the AI provider type (e.g., OpenAI, AzureOpenAI, Anthropic). + /// + public string ProviderType { get; set; } + + public AdvancedPasteEndpointUsageEvent(AIServiceType providerType) + { + ProviderType = providerType.ToString(); + } + + public PartA_PrivTags PartA_PrivTags => PartA_PrivTags.ProductAndServiceUsage; +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/ViewModels/OptionsViewModel.cs b/src/modules/AdvancedPaste/AdvancedPaste/ViewModels/OptionsViewModel.cs index 688c3047e2..ede006e960 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/ViewModels/OptionsViewModel.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/ViewModels/OptionsViewModel.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Diagnostics; +using System.Globalization; using System.IO.Abstractions; using System.Linq; using System.Runtime.InteropServices; @@ -22,6 +23,8 @@ using CommunityToolkit.Mvvm.Input; using ManagedCommon; using Microsoft.PowerToys.Settings.UI.Library; using Microsoft.UI.Xaml; +using Microsoft.UI.Xaml.Media; +using Microsoft.UI.Xaml.Media.Imaging; using Microsoft.Win32; using Windows.ApplicationModel.DataTransfer; using Windows.System; @@ -37,12 +40,20 @@ namespace AdvancedPaste.ViewModels private readonly DispatcherTimer _clipboardTimer; private readonly IUserSettings _userSettings; private readonly IPasteFormatExecutor _pasteFormatExecutor; - private readonly IAICredentialsProvider _aiCredentialsProvider; + private readonly IAICredentialsProvider _credentialsProvider; private CancellationTokenSource _pasteActionCancellationTokenSource; + private string _currentClipboardHistoryId; + private DateTimeOffset? _currentClipboardTimestamp; + private ClipboardFormat _lastClipboardFormats = ClipboardFormat.None; + private bool _clipboardHistoryUnavailableLogged; + public DataPackageView ClipboardData { get; set; } + [ObservableProperty] + private ClipboardItem _currentClipboardItem; + [ObservableProperty] [NotifyPropertyChangedFor(nameof(IsCustomAIAvailable))] [NotifyPropertyChangedFor(nameof(ClipboardHasData))] @@ -58,6 +69,8 @@ namespace AdvancedPaste.ViewModels [NotifyPropertyChangedFor(nameof(CustomAIUnavailableErrorText))] [NotifyPropertyChangedFor(nameof(IsCustomAIServiceEnabled))] [NotifyPropertyChangedFor(nameof(IsCustomAIAvailable))] + [NotifyPropertyChangedFor(nameof(AllowedAIProviders))] + [NotifyPropertyChangedFor(nameof(ActiveAIProvider))] private bool _isAllowedByGPO; [ObservableProperty] @@ -79,11 +92,100 @@ namespace AdvancedPaste.ViewModels public ObservableCollection CustomActionPasteFormats { get; } = []; - public bool IsCustomAIServiceEnabled => IsAllowedByGPO && _aiCredentialsProvider.IsConfigured; + public bool IsCustomAIServiceEnabled + { + get + { + if (!IsAllowedByGPO || !_userSettings.IsAIEnabled) + { + return false; + } + + // Check if there are any allowed providers + if (!AllowedAIProviders.Any()) + { + return false; + } + + // We should handle the IsAIEnabled logic in settings, don't check again here. + // If setting says yes, and here should pass check, and if error happens, it happens. + return true; + } + } public bool IsCustomAIAvailable => IsCustomAIServiceEnabled && ClipboardHasDataForCustomAI; - public bool IsAdvancedAIEnabled => IsCustomAIServiceEnabled && _userSettings.IsAdvancedAIEnabled; + public bool IsAdvancedAIEnabled + { + get + { + if (!IsAllowedByGPO || !_userSettings.IsAIEnabled) + { + return false; + } + + if (!TryResolveAdvancedAIProvider(out _)) + { + return false; + } + + return _credentialsProvider.IsConfigured(); + } + } + + public ObservableCollection AIProviders => _userSettings?.PasteAIConfiguration?.Providers ?? new ObservableCollection(); + + public IEnumerable AllowedAIProviders + { + get + { + var providers = AIProviders; + if (providers is null || providers.Count == 0) + { + return Enumerable.Empty(); + } + + return providers.Where(IsProviderAllowedByGPO); + } + } + + public PasteAIProviderDefinition ActiveAIProvider + { + get + { + var provider = _userSettings?.PasteAIConfiguration?.ActiveProvider; + if (provider is null || !IsProviderAllowedByGPO(provider)) + { + return null; + } + + return provider; + } + } + + public string ActiveAIProviderTooltip + { + get + { + var resourceLoader = ResourceLoaderInstance.ResourceLoader; + var provider = ActiveAIProvider; + + if (provider is null) + { + return resourceLoader.GetString("AIProviderButtonTooltipEmpty"); + } + + var format = resourceLoader.GetString("AIProviderButtonTooltipFormat"); + var displayName = provider.DisplayName; + + if (!string.IsNullOrEmpty(format)) + { + return string.Format(CultureInfo.CurrentCulture, format, displayName); + } + + return displayName; + } + } public bool ClipboardHasData => AvailableClipboardFormats != ClipboardFormat.None; @@ -91,7 +193,10 @@ namespace AdvancedPaste.ViewModels public bool HasIndeterminateTransformProgress => double.IsNaN(TransformProgress); - private PasteFormats CustomAIFormat => _userSettings.IsAdvancedAIEnabled ? PasteFormats.KernelQuery : PasteFormats.CustomTextTransformation; + private PasteFormats CustomAIFormat => + _userSettings.IsAIEnabled && TryResolveAdvancedAIProvider(out _) + ? PasteFormats.KernelQuery + : PasteFormats.CustomTextTransformation; private bool Visible { @@ -110,9 +215,9 @@ namespace AdvancedPaste.ViewModels public event EventHandler PreviewRequested; - public OptionsViewModel(IFileSystem fileSystem, IAICredentialsProvider aiCredentialsProvider, IUserSettings userSettings, IPasteFormatExecutor pasteFormatExecutor) + public OptionsViewModel(IFileSystem fileSystem, IAICredentialsProvider credentialsProvider, IUserSettings userSettings, IPasteFormatExecutor pasteFormatExecutor) { - _aiCredentialsProvider = aiCredentialsProvider; + _credentialsProvider = credentialsProvider; _userSettings = userSettings; _pasteFormatExecutor = pasteFormatExecutor; @@ -130,6 +235,7 @@ namespace AdvancedPaste.ViewModels _clipboardTimer.Start(); RefreshPasteFormats(); + UpdateAIProviderActiveFlags(); _userSettings.Changed += UserSettings_Changed; PropertyChanged += (_, e) => { @@ -158,15 +264,20 @@ namespace AdvancedPaste.ViewModels if (Visible) { await ReadClipboardAsync(); - UpdateAllowedByGPO(); } } private void UserSettings_Changed(object sender, EventArgs e) { + UpdateAIProviderActiveFlags(); + OnPropertyChanged(nameof(IsCustomAIServiceEnabled)); OnPropertyChanged(nameof(ClipboardHasDataForCustomAI)); OnPropertyChanged(nameof(IsCustomAIAvailable)); OnPropertyChanged(nameof(IsAdvancedAIEnabled)); + OnPropertyChanged(nameof(AIProviders)); + OnPropertyChanged(nameof(AllowedAIProviders)); + OnPropertyChanged(nameof(ActiveAIProvider)); + OnPropertyChanged(nameof(ActiveAIProviderTooltip)); EnqueueRefreshPasteFormats(); } @@ -192,6 +303,23 @@ namespace AdvancedPaste.ViewModels private PasteFormat CreateCustomAIPasteFormat(string name, string prompt, bool isSavedQuery) => PasteFormat.CreateCustomAIFormat(CustomAIFormat, name, prompt, isSavedQuery, AvailableClipboardFormats, IsCustomAIServiceEnabled); + private void UpdateAIProviderActiveFlags() + { + var providers = _userSettings?.PasteAIConfiguration?.Providers; + if (providers is not null) + { + var activeId = ActiveAIProvider?.Id; + + foreach (var provider in providers) + { + provider.IsActive = !string.IsNullOrEmpty(activeId) && string.Equals(provider.Id, activeId, StringComparison.OrdinalIgnoreCase); + } + } + + OnPropertyChanged(nameof(ActiveAIProvider)); + OnPropertyChanged(nameof(ActiveAIProviderTooltip)); + } + private void RefreshPasteFormats() { var ctrlString = ResourceLoaderInstance.ResourceLoader.GetString("CtrlKey"); @@ -253,8 +381,96 @@ namespace AdvancedPaste.ViewModels return; } - ClipboardData = Clipboard.GetContent(); - AvailableClipboardFormats = await ClipboardData.GetAvailableFormatsAsync(); + try + { + ClipboardData = Clipboard.GetContent(); + AvailableClipboardFormats = ClipboardData != null ? await ClipboardData.GetAvailableFormatsAsync() : ClipboardFormat.None; + } + catch (Exception ex) when (ex is COMException or InvalidOperationException) + { + // Logger.LogDebug("Failed to read clipboard content", ex); + ClipboardData = null; + AvailableClipboardFormats = ClipboardFormat.None; + } + + await UpdateClipboardPreviewAsync(); + } + + private async Task UpdateClipboardPreviewAsync() + { + if (ClipboardData is null || !ClipboardHasData) + { + ResetClipboardPreview(); + _currentClipboardHistoryId = null; + _currentClipboardTimestamp = null; + _lastClipboardFormats = ClipboardFormat.None; + return; + } + + var formatsChanged = AvailableClipboardFormats != _lastClipboardFormats; + _lastClipboardFormats = AvailableClipboardFormats; + + var clipboardChanged = await UpdateClipboardTimestampAsync(formatsChanged); + + // Create ClipboardItem directly from current clipboard data using helper + CurrentClipboardItem = await ClipboardItemHelper.CreateFromCurrentClipboardAsync( + ClipboardData, + AvailableClipboardFormats, + _currentClipboardTimestamp, + clipboardChanged ? null : CurrentClipboardItem?.Image); + } + + private async Task UpdateClipboardTimestampAsync(bool formatsChanged) + { + bool clipboardChanged = formatsChanged; + + if (Clipboard.IsHistoryEnabled()) + { + try + { + var historyItems = await Clipboard.GetHistoryItemsAsync(); + if (historyItems.Status == ClipboardHistoryItemsResultStatus.Success && historyItems.Items.Count > 0) + { + var latest = historyItems.Items[0]; + if (_currentClipboardHistoryId != latest.Id) + { + clipboardChanged = true; + _currentClipboardHistoryId = latest.Id; + } + + _currentClipboardTimestamp = latest.Timestamp; + _clipboardHistoryUnavailableLogged = false; + return clipboardChanged; + } + } + catch (Exception ex) + { + if (!_clipboardHistoryUnavailableLogged) + { + Logger.LogDebug("Failed to access clipboard history timestamp", ex.Message); + _clipboardHistoryUnavailableLogged = true; + } + } + } + + if (!_currentClipboardTimestamp.HasValue || clipboardChanged) + { + _currentClipboardTimestamp = DateTimeOffset.Now; + clipboardChanged = true; + } + + return clipboardChanged; + } + + private void ResetClipboardPreview() + { + // Clear to avoid leaks due to Garbage Collection not clearing the bitmap from memory + if (CurrentClipboardItem?.Image is not null) + { + CurrentClipboardItem.Image.ClearValue(BitmapImage.UriSourceProperty); + } + + CurrentClipboardItem = null; } public async Task OnShowAsync() @@ -270,7 +486,7 @@ namespace AdvancedPaste.ViewModels _dispatcherQueue.TryEnqueue(() => { - GetMainWindow()?.FinishLoading(_aiCredentialsProvider.IsConfigured); + GetMainWindow()?.FinishLoading(IsCustomAIServiceEnabled); OnPropertyChanged(nameof(InputTxtBoxPlaceholderText)); OnPropertyChanged(nameof(CustomAIUnavailableErrorText)); OnPropertyChanged(nameof(IsCustomAIServiceEnabled)); @@ -319,7 +535,7 @@ namespace AdvancedPaste.ViewModels return ResourceLoaderInstance.ResourceLoader.GetString("OpenAIGpoDisabled"); } - if (!_aiCredentialsProvider.IsConfigured) + if (!IsCustomAIServiceEnabled) { return ResourceLoaderInstance.ResourceLoader.GetString("OpenAINotConfigured"); } @@ -515,11 +731,113 @@ namespace AdvancedPaste.ViewModels IsAllowedByGPO = PowerToys.GPOWrapper.GPOWrapper.GetAllowedAdvancedPasteOnlineAIModelsValue() != PowerToys.GPOWrapper.GpoRuleConfigured.Disabled; } + private bool IsProviderAllowedByGPO(PasteAIProviderDefinition provider) + { + if (provider is null) + { + return false; + } + + var serviceType = provider.ServiceType.ToAIServiceType(); + var metadata = AIServiceTypeRegistry.GetMetadata(serviceType); + + // Check global online AI GPO for online services + if (metadata.IsOnlineService && !IsAllowedByGPO) + { + return false; + } + + // Check individual endpoint GPO + return serviceType switch + { + AIServiceType.OpenAI => PowerToys.GPOWrapper.GPOWrapper.GetAllowedAdvancedPasteOpenAIValue() != PowerToys.GPOWrapper.GpoRuleConfigured.Disabled, + AIServiceType.AzureOpenAI => PowerToys.GPOWrapper.GPOWrapper.GetAllowedAdvancedPasteAzureOpenAIValue() != PowerToys.GPOWrapper.GpoRuleConfigured.Disabled, + AIServiceType.AzureAIInference => PowerToys.GPOWrapper.GPOWrapper.GetAllowedAdvancedPasteAzureAIInferenceValue() != PowerToys.GPOWrapper.GpoRuleConfigured.Disabled, + AIServiceType.Mistral => PowerToys.GPOWrapper.GPOWrapper.GetAllowedAdvancedPasteMistralValue() != PowerToys.GPOWrapper.GpoRuleConfigured.Disabled, + AIServiceType.Google => PowerToys.GPOWrapper.GPOWrapper.GetAllowedAdvancedPasteGoogleValue() != PowerToys.GPOWrapper.GpoRuleConfigured.Disabled, + AIServiceType.Anthropic => PowerToys.GPOWrapper.GPOWrapper.GetAllowedAdvancedPasteAnthropicValue() != PowerToys.GPOWrapper.GpoRuleConfigured.Disabled, + AIServiceType.Ollama => PowerToys.GPOWrapper.GPOWrapper.GetAllowedAdvancedPasteOllamaValue() != PowerToys.GPOWrapper.GpoRuleConfigured.Disabled, + AIServiceType.FoundryLocal => PowerToys.GPOWrapper.GPOWrapper.GetAllowedAdvancedPasteFoundryLocalValue() != PowerToys.GPOWrapper.GpoRuleConfigured.Disabled, + _ => true, // Allow unknown types by default + }; + } + + private bool TryResolveAdvancedAIProvider(out PasteAIProviderDefinition provider) + { + provider = null; + + var configuration = _userSettings?.PasteAIConfiguration; + if (configuration is null) + { + return false; + } + + var activeProvider = configuration.ActiveProvider; + if (IsAdvancedAIProvider(activeProvider)) + { + provider = activeProvider; + return true; + } + + if (activeProvider is not null) + { + return false; + } + + var fallback = configuration.Providers?.FirstOrDefault(IsAdvancedAIProvider); + if (fallback is not null) + { + provider = fallback; + return true; + } + + return false; + } + + private static bool IsAdvancedAIProvider(PasteAIProviderDefinition provider) + { + return provider is not null && provider.EnableAdvancedAI && SupportsAdvancedAI(provider.ServiceTypeKind); + } + + private static bool SupportsAdvancedAI(AIServiceType serviceType) + { + return serviceType is AIServiceType.OpenAI + or AIServiceType.AzureOpenAI; + } + private bool UpdateOpenAIKey() { UpdateAllowedByGPO(); - return IsAllowedByGPO && _aiCredentialsProvider.Refresh(); + return _credentialsProvider.Refresh(); + } + + [RelayCommand] + private async Task SetActiveProviderAsync(PasteAIProviderDefinition provider) + { + if (provider is null || string.IsNullOrEmpty(provider.Id)) + { + return; + } + + if (string.Equals(ActiveAIProvider?.Id, provider.Id, StringComparison.OrdinalIgnoreCase)) + { + return; + } + + try + { + await _userSettings.SetActiveAIProviderAsync(provider.Id); + } + catch (Exception ex) + { + Logger.LogError("Failed to activate AI provider", ex); + return; + } + + UpdateAIProviderActiveFlags(); + OnPropertyChanged(nameof(AIProviders)); + EnqueueRefreshPasteFormats(); } public async Task CancelPasteActionAsync() diff --git a/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/AdvancedPasteModuleInterface.vcxproj b/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/AdvancedPasteModuleInterface.vcxproj index 083aa868d3..2cf2920673 100644 --- a/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/AdvancedPasteModuleInterface.vcxproj +++ b/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/AdvancedPasteModuleInterface.vcxproj @@ -2,7 +2,7 @@ - + 15.0 diff --git a/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/dllmain.cpp b/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/dllmain.cpp index 6af0d636ac..c7d22d474f 100644 --- a/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/dllmain.cpp +++ b/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/dllmain.cpp @@ -16,7 +16,8 @@ #include #include -#include +#include +#include #include BOOL APIENTRY DllMain(HMODULE /*hModule*/, DWORD ul_reason_for_call, LPVOID /*lpReserved*/) @@ -54,12 +55,14 @@ namespace const wchar_t JSON_KEY_ADVANCED_PASTE_UI_HOTKEY[] = L"advanced-paste-ui-hotkey"; const wchar_t JSON_KEY_PASTE_AS_MARKDOWN_HOTKEY[] = L"paste-as-markdown-hotkey"; const wchar_t JSON_KEY_PASTE_AS_JSON_HOTKEY[] = L"paste-as-json-hotkey"; - const wchar_t JSON_KEY_IS_ADVANCED_AI_ENABLED[] = L"IsAdvancedAIEnabled"; + const wchar_t JSON_KEY_IS_AI_ENABLED[] = L"IsAIEnabled"; + const wchar_t JSON_KEY_IS_OPEN_AI_ENABLED[] = L"IsOpenAIEnabled"; const wchar_t JSON_KEY_SHOW_CUSTOM_PREVIEW[] = L"ShowCustomPreview"; + const wchar_t JSON_KEY_PASTE_AI_CONFIGURATION[] = L"paste-ai-configuration"; + const wchar_t JSON_KEY_PROVIDERS[] = L"providers"; + const wchar_t JSON_KEY_SERVICE_TYPE[] = L"service-type"; + const wchar_t JSON_KEY_ENABLE_ADVANCED_AI[] = L"enable-advanced-ai"; const wchar_t JSON_KEY_VALUE[] = L"value"; - - const wchar_t OPENAI_VAULT_RESOURCE[] = L"https://platform.openai.com/api-keys"; - const wchar_t OPENAI_VAULT_USERNAME[] = L"PowerToys_AdvancedPaste_OpenAIKey"; } class AdvancedPaste : public PowertoyModuleIface @@ -94,6 +97,7 @@ private: using CustomAction = ActionData; std::vector m_custom_actions; + bool m_is_ai_enabled = false; bool m_is_advanced_ai_enabled = false; bool m_preview_custom_format_output = true; @@ -145,32 +149,11 @@ private: return jsonObject; } - static bool open_ai_key_exists() - { - try - { - winrt::Windows::Security::Credentials::PasswordVault().Retrieve(OPENAI_VAULT_RESOURCE, OPENAI_VAULT_USERNAME); - return true; - } - catch (const winrt::hresult_error& ex) - { - // Looks like the only way to access the PasswordVault is through an API that throws an exception in case the resource doesn't exist. - // If the debugger breaks here, just continue. - // If you want to disable breaking here in a more permanent way, just add a condition in Visual Studio's Exception Settings to not break on win::hresult_error, but that might make you not hit other exceptions you might want to catch. - if (ex.code() == HRESULT_FROM_WIN32(ERROR_NOT_FOUND)) - { - return false; // Credential doesn't exist. - } - Logger::error("Unexpected error while retrieving OpenAI key from vault: {}", winrt::to_string(ex.message())); - return false; - } - } - - bool is_open_ai_enabled() + bool is_ai_enabled() { return gpo_policy_enabled_configuration() != powertoys_gpo::gpo_rule_configured_disabled && powertoys_gpo::getAllowedAdvancedPasteOnlineAIModelsValue() != powertoys_gpo::gpo_rule_configured_disabled && - open_ai_key_exists(); + m_is_ai_enabled; } static std::wstring kebab_to_pascal_case(const std::wstring& kebab_str) @@ -201,6 +184,13 @@ private: return result; } + static std::wstring to_lower_case(const std::wstring& value) + { + std::wstring result = value; + std::transform(result.begin(), result.end(), result.begin(), [](wchar_t ch) { return std::towlower(ch); }); + return result; + } + bool migrate_data_and_remove_data_file(Hotkey& old_paste_as_plain_hotkey) { const wchar_t OLD_JSON_KEY_ACTIVATION_SHORTCUT[] = L"ActivationShortcut"; @@ -267,6 +257,61 @@ private: } } + bool has_advanced_ai_provider(const winrt::Windows::Data::Json::JsonObject& propertiesObject) + { + if (!propertiesObject.HasKey(JSON_KEY_PASTE_AI_CONFIGURATION)) + { + return false; + } + + const auto configValue = propertiesObject.GetNamedValue(JSON_KEY_PASTE_AI_CONFIGURATION); + if (configValue.ValueType() != winrt::Windows::Data::Json::JsonValueType::Object) + { + return false; + } + + const auto configObject = configValue.GetObjectW(); + if (!configObject.HasKey(JSON_KEY_PROVIDERS)) + { + return false; + } + + const auto providersValue = configObject.GetNamedValue(JSON_KEY_PROVIDERS); + if (providersValue.ValueType() != winrt::Windows::Data::Json::JsonValueType::Array) + { + return false; + } + + const auto providers = providersValue.GetArray(); + for (const auto providerValue : providers) + { + if (providerValue.ValueType() != winrt::Windows::Data::Json::JsonValueType::Object) + { + continue; + } + + const auto providerObject = providerValue.GetObjectW(); + if (!providerObject.GetNamedBoolean(JSON_KEY_ENABLE_ADVANCED_AI, false)) + { + continue; + } + + if (!providerObject.HasKey(JSON_KEY_SERVICE_TYPE)) + { + continue; + } + + const std::wstring serviceType = providerObject.GetNamedString(JSON_KEY_SERVICE_TYPE, L"").c_str(); + const auto normalizedServiceType = to_lower_case(serviceType); + if (normalizedServiceType == L"openai" || normalizedServiceType == L"azureopenai") + { + return true; + } + } + + return false; + } + void read_settings(PowerToysSettings::PowerToyValues& settings) { const auto settingsObject = settings.get_raw_json(); @@ -341,7 +386,7 @@ private: if (propertiesObject.HasKey(JSON_KEY_CUSTOM_ACTIONS)) { const auto customActions = propertiesObject.GetNamedObject(JSON_KEY_CUSTOM_ACTIONS).GetNamedArray(JSON_KEY_VALUE); - if (customActions.Size() > 0 && is_open_ai_enabled()) + if (customActions.Size() > 0 && is_ai_enabled()) { for (const auto& customAction : customActions) { @@ -365,9 +410,19 @@ private: { const auto propertiesObject = settingsObject.GetNamedObject(JSON_KEY_PROPERTIES); - if (propertiesObject.HasKey(JSON_KEY_IS_ADVANCED_AI_ENABLED)) + m_is_advanced_ai_enabled = has_advanced_ai_provider(propertiesObject); + + if (propertiesObject.HasKey(JSON_KEY_IS_AI_ENABLED)) { - m_is_advanced_ai_enabled = propertiesObject.GetNamedObject(JSON_KEY_IS_ADVANCED_AI_ENABLED).GetNamedBoolean(JSON_KEY_VALUE); + m_is_ai_enabled = propertiesObject.GetNamedObject(JSON_KEY_IS_AI_ENABLED).GetNamedBoolean(JSON_KEY_VALUE, false); + } + else if (propertiesObject.HasKey(JSON_KEY_IS_OPEN_AI_ENABLED)) + { + m_is_ai_enabled = propertiesObject.GetNamedObject(JSON_KEY_IS_OPEN_AI_ENABLED).GetNamedBoolean(JSON_KEY_VALUE, false); + } + else + { + m_is_ai_enabled = false; } if (propertiesObject.HasKey(JSON_KEY_SHOW_CUSTOM_PREVIEW)) diff --git a/src/modules/AdvancedPaste/UITest-AdvancedPaste/TestFiles/settings.json b/src/modules/AdvancedPaste/UITest-AdvancedPaste/TestFiles/settings.json index 31ad05c701..bc0803796e 100644 --- a/src/modules/AdvancedPaste/UITest-AdvancedPaste/TestFiles/settings.json +++ b/src/modules/AdvancedPaste/UITest-AdvancedPaste/TestFiles/settings.json @@ -1 +1 @@ -{"properties":{"IsAdvancedAIEnabled":{"value":false},"ShowCustomPreview":{"value":true},"CloseAfterLosingFocus":{"value":false},"advanced-paste-ui-hotkey":{"win":true,"ctrl":false,"alt":false,"shift":true,"code":86,"key":""},"paste-as-plain-hotkey":{"win":true,"ctrl":true,"alt":true,"shift":false,"code":79,"key":""},"paste-as-markdown-hotkey":{"win":true,"ctrl":true,"alt":true,"shift":false,"code":77,"key":""},"paste-as-json-hotkey":{"win":true,"ctrl":true,"alt":true,"shift":false,"code":74,"key":""},"custom-actions":{"value":[]},"additional-actions":{"image-to-text":{"shortcut":{"win":false,"ctrl":false,"alt":false,"shift":false,"code":0,"key":""},"isShown":true},"paste-as-file":{"isShown":true,"paste-as-txt-file":{"shortcut":{"win":false,"ctrl":false,"alt":false,"shift":false,"code":0,"key":""},"isShown":true},"paste-as-png-file":{"shortcut":{"win":false,"ctrl":false,"alt":false,"shift":false,"code":0,"key":""},"isShown":true},"paste-as-html-file":{"shortcut":{"win":false,"ctrl":false,"alt":false,"shift":false,"code":0,"key":""},"isShown":true}},"transcode":{"isShown":true,"transcode-to-mp3":{"shortcut":{"win":false,"ctrl":false,"alt":false,"shift":false,"code":0,"key":""},"isShown":true},"transcode-to-mp4":{"shortcut":{"win":false,"ctrl":false,"alt":false,"shift":false,"code":0,"key":""},"isShown":true}}}},"name":"AdvancedPaste","version":"1"} \ No newline at end of file +{"properties":{"IsAIEnabled":{"value":false},"ShowCustomPreview":{"value":true},"CloseAfterLosingFocus":{"value":false},"advanced-paste-ui-hotkey":{"win":true,"ctrl":false,"alt":false,"shift":true,"code":86,"key":""},"paste-as-plain-hotkey":{"win":true,"ctrl":true,"alt":true,"shift":false,"code":79,"key":""},"paste-as-markdown-hotkey":{"win":true,"ctrl":true,"alt":true,"shift":false,"code":77,"key":""},"paste-as-json-hotkey":{"win":true,"ctrl":true,"alt":true,"shift":false,"code":74,"key":""},"custom-actions":{"value":[]},"additional-actions":{"image-to-text":{"shortcut":{"win":false,"ctrl":false,"alt":false,"shift":false,"code":0,"key":""},"isShown":true},"paste-as-file":{"isShown":true,"paste-as-txt-file":{"shortcut":{"win":false,"ctrl":false,"alt":false,"shift":false,"code":0,"key":""},"isShown":true},"paste-as-png-file":{"shortcut":{"win":false,"ctrl":false,"alt":false,"shift":false,"code":0,"key":""},"isShown":true},"paste-as-html-file":{"shortcut":{"win":false,"ctrl":false,"alt":false,"shift":false,"code":0,"key":""},"isShown":true}},"transcode":{"isShown":true,"transcode-to-mp3":{"shortcut":{"win":false,"ctrl":false,"alt":false,"shift":false,"code":0,"key":""},"isShown":true},"transcode-to-mp4":{"shortcut":{"win":false,"ctrl":false,"alt":false,"shift":false,"code":0,"key":""},"isShown":true}}},"paste-ai-configuration":{"active-provider-id":"","providers":[],"use-shared-credentials":true}},"name":"AdvancedPaste","version":"1"} \ No newline at end of file diff --git a/src/modules/cmdpal/ExtensionTemplate/TemplateCmdPalExtension/nuget.config b/src/modules/cmdpal/ExtensionTemplate/TemplateCmdPalExtension/nuget.config deleted file mode 100644 index e6a17ffdfe..0000000000 --- a/src/modules/cmdpal/ExtensionTemplate/TemplateCmdPalExtension/nuget.config +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - diff --git a/src/modules/cmdpal/Microsoft.CmdPal.UI/Microsoft.CmdPal.UI.csproj b/src/modules/cmdpal/Microsoft.CmdPal.UI/Microsoft.CmdPal.UI.csproj index 15bc3920fd..bd7402e4fd 100644 --- a/src/modules/cmdpal/Microsoft.CmdPal.UI/Microsoft.CmdPal.UI.csproj +++ b/src/modules/cmdpal/Microsoft.CmdPal.UI/Microsoft.CmdPal.UI.csproj @@ -111,6 +111,10 @@ + + + + diff --git a/src/settings-ui/Settings.UI.Library/AIProviderConfigurationSnapshot.cs b/src/settings-ui/Settings.UI.Library/AIProviderConfigurationSnapshot.cs new file mode 100644 index 0000000000..456632545f --- /dev/null +++ b/src/settings-ui/Settings.UI.Library/AIProviderConfigurationSnapshot.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text.Json.Serialization; + +namespace Microsoft.PowerToys.Settings.UI.Library +{ + /// + /// Stores provider-specific configuration overrides so each AI service can keep distinct settings. + /// + public class AIProviderConfigurationSnapshot + { + [JsonPropertyName("model-name")] + public string ModelName { get; set; } = string.Empty; + + [JsonPropertyName("endpoint-url")] + public string EndpointUrl { get; set; } = string.Empty; + + [JsonPropertyName("api-version")] + public string ApiVersion { get; set; } = string.Empty; + + [JsonPropertyName("deployment-name")] + public string DeploymentName { get; set; } = string.Empty; + + [JsonPropertyName("model-path")] + public string ModelPath { get; set; } = string.Empty; + + [JsonPropertyName("system-prompt")] + public string SystemPrompt { get; set; } = string.Empty; + + [JsonPropertyName("moderation-enabled")] + public bool ModerationEnabled { get; set; } = true; + } +} diff --git a/src/settings-ui/Settings.UI.Library/AIServiceType.cs b/src/settings-ui/Settings.UI.Library/AIServiceType.cs new file mode 100644 index 0000000000..5cbd6a855c --- /dev/null +++ b/src/settings-ui/Settings.UI.Library/AIServiceType.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.PowerToys.Settings.UI.Library +{ + /// + /// Supported AI service types for PowerToys AI experiences. + /// + public enum AIServiceType + { + Unknown = 0, + OpenAI, + AzureOpenAI, + Onnx, + ML, + FoundryLocal, + Mistral, + Google, + HuggingFace, + AzureAIInference, + Ollama, + Anthropic, + AmazonBedrock, + } +} diff --git a/src/settings-ui/Settings.UI.Library/AIServiceTypeExtensions.cs b/src/settings-ui/Settings.UI.Library/AIServiceTypeExtensions.cs new file mode 100644 index 0000000000..91f035d23a --- /dev/null +++ b/src/settings-ui/Settings.UI.Library/AIServiceTypeExtensions.cs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.PowerToys.Settings.UI.Library +{ + public static class AIServiceTypeExtensions + { + /// + /// Convert a persisted string value into an . + /// Supports historical casing and aliases. + /// + public static AIServiceType ToAIServiceType(this string serviceType) + { + if (string.IsNullOrWhiteSpace(serviceType)) + { + return AIServiceType.OpenAI; + } + + var normalized = serviceType.Trim().ToLowerInvariant(); + return normalized switch + { + "openai" => AIServiceType.OpenAI, + "azureopenai" or "azure" => AIServiceType.AzureOpenAI, + "onnx" => AIServiceType.Onnx, + "foundrylocal" or "foundry" or "fl" => AIServiceType.FoundryLocal, + "ml" or "windowsml" or "winml" => AIServiceType.ML, + "mistral" => AIServiceType.Mistral, + "google" or "googleai" or "googlegemini" => AIServiceType.Google, + "huggingface" => AIServiceType.HuggingFace, + "azureaiinference" or "azureinference" => AIServiceType.AzureAIInference, + "ollama" => AIServiceType.Ollama, + "anthropic" => AIServiceType.Anthropic, + "amazonbedrock" or "bedrock" => AIServiceType.AmazonBedrock, + _ => AIServiceType.Unknown, + }; + } + + /// + /// Convert an to the canonical string used for persistence. + /// + public static string ToConfigurationString(this AIServiceType serviceType) + { + return serviceType switch + { + AIServiceType.OpenAI => "OpenAI", + AIServiceType.AzureOpenAI => "AzureOpenAI", + AIServiceType.Onnx => "Onnx", + AIServiceType.FoundryLocal => "FoundryLocal", + AIServiceType.ML => "ML", + AIServiceType.Mistral => "Mistral", + AIServiceType.Google => "Google", + AIServiceType.HuggingFace => "HuggingFace", + AIServiceType.AzureAIInference => "AzureAIInference", + AIServiceType.Ollama => "Ollama", + AIServiceType.Anthropic => "Anthropic", + AIServiceType.AmazonBedrock => "AmazonBedrock", + AIServiceType.Unknown => string.Empty, + _ => throw new ArgumentOutOfRangeException(nameof(serviceType), serviceType, "Unsupported AI service type."), + }; + } + + /// + /// Convert an into the normalized key used internally. + /// + public static string ToNormalizedKey(this AIServiceType serviceType) + { + return serviceType switch + { + AIServiceType.OpenAI => "openai", + AIServiceType.AzureOpenAI => "azureopenai", + AIServiceType.Onnx => "onnx", + AIServiceType.FoundryLocal => "foundrylocal", + AIServiceType.ML => "ml", + AIServiceType.Mistral => "mistral", + AIServiceType.Google => "google", + AIServiceType.HuggingFace => "huggingface", + AIServiceType.AzureAIInference => "azureaiinference", + AIServiceType.Ollama => "ollama", + AIServiceType.Anthropic => "anthropic", + AIServiceType.AmazonBedrock => "amazonbedrock", + _ => string.Empty, + }; + } + } +} diff --git a/src/settings-ui/Settings.UI.Library/AIServiceTypeMetadata.cs b/src/settings-ui/Settings.UI.Library/AIServiceTypeMetadata.cs new file mode 100644 index 0000000000..df01b1816a --- /dev/null +++ b/src/settings-ui/Settings.UI.Library/AIServiceTypeMetadata.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.PowerToys.Settings.UI.Library +{ + /// + /// Metadata information for an AI service type. + /// + public class AIServiceTypeMetadata + { + public AIServiceType ServiceType { get; init; } + + public string DisplayName { get; init; } + + public string IconPath { get; init; } + + public bool IsOnlineService { get; init; } + + public bool IsAvailableInUI { get; init; } = true; + + public bool IsLocalModel { get; init; } + + public string LegalDescription { get; init; } + + public string TermsLabel { get; init; } + + public Uri TermsUri { get; init; } + + public string PrivacyLabel { get; init; } + + public Uri PrivacyUri { get; init; } + + public bool HasLegalInfo => !string.IsNullOrWhiteSpace(LegalDescription); + + public bool HasTermsLink => TermsUri is not null && !string.IsNullOrEmpty(TermsLabel); + + public bool HasPrivacyLink => PrivacyUri is not null && !string.IsNullOrEmpty(PrivacyLabel); + } +} diff --git a/src/settings-ui/Settings.UI.Library/AIServiceTypeRegistry.cs b/src/settings-ui/Settings.UI.Library/AIServiceTypeRegistry.cs new file mode 100644 index 0000000000..ca0bd9e011 --- /dev/null +++ b/src/settings-ui/Settings.UI.Library/AIServiceTypeRegistry.cs @@ -0,0 +1,222 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.PowerToys.Settings.UI.Library; + +/// +/// Centralized registry for AI service type metadata. +/// +public static class AIServiceTypeRegistry +{ + private static readonly Dictionary MetadataMap = new() + { + [AIServiceType.AmazonBedrock] = new AIServiceTypeMetadata + { + ServiceType = AIServiceType.AmazonBedrock, + DisplayName = "Amazon Bedrock", + IsAvailableInUI = false, // Currently disabled in UI + IconPath = "ms-appx:///Assets/Settings/Icons/Models/Bedrock.svg", + IsOnlineService = true, + LegalDescription = "AdvancedPaste_AmazonBedrock_LegalDescription", + TermsLabel = "AdvancedPaste_AmazonBedrock_TermsLabel", + TermsUri = new Uri("https://aws.amazon.com/service-terms/"), + PrivacyLabel = "AdvancedPaste_AmazonBedrock_PrivacyLabel", + PrivacyUri = new Uri("https://aws.amazon.com/privacy/"), + }, + [AIServiceType.Anthropic] = new AIServiceTypeMetadata + { + ServiceType = AIServiceType.Anthropic, + DisplayName = "Anthropic", + IconPath = "ms-appx:///Assets/Settings/Icons/Models/Anthropic.svg", + IsOnlineService = true, + LegalDescription = "AdvancedPaste_Anthropic_LegalDescription", + TermsLabel = "AdvancedPaste_Anthropic_TermsLabel", + TermsUri = new Uri("https://www.anthropic.com/legal/terms-of-service"), + PrivacyLabel = "AdvancedPaste_Anthropic_PrivacyLabel", + PrivacyUri = new Uri("https://www.anthropic.com/legal/privacy"), + }, + [AIServiceType.AzureAIInference] = new AIServiceTypeMetadata + { + ServiceType = AIServiceType.AzureAIInference, + DisplayName = "Azure AI Inference", + IconPath = "ms-appx:///Assets/Settings/Icons/Models/FoundryLocal.svg", // No icon for Azure AI Inference, use Foundry Local temporarily + IsOnlineService = true, + LegalDescription = "AdvancedPaste_AzureAIInference_LegalDescription", + TermsLabel = "AdvancedPaste_AzureAIInference_TermsLabel", + TermsUri = new Uri("https://azure.microsoft.com/support/legal/"), + PrivacyLabel = "AdvancedPaste_AzureAIInference_PrivacyLabel", + PrivacyUri = new Uri("https://privacy.microsoft.com/privacystatement"), + }, + [AIServiceType.AzureOpenAI] = new AIServiceTypeMetadata + { + ServiceType = AIServiceType.AzureOpenAI, + DisplayName = "Azure OpenAI", + IconPath = "ms-appx:///Assets/Settings/Icons/Models/AzureAI.svg", + IsOnlineService = true, + LegalDescription = "AdvancedPaste_AzureOpenAI_LegalDescription", + TermsLabel = "AdvancedPaste_AzureOpenAI_TermsLabel", + TermsUri = new Uri("https://azure.microsoft.com/support/legal/"), + PrivacyLabel = "AdvancedPaste_AzureOpenAI_PrivacyLabel", + PrivacyUri = new Uri("https://privacy.microsoft.com/privacystatement"), + }, + [AIServiceType.FoundryLocal] = new AIServiceTypeMetadata + { + ServiceType = AIServiceType.FoundryLocal, + DisplayName = "Foundry Local", + IconPath = "ms-appx:///Assets/Settings/Icons/Models/FoundryLocal.svg", + IsOnlineService = false, + IsLocalModel = true, + LegalDescription = "AdvancedPaste_FoundryLocal_LegalDescription", // Resource key for localized description + }, + [AIServiceType.Google] = new AIServiceTypeMetadata + { + ServiceType = AIServiceType.Google, + DisplayName = "Google", + IconPath = "ms-appx:///Assets/Settings/Icons/Models/Gemini.svg", + IsOnlineService = true, + LegalDescription = "AdvancedPaste_Google_LegalDescription", + TermsLabel = "AdvancedPaste_Google_TermsLabel", + TermsUri = new Uri("https://policies.google.com/terms"), + PrivacyLabel = "AdvancedPaste_Google_PrivacyLabel", + PrivacyUri = new Uri("https://policies.google.com/privacy"), + }, + [AIServiceType.HuggingFace] = new AIServiceTypeMetadata + { + ServiceType = AIServiceType.HuggingFace, + DisplayName = "Hugging Face", + IconPath = "ms-appx:///Assets/Settings/Icons/Models/HuggingFace.svg", + IsOnlineService = true, + IsAvailableInUI = false, // Currently disabled in UI + }, + [AIServiceType.Mistral] = new AIServiceTypeMetadata + { + ServiceType = AIServiceType.Mistral, + DisplayName = "Mistral", + IconPath = "ms-appx:///Assets/Settings/Icons/Models/Mistral.svg", + IsOnlineService = true, + LegalDescription = "AdvancedPaste_Mistral_LegalDescription", + TermsLabel = "AdvancedPaste_Mistral_TermsLabel", + TermsUri = new Uri("https://mistral.ai/terms-of-service/"), + PrivacyLabel = "AdvancedPaste_Mistral_PrivacyLabel", + PrivacyUri = new Uri("https://mistral.ai/privacy-policy/"), + }, + [AIServiceType.ML] = new AIServiceTypeMetadata + { + ServiceType = AIServiceType.ML, + DisplayName = "Windows ML", + IconPath = "ms-appx:///Assets/Settings/Icons/Models/WindowsML.svg", + LegalDescription = "AdvancedPaste_LocalModel_LegalDescription", + IsAvailableInUI = false, + IsOnlineService = false, + IsLocalModel = true, + }, + [AIServiceType.Ollama] = new AIServiceTypeMetadata + { + ServiceType = AIServiceType.Ollama, + DisplayName = "Ollama", + IconPath = "ms-appx:///Assets/Settings/Icons/Models/Ollama.svg", + + // Ollama provide online service, but we treat it as local model at first version since it can is known for local model. + IsOnlineService = false, + IsLocalModel = true, + LegalDescription = "AdvancedPaste_LocalModel_LegalDescription", + TermsLabel = "AdvancedPaste_Ollama_TermsLabel", + TermsUri = new Uri("https://ollama.com/terms"), + PrivacyLabel = "AdvancedPaste_Ollama_PrivacyLabel", + PrivacyUri = new Uri("https://ollama.com/privacy"), + }, + [AIServiceType.Onnx] = new AIServiceTypeMetadata + { + ServiceType = AIServiceType.Onnx, + DisplayName = "ONNX", + LegalDescription = "AdvancedPaste_LocalModel_LegalDescription", + IconPath = "ms-appx:///Assets/Settings/Icons/Models/Onnx.svg", + IsOnlineService = false, + IsAvailableInUI = false, + }, + [AIServiceType.OpenAI] = new AIServiceTypeMetadata + { + ServiceType = AIServiceType.OpenAI, + DisplayName = "OpenAI", + IconPath = "ms-appx:///Assets/Settings/Icons/Models/OpenAI.light.svg", + IsOnlineService = true, + LegalDescription = "AdvancedPaste_OpenAI_LegalDescription", + TermsLabel = "AdvancedPaste_OpenAI_TermsLabel", + TermsUri = new Uri("https://openai.com/terms"), + PrivacyLabel = "AdvancedPaste_OpenAI_PrivacyLabel", + PrivacyUri = new Uri("https://openai.com/privacy"), + }, + [AIServiceType.Unknown] = new AIServiceTypeMetadata + { + ServiceType = AIServiceType.Unknown, + DisplayName = "Unknown", + IconPath = "ms-appx:///Assets/Settings/Icons/Models/OpenAI.light.svg", + IsOnlineService = false, + IsAvailableInUI = false, + }, + }; + + /// + /// Get metadata for a specific service type. + /// + public static AIServiceTypeMetadata GetMetadata(AIServiceType serviceType) + { + return MetadataMap.TryGetValue(serviceType, out var metadata) + ? metadata + : MetadataMap[AIServiceType.Unknown]; + } + + /// + /// Get metadata for a service type from its string representation. + /// + public static AIServiceTypeMetadata GetMetadata(string serviceType) + { + var type = serviceType.ToAIServiceType(); + return GetMetadata(type); + } + + /// + /// Get icon path for a service type. + /// + public static string GetIconPath(AIServiceType serviceType) + { + return GetMetadata(serviceType).IconPath; + } + + /// + /// Get icon path for a service type from its string representation. + /// + public static string GetIconPath(string serviceType) + { + return GetMetadata(serviceType).IconPath; + } + + /// + /// Get all service types available in the UI. + /// + public static IEnumerable GetAvailableServiceTypes() + { + return MetadataMap.Values.Where(m => m.IsAvailableInUI); + } + + /// + /// Get all online service types available in the UI. + /// + public static IEnumerable GetOnlineServiceTypes() + { + return GetAvailableServiceTypes().Where(m => m.IsOnlineService); + } + + /// + /// Get all local service types available in the UI. + /// + public static IEnumerable GetLocalServiceTypes() + { + return GetAvailableServiceTypes().Where(m => m.IsLocalModel); + } +} diff --git a/src/settings-ui/Settings.UI.Library/AdvancedPasteCustomAction.cs b/src/settings-ui/Settings.UI.Library/AdvancedPasteCustomAction.cs index 43baf89351..c981295906 100644 --- a/src/settings-ui/Settings.UI.Library/AdvancedPasteCustomAction.cs +++ b/src/settings-ui/Settings.UI.Library/AdvancedPasteCustomAction.cs @@ -14,6 +14,7 @@ public sealed class AdvancedPasteCustomAction : Observable, IAdvancedPasteAction { private int _id; private string _name = string.Empty; + private string _description = string.Empty; private string _prompt = string.Empty; private HotkeySettings _shortcut = new(); private bool _isShown; @@ -43,6 +44,13 @@ public sealed class AdvancedPasteCustomAction : Observable, IAdvancedPasteAction } } + [JsonPropertyName("description")] + public string Description + { + get => _description; + set => Set(ref _description, value ?? string.Empty); + } + [JsonPropertyName("prompt")] public string Prompt { @@ -128,6 +136,7 @@ public sealed class AdvancedPasteCustomAction : Observable, IAdvancedPasteAction { Id = other.Id; Name = other.Name; + Description = other.Description; Prompt = other.Prompt; Shortcut = other.GetShortcutClone(); IsShown = other.IsShown; diff --git a/src/settings-ui/Settings.UI.Library/AdvancedPasteProperties.cs b/src/settings-ui/Settings.UI.Library/AdvancedPasteProperties.cs index d40bd686d3..200e5e459d 100644 --- a/src/settings-ui/Settings.UI.Library/AdvancedPasteProperties.cs +++ b/src/settings-ui/Settings.UI.Library/AdvancedPasteProperties.cs @@ -2,6 +2,7 @@ // The Microsoft Corporation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System.Collections.Generic; using System.Text.Json; using System.Text.Json.Serialization; @@ -23,13 +24,38 @@ namespace Microsoft.PowerToys.Settings.UI.Library PasteAsJsonShortcut = new(); CustomActions = new(); AdditionalActions = new(); - IsAdvancedAIEnabled = false; + IsAIEnabled = false; ShowCustomPreview = true; CloseAfterLosingFocus = false; + PasteAIConfiguration = new(); } [JsonConverter(typeof(BoolPropertyJsonConverter))] - public bool IsAdvancedAIEnabled { get; set; } + public bool IsAIEnabled { get; set; } + + [JsonExtensionData] + public Dictionary ExtensionData + { + get => _extensionData; + set + { + _extensionData = value; + + if (_extensionData != null && _extensionData.TryGetValue("IsOpenAIEnabled", out var legacyElement) && legacyElement.ValueKind == JsonValueKind.Object && legacyElement.TryGetProperty("value", out var valueElement)) + { + IsAIEnabled = valueElement.ValueKind switch + { + JsonValueKind.True => true, + JsonValueKind.False => false, + _ => IsAIEnabled, + }; + + _extensionData.Remove("IsOpenAIEnabled"); + } + } + } + + private Dictionary _extensionData; [JsonConverter(typeof(BoolPropertyJsonConverter))] public bool ShowCustomPreview { get; set; } @@ -57,6 +83,10 @@ namespace Microsoft.PowerToys.Settings.UI.Library [CmdConfigureIgnoreAttribute] public AdvancedPasteAdditionalActions AdditionalActions { get; init; } + [JsonPropertyName("paste-ai-configuration")] + [CmdConfigureIgnoreAttribute] + public PasteAIConfiguration PasteAIConfiguration { get; set; } + public override string ToString() => JsonSerializer.Serialize(this); } diff --git a/src/settings-ui/Settings.UI.Library/PasteAIConfiguration.cs b/src/settings-ui/Settings.UI.Library/PasteAIConfiguration.cs new file mode 100644 index 0000000000..3ed0937280 --- /dev/null +++ b/src/settings-ui/Settings.UI.Library/PasteAIConfiguration.cs @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.ComponentModel; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.PowerToys.Settings.UI.Library +{ + /// + /// Configuration for Paste AI features (custom action transformations like custom prompt processing) + /// + public class PasteAIConfiguration : INotifyPropertyChanged + { + private string _activeProviderId = string.Empty; + private ObservableCollection _providers = new(); + private bool _useSharedCredentials = true; + private Dictionary _legacyProviderConfigurations; + + public event PropertyChangedEventHandler PropertyChanged; + + [JsonPropertyName("active-provider-id")] + public string ActiveProviderId + { + get => _activeProviderId; + set => SetProperty(ref _activeProviderId, value ?? string.Empty); + } + + [JsonPropertyName("providers")] + public ObservableCollection Providers + { + get => _providers; + set => SetProperty(ref _providers, value ?? new ObservableCollection()); + } + + [JsonPropertyName("use-shared-credentials")] + public bool UseSharedCredentials + { + get => _useSharedCredentials; + set => SetProperty(ref _useSharedCredentials, value); + } + + [JsonPropertyName("provider-configurations")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Dictionary LegacyProviderConfigurations + { + get => _legacyProviderConfigurations; + set => _legacyProviderConfigurations = value; + } + + [JsonIgnore] + public PasteAIProviderDefinition ActiveProvider + { + get + { + if (_providers is null || _providers.Count == 0) + { + return null; + } + + if (!string.IsNullOrWhiteSpace(_activeProviderId)) + { + var match = _providers.FirstOrDefault(provider => string.Equals(provider.Id, _activeProviderId, StringComparison.OrdinalIgnoreCase)); + if (match is not null) + { + return match; + } + } + + return _providers[0]; + } + } + + [JsonIgnore] + public AIServiceType ActiveServiceTypeKind => ActiveProvider?.ServiceTypeKind ?? AIServiceType.OpenAI; + + public override string ToString() + => JsonSerializer.Serialize(this); + + protected bool SetProperty(ref T field, T value, [CallerMemberName] string propertyName = null) + { + if (EqualityComparer.Default.Equals(field, value)) + { + return false; + } + + field = value; + OnPropertyChanged(propertyName); + return true; + } + + protected void OnPropertyChanged(string propertyName) + { + PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName)); + } + } +} diff --git a/src/settings-ui/Settings.UI.Library/PasteAIProviderDefinition.cs b/src/settings-ui/Settings.UI.Library/PasteAIProviderDefinition.cs new file mode 100644 index 0000000000..0fbb3328e7 --- /dev/null +++ b/src/settings-ui/Settings.UI.Library/PasteAIProviderDefinition.cs @@ -0,0 +1,175 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Runtime.CompilerServices; +using System.Text.Json.Serialization; + +namespace Microsoft.PowerToys.Settings.UI.Library +{ + /// + /// Represents a single Paste AI provider configuration entry. + /// + public class PasteAIProviderDefinition : INotifyPropertyChanged + { + private string _id = Guid.NewGuid().ToString("N"); + private string _serviceType = "OpenAI"; + private string _modelName = string.Empty; + private string _endpointUrl = string.Empty; + private string _apiVersion = string.Empty; + private string _deploymentName = string.Empty; + private string _modelPath = string.Empty; + private string _systemPrompt = string.Empty; + private bool _moderationEnabled = true; + private bool _isActive; + private bool _enableAdvancedAI; + private bool _isLocalModel; + + public event PropertyChangedEventHandler PropertyChanged; + + [JsonPropertyName("id")] + public string Id + { + get => _id; + set => SetProperty(ref _id, value); + } + + [JsonPropertyName("service-type")] + public string ServiceType + { + get => _serviceType; + set + { + if (SetProperty(ref _serviceType, string.IsNullOrWhiteSpace(value) ? "OpenAI" : value)) + { + OnPropertyChanged(nameof(DisplayName)); + } + } + } + + [JsonIgnore] + public AIServiceType ServiceTypeKind + { + get => ServiceType.ToAIServiceType(); + set => ServiceType = value.ToConfigurationString(); + } + + [JsonPropertyName("model-name")] + public string ModelName + { + get => _modelName; + set + { + if (SetProperty(ref _modelName, value ?? string.Empty)) + { + OnPropertyChanged(nameof(DisplayName)); + } + } + } + + [JsonPropertyName("endpoint-url")] + public string EndpointUrl + { + get => _endpointUrl; + set => SetProperty(ref _endpointUrl, value ?? string.Empty); + } + + [JsonPropertyName("api-version")] + public string ApiVersion + { + get => _apiVersion; + set => SetProperty(ref _apiVersion, value ?? string.Empty); + } + + [JsonPropertyName("deployment-name")] + public string DeploymentName + { + get => _deploymentName; + set => SetProperty(ref _deploymentName, value ?? string.Empty); + } + + [JsonPropertyName("model-path")] + public string ModelPath + { + get => _modelPath; + set => SetProperty(ref _modelPath, value ?? string.Empty); + } + + [JsonPropertyName("system-prompt")] + public string SystemPrompt + { + get => _systemPrompt; + set => SetProperty(ref _systemPrompt, value?.Trim() ?? string.Empty); + } + + [JsonPropertyName("moderation-enabled")] + public bool ModerationEnabled + { + get => _moderationEnabled; + set => SetProperty(ref _moderationEnabled, value); + } + + [JsonPropertyName("enable-advanced-ai")] + public bool EnableAdvancedAI + { + get => _enableAdvancedAI; + set => SetProperty(ref _enableAdvancedAI, value); + } + + [JsonPropertyName("is-local-model")] + public bool IsLocalModel + { + get => _isLocalModel; + set => SetProperty(ref _isLocalModel, value); + } + + [JsonIgnore] + public bool IsActive + { + get => _isActive; + set => SetProperty(ref _isActive, value); + } + + [JsonIgnore] + public string DisplayName => string.IsNullOrWhiteSpace(ModelName) ? ServiceType : ModelName; + + public PasteAIProviderDefinition Clone() + { + return new PasteAIProviderDefinition + { + Id = Id, + ServiceType = ServiceType, + ModelName = ModelName, + EndpointUrl = EndpointUrl, + ApiVersion = ApiVersion, + DeploymentName = DeploymentName, + ModelPath = ModelPath, + SystemPrompt = SystemPrompt, + ModerationEnabled = ModerationEnabled, + EnableAdvancedAI = EnableAdvancedAI, + IsLocalModel = IsLocalModel, + IsActive = IsActive, + }; + } + + protected bool SetProperty(ref T field, T value, [CallerMemberName] string propertyName = null) + { + if (EqualityComparer.Default.Equals(field, value)) + { + return false; + } + + field = value; + OnPropertyChanged(propertyName); + return true; + } + + protected void OnPropertyChanged(string propertyName) + { + PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName)); + } + } +} diff --git a/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Anthropic.svg b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Anthropic.svg new file mode 100644 index 0000000000..f990d4650f --- /dev/null +++ b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Anthropic.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Azure.svg b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Azure.svg new file mode 100644 index 0000000000..7497187ad7 --- /dev/null +++ b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Azure.svg @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/AzureAI.svg b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/AzureAI.svg new file mode 100644 index 0000000000..e6fd7121b2 --- /dev/null +++ b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/AzureAI.svg @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Bedrock.svg b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Bedrock.svg new file mode 100644 index 0000000000..d7a3d800d9 --- /dev/null +++ b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Bedrock.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/FoundryLocal.svg b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/FoundryLocal.svg new file mode 100644 index 0000000000..7066f294f9 --- /dev/null +++ b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/FoundryLocal.svg @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Gemini.svg b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Gemini.svg new file mode 100644 index 0000000000..56a5fe461b --- /dev/null +++ b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Gemini.svg @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/HuggingFace.svg b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/HuggingFace.svg new file mode 100644 index 0000000000..9fe0aff336 --- /dev/null +++ b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/HuggingFace.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Mistral.svg b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Mistral.svg new file mode 100644 index 0000000000..ce2471552e --- /dev/null +++ b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Mistral.svg @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Ollama.svg b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Ollama.svg new file mode 100644 index 0000000000..e44dda654d --- /dev/null +++ b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Ollama.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Onnx.svg b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Onnx.svg new file mode 100644 index 0000000000..301a40fd55 --- /dev/null +++ b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/Onnx.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/OpenAI.dark.svg b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/OpenAI.dark.svg new file mode 100644 index 0000000000..87aacb3a4f --- /dev/null +++ b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/OpenAI.dark.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/OpenAI.light.svg b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/OpenAI.light.svg new file mode 100644 index 0000000000..f72a3c64d1 --- /dev/null +++ b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/OpenAI.light.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/WindowsML.svg b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/WindowsML.svg new file mode 100644 index 0000000000..fafc16b59f --- /dev/null +++ b/src/settings-ui/Settings.UI/Assets/Settings/Icons/Models/WindowsML.svg @@ -0,0 +1,74 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/settings-ui/Settings.UI/Converters/ServiceTypeToIconConverter.cs b/src/settings-ui/Settings.UI/Converters/ServiceTypeToIconConverter.cs new file mode 100644 index 0000000000..7d632906c2 --- /dev/null +++ b/src/settings-ui/Settings.UI/Converters/ServiceTypeToIconConverter.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using Microsoft.PowerToys.Settings.UI.Library; +using Microsoft.UI.Xaml.Controls; +using Microsoft.UI.Xaml.Data; +using Microsoft.UI.Xaml.Media.Imaging; + +namespace Microsoft.PowerToys.Settings.UI.Converters; + +public partial class ServiceTypeToIconConverter : IValueConverter +{ + public object Convert(object value, Type targetType, object parameter, string language) + { + if (value is not string serviceType || string.IsNullOrWhiteSpace(serviceType)) + { + return new ImageIcon { Source = new SvgImageSource(new Uri(AIServiceTypeRegistry.GetIconPath(AIServiceType.OpenAI))) }; + } + + var iconPath = AIServiceTypeRegistry.GetIconPath(serviceType); + return new ImageIcon { Source = new SvgImageSource(new Uri(iconPath)) }; + } + + public object ConvertBack(object value, Type targetType, object parameter, string language) + { + throw new NotImplementedException(); + } +} diff --git a/src/settings-ui/Settings.UI/PowerToys.Settings.csproj b/src/settings-ui/Settings.UI/PowerToys.Settings.csproj index dd70af7533..057413a408 100644 --- a/src/settings-ui/Settings.UI/PowerToys.Settings.csproj +++ b/src/settings-ui/Settings.UI/PowerToys.Settings.csproj @@ -20,6 +20,12 @@ PowerToys.Settings.pri + + + + + + @@ -53,6 +59,13 @@ + + + + PreserveNewest + + + @@ -105,6 +118,7 @@ + @@ -197,4 +211,4 @@ - \ No newline at end of file + diff --git a/src/settings-ui/Settings.UI/SettingsXAML/Controls/ModelPicker/FoundryLocalModelPicker.xaml b/src/settings-ui/Settings.UI/SettingsXAML/Controls/ModelPicker/FoundryLocalModelPicker.xaml new file mode 100644 index 0000000000..f695301e3a --- /dev/null +++ b/src/settings-ui/Settings.UI/SettingsXAML/Controls/ModelPicker/FoundryLocalModelPicker.xaml @@ -0,0 +1,192 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/settings-ui/Settings.UI/SettingsXAML/Controls/ModelPicker/FoundryLocalModelPicker.xaml.cs b/src/settings-ui/Settings.UI/SettingsXAML/Controls/ModelPicker/FoundryLocalModelPicker.xaml.cs new file mode 100644 index 0000000000..3d1c4c2159 --- /dev/null +++ b/src/settings-ui/Settings.UI/SettingsXAML/Controls/ModelPicker/FoundryLocalModelPicker.xaml.cs @@ -0,0 +1,457 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Collections.Specialized; +using System.Diagnostics; +using System.Linq; +using LanguageModelProvider; +using Microsoft.UI.Xaml; +using Microsoft.UI.Xaml.Controls; + +namespace Microsoft.PowerToys.Settings.UI.Controls; + +public sealed partial class FoundryLocalModelPicker : UserControl +{ + private INotifyCollectionChanged _cachedModelsSubscription; + private INotifyCollectionChanged _downloadableModelsSubscription; + private bool _suppressSelection; + + public FoundryLocalModelPicker() + { + InitializeComponent(); + Loaded += (_, _) => UpdateVisualStates(); + } + + public delegate void ModelSelectionChangedEventHandler(object sender, ModelDetails model); + + public delegate void DownloadRequestedEventHandler(object sender, object payload); + + public delegate void LoadRequestedEventHandler(object sender, FoundryLoadRequestedEventArgs args); + + public event ModelSelectionChangedEventHandler SelectionChanged; + + public event LoadRequestedEventHandler LoadRequested; + + public IEnumerable CachedModels + { + get => (IEnumerable)GetValue(CachedModelsProperty); + set => SetValue(CachedModelsProperty, value); + } + + public static readonly DependencyProperty CachedModelsProperty = + DependencyProperty.Register(nameof(CachedModels), typeof(IEnumerable), typeof(FoundryLocalModelPicker), new PropertyMetadata(null, OnCachedModelsChanged)); + + public IEnumerable DownloadableModels + { + get => (IEnumerable)GetValue(DownloadableModelsProperty); + set => SetValue(DownloadableModelsProperty, value); + } + + public static readonly DependencyProperty DownloadableModelsProperty = + DependencyProperty.Register(nameof(DownloadableModels), typeof(IEnumerable), typeof(FoundryLocalModelPicker), new PropertyMetadata(null, OnDownloadableModelsChanged)); + + public ModelDetails SelectedModel + { + get => (ModelDetails)GetValue(SelectedModelProperty); + set => SetValue(SelectedModelProperty, value); + } + + public static readonly DependencyProperty SelectedModelProperty = + DependencyProperty.Register(nameof(SelectedModel), typeof(ModelDetails), typeof(FoundryLocalModelPicker), new PropertyMetadata(null, OnSelectedModelChanged)); + + public bool IsLoading + { + get => (bool)GetValue(IsLoadingProperty); + set => SetValue(IsLoadingProperty, value); + } + + public static readonly DependencyProperty IsLoadingProperty = + DependencyProperty.Register(nameof(IsLoading), typeof(bool), typeof(FoundryLocalModelPicker), new PropertyMetadata(false, OnStatePropertyChanged)); + + public bool IsAvailable + { + get => (bool)GetValue(IsAvailableProperty); + set => SetValue(IsAvailableProperty, value); + } + + public static readonly DependencyProperty IsAvailableProperty = + DependencyProperty.Register(nameof(IsAvailable), typeof(bool), typeof(FoundryLocalModelPicker), new PropertyMetadata(false, OnStatePropertyChanged)); + + public string StatusText + { + get => (string)GetValue(StatusTextProperty); + set => SetValue(StatusTextProperty, value); + } + + public static readonly DependencyProperty StatusTextProperty = + DependencyProperty.Register(nameof(StatusText), typeof(string), typeof(FoundryLocalModelPicker), new PropertyMetadata(string.Empty, OnStatePropertyChanged)); + + public bool HasCachedModels => CachedModels?.Any() ?? false; + + public bool HasDownloadableModels => DownloadableModels?.Cast().Any() ?? false; + + public void RequestLoad(bool refresh) + { + if (IsLoading) + { + // Allow refresh requests to continue even if already loading by cancelling via host. + } + else + { + IsLoading = true; + } + + IsAvailable = false; + StatusText = "Loading Foundry Local status..."; + LoadRequested?.Invoke(this, new FoundryLoadRequestedEventArgs(refresh)); + } + + private static void OnCachedModelsChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) + { + var control = (FoundryLocalModelPicker)d; + control.SubscribeToCachedModels(e.OldValue as IEnumerable, e.NewValue as IEnumerable); + control.UpdateVisualStates(); + } + + private static void OnDownloadableModelsChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) + { + var control = (FoundryLocalModelPicker)d; + control.SubscribeToDownloadableModels(e.OldValue as IEnumerable, e.NewValue as IEnumerable); + control.UpdateVisualStates(); + } + + private static void OnSelectedModelChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) + { + var control = (FoundryLocalModelPicker)d; + if (control._suppressSelection) + { + return; + } + + try + { + control._suppressSelection = true; + if (control.CachedModelsComboBox is not null) + { + control.CachedModelsComboBox.SelectedItem = e.NewValue; + } + } + finally + { + control._suppressSelection = false; + } + + control.UpdateSelectedModelDetails(); + } + + private static void OnStatePropertyChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) + { + var control = (FoundryLocalModelPicker)d; + control.UpdateVisualStates(); + } + + private void SubscribeToCachedModels(IEnumerable oldValue, IEnumerable newValue) + { + if (_cachedModelsSubscription is not null) + { + _cachedModelsSubscription.CollectionChanged -= CachedModels_CollectionChanged; + _cachedModelsSubscription = null; + } + + if (newValue is INotifyCollectionChanged observable) + { + observable.CollectionChanged += CachedModels_CollectionChanged; + _cachedModelsSubscription = observable; + } + } + + private void SubscribeToDownloadableModels(IEnumerable oldValue, IEnumerable newValue) + { + if (_downloadableModelsSubscription is not null) + { + _downloadableModelsSubscription.CollectionChanged -= DownloadableModels_CollectionChanged; + _downloadableModelsSubscription = null; + } + + if (newValue is INotifyCollectionChanged observable) + { + observable.CollectionChanged += DownloadableModels_CollectionChanged; + _downloadableModelsSubscription = observable; + } + } + + private void CachedModels_CollectionChanged(object sender, NotifyCollectionChangedEventArgs e) + { + UpdateVisualStates(); + } + + private void DownloadableModels_CollectionChanged(object sender, NotifyCollectionChangedEventArgs e) + { + UpdateVisualStates(); + } + + private void CachedModelsComboBox_SelectionChanged(object sender, SelectionChangedEventArgs e) + { + if (_suppressSelection) + { + return; + } + + try + { + _suppressSelection = true; + var selected = CachedModelsComboBox.SelectedItem as ModelDetails; + SetValue(SelectedModelProperty, selected); + SelectionChanged?.Invoke(this, selected); + } + finally + { + _suppressSelection = false; + } + + UpdateSelectedModelDetails(); + } + + private void UpdateSelectedModelDetails() + { + if (SelectedModelDetailsPanel is null || SelectedModelDescriptionText is null || SelectedModelTagsPanel is null) + { + return; + } + + if (!HasCachedModels || SelectedModel is not ModelDetails model) + { + SelectedModelDetailsPanel.Visibility = Visibility.Collapsed; + SelectedModelDescriptionText.Text = string.Empty; + SelectedModelTagsPanel.Children.Clear(); + SelectedModelTagsPanel.Visibility = Visibility.Collapsed; + return; + } + + SelectedModelDetailsPanel.Visibility = Visibility.Visible; + SelectedModelDescriptionText.Text = string.IsNullOrWhiteSpace(model.Description) + ? "No description provided." + : model.Description; + + SelectedModelTagsPanel.Children.Clear(); + + AddTag(GetModelSizeText(model.Size)); + AddTag(GetLicenseShortText(model.License), model.License); + + foreach (var deviceTag in GetDeviceTags(model.HardwareAccelerators)) + { + AddTag(deviceTag); + } + + SelectedModelTagsPanel.Visibility = SelectedModelTagsPanel.Children.Count > 0 ? Visibility.Visible : Visibility.Collapsed; + + void AddTag(string text, string tooltip = null) + { + if (string.IsNullOrWhiteSpace(text) || SelectedModelTagsPanel is null) + { + return; + } + + Border tag = new(); + if (Resources.TryGetValue("TagBorderStyle", out var borderStyleObj) && borderStyleObj is Style borderStyle) + { + tag.Style = borderStyle; + } + + TextBlock label = new() + { + Text = text, + }; + + if (Resources.TryGetValue("TagTextStyle", out var textStyleObj) && textStyleObj is Style textStyle) + { + label.Style = textStyle; + } + + tag.Child = label; + + if (!string.IsNullOrWhiteSpace(tooltip)) + { + ToolTipService.SetToolTip(tag, new TextBlock + { + Text = tooltip, + TextWrapping = TextWrapping.Wrap, + }); + } + + SelectedModelTagsPanel.Children.Add(tag); + } + } + + private void LaunchFoundryModelListButton_Click(object sender, RoutedEventArgs e) + { + try + { + ProcessStartInfo processInfo = new() + { + FileName = "powershell.exe", + Arguments = "-NoExit -Command \"foundry model list\"", + UseShellExecute = true, + }; + + Process.Start(processInfo); + StatusText = "Opening PowerShell and running 'foundry model list'..."; + } + catch (Exception ex) + { + StatusText = $"Unable to start PowerShell. {ex.Message}"; + Debug.WriteLine($"[FoundryLocalModelPicker] Failed to run 'foundry model list': {ex}"); + } + } + + private void RefreshModelsButton_Click(object sender, RoutedEventArgs e) + { + RequestLoad(refresh: true); + } + + private void UpdateVisualStates() + { + LoadingIndicator.IsActive = IsLoading; + + if (IsLoading) + { + VisualStateManager.GoToState(this, "ShowLoading", true); + } + else if (!IsAvailable) + { + VisualStateManager.GoToState(this, "ShowNotAvailable", true); + } + else + { + VisualStateManager.GoToState(this, "ShowModels", true); + } + + if (LoadingStatusTextBlock is not null) + { + LoadingStatusTextBlock.Text = string.IsNullOrWhiteSpace(StatusText) + ? "Loading Foundry Local status..." + : StatusText; + } + + NoModelsPanel.Visibility = HasCachedModels ? Visibility.Collapsed : Visibility.Visible; + if (CachedModelsComboBox is not null) + { + CachedModelsComboBox.Visibility = HasCachedModels ? Visibility.Visible : Visibility.Collapsed; + CachedModelsComboBox.IsEnabled = HasCachedModels; + } + + UpdateSelectedModelDetails(); + + Bindings.Update(); + } + + public static string GetModelSizeText(long size) + { + if (size <= 0) + { + return string.Empty; + } + + const long kiloByte = 1024; + const long megaByte = kiloByte * 1024; + const long gigaByte = megaByte * 1024; + + if (size >= gigaByte) + { + return $"{size / (double)gigaByte:0.##} GB"; + } + + if (size >= megaByte) + { + return $"{size / (double)megaByte:0.##} MB"; + } + + if (size >= kiloByte) + { + return $"{size / (double)kiloByte:0.##} KB"; + } + + return $"{size} B"; + } + + public static Visibility GetModelSizeVisibility(long size) + { + return size > 0 ? Visibility.Visible : Visibility.Collapsed; + } + + public static IEnumerable GetDeviceTags(IReadOnlyCollection accelerators) + { + if (accelerators is null || accelerators.Count == 0) + { + return Array.Empty(); + } + + HashSet tags = new(StringComparer.OrdinalIgnoreCase); + + foreach (var accelerator in accelerators) + { + switch (accelerator) + { + case HardwareAccelerator.CPU: + tags.Add("CPU"); + break; + case HardwareAccelerator.GPU: + case HardwareAccelerator.DML: + tags.Add("GPU"); + break; + case HardwareAccelerator.NPU: + case HardwareAccelerator.QNN: + tags.Add("NPU"); + break; + } + } + + return tags.Count > 0 ? tags.ToArray() : Array.Empty(); + } + + public static Visibility GetDeviceVisibility(IReadOnlyCollection accelerators) + { + return GetDeviceTags(accelerators).Any() ? Visibility.Visible : Visibility.Collapsed; + } + + public static string GetLicenseShortText(string license) + { + if (string.IsNullOrWhiteSpace(license)) + { + return string.Empty; + } + + var trimmed = license.Trim(); + int separatorIndex = trimmed.IndexOfAny(['(', '[', ':']); + if (separatorIndex > 0) + { + trimmed = trimmed[..separatorIndex].Trim(); + } + + if (trimmed.Length > 24) + { + trimmed = $"{trimmed[..24].TrimEnd()}…"; + } + + return trimmed; + } + + public static Visibility GetLicenseVisibility(string license) + { + return string.IsNullOrWhiteSpace(license) ? Visibility.Collapsed : Visibility.Visible; + } + + public sealed class FoundryLoadRequestedEventArgs : EventArgs + { + public FoundryLoadRequestedEventArgs(bool refresh) + { + Refresh = refresh; + } + + public bool Refresh { get; } + } +} diff --git a/src/settings-ui/Settings.UI/SettingsXAML/Views/AdvancedPastePage.xaml b/src/settings-ui/Settings.UI/SettingsXAML/Views/AdvancedPastePage.xaml index 628df84c01..8d6b0afa68 100644 --- a/src/settings-ui/Settings.UI/SettingsXAML/Views/AdvancedPastePage.xaml +++ b/src/settings-ui/Settings.UI/SettingsXAML/Views/AdvancedPastePage.xaml @@ -3,28 +3,43 @@ xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:controls="using:Microsoft.PowerToys.Settings.UI.Controls" + xmlns:converters="using:Microsoft.PowerToys.Settings.UI.Converters" xmlns:d="http://schemas.microsoft.com/expression/blend/2008" xmlns:local="using:Microsoft.PowerToys.Settings.UI.Helpers" xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" xmlns:models="using:Microsoft.PowerToys.Settings.UI.Library" xmlns:tkcontrols="using:CommunityToolkit.WinUI.Controls" xmlns:ui="using:CommunityToolkit.WinUI" + xmlns:viewmodels="using:Microsoft.PowerToys.Settings.UI.ViewModels" x:Name="RootPage" AutomationProperties.LandmarkType="Main" mc:Ignorable="d"> + + + ms-appx:///Assets/Settings/Modules/APDialog.dark.png + ms-appx:///Assets/Settings/Icons/Models/OpenAI.dark.svg ms-appx:///Assets/Settings/Modules/APDialog.light.png + ms-appx:///Assets/Settings/Icons/Models/OpenAI.light.svg ms-appx:///Assets/Settings/Modules/APDialog.light.png + ms-appx:///Assets/Settings/Icons/Models/OpenAI.light.svg + + + - + - + IsEnabled="{x:Bind ViewModel.IsOnlineAIModelsDisallowedByGPO, Mode=OneWay, Converter={StaticResource BoolNegationConverter}}" + IsExpanded="{x:Bind ViewModel.IsAIEnabled, Mode=OneWay}" + ItemsSource="{x:Bind ViewModel.PasteAIConfiguration.Providers, Mode=OneWay}"> + - - - - + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - + DataContext="{x:Bind ViewModel.AdditionalActions.ImageToText, Mode=OneWay}" + HeaderIcon="{ui:FontIcon Glyph=}"> + + + + + + + + + + + + - - - - - - - - - - - - - - - - | - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + 900 + 700 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +