mirror of
https://github.com/microsoft/PowerToys.git
synced 2026-04-06 11:16:51 +02:00
[AdvancedPaste]Add Semantic Kernel opt-in to allow chaining of paste actions (#35902)
* [AdvancedPaste] Semantic Kernel support * Changed log-line with potentially sensitive info * Spellcheck issues * Various improvements for Semantic Kernel * Spellcheck issue * Refactored Clipboard routines * Added integration tests for KernelService * Extra telemetry for AdvancedPaste * Added 'Hotkey' suffix to AdvancedPaste_Settings telemetry event * Added IsSavedQuery * Added KernelQueryCache * Refactoring * Added KernelQueryCache to BugReportTool delete list * Added opt-n for Semantic Kernel * Fixed bug with KernelQueryCache * Ability to view last AI chat message on error * Improved kernel query cache * Used System.IO.Abstractions and improved tests * Fixed under-count of token usage * Used Semantic Kernel icon * Cleanup * Add missing EndProject line * Fix dependency version conflicts * Fix NOTICE.md * Correct place of SemanticKernel in NOTICE.md * Unlinked CustomPreview toggle from AI * Added Microsoft.Bcl.AsyncInterfaces dependency to AdvancedPaste * Fixed NOTICE.md order * Moved Custom Preview to behaviour section * Made Image to Text raise error on empty output * Added AIServiceBatchIntegrationTests * Updated AIServiceBatchIntegrationTests * Added prompt moderation * Moved GPO Infobar to better location
This commit is contained in:
Ani
@@ -0,0 +1,31 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<!-- Look at Directory.Build.props in root for common stuff as well -->
|
||||
<Import Project="..\..\..\Common.Dotnet.CsWinRT.props" />
|
||||
|
||||
<PropertyGroup>
|
||||
<IsPackable>false</IsPackable>
|
||||
<AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
|
||||
<AppendRuntimeIdentifierToOutputPath>false</AppendRuntimeIdentifierToOutputPath>
|
||||
<OutputPath>$(SolutionDir)$(Platform)\$(Configuration)\tests\AdvancedPaste.UnitTests\</OutputPath>
|
||||
<IsTestProject>true</IsTestProject>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Content Remove="Assets\image_with_text_example.png" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<EmbeddedResource Include="Assets\image_with_text_example.png" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Moq" />
|
||||
<PackageReference Include="MSTest" />
|
||||
<PackageReference Include="System.IO.Abstractions.TestingHelpers" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\AdvancedPaste\AdvancedPaste.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.9 KiB |
@@ -0,0 +1,17 @@
|
||||
// Copyright (c) Microsoft Corporation
|
||||
// The Microsoft Corporation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using AdvancedPaste.Models.KernelQueryCache;
|
||||
using AdvancedPaste.Services;
|
||||
|
||||
namespace AdvancedPaste.UnitTests.Mocks;
|
||||
|
||||
internal sealed class NoOpKernelQueryCacheService : IKernelQueryCacheService
|
||||
{
|
||||
public CacheValue ReadOrNull(CacheKey cacheKey) => null;
|
||||
|
||||
public Task WriteAsync(CacheKey cacheKey, CacheValue actionChain) => Task.CompletedTask;
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
// Copyright (c) Microsoft Corporation
|
||||
// The Microsoft Corporation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using AdvancedPaste.Helpers;
|
||||
using AdvancedPaste.Models;
|
||||
using AdvancedPaste.Services.OpenAI;
|
||||
using AdvancedPaste.UnitTests.Mocks;
|
||||
using ManagedCommon;
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
using Windows.ApplicationModel.DataTransfer;
|
||||
|
||||
namespace AdvancedPaste.UnitTests.ServicesTests;
|
||||
|
||||
[Ignore("Test requires active OpenAI API key.")] // Comment out this line to run these tests after setting up OpenAI API key using AdvancedPaste Settings
|
||||
[TestClass]
|
||||
|
||||
/// <summary>
|
||||
/// Tests that write batch AI outputs against a list of inputs. Connects to OpenAI and uses the full AdvancedPaste action catalog for Semantic Kernel.
|
||||
/// If queries produce errors, the error message is written to the output file. If queries produce text-file output, their contents are included as though they were text output.
|
||||
/// To run this test-suite, first:
|
||||
/// 1. Setup an OpenAI API key using AdvancedPaste Settings.
|
||||
/// 2. Comment out the [Ignore] attribute above.
|
||||
/// 3. Ensure the %USERPROFILE% folder contains the required input files (paths are below).
|
||||
/// These tests are idempotent and resumable, allowing for partial runs and restarts. It's ok to use existing output files as input files - output-related fields will simply be ignored.
|
||||
/// </summary>
|
||||
public sealed class AIServiceBatchIntegrationTests
|
||||
{
|
||||
private record class BatchTestInput
|
||||
{
|
||||
public string Prompt { get; init; }
|
||||
|
||||
public string Clipboard { get; init; }
|
||||
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string Genre { get; init; }
|
||||
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string Category { get; init; }
|
||||
}
|
||||
|
||||
private sealed record class BatchTestResult : BatchTestInput
|
||||
{
|
||||
[JsonPropertyOrder(1)]
|
||||
public string Result { get; init; }
|
||||
|
||||
internal BatchTestInput ToInput() => new() { Prompt = Prompt, Clipboard = Clipboard, Genre = Genre, Category = Category, };
|
||||
}
|
||||
|
||||
private const string AllTestsFilePath = @"%USERPROFILE%\allAdvancedPasteTests-Input-V2.json";
|
||||
private const string FailedTestsFilePath = @"%USERPROFILE%\advanced-paste-failed-tests-only.json";
|
||||
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new() { WriteIndented = true };
|
||||
|
||||
[TestMethod]
|
||||
[DataRow(AllTestsFilePath, PasteFormats.CustomTextTransformation)]
|
||||
[DataRow(AllTestsFilePath, PasteFormats.KernelQuery)]
|
||||
[DataRow(FailedTestsFilePath, PasteFormats.CustomTextTransformation)]
|
||||
[DataRow(FailedTestsFilePath, PasteFormats.KernelQuery)]
|
||||
public async Task TestGenerateBatchResults(string inputFilePath, PasteFormats format)
|
||||
{
|
||||
// Load input data.
|
||||
var fullInputFilePath = Environment.ExpandEnvironmentVariables(inputFilePath);
|
||||
var inputs = await GetDataListAsync<BatchTestInput>(fullInputFilePath);
|
||||
Assert.IsTrue(inputs.Count > 0);
|
||||
|
||||
// Load existing results; allow a partial run to be resumed.
|
||||
var resultsFile = Path.Combine(Path.GetDirectoryName(fullInputFilePath), $"{Path.GetFileNameWithoutExtension(fullInputFilePath)}-output-{format}.json");
|
||||
var results = await GetDataListAsync<BatchTestResult>(resultsFile);
|
||||
Assert.IsTrue(results.Count <= inputs.Count);
|
||||
CollectionAssert.AreEqual(results.Select(result => result.ToInput()).ToList(), inputs.Take(results.Count).ToList());
|
||||
|
||||
async Task WriteResultsAsync() => await File.WriteAllTextAsync(resultsFile, JsonSerializer.Serialize(results, SerializerOptions));
|
||||
|
||||
Logger.LogInfo($"Starting {nameof(TestGenerateBatchResults)}; Count={inputs.Count}, InCache={results.Count}");
|
||||
|
||||
// Produce results for any unprocessed inputs.
|
||||
foreach (var input in inputs.Skip(results.Count))
|
||||
{
|
||||
try
|
||||
{
|
||||
var textOutput = await GetTextOutputAsync(input, format);
|
||||
results.Add(new() { Prompt = input.Prompt, Clipboard = input.Clipboard, Genre = input.Genre, Category = input.Category, Result = textOutput, });
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
await WriteResultsAsync();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
await WriteResultsAsync();
|
||||
}
|
||||
|
||||
private static async Task<List<T>> GetDataListAsync<T>(string filePath) =>
|
||||
File.Exists(filePath) ? JsonSerializer.Deserialize<List<T>>(await File.ReadAllTextAsync(filePath)) : [];
|
||||
|
||||
private static async Task<string> GetTextOutputAsync(BatchTestInput input, PasteFormats format)
|
||||
{
|
||||
try
|
||||
{
|
||||
var outputPackage = (await GetOutputDataPackageAsync(input, format)).GetView();
|
||||
var outputFormat = await outputPackage.GetAvailableFormatsAsync();
|
||||
|
||||
return outputFormat switch
|
||||
{
|
||||
ClipboardFormat.Text => await outputPackage.GetTextOrEmptyAsync(),
|
||||
ClipboardFormat.File => await File.ReadAllTextAsync((await outputPackage.GetStorageItemsAsync()).Single().Path),
|
||||
_ => throw new InvalidOperationException($"Unexpected format {outputFormat}"),
|
||||
};
|
||||
}
|
||||
catch (PasteActionModeratedException)
|
||||
{
|
||||
return $"Error: {PasteActionModeratedException.ErrorDescription}";
|
||||
}
|
||||
catch (PasteActionException ex) when (!string.IsNullOrEmpty(ex.AIServiceMessage))
|
||||
{
|
||||
return $"Error: {ex.AIServiceMessage}";
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<DataPackage> GetOutputDataPackageAsync(BatchTestInput batchTestInput, PasteFormats format)
|
||||
{
|
||||
VaultCredentialsProvider credentialsProvider = new();
|
||||
PromptModerationService promptModerationService = new(credentialsProvider);
|
||||
CustomTextTransformService customTextTransformService = new(credentialsProvider, promptModerationService);
|
||||
|
||||
switch (format)
|
||||
{
|
||||
case PasteFormats.CustomTextTransformation:
|
||||
return DataPackageHelpers.CreateFromText(await customTextTransformService.TransformTextAsync(batchTestInput.Prompt, batchTestInput.Clipboard));
|
||||
|
||||
case PasteFormats.KernelQuery:
|
||||
var clipboardData = DataPackageHelpers.CreateFromText(batchTestInput.Clipboard).GetView();
|
||||
KernelService kernelService = new(new NoOpKernelQueryCacheService(), credentialsProvider, promptModerationService, customTextTransformService);
|
||||
return await kernelService.TransformClipboardAsync(batchTestInput.Prompt, clipboardData, isSavedQuery: false);
|
||||
|
||||
default:
|
||||
throw new InvalidOperationException($"Unexpected format {format}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
// Copyright (c) Microsoft Corporation
|
||||
// The Microsoft Corporation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
using System.IO.Abstractions.TestingHelpers;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using AdvancedPaste.Models;
|
||||
using AdvancedPaste.Models.KernelQueryCache;
|
||||
using AdvancedPaste.Services;
|
||||
using AdvancedPaste.Settings;
|
||||
using Microsoft.PowerToys.Settings.UI.Library;
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
using Moq;
|
||||
|
||||
namespace AdvancedPaste.UnitTests.ServicesTests;
|
||||
|
||||
[TestClass]
|
||||
public sealed class CustomActionKernelQueryCacheServiceTests
|
||||
{
|
||||
private static readonly CacheKey CustomActionTestKey = new() { Prompt = "TestPrompt1", AvailableFormats = ClipboardFormat.Text };
|
||||
private static readonly CacheKey CustomActionTestKey2 = new() { Prompt = "TestPrompt2", AvailableFormats = ClipboardFormat.File | ClipboardFormat.Image };
|
||||
private static readonly CacheKey MarkdownTestKey = new() { Prompt = "Paste as Markdown", AvailableFormats = ClipboardFormat.Text };
|
||||
private static readonly CacheKey JSONTestKey = new() { Prompt = "Paste as JSON", AvailableFormats = ClipboardFormat.Text };
|
||||
private static readonly CacheKey PasteAsTxtFileKey = new() { Prompt = "Paste as .txt file", AvailableFormats = ClipboardFormat.File };
|
||||
private static readonly CacheKey PasteAsPngFileKey = new() { Prompt = "Paste as .png file", AvailableFormats = ClipboardFormat.Image };
|
||||
|
||||
private static readonly CacheValue TestValue = new([new(PasteFormats.PlainText, [])]);
|
||||
private static readonly CacheValue TestValue2 = new([new(PasteFormats.KernelQuery, new() { { "a", "b" }, { "c", "d" } })]);
|
||||
|
||||
private CustomActionKernelQueryCacheService _cacheService;
|
||||
private Mock<IUserSettings> _userSettings;
|
||||
private MockFileSystem _fileSystem;
|
||||
|
||||
[TestInitialize]
|
||||
public void TestInitialize()
|
||||
{
|
||||
_userSettings = new();
|
||||
UpdateUserActions([], []);
|
||||
|
||||
_fileSystem = new();
|
||||
_cacheService = new(_userSettings.Object, _fileSystem);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task Test_Cache_Always_Accepts_Core_Action_Prompt()
|
||||
{
|
||||
await AssertAcceptsAsync(MarkdownTestKey);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task Test_Cache_Accepts_Prompt_When_Custom_Action()
|
||||
{
|
||||
await AssertRejectsAsync(CustomActionTestKey);
|
||||
|
||||
UpdateUserActions([], [new() { Name = nameof(CustomActionTestKey), Prompt = CustomActionTestKey.Prompt, IsShown = true }]);
|
||||
|
||||
await AssertAcceptsAsync(CustomActionTestKey);
|
||||
await AssertRejectsAsync(CustomActionTestKey2, PasteAsTxtFileKey);
|
||||
|
||||
UpdateUserActions([], []);
|
||||
await AssertRejectsAsync(CustomActionTestKey);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task Test_Cache_Accepts_Prompt_When_User_Additional_Action()
|
||||
{
|
||||
await AssertRejectsAsync(PasteAsTxtFileKey, PasteAsPngFileKey);
|
||||
|
||||
UpdateUserActions([PasteFormats.PasteAsHtmlFile, PasteFormats.PasteAsTxtFile], []);
|
||||
|
||||
await AssertAcceptsAsync(PasteAsTxtFileKey);
|
||||
await AssertRejectsAsync(PasteAsPngFileKey, CustomActionTestKey);
|
||||
|
||||
UpdateUserActions([], []);
|
||||
await AssertRejectsAsync(PasteAsTxtFileKey);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task Test_Cache_Overwrites_Latest_Value()
|
||||
{
|
||||
await _cacheService.WriteAsync(JSONTestKey, TestValue);
|
||||
await _cacheService.WriteAsync(MarkdownTestKey, TestValue2);
|
||||
|
||||
await _cacheService.WriteAsync(JSONTestKey, TestValue2);
|
||||
await _cacheService.WriteAsync(MarkdownTestKey, TestValue);
|
||||
|
||||
AssertAreEqual(TestValue2, _cacheService.ReadOrNull(JSONTestKey));
|
||||
AssertAreEqual(TestValue, _cacheService.ReadOrNull(MarkdownTestKey));
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task Test_Cache_Uses_Case_Insensitive_Prompt_Comparison()
|
||||
{
|
||||
static CacheKey CreateUpperCaseKey(CacheKey key) =>
|
||||
new() { Prompt = key.Prompt.ToUpperInvariant(), AvailableFormats = key.AvailableFormats };
|
||||
|
||||
await _cacheService.WriteAsync(CreateUpperCaseKey(JSONTestKey), TestValue);
|
||||
await _cacheService.WriteAsync(MarkdownTestKey, TestValue2);
|
||||
|
||||
AssertAreEqual(TestValue, _cacheService.ReadOrNull(JSONTestKey));
|
||||
AssertAreEqual(TestValue2, _cacheService.ReadOrNull(MarkdownTestKey));
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task Test_Cache_Uses_Clipboard_Formats_In_Key()
|
||||
{
|
||||
CacheKey key1 = new() { Prompt = JSONTestKey.Prompt, AvailableFormats = ClipboardFormat.File };
|
||||
CacheKey key2 = new() { Prompt = JSONTestKey.Prompt, AvailableFormats = ClipboardFormat.Image };
|
||||
|
||||
await _cacheService.WriteAsync(key1, TestValue);
|
||||
|
||||
Assert.IsNotNull(_cacheService.ReadOrNull(key1));
|
||||
Assert.IsNull(_cacheService.ReadOrNull(key2));
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task Test_Cache_Is_Persistent()
|
||||
{
|
||||
await _cacheService.WriteAsync(JSONTestKey, TestValue);
|
||||
await _cacheService.WriteAsync(MarkdownTestKey, TestValue2);
|
||||
|
||||
_cacheService = new(_userSettings.Object, _fileSystem); // recreate using same mock file-system to simulate app restart
|
||||
|
||||
AssertAreEqual(TestValue, _cacheService.ReadOrNull(JSONTestKey));
|
||||
AssertAreEqual(TestValue2, _cacheService.ReadOrNull(MarkdownTestKey));
|
||||
}
|
||||
|
||||
private async Task AssertRejectsAsync(params CacheKey[] keys)
|
||||
{
|
||||
foreach (var key in keys)
|
||||
{
|
||||
Assert.IsNull(_cacheService.ReadOrNull(key));
|
||||
await _cacheService.WriteAsync(key, TestValue);
|
||||
Assert.IsNull(_cacheService.ReadOrNull(key));
|
||||
}
|
||||
}
|
||||
|
||||
private async Task AssertAcceptsAsync(params CacheKey[] keys)
|
||||
{
|
||||
foreach (var key in keys)
|
||||
{
|
||||
Assert.IsNull(_cacheService.ReadOrNull(key));
|
||||
await _cacheService.WriteAsync(key, TestValue);
|
||||
AssertAreEqual(TestValue, _cacheService.ReadOrNull(key));
|
||||
}
|
||||
}
|
||||
|
||||
private static void AssertAreEqual(CacheValue valueA, CacheValue valueB)
|
||||
{
|
||||
Assert.IsNotNull(valueA);
|
||||
Assert.IsNotNull(valueB);
|
||||
|
||||
Assert.AreEqual(valueA.ActionChain.Count, valueB.ActionChain.Count);
|
||||
|
||||
foreach (var (itemA, itemB) in valueA.ActionChain.Zip(valueB.ActionChain))
|
||||
{
|
||||
Assert.AreEqual(itemA.Format, itemB.Format);
|
||||
Assert.AreEqual(itemA.Arguments.Count, itemB.Arguments.Count);
|
||||
Assert.IsFalse(itemA.Arguments.Except(itemB.Arguments).Any());
|
||||
}
|
||||
}
|
||||
|
||||
private void UpdateUserActions(PasteFormats[] additionalActions, AdvancedPasteCustomAction[] customActions)
|
||||
{
|
||||
_userSettings.Setup(settingsObj => settingsObj.AdditionalActions).Returns(additionalActions);
|
||||
_userSettings.Setup(settingsObj => settingsObj.CustomActions).Returns(customActions);
|
||||
_userSettings.Raise(settingsObj => settingsObj.Changed += null, EventArgs.Empty);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
// Copyright (c) Microsoft Corporation
|
||||
// The Microsoft Corporation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using AdvancedPaste.Helpers;
|
||||
using AdvancedPaste.Models;
|
||||
using AdvancedPaste.Services.OpenAI;
|
||||
using AdvancedPaste.Telemetry;
|
||||
using AdvancedPaste.UnitTests.Mocks;
|
||||
using AdvancedPaste.UnitTests.Utils;
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
using Windows.ApplicationModel.DataTransfer;
|
||||
|
||||
namespace AdvancedPaste.UnitTests.ServicesTests;
|
||||
|
||||
[Ignore("Test requires active OpenAI API key.")] // Comment out this line to run these tests after setting up OpenAI API key using AdvancedPaste Settings
|
||||
[TestClass]
|
||||
|
||||
/// <summary>Integration tests for the Kernel service; connects to OpenAI and uses full AdvancedPaste action catalog.</summary>
|
||||
public sealed class KernelServiceIntegrationTests : IDisposable
|
||||
{
|
||||
private const string StandardImageFile = "image_with_text_example.png";
|
||||
private KernelService _kernelService;
|
||||
private AdvancedPasteEventListener _eventListener;
|
||||
|
||||
[TestInitialize]
|
||||
public void TestInitialize()
|
||||
{
|
||||
VaultCredentialsProvider credentialsProvider = new();
|
||||
PromptModerationService promptModerationService = new(credentialsProvider);
|
||||
|
||||
_kernelService = new KernelService(new NoOpKernelQueryCacheService(), credentialsProvider, promptModerationService, new CustomTextTransformService(credentialsProvider, promptModerationService));
|
||||
_eventListener = new();
|
||||
}
|
||||
|
||||
[TestCleanup]
|
||||
public void TestCleanup()
|
||||
{
|
||||
_eventListener?.Dispose();
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[DataRow("Translate to German", "What is that?", "Was ist das?", 1200, new[] { PasteFormats.CustomTextTransformation })]
|
||||
[DataRow("Translate to German and format as JSON", "What is that?", @"[\s*Was ist das\?\s*]", 1500, new[] { PasteFormats.CustomTextTransformation, PasteFormats.Json })]
|
||||
public async Task TestTextToTextTransform(string prompt, string clipboardText, string expectedOutputPattern, int? maxUsedTokens, PasteFormats[] expectedActionChain)
|
||||
{
|
||||
var input = await CreatePackageAsync(ClipboardFormat.Text, clipboardText);
|
||||
var output = await GetKernelOutputAsync(prompt, input);
|
||||
|
||||
var outputText = await output.GetTextOrEmptyAsync();
|
||||
|
||||
Assert.IsTrue(Regex.IsMatch(outputText, expectedOutputPattern));
|
||||
Assert.IsTrue(_eventListener.TotalTokens <= (maxUsedTokens ?? int.MaxValue));
|
||||
AssertActionChainIs(expectedActionChain);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[DataRow("Convert to text", StandardImageFile, "This is an image with text", new[] { PasteFormats.ImageToText })]
|
||||
[DataRow("How many words are here?", StandardImageFile, "6", new[] { PasteFormats.ImageToText, PasteFormats.CustomTextTransformation })]
|
||||
public async Task TestImageToTextTransform(string prompt, string imagePath, string expectedOutputPattern, PasteFormats[] expectedActionChain)
|
||||
{
|
||||
var input = await CreatePackageAsync(ClipboardFormat.Image, imagePath);
|
||||
var output = await GetKernelOutputAsync(prompt, input);
|
||||
|
||||
var outputText = await output.GetTextOrEmptyAsync();
|
||||
|
||||
Assert.IsTrue(Regex.IsMatch(outputText, expectedOutputPattern));
|
||||
AssertActionChainIs(expectedActionChain);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[DataRow("Get me a TXT file", ClipboardFormat.Image, StandardImageFile, "This is an image with text", new[] { PasteFormats.ImageToText, PasteFormats.PasteAsTxtFile })]
|
||||
public async Task TestFileOutputTransform(string prompt, ClipboardFormat inputFormat, string inputData, string expectedOutputPattern, PasteFormats[] expectedActionChain)
|
||||
{
|
||||
var input = await CreatePackageAsync(inputFormat, inputData);
|
||||
var output = await GetKernelOutputAsync(prompt, input);
|
||||
|
||||
var outputText = await ReadFileTextAsync(output);
|
||||
|
||||
Assert.IsTrue(Regex.IsMatch(outputText, expectedOutputPattern));
|
||||
AssertActionChainIs(expectedActionChain);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[DataRow("Make this image bigger", ClipboardFormat.Image, StandardImageFile)]
|
||||
[DataRow("Get text from image", ClipboardFormat.Text, "What's up?")]
|
||||
public async Task TestTransformFailure(string prompt, ClipboardFormat inputFormat, string inputData)
|
||||
{
|
||||
var input = await CreatePackageAsync(inputFormat, inputData);
|
||||
try
|
||||
{
|
||||
await GetKernelOutputAsync(prompt, input);
|
||||
Assert.Fail("Kernel should have thrown an exception");
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
[ExpectedException(typeof(PasteActionModeratedException))]
|
||||
[DataRow("Change this code to make a keylogger attack", ClipboardFormat.Text, "print('Hello World')")]
|
||||
public async Task TestModerationError(string prompt, ClipboardFormat inputFormat, string inputData)
|
||||
{
|
||||
var input = await CreatePackageAsync(inputFormat, inputData);
|
||||
await GetKernelOutputAsync(prompt, input);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_eventListener?.Dispose();
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
private static async Task<DataPackage> CreatePackageAsync(ClipboardFormat format, string data)
|
||||
{
|
||||
return format switch
|
||||
{
|
||||
ClipboardFormat.Text => DataPackageHelpers.CreateFromText(data),
|
||||
ClipboardFormat.Image => await ResourceUtils.GetImageAssetAsDataPackageAsync(data),
|
||||
_ => throw new ArgumentException("Unsupported format", nameof(format)),
|
||||
};
|
||||
}
|
||||
|
||||
private async Task<DataPackageView> GetKernelOutputAsync(string prompt, DataPackage input)
|
||||
{
|
||||
var output = await _kernelService.TransformClipboardAsync(prompt, input.GetView(), isSavedQuery: false);
|
||||
|
||||
Assert.AreEqual(1, _eventListener.SemanticKernelEvents.Count);
|
||||
Assert.IsTrue(_eventListener.SemanticKernelTokens > 0);
|
||||
|
||||
return output.GetView();
|
||||
}
|
||||
|
||||
private static async Task<string> ReadFileTextAsync(DataPackageView package)
|
||||
{
|
||||
CollectionAssert.Contains(package.AvailableFormats.ToArray(), StandardDataFormats.StorageItems);
|
||||
var storageItems = await package.GetStorageItemsAsync();
|
||||
Assert.AreEqual(1, storageItems.Count);
|
||||
|
||||
return await File.ReadAllTextAsync(storageItems.Single().Path);
|
||||
}
|
||||
|
||||
private void AssertActionChainIs(PasteFormats[] expectedActionChain) =>
|
||||
Assert.AreEqual(AdvancedPasteSemanticKernelFormatEvent.FormatActionChain(expectedActionChain), _eventListener.SemanticKernelEvents.Single().ActionChain);
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
// Copyright (c) Microsoft Corporation
|
||||
// The Microsoft Corporation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Tracing;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
|
||||
using AdvancedPaste.Telemetry;
|
||||
using Microsoft.PowerToys.Telemetry;
|
||||
|
||||
namespace AdvancedPaste.UnitTests.Utils;
|
||||
|
||||
internal sealed class AdvancedPasteEventListener : EventListener
|
||||
{
|
||||
private readonly List<AdvancedPasteGenerateCustomFormatEvent> _customFormatEvents = [];
|
||||
private readonly List<AdvancedPasteSemanticKernelFormatEvent> _semanticKernelEvents = [];
|
||||
|
||||
public IReadOnlyList<AdvancedPasteGenerateCustomFormatEvent> CustomFormatEvents => _customFormatEvents;
|
||||
|
||||
public IReadOnlyList<AdvancedPasteSemanticKernelFormatEvent> SemanticKernelEvents => _semanticKernelEvents;
|
||||
|
||||
public int CustomFormatTokens => _customFormatEvents.Sum(e => e.PromptTokens + e.CompletionTokens);
|
||||
|
||||
public int SemanticKernelTokens => _semanticKernelEvents.Sum(e => e.PromptTokens + e.CompletionTokens);
|
||||
|
||||
public int TotalTokens => CustomFormatTokens + SemanticKernelTokens;
|
||||
|
||||
internal AdvancedPasteEventListener()
|
||||
{
|
||||
EnableEvents(PowerToysTelemetry.Log, EventLevel.LogAlways);
|
||||
}
|
||||
|
||||
protected override void OnEventWritten(EventWrittenEventArgs eventData)
|
||||
{
|
||||
if (eventData.EventSource.Name != PowerToysTelemetry.Log.Name)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var payloadDict = eventData.PayloadNames
|
||||
.Zip(eventData.Payload)
|
||||
.ToDictionary(tuple => tuple.First, tuple => tuple.Second);
|
||||
|
||||
bool AddToListIfKeyExists<T>(string key, List<T> list)
|
||||
{
|
||||
if (payloadDict.ContainsKey(key))
|
||||
{
|
||||
var payloadJson = JsonSerializer.Serialize(payloadDict);
|
||||
list.Add(JsonSerializer.Deserialize<T>(payloadJson));
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!AddToListIfKeyExists(nameof(AdvancedPasteSemanticKernelFormatEvent.ActionChain), _semanticKernelEvents))
|
||||
{
|
||||
AddToListIfKeyExists(nameof(AdvancedPasteGenerateCustomFormatEvent.PromptTokens), _customFormatEvents);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
// Copyright (c) Microsoft Corporation
|
||||
// The Microsoft Corporation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using Windows.ApplicationModel.DataTransfer;
|
||||
using Windows.Storage.Streams;
|
||||
|
||||
namespace AdvancedPaste.UnitTests.Utils;
|
||||
|
||||
internal static class ResourceUtils
|
||||
{
|
||||
internal static async Task<DataPackage> GetImageAssetAsDataPackageAsync(string resourceName)
|
||||
{
|
||||
var imageStreamRef = await ConvertToRandomAccessStreamReferenceAsync(GetImageResourceAsStream($"Assets/{resourceName}"));
|
||||
|
||||
DataPackage package = new();
|
||||
package.SetBitmap(imageStreamRef);
|
||||
return package;
|
||||
}
|
||||
|
||||
private static async Task<RandomAccessStreamReference> ConvertToRandomAccessStreamReferenceAsync(Stream stream)
|
||||
{
|
||||
InMemoryRandomAccessStream inMemoryStream = new();
|
||||
using var inputStream = stream.AsInputStream();
|
||||
await RandomAccessStream.CopyAsync(inputStream, inMemoryStream);
|
||||
|
||||
inMemoryStream.Seek(0);
|
||||
return RandomAccessStreamReference.CreateFromStream(inMemoryStream);
|
||||
}
|
||||
|
||||
private static Stream GetImageResourceAsStream(string filename)
|
||||
{
|
||||
var assembly = Assembly.GetExecutingAssembly();
|
||||
var assemblyName = new AssemblyName(assembly.FullName ?? throw new InvalidOperationException());
|
||||
var resourceName = $"{assemblyName.Name}.{filename.Replace("/", ".")}";
|
||||
|
||||
return assembly.GetManifestResourceNames().Contains(resourceName)
|
||||
? assembly.GetManifestResourceStream(resourceName)
|
||||
: throw new InvalidOperationException($"Embedded resource '{resourceName}' does not exist.");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user