diff --git a/DATA_AND_PRIVACY.md b/DATA_AND_PRIVACY.md index 56a2eb9eee..0ad4bda9c9 100644 --- a/DATA_AND_PRIVACY.md +++ b/DATA_AND_PRIVACY.md @@ -147,6 +147,18 @@ _If you want to find diagnostic data events in the source code, these two links Microsoft.PowerToys.AdvancedPasteSemanticKernelFormatEvent Triggered when Advanced Paste leverages the Semantic Kernel. + + Microsoft.PowerToys.AdvancedPasteSemanticKernelErrorEvent + Occurs when the Semantic Kernel workflow encounters an error. + + + Microsoft.PowerToys.AdvancedPasteEndpointUsageEvent + Logs the AI provider, model, and processing duration for each endpoint call. + + + Microsoft.PowerToys.AdvancedPasteCustomActionErrorEvent + Records provider, model, and status details when a custom action fails. + ### Always on Top diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AIServiceFormatEvent.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AIServiceFormatEvent.cs index 1ab58bf269..b74192213b 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AIServiceFormatEvent.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AIServiceFormatEvent.cs @@ -18,6 +18,7 @@ namespace AdvancedPaste.Helpers PromptTokens = semanticKernelFormatEvent.PromptTokens; CompletionTokens = semanticKernelFormatEvent.CompletionTokens; ModelName = semanticKernelFormatEvent.ModelName; + ProviderType = semanticKernelFormatEvent.ProviderType; ActionChain = semanticKernelFormatEvent.ActionChain; } @@ -38,6 +39,8 @@ namespace AdvancedPaste.Helpers public string ModelName { get; set; } + public string ProviderType { get; set; } + public string ActionChain { get; set; } public string ToJsonString() => JsonSerializer.Serialize(this, SourceGenerationContext.Default.AIServiceFormatEvent); diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/CustomActionTransformService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/CustomActionTransformService.cs index 65d37bfdd8..57d55492a4 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/CustomActionTransformService.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/CustomActionTransformService.cs @@ -83,32 +83,39 @@ namespace AdvancedPaste.Services.CustomActions SystemPrompt = systemPrompt, }; + var operationStart = DateTime.UtcNow; + var providerContent = await provider.ProcessPasteAsync( request, cancellationToken, progress); + var durationMs = (int)Math.Round((DateTime.UtcNow - operationStart).TotalMilliseconds); + var usage = request.Usage; var content = providerContent ?? string.Empty; - // Log endpoint usage - var endpointEvent = new AdvancedPasteEndpointUsageEvent(providerConfig.ProviderType); + // Log endpoint usage (custom action pipeline is not the advanced SK flow) + var endpointEvent = new AdvancedPasteEndpointUsageEvent(providerConfig.ProviderType, providerConfig.Model ?? string.Empty, isAdvanced: false, durationMs: durationMs); PowerToysTelemetry.Log.WriteEvent(endpointEvent); - Logger.LogDebug($"{nameof(CustomActionTransformService)}.{nameof(TransformAsync)} complete; ModelName={providerConfig.Model ?? string.Empty}, PromptTokens={usage.PromptTokens}, CompletionTokens={usage.CompletionTokens}"); + Logger.LogDebug($"{nameof(CustomActionTransformService)}.{nameof(TransformAsync)} complete; ModelName={providerConfig.Model ?? string.Empty}, PromptTokens={usage.PromptTokens}, CompletionTokens={usage.CompletionTokens}, DurationMs={durationMs}"); return new CustomActionTransformResult(content, usage); } catch (Exception ex) { Logger.LogError($"{nameof(CustomActionTransformService)}.{nameof(TransformAsync)} failed", ex); + var statusCode = ExtractStatusCode(ex); + var modelName = providerConfig.Model ?? string.Empty; + AdvancedPasteCustomActionErrorEvent errorEvent = new(providerConfig.ProviderType, modelName, statusCode, ex is PasteActionModeratedException ? PasteActionModeratedException.ErrorDescription : ex.Message); + PowerToysTelemetry.Log.WriteEvent(errorEvent); if (ex is PasteActionException or OperationCanceledException) { throw; } - var statusCode = ExtractStatusCode(ex); var failureMessage = providerConfig.ProviderType switch { AIServiceType.OpenAI or AIServiceType.AzureOpenAI => ErrorHelpers.TranslateErrorText(statusCode), diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs index 0ea9ef40bc..6b293cbbe2 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs @@ -186,12 +186,20 @@ public abstract class KernelServiceBase( private void LogResult(bool cacheUsed, bool isSavedQuery, IEnumerable actionChain, AIServiceUsage usage) { - AdvancedPasteSemanticKernelFormatEvent telemetryEvent = new(cacheUsed, isSavedQuery, usage.PromptTokens, usage.CompletionTokens, AdvancedAIModelName, AdvancedPasteSemanticKernelFormatEvent.FormatActionChain(actionChain)); + var runtimeConfig = GetRuntimeConfiguration(); + + AdvancedPasteSemanticKernelFormatEvent telemetryEvent = new( + cacheUsed, + isSavedQuery, + usage.PromptTokens, + usage.CompletionTokens, + AdvancedAIModelName, + runtimeConfig.ServiceType.ToString(), + AdvancedPasteSemanticKernelFormatEvent.FormatActionChain(actionChain)); PowerToysTelemetry.Log.WriteEvent(telemetryEvent); // Log endpoint usage - var runtimeConfig = GetRuntimeConfiguration(); - var endpointEvent = new AdvancedPasteEndpointUsageEvent(runtimeConfig.ServiceType); + var endpointEvent = new AdvancedPasteEndpointUsageEvent(runtimeConfig.ServiceType, AdvancedAIModelName, isAdvanced: true); PowerToysTelemetry.Log.WriteEvent(endpointEvent); var logEvent = new AIServiceFormatEvent(telemetryEvent); diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteCustomActionErrorEvent.cs b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteCustomActionErrorEvent.cs new file mode 100644 index 0000000000..06f45a98ae --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteCustomActionErrorEvent.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Tracing; +using Microsoft.PowerToys.Settings.UI.Library; +using Microsoft.PowerToys.Telemetry; +using Microsoft.PowerToys.Telemetry.Events; + +namespace AdvancedPaste.Telemetry; + +[EventData] +[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties)] +public sealed class AdvancedPasteCustomActionErrorEvent : EventBase, IEvent +{ + public AdvancedPasteCustomActionErrorEvent(AIServiceType providerType, string modelName, int statusCode, string error) + { + ProviderType = providerType.ToString(); + ModelName = modelName; + StatusCode = statusCode; + Error = error; + } + + public string ProviderType { get; set; } + + public string ModelName { get; set; } + + public int StatusCode { get; set; } + + public string Error { get; set; } + + public PartA_PrivTags PartA_PrivTags => PartA_PrivTags.ProductAndServiceUsage; +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteEndpointUsageEvent.cs b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteEndpointUsageEvent.cs index 7b179a7cd5..671f6a7b9c 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteEndpointUsageEvent.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteEndpointUsageEvent.cs @@ -19,9 +19,27 @@ public class AdvancedPasteEndpointUsageEvent : EventBase, IEvent /// public string ProviderType { get; set; } - public AdvancedPasteEndpointUsageEvent(AIServiceType providerType) + /// + /// Gets or sets the configured model name. + /// + public string ModelName { get; set; } + + /// + /// Gets or sets a value indicating whether the advanced AI pipeline was used. + /// + public bool IsAdvanced { get; set; } + + /// + /// Gets or sets the total duration in milliseconds, or -1 if unavailable. + /// + public int DurationMs { get; set; } + + public AdvancedPasteEndpointUsageEvent(AIServiceType providerType, string modelName, bool isAdvanced, int durationMs = -1) { ProviderType = providerType.ToString(); + ModelName = modelName; + IsAdvanced = isAdvanced; + DurationMs = durationMs; } public PartA_PrivTags PartA_PrivTags => PartA_PrivTags.ProductAndServiceUsage; diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteSemanticKernelFormatEvent.cs b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteSemanticKernelFormatEvent.cs index 70542da6c8..53b4008782 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteSemanticKernelFormatEvent.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteSemanticKernelFormatEvent.cs @@ -14,7 +14,7 @@ namespace AdvancedPaste.Telemetry; [EventData] [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties)] -public class AdvancedPasteSemanticKernelFormatEvent(bool cacheUsed, bool isSavedQuery, int promptTokens, int completionTokens, string modelName, string actionChain) : EventBase, IEvent +public class AdvancedPasteSemanticKernelFormatEvent(bool cacheUsed, bool isSavedQuery, int promptTokens, int completionTokens, string modelName, string providerType, string actionChain) : EventBase, IEvent { public static string FormatActionChain(IEnumerable actionChain) => FormatActionChain(actionChain.Select(item => item.Format)); @@ -30,6 +30,8 @@ public class AdvancedPasteSemanticKernelFormatEvent(bool cacheUsed, bool isSaved public string ModelName { get; set; } = modelName; + public string ProviderType { get; set; } = providerType; + /// /// Gets or sets a comma-separated list of paste formats used - in the same order they were executed. /// Conceptually an array but formatted this way to work around https://github.com/dotnet/runtime/issues/10428