From 1c646ecb2a7d6eef231f980217bac767f1ac1033 Mon Sep 17 00:00:00 2001 From: leileizhang Date: Thu, 13 Nov 2025 10:09:50 +0800 Subject: [PATCH] Add more detailed telemetry for AdvancedPaste (#43498) ## Summary of the Pull Request Added telemetry to capture the following: - PasteAI: endpoint, model name, and processing time - PasteAI Errors - Advanced AI: endpoint, model name, and token usage - Advanced AI Errors ## PR Checklist - [ ] Closes: #xxx - [ ] **Communication:** I've discussed this with core contributors already. If the work hasn't been agreed, this work might be rejected - [ ] **Tests:** Added/updated and all pass - [ ] **Localization:** All end-user-facing strings can be localized - [ ] **Dev docs:** Added/updated - [ ] **New binaries:** Added on the required places - [ ] [JSON for signing](https://github.com/microsoft/PowerToys/blob/main/.pipelines/ESRPSigning_core.json) for new binaries - [ ] [WXS for installer](https://github.com/microsoft/PowerToys/blob/main/installer/PowerToysSetup/Product.wxs) for new binaries and localization folder - [ ] [YML for CI pipeline](https://github.com/microsoft/PowerToys/blob/main/.pipelines/ci/templates/build-powertoys-steps.yml) for new test projects - [ ] [YML for signed pipeline](https://github.com/microsoft/PowerToys/blob/main/.pipelines/release.yml) - [ ] **Documentation updated:** If checked, please file a pull request on [our docs repo](https://github.com/MicrosoftDocs/windows-uwp/tree/docs/hub/powertoys) and link it here: #xxx ## Detailed Description of the Pull Request / Additional comments ## Validation Steps Performed --- DATA_AND_PRIVACY.md | 12 +++++++ .../Helpers/AIServiceFormatEvent.cs | 3 ++ .../CustomActionTransformService.cs | 15 +++++--- .../Services/KernelServiceBase.cs | 14 ++++++-- .../AdvancedPasteCustomActionErrorEvent.cs | 34 +++++++++++++++++++ .../AdvancedPasteEndpointUsageEvent.cs | 20 ++++++++++- .../AdvancedPasteSemanticKernelFormatEvent.cs | 4 ++- 7 files changed, 93 insertions(+), 9 deletions(-) create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteCustomActionErrorEvent.cs diff --git a/DATA_AND_PRIVACY.md b/DATA_AND_PRIVACY.md index 56a2eb9eee..0ad4bda9c9 100644 --- a/DATA_AND_PRIVACY.md +++ b/DATA_AND_PRIVACY.md @@ -147,6 +147,18 @@ _If you want to find diagnostic data events in the source code, these two links Microsoft.PowerToys.AdvancedPasteSemanticKernelFormatEvent Triggered when Advanced Paste leverages the Semantic Kernel. + + Microsoft.PowerToys.AdvancedPasteSemanticKernelErrorEvent + Occurs when the Semantic Kernel workflow encounters an error. + + + Microsoft.PowerToys.AdvancedPasteEndpointUsageEvent + Logs the AI provider, model, and processing duration for each endpoint call. + + + Microsoft.PowerToys.AdvancedPasteCustomActionErrorEvent + Records provider, model, and status details when a custom action fails. + ### Always on Top diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AIServiceFormatEvent.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AIServiceFormatEvent.cs index 1ab58bf269..b74192213b 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AIServiceFormatEvent.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AIServiceFormatEvent.cs @@ -18,6 +18,7 @@ namespace AdvancedPaste.Helpers PromptTokens = semanticKernelFormatEvent.PromptTokens; CompletionTokens = semanticKernelFormatEvent.CompletionTokens; ModelName = semanticKernelFormatEvent.ModelName; + ProviderType = semanticKernelFormatEvent.ProviderType; ActionChain = semanticKernelFormatEvent.ActionChain; } @@ -38,6 +39,8 @@ namespace AdvancedPaste.Helpers public string ModelName { get; set; } + public string ProviderType { get; set; } + public string ActionChain { get; set; } public string ToJsonString() => JsonSerializer.Serialize(this, SourceGenerationContext.Default.AIServiceFormatEvent); diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/CustomActionTransformService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/CustomActionTransformService.cs index 65d37bfdd8..57d55492a4 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/CustomActionTransformService.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/CustomActionTransformService.cs @@ -83,32 +83,39 @@ namespace AdvancedPaste.Services.CustomActions SystemPrompt = systemPrompt, }; + var operationStart = DateTime.UtcNow; + var providerContent = await provider.ProcessPasteAsync( request, cancellationToken, progress); + var durationMs = (int)Math.Round((DateTime.UtcNow - operationStart).TotalMilliseconds); + var usage = request.Usage; var content = providerContent ?? string.Empty; - // Log endpoint usage - var endpointEvent = new AdvancedPasteEndpointUsageEvent(providerConfig.ProviderType); + // Log endpoint usage (custom action pipeline is not the advanced SK flow) + var endpointEvent = new AdvancedPasteEndpointUsageEvent(providerConfig.ProviderType, providerConfig.Model ?? string.Empty, isAdvanced: false, durationMs: durationMs); PowerToysTelemetry.Log.WriteEvent(endpointEvent); - Logger.LogDebug($"{nameof(CustomActionTransformService)}.{nameof(TransformAsync)} complete; ModelName={providerConfig.Model ?? string.Empty}, PromptTokens={usage.PromptTokens}, CompletionTokens={usage.CompletionTokens}"); + Logger.LogDebug($"{nameof(CustomActionTransformService)}.{nameof(TransformAsync)} complete; ModelName={providerConfig.Model ?? string.Empty}, PromptTokens={usage.PromptTokens}, CompletionTokens={usage.CompletionTokens}, DurationMs={durationMs}"); return new CustomActionTransformResult(content, usage); } catch (Exception ex) { Logger.LogError($"{nameof(CustomActionTransformService)}.{nameof(TransformAsync)} failed", ex); + var statusCode = ExtractStatusCode(ex); + var modelName = providerConfig.Model ?? string.Empty; + AdvancedPasteCustomActionErrorEvent errorEvent = new(providerConfig.ProviderType, modelName, statusCode, ex is PasteActionModeratedException ? PasteActionModeratedException.ErrorDescription : ex.Message); + PowerToysTelemetry.Log.WriteEvent(errorEvent); if (ex is PasteActionException or OperationCanceledException) { throw; } - var statusCode = ExtractStatusCode(ex); var failureMessage = providerConfig.ProviderType switch { AIServiceType.OpenAI or AIServiceType.AzureOpenAI => ErrorHelpers.TranslateErrorText(statusCode), diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs index 0ea9ef40bc..6b293cbbe2 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs @@ -186,12 +186,20 @@ public abstract class KernelServiceBase( private void LogResult(bool cacheUsed, bool isSavedQuery, IEnumerable actionChain, AIServiceUsage usage) { - AdvancedPasteSemanticKernelFormatEvent telemetryEvent = new(cacheUsed, isSavedQuery, usage.PromptTokens, usage.CompletionTokens, AdvancedAIModelName, AdvancedPasteSemanticKernelFormatEvent.FormatActionChain(actionChain)); + var runtimeConfig = GetRuntimeConfiguration(); + + AdvancedPasteSemanticKernelFormatEvent telemetryEvent = new( + cacheUsed, + isSavedQuery, + usage.PromptTokens, + usage.CompletionTokens, + AdvancedAIModelName, + runtimeConfig.ServiceType.ToString(), + AdvancedPasteSemanticKernelFormatEvent.FormatActionChain(actionChain)); PowerToysTelemetry.Log.WriteEvent(telemetryEvent); // Log endpoint usage - var runtimeConfig = GetRuntimeConfiguration(); - var endpointEvent = new AdvancedPasteEndpointUsageEvent(runtimeConfig.ServiceType); + var endpointEvent = new AdvancedPasteEndpointUsageEvent(runtimeConfig.ServiceType, AdvancedAIModelName, isAdvanced: true); PowerToysTelemetry.Log.WriteEvent(endpointEvent); var logEvent = new AIServiceFormatEvent(telemetryEvent); diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteCustomActionErrorEvent.cs b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteCustomActionErrorEvent.cs new file mode 100644 index 0000000000..06f45a98ae --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteCustomActionErrorEvent.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Tracing; +using Microsoft.PowerToys.Settings.UI.Library; +using Microsoft.PowerToys.Telemetry; +using Microsoft.PowerToys.Telemetry.Events; + +namespace AdvancedPaste.Telemetry; + +[EventData] +[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties)] +public sealed class AdvancedPasteCustomActionErrorEvent : EventBase, IEvent +{ + public AdvancedPasteCustomActionErrorEvent(AIServiceType providerType, string modelName, int statusCode, string error) + { + ProviderType = providerType.ToString(); + ModelName = modelName; + StatusCode = statusCode; + Error = error; + } + + public string ProviderType { get; set; } + + public string ModelName { get; set; } + + public int StatusCode { get; set; } + + public string Error { get; set; } + + public PartA_PrivTags PartA_PrivTags => PartA_PrivTags.ProductAndServiceUsage; +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteEndpointUsageEvent.cs b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteEndpointUsageEvent.cs index 7b179a7cd5..671f6a7b9c 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteEndpointUsageEvent.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteEndpointUsageEvent.cs @@ -19,9 +19,27 @@ public class AdvancedPasteEndpointUsageEvent : EventBase, IEvent /// public string ProviderType { get; set; } - public AdvancedPasteEndpointUsageEvent(AIServiceType providerType) + /// + /// Gets or sets the configured model name. + /// + public string ModelName { get; set; } + + /// + /// Gets or sets a value indicating whether the advanced AI pipeline was used. + /// + public bool IsAdvanced { get; set; } + + /// + /// Gets or sets the total duration in milliseconds, or -1 if unavailable. + /// + public int DurationMs { get; set; } + + public AdvancedPasteEndpointUsageEvent(AIServiceType providerType, string modelName, bool isAdvanced, int durationMs = -1) { ProviderType = providerType.ToString(); + ModelName = modelName; + IsAdvanced = isAdvanced; + DurationMs = durationMs; } public PartA_PrivTags PartA_PrivTags => PartA_PrivTags.ProductAndServiceUsage; diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteSemanticKernelFormatEvent.cs b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteSemanticKernelFormatEvent.cs index 70542da6c8..53b4008782 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteSemanticKernelFormatEvent.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteSemanticKernelFormatEvent.cs @@ -14,7 +14,7 @@ namespace AdvancedPaste.Telemetry; [EventData] [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties)] -public class AdvancedPasteSemanticKernelFormatEvent(bool cacheUsed, bool isSavedQuery, int promptTokens, int completionTokens, string modelName, string actionChain) : EventBase, IEvent +public class AdvancedPasteSemanticKernelFormatEvent(bool cacheUsed, bool isSavedQuery, int promptTokens, int completionTokens, string modelName, string providerType, string actionChain) : EventBase, IEvent { public static string FormatActionChain(IEnumerable actionChain) => FormatActionChain(actionChain.Select(item => item.Format)); @@ -30,6 +30,8 @@ public class AdvancedPasteSemanticKernelFormatEvent(bool cacheUsed, bool isSaved public string ModelName { get; set; } = modelName; + public string ProviderType { get; set; } = providerType; + /// /// Gets or sets a comma-separated list of paste formats used - in the same order they were executed. /// Conceptually an array but formatted this way to work around https://github.com/dotnet/runtime/issues/10428