update doc

DATA_AND_PRIVACY.md

@@ -147,6 +147,18 @@ _If you want to find diagnostic data events in the source code, these two links
     <td>Microsoft.PowerToys.AdvancedPasteSemanticKernelFormatEvent</td>
     <td>Triggered when Advanced Paste leverages the Semantic Kernel.</td>
   </tr> 
+  <tr>
+    <td>Microsoft.PowerToys.AdvancedPasteSemanticKernelErrorEvent</td>
+    <td>Occurs when the Semantic Kernel workflow encounters an error.</td>
+  </tr>
+  <tr>
+    <td>Microsoft.PowerToys.AdvancedPasteEndpointUsageEvent</td>
+    <td>Logs the AI provider, model, and processing duration for each endpoint call.</td>
+  </tr>
+  <tr>
+    <td>Microsoft.PowerToys.AdvancedPasteCustomActionErrorEvent</td>
+    <td>Records provider, model, and status details when a custom action fails.</td>
+  </tr>
 </table>
 
 ### Always on Top

src/modules/AdvancedPaste/AdvancedPaste/Helpers/AIServiceFormatEvent.cs

@@ -18,6 +18,7 @@ namespace AdvancedPaste.Helpers
             PromptTokens = semanticKernelFormatEvent.PromptTokens;
             CompletionTokens = semanticKernelFormatEvent.CompletionTokens;
             ModelName = semanticKernelFormatEvent.ModelName;
+            ProviderType = semanticKernelFormatEvent.ProviderType;
             ActionChain = semanticKernelFormatEvent.ActionChain;
         }
 
@@ -38,6 +39,8 @@ namespace AdvancedPaste.Helpers
 
         public string ModelName { get; set; }
 
+        public string ProviderType { get; set; }
+
         public string ActionChain { get; set; }
 
         public string ToJsonString() => JsonSerializer.Serialize(this, SourceGenerationContext.Default.AIServiceFormatEvent);

src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActions/CustomActionTransformService.cs

@@ -83,32 +83,39 @@ namespace AdvancedPaste.Services.CustomActions
                     SystemPrompt = systemPrompt,
                 };
 
+                var operationStart = DateTime.UtcNow;
+
                 var providerContent = await provider.ProcessPasteAsync(
                     request,
                     cancellationToken,
                     progress);
 
+                var durationMs = (int)Math.Round((DateTime.UtcNow - operationStart).TotalMilliseconds);
+
                 var usage = request.Usage;
                 var content = providerContent ?? string.Empty;
 
-                // Log endpoint usage
-                var endpointEvent = new AdvancedPasteEndpointUsageEvent(providerConfig.ProviderType);
+                // Log endpoint usage (custom action pipeline is not the advanced SK flow)
+                var endpointEvent = new AdvancedPasteEndpointUsageEvent(providerConfig.ProviderType, providerConfig.Model ?? string.Empty, isAdvanced: false, durationMs: durationMs);
                 PowerToysTelemetry.Log.WriteEvent(endpointEvent);
 
-                Logger.LogDebug($"{nameof(CustomActionTransformService)}.{nameof(TransformAsync)} complete; ModelName={providerConfig.Model ?? string.Empty}, PromptTokens={usage.PromptTokens}, CompletionTokens={usage.CompletionTokens}");
+                Logger.LogDebug($"{nameof(CustomActionTransformService)}.{nameof(TransformAsync)} complete; ModelName={providerConfig.Model ?? string.Empty}, PromptTokens={usage.PromptTokens}, CompletionTokens={usage.CompletionTokens}, DurationMs={durationMs}");
 
                 return new CustomActionTransformResult(content, usage);
             }
             catch (Exception ex)
             {
                 Logger.LogError($"{nameof(CustomActionTransformService)}.{nameof(TransformAsync)} failed", ex);
+                var statusCode = ExtractStatusCode(ex);
+                var modelName = providerConfig.Model ?? string.Empty;
+                AdvancedPasteCustomActionErrorEvent errorEvent = new(providerConfig.ProviderType, modelName, statusCode, ex is PasteActionModeratedException ? PasteActionModeratedException.ErrorDescription : ex.Message);
+                PowerToysTelemetry.Log.WriteEvent(errorEvent);
 
                 if (ex is PasteActionException or OperationCanceledException)
                 {
                     throw;
                 }
 
-                var statusCode = ExtractStatusCode(ex);
                 var failureMessage = providerConfig.ProviderType switch
                 {
                     AIServiceType.OpenAI or AIServiceType.AzureOpenAI => ErrorHelpers.TranslateErrorText(statusCode),

src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs

@@ -186,12 +186,20 @@ public abstract class KernelServiceBase(
 
     private void LogResult(bool cacheUsed, bool isSavedQuery, IEnumerable<ActionChainItem> actionChain, AIServiceUsage usage)
     {
-        AdvancedPasteSemanticKernelFormatEvent telemetryEvent = new(cacheUsed, isSavedQuery, usage.PromptTokens, usage.CompletionTokens, AdvancedAIModelName, AdvancedPasteSemanticKernelFormatEvent.FormatActionChain(actionChain));
+        var runtimeConfig = GetRuntimeConfiguration();
+
+        AdvancedPasteSemanticKernelFormatEvent telemetryEvent = new(
+            cacheUsed,
+            isSavedQuery,
+            usage.PromptTokens,
+            usage.CompletionTokens,
+            AdvancedAIModelName,
+            runtimeConfig.ServiceType.ToString(),
+            AdvancedPasteSemanticKernelFormatEvent.FormatActionChain(actionChain));
         PowerToysTelemetry.Log.WriteEvent(telemetryEvent);
 
         // Log endpoint usage
-        var runtimeConfig = GetRuntimeConfiguration();
-        var endpointEvent = new AdvancedPasteEndpointUsageEvent(runtimeConfig.ServiceType);
+        var endpointEvent = new AdvancedPasteEndpointUsageEvent(runtimeConfig.ServiceType, AdvancedAIModelName, isAdvanced: true);
         PowerToysTelemetry.Log.WriteEvent(endpointEvent);
 
         var logEvent = new AIServiceFormatEvent(telemetryEvent);

src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteCustomActionErrorEvent.cs

@@ -0,0 +1,34 @@
+// Copyright (c) Microsoft Corporation
+// The Microsoft Corporation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Diagnostics.CodeAnalysis;
+using System.Diagnostics.Tracing;
+using Microsoft.PowerToys.Settings.UI.Library;
+using Microsoft.PowerToys.Telemetry;
+using Microsoft.PowerToys.Telemetry.Events;
+
+namespace AdvancedPaste.Telemetry;
+
+[EventData]
+[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties)]
+public sealed class AdvancedPasteCustomActionErrorEvent : EventBase, IEvent
+{
+    public AdvancedPasteCustomActionErrorEvent(AIServiceType providerType, string modelName, int statusCode, string error)
+    {
+        ProviderType = providerType.ToString();
+        ModelName = modelName;
+        StatusCode = statusCode;
+        Error = error;
+    }
+
+    public string ProviderType { get; set; }
+
+    public string ModelName { get; set; }
+
+    public int StatusCode { get; set; }
+
+    public string Error { get; set; }
+
+    public PartA_PrivTags PartA_PrivTags => PartA_PrivTags.ProductAndServiceUsage;
+}

src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteEndpointUsageEvent.cs

@@ -19,9 +19,27 @@ public class AdvancedPasteEndpointUsageEvent : EventBase, IEvent
     /// </summary>
     public string ProviderType { get; set; }
 
-    public AdvancedPasteEndpointUsageEvent(AIServiceType providerType)
+    /// <summary>
+    /// Gets or sets the configured model name.
+    /// </summary>
+    public string ModelName { get; set; }
+
+    /// <summary>
+    /// Gets or sets a value indicating whether the advanced AI pipeline was used.
+    /// </summary>
+    public bool IsAdvanced { get; set; }
+
+    /// <summary>
+    /// Gets or sets the total duration in milliseconds, or -1 if unavailable.
+    /// </summary>
+    public int DurationMs { get; set; }
+
+    public AdvancedPasteEndpointUsageEvent(AIServiceType providerType, string modelName, bool isAdvanced, int durationMs = -1)
     {
         ProviderType = providerType.ToString();
+        ModelName = modelName;
+        IsAdvanced = isAdvanced;
+        DurationMs = durationMs;
     }
 
     public PartA_PrivTags PartA_PrivTags => PartA_PrivTags.ProductAndServiceUsage;

src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteSemanticKernelFormatEvent.cs

@@ -14,7 +14,7 @@ namespace AdvancedPaste.Telemetry;
 
 [EventData]
 [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties)]
-public class AdvancedPasteSemanticKernelFormatEvent(bool cacheUsed, bool isSavedQuery, int promptTokens, int completionTokens, string modelName, string actionChain) : EventBase, IEvent
+public class AdvancedPasteSemanticKernelFormatEvent(bool cacheUsed, bool isSavedQuery, int promptTokens, int completionTokens, string modelName, string providerType, string actionChain) : EventBase, IEvent
 {
     public static string FormatActionChain(IEnumerable<ActionChainItem> actionChain) => FormatActionChain(actionChain.Select(item => item.Format));
 
@@ -30,6 +30,8 @@ public class AdvancedPasteSemanticKernelFormatEvent(bool cacheUsed, bool isSaved
 
     public string ModelName { get; set; } = modelName;
 
+    public string ProviderType { get; set; } = providerType;
+
     /// <summary>
     /// Gets or sets a comma-separated list of paste formats used - in the same order they were executed.
     /// Conceptually an array but formatted this way to work around https://github.com/dotnet/runtime/issues/10428