Extends OpenAITelemetryPlugin with token report. Closes #1308

DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs

@@ -15,6 +15,7 @@
 using OpenTelemetry.Metrics;
 using OpenTelemetry.Resources;
 using OpenTelemetry.Trace;
+using System.Collections.Concurrent;
 using System.Diagnostics;
 using System.Diagnostics.Metrics;
 using System.Text.Json;
@@ -46,7 +47,7 @@ public sealed class OpenAITelemetryPlugin(
     ISet<UrlToWatch> urlsToWatch,
     IProxyConfiguration proxyConfiguration,
     IConfigurationSection pluginConfigurationSection) :
-    BasePlugin<OpenAITelemetryPluginConfiguration>(
+    BaseReportingPlugin<OpenAITelemetryPluginConfiguration>(
         httpClient,
         logger,
         urlsToWatch,
@@ -65,6 +66,7 @@ public sealed class OpenAITelemetryPlugin(
     private LanguageModelPricesLoader? _loader;
     private MeterProvider? _meterProvider;
     private TracerProvider? _tracerProvider;
+    private readonly ConcurrentDictionary<string, List<OpenAITelemetryPluginReportModelUsageInformation>> _modelUsage = [];
 
     public override string Name => nameof(OpenAITelemetryPlugin);
 
@@ -189,6 +191,26 @@ public override Task AfterResponseAsync(ProxyResponseArgs e, CancellationToken c
         return Task.CompletedTask;
     }
 
+    public override Task AfterRecordingStopAsync(RecordingArgs e, CancellationToken cancellationToken)
+    {
+        Logger.LogTrace("{Method} called", nameof(AfterRecordingStopAsync));
+
+        var report = new OpenAITelemetryPluginReport
+        {
+            Application = Configuration.Application,
+            Environment = Configuration.Environment,
+            Currency = Configuration.Currency,
+            IncludeCosts = Configuration.IncludeCosts,
+            ModelUsage = _modelUsage.ToDictionary()
+        };
+
+        StoreReport(report, e);
+        _modelUsage.Clear();
+
+        Logger.LogTrace("Left {Name}", nameof(AfterRecordingStopAsync));
+        return Task.CompletedTask;
+    }
+
     private void InitializeOpenTelemetryExporter()
     {
         Logger.LogTrace("InitializeOpenTelemetryExporter() called");
@@ -811,6 +833,15 @@ private void RecordUsageMetrics(Activity activity, OpenAIRequest request, OpenAI
             .SetTag(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, usage.CompletionTokens)
             .SetTag(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS, usage.TotalTokens);
 
+        var reportModelUsageInformation = new OpenAITelemetryPluginReportModelUsageInformation
+        {
+            Model = response.Model,
+            PromptTokens = usage.PromptTokens,
+            CompletionTokens = usage.CompletionTokens
+        };
+        var usagePerModel = _modelUsage.GetOrAdd(response.Model, model => []);
+        usagePerModel.Add(reportModelUsageInformation);
+
         if (!Configuration.IncludeCosts || Configuration.Prices is null)
         {
             Logger.LogDebug("Cost tracking is disabled or prices data is not available");
@@ -847,6 +878,7 @@ private void RecordUsageMetrics(Activity activity, OpenAIRequest request, OpenAI
                 new(SemanticConvention.GEN_AI_REQUEST_MODEL, request.Model),
                 new(SemanticConvention.GEN_AI_RESPONSE_MODEL, response.Model)
             ]);
+            reportModelUsageInformation.Cost = totalCost;
         }
         else
         {

DevProxy.Plugins/Inspection/OpenAITelemetryPluginReport.cs

@@ -0,0 +1,165 @@
+using DevProxy.Abstractions.Plugins;
+using DevProxy.Abstractions.Utils;
+using System.Globalization;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace DevProxy.Plugins.Inspection;
+
+public class OpenAITelemetryPluginReportModelUsageInformation
+{
+    public required long CompletionTokens { get; init; }
+    public double Cost { get; set; }
+    public required string Model { get; init; }
+    public required long PromptTokens { get; init; }
+}
+
+public class OpenAITelemetryPluginReport : IMarkdownReport, IPlainTextReport, IJsonReport
+{
+    public required string Application { get; init; }
+    public required string Currency { get; init; }
+    public required string Environment { get; init; }
+    [JsonIgnore]
+    public bool IncludeCosts { get; set; }
+    public required Dictionary<string, List<OpenAITelemetryPluginReportModelUsageInformation>> ModelUsage { get; init; } = [];
+
+    public string FileExtension => ".json";
+
+    public object ToJson() => JsonSerializer.Serialize(this, ProxyUtils.JsonSerializerOptions);
+
+    public string? ToMarkdown()
+    {
+        var totalTokens = 0L;
+        var totalPromptTokens = 0L;
+        var totalCompletionTokens = 0L;
+        var totalCost = 0.0;
+        var totalRequests = 0;
+
+        var sb = new StringBuilder();
+        _ = sb
+            .AppendLine(CultureInfo.InvariantCulture, $"# LLM usage report for {Application} in {Environment}")
+            .AppendLine()
+            .Append("Model|Requests|Prompt Tokens|Completion Tokens|Total Tokens");
+
+        if (IncludeCosts)
+        {
+            _ = sb.Append("|Total Cost");
+        }
+
+        _ = sb
+            .AppendLine()
+            .Append(":----|-------:|------------:|----------------:|-----------:");
+
+        if (IncludeCosts)
+        {
+            _ = sb.Append("|---------:");
+        }
+
+        _ = sb.AppendLine();
+
+        foreach (var modelUsage in ModelUsage.OrderBy(m => m.Key))
+        {
+            var promptTokens = modelUsage.Value.Sum(u => u.PromptTokens);
+            var completionTokens = modelUsage.Value.Sum(u => u.CompletionTokens);
+            var tokens = promptTokens + completionTokens;
+
+            totalPromptTokens += promptTokens;
+            totalCompletionTokens += completionTokens;
+            totalTokens += tokens;
+            totalRequests += modelUsage.Value.Count;
+
+            _ = sb
+                .Append(modelUsage.Key)
+                .Append('|').Append(totalRequests)
+                .Append('|').Append(promptTokens)
+                .Append('|').Append(completionTokens)
+                .Append('|').Append(tokens);
+
+            if (IncludeCosts)
+            {
+                var cost = modelUsage.Value.Sum(u => u.Cost);
+                totalCost += cost;
+                _ = sb.Append('|').Append(FormatCost(cost, Currency));
+            }
+
+            _ = sb.AppendLine();
+        }
+
+        _ = sb
+            .Append("**Total**")
+            .Append('|').Append(CultureInfo.CurrentCulture, $"**{totalRequests}**")
+            .Append('|').Append(CultureInfo.CurrentCulture, $"**{totalPromptTokens}**")
+            .Append('|').Append(CultureInfo.CurrentCulture, $"**{totalCompletionTokens}**")
+            .Append('|').Append(CultureInfo.CurrentCulture, $"**{totalTokens}**");
+
+        if (IncludeCosts)
+        {
+            _ = sb.Append('|').Append(CultureInfo.CurrentCulture, $"**{FormatCost(totalCost, Currency)}**");
+        }
+
+        _ = sb.AppendLine();
+
+        return sb.ToString();
+    }
+
+    public string? ToPlainText()
+    {
+        var totalTokens = 0L;
+        var totalPromptTokens = 0L;
+        var totalCompletionTokens = 0L;
+        var totalCost = 0.0;
+
+        var sb = new StringBuilder();
+        _ = sb
+            .AppendLine(CultureInfo.InvariantCulture, $"LLM USAGE REPORT FOR {Application} IN {Environment}")
+            .AppendLine()
+            .AppendLine("PER MODEL USAGE")
+            .AppendLine();
+
+        foreach (var modelUsage in ModelUsage.OrderBy(m => m.Key))
+        {
+            var promptTokens = modelUsage.Value.Sum(u => u.PromptTokens);
+            var completionTokens = modelUsage.Value.Sum(u => u.CompletionTokens);
+            var tokens = promptTokens + completionTokens;
+
+            totalPromptTokens += promptTokens;
+            totalCompletionTokens += completionTokens;
+            totalTokens += tokens;
+
+            _ = sb.AppendLine(CultureInfo.InvariantCulture, $"MODEL: {modelUsage.Key}")
+                .AppendLine()
+                .AppendLine(CultureInfo.InvariantCulture, $"Requests: {modelUsage.Value.Count}")
+                .AppendLine(CultureInfo.InvariantCulture, $"Prompt Tokens: {promptTokens}")
+                .AppendLine(CultureInfo.InvariantCulture, $"Completion Tokens: {completionTokens}")
+                .AppendLine(CultureInfo.InvariantCulture, $"Total Tokens: {tokens}");
+
+            if (IncludeCosts)
+            {
+                var cost = modelUsage.Value.Sum(u => u.Cost);
+                totalCost += cost;
+                _ = sb.AppendLine(CultureInfo.InvariantCulture, $"Total Cost: {FormatCost(cost, Currency)}");
+            }
+
+            _ = sb.AppendLine();
+        }
+
+        _ = sb
+            .AppendLine("TOTALS")
+            .AppendLine()
+            .AppendLine(CultureInfo.InvariantCulture, $"Prompt Tokens: {totalPromptTokens}")
+            .AppendLine(CultureInfo.InvariantCulture, $"Completion Tokens: {totalCompletionTokens}")
+            .AppendLine(CultureInfo.InvariantCulture, $"Total Tokens: {totalTokens}");
+
+        if (IncludeCosts)
+        {
+            _ = sb.AppendLine(CultureInfo.InvariantCulture, $"Total Cost: {FormatCost(totalCost, Currency)}");
+        }
+        return sb.ToString();
+    }
+
+    private static string FormatCost(double cost, string currency)
+    {
+        return $"{cost.ToString("#,##0.00########", CultureInfo.InvariantCulture)} {currency}";
+    }
+}