diff --git a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs index 1b1d75bd..d560196a 100644 --- a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs +++ b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs @@ -15,6 +15,7 @@ using OpenTelemetry.Metrics; using OpenTelemetry.Resources; using OpenTelemetry.Trace; +using System.Collections.Concurrent; using System.Diagnostics; using System.Diagnostics.Metrics; using System.Text.Json; @@ -46,7 +47,7 @@ public sealed class OpenAITelemetryPlugin( ISet urlsToWatch, IProxyConfiguration proxyConfiguration, IConfigurationSection pluginConfigurationSection) : - BasePlugin( + BaseReportingPlugin( httpClient, logger, urlsToWatch, @@ -65,6 +66,7 @@ public sealed class OpenAITelemetryPlugin( private LanguageModelPricesLoader? _loader; private MeterProvider? _meterProvider; private TracerProvider? _tracerProvider; + private readonly ConcurrentDictionary> _modelUsage = []; public override string Name => nameof(OpenAITelemetryPlugin); @@ -189,6 +191,26 @@ public override Task AfterResponseAsync(ProxyResponseArgs e, CancellationToken c return Task.CompletedTask; } + public override Task AfterRecordingStopAsync(RecordingArgs e, CancellationToken cancellationToken) + { + Logger.LogTrace("{Method} called", nameof(AfterRecordingStopAsync)); + + var report = new OpenAITelemetryPluginReport + { + Application = Configuration.Application, + Environment = Configuration.Environment, + Currency = Configuration.Currency, + IncludeCosts = Configuration.IncludeCosts, + ModelUsage = _modelUsage.ToDictionary() + }; + + StoreReport(report, e); + _modelUsage.Clear(); + + Logger.LogTrace("Left {Name}", nameof(AfterRecordingStopAsync)); + return Task.CompletedTask; + } + private void InitializeOpenTelemetryExporter() { Logger.LogTrace("InitializeOpenTelemetryExporter() called"); @@ -811,6 +833,15 @@ private void RecordUsageMetrics(Activity activity, OpenAIRequest request, OpenAI .SetTag(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, usage.CompletionTokens) .SetTag(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS, usage.TotalTokens); + var reportModelUsageInformation = new OpenAITelemetryPluginReportModelUsageInformation + { + Model = response.Model, + PromptTokens = usage.PromptTokens, + CompletionTokens = usage.CompletionTokens + }; + var usagePerModel = _modelUsage.GetOrAdd(response.Model, model => []); + usagePerModel.Add(reportModelUsageInformation); + if (!Configuration.IncludeCosts || Configuration.Prices is null) { Logger.LogDebug("Cost tracking is disabled or prices data is not available"); @@ -847,6 +878,7 @@ private void RecordUsageMetrics(Activity activity, OpenAIRequest request, OpenAI new(SemanticConvention.GEN_AI_REQUEST_MODEL, request.Model), new(SemanticConvention.GEN_AI_RESPONSE_MODEL, response.Model) ]); + reportModelUsageInformation.Cost = totalCost; } else { diff --git a/DevProxy.Plugins/Inspection/OpenAITelemetryPluginReport.cs b/DevProxy.Plugins/Inspection/OpenAITelemetryPluginReport.cs new file mode 100644 index 00000000..89dd5991 --- /dev/null +++ b/DevProxy.Plugins/Inspection/OpenAITelemetryPluginReport.cs @@ -0,0 +1,165 @@ +using DevProxy.Abstractions.Plugins; +using DevProxy.Abstractions.Utils; +using System.Globalization; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace DevProxy.Plugins.Inspection; + +public class OpenAITelemetryPluginReportModelUsageInformation +{ + public required long CompletionTokens { get; init; } + public double Cost { get; set; } + public required string Model { get; init; } + public required long PromptTokens { get; init; } +} + +public class OpenAITelemetryPluginReport : IMarkdownReport, IPlainTextReport, IJsonReport +{ + public required string Application { get; init; } + public required string Currency { get; init; } + public required string Environment { get; init; } + [JsonIgnore] + public bool IncludeCosts { get; set; } + public required Dictionary> ModelUsage { get; init; } = []; + + public string FileExtension => ".json"; + + public object ToJson() => JsonSerializer.Serialize(this, ProxyUtils.JsonSerializerOptions); + + public string? ToMarkdown() + { + var totalTokens = 0L; + var totalPromptTokens = 0L; + var totalCompletionTokens = 0L; + var totalCost = 0.0; + var totalRequests = 0; + + var sb = new StringBuilder(); + _ = sb + .AppendLine(CultureInfo.InvariantCulture, $"# LLM usage report for {Application} in {Environment}") + .AppendLine() + .Append("Model|Requests|Prompt Tokens|Completion Tokens|Total Tokens"); + + if (IncludeCosts) + { + _ = sb.Append("|Total Cost"); + } + + _ = sb + .AppendLine() + .Append(":----|-------:|------------:|----------------:|-----------:"); + + if (IncludeCosts) + { + _ = sb.Append("|---------:"); + } + + _ = sb.AppendLine(); + + foreach (var modelUsage in ModelUsage.OrderBy(m => m.Key)) + { + var promptTokens = modelUsage.Value.Sum(u => u.PromptTokens); + var completionTokens = modelUsage.Value.Sum(u => u.CompletionTokens); + var tokens = promptTokens + completionTokens; + + totalPromptTokens += promptTokens; + totalCompletionTokens += completionTokens; + totalTokens += tokens; + totalRequests += modelUsage.Value.Count; + + _ = sb + .Append(modelUsage.Key) + .Append('|').Append(totalRequests) + .Append('|').Append(promptTokens) + .Append('|').Append(completionTokens) + .Append('|').Append(tokens); + + if (IncludeCosts) + { + var cost = modelUsage.Value.Sum(u => u.Cost); + totalCost += cost; + _ = sb.Append('|').Append(FormatCost(cost, Currency)); + } + + _ = sb.AppendLine(); + } + + _ = sb + .Append("**Total**") + .Append('|').Append(CultureInfo.CurrentCulture, $"**{totalRequests}**") + .Append('|').Append(CultureInfo.CurrentCulture, $"**{totalPromptTokens}**") + .Append('|').Append(CultureInfo.CurrentCulture, $"**{totalCompletionTokens}**") + .Append('|').Append(CultureInfo.CurrentCulture, $"**{totalTokens}**"); + + if (IncludeCosts) + { + _ = sb.Append('|').Append(CultureInfo.CurrentCulture, $"**{FormatCost(totalCost, Currency)}**"); + } + + _ = sb.AppendLine(); + + return sb.ToString(); + } + + public string? ToPlainText() + { + var totalTokens = 0L; + var totalPromptTokens = 0L; + var totalCompletionTokens = 0L; + var totalCost = 0.0; + + var sb = new StringBuilder(); + _ = sb + .AppendLine(CultureInfo.InvariantCulture, $"LLM USAGE REPORT FOR {Application} IN {Environment}") + .AppendLine() + .AppendLine("PER MODEL USAGE") + .AppendLine(); + + foreach (var modelUsage in ModelUsage.OrderBy(m => m.Key)) + { + var promptTokens = modelUsage.Value.Sum(u => u.PromptTokens); + var completionTokens = modelUsage.Value.Sum(u => u.CompletionTokens); + var tokens = promptTokens + completionTokens; + + totalPromptTokens += promptTokens; + totalCompletionTokens += completionTokens; + totalTokens += tokens; + + _ = sb.AppendLine(CultureInfo.InvariantCulture, $"MODEL: {modelUsage.Key}") + .AppendLine() + .AppendLine(CultureInfo.InvariantCulture, $"Requests: {modelUsage.Value.Count}") + .AppendLine(CultureInfo.InvariantCulture, $"Prompt Tokens: {promptTokens}") + .AppendLine(CultureInfo.InvariantCulture, $"Completion Tokens: {completionTokens}") + .AppendLine(CultureInfo.InvariantCulture, $"Total Tokens: {tokens}"); + + if (IncludeCosts) + { + var cost = modelUsage.Value.Sum(u => u.Cost); + totalCost += cost; + _ = sb.AppendLine(CultureInfo.InvariantCulture, $"Total Cost: {FormatCost(cost, Currency)}"); + } + + _ = sb.AppendLine(); + } + + _ = sb + .AppendLine("TOTALS") + .AppendLine() + .AppendLine(CultureInfo.InvariantCulture, $"Prompt Tokens: {totalPromptTokens}") + .AppendLine(CultureInfo.InvariantCulture, $"Completion Tokens: {totalCompletionTokens}") + .AppendLine(CultureInfo.InvariantCulture, $"Total Tokens: {totalTokens}"); + + if (IncludeCosts) + { + _ = sb.AppendLine(CultureInfo.InvariantCulture, $"Total Cost: {FormatCost(totalCost, Currency)}"); + } + return sb.ToString(); + } + + private static string FormatCost(double cost, string currency) + { + return $"{cost.ToString("#,##0.00########", CultureInfo.InvariantCulture)} {currency}"; + } +}