From 769b29a798d7f87525676ba14a994c5873b82b2f Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 26 Sep 2025 16:19:23 -0500 Subject: [PATCH 1/2] add realtime options --- .../MLTasks/IRealTimeCompletion.cs | 1 + .../BotSharp.Abstraction/Realtime/IRealtimeHub.cs | 3 ++- .../Realtime/Models/RealtimeOptions.cs | 12 ++++++++++++ .../Services/RealtimeHub.cs | 7 ++++++- .../ChatStreamMiddleware.cs | 7 +++---- .../Models/Stream/ChatStreamRequest.cs | 4 ++++ .../Realtime/RealTimeCompletionProvider.cs | 15 ++++++++++----- .../Realtime/RealTimeCompletionProvider.cs | 10 ++++++++-- 8 files changed, 46 insertions(+), 13 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeOptions.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index bb3104d36..65ae2dbc1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -7,6 +7,7 @@ public interface IRealTimeCompletion string Provider { get; } string Model { get; } void SetModelName(string model); + void SetOptions(RealtimeOptions? options); Task Connect( RealtimeHubConnection conn, diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs index 9d64b6785..e75f46998 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs @@ -13,5 +13,6 @@ public interface IRealtimeHub IRealTimeCompletion Completer { get; } - Task ConnectToModel(Func? responseToUser = null, Func? init = null, List? initStates = null); + Task ConnectToModel(Func? responseToUser = null, Func? init = null, + List? initStates = null, RealtimeOptions? options = null); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeOptions.cs new file mode 100644 index 000000000..a720c88b0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeOptions.cs @@ -0,0 +1,12 @@ +namespace BotSharp.Abstraction.Realtime.Models; + +public class RealtimeOptions +{ + [JsonPropertyName("input_audio_format")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? InputAudioFormat { get; set; } + + [JsonPropertyName("output_audio_format")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? OutputAudioFormat { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 8c057b950..b34f5fba4 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -24,7 +24,11 @@ public RealtimeHub(IServiceProvider services, ILogger logger) _logger = logger; } - public async Task ConnectToModel(Func? responseToUser = null, Func? init = null, List? initStates = null) + public async Task ConnectToModel( + Func? responseToUser = null, + Func? init = null, + List? initStates = null, + RealtimeOptions? options = null) { var convService = _services.GetRequiredService(); convService.SetConversationId(_conn.ConversationId, initStates ?? []); @@ -43,6 +47,7 @@ public async Task ConnectToModel(Func? responseToUser = null, Func var settings = _services.GetRequiredService(); _completer = _services.GetServices().First(x => x.Provider == settings.Provider); + _completer.SetOptions(options); await _completer.Connect( conn: _conn, diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs index 44a6195a7..a08fda080 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Models; using BotSharp.Abstraction.Realtime.Models.Session; using BotSharp.Core.Session; using Microsoft.AspNetCore.Http; @@ -84,7 +83,7 @@ private async Task HandleWebSocket(IServiceProvider services, string agentId, st _logger.LogCritical($"Start chat stream connection for conversation ({conversationId})"); #endif var request = InitRequest(data, conversationId); - await ConnectToModel(hub, session, request?.States); + await ConnectToModel(hub, session, request); } else if (eventType == "media") { @@ -107,7 +106,7 @@ private async Task HandleWebSocket(IServiceProvider services, string agentId, st await session.DisconnectAsync(); } - private async Task ConnectToModel(IRealtimeHub hub, BotSharpRealtimeSession session, List? states = null) + private async Task ConnectToModel(IRealtimeHub hub, BotSharpRealtimeSession session, ChatStreamRequest? request) { await hub.ConnectToModel(responseToUser: async data => { @@ -115,7 +114,7 @@ await hub.ConnectToModel(responseToUser: async data => { await session.SendEventAsync(data); } - }, initStates: states); + }, initStates: request?.States, options: request?.Options); } private (string, string) MapEvents(RealtimeHubConnection conn, string receivedText, string conversationId) diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Models/Stream/ChatStreamRequest.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Models/Stream/ChatStreamRequest.cs index 1a72d7dce..13ef8f0e7 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Models/Stream/ChatStreamRequest.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Models/Stream/ChatStreamRequest.cs @@ -7,4 +7,8 @@ public class ChatStreamRequest { [JsonPropertyName("states")] public List States { get; set; } = []; + + [JsonPropertyName("realtime_options")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public RealtimeOptions? Options { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index 983b203a5..90395d49b 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -58,11 +58,6 @@ public GoogleRealTimeProvider( _logger = logger; } - public void SetModelName(string model) - { - _model = model; - } - public async Task Connect( RealtimeHubConnection conn, Func onModelReady, @@ -420,6 +415,16 @@ await SendEventToModel(new RealtimeClientPayload } } + public void SetModelName(string model) + { + _model = model; + } + + public void SetOptions(RealtimeOptions? options) + { + + } + #region Private methods private List OnFunctionCall(RealtimeHubConnection conn, RealtimeFunctionCall functionCall) { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 890a8619c..1e246c630 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -18,6 +18,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion private string _model = "gpt-4o-mini-realtime-preview"; private LlmRealtimeSession _session; + private RealtimeOptions? _realtimeOptions; private bool _isBlocking = false; private RealtimeHubConnection _conn; @@ -338,8 +339,8 @@ public async Task UpdateSession(RealtimeHubConnection conn, bool isInit type = "session.update", session = new RealtimeSessionUpdateRequest { - InputAudioFormat = realtimeModelSettings.InputAudioFormat, - OutputAudioFormat = realtimeModelSettings.OutputAudioFormat, + InputAudioFormat = _realtimeOptions?.InputAudioFormat ?? realtimeModelSettings.InputAudioFormat, + OutputAudioFormat = _realtimeOptions?.OutputAudioFormat ?? realtimeModelSettings.OutputAudioFormat, Voice = realtimeModelSettings.Voice, Instructions = instruction, ToolChoice = "auto", @@ -457,6 +458,11 @@ public void SetModelName(string model) _model = model; } + public void SetOptions(RealtimeOptions? options) + { + _realtimeOptions = options; + } + #region Private methods private async Task> OnResponsedDone(RealtimeHubConnection conn, string response) { From 2f9183f7d2d8c5a4b4fc4a1e195815a30e4aa507 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 26 Sep 2025 16:45:22 -0500 Subject: [PATCH 2/2] add realtime model settings --- src/WebStarter/appsettings.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index c55e9e2c1..59d7d3f55 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -570,6 +570,20 @@ } }, + "RealtimeModel": { + "Provider": "openai", + "Model": "gpt-realtime", + "InputAudioFormat": "pcm16", + "OutputAudioFormat": "pcm16", + "InterruptResponse": true, + "MaxResponseOutputTokens": 4096, + "InputAudioTranscribe": true, + "InputAudioTranscription": { + "Model": "whisper-1", + "Language": "en" + } + }, + "PluginLoader": { "Assemblies": [ "BotSharp.Core",