Skip to content

Commit cd604bf

Browse files
authored
Merge pull request #1174 from iceljc/master
add realtime options
2 parents 7553fce + 2f9183f commit cd604bf

File tree

9 files changed

+60
-13
lines changed

9 files changed

+60
-13
lines changed

src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ public interface IRealTimeCompletion
77
string Provider { get; }
88
string Model { get; }
99
void SetModelName(string model);
10+
void SetOptions(RealtimeOptions? options);
1011

1112
Task Connect(
1213
RealtimeHubConnection conn,

src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,6 @@ public interface IRealtimeHub
1313

1414
IRealTimeCompletion Completer { get; }
1515

16-
Task ConnectToModel(Func<string, Task>? responseToUser = null, Func<string, Task>? init = null, List<MessageState>? initStates = null);
16+
Task ConnectToModel(Func<string, Task>? responseToUser = null, Func<string, Task>? init = null,
17+
List<MessageState>? initStates = null, RealtimeOptions? options = null);
1718
}
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
namespace BotSharp.Abstraction.Realtime.Models;
2+
3+
public class RealtimeOptions
4+
{
5+
[JsonPropertyName("input_audio_format")]
6+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
7+
public string? InputAudioFormat { get; set; }
8+
9+
[JsonPropertyName("output_audio_format")]
10+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
11+
public string? OutputAudioFormat { get; set; }
12+
}

src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,11 @@ public RealtimeHub(IServiceProvider services, ILogger<RealtimeHub> logger)
2424
_logger = logger;
2525
}
2626

27-
public async Task ConnectToModel(Func<string, Task>? responseToUser = null, Func<string, Task>? init = null, List<MessageState>? initStates = null)
27+
public async Task ConnectToModel(
28+
Func<string, Task>? responseToUser = null,
29+
Func<string, Task>? init = null,
30+
List<MessageState>? initStates = null,
31+
RealtimeOptions? options = null)
2832
{
2933
var convService = _services.GetRequiredService<IConversationService>();
3034
convService.SetConversationId(_conn.ConversationId, initStates ?? []);
@@ -43,6 +47,7 @@ public async Task ConnectToModel(Func<string, Task>? responseToUser = null, Func
4347
var settings = _services.GetRequiredService<RealtimeModelSettings>();
4448

4549
_completer = _services.GetServices<IRealTimeCompletion>().First(x => x.Provider == settings.Provider);
50+
_completer.SetOptions(options);
4651

4752
await _completer.Connect(
4853
conn: _conn,

src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
using BotSharp.Abstraction.Models;
21
using BotSharp.Abstraction.Realtime.Models.Session;
32
using BotSharp.Core.Session;
43
using Microsoft.AspNetCore.Http;
@@ -84,7 +83,7 @@ private async Task HandleWebSocket(IServiceProvider services, string agentId, st
8483
_logger.LogCritical($"Start chat stream connection for conversation ({conversationId})");
8584
#endif
8685
var request = InitRequest(data, conversationId);
87-
await ConnectToModel(hub, session, request?.States);
86+
await ConnectToModel(hub, session, request);
8887
}
8988
else if (eventType == "media")
9089
{
@@ -107,15 +106,15 @@ private async Task HandleWebSocket(IServiceProvider services, string agentId, st
107106
await session.DisconnectAsync();
108107
}
109108

110-
private async Task ConnectToModel(IRealtimeHub hub, BotSharpRealtimeSession session, List<MessageState>? states = null)
109+
private async Task ConnectToModel(IRealtimeHub hub, BotSharpRealtimeSession session, ChatStreamRequest? request)
111110
{
112111
await hub.ConnectToModel(responseToUser: async data =>
113112
{
114113
if (session != null)
115114
{
116115
await session.SendEventAsync(data);
117116
}
118-
}, initStates: states);
117+
}, initStates: request?.States, options: request?.Options);
119118
}
120119

121120
private (string, string) MapEvents(RealtimeHubConnection conn, string receivedText, string conversationId)

src/Plugins/BotSharp.Plugin.ChatHub/Models/Stream/ChatStreamRequest.cs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,8 @@ public class ChatStreamRequest
77
{
88
[JsonPropertyName("states")]
99
public List<MessageState> States { get; set; } = [];
10+
11+
[JsonPropertyName("realtime_options")]
12+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
13+
public RealtimeOptions? Options { get; set; }
1014
}

src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,6 @@ public GoogleRealTimeProvider(
5858
_logger = logger;
5959
}
6060

61-
public void SetModelName(string model)
62-
{
63-
_model = model;
64-
}
65-
6661
public async Task Connect(
6762
RealtimeHubConnection conn,
6863
Func<Task> onModelReady,
@@ -420,6 +415,16 @@ await SendEventToModel(new RealtimeClientPayload
420415
}
421416
}
422417

418+
public void SetModelName(string model)
419+
{
420+
_model = model;
421+
}
422+
423+
public void SetOptions(RealtimeOptions? options)
424+
{
425+
426+
}
427+
423428
#region Private methods
424429
private List<RoleDialogModel> OnFunctionCall(RealtimeHubConnection conn, RealtimeFunctionCall functionCall)
425430
{

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
1818

1919
private string _model = "gpt-4o-mini-realtime-preview";
2020
private LlmRealtimeSession _session;
21+
private RealtimeOptions? _realtimeOptions;
2122
private bool _isBlocking = false;
2223

2324
private RealtimeHubConnection _conn;
@@ -338,8 +339,8 @@ public async Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit
338339
type = "session.update",
339340
session = new RealtimeSessionUpdateRequest
340341
{
341-
InputAudioFormat = realtimeModelSettings.InputAudioFormat,
342-
OutputAudioFormat = realtimeModelSettings.OutputAudioFormat,
342+
InputAudioFormat = _realtimeOptions?.InputAudioFormat ?? realtimeModelSettings.InputAudioFormat,
343+
OutputAudioFormat = _realtimeOptions?.OutputAudioFormat ?? realtimeModelSettings.OutputAudioFormat,
343344
Voice = realtimeModelSettings.Voice,
344345
Instructions = instruction,
345346
ToolChoice = "auto",
@@ -457,6 +458,11 @@ public void SetModelName(string model)
457458
_model = model;
458459
}
459460

461+
public void SetOptions(RealtimeOptions? options)
462+
{
463+
_realtimeOptions = options;
464+
}
465+
460466
#region Private methods
461467
private async Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection conn, string response)
462468
{

src/WebStarter/appsettings.json

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -570,6 +570,20 @@
570570
}
571571
},
572572

573+
"RealtimeModel": {
574+
"Provider": "openai",
575+
"Model": "gpt-realtime",
576+
"InputAudioFormat": "pcm16",
577+
"OutputAudioFormat": "pcm16",
578+
"InterruptResponse": true,
579+
"MaxResponseOutputTokens": 4096,
580+
"InputAudioTranscribe": true,
581+
"InputAudioTranscription": {
582+
"Model": "whisper-1",
583+
"Language": "en"
584+
}
585+
},
586+
573587
"PluginLoader": {
574588
"Assemblies": [
575589
"BotSharp.Core",

0 commit comments

Comments
 (0)