From 087c83018c01bdb11eef3655e5b854fdd31d5095 Mon Sep 17 00:00:00 2001 From: "j.foucher" Date: Thu, 2 Jul 2026 17:18:10 +0200 Subject: [PATCH] ConvAgent: fire the voice-onset interrupt server-side (dedicated-server support) The agent-to-agent dialogue interrupt was detected only in FeedExternalAudio, which runs client-side for a remote player. On a DEDICATED server (no host player) no one could trigger the reliable VAD interrupt -- everyone fell back to the slower ElevenLabs transcript path. Also broadcast OnUserVoiceDetected from ServerSendMicAudioFromPlayer, where the player's mic (the host's OR a remote client's, relayed) always arrives on the server. Shares the LastUserVoiceOnset debounce so the host does not double-fire. The orchestrator still attributes the correct player (ResolveSpeakingPlayer finds the connected client via NetConnectedPawns). Co-Authored-By: Claude Opus 4.8 --- .../PS_AI_ConvAgent_ElevenLabsComponent.cpp | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp index 6e9293f..d89d228 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp @@ -2525,6 +2525,32 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ServerSendMicAudioFromPlayer( { if (!WebSocketProxy || !WebSocketProxy->IsConnected()) return; + // Server-side voice-onset. This runs on the SERVER (only the server has a live + // WebSocket — clients returned above), and the player's mic — the host's OR a + // remote client's, relayed here — always passes through this function. Broadcasting + // OnUserVoiceDetected here is what makes the agent-to-agent dialogue interrupt work + // on a DEDICATED server (no host player). Debounced (shares LastUserVoiceOnset with + // the FeedExternalAudio path, so the host doesn't double-fire). + { + const int32 OnsetSamples = PCMBytes.Num() / (int32)sizeof(int16); + if (OnsetSamples > 0) + { + const int16* OS = reinterpret_cast(PCMBytes.GetData()); + double OnsetSumSq = 0.0; + for (int32 i = 0; i < OnsetSamples; ++i) { const double v = OS[i] / 32768.0; OnsetSumSq += v * v; } + const double OnsetRms = FMath::Sqrt(OnsetSumSq / OnsetSamples); + if (OnsetRms >= 0.02) + { + const double NowSec = FPlatformTime::Seconds(); + if (NowSec - LastUserVoiceOnset > 0.5) + { + LastUserVoiceOnset = NowSec; + OnUserVoiceDetected.Broadcast(); + } + } + } + } + // Standalone / single-player: bypass speaker arbitration entirely. // There's only one player — no need for Contains check or speaker switching. if (NetConnectedPawns.Num() <= 1)