Compare commits

...

2 Commits

Author SHA1 Message Date
4854a7fa9a ConvAgent: send agent audio multicast Unreliable + add size debug log
- MulticastReceiveAgentAudio: Reliable -> Unreliable. Realtime audio: a lost or
  oversized packet is dropped (minor glitch) instead of backing up the reliable
  buffer and dropping the client connection.
- Raw-PCM fallback chunk size 32000 -> 4000 bytes so an unreliable RPC is not
  auto-promoted to reliable when fragmented into many partial bunches (which
  would reintroduce the overflow).
- Add net-audio size log gated behind ps.ai.ConvAgent.Debug.ElevenLabs; fires on
  Authority even in solo PIE: "[NET-SRV] Opus audio chunk: N bytes (raw M -> Xx)".

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-26 19:52:48 +02:00
41900991ef ConvAgent: enable Voice module so the Opus codec initializes
The project had no [Voice] section, so the engine default bEnabled=false left
FVoiceModule::CreateVoiceEncoder() returning null (Encoder/Decoder NULL).
Networked agent audio then fell back to raw PCM in large reliable multicast
RPCs, overflowing the reliable buffer and dropping VR client connections the
moment the avatar spoke. Enabling Voice lets Opus compress (~13-20x), keeping
RPCs small.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-26 19:52:48 +02:00
3 changed files with 44 additions and 5 deletions

View File

@@ -166,6 +166,14 @@ DefaultPlatformService=Null
[OnlineSubsystemNull] [OnlineSubsystemNull]
bEnabled=true bEnabled=true
[Voice]
; Enable the Voice module so FVoiceModule::CreateVoiceEncoder() returns a valid
; Opus encoder/decoder. The engine default is bEnabled=false, which leaves the
; encoder NULL and makes networked agent audio fall back to raw PCM in large
; reliable multicast RPCs — these overflow the reliable buffer and drop VR
; client connections the moment the avatar speaks. With Opus, chunks are ~1-4KB.
bEnabled=true
[/Script/AndroidFileServerEditor.AndroidFileServerRuntimeSettings] [/Script/AndroidFileServerEditor.AndroidFileServerRuntimeSettings]
bEnablePlugin=True bEnablePlugin=True
bAllowNetworkConnection=True bAllowNetworkConnection=True

View File

@@ -1224,6 +1224,11 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray<uint
// Network: broadcast audio to all clients. // Network: broadcast audio to all clients.
if (GetOwnerRole() == ROLE_Authority) if (GetOwnerRole() == ROLE_Authority)
{ {
// Net-audio size debug: toggle at runtime with
// ps.ai.ConvAgent.Debug.ElevenLabs 1 (or set bDebug on the component).
const int32 NetDbgCVar = CVarDebugElevenLabs.GetValueOnGameThread();
const bool bNetAudioDebug = (NetDbgCVar >= 0) ? (NetDbgCVar > 0) : bDebug;
if (OpusEncoder.IsValid()) if (OpusEncoder.IsValid())
{ {
// Opus path: compress then send. // Opus path: compress then send.
@@ -1235,14 +1240,28 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray<uint
{ {
TArray<uint8> CompressedData; TArray<uint8> CompressedData;
CompressedData.Append(OpusWorkBuffer.GetData(), CompressedSize); CompressedData.Append(OpusWorkBuffer.GetData(), CompressedSize);
if (bNetAudioDebug)
{
const float Ratio = (CompressedSize > 0)
? static_cast<float>(PCMData.Num()) / static_cast<float>(CompressedSize)
: 0.0f;
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
TEXT("[NET-SRV] Opus audio chunk: %u bytes sent (raw %d → %.1fx compression)."),
CompressedSize, PCMData.Num(), Ratio);
}
MulticastReceiveAgentAudio(CompressedData); MulticastReceiveAgentAudio(CompressedData);
} }
} }
else else
{ {
// Fallback: send raw PCM (no compression). ~32 KB/s at 16kHz 16-bit mono. // Fallback: send raw PCM (no compression). ~32 KB/s at 16kHz 16-bit mono.
// Fine for LAN; revisit with proper Opus if internet play is needed. // Only hit if Opus init failed. Chunk small so the Unreliable RPC is
// UE5 limits replicated TArrays to 65535 elements, so we must chunk. // NOT auto-promoted to reliable when fragmented: UE re-sends an
// unreliable bunch reliably once it splits into more than ~8 partial
// bunches, which would reintroduce the reliable-buffer overflow.
// ~4 KB ≈ 4 bunches → stays unreliable.
static bool bWarnedOnce = false; static bool bWarnedOnce = false;
if (!bWarnedOnce) if (!bWarnedOnce)
{ {
@@ -1251,13 +1270,21 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray<uint
TEXT("[NET-SRV] Opus encoder unavailable — sending raw PCM in chunks (no compression).")); TEXT("[NET-SRV] Opus encoder unavailable — sending raw PCM in chunks (no compression)."));
} }
static constexpr int32 MaxChunkBytes = 32000; // 1s of 16kHz 16-bit mono, well under 65535 limit static constexpr int32 MaxChunkBytes = 4000; // small enough to stay unreliable (avoid partial-bunch reliable promotion)
int32 Offset = 0; int32 Offset = 0;
while (Offset < PCMData.Num()) while (Offset < PCMData.Num())
{ {
const int32 ChunkSize = FMath::Min(MaxChunkBytes, PCMData.Num() - Offset); const int32 ChunkSize = FMath::Min(MaxChunkBytes, PCMData.Num() - Offset);
TArray<uint8> Chunk; TArray<uint8> Chunk;
Chunk.Append(PCMData.GetData() + Offset, ChunkSize); Chunk.Append(PCMData.GetData() + Offset, ChunkSize);
if (bNetAudioDebug)
{
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
TEXT("[NET-SRV] RAW PCM chunk: %d bytes sent (NO compression — Opus disabled)."),
ChunkSize);
}
MulticastReceiveAgentAudio(Chunk); MulticastReceiveAgentAudio(Chunk);
Offset += ChunkSize; Offset += ChunkSize;
} }

View File

@@ -484,8 +484,12 @@ public:
UFUNCTION(Server, Reliable) UFUNCTION(Server, Reliable)
void ServerRequestInterrupt(); void ServerRequestInterrupt();
/** Broadcast agent audio to all clients (Opus-compressed or raw PCM fallback). */ /** Broadcast agent audio to all clients (Opus-compressed or raw PCM fallback).
UFUNCTION(NetMulticast, Reliable) * Unreliable: realtime audio — a lost/oversized packet is dropped (minor
* glitch) rather than backing up the reliable buffer and dropping the
* connection. Keep each call small (Opus ~1-4KB) so it isn't auto-promoted
* to reliable when fragmented into partial bunches. */
UFUNCTION(NetMulticast, Unreliable)
void MulticastReceiveAgentAudio(const TArray<uint8>& AudioData); void MulticastReceiveAgentAudio(const TArray<uint8>& AudioData);
/** Notify all clients that the agent started speaking (first audio chunk). */ /** Notify all clients that the agent started speaking (first audio chunk). */