From 5b3a11a733dbae4f8437b5e2eff537971e7f3767 Mon Sep 17 00:00:00 2001
From: "j.foucher" <j.foucher@polymorph.fr>
Date: Wed, 1 Jul 2026 14:31:02 +0200
Subject: [PATCH] ConvAgent: agent audio multicast back to Reliable (fix Opus
 crackle/gaps)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Sending Opus over Unreliable multicast caused missing words and crackling:
Opus is a stateful inter-frame codec, so a dropped or reordered packet
desyncs the decoder and corrupts all following frames until the stream
resets. Unreliable was originally adopted only to stop a reliable-buffer
overflow — but that overflow came from 32-152KB raw-PCM RPCs (pre-Opus).
With Opus + sub-chunking each RPC is ~2KB (2-3 partial bunches) at
~1.25 RPC/s, so the 512-bunch reliable buffer stays in single digits and
cannot overflow. Reliable restores ordered, lossless delivery.

- MulticastReceiveAgentAudio: Unreliable -> Reliable.
- Keep the 40-frame (0.8s, ~2KB) sub-chunking — it is what keeps packets
  small enough for Reliable to be safe.
- Harden the raw-PCM fallback (only reached if Opus init fails): hard-cap
  one message at 320KB and drop the tail so it can never overflow the
  reliable buffer, and warn on every fallback (config bug: [Voice]).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 .../PS_AI_ConvAgent_ElevenLabsComponent.cpp   | 37 ++++++++++++-------
 .../PS_AI_ConvAgent_ElevenLabsComponent.h     | 16 +++++---
 2 files changed, 34 insertions(+), 19 deletions(-)

diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp
index ab04703..a8b5590 100644
--- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp
+++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp
@@ -1278,24 +1278,33 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray<uint
 		}
 		else
 		{
-			// Fallback: send raw PCM (no compression). ~32 KB/s at 16kHz 16-bit mono.
-			// Only hit if Opus init failed. Chunk small so the Unreliable RPC is
-			// NOT auto-promoted to reliable when fragmented: UE re-sends an
-			// unreliable bunch reliably once it splits into more than ~8 partial
-			// bunches, which would reintroduce the reliable-buffer overflow.
-			// ~4 KB ≈ 4 bunches → stays unreliable.
-			static bool bWarnedOnce = false;
-			if (!bWarnedOnce)
-			{
-				bWarnedOnce = true;
-				UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
-					TEXT("[NET-SRV] Opus encoder unavailable — sending raw PCM in chunks (no compression)."));
-			}
+			// Fallback: Opus init failed → send raw PCM (no compression, ~32 KB/s).
+			// This should not happen in the working config — it means [Voice]
+			// bEnabled=true is missing in DefaultEngine.ini. Warn on every message
+			// so it is loud, not hidden behind a one-shot flag.
+			// The RPC is now Reliable, so we must protect the reliable buffer (512
+			// bunches): each ~4 KB chunk ≈ ~4 partial bunches. Hard-cap one message
+			// at MaxFallbackBytes and drop the tail so a single message can never
+			// approach the buffer and trigger a reliable-buffer overflow (the exact
+			// cause of the old VR disconnect).
+			UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
+				TEXT("[NET-SRV] Opus encoder unavailable — sending raw PCM (no compression). "
+					 "Set [Voice] bEnabled=true in DefaultEngine.ini to restore Opus."));
 
-			static constexpr int32 MaxChunkBytes = 4000; // small enough to stay unreliable (avoid partial-bunch reliable promotion)
+			static constexpr int32 MaxChunkBytes = 4000;
+			static constexpr int32 MaxFallbackBytes = 320000; // ~80 reliable chunks/message, well under the 512-bunch buffer
 			int32 Offset = 0;
 			while (Offset < PCMData.Num())
 			{
+				if (Offset >= MaxFallbackBytes)
+				{
+					UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
+						TEXT("[NET-SRV] Raw-PCM fallback capped at %d bytes — dropping %d-byte tail "
+							 "to protect the reliable buffer."),
+						MaxFallbackBytes, PCMData.Num() - Offset);
+					break;
+				}
+
 				const int32 ChunkSize = FMath::Min(MaxChunkBytes, PCMData.Num() - Offset);
 				TArray<uint8> Chunk;
 				Chunk.Append(PCMData.GetData() + Offset, ChunkSize);
diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h
index 2712eb0..579db08 100644
--- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h
+++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h
@@ -485,11 +485,17 @@ public:
 	void ServerRequestInterrupt();
 
 	/** Broadcast agent audio to all clients (Opus-compressed or raw PCM fallback).
-	 *  Unreliable: realtime audio — a lost/oversized packet is dropped (minor
-	 *  glitch) rather than backing up the reliable buffer and dropping the
-	 *  connection. Keep each call small (Opus ~1-4KB) so it isn't auto-promoted
-	 *  to reliable when fragmented into partial bunches. */
-	UFUNCTION(NetMulticast, Unreliable)
+	 *  Reliable: agent TTS must arrive complete and IN ORDER. Opus is a stateful
+	 *  (inter-frame-predictive) codec, so a dropped or reordered packet on an
+	 *  Unreliable channel desyncs the decoder → missing words + crackle. Reliable
+	 *  guarantees ordered, lossless delivery, keeping the decoder in sync.
+	 *  This does NOT reintroduce the old reliable-buffer overflow (which dropped
+	 *  VR connections): that came from 32-152 KB raw-PCM RPCs. With Opus each
+	 *  sub-chunk is ~2 KB (2-3 partial bunches, below the 8-bunch reliable-
+	 *  promotion threshold) at ~1.25 RPC/s, so in-flight bunches stay in single
+	 *  digits vs the 512-bunch buffer. Sub-chunking in HandleAudioReceived is what
+	 *  keeps packets this small — it must stay. */
+	UFUNCTION(NetMulticast, Reliable)
 	void MulticastReceiveAgentAudio(const TArray<uint8>& AudioData);
 
 	/** Notify all clients that the agent started speaking (first audio chunk). */