j.foucher/PS_AI_Agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ConvAgent: send agent audio multicast Unreliable + add size debug log

Browse Source 
- MulticastReceiveAgentAudio: Reliable -> Unreliable. Realtime audio: a lost or
  oversized packet is dropped (minor glitch) instead of backing up the reliable
  buffer and dropping the client connection.
- Raw-PCM fallback chunk size 32000 -> 4000 bytes so an unreliable RPC is not
  auto-promoted to reliable when fragmented into many partial bunches (which
  would reintroduce the overflow).
- Add net-audio size log gated behind ps.ai.ConvAgent.Debug.ElevenLabs; fires on
  Authority even in solo PIE: "[NET-SRV] Opus audio chunk: N bytes (raw M -> Xx)".

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
j.foucher
2026-06-26 19:52:48 +02:00
parent 41900991ef
commit 4854a7fa9a
2 changed files with 36 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp
View File

@@ -1224,6 +1224,11 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray<uint
// Network: broadcast audio to all clients.
if (GetOwnerRole() == ROLE_Authority)
{
// Net-audio size debug: toggle at runtime with
// ps.ai.ConvAgent.Debug.ElevenLabs 1 (or set bDebug on the component).
const int32 NetDbgCVar = CVarDebugElevenLabs.GetValueOnGameThread();
const bool bNetAudioDebug = (NetDbgCVar >= 0) ? (NetDbgCVar > 0) : bDebug;
if (OpusEncoder.IsValid())
{
// Opus path: compress then send.
@@ -1235,14 +1240,28 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray<uint
{
TArray<uint8> CompressedData;
CompressedData.Append(OpusWorkBuffer.GetData(), CompressedSize);
if (bNetAudioDebug)
{
const float Ratio = (CompressedSize > 0)
? static_cast<float>(PCMData.Num()) / static_cast<float>(CompressedSize)
: 0.0f;
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
TEXT("[NET-SRV] Opus audio chunk: %u bytes sent (raw %d → %.1fx compression)."),
CompressedSize, PCMData.Num(), Ratio);
}
MulticastReceiveAgentAudio(CompressedData);
}
}
else
{
// Fallback: send raw PCM (no compression). ~32 KB/s at 16kHz 16-bit mono.
// Fine for LAN; revisit with proper Opus if internet play is needed.
// UE5 limits replicated TArrays to 65535 elements, so we must chunk.
// Only hit if Opus init failed. Chunk small so the Unreliable RPC is
// NOT auto-promoted to reliable when fragmented: UE re-sends an
// unreliable bunch reliably once it splits into more than ~8 partial
// bunches, which would reintroduce the reliable-buffer overflow.
// ~4 KB ≈ 4 bunches → stays unreliable.
static bool bWarnedOnce = false;
if (!bWarnedOnce)
{
@@ -1251,13 +1270,21 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray<uint
TEXT("[NET-SRV] Opus encoder unavailable — sending raw PCM in chunks (no compression)."));
}
static constexpr int32 MaxChunkBytes = 32000; // 1s of 16kHz 16-bit mono, well under 65535 limit
static constexpr int32 MaxChunkBytes = 4000; // small enough to stay unreliable (avoid partial-bunch reliable promotion)
int32 Offset = 0;
while (Offset < PCMData.Num())
{
const int32 ChunkSize = FMath::Min(MaxChunkBytes, PCMData.Num() - Offset);
TArray<uint8> Chunk;
Chunk.Append(PCMData.GetData() + Offset, ChunkSize);
if (bNetAudioDebug)
{
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
TEXT("[NET-SRV] RAW PCM chunk: %d bytes sent (NO compression — Opus disabled)."),
ChunkSize);
}
MulticastReceiveAgentAudio(Chunk);
Offset += ChunkSize;
}

8
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h
View File

@@ -484,8 +484,12 @@ public:
UFUNCTION(Server, Reliable)
void ServerRequestInterrupt();
/** Broadcast agent audio to all clients (Opus-compressed or raw PCM fallback). */
UFUNCTION(NetMulticast, Reliable)
/** Broadcast agent audio to all clients (Opus-compressed or raw PCM fallback).
* Unreliable: realtime audio — a lost/oversized packet is dropped (minor
* glitch) rather than backing up the reliable buffer and dropping the
* connection. Keep each call small (Opus ~1-4KB) so it isn't auto-promoted
* to reliable when fragmented into partial bunches. */
UFUNCTION(NetMulticast, Unreliable)
void MulticastReceiveAgentAudio(const TArray<uint8>& AudioData);
/** Notify all clients that the agent started speaking (first audio chunk). */