ConvAgent: send agent audio multicast Unreliable + add size debug log
- MulticastReceiveAgentAudio: Reliable -> Unreliable. Realtime audio: a lost or oversized packet is dropped (minor glitch) instead of backing up the reliable buffer and dropping the client connection. - Raw-PCM fallback chunk size 32000 -> 4000 bytes so an unreliable RPC is not auto-promoted to reliable when fragmented into many partial bunches (which would reintroduce the overflow). - Add net-audio size log gated behind ps.ai.ConvAgent.Debug.ElevenLabs; fires on Authority even in solo PIE: "[NET-SRV] Opus audio chunk: N bytes (raw M -> Xx)". Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -1224,6 +1224,11 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray<uint
|
|||||||
// Network: broadcast audio to all clients.
|
// Network: broadcast audio to all clients.
|
||||||
if (GetOwnerRole() == ROLE_Authority)
|
if (GetOwnerRole() == ROLE_Authority)
|
||||||
{
|
{
|
||||||
|
// Net-audio size debug: toggle at runtime with
|
||||||
|
// ps.ai.ConvAgent.Debug.ElevenLabs 1 (or set bDebug on the component).
|
||||||
|
const int32 NetDbgCVar = CVarDebugElevenLabs.GetValueOnGameThread();
|
||||||
|
const bool bNetAudioDebug = (NetDbgCVar >= 0) ? (NetDbgCVar > 0) : bDebug;
|
||||||
|
|
||||||
if (OpusEncoder.IsValid())
|
if (OpusEncoder.IsValid())
|
||||||
{
|
{
|
||||||
// Opus path: compress then send.
|
// Opus path: compress then send.
|
||||||
@@ -1235,14 +1240,28 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray<uint
|
|||||||
{
|
{
|
||||||
TArray<uint8> CompressedData;
|
TArray<uint8> CompressedData;
|
||||||
CompressedData.Append(OpusWorkBuffer.GetData(), CompressedSize);
|
CompressedData.Append(OpusWorkBuffer.GetData(), CompressedSize);
|
||||||
|
|
||||||
|
if (bNetAudioDebug)
|
||||||
|
{
|
||||||
|
const float Ratio = (CompressedSize > 0)
|
||||||
|
? static_cast<float>(PCMData.Num()) / static_cast<float>(CompressedSize)
|
||||||
|
: 0.0f;
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
||||||
|
TEXT("[NET-SRV] Opus audio chunk: %u bytes sent (raw %d → %.1fx compression)."),
|
||||||
|
CompressedSize, PCMData.Num(), Ratio);
|
||||||
|
}
|
||||||
|
|
||||||
MulticastReceiveAgentAudio(CompressedData);
|
MulticastReceiveAgentAudio(CompressedData);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
// Fallback: send raw PCM (no compression). ~32 KB/s at 16kHz 16-bit mono.
|
// Fallback: send raw PCM (no compression). ~32 KB/s at 16kHz 16-bit mono.
|
||||||
// Fine for LAN; revisit with proper Opus if internet play is needed.
|
// Only hit if Opus init failed. Chunk small so the Unreliable RPC is
|
||||||
// UE5 limits replicated TArrays to 65535 elements, so we must chunk.
|
// NOT auto-promoted to reliable when fragmented: UE re-sends an
|
||||||
|
// unreliable bunch reliably once it splits into more than ~8 partial
|
||||||
|
// bunches, which would reintroduce the reliable-buffer overflow.
|
||||||
|
// ~4 KB ≈ 4 bunches → stays unreliable.
|
||||||
static bool bWarnedOnce = false;
|
static bool bWarnedOnce = false;
|
||||||
if (!bWarnedOnce)
|
if (!bWarnedOnce)
|
||||||
{
|
{
|
||||||
@@ -1251,13 +1270,21 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray<uint
|
|||||||
TEXT("[NET-SRV] Opus encoder unavailable — sending raw PCM in chunks (no compression)."));
|
TEXT("[NET-SRV] Opus encoder unavailable — sending raw PCM in chunks (no compression)."));
|
||||||
}
|
}
|
||||||
|
|
||||||
static constexpr int32 MaxChunkBytes = 32000; // 1s of 16kHz 16-bit mono, well under 65535 limit
|
static constexpr int32 MaxChunkBytes = 4000; // small enough to stay unreliable (avoid partial-bunch reliable promotion)
|
||||||
int32 Offset = 0;
|
int32 Offset = 0;
|
||||||
while (Offset < PCMData.Num())
|
while (Offset < PCMData.Num())
|
||||||
{
|
{
|
||||||
const int32 ChunkSize = FMath::Min(MaxChunkBytes, PCMData.Num() - Offset);
|
const int32 ChunkSize = FMath::Min(MaxChunkBytes, PCMData.Num() - Offset);
|
||||||
TArray<uint8> Chunk;
|
TArray<uint8> Chunk;
|
||||||
Chunk.Append(PCMData.GetData() + Offset, ChunkSize);
|
Chunk.Append(PCMData.GetData() + Offset, ChunkSize);
|
||||||
|
|
||||||
|
if (bNetAudioDebug)
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
|
||||||
|
TEXT("[NET-SRV] RAW PCM chunk: %d bytes sent (NO compression — Opus disabled)."),
|
||||||
|
ChunkSize);
|
||||||
|
}
|
||||||
|
|
||||||
MulticastReceiveAgentAudio(Chunk);
|
MulticastReceiveAgentAudio(Chunk);
|
||||||
Offset += ChunkSize;
|
Offset += ChunkSize;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -484,8 +484,12 @@ public:
|
|||||||
UFUNCTION(Server, Reliable)
|
UFUNCTION(Server, Reliable)
|
||||||
void ServerRequestInterrupt();
|
void ServerRequestInterrupt();
|
||||||
|
|
||||||
/** Broadcast agent audio to all clients (Opus-compressed or raw PCM fallback). */
|
/** Broadcast agent audio to all clients (Opus-compressed or raw PCM fallback).
|
||||||
UFUNCTION(NetMulticast, Reliable)
|
* Unreliable: realtime audio — a lost/oversized packet is dropped (minor
|
||||||
|
* glitch) rather than backing up the reliable buffer and dropping the
|
||||||
|
* connection. Keep each call small (Opus ~1-4KB) so it isn't auto-promoted
|
||||||
|
* to reliable when fragmented into partial bunches. */
|
||||||
|
UFUNCTION(NetMulticast, Unreliable)
|
||||||
void MulticastReceiveAgentAudio(const TArray<uint8>& AudioData);
|
void MulticastReceiveAgentAudio(const TArray<uint8>& AudioData);
|
||||||
|
|
||||||
/** Notify all clients that the agent started speaking (first audio chunk). */
|
/** Notify all clients that the agent started speaking (first audio chunk). */
|
||||||
|
|||||||
Reference in New Issue
Block a user