ConvAgent: fix agent voice truncation on remote clients (Opus)
Long ElevenLabs audio messages were cut off on remote clients: - the client Opus decode buffer was only 32000 bytes (~0.88s), so the engine decoder aborted with "Decompression buffer too small" and dropped the tail of any packet longer than that; - the server also encoded a whole message as a single Opus packet, which the engine encoder caps at 76 frames (~1.52s), silently dropping the rest. The host/authority plays raw PCM and never decodes, so standalone and listen-server hosts were unaffected, which masked the bug. Fix (HandleAudioReceived + MulticastReceiveAgentAudio_Implementation): - sub-chunk the Opus encode into <=40-frame (25600-byte, 0.8s) blocks, one RPC each: stays under both the 76-frame encode cap and the client decode buffer, and keeps each Unreliable RPC small; - raise the client decode buffer 32000 -> 64*1024 as headroom. Empirically confirmed: networked client logged 16x "Decompression buffer too small" with Opus active; standalone (Opus equally active) logged zero. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -1231,27 +1231,49 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray<uint
|
|||||||
|
|
||||||
if (OpusEncoder.IsValid())
|
if (OpusEncoder.IsValid())
|
||||||
{
|
{
|
||||||
// Opus path: compress then send.
|
// Opus path: compress then send, in sub-blocks.
|
||||||
uint32 CompressedSize = static_cast<uint32>(OpusWorkBuffer.Num());
|
//
|
||||||
OpusEncoder->Encode(PCMData.GetData(), PCMData.Num(),
|
// A whole ElevenLabs message can be several seconds. Two hard limits
|
||||||
OpusWorkBuffer.GetData(), CompressedSize);
|
// force us to split it before each Encode()/RPC:
|
||||||
|
// 1. UE's Opus encoder caps ONE packet at 76 frames (~1.52s) — frames
|
||||||
|
// beyond that are silently dropped at encode (VoiceCodecOpus.cpp).
|
||||||
|
// 2. The client decode buffer only holds ~44 frames (~0.88s) before
|
||||||
|
// the engine decoder aborts with "Decompression buffer too small"
|
||||||
|
// and drops the packet tail (MulticastReceiveAgentAudio_Implementation).
|
||||||
|
// 40 frames (25600 bytes, 0.8s) stays safely under BOTH, so nothing is
|
||||||
|
// truncated on server or client. It also keeps each Unreliable RPC small.
|
||||||
|
static constexpr int32 BytesPerFrame =
|
||||||
|
(PS_AI_ConvAgent_Audio_ElevenLabs::SampleRate / 50)
|
||||||
|
* PS_AI_ConvAgent_Audio_ElevenLabs::Channels
|
||||||
|
* static_cast<int32>(sizeof(int16)); // 640 at 16kHz mono
|
||||||
|
static constexpr int32 MaxOpusChunkBytes = 40 * BytesPerFrame; // 25600 = 0.8s
|
||||||
|
|
||||||
if (CompressedSize > 0)
|
int32 Offset = 0;
|
||||||
|
while (Offset < PCMData.Num())
|
||||||
{
|
{
|
||||||
TArray<uint8> CompressedData;
|
const int32 BlockBytes = FMath::Min(MaxOpusChunkBytes, PCMData.Num() - Offset);
|
||||||
CompressedData.Append(OpusWorkBuffer.GetData(), CompressedSize);
|
|
||||||
|
|
||||||
if (bNetAudioDebug)
|
uint32 CompressedSize = static_cast<uint32>(OpusWorkBuffer.Num());
|
||||||
|
OpusEncoder->Encode(PCMData.GetData() + Offset, BlockBytes,
|
||||||
|
OpusWorkBuffer.GetData(), CompressedSize);
|
||||||
|
|
||||||
|
if (CompressedSize > 0)
|
||||||
{
|
{
|
||||||
const float Ratio = (CompressedSize > 0)
|
TArray<uint8> CompressedData;
|
||||||
? static_cast<float>(PCMData.Num()) / static_cast<float>(CompressedSize)
|
CompressedData.Append(OpusWorkBuffer.GetData(), CompressedSize);
|
||||||
: 0.0f;
|
|
||||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
if (bNetAudioDebug)
|
||||||
TEXT("[NET-SRV] Opus audio chunk: %u bytes sent (raw %d → %.1fx compression)."),
|
{
|
||||||
CompressedSize, PCMData.Num(), Ratio);
|
const float Ratio = static_cast<float>(BlockBytes) / static_cast<float>(CompressedSize);
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
||||||
|
TEXT("[NET-SRV] Opus audio chunk: %u bytes sent (raw %d → %.1fx compression)."),
|
||||||
|
CompressedSize, BlockBytes, Ratio);
|
||||||
|
}
|
||||||
|
|
||||||
|
MulticastReceiveAgentAudio(CompressedData);
|
||||||
}
|
}
|
||||||
|
|
||||||
MulticastReceiveAgentAudio(CompressedData);
|
Offset += BlockBytes;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
@@ -2533,7 +2555,12 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastReceiveAgentAudio_Implementa
|
|||||||
if (OpusDecoder.IsValid())
|
if (OpusDecoder.IsValid())
|
||||||
{
|
{
|
||||||
// Opus path: decode compressed audio.
|
// Opus path: decode compressed audio.
|
||||||
const uint32 MaxDecompressedSize = 16000 * 2;
|
// Buffer must hold a full packet's decoded PCM. The engine decoder aborts
|
||||||
|
// with "Decompression buffer too small to decode voice" (dropping the
|
||||||
|
// packet tail) if this is smaller than the packet needs. The server caps
|
||||||
|
// packets at 40 frames (25600 bytes); 64KB gives ample headroom even up to
|
||||||
|
// the engine's 76-frame (~48640 bytes) maximum, so no packet is truncated.
|
||||||
|
const uint32 MaxDecompressedSize = 64 * 1024;
|
||||||
PCMBuffer.SetNumUninitialized(MaxDecompressedSize);
|
PCMBuffer.SetNumUninitialized(MaxDecompressedSize);
|
||||||
uint32 DecompressedSize = MaxDecompressedSize;
|
uint32 DecompressedSize = MaxDecompressedSize;
|
||||||
OpusDecoder->Decode(AudioData.GetData(), AudioData.Num(),
|
OpusDecoder->Decode(AudioData.GetData(), AudioData.Num(),
|
||||||
|
|||||||
Reference in New Issue
Block a user