j.foucher/PS_AI_Agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix network audio: raw PCM fallback when Opus codecs unavailable

Browse Source 
UE5 5.5's FVoiceModule returns NULL encoder/decoder, breaking
Opus-based audio replication. This adds a transparent fallback:
- Server sends raw PCM when OpusEncoder is null (~32KB/s, fine for LAN)
- Client accepts raw PCM when OpusDecoder is null
- Changed MulticastReceiveAgentAudio from Unreliable to Reliable
  to handle larger uncompressed payloads without packet loss
- Added OnlineSubsystemNull config for future Opus compatibility
- Removed premature bAgentSpeaking=true from MulticastAgentStartedSpeaking
  to fix race condition with audio initialization

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
j.foucher 2026-03-02 14:58:21 +01:00
parent 5e1c50edf8
commit 6cac56fa06
3 changed files with 43 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Unreal/PS_AI_Agent/Config/DefaultEngine.ini
View File

@ -153,6 +153,12 @@ FontDPI=72
+EnumRedirects=(OldName="EPS_AI_Agent_Emotion", NewName="EPS_AI_ConvAgent_Emotion")
+EnumRedirects=(OldName="EPS_AI_Agent_EmotionIntensity", NewName="EPS_AI_ConvAgent_EmotionIntensity")
[OnlineSubsystem]
DefaultPlatformService=Null
[OnlineSubsystemNull]
bEnabled=true
[/Script/AndroidFileServerEditor.AndroidFileServerRuntimeSettings]
bEnablePlugin=True
bAllowNetworkConnection=True

86
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp
View File

@ -637,46 +637,35 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray<uint
QueueBefore, (static_cast<float>(QueueBefore) / 16000.0f) * 1000.0f);
}
// Network: Opus-compress and broadcast to all clients before local playback.
if (GetOwnerRole() != ROLE_Authority || !OpusEncoder.IsValid())
// Network: broadcast audio to all clients.
if (GetOwnerRole() == ROLE_Authority)
{
static bool bWarnedOnce = false;
if (!bWarnedOnce)
{
bWarnedOnce = true;
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
TEXT("[NET-SRV] Cannot multicast audio! Role=%d (need %d=Authority), OpusEncoder=%s, FVoiceModule=%s"),
static_cast<int32>(GetOwnerRole()), static_cast<int32>(ROLE_Authority),
OpusEncoder.IsValid() ? TEXT("VALID") : TEXT("NULL"),
FVoiceModule::IsAvailable() ? TEXT("available") : TEXT("UNAVAILABLE"));
}
}
if (GetOwnerRole() == ROLE_Authority && OpusEncoder.IsValid())
if (OpusEncoder.IsValid())
{
// Opus path: compress then send.
uint32 CompressedSize = static_cast<uint32>(OpusWorkBuffer.Num());
int32 Remainder = OpusEncoder->Encode(PCMData.GetData(), PCMData.Num(),
OpusEncoder->Encode(PCMData.GetData(), PCMData.Num(),
OpusWorkBuffer.GetData(), CompressedSize);
if (CompressedSize > 0)
{
TArray<uint8> CompressedData;
CompressedData.Append(OpusWorkBuffer.GetData(), CompressedSize);
if (bDebug)
{
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
TEXT("[NET-SRV] Multicasting Opus audio: %d bytes PCM → %d bytes Opus (%.1f:1 ratio)"),
PCMData.Num(), CompressedSize,
PCMData.Num() > 0 ? static_cast<float>(PCMData.Num()) / CompressedSize : 0.f);
}
MulticastReceiveAgentAudio(CompressedData);
}
}
else
{
// Fallback: send raw PCM (no compression). ~32 KB/s at 16kHz 16-bit mono.
// Fine for LAN; revisit with proper Opus if internet play is needed.
static bool bWarnedOnce = false;
if (!bWarnedOnce)
{
bWarnedOnce = true;
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
TEXT("[NET-SRV] Opus encode produced 0 bytes from %d bytes PCM — audio not sent."),
PCMData.Num());
TEXT("[NET-SRV] Opus encoder unavailable — sending raw PCM (no compression)."));
}
MulticastReceiveAgentAudio(PCMData);
}
}
@ -1346,57 +1335,36 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ClientConversationFailed_Implementati
// Network: Multicast RPCs
// ─────────────────────────────────────────────────────────────────────────────
void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastReceiveAgentAudio_Implementation(
const TArray<uint8>& OpusData)
const TArray<uint8>& AudioData)
{
// Server already handled playback in HandleAudioReceived.
if (GetOwnerRole() == ROLE_Authority) return;
if (!OpusDecoder.IsValid())
{
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
TEXT("[NET] MulticastReceiveAgentAudio: OpusDecoder is INVALID — audio dropped. FVoiceModule available: %s"),
FVoiceModule::IsAvailable() ? TEXT("YES") : TEXT("NO"));
return;
}
// LOD: skip audio if too far (unless this client is the speaker).
const float Dist = GetDistanceToLocalPlayer();
const bool bIsSpeaker = IsLocalPlayerConversating();
if (!bIsSpeaker && AudioLODCullDistance > 0.f && Dist > AudioLODCullDistance)
{
if (bDebug && DebugVerbosity >= 2)
{
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
TEXT("[NET] MulticastReceiveAgentAudio: LOD culled (dist=%.0f > cull=%.0f)"),
Dist, AudioLODCullDistance);
}
return;
}
// Decode Opus → PCM.
const uint32 MaxDecompressedSize = 16000 * 2; // 1 second of 16kHz 16-bit mono
TArray<uint8> PCMBuffer;
if (OpusDecoder.IsValid())
{
// Opus path: decode compressed audio.
const uint32 MaxDecompressedSize = 16000 * 2;
PCMBuffer.SetNumUninitialized(MaxDecompressedSize);
uint32 DecompressedSize = MaxDecompressedSize;
OpusDecoder->Decode(OpusData.GetData(), OpusData.Num(),
OpusDecoder->Decode(AudioData.GetData(), AudioData.Num(),
PCMBuffer.GetData(), DecompressedSize);
if (DecompressedSize == 0)
{
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
TEXT("[NET] MulticastReceiveAgentAudio: Opus decode failed (0 bytes output from %d bytes input)"),
OpusData.Num());
return;
}
if (DecompressedSize == 0) return;
PCMBuffer.SetNum(DecompressedSize);
if (bDebug)
}
else
{
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
TEXT("[NET] MulticastReceiveAgentAudio: decoded %d bytes Opus → %d bytes PCM | bAgentSpeaking=%s | AudioComp playing=%s"),
OpusData.Num(), DecompressedSize,
bAgentSpeaking ? TEXT("true") : TEXT("false"),
(AudioPlaybackComponent && AudioPlaybackComponent->IsPlaying()) ? TEXT("true") : TEXT("false"));
// Fallback: data is already raw PCM (sent uncompressed by server).
PCMBuffer = AudioData;
}
// Local playback.

6
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h
View File

@ -339,9 +339,9 @@ public:
UFUNCTION(Server, Reliable)
void ServerRequestInterrupt();
/** Broadcast Opus-compressed agent audio to all clients. */
UFUNCTION(NetMulticast, Unreliable)
void MulticastReceiveAgentAudio(const TArray<uint8>& OpusData);
/** Broadcast agent audio to all clients (Opus-compressed or raw PCM fallback). */
UFUNCTION(NetMulticast, Reliable)
void MulticastReceiveAgentAudio(const TArray<uint8>& AudioData);
/** Notify all clients that the agent started speaking (first audio chunk). */
UFUNCTION(NetMulticast, Reliable)