From bcb6c44f5715cee377229ce2d159fb36ac5d553c Mon Sep 17 00:00:00 2001 From: "j.foucher" Date: Wed, 1 Jul 2026 16:41:25 +0200 Subject: [PATCH] ConvAgent: carry Opus sub-frame remainder across messages (fix pops) The remaining pops were per-MESSAGE, not per-sub-chunk: the Opus encoder processes whole 640-byte frames only, so each ElevenLabs message dropped its <1-frame (<20ms) tail. The lost ~10-16ms per message left a waveform discontinuity at each message splice -> audible click (2-3 per sentence). Confirmed by log (message sizes not 640-aligned; no underflow) and the standalone-vs-client asymmetry: both use the same playback queue, but the host plays raw PCM (no drop) and is clean while clients play the Opus-encoded stream and pop. Fix (server-only path): carry the sub-frame remainder to the next message. Prepend the previous message's leftover before encoding, emit only whole frames, and stash the new tail. No samples dropped -> continuous stream -> no splice click. Leftover reset per turn (new-turn guard + StopAgentAudio). Co-Authored-By: Claude Opus 4.8 --- .../PS_AI_ConvAgent_ElevenLabsComponent.cpp | 31 ++++++++++++++----- .../PS_AI_ConvAgent_ElevenLabsComponent.h | 7 +++++ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp index 2c55973..f06aeb6 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp @@ -1248,12 +1248,26 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray(sizeof(int16)); // 640 at 16kHz mono static constexpr int32 MaxOpusChunkBytes = 40 * BytesPerFrame; // 25600 = 0.8s - // Frame-align the message length (Encode drops any sub-frame remainder - // anyway). A frame-aligned length guarantees every sub-chunk holds >=1 - // whole frame and lets us reliably mark the FINAL sub-chunk (bLast), so - // the client reassembles one message into a single enqueue (no per-chunk - // playback re-arm → no pops). - const int32 UsableBytes = (PCMData.Num() / BytesPerFrame) * BytesPerFrame; + // Carry the sub-frame remainder from the PREVIOUS message so no samples + // are dropped at message boundaries. Opus encodes whole 640-byte frames + // only; dropping each message's <1-frame tail left a ~10-16ms hole and a + // waveform discontinuity at every message splice → audible pops on clients + // (the host plays raw PCM directly, so standalone/host never had it). + // Prepend the leftover, encode only whole frames, stash the new tail. + if (!bAgentSpeaking) + { + // First message of a new turn — don't glue on the previous turn's tail. + AgentAudioEncodeLeftover.Reset(); + } + TArray ContinuousPCM = MoveTemp(AgentAudioEncodeLeftover); + ContinuousPCM.Append(PCMData); + + const int32 UsableBytes = (ContinuousPCM.Num() / BytesPerFrame) * BytesPerFrame; + if (ContinuousPCM.Num() > UsableBytes) + { + AgentAudioEncodeLeftover.Append( + ContinuousPCM.GetData() + UsableBytes, ContinuousPCM.Num() - UsableBytes); + } int32 Offset = 0; while (Offset < UsableBytes) @@ -1262,7 +1276,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray= UsableBytes); uint32 CompressedSize = static_cast(OpusWorkBuffer.Num()); - OpusEncoder->Encode(PCMData.GetData() + Offset, BlockBytes, + OpusEncoder->Encode(ContinuousPCM.GetData() + Offset, BlockBytes, OpusWorkBuffer.GetData(), CompressedSize); if (CompressedSize > 0) @@ -1892,7 +1906,8 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StopAgentAudio() bool bWasSpeaking = false; double Now = 0.0; bPreBuffering = false; // Clear pre-buffer state on stop. - ClientReassemblyPCM.Reset(); // Drop any half-received message so it can't leak into the next turn. + ClientReassemblyPCM.Reset(); // Client: drop any half-received message so it can't leak into the next turn. + AgentAudioEncodeLeftover.Reset(); // Server: drop the carried sub-frame tail so it can't glue onto the next turn. { FScopeLock Lock(&AudioQueueLock); AudioQueue.Empty(); diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h index b241f61..b159b6e 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h @@ -812,6 +812,13 @@ private: // sub-chunk (bLastSubChunk). Game-thread only. Reset in StopAgentAudio(). TArray ClientReassemblyPCM; + // Server-only: sub-frame (<640 byte) tail carried from the previous agent + // audio message. Opus encodes whole frames only; without carry-over each + // message dropped its <1-frame tail, leaving a ~10-16ms hole + waveform + // discontinuity at every message splice → audible pops on clients. Reset + // per turn (new-turn guard in HandleAudioReceived + StopAgentAudio()). + TArray AgentAudioEncodeLeftover; + // Pre-buffer state: delay playback start to absorb TTS inter-chunk gaps. bool bPreBuffering = false; double PreBufferStartTime = 0.0;