ConvAgent: carry Opus sub-frame remainder across messages (fix pops)
The remaining pops were per-MESSAGE, not per-sub-chunk: the Opus encoder processes whole 640-byte frames only, so each ElevenLabs message dropped its <1-frame (<20ms) tail. The lost ~10-16ms per message left a waveform discontinuity at each message splice -> audible click (2-3 per sentence). Confirmed by log (message sizes not 640-aligned; no underflow) and the standalone-vs-client asymmetry: both use the same playback queue, but the host plays raw PCM (no drop) and is clean while clients play the Opus-encoded stream and pop. Fix (server-only path): carry the sub-frame remainder to the next message. Prepend the previous message's leftover before encoding, emit only whole frames, and stash the new tail. No samples dropped -> continuous stream -> no splice click. Leftover reset per turn (new-turn guard + StopAgentAudio). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -1248,12 +1248,26 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray<uint
|
|||||||
* static_cast<int32>(sizeof(int16)); // 640 at 16kHz mono
|
* static_cast<int32>(sizeof(int16)); // 640 at 16kHz mono
|
||||||
static constexpr int32 MaxOpusChunkBytes = 40 * BytesPerFrame; // 25600 = 0.8s
|
static constexpr int32 MaxOpusChunkBytes = 40 * BytesPerFrame; // 25600 = 0.8s
|
||||||
|
|
||||||
// Frame-align the message length (Encode drops any sub-frame remainder
|
// Carry the sub-frame remainder from the PREVIOUS message so no samples
|
||||||
// anyway). A frame-aligned length guarantees every sub-chunk holds >=1
|
// are dropped at message boundaries. Opus encodes whole 640-byte frames
|
||||||
// whole frame and lets us reliably mark the FINAL sub-chunk (bLast), so
|
// only; dropping each message's <1-frame tail left a ~10-16ms hole and a
|
||||||
// the client reassembles one message into a single enqueue (no per-chunk
|
// waveform discontinuity at every message splice → audible pops on clients
|
||||||
// playback re-arm → no pops).
|
// (the host plays raw PCM directly, so standalone/host never had it).
|
||||||
const int32 UsableBytes = (PCMData.Num() / BytesPerFrame) * BytesPerFrame;
|
// Prepend the leftover, encode only whole frames, stash the new tail.
|
||||||
|
if (!bAgentSpeaking)
|
||||||
|
{
|
||||||
|
// First message of a new turn — don't glue on the previous turn's tail.
|
||||||
|
AgentAudioEncodeLeftover.Reset();
|
||||||
|
}
|
||||||
|
TArray<uint8> ContinuousPCM = MoveTemp(AgentAudioEncodeLeftover);
|
||||||
|
ContinuousPCM.Append(PCMData);
|
||||||
|
|
||||||
|
const int32 UsableBytes = (ContinuousPCM.Num() / BytesPerFrame) * BytesPerFrame;
|
||||||
|
if (ContinuousPCM.Num() > UsableBytes)
|
||||||
|
{
|
||||||
|
AgentAudioEncodeLeftover.Append(
|
||||||
|
ContinuousPCM.GetData() + UsableBytes, ContinuousPCM.Num() - UsableBytes);
|
||||||
|
}
|
||||||
|
|
||||||
int32 Offset = 0;
|
int32 Offset = 0;
|
||||||
while (Offset < UsableBytes)
|
while (Offset < UsableBytes)
|
||||||
@@ -1262,7 +1276,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray<uint
|
|||||||
const bool bLast = (Offset + BlockBytes >= UsableBytes);
|
const bool bLast = (Offset + BlockBytes >= UsableBytes);
|
||||||
|
|
||||||
uint32 CompressedSize = static_cast<uint32>(OpusWorkBuffer.Num());
|
uint32 CompressedSize = static_cast<uint32>(OpusWorkBuffer.Num());
|
||||||
OpusEncoder->Encode(PCMData.GetData() + Offset, BlockBytes,
|
OpusEncoder->Encode(ContinuousPCM.GetData() + Offset, BlockBytes,
|
||||||
OpusWorkBuffer.GetData(), CompressedSize);
|
OpusWorkBuffer.GetData(), CompressedSize);
|
||||||
|
|
||||||
if (CompressedSize > 0)
|
if (CompressedSize > 0)
|
||||||
@@ -1892,7 +1906,8 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StopAgentAudio()
|
|||||||
bool bWasSpeaking = false;
|
bool bWasSpeaking = false;
|
||||||
double Now = 0.0;
|
double Now = 0.0;
|
||||||
bPreBuffering = false; // Clear pre-buffer state on stop.
|
bPreBuffering = false; // Clear pre-buffer state on stop.
|
||||||
ClientReassemblyPCM.Reset(); // Drop any half-received message so it can't leak into the next turn.
|
ClientReassemblyPCM.Reset(); // Client: drop any half-received message so it can't leak into the next turn.
|
||||||
|
AgentAudioEncodeLeftover.Reset(); // Server: drop the carried sub-frame tail so it can't glue onto the next turn.
|
||||||
{
|
{
|
||||||
FScopeLock Lock(&AudioQueueLock);
|
FScopeLock Lock(&AudioQueueLock);
|
||||||
AudioQueue.Empty();
|
AudioQueue.Empty();
|
||||||
|
|||||||
@@ -812,6 +812,13 @@ private:
|
|||||||
// sub-chunk (bLastSubChunk). Game-thread only. Reset in StopAgentAudio().
|
// sub-chunk (bLastSubChunk). Game-thread only. Reset in StopAgentAudio().
|
||||||
TArray<uint8> ClientReassemblyPCM;
|
TArray<uint8> ClientReassemblyPCM;
|
||||||
|
|
||||||
|
// Server-only: sub-frame (<640 byte) tail carried from the previous agent
|
||||||
|
// audio message. Opus encodes whole frames only; without carry-over each
|
||||||
|
// message dropped its <1-frame tail, leaving a ~10-16ms hole + waveform
|
||||||
|
// discontinuity at every message splice → audible pops on clients. Reset
|
||||||
|
// per turn (new-turn guard in HandleAudioReceived + StopAgentAudio()).
|
||||||
|
TArray<uint8> AgentAudioEncodeLeftover;
|
||||||
|
|
||||||
// Pre-buffer state: delay playback start to absorb TTS inter-chunk gaps.
|
// Pre-buffer state: delay playback start to absorb TTS inter-chunk gaps.
|
||||||
bool bPreBuffering = false;
|
bool bPreBuffering = false;
|
||||||
double PreBufferStartTime = 0.0;
|
double PreBufferStartTime = 0.0;
|
||||||
|
|||||||
Reference in New Issue
Block a user