Compare commits

..

2 Commits

Author SHA1 Message Date
6d92ec3c26 ConvAgent: add waveform-splice discontinuity debug detector
Diagnostic for the audio "pops": EnqueueAgentAudio compares the last sample of
the previous enqueued block with the first sample of the new one and logs
[AUDIO-DBG] with the delta + a running count when the jump exceeds a threshold
(gated behind ps.ai.ConvAgent.Debug.ElevenLabs; role-tagged). The host's raw-PCM
stream is continuous so its count stays ~0; a client whose count climbs at
message cadence still has a splice problem. Confirms the carry-over fix
(bcb6c44) at runtime.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-01 16:45:16 +02:00
bcb6c44f57 ConvAgent: carry Opus sub-frame remainder across messages (fix pops)
The remaining pops were per-MESSAGE, not per-sub-chunk: the Opus encoder
processes whole 640-byte frames only, so each ElevenLabs message dropped its
<1-frame (<20ms) tail. The lost ~10-16ms per message left a waveform
discontinuity at each message splice -> audible click (2-3 per sentence).
Confirmed by log (message sizes not 640-aligned; no underflow) and the
standalone-vs-client asymmetry: both use the same playback queue, but the host
plays raw PCM (no drop) and is clean while clients play the Opus-encoded stream
and pop.

Fix (server-only path): carry the sub-frame remainder to the next message.
Prepend the previous message's leftover before encoding, emit only whole frames,
and stash the new tail. No samples dropped -> continuous stream -> no splice
click. Leftover reset per turn (new-turn guard + StopAgentAudio).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-01 16:41:25 +02:00
2 changed files with 75 additions and 8 deletions

View File

@@ -1248,12 +1248,26 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray<uint
* static_cast<int32>(sizeof(int16)); // 640 at 16kHz mono * static_cast<int32>(sizeof(int16)); // 640 at 16kHz mono
static constexpr int32 MaxOpusChunkBytes = 40 * BytesPerFrame; // 25600 = 0.8s static constexpr int32 MaxOpusChunkBytes = 40 * BytesPerFrame; // 25600 = 0.8s
// Frame-align the message length (Encode drops any sub-frame remainder // Carry the sub-frame remainder from the PREVIOUS message so no samples
// anyway). A frame-aligned length guarantees every sub-chunk holds >=1 // are dropped at message boundaries. Opus encodes whole 640-byte frames
// whole frame and lets us reliably mark the FINAL sub-chunk (bLast), so // only; dropping each message's <1-frame tail left a ~10-16ms hole and a
// the client reassembles one message into a single enqueue (no per-chunk // waveform discontinuity at every message splice → audible pops on clients
// playback re-arm → no pops). // (the host plays raw PCM directly, so standalone/host never had it).
const int32 UsableBytes = (PCMData.Num() / BytesPerFrame) * BytesPerFrame; // Prepend the leftover, encode only whole frames, stash the new tail.
if (!bAgentSpeaking)
{
// First message of a new turn — don't glue on the previous turn's tail.
AgentAudioEncodeLeftover.Reset();
}
TArray<uint8> ContinuousPCM = MoveTemp(AgentAudioEncodeLeftover);
ContinuousPCM.Append(PCMData);
const int32 UsableBytes = (ContinuousPCM.Num() / BytesPerFrame) * BytesPerFrame;
if (ContinuousPCM.Num() > UsableBytes)
{
AgentAudioEncodeLeftover.Append(
ContinuousPCM.GetData() + UsableBytes, ContinuousPCM.Num() - UsableBytes);
}
int32 Offset = 0; int32 Offset = 0;
while (Offset < UsableBytes) while (Offset < UsableBytes)
@@ -1262,7 +1276,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray<uint
const bool bLast = (Offset + BlockBytes >= UsableBytes); const bool bLast = (Offset + BlockBytes >= UsableBytes);
uint32 CompressedSize = static_cast<uint32>(OpusWorkBuffer.Num()); uint32 CompressedSize = static_cast<uint32>(OpusWorkBuffer.Num());
OpusEncoder->Encode(PCMData.GetData() + Offset, BlockBytes, OpusEncoder->Encode(ContinuousPCM.GetData() + Offset, BlockBytes,
OpusWorkBuffer.GetData(), CompressedSize); OpusWorkBuffer.GetData(), CompressedSize);
if (CompressedSize > 0) if (CompressedSize > 0)
@@ -1718,6 +1732,42 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::OnProceduralUnderflow(
void UPS_AI_ConvAgent_ElevenLabsComponent::EnqueueAgentAudio(const TArray<uint8>& PCMData) void UPS_AI_ConvAgent_ElevenLabsComponent::EnqueueAgentAudio(const TArray<uint8>& PCMData)
{ {
// Debug: flag a waveform discontinuity at the splice between the previous
// enqueued block and this one — the signature of the audio pops. The host's
// raw-PCM stream is continuous so its count should stay ~0; if a client's
// count climbs at message cadence, a splice problem remains.
{
const int32 DbgVal = CVarDebugElevenLabs.GetValueOnGameThread();
const bool bAudioDbg = (DbgVal >= 0) ? (DbgVal > 0) : bDebug;
if (bAudioDbg && PCMData.Num() >= static_cast<int32>(sizeof(int16)))
{
const int16* Samples = reinterpret_cast<const int16*>(PCMData.GetData());
const int32 NumSamples = PCMData.Num() / static_cast<int32>(sizeof(int16));
if (bHasLastEnqueuedSample)
{
const int32 Delta = FMath::Abs(static_cast<int32>(Samples[0]) - static_cast<int32>(LastEnqueuedSample));
static constexpr int32 DiscontinuityThreshold = 3000; // |delta| on the int16 scale
if (Delta > DiscontinuityThreshold)
{
++AudioSpliceDiscontinuityCount;
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
TEXT("[AUDIO-DBG] Role=%d splice discontinuity: |delta|=%d (prev=%d first=%d) count=%d — pop likely here."),
static_cast<int32>(GetOwnerRole()), Delta,
static_cast<int32>(LastEnqueuedSample), static_cast<int32>(Samples[0]),
AudioSpliceDiscontinuityCount);
}
else if (DbgVal >= 2)
{
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
TEXT("[AUDIO-DBG] Role=%d splice ok: |delta|=%d"),
static_cast<int32>(GetOwnerRole()), Delta);
}
}
LastEnqueuedSample = Samples[NumSamples - 1];
bHasLastEnqueuedSample = true;
}
}
{ {
FScopeLock Lock(&AudioQueueLock); FScopeLock Lock(&AudioQueueLock);
AudioQueue.Append(PCMData); AudioQueue.Append(PCMData);
@@ -1892,7 +1942,9 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StopAgentAudio()
bool bWasSpeaking = false; bool bWasSpeaking = false;
double Now = 0.0; double Now = 0.0;
bPreBuffering = false; // Clear pre-buffer state on stop. bPreBuffering = false; // Clear pre-buffer state on stop.
ClientReassemblyPCM.Reset(); // Drop any half-received message so it can't leak into the next turn. ClientReassemblyPCM.Reset(); // Client: drop any half-received message so it can't leak into the next turn.
AgentAudioEncodeLeftover.Reset(); // Server: drop the carried sub-frame tail so it can't glue onto the next turn.
bHasLastEnqueuedSample = false; // Debug: don't count the inter-turn splice as a discontinuity.
{ {
FScopeLock Lock(&AudioQueueLock); FScopeLock Lock(&AudioQueueLock);
AudioQueue.Empty(); AudioQueue.Empty();

View File

@@ -812,6 +812,21 @@ private:
// sub-chunk (bLastSubChunk). Game-thread only. Reset in StopAgentAudio(). // sub-chunk (bLastSubChunk). Game-thread only. Reset in StopAgentAudio().
TArray<uint8> ClientReassemblyPCM; TArray<uint8> ClientReassemblyPCM;
// Server-only: sub-frame (<640 byte) tail carried from the previous agent
// audio message. Opus encodes whole frames only; without carry-over each
// message dropped its <1-frame tail, leaving a ~10-16ms hole + waveform
// discontinuity at every message splice → audible pops on clients. Reset
// per turn (new-turn guard in HandleAudioReceived + StopAgentAudio()).
TArray<uint8> AgentAudioEncodeLeftover;
// Debug: detects waveform discontinuities at enqueue/message splices (the
// signature of the audio "pops"). Tracks the last enqueued sample; a large
// jump to the next block's first sample is flagged and counted. Toggle with
// ps.ai.ConvAgent.Debug.ElevenLabs. Reset in StopAgentAudio().
int16 LastEnqueuedSample = 0;
bool bHasLastEnqueuedSample = false;
int32 AudioSpliceDiscontinuityCount = 0;
// Pre-buffer state: delay playback start to absorb TTS inter-chunk gaps. // Pre-buffer state: delay playback start to absorb TTS inter-chunk gaps.
bool bPreBuffering = false; bool bPreBuffering = false;
double PreBufferStartTime = 0.0; double PreBufferStartTime = 0.0;