|
|
|
|
@@ -1248,12 +1248,26 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray<uint
|
|
|
|
|
* static_cast<int32>(sizeof(int16)); // 640 at 16kHz mono
|
|
|
|
|
static constexpr int32 MaxOpusChunkBytes = 40 * BytesPerFrame; // 25600 = 0.8s
|
|
|
|
|
|
|
|
|
|
// Frame-align the message length (Encode drops any sub-frame remainder
|
|
|
|
|
// anyway). A frame-aligned length guarantees every sub-chunk holds >=1
|
|
|
|
|
// whole frame and lets us reliably mark the FINAL sub-chunk (bLast), so
|
|
|
|
|
// the client reassembles one message into a single enqueue (no per-chunk
|
|
|
|
|
// playback re-arm → no pops).
|
|
|
|
|
const int32 UsableBytes = (PCMData.Num() / BytesPerFrame) * BytesPerFrame;
|
|
|
|
|
// Carry the sub-frame remainder from the PREVIOUS message so no samples
|
|
|
|
|
// are dropped at message boundaries. Opus encodes whole 640-byte frames
|
|
|
|
|
// only; dropping each message's <1-frame tail left a ~10-16ms hole and a
|
|
|
|
|
// waveform discontinuity at every message splice → audible pops on clients
|
|
|
|
|
// (the host plays raw PCM directly, so standalone/host never had it).
|
|
|
|
|
// Prepend the leftover, encode only whole frames, stash the new tail.
|
|
|
|
|
if (!bAgentSpeaking)
|
|
|
|
|
{
|
|
|
|
|
// First message of a new turn — don't glue on the previous turn's tail.
|
|
|
|
|
AgentAudioEncodeLeftover.Reset();
|
|
|
|
|
}
|
|
|
|
|
TArray<uint8> ContinuousPCM = MoveTemp(AgentAudioEncodeLeftover);
|
|
|
|
|
ContinuousPCM.Append(PCMData);
|
|
|
|
|
|
|
|
|
|
const int32 UsableBytes = (ContinuousPCM.Num() / BytesPerFrame) * BytesPerFrame;
|
|
|
|
|
if (ContinuousPCM.Num() > UsableBytes)
|
|
|
|
|
{
|
|
|
|
|
AgentAudioEncodeLeftover.Append(
|
|
|
|
|
ContinuousPCM.GetData() + UsableBytes, ContinuousPCM.Num() - UsableBytes);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int32 Offset = 0;
|
|
|
|
|
while (Offset < UsableBytes)
|
|
|
|
|
@@ -1262,7 +1276,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray<uint
|
|
|
|
|
const bool bLast = (Offset + BlockBytes >= UsableBytes);
|
|
|
|
|
|
|
|
|
|
uint32 CompressedSize = static_cast<uint32>(OpusWorkBuffer.Num());
|
|
|
|
|
OpusEncoder->Encode(PCMData.GetData() + Offset, BlockBytes,
|
|
|
|
|
OpusEncoder->Encode(ContinuousPCM.GetData() + Offset, BlockBytes,
|
|
|
|
|
OpusWorkBuffer.GetData(), CompressedSize);
|
|
|
|
|
|
|
|
|
|
if (CompressedSize > 0)
|
|
|
|
|
@@ -1718,6 +1732,42 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::OnProceduralUnderflow(
|
|
|
|
|
|
|
|
|
|
void UPS_AI_ConvAgent_ElevenLabsComponent::EnqueueAgentAudio(const TArray<uint8>& PCMData)
|
|
|
|
|
{
|
|
|
|
|
// Debug: flag a waveform discontinuity at the splice between the previous
|
|
|
|
|
// enqueued block and this one — the signature of the audio pops. The host's
|
|
|
|
|
// raw-PCM stream is continuous so its count should stay ~0; if a client's
|
|
|
|
|
// count climbs at message cadence, a splice problem remains.
|
|
|
|
|
{
|
|
|
|
|
const int32 DbgVal = CVarDebugElevenLabs.GetValueOnGameThread();
|
|
|
|
|
const bool bAudioDbg = (DbgVal >= 0) ? (DbgVal > 0) : bDebug;
|
|
|
|
|
if (bAudioDbg && PCMData.Num() >= static_cast<int32>(sizeof(int16)))
|
|
|
|
|
{
|
|
|
|
|
const int16* Samples = reinterpret_cast<const int16*>(PCMData.GetData());
|
|
|
|
|
const int32 NumSamples = PCMData.Num() / static_cast<int32>(sizeof(int16));
|
|
|
|
|
if (bHasLastEnqueuedSample)
|
|
|
|
|
{
|
|
|
|
|
const int32 Delta = FMath::Abs(static_cast<int32>(Samples[0]) - static_cast<int32>(LastEnqueuedSample));
|
|
|
|
|
static constexpr int32 DiscontinuityThreshold = 3000; // |delta| on the int16 scale
|
|
|
|
|
if (Delta > DiscontinuityThreshold)
|
|
|
|
|
{
|
|
|
|
|
++AudioSpliceDiscontinuityCount;
|
|
|
|
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
|
|
|
|
|
TEXT("[AUDIO-DBG] Role=%d splice discontinuity: |delta|=%d (prev=%d first=%d) count=%d — pop likely here."),
|
|
|
|
|
static_cast<int32>(GetOwnerRole()), Delta,
|
|
|
|
|
static_cast<int32>(LastEnqueuedSample), static_cast<int32>(Samples[0]),
|
|
|
|
|
AudioSpliceDiscontinuityCount);
|
|
|
|
|
}
|
|
|
|
|
else if (DbgVal >= 2)
|
|
|
|
|
{
|
|
|
|
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
|
|
|
|
TEXT("[AUDIO-DBG] Role=%d splice ok: |delta|=%d"),
|
|
|
|
|
static_cast<int32>(GetOwnerRole()), Delta);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
LastEnqueuedSample = Samples[NumSamples - 1];
|
|
|
|
|
bHasLastEnqueuedSample = true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
{
|
|
|
|
|
FScopeLock Lock(&AudioQueueLock);
|
|
|
|
|
AudioQueue.Append(PCMData);
|
|
|
|
|
@@ -1892,7 +1942,9 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StopAgentAudio()
|
|
|
|
|
bool bWasSpeaking = false;
|
|
|
|
|
double Now = 0.0;
|
|
|
|
|
bPreBuffering = false; // Clear pre-buffer state on stop.
|
|
|
|
|
ClientReassemblyPCM.Reset(); // Drop any half-received message so it can't leak into the next turn.
|
|
|
|
|
ClientReassemblyPCM.Reset(); // Client: drop any half-received message so it can't leak into the next turn.
|
|
|
|
|
AgentAudioEncodeLeftover.Reset(); // Server: drop the carried sub-frame tail so it can't glue onto the next turn.
|
|
|
|
|
bHasLastEnqueuedSample = false; // Debug: don't count the inter-turn splice as a discontinuity.
|
|
|
|
|
{
|
|
|
|
|
FScopeLock Lock(&AudioQueueLock);
|
|
|
|
|
AudioQueue.Empty();
|
|
|
|
|
|