diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp index a8b5590..2c55973 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp @@ -1248,10 +1248,18 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray(sizeof(int16)); // 640 at 16kHz mono static constexpr int32 MaxOpusChunkBytes = 40 * BytesPerFrame; // 25600 = 0.8s + // Frame-align the message length (Encode drops any sub-frame remainder + // anyway). A frame-aligned length guarantees every sub-chunk holds >=1 + // whole frame and lets us reliably mark the FINAL sub-chunk (bLast), so + // the client reassembles one message into a single enqueue (no per-chunk + // playback re-arm → no pops). + const int32 UsableBytes = (PCMData.Num() / BytesPerFrame) * BytesPerFrame; + int32 Offset = 0; - while (Offset < PCMData.Num()) + while (Offset < UsableBytes) { - const int32 BlockBytes = FMath::Min(MaxOpusChunkBytes, PCMData.Num() - Offset); + const int32 BlockBytes = FMath::Min(MaxOpusChunkBytes, UsableBytes - Offset); + const bool bLast = (Offset + BlockBytes >= UsableBytes); uint32 CompressedSize = static_cast(OpusWorkBuffer.Num()); OpusEncoder->Encode(PCMData.GetData() + Offset, BlockBytes, @@ -1266,11 +1274,11 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray(BlockBytes) / static_cast(CompressedSize); UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log, - TEXT("[NET-SRV] Opus audio chunk: %u bytes sent (raw %d → %.1fx compression)."), - CompressedSize, BlockBytes, Ratio); + TEXT("[NET-SRV] Opus audio chunk: %u bytes sent (raw %d → %.1fx compression, last=%d)."), + CompressedSize, BlockBytes, Ratio, bLast ? 1 : 0); } - MulticastReceiveAgentAudio(CompressedData); + MulticastReceiveAgentAudio(CompressedData, bLast); } Offset += BlockBytes; @@ -1293,30 +1301,32 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray= MaxFallbackBytes) - { - UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning, - TEXT("[NET-SRV] Raw-PCM fallback capped at %d bytes — dropping %d-byte tail " - "to protect the reliable buffer."), - MaxFallbackBytes, PCMData.Num() - Offset); - break; - } - const int32 ChunkSize = FMath::Min(MaxChunkBytes, PCMData.Num() - Offset); + const int32 SendableBytes = FMath::Min(PCMData.Num(), MaxFallbackBytes); + if (PCMData.Num() > MaxFallbackBytes) + { + UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning, + TEXT("[NET-SRV] Raw-PCM fallback capped at %d bytes — dropping %d-byte tail " + "to protect the reliable buffer."), + MaxFallbackBytes, PCMData.Num() - MaxFallbackBytes); + } + + int32 Offset = 0; + while (Offset < SendableBytes) + { + const int32 ChunkSize = FMath::Min(MaxChunkBytes, SendableBytes - Offset); + const bool bLast = (Offset + ChunkSize >= SendableBytes); TArray Chunk; Chunk.Append(PCMData.GetData() + Offset, ChunkSize); if (bNetAudioDebug) { UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning, - TEXT("[NET-SRV] RAW PCM chunk: %d bytes sent (NO compression — Opus disabled)."), - ChunkSize); + TEXT("[NET-SRV] RAW PCM chunk: %d bytes sent (NO compression — Opus disabled, last=%d)."), + ChunkSize, bLast ? 1 : 0); } - MulticastReceiveAgentAudio(Chunk); + MulticastReceiveAgentAudio(Chunk, bLast); Offset += ChunkSize; } } @@ -1882,6 +1892,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StopAgentAudio() bool bWasSpeaking = false; double Now = 0.0; bPreBuffering = false; // Clear pre-buffer state on stop. + ClientReassemblyPCM.Reset(); // Drop any half-received message so it can't leak into the next turn. { FScopeLock Lock(&AudioQueueLock); AudioQueue.Empty(); @@ -2546,7 +2557,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ClientConversationFailed_Implementati // Network: Multicast RPCs // ───────────────────────────────────────────────────────────────────────────── void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastReceiveAgentAudio_Implementation( - const TArray& AudioData) + const TArray& AudioData, bool bLastSubChunk) { // Server already handled playback in HandleAudioReceived. if (GetOwnerRole() == ROLE_Authority) return; @@ -2574,8 +2585,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastReceiveAgentAudio_Implementa uint32 DecompressedSize = MaxDecompressedSize; OpusDecoder->Decode(AudioData.GetData(), AudioData.Num(), PCMBuffer.GetData(), DecompressedSize); - if (DecompressedSize == 0) return; - PCMBuffer.SetNum(DecompressedSize); + PCMBuffer.SetNum(DecompressedSize); // may be 0 — handled by reassembly below } else { @@ -2583,13 +2593,33 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastReceiveAgentAudio_Implementa PCMBuffer = AudioData; } - // Local playback. - EnqueueAgentAudio(PCMBuffer); - - // Feed lip-sync (within LOD or speaker). - if (bIsSpeaker || LipSyncLODDistance <= 0.f || Dist <= LipSyncLODDistance) + // Reassemble one ElevenLabs message from its N sub-chunks, then enqueue ONCE. + // The server splits a message into sub-chunks (to respect the Opus 76-frame / + // client decode-buffer limits), sent back-to-back in the same tick. Enqueuing + // each sub-chunk separately re-ran the playback state machine → a pop at each + // ~0.8s seam. Accumulating and enqueuing once per message plays it contiguously. + if (PCMBuffer.Num() > 0) { - OnAgentAudioData.Broadcast(PCMBuffer); + ClientReassemblyPCM.Append(PCMBuffer); + } + + if (!bLastSubChunk) + { + return; // wait for the remaining sub-chunks of this message + } + + if (ClientReassemblyPCM.Num() > 0) + { + // Local playback (one contiguous block per message). + EnqueueAgentAudio(ClientReassemblyPCM); + + // Feed lip-sync (within LOD or speaker). + if (bIsSpeaker || LipSyncLODDistance <= 0.f || Dist <= LipSyncLODDistance) + { + OnAgentAudioData.Broadcast(ClientReassemblyPCM); + } + + ClientReassemblyPCM.Reset(); } } diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h index 579db08..b241f61 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h @@ -494,9 +494,12 @@ public: * sub-chunk is ~2 KB (2-3 partial bunches, below the 8-bunch reliable- * promotion threshold) at ~1.25 RPC/s, so in-flight bunches stay in single * digits vs the 512-bunch buffer. Sub-chunking in HandleAudioReceived is what - * keeps packets this small — it must stay. */ + * keeps packets this small — it must stay. + * bLastSubChunk marks the final sub-chunk of one ElevenLabs message so the + * client can reassemble all sub-chunks and enqueue ONCE per message (enqueuing + * each sub-chunk separately re-ran the playback state machine → pops). */ UFUNCTION(NetMulticast, Reliable) - void MulticastReceiveAgentAudio(const TArray& AudioData); + void MulticastReceiveAgentAudio(const TArray& AudioData, bool bLastSubChunk); /** Notify all clients that the agent started speaking (first audio chunk). */ UFUNCTION(NetMulticast, Reliable) @@ -804,6 +807,11 @@ private: int32 AudioQueueReadOffset = 0; FCriticalSection AudioQueueLock; + // Client-only: accumulates decoded PCM across the N sub-chunks of one + // ElevenLabs message; flushed to EnqueueAgentAudio() once, on the last + // sub-chunk (bLastSubChunk). Game-thread only. Reset in StopAgentAudio(). + TArray ClientReassemblyPCM; + // Pre-buffer state: delay playback start to absorb TTS inter-chunk gaps. bool bPreBuffering = false; double PreBufferStartTime = 0.0;