ConvAgent: cut BOTH agents' speech when the player interrupts the dialogue

On a player barge-in, the engaged agent stops (its mic feeds the player's voice to ElevenLabs, which interrupts it), but the witness agent never hears the player and kept finishing its line (e.g. '...I'll give you the cash...') mid-scene. InterruptForPlayer now calls InterruptAgent() on both CompA and CompB. Conversations stay open; the witness still receives the contextual_updates.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_AgentDialogue.cpp

@@ -255,6 +255,14 @@ void APS_AI_ConvAgent_AgentDialogue::InterruptForPlayer(EFloor Engaged)
 	{
 		W->GetTimerManager().ClearTimer(TurnTimer);
 	}
+
+	// The player barged in — cut BOTH agents' current speech now. The engaged agent
+	// is already interrupted by the player's mic (ElevenLabs), but the WITNESS agent
+	// never hears the player, so without this it keeps finishing its dealer<->client
+	// line ("...I'll give you the cash...") while the player is talking. Interrupting
+	// leaves the conversations open; the witness still receives the contextual_updates.
+	if (CompA) { CompA->InterruptAgent(); }
+	if (CompB) { CompB->InterruptAgent(); }
 	// Keep the agent bindings: in floor mode the text/transcript handlers RELAY to the
 	// witness instead of forwarding (their dialogue logic no-ops while !bRunning).
 

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp

@@ -2525,6 +2525,38 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ServerSendMicAudioFromPlayer(
 {
 	if (!WebSocketProxy || !WebSocketProxy->IsConnected()) return;
 
+	// Server-side voice-onset. This runs on the SERVER (only the server has a live
+	// WebSocket — clients returned above), and the player's mic — the host's OR a
+	// remote client's, relayed here — always passes through this function. Broadcasting
+	// OnUserVoiceDetected here is what makes the agent-to-agent dialogue interrupt work
+	// on a DEDICATED server (no host player). Debounced (shares LastUserVoiceOnset with
+	// the FeedExternalAudio path, so the host doesn't double-fire).
+	{
+		const int32 OnsetSamples = PCMBytes.Num() / (int32)sizeof(int16);
+		if (OnsetSamples > 0)
+		{
+			const int16* OS = reinterpret_cast<const int16*>(PCMBytes.GetData());
+			double OnsetSumSq = 0.0;
+			for (int32 i = 0; i < OnsetSamples; ++i) { const double v = OS[i] / 32768.0; OnsetSumSq += v * v; }
+			const double OnsetRms = FMath::Sqrt(OnsetSumSq / OnsetSamples);
+			if (OnsetRms >= 0.02)
+			{
+				const double NowSec = FPlatformTime::Seconds();
+				if (NowSec - LastUserVoiceOnset > 0.5)
+				{
+					LastUserVoiceOnset = NowSec;
+					OnUserVoiceDetected.Broadcast();
+				}
+			}
+		}
+	}
+
+	// Replay capture: expose every relayed player mic chunk (with its pawn) so an
+	// external recorder (PS_AI_ConvReplay) can save player speech. Server-only — clients
+	// have no live WebSocket and returned above. Fired before arbitration so audio from
+	// every player is captured, even a non-active speaker.
+	OnPlayerAudioCaptured.Broadcast(SpeakerPawn, PCMBytes);
+
 	// Standalone / single-player: bypass speaker arbitration entirely.
 	// There's only one player — no need for Contains check or speaker switching.
 	if (NetConnectedPawns.Num() <= 1)
@@ -2735,6 +2767,45 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastReceiveAgentAudio_Implementa
 	}
 }
 
+// ─────────────────────────────────────────────────────────────────────────────
+// Replay support (PS_AI_ConvReplay bridge)
+// ─────────────────────────────────────────────────────────────────────────────
+void UPS_AI_ConvAgent_ElevenLabsComponent::PlayExternalAudioPCM(const TArray<uint8>& PCM16)
+{
+	if (PCM16.Num() == 0) return;
+
+	// Same tail as MulticastReceiveAgentAudio_Implementation, minus the LOD culls:
+	// enqueue for playback (drives body expression + start/stop via bAgentSpeaking) and
+	// broadcast the raw PCM so the LipSync component animates visemes.
+	EnqueueAgentAudio(PCM16);
+
+	// During replay there is no live agent_response message, so the normal
+	// silence-detection stop path (TickComponent) would otherwise wait for the long
+	// hard-timeout. Mark the response as received so OnAgentStoppedSpeaking fires
+	// shortly after the buffer drains and body expression returns to idle.
+	// (Set AFTER EnqueueAgentAudio, which resets this flag on the first chunk.)
+	bAgentResponseReceived = true;
+
+	OnAgentAudioData.Broadcast(PCM16);
+}
+
+void UPS_AI_ConvAgent_ElevenLabsComponent::StopExternalAudio()
+{
+	StopAgentAudio();
+}
+
+void UPS_AI_ConvAgent_ElevenLabsComponent::TriggerReplayedAction(const FString& ActionName)
+{
+	PendingActionName = ActionName;
+	OnAgentActionRequested.Broadcast(ActionName);
+}
+
+void UPS_AI_ConvAgent_ElevenLabsComponent::TriggerReplayedClientToolCall(
+	const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall)
+{
+	OnAgentClientToolCall.Broadcast(ToolCall);
+}
+
 void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastAgentStartedSpeaking_Implementation()
 {
 	if (GetOwnerRole() == ROLE_Authority) return;

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp

@@ -21,6 +21,17 @@ static TAutoConsoleVariable<int32> CVarDebugInteraction(
 	TEXT("Debug HUD for Interaction. -1=use property, 0=off, 1-3=verbosity."),
 	ECVF_Default);
 
+// Convert normalized float mic samples to int16 little-endian bytes (16kHz mono).
+static void ConvReplay_FloatToInt16Bytes(const TArray<float>& In, TArray<uint8>& Out)
+{
+	Out.SetNumUninitialized(In.Num() * static_cast<int32>(sizeof(int16)));
+	int16* D = reinterpret_cast<int16*>(Out.GetData());
+	for (int32 i = 0; i < In.Num(); ++i)
+	{
+		D[i] = static_cast<int16>(FMath::Clamp<int32>(FMath::RoundToInt(In[i] * 32767.0f), -32768, 32767));
+	}
+}
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Constructor
 // ─────────────────────────────────────────────────────────────────────────────
@@ -132,6 +143,9 @@ void UPS_AI_ConvAgent_InteractionComponent::TickComponent(float DeltaTime, ELeve
 
 		bInitialized = true;
 
+		// If a replay recording started before the mic existed, begin capturing now.
+		UpdateReplayCapture();
+
 		if (bDebug)
 		{
 			UE_LOG(LogPS_AI_ConvAgent_Select, Log,
@@ -1062,10 +1076,34 @@ void UPS_AI_ConvAgent_InteractionComponent::CleanupRetainedGaze(UPS_AI_ConvAgent
 // ─────────────────────────────────────────────────────────────────────────────
 void UPS_AI_ConvAgent_InteractionComponent::OnMicAudioCaptured(const TArray<float>& FloatPCM)
 {
-	UPS_AI_ConvAgent_ElevenLabsComponent* Agent = SelectedAgent.Get();
+	if (UPS_AI_ConvAgent_ElevenLabsComponent* Agent = SelectedAgent.Get())
-	if (!Agent) return;
+	{
-
 		Agent->FeedExternalAudio(FloatPCM);
+		return;
+	}
+
+	// No agent selected: if a replay is being recorded, still capture this pawn's
+	// voice for the recorder. Convert to int16, accumulate, and relay in chunks —
+	// the host broadcasts locally; a remote client sends a Server RPC.
+	if (!bReplayRecordActive) return;
+
+	TArray<uint8> Bytes;
+	ConvReplay_FloatToInt16Bytes(FloatPCM, Bytes);
+	RecordMicBuffer.Append(Bytes);
+
+	static constexpr int32 RecordChunkMinBytes = 3200; // ~100ms @ 16kHz mono int16
+	if (RecordMicBuffer.Num() >= RecordChunkMinBytes)
+	{
+		if (GetOwnerRole() == ROLE_Authority)
+		{
+			OnPlayerVoiceForRecording.Broadcast(Cast<APawn>(GetOwner()), RecordMicBuffer);
+		}
+		else
+		{
+			ServerRecordVoice(RecordMicBuffer);
+		}
+		RecordMicBuffer.Reset();
+	}
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -1154,7 +1192,7 @@ void UPS_AI_ConvAgent_InteractionComponent::GetLifetimeReplicatedProps(
 	TArray<FLifetimeProperty>& OutLifetimeProps) const
 {
 	Super::GetLifetimeReplicatedProps(OutLifetimeProps);
-	// No replicated properties — but the component must be replicated for RPCs.
+	DOREPLIFETIME(UPS_AI_ConvAgent_InteractionComponent, bReplayRecordActive);
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -1231,6 +1269,50 @@ void UPS_AI_ConvAgent_InteractionComponent::ServerRelayMicAudio_Implementation(
 	Agent->ServerSendMicAudioFromPlayer(SenderPawn, PCMToSend);
 }
 
+// ─────────────────────────────────────────────────────────────────────────────
+// Agent-less replay recording (driven by PS_AI_ConvReplay)
+// ─────────────────────────────────────────────────────────────────────────────
+void UPS_AI_ConvAgent_InteractionComponent::SetReplayRecordActive(bool bActive)
+{
+	if (GetOwnerRole() != ROLE_Authority) return;
+	if (bReplayRecordActive == bActive) return;
+	bReplayRecordActive = bActive;
+	// Authority (host player) receives no OnRep — apply locally now.
+	UpdateReplayCapture();
+}
+
+void UPS_AI_ConvAgent_InteractionComponent::OnRep_ReplayRecordActive()
+{
+	UpdateReplayCapture();
+}
+
+void UPS_AI_ConvAgent_InteractionComponent::UpdateReplayCapture()
+{
+	if (!bReplayRecordActive)
+	{
+		RecordMicBuffer.Reset();
+		// Only stop the mic if no agent conversation still needs it.
+		if (MicComponent && MicComponent->IsCapturing() && !SelectedAgent.IsValid())
+		{
+			MicComponent->StopCapture();
+		}
+		return;
+	}
+
+	// Recording: make sure the mic is capturing. On a remote pawn's server-side
+	// replica MicComponent is null (tick disabled) and this is a harmless no-op —
+	// that client captures on its own machine and relays via ServerRecordVoice.
+	if (MicComponent && !MicComponent->IsCapturing())
+	{
+		MicComponent->StartCapture();
+	}
+}
+
+void UPS_AI_ConvAgent_InteractionComponent::ServerRecordVoice_Implementation(const TArray<uint8>& PCMBytes)
+{
+	OnPlayerVoiceForRecording.Broadcast(Cast<APawn>(GetOwner()), PCMBytes);
+}
+
 void UPS_AI_ConvAgent_InteractionComponent::ServerRelaySendText_Implementation(
 	AActor* AgentActor, const FString& Text)
 {

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h

@@ -131,6 +131,12 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnUserVoiceDetected);
 // Delivers PCM chunks as int16, 16kHz mono, little-endian.
 DECLARE_MULTICAST_DELEGATE_OneParam(FOnAgentAudioData, const TArray<uint8>& /*PCMData*/);
 
+/** Fired on the SERVER for each player mic chunk relayed via ServerSendMicAudioFromPlayer
+ *  (int16, 16kHz mono). Carries the speaking pawn so an external recorder (e.g. the
+ *  PS_AI_ConvReplay bridge) can capture player speech for replay. */
+DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnPlayerAudioCaptured,
+	APawn*, SpeakerPawn, const TArray<uint8>&, PCMData);
+
 // ─────────────────────────────────────────────────────────────────────────────
 // UPS_AI_ConvAgent_ElevenLabsComponent
 //
@@ -399,6 +405,12 @@ public:
 	 *  Used internally by UPS_AI_ConvAgent_LipSyncComponent for spectral analysis. */
 	FOnAgentAudioData OnAgentAudioData;
 
+	/** Fired on the SERVER for each player mic chunk relayed via ServerSendMicAudioFromPlayer
+	 *  (int16, 16kHz mono). Lets an external recorder capture player speech for replay.
+	 *  Not fired on clients — only the server holds the relayed mic stream. */
+	UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events")
+	FOnPlayerAudioCaptured OnPlayerAudioCaptured;
+
 	// ── Network state (replicated) ───────────────────────────────────────────
 
 	/** True when one or more players are in conversation with this NPC.
@@ -605,6 +617,28 @@ public:
 	UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
 	void InterruptAgent();
 
+	// ── Replay support (PS_AI_ConvReplay bridge) ─────────────────────────────
+	/**
+	 * Play an external PCM buffer through this agent's voice, exactly as if it were
+	 * live TTS: drives audio playback, lip-sync, body expression and start/stop events.
+	 * Role-agnostic (works on a replay/simulated proxy); no LOD culling.
+	 * @param PCM16 int16 samples, 16000 Hz mono, little-endian.
+	 */
+	UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Replay")
+	void PlayExternalAudioPCM(const TArray<uint8>& PCM16);
+
+	/** Stop/flush any audio currently playing (used when scrubbing a replay). */
+	UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Replay")
+	void StopExternalAudio();
+
+	/** Re-broadcast a recorded perform_action reaction during replay (fires OnAgentActionRequested). */
+	UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Replay")
+	void TriggerReplayedAction(const FString& ActionName);
+
+	/** Re-broadcast a recorded custom client tool call during replay (fires OnAgentClientToolCall). */
+	UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Replay")
+	void TriggerReplayedClientToolCall(const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall);
+
 	/**
 	 * Cleanly disable conversation on this agent.
 	 * Ends the WebSocket connection, stops all audio, and begins blending all

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionComponent.h

@@ -26,6 +26,12 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnConvAgentDeselected,
 /** Fired when no agent is within interaction range/view. */
 DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnNoConvAgentInRange);
 
+/** Fired on the SERVER for each player mic chunk captured for agent-less replay
+ *  recording (int16, 16kHz mono). Lets an external recorder (PS_AI_ConvReplay)
+ *  save player speech even when no agent is present in the scene. */
+DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnConvAgentPlayerVoiceForRecording,
+	APawn*, SpeakerPawn, const TArray<uint8>&, PCMData);
+
 // ─────────────────────────────────────────────────────────────────────────────
 // UPS_AI_ConvAgent_InteractionComponent
 //
@@ -195,6 +201,11 @@ public:
 		meta = (ToolTip = "Fires when no agent meets the distance/view criteria."))
 	FOnNoConvAgentInRange OnNoAgentInRange;
 
+	/** Fired on the SERVER for each player mic chunk captured for agent-less replay
+	 *  recording. An external recorder (PS_AI_ConvReplay) subscribes to save player voice. */
+	UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|Interaction|Events")
+	FOnConvAgentPlayerVoiceForRecording OnPlayerVoiceForRecording;
+
 	// ── Blueprint API ─────────────────────────────────────────────────────────
 
 	/** Get the currently selected agent (null if none). */
@@ -225,6 +236,12 @@ public:
 	UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Interaction")
 	void ClearSelection();
 
+	/** Enable/disable capturing this pawn's mic for replay recording even when no
+	 *  agent is selected. Server authority only; the flag replicates so the owning
+	 *  client starts capturing + relaying its mic. Driven by PS_AI_ConvReplay. */
+	UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Interaction")
+	void SetReplayRecordActive(bool bActive);
+
 	// ── Network relay RPCs ───────────────────────────────────────────────────
 	// UE5 clients can only call Server RPCs on actors they own. NPC actors are
 	// server-owned, so clients can't call RPCs on them directly.
@@ -253,6 +270,11 @@ public:
 	UFUNCTION(Server, Unreliable)
 	void ServerRelayMicAudio(AActor* AgentActor, const TArray<uint8>& PCMBytes);
 
+	/** Relay: stream this pawn's mic to the server purely for replay recording
+	 *  (no agent). The server broadcasts OnPlayerVoiceForRecording for the recorder. */
+	UFUNCTION(Server, Unreliable)
+	void ServerRecordVoice(const TArray<uint8>& PCMBytes);
+
 	/** Relay: send text message to an NPC agent. */
 	UFUNCTION(Server, Reliable)
 	void ServerRelaySendText(AActor* AgentActor, const FString& Text);
@@ -317,6 +339,22 @@ private:
 	UPROPERTY()
 	UPS_AI_ConvAgent_MicrophoneCaptureComponent* MicComponent = nullptr;
 
+	// ── Agent-less replay recording ──────────────────────────────────────────
+	/** Replicated: the server sets this while a replay is recording so the owning
+	 *  client captures + relays its mic even without a selected agent. */
+	UPROPERTY(ReplicatedUsing = OnRep_ReplayRecordActive)
+	bool bReplayRecordActive = false;
+
+	UFUNCTION()
+	void OnRep_ReplayRecordActive();
+
+	/** Start/stop mic capture to honour bReplayRecordActive (never stops the mic
+	 *  while an agent conversation still needs it). */
+	void UpdateReplayCapture();
+
+	/** Accumulates int16 mic bytes for the recording relay, chunked to limit RPC count. */
+	TArray<uint8> RecordMicBuffer;
+
 	/** True once the one-time lazy init in TickComponent has completed.
 	 *  Deferred from BeginPlay because IsLocallyControlled() may return false
 	 *  before the PlayerController has been replicated/possessed. */

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/PS_AI_ConvReplay.uplugin

@@ -0,0 +1,30 @@
+{
+	"FileVersion": 3,
+	"Version": 1,
+	"VersionName": "1.0",
+	"FriendlyName": "PS AI ConvReplay",
+	"Description": "Bridge that records ElevenLabs agent + player conversation audio (and AI reactions) into Unreal replays, and plays it back during replay. Keeps PS_AI_ConvAgent and PS_ReplaySystem independent.",
+	"Category": "ASTERION",
+	"CreatedBy": "ASTERION",
+	"CanContainContent": false,
+	"IsBetaVersion": false,
+	"Installed": false,
+	"EnabledByDefault": false,
+	"Modules": [
+		{
+			"Name": "PS_AI_ConvReplay",
+			"Type": "Runtime",
+			"LoadingPhase": "Default"
+		}
+	],
+	"Plugins": [
+		{
+			"Name": "PS_AI_ConvAgent",
+			"Enabled": true
+		},
+		{
+			"Name": "ReplaySystem",
+			"Enabled": true
+		}
+	]
+}

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/PS_AI_ConvReplay.Build.cs

@@ -0,0 +1,29 @@
+// Copyright ASTERION. All Rights Reserved.
+
+using UnrealBuildTool;
+
+public class PS_AI_ConvReplay : ModuleRules
+{
+	public PS_AI_ConvReplay(ReadOnlyTargetRules Target) : base(Target)
+	{
+		DefaultBuildSettings = BuildSettingsVersion.Latest;
+		PCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs;
+
+		PublicDependencyModuleNames.AddRange(new string[]
+		{
+			"Core",
+			"CoreUObject",
+			"Engine",
+			// Opus voice codec (own encoder/decoder instances — not shared with ConvAgent).
+			"Voice",
+			// The conversational-agent plugin whose delegates we tap and whose audio we replay.
+			"PS_AI_ConvAgent",
+			// Native replay wrapper (record/playback state, demo time, save path).
+			"ReplaySystem",
+		});
+
+		PrivateDependencyModuleNames.AddRange(new string[]
+		{
+		});
+	}
+}

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Private/PS_AI_ConvReplay_AgentTag.cpp

@@ -0,0 +1,117 @@
+// Copyright ASTERION. All Rights Reserved.
+
+#include "PS_AI_ConvReplay_AgentTag.h"
+#include "PS_AI_ConvReplay_Subsystem.h"
+#include "GameFramework/Actor.h"
+
+UPS_AI_ConvReplay_AgentTag::UPS_AI_ConvReplay_AgentTag()
+{
+	PrimaryComponentTick.bCanEverTick = false;
+	// Exists on server (recording) and on replay clients (identity lookup). It is not
+	// itself replicated — the owner agent already replicates; the tag only needs to be
+	// present on both ends with the same resolved id.
+	SetIsReplicatedByDefault(false);
+}
+
+FString UPS_AI_ConvReplay_AgentTag::GetResolvedId() const
+{
+	if (!AgentReplayId.IsNone())
+	{
+		return AgentReplayId.ToString();
+	}
+	return GetOwner() ? GetOwner()->GetName() : FString();
+}
+
+UPS_AI_ConvAgent_ElevenLabsComponent* UPS_AI_ConvReplay_AgentTag::GetAgentComponent() const
+{
+	if (AActor* Owner = GetOwner())
+	{
+		return Owner->FindComponentByClass<UPS_AI_ConvAgent_ElevenLabsComponent>();
+	}
+	return nullptr;
+}
+
+void UPS_AI_ConvReplay_AgentTag::BeginRecording(UPS_AI_ConvReplay_Subsystem* InSubsystem)
+{
+	Subsystem = InSubsystem;
+	Agent = GetAgentComponent();
+	if (!Agent || bBound)
+	{
+		return;
+	}
+
+	Agent->OnAgentStartedSpeaking.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleStarted);
+	Agent->OnAgentStoppedSpeaking.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleStopped);
+	Agent->OnAgentActionRequested.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleActionRequested);
+	Agent->OnAgentClientToolCall.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleClientToolCall);
+	Agent->OnAgentTextResponse.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleTextResponse);
+	Agent->OnAgentTranscript.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleTranscript);
+	Agent->OnPlayerAudioCaptured.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandlePlayerAudio);
+
+	// Raw PCM is a non-dynamic C++ multicast delegate.
+	AudioDataHandle = Agent->OnAgentAudioData.AddUObject(this, &UPS_AI_ConvReplay_AgentTag::HandleAgentAudioData);
+
+	bBound = true;
+}
+
+void UPS_AI_ConvReplay_AgentTag::EndRecording()
+{
+	if (Agent && bBound)
+	{
+		Agent->OnAgentStartedSpeaking.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleStarted);
+		Agent->OnAgentStoppedSpeaking.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleStopped);
+		Agent->OnAgentActionRequested.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleActionRequested);
+		Agent->OnAgentClientToolCall.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleClientToolCall);
+		Agent->OnAgentTextResponse.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleTextResponse);
+		Agent->OnAgentTranscript.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleTranscript);
+		Agent->OnPlayerAudioCaptured.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandlePlayerAudio);
+		Agent->OnAgentAudioData.Remove(AudioDataHandle);
+	}
+	AudioDataHandle.Reset();
+	bBound = false;
+	Subsystem = nullptr;
+}
+
+void UPS_AI_ConvReplay_AgentTag::HandleStarted()
+{
+	if (Subsystem) Subsystem->RecordAgentStarted(this);
+}
+
+void UPS_AI_ConvReplay_AgentTag::HandleStopped()
+{
+	if (Subsystem) Subsystem->RecordAgentStopped(this);
+}
+
+void UPS_AI_ConvReplay_AgentTag::HandleActionRequested(const FString& ActionName)
+{
+	if (Subsystem) Subsystem->RecordAction(this, ActionName);
+}
+
+void UPS_AI_ConvReplay_AgentTag::HandleClientToolCall(const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall)
+{
+	if (Subsystem) Subsystem->RecordToolCall(this, ToolCall);
+}
+
+void UPS_AI_ConvReplay_AgentTag::HandleTextResponse(const FString& Text)
+{
+	if (Subsystem) Subsystem->RecordAgentText(this, Text);
+}
+
+void UPS_AI_ConvReplay_AgentTag::HandleTranscript(const FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs& Segment)
+{
+	// Only final user transcripts become player-text markers.
+	if (Subsystem && Segment.Speaker == TEXT("user") && Segment.bIsFinal)
+	{
+		Subsystem->RecordUserTranscript(this, Segment.Text);
+	}
+}
+
+void UPS_AI_ConvReplay_AgentTag::HandlePlayerAudio(APawn* SpeakerPawn, const TArray<uint8>& PCM)
+{
+	if (Subsystem) Subsystem->RecordPlayerAudio(SpeakerPawn, PCM);
+}
+
+void UPS_AI_ConvReplay_AgentTag::HandleAgentAudioData(const TArray<uint8>& PCM)
+{
+	if (Subsystem) Subsystem->RecordAgentAudio(this, PCM);
+}

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Private/PS_AI_ConvReplay_Module.cpp

@@ -0,0 +1,5 @@
+// Copyright ASTERION. All Rights Reserved.
+
+#include "Modules/ModuleManager.h"
+
+IMPLEMENT_MODULE(FDefaultModuleImpl, PS_AI_ConvReplay);

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Public/PS_AI_ConvReplay_AgentTag.h

@@ -0,0 +1,71 @@
+// Copyright ASTERION. All Rights Reserved.
+
+#pragma once
+
+#include "CoreMinimal.h"
+#include "Components/ActorComponent.h"
+// Pulls in the ElevenLabs component + FPS_AI_ConvAgent_ClientToolCall_ElevenLabs /
+// FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs (needed for the UFUNCTION handler signatures).
+#include "PS_AI_ConvAgent_ElevenLabsComponent.h"
+#include "PS_AI_ConvReplay_AgentTag.generated.h"
+
+class UPS_AI_ConvReplay_Subsystem;
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Per-agent bridge component.
+//
+// Add one to each conversational agent to give it a stable, replay-safe identity
+// (AgentReplayId). During recording it binds the sibling ElevenLabs component's
+// delegates and forwards every conversation event to the subsystem, tagged with
+// this agent's identity. During replay the subsystem looks the agent up by id to
+// route recorded audio / reactions back to it.
+//
+// If you don't add one, the subsystem auto-provisions a tag on every agent at
+// record/replay start, using the owner actor name as the id (fine for
+// placed-in-level agents; add an explicit tag for runtime-spawned ones).
+// ─────────────────────────────────────────────────────────────────────────────
+UCLASS(ClassGroup = "ASTERION|PS_AI_ConvReplay", meta = (BlueprintSpawnableComponent),
+	DisplayName = "PS AI ConvReplay - Agent Tag")
+class PS_AI_CONVREPLAY_API UPS_AI_ConvReplay_AgentTag : public UActorComponent
+{
+	GENERATED_BODY()
+
+public:
+	UPS_AI_ConvReplay_AgentTag();
+
+	/** Stable id used to match recorded audio/reactions back to this agent on replay.
+	 *  Leave as None to fall back to the owner actor name (stable for placed agents). */
+	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvReplay")
+	FName AgentReplayId;
+
+	/** Resolved identity string (AgentReplayId if set, else the owner actor name). */
+	FString GetResolvedId() const;
+
+	/** The sibling ElevenLabs component on the owner actor (may be null). */
+	UPS_AI_ConvAgent_ElevenLabsComponent* GetAgentComponent() const;
+
+	/** Bind the sibling component's delegates and start forwarding events. Called by the subsystem. */
+	void BeginRecording(UPS_AI_ConvReplay_Subsystem* InSubsystem);
+
+	/** Unbind delegates and stop forwarding. Called by the subsystem. */
+	void EndRecording();
+
+private:
+	UPROPERTY(Transient) TObjectPtr<UPS_AI_ConvReplay_Subsystem> Subsystem;
+	UPROPERTY(Transient) TObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> Agent;
+
+	FDelegateHandle AudioDataHandle;
+	bool bBound = false;
+
+	// Dynamic-delegate handlers (must be UFUNCTION and match the delegate signatures).
+	UFUNCTION() void HandleStarted();
+	UFUNCTION() void HandleStopped();
+	UFUNCTION() void HandleActionRequested(const FString& ActionName);
+	UFUNCTION() void HandleClientToolCall(const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall);
+	UFUNCTION() void HandleTextResponse(const FString& Text);
+	UFUNCTION() void HandleTranscript(const FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs& Segment);
+	UFUNCTION() void HandlePlayerAudio(APawn* SpeakerPawn, const TArray<uint8>& PCM);
+
+	// Non-dynamic (raw C++ multicast) handler for agent PCM.
+	void HandleAgentAudioData(const TArray<uint8>& PCM);
+};

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Public/PS_AI_ConvReplay_Subsystem.h

@@ -0,0 +1,173 @@
+// Copyright ASTERION. All Rights Reserved.
+
+#pragma once
+
+#include "CoreMinimal.h"
+#include "Subsystems/WorldSubsystem.h"
+#include "PS_AI_ConvReplay_Types.h"
+#include "PS_AI_ConvReplay_Subsystem.generated.h"
+
+class UPS_AI_ConvReplay_AgentTag;
+class UPS_AI_ConvAgent_ElevenLabsComponent;
+class UPS_AI_ConvAgent_InteractionComponent;
+class UAudioComponent;
+class APawn;
+class IVoiceEncoder;
+class IVoiceDecoder;
+struct FPS_AI_ConvAgent_ClientToolCall_ElevenLabs;
+
+/** Fired on the replay client when a recorded line's timestamp is reached.
+ *  Use it to drive subtitles / a conversation log. Audio playback is automatic. */
+DECLARE_DYNAMIC_MULTICAST_DELEGATE_FourParams(FPS_AI_ConvReplay_OnReplayLinePlayed,
+	FName, AgentId, const FString&, Speaker, const FString&, Text, float, TimeSec);
+
+// ─────────────────────────────────────────────────────────────────────────────
+// World subsystem that records the conversation audio track while a replay is
+// being recorded, and plays it back while a replay is being played.
+//
+//  • Record: taps every agent's ElevenLabs delegates (via AgentTag), accumulates
+//    PCM per speaker, Opus-encodes each utterance and appends it to an external
+//    sidecar file (<ReplayName>.convreplay) next to the .replay — nothing heavy
+//    goes into the demo stream. Also records AI reactions (perform_action / custom
+//    tools) and player transcripts.
+//  • Replay: loads the sidecar, and each tick fires the records whose timestamp is
+//    reached — agent audio through the agent's own voice (drives lip-sync + body
+//    expression), player audio spatialized at the recorded location, reactions
+//    re-broadcast. Gaze / emotion / facial expression replay for free (replicated).
+// ─────────────────────────────────────────────────────────────────────────────
+UCLASS()
+class PS_AI_CONVREPLAY_API UPS_AI_ConvReplay_Subsystem : public UTickableWorldSubsystem
+{
+	GENERATED_BODY()
+
+public:
+	// USubsystem / UWorldSubsystem
+	virtual void Initialize(FSubsystemCollectionBase& Collection) override;
+	virtual void Deinitialize() override;
+	virtual bool DoesSupportWorldType(const EWorldType::Type WorldType) const override;
+
+	// FTickableGameObject
+	virtual void Tick(float DeltaTime) override;
+	virtual TStatId GetStatId() const override;
+
+	/** Fired on the replay client for each recorded line as its timestamp is reached. */
+	UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvReplay")
+	FPS_AI_ConvReplay_OnReplayLinePlayed OnReplayLinePlayed;
+
+	// ── Record callbacks (invoked by AgentTag on the recording machine) ──────
+	void RecordAgentStarted(UPS_AI_ConvReplay_AgentTag* Tag);
+	void RecordAgentAudio(UPS_AI_ConvReplay_AgentTag* Tag, const TArray<uint8>& PCM);
+	void RecordAgentText(UPS_AI_ConvReplay_AgentTag* Tag, const FString& Text);
+	void RecordAgentStopped(UPS_AI_ConvReplay_AgentTag* Tag);
+	void RecordAction(UPS_AI_ConvReplay_AgentTag* Tag, const FString& ActionName);
+	void RecordToolCall(UPS_AI_ConvReplay_AgentTag* Tag, const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall);
+	void RecordUserTranscript(UPS_AI_ConvReplay_AgentTag* Tag, const FString& Text);
+	void RecordPlayerAudio(APawn* SpeakerPawn, const TArray<uint8>& PCM);
+
+private:
+	// Session transitions.
+	void BeginRecordingSession();
+	void EndRecordingSession();
+	void BeginReplaySession();
+	void EndReplaySession();
+
+	void TickRecording();
+	void TickReplay();
+
+	// Agent-less player-voice capture (InteractionComponents on player pawns).
+	void RefreshRecordingMics();
+	UFUNCTION()
+	void HandlePlayerVoiceForRecording(APawn* SpeakerPawn, const TArray<uint8>& PCM);
+
+	// Agent discovery / lookup.
+	void RebuildTagList();
+	UPS_AI_ConvReplay_AgentTag* FindTagById(const FString& Id);
+	// Robust match for a recorded agent record: exact id first (placed agents /
+	// explicit AgentReplayId), else the nearest agent to the recorded position
+	// (handles runtime-spawned agents whose actor names differ on replay).
+	UPS_AI_ConvReplay_AgentTag* FindAgentForRecord(const FString& Id, const FVector& Pos);
+
+	// Sidecar IO.
+	FString GetSidecarPath() const;
+	void WriteRecord(FPS_AI_ConvReplay_Record& Rec);
+	void LoadRecords();
+
+	// Opus (own instances — not shared with ConvAgent's live encoder).
+	void EnsureCodec();
+	bool EncodeUtterance(const TArray<uint8>& PCM, TArray<uint8>& OutPackets);
+	bool DecodeUtterance(const TArray<uint8>& Packets, TArray<uint8>& OutPCM);
+
+	// Replay dispatch.
+	void DispatchDueRecords(double NowSec, bool bAudioAllowed);
+	void ResetPlaybackFrom(double NowSec);
+	void PlayPlayerAudioAtLocation(const FVector& Loc, const TArray<uint8>& PCM);
+
+	// Helpers.
+	double NowDemoSeconds() const;
+	void FlushAgentSegment(UPS_AI_ConvReplay_AgentTag* Tag);   // write + reset an agent utterance
+	void FlushPlayerSegment(APawn* Pawn);
+	static bool IsVoiced(const TArray<uint8>& PCM16);
+	static FString ResolvePlayerId(APawn* Pawn);
+	// Boost quiet mic capture: normalize a replayed player utterance toward a target
+	// RMS (peak-safe), then apply the ps.convreplay.PlayerGain CVar. In-place on int16 PCM.
+	static void NormalizePlayerPcm(TArray<uint8>& Pcm16);
+
+	// ── State ────────────────────────────────────────────────────────────────
+	bool bRecording = false;
+	bool bPlaying = false;
+	double LastReplayTime = -1.0;
+
+	IFileHandle* WriteHandle = nullptr;
+
+	TSharedPtr<IVoiceEncoder> Encoder;
+	TSharedPtr<IVoiceDecoder> Decoder;
+
+	UPROPERTY(Transient)
+	TArray<TObjectPtr<UPS_AI_ConvReplay_AgentTag>> Tags;
+
+	// Player InteractionComponents currently armed for agent-less voice recording.
+	UPROPERTY(Transient)
+	TArray<TObjectPtr<UPS_AI_ConvAgent_InteractionComponent>> RecordingMicComps;
+
+	// Per-agent recording accumulator.
+	struct FAgentAccum
+	{
+		double StartTime = 0.0;
+		double LastAudioTime = 0.0;  // demo time of the last appended chunk (gap detection)
+		FVector Pos = FVector::ZeroVector;
+		FString Text;
+		TArray<uint8> PCM;
+		bool bOpen = false;
+	};
+	TMap<TWeakObjectPtr<UPS_AI_ConvReplay_AgentTag>, FAgentAccum> AgentAccum;
+
+	// Per-player recording accumulator (segmented by silence).
+	struct FPlayerAccum
+	{
+		double StartTime = 0.0;
+		double LastVoiceTime = 0.0;
+		FVector Pos = FVector::ZeroVector;
+		FString Id;
+		TArray<uint8> PCM;
+		bool bActive = false;
+	};
+	TMap<TWeakObjectPtr<APawn>, FPlayerAccum> PlayerAccum;
+
+	// Loaded conversation track (replay).
+	TArray<FPS_AI_ConvReplay_Record> Records;
+
+	// Transient one-shot components used to play back player voices during replay.
+	UPROPERTY(Transient)
+	TArray<TObjectPtr<UAudioComponent>> PlayerAudioPool;
+
+	// Tunables.
+	static constexpr float AgentGapSplitSeconds = 1.5f;  // split an agent utterance if audio resumes after this gap (greeting vs reply)
+	static constexpr float PlayerSilenceFlushSec = 0.6f;
+	static constexpr float VoicedRmsThreshold = 0.02f;
+	static constexpr float ScrubBackThreshold = 0.05f;   // seconds jumped back => scrub
+	static constexpr float ScrubFwdThreshold = 1.5f;     // seconds jumped forward => scrub / fast-seek
+	static constexpr float NormalSpeedTolerance = 0.35f; // only play audio within this of 1x
+	// Target loudness (RMS, −1..1) is the live CVar ps.convreplay.PlayerTargetRms (default 0.14).
+	static constexpr float PlayerMinGain   = 0.1f;       // allow attenuating LOUD utterances (bidirectional AGC)
+	static constexpr float PlayerMaxGain   = 16.0f;      // cap so near-silence isn't blown up
+};

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Public/PS_AI_ConvReplay_Types.h

@@ -0,0 +1,70 @@
+// Copyright ASTERION. All Rights Reserved.
+
+#pragma once
+
+#include "CoreMinimal.h"
+
+// ─────────────────────────────────────────────────────────────────────────────
+// One entry in the conversation sidecar track (<ReplayName>.convreplay).
+//
+// The sidecar is an append-only sequence of these records, each serialized
+// back-to-back with FMemoryWriter (self-delimiting via FString/TArray length
+// prefixes) so it can be streamed while recording and read in one pass at replay.
+// Audio is stored as a sequence of length-prefixed Opus packets (see the
+// subsystem's EncodeUtterance/DecodeUtterance), NOT inside the demo metadata.
+// ─────────────────────────────────────────────────────────────────────────────
+enum class EPS_AI_ConvReplay_Kind : uint8
+{
+	AgentSpeech = 0,   // an agent utterance: Audio + Text, played through the agent's voice
+	PlayerSpeech = 1,  // a player utterance: Audio, played spatialized at Position
+	Action = 2,        // a perform_action reaction: Text = action name
+	ToolCall = 3,      // a custom client tool call: Text = tool name, Params = parameters
+	PlayerText = 4,    // a player transcript marker (subtitle only, no audio)
+};
+
+struct FPS_AI_ConvReplay_Record
+{
+	/** EPS_AI_ConvReplay_Kind. */
+	uint8 Kind = 0;
+
+	/** Start time in demo seconds (captured via GetCurrentReplayTime at record time). */
+	double TimeSec = 0.0;
+
+	/** Audio duration in seconds (0 for non-audio records). */
+	double DurationSec = 0.0;
+
+	/** Agent id (resolved from AgentTag / actor name) or player id. */
+	FString Id;
+
+	/** 0 = agent, 1 = user. */
+	uint8 Speaker = 0;
+
+	/** Transcript, or action name, or tool name depending on Kind. */
+	FString Text;
+
+	/** World location of the speaker at utterance start (used to spatialize player audio). */
+	FVector Position = FVector::ZeroVector;
+
+	/** Tool-call parameters (empty otherwise). */
+	TMap<FString, FString> Params;
+
+	/** Sequence of length-prefixed Opus packets (empty for non-audio records). */
+	TArray<uint8> Audio;
+
+	/** Runtime-only: set once this record has been dispatched during playback. NOT serialized. */
+	bool bPlayed = false;
+
+	friend FArchive& operator<<(FArchive& Ar, FPS_AI_ConvReplay_Record& R)
+	{
+		Ar << R.Kind;
+		Ar << R.TimeSec;
+		Ar << R.DurationSec;
+		Ar << R.Id;
+		Ar << R.Speaker;
+		Ar << R.Text;
+		Ar << R.Position;
+		Ar << R.Params;
+		Ar << R.Audio;
+		return Ar;
+	}
+};