ConvReplay: agent-less player voice recording + audio polish

Record every player's mic into the replay even with no agent in the scene (listen-server/standalone). The bridge arms each player pawn's InteractionComponent via a replicated flag; the host captures locally, remote clients relay via a new ServerRecordVoice RPC, and the server broadcasts OnPlayerVoiceForRecording into the existing player-audio pipeline. Agent and agent-less paths are mutually exclusive (no double-record).

Replay routing: match a recorded agent by id, else by nearest position (runtime-spawned NPCs get unstable names on replay), keeping lip-sync.

Player audio: per-utterance bidirectional AGC toward a target RMS with an envelope peak limiter (instant attack, ~9ms release) instead of hard clipping. Live CVars: ps.convreplay.PlayerTargetRms (default 0.08), ps.convreplay.PlayerGain, ps.convreplay.Debug (per-record logs off by default).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp

@@ -21,6 +21,17 @@ static TAutoConsoleVariable<int32> CVarDebugInteraction(
 	TEXT("Debug HUD for Interaction. -1=use property, 0=off, 1-3=verbosity."),
 	ECVF_Default);
 
+// Convert normalized float mic samples to int16 little-endian bytes (16kHz mono).
+static void ConvReplay_FloatToInt16Bytes(const TArray<float>& In, TArray<uint8>& Out)
+{
+	Out.SetNumUninitialized(In.Num() * static_cast<int32>(sizeof(int16)));
+	int16* D = reinterpret_cast<int16*>(Out.GetData());
+	for (int32 i = 0; i < In.Num(); ++i)
+	{
+		D[i] = static_cast<int16>(FMath::Clamp<int32>(FMath::RoundToInt(In[i] * 32767.0f), -32768, 32767));
+	}
+}
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Constructor
 // ─────────────────────────────────────────────────────────────────────────────
@@ -132,6 +143,9 @@ void UPS_AI_ConvAgent_InteractionComponent::TickComponent(float DeltaTime, ELeve
 
 		bInitialized = true;
 
+		// If a replay recording started before the mic existed, begin capturing now.
+		UpdateReplayCapture();
+
 		if (bDebug)
 		{
 			UE_LOG(LogPS_AI_ConvAgent_Select, Log,
@@ -1062,10 +1076,34 @@ void UPS_AI_ConvAgent_InteractionComponent::CleanupRetainedGaze(UPS_AI_ConvAgent
 // ─────────────────────────────────────────────────────────────────────────────
 void UPS_AI_ConvAgent_InteractionComponent::OnMicAudioCaptured(const TArray<float>& FloatPCM)
 {
-	UPS_AI_ConvAgent_ElevenLabsComponent* Agent = SelectedAgent.Get();
-	if (!Agent) return;
-
+	if (UPS_AI_ConvAgent_ElevenLabsComponent* Agent = SelectedAgent.Get())
+	{
 		Agent->FeedExternalAudio(FloatPCM);
+		return;
+	}
+
+	// No agent selected: if a replay is being recorded, still capture this pawn's
+	// voice for the recorder. Convert to int16, accumulate, and relay in chunks —
+	// the host broadcasts locally; a remote client sends a Server RPC.
+	if (!bReplayRecordActive) return;
+
+	TArray<uint8> Bytes;
+	ConvReplay_FloatToInt16Bytes(FloatPCM, Bytes);
+	RecordMicBuffer.Append(Bytes);
+
+	static constexpr int32 RecordChunkMinBytes = 3200; // ~100ms @ 16kHz mono int16
+	if (RecordMicBuffer.Num() >= RecordChunkMinBytes)
+	{
+		if (GetOwnerRole() == ROLE_Authority)
+		{
+			OnPlayerVoiceForRecording.Broadcast(Cast<APawn>(GetOwner()), RecordMicBuffer);
+		}
+		else
+		{
+			ServerRecordVoice(RecordMicBuffer);
+		}
+		RecordMicBuffer.Reset();
+	}
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -1154,7 +1192,7 @@ void UPS_AI_ConvAgent_InteractionComponent::GetLifetimeReplicatedProps(
 	TArray<FLifetimeProperty>& OutLifetimeProps) const
 {
 	Super::GetLifetimeReplicatedProps(OutLifetimeProps);
-	// No replicated properties — but the component must be replicated for RPCs.
+	DOREPLIFETIME(UPS_AI_ConvAgent_InteractionComponent, bReplayRecordActive);
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -1231,6 +1269,50 @@ void UPS_AI_ConvAgent_InteractionComponent::ServerRelayMicAudio_Implementation(
 	Agent->ServerSendMicAudioFromPlayer(SenderPawn, PCMToSend);
 }
 
+// ─────────────────────────────────────────────────────────────────────────────
+// Agent-less replay recording (driven by PS_AI_ConvReplay)
+// ─────────────────────────────────────────────────────────────────────────────
+void UPS_AI_ConvAgent_InteractionComponent::SetReplayRecordActive(bool bActive)
+{
+	if (GetOwnerRole() != ROLE_Authority) return;
+	if (bReplayRecordActive == bActive) return;
+	bReplayRecordActive = bActive;
+	// Authority (host player) receives no OnRep — apply locally now.
+	UpdateReplayCapture();
+}
+
+void UPS_AI_ConvAgent_InteractionComponent::OnRep_ReplayRecordActive()
+{
+	UpdateReplayCapture();
+}
+
+void UPS_AI_ConvAgent_InteractionComponent::UpdateReplayCapture()
+{
+	if (!bReplayRecordActive)
+	{
+		RecordMicBuffer.Reset();
+		// Only stop the mic if no agent conversation still needs it.
+		if (MicComponent && MicComponent->IsCapturing() && !SelectedAgent.IsValid())
+		{
+			MicComponent->StopCapture();
+		}
+		return;
+	}
+
+	// Recording: make sure the mic is capturing. On a remote pawn's server-side
+	// replica MicComponent is null (tick disabled) and this is a harmless no-op —
+	// that client captures on its own machine and relays via ServerRecordVoice.
+	if (MicComponent && !MicComponent->IsCapturing())
+	{
+		MicComponent->StartCapture();
+	}
+}
+
+void UPS_AI_ConvAgent_InteractionComponent::ServerRecordVoice_Implementation(const TArray<uint8>& PCMBytes)
+{
+	OnPlayerVoiceForRecording.Broadcast(Cast<APawn>(GetOwner()), PCMBytes);
+}
+
 void UPS_AI_ConvAgent_InteractionComponent::ServerRelaySendText_Implementation(
 	AActor* AgentActor, const FString& Text)
 {

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionComponent.h

@@ -26,6 +26,12 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnConvAgentDeselected,
 /** Fired when no agent is within interaction range/view. */
 DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnNoConvAgentInRange);
 
+/** Fired on the SERVER for each player mic chunk captured for agent-less replay
+ *  recording (int16, 16kHz mono). Lets an external recorder (PS_AI_ConvReplay)
+ *  save player speech even when no agent is present in the scene. */
+DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnConvAgentPlayerVoiceForRecording,
+	APawn*, SpeakerPawn, const TArray<uint8>&, PCMData);
+
 // ─────────────────────────────────────────────────────────────────────────────
 // UPS_AI_ConvAgent_InteractionComponent
 //
@@ -195,6 +201,11 @@ public:
 		meta = (ToolTip = "Fires when no agent meets the distance/view criteria."))
 	FOnNoConvAgentInRange OnNoAgentInRange;
 
+	/** Fired on the SERVER for each player mic chunk captured for agent-less replay
+	 *  recording. An external recorder (PS_AI_ConvReplay) subscribes to save player voice. */
+	UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|Interaction|Events")
+	FOnConvAgentPlayerVoiceForRecording OnPlayerVoiceForRecording;
+
 	// ── Blueprint API ─────────────────────────────────────────────────────────
 
 	/** Get the currently selected agent (null if none). */
@@ -225,6 +236,12 @@ public:
 	UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Interaction")
 	void ClearSelection();
 
+	/** Enable/disable capturing this pawn's mic for replay recording even when no
+	 *  agent is selected. Server authority only; the flag replicates so the owning
+	 *  client starts capturing + relaying its mic. Driven by PS_AI_ConvReplay. */
+	UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Interaction")
+	void SetReplayRecordActive(bool bActive);
+
 	// ── Network relay RPCs ───────────────────────────────────────────────────
 	// UE5 clients can only call Server RPCs on actors they own. NPC actors are
 	// server-owned, so clients can't call RPCs on them directly.
@@ -253,6 +270,11 @@ public:
 	UFUNCTION(Server, Unreliable)
 	void ServerRelayMicAudio(AActor* AgentActor, const TArray<uint8>& PCMBytes);
 
+	/** Relay: stream this pawn's mic to the server purely for replay recording
+	 *  (no agent). The server broadcasts OnPlayerVoiceForRecording for the recorder. */
+	UFUNCTION(Server, Unreliable)
+	void ServerRecordVoice(const TArray<uint8>& PCMBytes);
+
 	/** Relay: send text message to an NPC agent. */
 	UFUNCTION(Server, Reliable)
 	void ServerRelaySendText(AActor* AgentActor, const FString& Text);
@@ -317,6 +339,22 @@ private:
 	UPROPERTY()
 	UPS_AI_ConvAgent_MicrophoneCaptureComponent* MicComponent = nullptr;
 
+	// ── Agent-less replay recording ──────────────────────────────────────────
+	/** Replicated: the server sets this while a replay is recording so the owning
+	 *  client captures + relays its mic even without a selected agent. */
+	UPROPERTY(ReplicatedUsing = OnRep_ReplayRecordActive)
+	bool bReplayRecordActive = false;
+
+	UFUNCTION()
+	void OnRep_ReplayRecordActive();
+
+	/** Start/stop mic capture to honour bReplayRecordActive (never stops the mic
+	 *  while an agent conversation still needs it). */
+	void UpdateReplayCapture();
+
+	/** Accumulates int16 mic bytes for the recording relay, chunked to limit RPC count. */
+	TArray<uint8> RecordMicBuffer;
+
 	/** True once the one-time lazy init in TickComponent has completed.
 	 *  Deferred from BeginPlay because IsLocallyControlled() may return false
 	 *  before the PlayerController has been replicated/possessed. */

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Private/PS_AI_ConvReplay_Subsystem.cpp

@@ -3,6 +3,7 @@
 #include "PS_AI_ConvReplay_Subsystem.h"
 #include "PS_AI_ConvReplay_AgentTag.h"
 #include "PS_AI_ConvAgent_ElevenLabsComponent.h"
+#include "PS_AI_ConvAgent_InteractionComponent.h"
 #include "ReplaySystemBPLibrary.h"
 
 #include "VoiceModule.h"
@@ -37,6 +38,21 @@ static TAutoConsoleVariable<float> CVarPlayerGain(
 	TEXT("Extra gain multiplier applied to replayed player mic audio (on top of auto-normalization). Default 1.0."),
 	ECVF_Default);
 
+// Target loudness (RMS, 0..1) each replayed player utterance is normalized toward.
+// Higher = louder (but boosts a quiet mic's noise/quantization more). Tune live to taste.
+static TAutoConsoleVariable<float> CVarPlayerTargetRms(
+	TEXT("ps.convreplay.PlayerTargetRms"),
+	0.08f,
+	TEXT("Target loudness (RMS, 0..1) for replayed player mic. Higher = louder. Default 0.08."),
+	ECVF_Default);
+
+// Verbose per-record logging (dispatch + per-utterance audio stats). Off by default.
+static TAutoConsoleVariable<int32> CVarConvReplayDebug(
+	TEXT("ps.convreplay.Debug"),
+	0,
+	TEXT("ConvReplay verbose logging: 1 = per-record dispatch + per-utterance audio stats. Default 0."),
+	ECVF_Default);
+
 namespace PS_AI_ConvReplay_Audio
 {
 	static constexpr int32 SampleRate      = 16000;
@@ -202,6 +218,37 @@ UPS_AI_ConvReplay_AgentTag* UPS_AI_ConvReplay_Subsystem::FindAgentForRecord(cons
 	return Best;
 }
 
+// ─────────────────────────────────────────────────────────────────────────────
+// Agent-less player-voice capture
+// ─────────────────────────────────────────────────────────────────────────────
+void UPS_AI_ConvReplay_Subsystem::RefreshRecordingMics()
+{
+	UWorld* World = GetWorld();
+	if (!World) return;
+
+	// Server-authoritative arming: for every player pawn's InteractionComponent not
+	// yet armed, enable agent-less capture (replicated to the owning client) and
+	// subscribe to its server-side voice delegate.
+	for (TActorIterator<APawn> It(World); It; ++It)
+	{
+		APawn* Pawn = *It;
+		if (!Pawn) continue;
+		UPS_AI_ConvAgent_InteractionComponent* Mic =
+			Pawn->FindComponentByClass<UPS_AI_ConvAgent_InteractionComponent>();
+		if (!Mic || RecordingMicComps.Contains(Mic)) continue;
+
+		Mic->SetReplayRecordActive(true);
+		Mic->OnPlayerVoiceForRecording.AddDynamic(this, &UPS_AI_ConvReplay_Subsystem::HandlePlayerVoiceForRecording);
+		RecordingMicComps.Add(Mic);
+	}
+}
+
+void UPS_AI_ConvReplay_Subsystem::HandlePlayerVoiceForRecording(APawn* SpeakerPawn, const TArray<uint8>& PCM)
+{
+	// Same pipeline as agent-routed player audio (segmentation, Opus, sidecar).
+	RecordPlayerAudio(SpeakerPawn, PCM);
+}
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Recording session
 // ─────────────────────────────────────────────────────────────────────────────
@@ -238,6 +285,9 @@ void UPS_AI_ConvReplay_Subsystem::BeginRecordingSession()
 	{
 		if (Tag) Tag->BeginRecording(this);
 	}
+
+	// Arm player mics for agent-less voice capture.
+	RefreshRecordingMics();
 }
 
 void UPS_AI_ConvReplay_Subsystem::EndRecordingSession()
@@ -255,6 +305,17 @@ void UPS_AI_ConvReplay_Subsystem::EndRecordingSession()
 		if (Tag) Tag->EndRecording();
 	}
 
+	// Disarm player mics.
+	for (UPS_AI_ConvAgent_InteractionComponent* Mic : RecordingMicComps)
+	{
+		if (Mic)
+		{
+			Mic->SetReplayRecordActive(false);
+			Mic->OnPlayerVoiceForRecording.RemoveDynamic(this, &UPS_AI_ConvReplay_Subsystem::HandlePlayerVoiceForRecording);
+		}
+	}
+	RecordingMicComps.Reset();
+
 	AgentAccum.Reset();
 	PlayerAccum.Reset();
 
@@ -270,6 +331,9 @@ void UPS_AI_ConvReplay_Subsystem::EndRecordingSession()
 
 void UPS_AI_ConvReplay_Subsystem::TickRecording()
 {
+	// Arm any player mics that appeared after recording began (late joiners).
+	RefreshRecordingMics();
+
 	// Flush player segments that have gone silent.
 	const double Now = NowDemoSeconds();
 	TArray<TWeakObjectPtr<APawn>> ToFlush;
@@ -499,9 +563,12 @@ void UPS_AI_ConvReplay_Subsystem::DispatchDueRecords(double NowSec, bool bAudioA
 		}
 		R.bPlayed = true;
 
+		if (CVarConvReplayDebug.GetValueOnGameThread() > 0)
+		{
 			UE_LOG(LogPS_AI_ConvReplay, Log,
 				TEXT("Dispatch kind=%d id='%s' t=%.2f (now=%.2f) audio=%dB speed-allowed=%d"),
 				R.Kind, *R.Id, R.TimeSec, NowSec, R.Audio.Num(), bAudioAllowed ? 1 : 0);
+		}
 
 		OnReplayLinePlayed.Broadcast(
 			FName(*R.Id),
@@ -521,10 +588,13 @@ void UPS_AI_ConvReplay_Subsystem::DispatchDueRecords(double NowSec, bool bAudioA
 				if (bDecoded)
 				{
 					Comp->PlayExternalAudioPCM(PCM);
+					if (CVarConvReplayDebug.GetValueOnGameThread() > 0)
+					{
 						UE_LOG(LogPS_AI_ConvReplay, Log,
 							TEXT("  agent speech → '%s' : played %d PCM bytes."),
 							*Tag->GetResolvedId(), PCM.Num());
 					}
+				}
 				else
 				{
 					UE_LOG(LogPS_AI_ConvReplay, Warning,
@@ -854,30 +924,36 @@ void UPS_AI_ConvReplay_Subsystem::NormalizePlayerPcm(TArray<uint8>& Pcm16)
 
 	// Bidirectional AGC: bring each utterance to a consistent target RMS — BOOST quiet
 	// ones and ATTENUATE loud ones — so speaking loud no longer runs hot. Then the live CVar.
-	double Gain = (Rms > 1e-5) ? (PlayerTargetRms / Rms) : 1.0;
+	const double TargetRms = FMath::Clamp(CVarPlayerTargetRms.GetValueOnGameThread(), 0.02f, 0.5f);
+	double Gain = (Rms > 1e-5) ? (TargetRms / Rms) : 1.0;
 	Gain = FMath::Clamp(Gain, static_cast<double>(PlayerMinGain), static_cast<double>(PlayerMaxGain));
 	Gain *= FMath::Max(0.0f, CVarPlayerGain.GetValueOnGameThread());
 
-	// Soft-knee limiter on the output: samples above the threshold are smoothly
-	// compressed toward full-scale (tanh) instead of hard-clipped, so residual peaks
-	// (or a hot CVar) don't produce harsh saturation.
-	const double T = static_cast<double>(PlayerSoftClipThreshold);
-	int32 ClippedOut = 0;
+	// Feed-forward peak limiter with an envelope follower (instant attack, slow release)
+	// instead of instantaneous waveshaping. It lowers the gain SMOOTHLY around transient
+	// peaks — never reshaping individual samples — so we reach the target loudness without
+	// the harmonic distortion a hard/soft clipper produces. The peak lands exactly on the
+	// ceiling (no overshoot), and gain recovers over ~40 ms (no audible pumping).
+	const double Ceiling = 0.97;
+	const double EnvRelease = 0.995;    // per-sample peak-envelope decay (~9 ms @ 16 kHz):
+	                                    // catches the rare brief spikes, lets the body stay loud between them
+	double Env = 0.0;
+	int32 Limited = 0;
 	for (int32 i = 0; i < N; ++i)
 	{
-		double x = (S[i] / 32768.0) * Gain;
-		const double a = FMath::Abs(x);
-		if (a > T)
-		{
-			const double s = (x < 0.0) ? -1.0 : 1.0;
-			x = s * (T + (1.0 - T) * FMath::Tanh((a - T) / (1.0 - T)));
-			++ClippedOut;
-		}
-		const int32 v = FMath::RoundToInt(x * 32767.0);
+		const double s = (S[i] / 32768.0) * Gain;
+		const double a = FMath::Abs(s);
+		Env = FMath::Max(a, Env * EnvRelease);            // instant attack, slow release
+		const double Gr = (Env > Ceiling) ? (Ceiling / Env) : 1.0;
+		if (Gr < 1.0) ++Limited;
+		const int32 v = FMath::RoundToInt(s * Gr * 32767.0);
 		S[i] = static_cast<int16>(FMath::Clamp(v, -32768, 32767));
 	}
 
+	if (CVarConvReplayDebug.GetValueOnGameThread() > 0)
+	{
 		UE_LOG(LogPS_AI_ConvReplay, Log,
-		TEXT("Player utterance: rms=%.3f peak=%d/32767 clipped-in=%d/%d gain=%.2f soft-limited=%d"),
-		Rms, Peak, ClippedIn, N, Gain, ClippedOut);
+			TEXT("Player utterance: rms=%.3f peak=%d/32767 clipped-in=%d/%d gain=%.2f limited=%d/%d"),
+			Rms, Peak, ClippedIn, N, Gain, Limited, N);
+	}
 }

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Public/PS_AI_ConvReplay_Subsystem.h

@@ -9,6 +9,7 @@
 
 class UPS_AI_ConvReplay_AgentTag;
 class UPS_AI_ConvAgent_ElevenLabsComponent;
+class UPS_AI_ConvAgent_InteractionComponent;
 class UAudioComponent;
 class APawn;
 class IVoiceEncoder;
@@ -73,6 +74,11 @@ private:
 	void TickRecording();
 	void TickReplay();
 
+	// Agent-less player-voice capture (InteractionComponents on player pawns).
+	void RefreshRecordingMics();
+	UFUNCTION()
+	void HandlePlayerVoiceForRecording(APawn* SpeakerPawn, const TArray<uint8>& PCM);
+
 	// Agent discovery / lookup.
 	void RebuildTagList();
 	UPS_AI_ConvReplay_AgentTag* FindTagById(const FString& Id);
@@ -118,6 +124,10 @@ private:
 	UPROPERTY(Transient)
 	TArray<TObjectPtr<UPS_AI_ConvReplay_AgentTag>> Tags;
 
+	// Player InteractionComponents currently armed for agent-less voice recording.
+	UPROPERTY(Transient)
+	TArray<TObjectPtr<UPS_AI_ConvAgent_InteractionComponent>> RecordingMicComps;
+
 	// Per-agent recording accumulator.
 	struct FAgentAccum
 	{
@@ -154,8 +164,7 @@ private:
 	static constexpr float ScrubBackThreshold = 0.05f;   // seconds jumped back => scrub
 	static constexpr float ScrubFwdThreshold = 1.5f;     // seconds jumped forward => scrub / fast-seek
 	static constexpr float NormalSpeedTolerance = 0.35f; // only play audio within this of 1x
-	static constexpr float PlayerTargetRms = 0.12f;      // target loudness for replayed player mic (−1..1 scale)
+	// Target loudness (RMS, −1..1) is the live CVar ps.convreplay.PlayerTargetRms (default 0.14).
 	static constexpr float PlayerMinGain   = 0.1f;       // allow attenuating LOUD utterances (bidirectional AGC)
-	static constexpr float PlayerMaxGain   = 12.0f;      // cap so near-silence isn't blown up
-	static constexpr float PlayerSoftClipThreshold = 0.8f; // above this (|x|), peaks are soft-limited, not hard-clipped
+	static constexpr float PlayerMaxGain   = 16.0f;      // cap so near-silence isn't blown up
 };