diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp index 1671c24..c52094f 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp @@ -21,6 +21,17 @@ static TAutoConsoleVariable CVarDebugInteraction( TEXT("Debug HUD for Interaction. -1=use property, 0=off, 1-3=verbosity."), ECVF_Default); +// Convert normalized float mic samples to int16 little-endian bytes (16kHz mono). +static void ConvReplay_FloatToInt16Bytes(const TArray& In, TArray& Out) +{ + Out.SetNumUninitialized(In.Num() * static_cast(sizeof(int16))); + int16* D = reinterpret_cast(Out.GetData()); + for (int32 i = 0; i < In.Num(); ++i) + { + D[i] = static_cast(FMath::Clamp(FMath::RoundToInt(In[i] * 32767.0f), -32768, 32767)); + } +} + // ───────────────────────────────────────────────────────────────────────────── // Constructor // ───────────────────────────────────────────────────────────────────────────── @@ -132,6 +143,9 @@ void UPS_AI_ConvAgent_InteractionComponent::TickComponent(float DeltaTime, ELeve bInitialized = true; + // If a replay recording started before the mic existed, begin capturing now. + UpdateReplayCapture(); + if (bDebug) { UE_LOG(LogPS_AI_ConvAgent_Select, Log, @@ -1062,10 +1076,34 @@ void UPS_AI_ConvAgent_InteractionComponent::CleanupRetainedGaze(UPS_AI_ConvAgent // ───────────────────────────────────────────────────────────────────────────── void UPS_AI_ConvAgent_InteractionComponent::OnMicAudioCaptured(const TArray& FloatPCM) { - UPS_AI_ConvAgent_ElevenLabsComponent* Agent = SelectedAgent.Get(); - if (!Agent) return; + if (UPS_AI_ConvAgent_ElevenLabsComponent* Agent = SelectedAgent.Get()) + { + Agent->FeedExternalAudio(FloatPCM); + return; + } - Agent->FeedExternalAudio(FloatPCM); + // No agent selected: if a replay is being recorded, still capture this pawn's + // voice for the recorder. Convert to int16, accumulate, and relay in chunks — + // the host broadcasts locally; a remote client sends a Server RPC. + if (!bReplayRecordActive) return; + + TArray Bytes; + ConvReplay_FloatToInt16Bytes(FloatPCM, Bytes); + RecordMicBuffer.Append(Bytes); + + static constexpr int32 RecordChunkMinBytes = 3200; // ~100ms @ 16kHz mono int16 + if (RecordMicBuffer.Num() >= RecordChunkMinBytes) + { + if (GetOwnerRole() == ROLE_Authority) + { + OnPlayerVoiceForRecording.Broadcast(Cast(GetOwner()), RecordMicBuffer); + } + else + { + ServerRecordVoice(RecordMicBuffer); + } + RecordMicBuffer.Reset(); + } } // ───────────────────────────────────────────────────────────────────────────── @@ -1154,7 +1192,7 @@ void UPS_AI_ConvAgent_InteractionComponent::GetLifetimeReplicatedProps( TArray& OutLifetimeProps) const { Super::GetLifetimeReplicatedProps(OutLifetimeProps); - // No replicated properties — but the component must be replicated for RPCs. + DOREPLIFETIME(UPS_AI_ConvAgent_InteractionComponent, bReplayRecordActive); } // ───────────────────────────────────────────────────────────────────────────── @@ -1231,6 +1269,50 @@ void UPS_AI_ConvAgent_InteractionComponent::ServerRelayMicAudio_Implementation( Agent->ServerSendMicAudioFromPlayer(SenderPawn, PCMToSend); } +// ───────────────────────────────────────────────────────────────────────────── +// Agent-less replay recording (driven by PS_AI_ConvReplay) +// ───────────────────────────────────────────────────────────────────────────── +void UPS_AI_ConvAgent_InteractionComponent::SetReplayRecordActive(bool bActive) +{ + if (GetOwnerRole() != ROLE_Authority) return; + if (bReplayRecordActive == bActive) return; + bReplayRecordActive = bActive; + // Authority (host player) receives no OnRep — apply locally now. + UpdateReplayCapture(); +} + +void UPS_AI_ConvAgent_InteractionComponent::OnRep_ReplayRecordActive() +{ + UpdateReplayCapture(); +} + +void UPS_AI_ConvAgent_InteractionComponent::UpdateReplayCapture() +{ + if (!bReplayRecordActive) + { + RecordMicBuffer.Reset(); + // Only stop the mic if no agent conversation still needs it. + if (MicComponent && MicComponent->IsCapturing() && !SelectedAgent.IsValid()) + { + MicComponent->StopCapture(); + } + return; + } + + // Recording: make sure the mic is capturing. On a remote pawn's server-side + // replica MicComponent is null (tick disabled) and this is a harmless no-op — + // that client captures on its own machine and relays via ServerRecordVoice. + if (MicComponent && !MicComponent->IsCapturing()) + { + MicComponent->StartCapture(); + } +} + +void UPS_AI_ConvAgent_InteractionComponent::ServerRecordVoice_Implementation(const TArray& PCMBytes) +{ + OnPlayerVoiceForRecording.Broadcast(Cast(GetOwner()), PCMBytes); +} + void UPS_AI_ConvAgent_InteractionComponent::ServerRelaySendText_Implementation( AActor* AgentActor, const FString& Text) { diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionComponent.h index 9f1d234..e536049 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionComponent.h @@ -26,6 +26,12 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnConvAgentDeselected, /** Fired when no agent is within interaction range/view. */ DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnNoConvAgentInRange); +/** Fired on the SERVER for each player mic chunk captured for agent-less replay + * recording (int16, 16kHz mono). Lets an external recorder (PS_AI_ConvReplay) + * save player speech even when no agent is present in the scene. */ +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnConvAgentPlayerVoiceForRecording, + APawn*, SpeakerPawn, const TArray&, PCMData); + // ───────────────────────────────────────────────────────────────────────────── // UPS_AI_ConvAgent_InteractionComponent // @@ -195,6 +201,11 @@ public: meta = (ToolTip = "Fires when no agent meets the distance/view criteria.")) FOnNoConvAgentInRange OnNoAgentInRange; + /** Fired on the SERVER for each player mic chunk captured for agent-less replay + * recording. An external recorder (PS_AI_ConvReplay) subscribes to save player voice. */ + UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|Interaction|Events") + FOnConvAgentPlayerVoiceForRecording OnPlayerVoiceForRecording; + // ── Blueprint API ───────────────────────────────────────────────────────── /** Get the currently selected agent (null if none). */ @@ -225,6 +236,12 @@ public: UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Interaction") void ClearSelection(); + /** Enable/disable capturing this pawn's mic for replay recording even when no + * agent is selected. Server authority only; the flag replicates so the owning + * client starts capturing + relaying its mic. Driven by PS_AI_ConvReplay. */ + UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Interaction") + void SetReplayRecordActive(bool bActive); + // ── Network relay RPCs ─────────────────────────────────────────────────── // UE5 clients can only call Server RPCs on actors they own. NPC actors are // server-owned, so clients can't call RPCs on them directly. @@ -253,6 +270,11 @@ public: UFUNCTION(Server, Unreliable) void ServerRelayMicAudio(AActor* AgentActor, const TArray& PCMBytes); + /** Relay: stream this pawn's mic to the server purely for replay recording + * (no agent). The server broadcasts OnPlayerVoiceForRecording for the recorder. */ + UFUNCTION(Server, Unreliable) + void ServerRecordVoice(const TArray& PCMBytes); + /** Relay: send text message to an NPC agent. */ UFUNCTION(Server, Reliable) void ServerRelaySendText(AActor* AgentActor, const FString& Text); @@ -317,6 +339,22 @@ private: UPROPERTY() UPS_AI_ConvAgent_MicrophoneCaptureComponent* MicComponent = nullptr; + // ── Agent-less replay recording ────────────────────────────────────────── + /** Replicated: the server sets this while a replay is recording so the owning + * client captures + relays its mic even without a selected agent. */ + UPROPERTY(ReplicatedUsing = OnRep_ReplayRecordActive) + bool bReplayRecordActive = false; + + UFUNCTION() + void OnRep_ReplayRecordActive(); + + /** Start/stop mic capture to honour bReplayRecordActive (never stops the mic + * while an agent conversation still needs it). */ + void UpdateReplayCapture(); + + /** Accumulates int16 mic bytes for the recording relay, chunked to limit RPC count. */ + TArray RecordMicBuffer; + /** True once the one-time lazy init in TickComponent has completed. * Deferred from BeginPlay because IsLocallyControlled() may return false * before the PlayerController has been replicated/possessed. */ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Private/PS_AI_ConvReplay_Subsystem.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Private/PS_AI_ConvReplay_Subsystem.cpp index 2aa4d03..468ebb6 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Private/PS_AI_ConvReplay_Subsystem.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Private/PS_AI_ConvReplay_Subsystem.cpp @@ -3,6 +3,7 @@ #include "PS_AI_ConvReplay_Subsystem.h" #include "PS_AI_ConvReplay_AgentTag.h" #include "PS_AI_ConvAgent_ElevenLabsComponent.h" +#include "PS_AI_ConvAgent_InteractionComponent.h" #include "ReplaySystemBPLibrary.h" #include "VoiceModule.h" @@ -37,6 +38,21 @@ static TAutoConsoleVariable CVarPlayerGain( TEXT("Extra gain multiplier applied to replayed player mic audio (on top of auto-normalization). Default 1.0."), ECVF_Default); +// Target loudness (RMS, 0..1) each replayed player utterance is normalized toward. +// Higher = louder (but boosts a quiet mic's noise/quantization more). Tune live to taste. +static TAutoConsoleVariable CVarPlayerTargetRms( + TEXT("ps.convreplay.PlayerTargetRms"), + 0.08f, + TEXT("Target loudness (RMS, 0..1) for replayed player mic. Higher = louder. Default 0.08."), + ECVF_Default); + +// Verbose per-record logging (dispatch + per-utterance audio stats). Off by default. +static TAutoConsoleVariable CVarConvReplayDebug( + TEXT("ps.convreplay.Debug"), + 0, + TEXT("ConvReplay verbose logging: 1 = per-record dispatch + per-utterance audio stats. Default 0."), + ECVF_Default); + namespace PS_AI_ConvReplay_Audio { static constexpr int32 SampleRate = 16000; @@ -202,6 +218,37 @@ UPS_AI_ConvReplay_AgentTag* UPS_AI_ConvReplay_Subsystem::FindAgentForRecord(cons return Best; } +// ───────────────────────────────────────────────────────────────────────────── +// Agent-less player-voice capture +// ───────────────────────────────────────────────────────────────────────────── +void UPS_AI_ConvReplay_Subsystem::RefreshRecordingMics() +{ + UWorld* World = GetWorld(); + if (!World) return; + + // Server-authoritative arming: for every player pawn's InteractionComponent not + // yet armed, enable agent-less capture (replicated to the owning client) and + // subscribe to its server-side voice delegate. + for (TActorIterator It(World); It; ++It) + { + APawn* Pawn = *It; + if (!Pawn) continue; + UPS_AI_ConvAgent_InteractionComponent* Mic = + Pawn->FindComponentByClass(); + if (!Mic || RecordingMicComps.Contains(Mic)) continue; + + Mic->SetReplayRecordActive(true); + Mic->OnPlayerVoiceForRecording.AddDynamic(this, &UPS_AI_ConvReplay_Subsystem::HandlePlayerVoiceForRecording); + RecordingMicComps.Add(Mic); + } +} + +void UPS_AI_ConvReplay_Subsystem::HandlePlayerVoiceForRecording(APawn* SpeakerPawn, const TArray& PCM) +{ + // Same pipeline as agent-routed player audio (segmentation, Opus, sidecar). + RecordPlayerAudio(SpeakerPawn, PCM); +} + // ───────────────────────────────────────────────────────────────────────────── // Recording session // ───────────────────────────────────────────────────────────────────────────── @@ -238,6 +285,9 @@ void UPS_AI_ConvReplay_Subsystem::BeginRecordingSession() { if (Tag) Tag->BeginRecording(this); } + + // Arm player mics for agent-less voice capture. + RefreshRecordingMics(); } void UPS_AI_ConvReplay_Subsystem::EndRecordingSession() @@ -255,6 +305,17 @@ void UPS_AI_ConvReplay_Subsystem::EndRecordingSession() if (Tag) Tag->EndRecording(); } + // Disarm player mics. + for (UPS_AI_ConvAgent_InteractionComponent* Mic : RecordingMicComps) + { + if (Mic) + { + Mic->SetReplayRecordActive(false); + Mic->OnPlayerVoiceForRecording.RemoveDynamic(this, &UPS_AI_ConvReplay_Subsystem::HandlePlayerVoiceForRecording); + } + } + RecordingMicComps.Reset(); + AgentAccum.Reset(); PlayerAccum.Reset(); @@ -270,6 +331,9 @@ void UPS_AI_ConvReplay_Subsystem::EndRecordingSession() void UPS_AI_ConvReplay_Subsystem::TickRecording() { + // Arm any player mics that appeared after recording began (late joiners). + RefreshRecordingMics(); + // Flush player segments that have gone silent. const double Now = NowDemoSeconds(); TArray> ToFlush; @@ -499,9 +563,12 @@ void UPS_AI_ConvReplay_Subsystem::DispatchDueRecords(double NowSec, bool bAudioA } R.bPlayed = true; - UE_LOG(LogPS_AI_ConvReplay, Log, - TEXT("Dispatch kind=%d id='%s' t=%.2f (now=%.2f) audio=%dB speed-allowed=%d"), - R.Kind, *R.Id, R.TimeSec, NowSec, R.Audio.Num(), bAudioAllowed ? 1 : 0); + if (CVarConvReplayDebug.GetValueOnGameThread() > 0) + { + UE_LOG(LogPS_AI_ConvReplay, Log, + TEXT("Dispatch kind=%d id='%s' t=%.2f (now=%.2f) audio=%dB speed-allowed=%d"), + R.Kind, *R.Id, R.TimeSec, NowSec, R.Audio.Num(), bAudioAllowed ? 1 : 0); + } OnReplayLinePlayed.Broadcast( FName(*R.Id), @@ -521,9 +588,12 @@ void UPS_AI_ConvReplay_Subsystem::DispatchDueRecords(double NowSec, bool bAudioA if (bDecoded) { Comp->PlayExternalAudioPCM(PCM); - UE_LOG(LogPS_AI_ConvReplay, Log, - TEXT(" agent speech → '%s' : played %d PCM bytes."), - *Tag->GetResolvedId(), PCM.Num()); + if (CVarConvReplayDebug.GetValueOnGameThread() > 0) + { + UE_LOG(LogPS_AI_ConvReplay, Log, + TEXT(" agent speech → '%s' : played %d PCM bytes."), + *Tag->GetResolvedId(), PCM.Num()); + } } else { @@ -854,30 +924,36 @@ void UPS_AI_ConvReplay_Subsystem::NormalizePlayerPcm(TArray& Pcm16) // Bidirectional AGC: bring each utterance to a consistent target RMS — BOOST quiet // ones and ATTENUATE loud ones — so speaking loud no longer runs hot. Then the live CVar. - double Gain = (Rms > 1e-5) ? (PlayerTargetRms / Rms) : 1.0; + const double TargetRms = FMath::Clamp(CVarPlayerTargetRms.GetValueOnGameThread(), 0.02f, 0.5f); + double Gain = (Rms > 1e-5) ? (TargetRms / Rms) : 1.0; Gain = FMath::Clamp(Gain, static_cast(PlayerMinGain), static_cast(PlayerMaxGain)); Gain *= FMath::Max(0.0f, CVarPlayerGain.GetValueOnGameThread()); - // Soft-knee limiter on the output: samples above the threshold are smoothly - // compressed toward full-scale (tanh) instead of hard-clipped, so residual peaks - // (or a hot CVar) don't produce harsh saturation. - const double T = static_cast(PlayerSoftClipThreshold); - int32 ClippedOut = 0; + // Feed-forward peak limiter with an envelope follower (instant attack, slow release) + // instead of instantaneous waveshaping. It lowers the gain SMOOTHLY around transient + // peaks — never reshaping individual samples — so we reach the target loudness without + // the harmonic distortion a hard/soft clipper produces. The peak lands exactly on the + // ceiling (no overshoot), and gain recovers over ~40 ms (no audible pumping). + const double Ceiling = 0.97; + const double EnvRelease = 0.995; // per-sample peak-envelope decay (~9 ms @ 16 kHz): + // catches the rare brief spikes, lets the body stay loud between them + double Env = 0.0; + int32 Limited = 0; for (int32 i = 0; i < N; ++i) { - double x = (S[i] / 32768.0) * Gain; - const double a = FMath::Abs(x); - if (a > T) - { - const double s = (x < 0.0) ? -1.0 : 1.0; - x = s * (T + (1.0 - T) * FMath::Tanh((a - T) / (1.0 - T))); - ++ClippedOut; - } - const int32 v = FMath::RoundToInt(x * 32767.0); + const double s = (S[i] / 32768.0) * Gain; + const double a = FMath::Abs(s); + Env = FMath::Max(a, Env * EnvRelease); // instant attack, slow release + const double Gr = (Env > Ceiling) ? (Ceiling / Env) : 1.0; + if (Gr < 1.0) ++Limited; + const int32 v = FMath::RoundToInt(s * Gr * 32767.0); S[i] = static_cast(FMath::Clamp(v, -32768, 32767)); } - UE_LOG(LogPS_AI_ConvReplay, Log, - TEXT("Player utterance: rms=%.3f peak=%d/32767 clipped-in=%d/%d gain=%.2f soft-limited=%d"), - Rms, Peak, ClippedIn, N, Gain, ClippedOut); + if (CVarConvReplayDebug.GetValueOnGameThread() > 0) + { + UE_LOG(LogPS_AI_ConvReplay, Log, + TEXT("Player utterance: rms=%.3f peak=%d/32767 clipped-in=%d/%d gain=%.2f limited=%d/%d"), + Rms, Peak, ClippedIn, N, Gain, Limited, N); + } } diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Public/PS_AI_ConvReplay_Subsystem.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Public/PS_AI_ConvReplay_Subsystem.h index e8ceb1b..cbdef49 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Public/PS_AI_ConvReplay_Subsystem.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Public/PS_AI_ConvReplay_Subsystem.h @@ -9,6 +9,7 @@ class UPS_AI_ConvReplay_AgentTag; class UPS_AI_ConvAgent_ElevenLabsComponent; +class UPS_AI_ConvAgent_InteractionComponent; class UAudioComponent; class APawn; class IVoiceEncoder; @@ -73,6 +74,11 @@ private: void TickRecording(); void TickReplay(); + // Agent-less player-voice capture (InteractionComponents on player pawns). + void RefreshRecordingMics(); + UFUNCTION() + void HandlePlayerVoiceForRecording(APawn* SpeakerPawn, const TArray& PCM); + // Agent discovery / lookup. void RebuildTagList(); UPS_AI_ConvReplay_AgentTag* FindTagById(const FString& Id); @@ -118,6 +124,10 @@ private: UPROPERTY(Transient) TArray> Tags; + // Player InteractionComponents currently armed for agent-less voice recording. + UPROPERTY(Transient) + TArray> RecordingMicComps; + // Per-agent recording accumulator. struct FAgentAccum { @@ -154,8 +164,7 @@ private: static constexpr float ScrubBackThreshold = 0.05f; // seconds jumped back => scrub static constexpr float ScrubFwdThreshold = 1.5f; // seconds jumped forward => scrub / fast-seek static constexpr float NormalSpeedTolerance = 0.35f; // only play audio within this of 1x - static constexpr float PlayerTargetRms = 0.12f; // target loudness for replayed player mic (−1..1 scale) + // Target loudness (RMS, −1..1) is the live CVar ps.convreplay.PlayerTargetRms (default 0.14). static constexpr float PlayerMinGain = 0.1f; // allow attenuating LOUD utterances (bidirectional AGC) - static constexpr float PlayerMaxGain = 12.0f; // cap so near-silence isn't blown up - static constexpr float PlayerSoftClipThreshold = 0.8f; // above this (|x|), peaks are soft-limited, not hard-clipped + static constexpr float PlayerMaxGain = 16.0f; // cap so near-silence isn't blown up };