diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp index d89d228..cdf8f04 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp @@ -2551,6 +2551,12 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ServerSendMicAudioFromPlayer( } } + // Replay capture: expose every relayed player mic chunk (with its pawn) so an + // external recorder (PS_AI_ConvReplay) can save player speech. Server-only — clients + // have no live WebSocket and returned above. Fired before arbitration so audio from + // every player is captured, even a non-active speaker. + OnPlayerAudioCaptured.Broadcast(SpeakerPawn, PCMBytes); + // Standalone / single-player: bypass speaker arbitration entirely. // There's only one player — no need for Contains check or speaker switching. if (NetConnectedPawns.Num() <= 1) @@ -2761,6 +2767,45 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastReceiveAgentAudio_Implementa } } +// ───────────────────────────────────────────────────────────────────────────── +// Replay support (PS_AI_ConvReplay bridge) +// ───────────────────────────────────────────────────────────────────────────── +void UPS_AI_ConvAgent_ElevenLabsComponent::PlayExternalAudioPCM(const TArray& PCM16) +{ + if (PCM16.Num() == 0) return; + + // Same tail as MulticastReceiveAgentAudio_Implementation, minus the LOD culls: + // enqueue for playback (drives body expression + start/stop via bAgentSpeaking) and + // broadcast the raw PCM so the LipSync component animates visemes. + EnqueueAgentAudio(PCM16); + + // During replay there is no live agent_response message, so the normal + // silence-detection stop path (TickComponent) would otherwise wait for the long + // hard-timeout. Mark the response as received so OnAgentStoppedSpeaking fires + // shortly after the buffer drains and body expression returns to idle. + // (Set AFTER EnqueueAgentAudio, which resets this flag on the first chunk.) + bAgentResponseReceived = true; + + OnAgentAudioData.Broadcast(PCM16); +} + +void UPS_AI_ConvAgent_ElevenLabsComponent::StopExternalAudio() +{ + StopAgentAudio(); +} + +void UPS_AI_ConvAgent_ElevenLabsComponent::TriggerReplayedAction(const FString& ActionName) +{ + PendingActionName = ActionName; + OnAgentActionRequested.Broadcast(ActionName); +} + +void UPS_AI_ConvAgent_ElevenLabsComponent::TriggerReplayedClientToolCall( + const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall) +{ + OnAgentClientToolCall.Broadcast(ToolCall); +} + void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastAgentStartedSpeaking_Implementation() { if (GetOwnerRole() == ROLE_Authority) return; diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h index 75a1c9a..6ddd10f 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h @@ -131,6 +131,12 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnUserVoiceDetected); // Delivers PCM chunks as int16, 16kHz mono, little-endian. DECLARE_MULTICAST_DELEGATE_OneParam(FOnAgentAudioData, const TArray& /*PCMData*/); +/** Fired on the SERVER for each player mic chunk relayed via ServerSendMicAudioFromPlayer + * (int16, 16kHz mono). Carries the speaking pawn so an external recorder (e.g. the + * PS_AI_ConvReplay bridge) can capture player speech for replay. */ +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnPlayerAudioCaptured, + APawn*, SpeakerPawn, const TArray&, PCMData); + // ───────────────────────────────────────────────────────────────────────────── // UPS_AI_ConvAgent_ElevenLabsComponent // @@ -399,6 +405,12 @@ public: * Used internally by UPS_AI_ConvAgent_LipSyncComponent for spectral analysis. */ FOnAgentAudioData OnAgentAudioData; + /** Fired on the SERVER for each player mic chunk relayed via ServerSendMicAudioFromPlayer + * (int16, 16kHz mono). Lets an external recorder capture player speech for replay. + * Not fired on clients — only the server holds the relayed mic stream. */ + UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events") + FOnPlayerAudioCaptured OnPlayerAudioCaptured; + // ── Network state (replicated) ─────────────────────────────────────────── /** True when one or more players are in conversation with this NPC. @@ -605,6 +617,28 @@ public: UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs") void InterruptAgent(); + // ── Replay support (PS_AI_ConvReplay bridge) ───────────────────────────── + /** + * Play an external PCM buffer through this agent's voice, exactly as if it were + * live TTS: drives audio playback, lip-sync, body expression and start/stop events. + * Role-agnostic (works on a replay/simulated proxy); no LOD culling. + * @param PCM16 int16 samples, 16000 Hz mono, little-endian. + */ + UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Replay") + void PlayExternalAudioPCM(const TArray& PCM16); + + /** Stop/flush any audio currently playing (used when scrubbing a replay). */ + UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Replay") + void StopExternalAudio(); + + /** Re-broadcast a recorded perform_action reaction during replay (fires OnAgentActionRequested). */ + UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Replay") + void TriggerReplayedAction(const FString& ActionName); + + /** Re-broadcast a recorded custom client tool call during replay (fires OnAgentClientToolCall). */ + UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Replay") + void TriggerReplayedClientToolCall(const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall); + /** * Cleanly disable conversation on this agent. * Ends the WebSocket connection, stops all audio, and begins blending all diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/PS_AI_ConvReplay.uplugin b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/PS_AI_ConvReplay.uplugin new file mode 100644 index 0000000..a4db7b2 --- /dev/null +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/PS_AI_ConvReplay.uplugin @@ -0,0 +1,30 @@ +{ + "FileVersion": 3, + "Version": 1, + "VersionName": "1.0", + "FriendlyName": "PS AI ConvReplay", + "Description": "Bridge that records ElevenLabs agent + player conversation audio (and AI reactions) into Unreal replays, and plays it back during replay. Keeps PS_AI_ConvAgent and PS_ReplaySystem independent.", + "Category": "ASTERION", + "CreatedBy": "ASTERION", + "CanContainContent": false, + "IsBetaVersion": false, + "Installed": false, + "EnabledByDefault": false, + "Modules": [ + { + "Name": "PS_AI_ConvReplay", + "Type": "Runtime", + "LoadingPhase": "Default" + } + ], + "Plugins": [ + { + "Name": "PS_AI_ConvAgent", + "Enabled": true + }, + { + "Name": "ReplaySystem", + "Enabled": true + } + ] +} diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/PS_AI_ConvReplay.Build.cs b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/PS_AI_ConvReplay.Build.cs new file mode 100644 index 0000000..aaf0de8 --- /dev/null +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/PS_AI_ConvReplay.Build.cs @@ -0,0 +1,29 @@ +// Copyright ASTERION. All Rights Reserved. + +using UnrealBuildTool; + +public class PS_AI_ConvReplay : ModuleRules +{ + public PS_AI_ConvReplay(ReadOnlyTargetRules Target) : base(Target) + { + DefaultBuildSettings = BuildSettingsVersion.Latest; + PCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs; + + PublicDependencyModuleNames.AddRange(new string[] + { + "Core", + "CoreUObject", + "Engine", + // Opus voice codec (own encoder/decoder instances — not shared with ConvAgent). + "Voice", + // The conversational-agent plugin whose delegates we tap and whose audio we replay. + "PS_AI_ConvAgent", + // Native replay wrapper (record/playback state, demo time, save path). + "ReplaySystem", + }); + + PrivateDependencyModuleNames.AddRange(new string[] + { + }); + } +} diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Private/PS_AI_ConvReplay_AgentTag.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Private/PS_AI_ConvReplay_AgentTag.cpp new file mode 100644 index 0000000..ec85fe7 --- /dev/null +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Private/PS_AI_ConvReplay_AgentTag.cpp @@ -0,0 +1,117 @@ +// Copyright ASTERION. All Rights Reserved. + +#include "PS_AI_ConvReplay_AgentTag.h" +#include "PS_AI_ConvReplay_Subsystem.h" +#include "GameFramework/Actor.h" + +UPS_AI_ConvReplay_AgentTag::UPS_AI_ConvReplay_AgentTag() +{ + PrimaryComponentTick.bCanEverTick = false; + // Exists on server (recording) and on replay clients (identity lookup). It is not + // itself replicated — the owner agent already replicates; the tag only needs to be + // present on both ends with the same resolved id. + SetIsReplicatedByDefault(false); +} + +FString UPS_AI_ConvReplay_AgentTag::GetResolvedId() const +{ + if (!AgentReplayId.IsNone()) + { + return AgentReplayId.ToString(); + } + return GetOwner() ? GetOwner()->GetName() : FString(); +} + +UPS_AI_ConvAgent_ElevenLabsComponent* UPS_AI_ConvReplay_AgentTag::GetAgentComponent() const +{ + if (AActor* Owner = GetOwner()) + { + return Owner->FindComponentByClass(); + } + return nullptr; +} + +void UPS_AI_ConvReplay_AgentTag::BeginRecording(UPS_AI_ConvReplay_Subsystem* InSubsystem) +{ + Subsystem = InSubsystem; + Agent = GetAgentComponent(); + if (!Agent || bBound) + { + return; + } + + Agent->OnAgentStartedSpeaking.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleStarted); + Agent->OnAgentStoppedSpeaking.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleStopped); + Agent->OnAgentActionRequested.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleActionRequested); + Agent->OnAgentClientToolCall.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleClientToolCall); + Agent->OnAgentTextResponse.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleTextResponse); + Agent->OnAgentTranscript.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleTranscript); + Agent->OnPlayerAudioCaptured.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandlePlayerAudio); + + // Raw PCM is a non-dynamic C++ multicast delegate. + AudioDataHandle = Agent->OnAgentAudioData.AddUObject(this, &UPS_AI_ConvReplay_AgentTag::HandleAgentAudioData); + + bBound = true; +} + +void UPS_AI_ConvReplay_AgentTag::EndRecording() +{ + if (Agent && bBound) + { + Agent->OnAgentStartedSpeaking.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleStarted); + Agent->OnAgentStoppedSpeaking.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleStopped); + Agent->OnAgentActionRequested.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleActionRequested); + Agent->OnAgentClientToolCall.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleClientToolCall); + Agent->OnAgentTextResponse.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleTextResponse); + Agent->OnAgentTranscript.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleTranscript); + Agent->OnPlayerAudioCaptured.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandlePlayerAudio); + Agent->OnAgentAudioData.Remove(AudioDataHandle); + } + AudioDataHandle.Reset(); + bBound = false; + Subsystem = nullptr; +} + +void UPS_AI_ConvReplay_AgentTag::HandleStarted() +{ + if (Subsystem) Subsystem->RecordAgentStarted(this); +} + +void UPS_AI_ConvReplay_AgentTag::HandleStopped() +{ + if (Subsystem) Subsystem->RecordAgentStopped(this); +} + +void UPS_AI_ConvReplay_AgentTag::HandleActionRequested(const FString& ActionName) +{ + if (Subsystem) Subsystem->RecordAction(this, ActionName); +} + +void UPS_AI_ConvReplay_AgentTag::HandleClientToolCall(const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall) +{ + if (Subsystem) Subsystem->RecordToolCall(this, ToolCall); +} + +void UPS_AI_ConvReplay_AgentTag::HandleTextResponse(const FString& Text) +{ + if (Subsystem) Subsystem->RecordAgentText(this, Text); +} + +void UPS_AI_ConvReplay_AgentTag::HandleTranscript(const FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs& Segment) +{ + // Only final user transcripts become player-text markers. + if (Subsystem && Segment.Speaker == TEXT("user") && Segment.bIsFinal) + { + Subsystem->RecordUserTranscript(this, Segment.Text); + } +} + +void UPS_AI_ConvReplay_AgentTag::HandlePlayerAudio(APawn* SpeakerPawn, const TArray& PCM) +{ + if (Subsystem) Subsystem->RecordPlayerAudio(SpeakerPawn, PCM); +} + +void UPS_AI_ConvReplay_AgentTag::HandleAgentAudioData(const TArray& PCM) +{ + if (Subsystem) Subsystem->RecordAgentAudio(this, PCM); +} diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Private/PS_AI_ConvReplay_Module.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Private/PS_AI_ConvReplay_Module.cpp new file mode 100644 index 0000000..13b2e5d --- /dev/null +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Private/PS_AI_ConvReplay_Module.cpp @@ -0,0 +1,5 @@ +// Copyright ASTERION. All Rights Reserved. + +#include "Modules/ModuleManager.h" + +IMPLEMENT_MODULE(FDefaultModuleImpl, PS_AI_ConvReplay); diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Private/PS_AI_ConvReplay_Subsystem.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Private/PS_AI_ConvReplay_Subsystem.cpp new file mode 100644 index 0000000..2aa4d03 --- /dev/null +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Private/PS_AI_ConvReplay_Subsystem.cpp @@ -0,0 +1,883 @@ +// Copyright ASTERION. All Rights Reserved. + +#include "PS_AI_ConvReplay_Subsystem.h" +#include "PS_AI_ConvReplay_AgentTag.h" +#include "PS_AI_ConvAgent_ElevenLabsComponent.h" +#include "ReplaySystemBPLibrary.h" + +#include "VoiceModule.h" +#include "Interfaces/VoiceCodec.h" + +#include "Sound/SoundWaveProcedural.h" +#include "Components/AudioComponent.h" +#include "Kismet/GameplayStatics.h" + +#include "GameFramework/Pawn.h" +#include "GameFramework/PlayerState.h" +#include "GameFramework/Actor.h" +#include "Engine/World.h" +#include "EngineUtils.h" + +#include "Stats/Stats.h" +#include "Serialization/MemoryWriter.h" +#include "Serialization/MemoryReader.h" +#include "HAL/PlatformFileManager.h" +#include "HAL/IConsoleManager.h" +#include "GenericPlatform/GenericPlatformFile.h" +#include "Misc/FileHelper.h" +#include "Misc/Paths.h" + +DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvReplay, Log, All); + +// Live-tunable extra gain on replayed player mic audio (on top of auto-normalization). +// e.g. console: ps.convreplay.PlayerGain 1.5 +static TAutoConsoleVariable CVarPlayerGain( + TEXT("ps.convreplay.PlayerGain"), + 1.0f, + TEXT("Extra gain multiplier applied to replayed player mic audio (on top of auto-normalization). Default 1.0."), + ECVF_Default); + +namespace PS_AI_ConvReplay_Audio +{ + static constexpr int32 SampleRate = 16000; + static constexpr int32 Channels = 1; + static constexpr int32 BytesPerFrame = 640; // 320 samples * 2 bytes (20ms @16k mono) + static constexpr int32 FramesPerPacket = 40; // matches ConvAgent's live path, under the 76-frame cap + static constexpr int32 PacketPCMBytes = BytesPerFrame * FramesPerPacket; // 25600 + static constexpr int32 WorkBufferBytes = 8 * 1024; + static constexpr int32 DecodeBufferBytes = 64 * 1024; +} + +// ───────────────────────────────────────────────────────────────────────────── +// USubsystem +// ───────────────────────────────────────────────────────────────────────────── +void UPS_AI_ConvReplay_Subsystem::Initialize(FSubsystemCollectionBase& Collection) +{ + Super::Initialize(Collection); + bRecording = false; + bPlaying = false; + LastReplayTime = -1.0; +} + +void UPS_AI_ConvReplay_Subsystem::Deinitialize() +{ + if (bRecording) EndRecordingSession(); + if (bPlaying) EndReplaySession(); + + if (WriteHandle) + { + delete WriteHandle; + WriteHandle = nullptr; + } + Encoder.Reset(); + Decoder.Reset(); + + Super::Deinitialize(); +} + +bool UPS_AI_ConvReplay_Subsystem::DoesSupportWorldType(const EWorldType::Type WorldType) const +{ + return WorldType == EWorldType::Game || WorldType == EWorldType::PIE; +} + +TStatId UPS_AI_ConvReplay_Subsystem::GetStatId() const +{ + RETURN_QUICK_DECLARE_CYCLE_STAT(UPS_AI_ConvReplay_Subsystem, STATGROUP_Tickables); +} + +double UPS_AI_ConvReplay_Subsystem::NowDemoSeconds() const +{ + return static_cast(UReplaySystemBPLibrary::GetCurrentReplayTime(GetWorld())); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Tick — drive record / replay based on the current demo state +// ───────────────────────────────────────────────────────────────────────────── +void UPS_AI_ConvReplay_Subsystem::Tick(float DeltaTime) +{ + UWorld* World = GetWorld(); + if (!World) return; + + const bool bRecNow = UReplaySystemBPLibrary::IsRecordingReplay(World); + const bool bPlayNow = UReplaySystemBPLibrary::IsPlayingReplay(World); + + if (bRecNow && !bRecording) BeginRecordingSession(); + else if (!bRecNow && bRecording) EndRecordingSession(); + + if (bPlayNow && !bPlaying) BeginReplaySession(); + else if (!bPlayNow && bPlaying) EndReplaySession(); + + if (bRecording) TickRecording(); + if (bPlaying) TickReplay(); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Agent discovery +// ───────────────────────────────────────────────────────────────────────────── +void UPS_AI_ConvReplay_Subsystem::RebuildTagList() +{ + Tags.Reset(); + UWorld* World = GetWorld(); + if (!World) return; + + for (TActorIterator It(World); It; ++It) + { + AActor* Actor = *It; + if (!Actor || !Actor->FindComponentByClass()) + { + continue; + } + + UPS_AI_ConvReplay_AgentTag* Tag = Actor->FindComponentByClass(); + if (!Tag) + { + // Auto-provision a tag (id = actor name) so agents work with zero setup. + Tag = NewObject(Actor); + if (Tag) + { + Tag->RegisterComponent(); + } + } + if (Tag) + { + Tags.AddUnique(Tag); + } + } +} + +UPS_AI_ConvReplay_AgentTag* UPS_AI_ConvReplay_Subsystem::FindTagById(const FString& Id) +{ + for (UPS_AI_ConvReplay_AgentTag* Tag : Tags) + { + if (Tag && Tag->GetResolvedId() == Id) + { + return Tag; + } + } + // Lazy rescan in case agents spawned after the session started. + RebuildTagList(); + for (UPS_AI_ConvReplay_AgentTag* Tag : Tags) + { + if (Tag && Tag->GetResolvedId() == Id) + { + return Tag; + } + } + return nullptr; +} + +UPS_AI_ConvReplay_AgentTag* UPS_AI_ConvReplay_Subsystem::FindAgentForRecord(const FString& Id, const FVector& Pos) +{ + // Exact id first (FindTagById also (re)populates Tags on a miss). + if (UPS_AI_ConvReplay_AgentTag* Exact = FindTagById(Id)) + { + return Exact; + } + + // Fallback: nearest agent to the recorded utterance position. Runtime-spawned + // agents get a different actor-name suffix on replay, so the id won't match — + // but the demo replays their transform, so the speaker is at (about) the same + // world position when this record fires. + UPS_AI_ConvReplay_AgentTag* Best = nullptr; + double BestDistSq = TNumericLimits::Max(); + for (UPS_AI_ConvReplay_AgentTag* Tag : Tags) + { + if (!Tag) continue; + if (const AActor* Owner = Tag->GetOwner()) + { + const double D = FVector::DistSquared(Owner->GetActorLocation(), Pos); + if (D < BestDistSq) + { + BestDistSq = D; + Best = Tag; + } + } + } + if (Best) + { + UE_LOG(LogPS_AI_ConvReplay, Verbose, + TEXT("Agent id '%s' not found by name; matched nearest agent '%s' (%.0f cm) to recorded position."), + *Id, *Best->GetResolvedId(), FMath::Sqrt(BestDistSq)); + } + return Best; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Recording session +// ───────────────────────────────────────────────────────────────────────────── +void UPS_AI_ConvReplay_Subsystem::BeginRecordingSession() +{ + bRecording = true; + AgentAccum.Reset(); + PlayerAccum.Reset(); + + const FString Path = GetSidecarPath(); + if (Path.IsEmpty()) + { + UE_LOG(LogPS_AI_ConvReplay, Warning, + TEXT("Recording started but no active replay name — conversation track disabled.")); + return; + } + + IPlatformFile& PF = FPlatformFileManager::Get().GetPlatformFile(); + PF.CreateDirectoryTree(*FPaths::GetPath(Path)); + if (WriteHandle) { delete WriteHandle; WriteHandle = nullptr; } + WriteHandle = PF.OpenWrite(*Path, /*bAppend=*/false, /*bAllowRead=*/true); + + if (!WriteHandle) + { + UE_LOG(LogPS_AI_ConvReplay, Warning, TEXT("Could not open sidecar for write: %s"), *Path); + } + else + { + UE_LOG(LogPS_AI_ConvReplay, Log, TEXT("Recording conversation track → %s"), *Path); + } + + RebuildTagList(); + for (UPS_AI_ConvReplay_AgentTag* Tag : Tags) + { + if (Tag) Tag->BeginRecording(this); + } +} + +void UPS_AI_ConvReplay_Subsystem::EndRecordingSession() +{ + // Flush any player segment still open. + TArray> Pending; + PlayerAccum.GetKeys(Pending); + for (const TWeakObjectPtr& P : Pending) + { + FlushPlayerSegment(P.Get()); + } + + for (UPS_AI_ConvReplay_AgentTag* Tag : Tags) + { + if (Tag) Tag->EndRecording(); + } + + AgentAccum.Reset(); + PlayerAccum.Reset(); + + if (WriteHandle) + { + WriteHandle->Flush(); + delete WriteHandle; + WriteHandle = nullptr; + } + bRecording = false; + UE_LOG(LogPS_AI_ConvReplay, Log, TEXT("Conversation track recording stopped.")); +} + +void UPS_AI_ConvReplay_Subsystem::TickRecording() +{ + // Flush player segments that have gone silent. + const double Now = NowDemoSeconds(); + TArray> ToFlush; + for (TPair, FPlayerAccum>& Pair : PlayerAccum) + { + FPlayerAccum& A = Pair.Value; + if (A.bActive && A.PCM.Num() > 0 && (Now - A.LastVoiceTime) > PlayerSilenceFlushSec) + { + ToFlush.Add(Pair.Key); + } + } + for (const TWeakObjectPtr& P : ToFlush) + { + FlushPlayerSegment(P.Get()); + } +} + +// ── Record callbacks ───────────────────────────────────────────────────────── +void UPS_AI_ConvReplay_Subsystem::RecordAgentStarted(UPS_AI_ConvReplay_AgentTag* Tag) +{ + if (!bRecording || !Tag) return; + FAgentAccum& A = AgentAccum.FindOrAdd(Tag); + A.StartTime = NowDemoSeconds(); + A.PCM.Reset(); + A.bOpen = true; + if (AActor* Owner = Tag->GetOwner()) + { + A.Pos = Owner->GetActorLocation(); + } +} + +void UPS_AI_ConvReplay_Subsystem::RecordAgentAudio(UPS_AI_ConvReplay_AgentTag* Tag, const TArray& PCM) +{ + if (!bRecording || !Tag || PCM.Num() == 0) return; + FAgentAccum& A = AgentAccum.FindOrAdd(Tag); + if (!A.bOpen) + { + // Audio before an explicit start — open a segment defensively. + A.StartTime = NowDemoSeconds(); + A.bOpen = true; + if (AActor* Owner = Tag->GetOwner()) A.Pos = Owner->GetActorLocation(); + } + A.PCM.Append(PCM); +} + +void UPS_AI_ConvReplay_Subsystem::RecordAgentText(UPS_AI_ConvReplay_AgentTag* Tag, const FString& Text) +{ + if (!bRecording || !Tag) return; + AgentAccum.FindOrAdd(Tag).Text = Text; +} + +void UPS_AI_ConvReplay_Subsystem::RecordAgentStopped(UPS_AI_ConvReplay_AgentTag* Tag) +{ + if (!bRecording || !Tag) return; + FAgentAccum* A = AgentAccum.Find(Tag); + if (!A || !A->bOpen || A->PCM.Num() == 0) + { + if (A) { A->bOpen = false; A->PCM.Reset(); A->Text.Reset(); } + return; + } + + FPS_AI_ConvReplay_Record R; + R.Kind = static_cast(EPS_AI_ConvReplay_Kind::AgentSpeech); + R.TimeSec = A->StartTime; + R.DurationSec = static_cast(A->PCM.Num() / 2) / PS_AI_ConvReplay_Audio::SampleRate; + R.Id = Tag->GetResolvedId(); + R.Speaker = 0; + R.Text = A->Text; + R.Position = A->Pos; + EncodeUtterance(A->PCM, R.Audio); + WriteRecord(R); + + A->bOpen = false; + A->PCM.Reset(); + A->Text.Reset(); +} + +void UPS_AI_ConvReplay_Subsystem::RecordAction(UPS_AI_ConvReplay_AgentTag* Tag, const FString& ActionName) +{ + if (!bRecording || !Tag) return; + FPS_AI_ConvReplay_Record R; + R.Kind = static_cast(EPS_AI_ConvReplay_Kind::Action); + R.TimeSec = NowDemoSeconds(); + R.Id = Tag->GetResolvedId(); + R.Speaker = 0; + R.Text = ActionName; + if (const AActor* Owner = Tag->GetOwner()) R.Position = Owner->GetActorLocation(); + WriteRecord(R); +} + +void UPS_AI_ConvReplay_Subsystem::RecordToolCall(UPS_AI_ConvReplay_AgentTag* Tag, + const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall) +{ + if (!bRecording || !Tag) return; + FPS_AI_ConvReplay_Record R; + R.Kind = static_cast(EPS_AI_ConvReplay_Kind::ToolCall); + R.TimeSec = NowDemoSeconds(); + R.Id = Tag->GetResolvedId(); + R.Speaker = 0; + R.Text = ToolCall.ToolName; + R.Params = ToolCall.Parameters; + R.Params.Add(TEXT("__toolCallId"), ToolCall.ToolCallId); // stashed so we can rebuild the struct on replay + if (const AActor* Owner = Tag->GetOwner()) R.Position = Owner->GetActorLocation(); + WriteRecord(R); +} + +void UPS_AI_ConvReplay_Subsystem::RecordUserTranscript(UPS_AI_ConvReplay_AgentTag* Tag, const FString& Text) +{ + if (!bRecording || !Tag || Text.IsEmpty()) return; + FPS_AI_ConvReplay_Record R; + R.Kind = static_cast(EPS_AI_ConvReplay_Kind::PlayerText); + R.TimeSec = NowDemoSeconds(); + R.Speaker = 1; + R.Text = Text; + if (UPS_AI_ConvAgent_ElevenLabsComponent* Comp = Tag->GetAgentComponent()) + { + if (APawn* SP = Comp->NetActiveSpeakerPawn) + { + R.Id = ResolvePlayerId(SP); + R.Position = SP->GetActorLocation(); + } + } + WriteRecord(R); +} + +void UPS_AI_ConvReplay_Subsystem::RecordPlayerAudio(APawn* SpeakerPawn, const TArray& PCM) +{ + if (!bRecording || !SpeakerPawn || PCM.Num() == 0) return; + + const bool bVoiced = IsVoiced(PCM); + FPlayerAccum& A = PlayerAccum.FindOrAdd(SpeakerPawn); + const double Now = NowDemoSeconds(); + + if (!A.bActive) + { + if (!bVoiced) return; // don't open a segment on ambient/silence + A.bActive = true; + A.StartTime = Now; + A.LastVoiceTime = Now; + A.Pos = SpeakerPawn->GetActorLocation(); + A.Id = ResolvePlayerId(SpeakerPawn); + A.PCM.Reset(); + } + + A.PCM.Append(PCM); + if (bVoiced) + { + A.LastVoiceTime = Now; + } +} + +void UPS_AI_ConvReplay_Subsystem::FlushPlayerSegment(APawn* Pawn) +{ + FPlayerAccum* A = PlayerAccum.Find(Pawn); + if (!A) return; + if (A->bActive && A->PCM.Num() > 0) + { + FPS_AI_ConvReplay_Record R; + R.Kind = static_cast(EPS_AI_ConvReplay_Kind::PlayerSpeech); + R.TimeSec = A->StartTime; + R.DurationSec = static_cast(A->PCM.Num() / 2) / PS_AI_ConvReplay_Audio::SampleRate; + R.Id = A->Id; + R.Speaker = 1; + R.Position = A->Pos; + EncodeUtterance(A->PCM, R.Audio); + WriteRecord(R); + } + A->bActive = false; + A->PCM.Reset(); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Replay session +// ───────────────────────────────────────────────────────────────────────────── +void UPS_AI_ConvReplay_Subsystem::BeginReplaySession() +{ + bPlaying = true; + LastReplayTime = -1.0; + PlayerAudioPool.Reset(); + RebuildTagList(); + LoadRecords(); + UE_LOG(LogPS_AI_ConvReplay, Log, TEXT("Replaying conversation track: %d records."), Records.Num()); +} + +void UPS_AI_ConvReplay_Subsystem::EndReplaySession() +{ + for (UAudioComponent* AC : PlayerAudioPool) + { + if (AC) AC->Stop(); + } + PlayerAudioPool.Reset(); + Records.Reset(); + bPlaying = false; + LastReplayTime = -1.0; +} + +void UPS_AI_ConvReplay_Subsystem::TickReplay() +{ + UWorld* World = GetWorld(); + if (!World) return; + + const double Now = static_cast(UReplaySystemBPLibrary::GetCurrentReplayTime(World)); + + // Scrub detection: a jump backward, or a large forward jump, resets playback. + if (LastReplayTime >= 0.0) + { + const double Delta = Now - LastReplayTime; + if (Delta < -static_cast(ScrubBackThreshold) || Delta > static_cast(ScrubFwdThreshold)) + { + ResetPlaybackFrom(Now); + } + } + LastReplayTime = Now; + + const float Speed = UReplaySystemBPLibrary::GetPlaybackSpeed(World); + const bool bAudioAllowed = FMath::IsNearlyEqual(Speed, 1.0f, NormalSpeedTolerance); + DispatchDueRecords(Now, bAudioAllowed); +} + +void UPS_AI_ConvReplay_Subsystem::DispatchDueRecords(double NowSec, bool bAudioAllowed) +{ + for (FPS_AI_ConvReplay_Record& R : Records) + { + if (R.bPlayed || R.TimeSec > NowSec) + { + continue; + } + R.bPlayed = true; + + UE_LOG(LogPS_AI_ConvReplay, Log, + TEXT("Dispatch kind=%d id='%s' t=%.2f (now=%.2f) audio=%dB speed-allowed=%d"), + R.Kind, *R.Id, R.TimeSec, NowSec, R.Audio.Num(), bAudioAllowed ? 1 : 0); + + OnReplayLinePlayed.Broadcast( + FName(*R.Id), + R.Speaker == 1 ? TEXT("user") : TEXT("agent"), + R.Text, + static_cast(R.TimeSec)); + + switch (static_cast(R.Kind)) + { + case EPS_AI_ConvReplay_Kind::AgentSpeech: + if (bAudioAllowed && R.Audio.Num() > 0) + { + UPS_AI_ConvReplay_AgentTag* Tag = FindAgentForRecord(R.Id, R.Position); + UPS_AI_ConvAgent_ElevenLabsComponent* Comp = Tag ? Tag->GetAgentComponent() : nullptr; + TArray PCM; + const bool bDecoded = Comp && DecodeUtterance(R.Audio, PCM); + if (bDecoded) + { + Comp->PlayExternalAudioPCM(PCM); + UE_LOG(LogPS_AI_ConvReplay, Log, + TEXT(" agent speech → '%s' : played %d PCM bytes."), + *Tag->GetResolvedId(), PCM.Num()); + } + else + { + UE_LOG(LogPS_AI_ConvReplay, Warning, + TEXT(" agent speech NOT played (agent=%s decoded=%d) — %d agent tags known."), + Comp ? TEXT("found") : TEXT("NOT FOUND"), PCM.Num(), Tags.Num()); + } + } + break; + + case EPS_AI_ConvReplay_Kind::PlayerSpeech: + if (bAudioAllowed && R.Audio.Num() > 0) + { + TArray PCM; + if (DecodeUtterance(R.Audio, PCM)) + { + PlayPlayerAudioAtLocation(R.Position, PCM); + } + } + break; + + case EPS_AI_ConvReplay_Kind::Action: + if (UPS_AI_ConvReplay_AgentTag* Tag = FindAgentForRecord(R.Id, R.Position)) + { + if (UPS_AI_ConvAgent_ElevenLabsComponent* Comp = Tag->GetAgentComponent()) + { + Comp->TriggerReplayedAction(R.Text); + } + } + break; + + case EPS_AI_ConvReplay_Kind::ToolCall: + if (UPS_AI_ConvReplay_AgentTag* Tag = FindAgentForRecord(R.Id, R.Position)) + { + if (UPS_AI_ConvAgent_ElevenLabsComponent* Comp = Tag->GetAgentComponent()) + { + FPS_AI_ConvAgent_ClientToolCall_ElevenLabs TC; + TC.ToolName = R.Text; + TC.Parameters = R.Params; + if (FString* Cid = TC.Parameters.Find(TEXT("__toolCallId"))) + { + TC.ToolCallId = *Cid; + TC.Parameters.Remove(TEXT("__toolCallId")); + } + Comp->TriggerReplayedClientToolCall(TC); + } + } + break; + + case EPS_AI_ConvReplay_Kind::PlayerText: + default: + break; // subtitle only (already broadcast above) + } + } +} + +void UPS_AI_ConvReplay_Subsystem::ResetPlaybackFrom(double NowSec) +{ + for (FPS_AI_ConvReplay_Record& R : Records) + { + // Records at/behind the new time are considered already elapsed (skipped), + // records ahead of it will fire as time advances. + R.bPlayed = (R.TimeSec <= NowSec); + } + for (UPS_AI_ConvReplay_AgentTag* Tag : Tags) + { + if (Tag) + { + if (UPS_AI_ConvAgent_ElevenLabsComponent* Comp = Tag->GetAgentComponent()) + { + Comp->StopExternalAudio(); + } + } + } + for (UAudioComponent* AC : PlayerAudioPool) + { + if (AC) AC->Stop(); + } + PlayerAudioPool.Reset(); +} + +void UPS_AI_ConvReplay_Subsystem::PlayPlayerAudioAtLocation(const FVector& Loc, const TArray& PCM) +{ + UWorld* World = GetWorld(); + if (!World || PCM.Num() == 0) return; + + // Drop finished/dead components from the pool. + PlayerAudioPool.RemoveAll([](const TObjectPtr& AC) + { + return AC == nullptr || !AC->IsPlaying(); + }); + + // Mic capture is often quiet — normalize this utterance to a consistent level. + TArray Boosted = PCM; + NormalizePlayerPcm(Boosted); + + USoundWaveProcedural* Wave = NewObject(this); + Wave->SetSampleRate(PS_AI_ConvReplay_Audio::SampleRate); + Wave->NumChannels = PS_AI_ConvReplay_Audio::Channels; + Wave->Duration = static_cast(Boosted.Num() / 2) / PS_AI_ConvReplay_Audio::SampleRate; + Wave->SoundGroup = SOUNDGROUP_Voice; + Wave->bLooping = false; + Wave->QueueAudio(Boosted.GetData(), Boosted.Num()); + + UAudioComponent* AC = UGameplayStatics::SpawnSoundAtLocation(World, Wave, Loc); + if (AC) + { + PlayerAudioPool.Add(AC); + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Sidecar IO +// ───────────────────────────────────────────────────────────────────────────── +FString UPS_AI_ConvReplay_Subsystem::GetSidecarPath() const +{ + UWorld* World = GetWorld(); + if (!World) return FString(); + + FString Name = UReplaySystemBPLibrary::GetActiveReplayName(World); + if (Name.IsEmpty() || Name == TEXT("None")) + { + return FString(); + } + + FString Dir = UReplaySystemBPLibrary::GetReplaySavePath(World); + if (Dir.IsEmpty()) + { + Dir = FPaths::Combine(FPaths::ProjectSavedDir(), TEXT("Demos")); + } + return FPaths::Combine(Dir, Name + TEXT(".convreplay")); +} + +void UPS_AI_ConvReplay_Subsystem::WriteRecord(FPS_AI_ConvReplay_Record& Rec) +{ + if (!WriteHandle) return; + TArray Blob; + FMemoryWriter Writer(Blob, /*bIsPersistent=*/true); + Writer << Rec; + WriteHandle->Write(Blob.GetData(), Blob.Num()); + WriteHandle->Flush(); +} + +void UPS_AI_ConvReplay_Subsystem::LoadRecords() +{ + Records.Reset(); + const FString Path = GetSidecarPath(); + if (Path.IsEmpty()) return; + + TArray Data; + if (!FFileHelper::LoadFileToArray(Data, *Path)) + { + UE_LOG(LogPS_AI_ConvReplay, Log, TEXT("No conversation track found at %s."), *Path); + return; + } + + FMemoryReader Reader(Data, /*bIsPersistent=*/true); + while (!Reader.AtEnd()) + { + FPS_AI_ConvReplay_Record R; + Reader << R; + Records.Add(MoveTemp(R)); + } + + Records.Sort([](const FPS_AI_ConvReplay_Record& A, const FPS_AI_ConvReplay_Record& B) + { + return A.TimeSec < B.TimeSec; + }); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Opus codec (own instances) +// ───────────────────────────────────────────────────────────────────────────── +void UPS_AI_ConvReplay_Subsystem::EnsureCodec() +{ + if (Encoder.IsValid() && Decoder.IsValid()) return; + if (!FVoiceModule::IsAvailable()) + { + UE_LOG(LogPS_AI_ConvReplay, Warning, + TEXT("FVoiceModule not available — conversation audio cannot be (de)compressed. Check [Voice] bEnabled=true.")); + return; + } + FVoiceModule& VM = FVoiceModule::Get(); + if (!Encoder.IsValid()) + { + Encoder = VM.CreateVoiceEncoder(PS_AI_ConvReplay_Audio::SampleRate, + PS_AI_ConvReplay_Audio::Channels, EAudioEncodeHint::VoiceEncode_Voice); + } + if (!Decoder.IsValid()) + { + Decoder = VM.CreateVoiceDecoder(PS_AI_ConvReplay_Audio::SampleRate, + PS_AI_ConvReplay_Audio::Channels); + } +} + +bool UPS_AI_ConvReplay_Subsystem::EncodeUtterance(const TArray& PCM, TArray& OutPackets) +{ + OutPackets.Reset(); + EnsureCodec(); + if (!Encoder.IsValid() || PCM.Num() == 0) return false; + + using namespace PS_AI_ConvReplay_Audio; + + Encoder->Reset(); // make this utterance self-contained (independent of the previous one) + + // Pad to a whole number of Opus frames (the encoder only consumes full frames). + TArray In = PCM; + const int32 Rem = In.Num() % BytesPerFrame; + if (Rem != 0) + { + In.AddZeroed(BytesPerFrame - Rem); + } + + TArray Work; + Work.SetNumUninitialized(WorkBufferBytes); + + FMemoryWriter Writer(OutPackets, /*bIsPersistent=*/true); + int32 Offset = 0; + while (Offset < In.Num()) + { + const int32 Block = FMath::Min(PacketPCMBytes, In.Num() - Offset); + uint32 CompSize = static_cast(Work.Num()); + Encoder->Encode(In.GetData() + Offset, static_cast(Block), Work.GetData(), CompSize); + if (CompSize > 0) + { + int32 Len = static_cast(CompSize); + Writer << Len; + Writer.Serialize(Work.GetData(), Len); + } + Offset += Block; + } + return OutPackets.Num() > 0; +} + +bool UPS_AI_ConvReplay_Subsystem::DecodeUtterance(const TArray& Packets, TArray& OutPCM) +{ + OutPCM.Reset(); + EnsureCodec(); + if (!Decoder.IsValid() || Packets.Num() == 0) return false; + + using namespace PS_AI_ConvReplay_Audio; + + Decoder->Reset(); + + TArray Frame; + Frame.SetNumUninitialized(DecodeBufferBytes); + + FMemoryReader Reader(Packets, /*bIsPersistent=*/true); + while (!Reader.AtEnd()) + { + int32 Len = 0; + Reader << Len; + if (Len <= 0 || Reader.Tell() + Len > Packets.Num()) + { + break; + } + TArray Packet; + Packet.SetNumUninitialized(Len); + Reader.Serialize(Packet.GetData(), Len); + + uint32 OutSize = static_cast(Frame.Num()); + Decoder->Decode(Packet.GetData(), static_cast(Len), Frame.GetData(), OutSize); + if (OutSize > 0) + { + OutPCM.Append(Frame.GetData(), OutSize); + } + } + return OutPCM.Num() > 0; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Helpers +// ───────────────────────────────────────────────────────────────────────────── +bool UPS_AI_ConvReplay_Subsystem::IsVoiced(const TArray& PCM16) +{ + const int32 NumSamples = PCM16.Num() / static_cast(sizeof(int16)); + if (NumSamples <= 0) return false; + const int16* S = reinterpret_cast(PCM16.GetData()); + double SumSq = 0.0; + for (int32 i = 0; i < NumSamples; ++i) + { + const double v = S[i] / 32768.0; + SumSq += v * v; + } + const double Rms = FMath::Sqrt(SumSq / NumSamples); + return Rms >= VoicedRmsThreshold; +} + +FString UPS_AI_ConvReplay_Subsystem::ResolvePlayerId(APawn* Pawn) +{ + // Player audio is replayed at its recorded world position, so this id is only a + // human-readable label for subtitles — the player name (or pawn name) is enough, + // and avoids pulling in the online-subsystem net-id types. + if (!Pawn) return TEXT("player"); + if (APlayerState* PS = Pawn->GetPlayerState()) + { + const FString PlayerName = PS->GetPlayerName(); + if (!PlayerName.IsEmpty()) + { + return PlayerName; + } + } + return Pawn->GetName(); +} + +void UPS_AI_ConvReplay_Subsystem::NormalizePlayerPcm(TArray& Pcm16) +{ + const int32 N = Pcm16.Num() / static_cast(sizeof(int16)); + if (N <= 0) return; + + int16* S = reinterpret_cast(Pcm16.GetData()); + + // Measure loudness (RMS), peak, and how many samples are already at/near full-scale + // (a signature of clipping AT CAPTURE — which no playback processing can undo). + double SumSq = 0.0; + int32 Peak = 0; + int32 ClippedIn = 0; + for (int32 i = 0; i < N; ++i) + { + const int32 A = FMath::Abs(static_cast(S[i])); + Peak = FMath::Max(Peak, A); + if (A >= 32000) ++ClippedIn; + const double v = S[i] / 32768.0; + SumSq += v * v; + } + if (Peak == 0) return; + const double Rms = FMath::Sqrt(SumSq / N); + + // Bidirectional AGC: bring each utterance to a consistent target RMS — BOOST quiet + // ones and ATTENUATE loud ones — so speaking loud no longer runs hot. Then the live CVar. + double Gain = (Rms > 1e-5) ? (PlayerTargetRms / Rms) : 1.0; + Gain = FMath::Clamp(Gain, static_cast(PlayerMinGain), static_cast(PlayerMaxGain)); + Gain *= FMath::Max(0.0f, CVarPlayerGain.GetValueOnGameThread()); + + // Soft-knee limiter on the output: samples above the threshold are smoothly + // compressed toward full-scale (tanh) instead of hard-clipped, so residual peaks + // (or a hot CVar) don't produce harsh saturation. + const double T = static_cast(PlayerSoftClipThreshold); + int32 ClippedOut = 0; + for (int32 i = 0; i < N; ++i) + { + double x = (S[i] / 32768.0) * Gain; + const double a = FMath::Abs(x); + if (a > T) + { + const double s = (x < 0.0) ? -1.0 : 1.0; + x = s * (T + (1.0 - T) * FMath::Tanh((a - T) / (1.0 - T))); + ++ClippedOut; + } + const int32 v = FMath::RoundToInt(x * 32767.0); + S[i] = static_cast(FMath::Clamp(v, -32768, 32767)); + } + + UE_LOG(LogPS_AI_ConvReplay, Log, + TEXT("Player utterance: rms=%.3f peak=%d/32767 clipped-in=%d/%d gain=%.2f soft-limited=%d"), + Rms, Peak, ClippedIn, N, Gain, ClippedOut); +} diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Public/PS_AI_ConvReplay_AgentTag.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Public/PS_AI_ConvReplay_AgentTag.h new file mode 100644 index 0000000..f25e2cb --- /dev/null +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Public/PS_AI_ConvReplay_AgentTag.h @@ -0,0 +1,71 @@ +// Copyright ASTERION. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Components/ActorComponent.h" +// Pulls in the ElevenLabs component + FPS_AI_ConvAgent_ClientToolCall_ElevenLabs / +// FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs (needed for the UFUNCTION handler signatures). +#include "PS_AI_ConvAgent_ElevenLabsComponent.h" +#include "PS_AI_ConvReplay_AgentTag.generated.h" + +class UPS_AI_ConvReplay_Subsystem; + +// ───────────────────────────────────────────────────────────────────────────── +// Per-agent bridge component. +// +// Add one to each conversational agent to give it a stable, replay-safe identity +// (AgentReplayId). During recording it binds the sibling ElevenLabs component's +// delegates and forwards every conversation event to the subsystem, tagged with +// this agent's identity. During replay the subsystem looks the agent up by id to +// route recorded audio / reactions back to it. +// +// If you don't add one, the subsystem auto-provisions a tag on every agent at +// record/replay start, using the owner actor name as the id (fine for +// placed-in-level agents; add an explicit tag for runtime-spawned ones). +// ───────────────────────────────────────────────────────────────────────────── +UCLASS(ClassGroup = "ASTERION|PS_AI_ConvReplay", meta = (BlueprintSpawnableComponent), + DisplayName = "PS AI ConvReplay - Agent Tag") +class PS_AI_CONVREPLAY_API UPS_AI_ConvReplay_AgentTag : public UActorComponent +{ + GENERATED_BODY() + +public: + UPS_AI_ConvReplay_AgentTag(); + + /** Stable id used to match recorded audio/reactions back to this agent on replay. + * Leave as None to fall back to the owner actor name (stable for placed agents). */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvReplay") + FName AgentReplayId; + + /** Resolved identity string (AgentReplayId if set, else the owner actor name). */ + FString GetResolvedId() const; + + /** The sibling ElevenLabs component on the owner actor (may be null). */ + UPS_AI_ConvAgent_ElevenLabsComponent* GetAgentComponent() const; + + /** Bind the sibling component's delegates and start forwarding events. Called by the subsystem. */ + void BeginRecording(UPS_AI_ConvReplay_Subsystem* InSubsystem); + + /** Unbind delegates and stop forwarding. Called by the subsystem. */ + void EndRecording(); + +private: + UPROPERTY(Transient) TObjectPtr Subsystem; + UPROPERTY(Transient) TObjectPtr Agent; + + FDelegateHandle AudioDataHandle; + bool bBound = false; + + // Dynamic-delegate handlers (must be UFUNCTION and match the delegate signatures). + UFUNCTION() void HandleStarted(); + UFUNCTION() void HandleStopped(); + UFUNCTION() void HandleActionRequested(const FString& ActionName); + UFUNCTION() void HandleClientToolCall(const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall); + UFUNCTION() void HandleTextResponse(const FString& Text); + UFUNCTION() void HandleTranscript(const FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs& Segment); + UFUNCTION() void HandlePlayerAudio(APawn* SpeakerPawn, const TArray& PCM); + + // Non-dynamic (raw C++ multicast) handler for agent PCM. + void HandleAgentAudioData(const TArray& PCM); +}; diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Public/PS_AI_ConvReplay_Subsystem.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Public/PS_AI_ConvReplay_Subsystem.h new file mode 100644 index 0000000..e8ceb1b --- /dev/null +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Public/PS_AI_ConvReplay_Subsystem.h @@ -0,0 +1,161 @@ +// Copyright ASTERION. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Subsystems/WorldSubsystem.h" +#include "PS_AI_ConvReplay_Types.h" +#include "PS_AI_ConvReplay_Subsystem.generated.h" + +class UPS_AI_ConvReplay_AgentTag; +class UPS_AI_ConvAgent_ElevenLabsComponent; +class UAudioComponent; +class APawn; +class IVoiceEncoder; +class IVoiceDecoder; +struct FPS_AI_ConvAgent_ClientToolCall_ElevenLabs; + +/** Fired on the replay client when a recorded line's timestamp is reached. + * Use it to drive subtitles / a conversation log. Audio playback is automatic. */ +DECLARE_DYNAMIC_MULTICAST_DELEGATE_FourParams(FPS_AI_ConvReplay_OnReplayLinePlayed, + FName, AgentId, const FString&, Speaker, const FString&, Text, float, TimeSec); + +// ───────────────────────────────────────────────────────────────────────────── +// World subsystem that records the conversation audio track while a replay is +// being recorded, and plays it back while a replay is being played. +// +// • Record: taps every agent's ElevenLabs delegates (via AgentTag), accumulates +// PCM per speaker, Opus-encodes each utterance and appends it to an external +// sidecar file (.convreplay) next to the .replay — nothing heavy +// goes into the demo stream. Also records AI reactions (perform_action / custom +// tools) and player transcripts. +// • Replay: loads the sidecar, and each tick fires the records whose timestamp is +// reached — agent audio through the agent's own voice (drives lip-sync + body +// expression), player audio spatialized at the recorded location, reactions +// re-broadcast. Gaze / emotion / facial expression replay for free (replicated). +// ───────────────────────────────────────────────────────────────────────────── +UCLASS() +class PS_AI_CONVREPLAY_API UPS_AI_ConvReplay_Subsystem : public UTickableWorldSubsystem +{ + GENERATED_BODY() + +public: + // USubsystem / UWorldSubsystem + virtual void Initialize(FSubsystemCollectionBase& Collection) override; + virtual void Deinitialize() override; + virtual bool DoesSupportWorldType(const EWorldType::Type WorldType) const override; + + // FTickableGameObject + virtual void Tick(float DeltaTime) override; + virtual TStatId GetStatId() const override; + + /** Fired on the replay client for each recorded line as its timestamp is reached. */ + UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvReplay") + FPS_AI_ConvReplay_OnReplayLinePlayed OnReplayLinePlayed; + + // ── Record callbacks (invoked by AgentTag on the recording machine) ────── + void RecordAgentStarted(UPS_AI_ConvReplay_AgentTag* Tag); + void RecordAgentAudio(UPS_AI_ConvReplay_AgentTag* Tag, const TArray& PCM); + void RecordAgentText(UPS_AI_ConvReplay_AgentTag* Tag, const FString& Text); + void RecordAgentStopped(UPS_AI_ConvReplay_AgentTag* Tag); + void RecordAction(UPS_AI_ConvReplay_AgentTag* Tag, const FString& ActionName); + void RecordToolCall(UPS_AI_ConvReplay_AgentTag* Tag, const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall); + void RecordUserTranscript(UPS_AI_ConvReplay_AgentTag* Tag, const FString& Text); + void RecordPlayerAudio(APawn* SpeakerPawn, const TArray& PCM); + +private: + // Session transitions. + void BeginRecordingSession(); + void EndRecordingSession(); + void BeginReplaySession(); + void EndReplaySession(); + + void TickRecording(); + void TickReplay(); + + // Agent discovery / lookup. + void RebuildTagList(); + UPS_AI_ConvReplay_AgentTag* FindTagById(const FString& Id); + // Robust match for a recorded agent record: exact id first (placed agents / + // explicit AgentReplayId), else the nearest agent to the recorded position + // (handles runtime-spawned agents whose actor names differ on replay). + UPS_AI_ConvReplay_AgentTag* FindAgentForRecord(const FString& Id, const FVector& Pos); + + // Sidecar IO. + FString GetSidecarPath() const; + void WriteRecord(FPS_AI_ConvReplay_Record& Rec); + void LoadRecords(); + + // Opus (own instances — not shared with ConvAgent's live encoder). + void EnsureCodec(); + bool EncodeUtterance(const TArray& PCM, TArray& OutPackets); + bool DecodeUtterance(const TArray& Packets, TArray& OutPCM); + + // Replay dispatch. + void DispatchDueRecords(double NowSec, bool bAudioAllowed); + void ResetPlaybackFrom(double NowSec); + void PlayPlayerAudioAtLocation(const FVector& Loc, const TArray& PCM); + + // Helpers. + double NowDemoSeconds() const; + void FlushPlayerSegment(APawn* Pawn); + static bool IsVoiced(const TArray& PCM16); + static FString ResolvePlayerId(APawn* Pawn); + // Boost quiet mic capture: normalize a replayed player utterance toward a target + // RMS (peak-safe), then apply the ps.convreplay.PlayerGain CVar. In-place on int16 PCM. + static void NormalizePlayerPcm(TArray& Pcm16); + + // ── State ──────────────────────────────────────────────────────────────── + bool bRecording = false; + bool bPlaying = false; + double LastReplayTime = -1.0; + + IFileHandle* WriteHandle = nullptr; + + TSharedPtr Encoder; + TSharedPtr Decoder; + + UPROPERTY(Transient) + TArray> Tags; + + // Per-agent recording accumulator. + struct FAgentAccum + { + double StartTime = 0.0; + FVector Pos = FVector::ZeroVector; + FString Text; + TArray PCM; + bool bOpen = false; + }; + TMap, FAgentAccum> AgentAccum; + + // Per-player recording accumulator (segmented by silence). + struct FPlayerAccum + { + double StartTime = 0.0; + double LastVoiceTime = 0.0; + FVector Pos = FVector::ZeroVector; + FString Id; + TArray PCM; + bool bActive = false; + }; + TMap, FPlayerAccum> PlayerAccum; + + // Loaded conversation track (replay). + TArray Records; + + // Transient one-shot components used to play back player voices during replay. + UPROPERTY(Transient) + TArray> PlayerAudioPool; + + // Tunables. + static constexpr float PlayerSilenceFlushSec = 0.6f; + static constexpr float VoicedRmsThreshold = 0.02f; + static constexpr float ScrubBackThreshold = 0.05f; // seconds jumped back => scrub + static constexpr float ScrubFwdThreshold = 1.5f; // seconds jumped forward => scrub / fast-seek + static constexpr float NormalSpeedTolerance = 0.35f; // only play audio within this of 1x + static constexpr float PlayerTargetRms = 0.12f; // target loudness for replayed player mic (−1..1 scale) + static constexpr float PlayerMinGain = 0.1f; // allow attenuating LOUD utterances (bidirectional AGC) + static constexpr float PlayerMaxGain = 12.0f; // cap so near-silence isn't blown up + static constexpr float PlayerSoftClipThreshold = 0.8f; // above this (|x|), peaks are soft-limited, not hard-clipped +}; diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Public/PS_AI_ConvReplay_Types.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Public/PS_AI_ConvReplay_Types.h new file mode 100644 index 0000000..ddaf544 --- /dev/null +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Public/PS_AI_ConvReplay_Types.h @@ -0,0 +1,70 @@ +// Copyright ASTERION. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" + +// ───────────────────────────────────────────────────────────────────────────── +// One entry in the conversation sidecar track (.convreplay). +// +// The sidecar is an append-only sequence of these records, each serialized +// back-to-back with FMemoryWriter (self-delimiting via FString/TArray length +// prefixes) so it can be streamed while recording and read in one pass at replay. +// Audio is stored as a sequence of length-prefixed Opus packets (see the +// subsystem's EncodeUtterance/DecodeUtterance), NOT inside the demo metadata. +// ───────────────────────────────────────────────────────────────────────────── +enum class EPS_AI_ConvReplay_Kind : uint8 +{ + AgentSpeech = 0, // an agent utterance: Audio + Text, played through the agent's voice + PlayerSpeech = 1, // a player utterance: Audio, played spatialized at Position + Action = 2, // a perform_action reaction: Text = action name + ToolCall = 3, // a custom client tool call: Text = tool name, Params = parameters + PlayerText = 4, // a player transcript marker (subtitle only, no audio) +}; + +struct FPS_AI_ConvReplay_Record +{ + /** EPS_AI_ConvReplay_Kind. */ + uint8 Kind = 0; + + /** Start time in demo seconds (captured via GetCurrentReplayTime at record time). */ + double TimeSec = 0.0; + + /** Audio duration in seconds (0 for non-audio records). */ + double DurationSec = 0.0; + + /** Agent id (resolved from AgentTag / actor name) or player id. */ + FString Id; + + /** 0 = agent, 1 = user. */ + uint8 Speaker = 0; + + /** Transcript, or action name, or tool name depending on Kind. */ + FString Text; + + /** World location of the speaker at utterance start (used to spatialize player audio). */ + FVector Position = FVector::ZeroVector; + + /** Tool-call parameters (empty otherwise). */ + TMap Params; + + /** Sequence of length-prefixed Opus packets (empty for non-audio records). */ + TArray Audio; + + /** Runtime-only: set once this record has been dispatched during playback. NOT serialized. */ + bool bPlayed = false; + + friend FArchive& operator<<(FArchive& Ar, FPS_AI_ConvReplay_Record& R) + { + Ar << R.Kind; + Ar << R.TimeSec; + Ar << R.DurationSec; + Ar << R.Id; + Ar << R.Speaker; + Ar << R.Text; + Ar << R.Position; + Ar << R.Params; + Ar << R.Audio; + return Ar; + } +};