ConvReplay: record & replay agent + player conversation audio in demos
New PS_AI_ConvReplay bridge plugin records ElevenLabs agent TTS and player mic audio (Opus) into a sidecar next to the .replay, and plays it back during DemoPlay with lip-sync / body-expression / gaze / emotion. Agent audio is re-fed through the agent's own voice; player audio is spatialized at the recorded position. ConvAgent hooks (generic): PlayExternalAudioPCM, StopExternalAudio, TriggerReplayedAction/TriggerReplayedClientToolCall, and an OnPlayerAudioCaptured delegate broadcast in ServerSendMicAudioFromPlayer. Agent matching is id-or-nearest-position (runtime-spawned NPCs get unstable names on replay). Player level normalized per-utterance (bidirectional AGC + soft limiter), tunable live via ps.convreplay.PlayerGain. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -2551,6 +2551,12 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ServerSendMicAudioFromPlayer(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Replay capture: expose every relayed player mic chunk (with its pawn) so an
|
||||||
|
// external recorder (PS_AI_ConvReplay) can save player speech. Server-only — clients
|
||||||
|
// have no live WebSocket and returned above. Fired before arbitration so audio from
|
||||||
|
// every player is captured, even a non-active speaker.
|
||||||
|
OnPlayerAudioCaptured.Broadcast(SpeakerPawn, PCMBytes);
|
||||||
|
|
||||||
// Standalone / single-player: bypass speaker arbitration entirely.
|
// Standalone / single-player: bypass speaker arbitration entirely.
|
||||||
// There's only one player — no need for Contains check or speaker switching.
|
// There's only one player — no need for Contains check or speaker switching.
|
||||||
if (NetConnectedPawns.Num() <= 1)
|
if (NetConnectedPawns.Num() <= 1)
|
||||||
@@ -2761,6 +2767,45 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastReceiveAgentAudio_Implementa
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Replay support (PS_AI_ConvReplay bridge)
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
void UPS_AI_ConvAgent_ElevenLabsComponent::PlayExternalAudioPCM(const TArray<uint8>& PCM16)
|
||||||
|
{
|
||||||
|
if (PCM16.Num() == 0) return;
|
||||||
|
|
||||||
|
// Same tail as MulticastReceiveAgentAudio_Implementation, minus the LOD culls:
|
||||||
|
// enqueue for playback (drives body expression + start/stop via bAgentSpeaking) and
|
||||||
|
// broadcast the raw PCM so the LipSync component animates visemes.
|
||||||
|
EnqueueAgentAudio(PCM16);
|
||||||
|
|
||||||
|
// During replay there is no live agent_response message, so the normal
|
||||||
|
// silence-detection stop path (TickComponent) would otherwise wait for the long
|
||||||
|
// hard-timeout. Mark the response as received so OnAgentStoppedSpeaking fires
|
||||||
|
// shortly after the buffer drains and body expression returns to idle.
|
||||||
|
// (Set AFTER EnqueueAgentAudio, which resets this flag on the first chunk.)
|
||||||
|
bAgentResponseReceived = true;
|
||||||
|
|
||||||
|
OnAgentAudioData.Broadcast(PCM16);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_ElevenLabsComponent::StopExternalAudio()
|
||||||
|
{
|
||||||
|
StopAgentAudio();
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_ElevenLabsComponent::TriggerReplayedAction(const FString& ActionName)
|
||||||
|
{
|
||||||
|
PendingActionName = ActionName;
|
||||||
|
OnAgentActionRequested.Broadcast(ActionName);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_ElevenLabsComponent::TriggerReplayedClientToolCall(
|
||||||
|
const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall)
|
||||||
|
{
|
||||||
|
OnAgentClientToolCall.Broadcast(ToolCall);
|
||||||
|
}
|
||||||
|
|
||||||
void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastAgentStartedSpeaking_Implementation()
|
void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastAgentStartedSpeaking_Implementation()
|
||||||
{
|
{
|
||||||
if (GetOwnerRole() == ROLE_Authority) return;
|
if (GetOwnerRole() == ROLE_Authority) return;
|
||||||
|
|||||||
@@ -131,6 +131,12 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnUserVoiceDetected);
|
|||||||
// Delivers PCM chunks as int16, 16kHz mono, little-endian.
|
// Delivers PCM chunks as int16, 16kHz mono, little-endian.
|
||||||
DECLARE_MULTICAST_DELEGATE_OneParam(FOnAgentAudioData, const TArray<uint8>& /*PCMData*/);
|
DECLARE_MULTICAST_DELEGATE_OneParam(FOnAgentAudioData, const TArray<uint8>& /*PCMData*/);
|
||||||
|
|
||||||
|
/** Fired on the SERVER for each player mic chunk relayed via ServerSendMicAudioFromPlayer
|
||||||
|
* (int16, 16kHz mono). Carries the speaking pawn so an external recorder (e.g. the
|
||||||
|
* PS_AI_ConvReplay bridge) can capture player speech for replay. */
|
||||||
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnPlayerAudioCaptured,
|
||||||
|
APawn*, SpeakerPawn, const TArray<uint8>&, PCMData);
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
// UPS_AI_ConvAgent_ElevenLabsComponent
|
// UPS_AI_ConvAgent_ElevenLabsComponent
|
||||||
//
|
//
|
||||||
@@ -399,6 +405,12 @@ public:
|
|||||||
* Used internally by UPS_AI_ConvAgent_LipSyncComponent for spectral analysis. */
|
* Used internally by UPS_AI_ConvAgent_LipSyncComponent for spectral analysis. */
|
||||||
FOnAgentAudioData OnAgentAudioData;
|
FOnAgentAudioData OnAgentAudioData;
|
||||||
|
|
||||||
|
/** Fired on the SERVER for each player mic chunk relayed via ServerSendMicAudioFromPlayer
|
||||||
|
* (int16, 16kHz mono). Lets an external recorder capture player speech for replay.
|
||||||
|
* Not fired on clients — only the server holds the relayed mic stream. */
|
||||||
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events")
|
||||||
|
FOnPlayerAudioCaptured OnPlayerAudioCaptured;
|
||||||
|
|
||||||
// ── Network state (replicated) ───────────────────────────────────────────
|
// ── Network state (replicated) ───────────────────────────────────────────
|
||||||
|
|
||||||
/** True when one or more players are in conversation with this NPC.
|
/** True when one or more players are in conversation with this NPC.
|
||||||
@@ -605,6 +617,28 @@ public:
|
|||||||
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
void InterruptAgent();
|
void InterruptAgent();
|
||||||
|
|
||||||
|
// ── Replay support (PS_AI_ConvReplay bridge) ─────────────────────────────
|
||||||
|
/**
|
||||||
|
* Play an external PCM buffer through this agent's voice, exactly as if it were
|
||||||
|
* live TTS: drives audio playback, lip-sync, body expression and start/stop events.
|
||||||
|
* Role-agnostic (works on a replay/simulated proxy); no LOD culling.
|
||||||
|
* @param PCM16 int16 samples, 16000 Hz mono, little-endian.
|
||||||
|
*/
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Replay")
|
||||||
|
void PlayExternalAudioPCM(const TArray<uint8>& PCM16);
|
||||||
|
|
||||||
|
/** Stop/flush any audio currently playing (used when scrubbing a replay). */
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Replay")
|
||||||
|
void StopExternalAudio();
|
||||||
|
|
||||||
|
/** Re-broadcast a recorded perform_action reaction during replay (fires OnAgentActionRequested). */
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Replay")
|
||||||
|
void TriggerReplayedAction(const FString& ActionName);
|
||||||
|
|
||||||
|
/** Re-broadcast a recorded custom client tool call during replay (fires OnAgentClientToolCall). */
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Replay")
|
||||||
|
void TriggerReplayedClientToolCall(const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Cleanly disable conversation on this agent.
|
* Cleanly disable conversation on this agent.
|
||||||
* Ends the WebSocket connection, stops all audio, and begins blending all
|
* Ends the WebSocket connection, stops all audio, and begins blending all
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
{
|
||||||
|
"FileVersion": 3,
|
||||||
|
"Version": 1,
|
||||||
|
"VersionName": "1.0",
|
||||||
|
"FriendlyName": "PS AI ConvReplay",
|
||||||
|
"Description": "Bridge that records ElevenLabs agent + player conversation audio (and AI reactions) into Unreal replays, and plays it back during replay. Keeps PS_AI_ConvAgent and PS_ReplaySystem independent.",
|
||||||
|
"Category": "ASTERION",
|
||||||
|
"CreatedBy": "ASTERION",
|
||||||
|
"CanContainContent": false,
|
||||||
|
"IsBetaVersion": false,
|
||||||
|
"Installed": false,
|
||||||
|
"EnabledByDefault": false,
|
||||||
|
"Modules": [
|
||||||
|
{
|
||||||
|
"Name": "PS_AI_ConvReplay",
|
||||||
|
"Type": "Runtime",
|
||||||
|
"LoadingPhase": "Default"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"Plugins": [
|
||||||
|
{
|
||||||
|
"Name": "PS_AI_ConvAgent",
|
||||||
|
"Enabled": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Name": "ReplaySystem",
|
||||||
|
"Enabled": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
using UnrealBuildTool;
|
||||||
|
|
||||||
|
public class PS_AI_ConvReplay : ModuleRules
|
||||||
|
{
|
||||||
|
public PS_AI_ConvReplay(ReadOnlyTargetRules Target) : base(Target)
|
||||||
|
{
|
||||||
|
DefaultBuildSettings = BuildSettingsVersion.Latest;
|
||||||
|
PCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs;
|
||||||
|
|
||||||
|
PublicDependencyModuleNames.AddRange(new string[]
|
||||||
|
{
|
||||||
|
"Core",
|
||||||
|
"CoreUObject",
|
||||||
|
"Engine",
|
||||||
|
// Opus voice codec (own encoder/decoder instances — not shared with ConvAgent).
|
||||||
|
"Voice",
|
||||||
|
// The conversational-agent plugin whose delegates we tap and whose audio we replay.
|
||||||
|
"PS_AI_ConvAgent",
|
||||||
|
// Native replay wrapper (record/playback state, demo time, save path).
|
||||||
|
"ReplaySystem",
|
||||||
|
});
|
||||||
|
|
||||||
|
PrivateDependencyModuleNames.AddRange(new string[]
|
||||||
|
{
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#include "PS_AI_ConvReplay_AgentTag.h"
|
||||||
|
#include "PS_AI_ConvReplay_Subsystem.h"
|
||||||
|
#include "GameFramework/Actor.h"
|
||||||
|
|
||||||
|
UPS_AI_ConvReplay_AgentTag::UPS_AI_ConvReplay_AgentTag()
|
||||||
|
{
|
||||||
|
PrimaryComponentTick.bCanEverTick = false;
|
||||||
|
// Exists on server (recording) and on replay clients (identity lookup). It is not
|
||||||
|
// itself replicated — the owner agent already replicates; the tag only needs to be
|
||||||
|
// present on both ends with the same resolved id.
|
||||||
|
SetIsReplicatedByDefault(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
FString UPS_AI_ConvReplay_AgentTag::GetResolvedId() const
|
||||||
|
{
|
||||||
|
if (!AgentReplayId.IsNone())
|
||||||
|
{
|
||||||
|
return AgentReplayId.ToString();
|
||||||
|
}
|
||||||
|
return GetOwner() ? GetOwner()->GetName() : FString();
|
||||||
|
}
|
||||||
|
|
||||||
|
UPS_AI_ConvAgent_ElevenLabsComponent* UPS_AI_ConvReplay_AgentTag::GetAgentComponent() const
|
||||||
|
{
|
||||||
|
if (AActor* Owner = GetOwner())
|
||||||
|
{
|
||||||
|
return Owner->FindComponentByClass<UPS_AI_ConvAgent_ElevenLabsComponent>();
|
||||||
|
}
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_AgentTag::BeginRecording(UPS_AI_ConvReplay_Subsystem* InSubsystem)
|
||||||
|
{
|
||||||
|
Subsystem = InSubsystem;
|
||||||
|
Agent = GetAgentComponent();
|
||||||
|
if (!Agent || bBound)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Agent->OnAgentStartedSpeaking.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleStarted);
|
||||||
|
Agent->OnAgentStoppedSpeaking.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleStopped);
|
||||||
|
Agent->OnAgentActionRequested.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleActionRequested);
|
||||||
|
Agent->OnAgentClientToolCall.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleClientToolCall);
|
||||||
|
Agent->OnAgentTextResponse.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleTextResponse);
|
||||||
|
Agent->OnAgentTranscript.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleTranscript);
|
||||||
|
Agent->OnPlayerAudioCaptured.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandlePlayerAudio);
|
||||||
|
|
||||||
|
// Raw PCM is a non-dynamic C++ multicast delegate.
|
||||||
|
AudioDataHandle = Agent->OnAgentAudioData.AddUObject(this, &UPS_AI_ConvReplay_AgentTag::HandleAgentAudioData);
|
||||||
|
|
||||||
|
bBound = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_AgentTag::EndRecording()
|
||||||
|
{
|
||||||
|
if (Agent && bBound)
|
||||||
|
{
|
||||||
|
Agent->OnAgentStartedSpeaking.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleStarted);
|
||||||
|
Agent->OnAgentStoppedSpeaking.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleStopped);
|
||||||
|
Agent->OnAgentActionRequested.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleActionRequested);
|
||||||
|
Agent->OnAgentClientToolCall.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleClientToolCall);
|
||||||
|
Agent->OnAgentTextResponse.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleTextResponse);
|
||||||
|
Agent->OnAgentTranscript.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleTranscript);
|
||||||
|
Agent->OnPlayerAudioCaptured.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandlePlayerAudio);
|
||||||
|
Agent->OnAgentAudioData.Remove(AudioDataHandle);
|
||||||
|
}
|
||||||
|
AudioDataHandle.Reset();
|
||||||
|
bBound = false;
|
||||||
|
Subsystem = nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_AgentTag::HandleStarted()
|
||||||
|
{
|
||||||
|
if (Subsystem) Subsystem->RecordAgentStarted(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_AgentTag::HandleStopped()
|
||||||
|
{
|
||||||
|
if (Subsystem) Subsystem->RecordAgentStopped(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_AgentTag::HandleActionRequested(const FString& ActionName)
|
||||||
|
{
|
||||||
|
if (Subsystem) Subsystem->RecordAction(this, ActionName);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_AgentTag::HandleClientToolCall(const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall)
|
||||||
|
{
|
||||||
|
if (Subsystem) Subsystem->RecordToolCall(this, ToolCall);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_AgentTag::HandleTextResponse(const FString& Text)
|
||||||
|
{
|
||||||
|
if (Subsystem) Subsystem->RecordAgentText(this, Text);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_AgentTag::HandleTranscript(const FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs& Segment)
|
||||||
|
{
|
||||||
|
// Only final user transcripts become player-text markers.
|
||||||
|
if (Subsystem && Segment.Speaker == TEXT("user") && Segment.bIsFinal)
|
||||||
|
{
|
||||||
|
Subsystem->RecordUserTranscript(this, Segment.Text);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_AgentTag::HandlePlayerAudio(APawn* SpeakerPawn, const TArray<uint8>& PCM)
|
||||||
|
{
|
||||||
|
if (Subsystem) Subsystem->RecordPlayerAudio(SpeakerPawn, PCM);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_AgentTag::HandleAgentAudioData(const TArray<uint8>& PCM)
|
||||||
|
{
|
||||||
|
if (Subsystem) Subsystem->RecordAgentAudio(this, PCM);
|
||||||
|
}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#include "Modules/ModuleManager.h"
|
||||||
|
|
||||||
|
IMPLEMENT_MODULE(FDefaultModuleImpl, PS_AI_ConvReplay);
|
||||||
@@ -0,0 +1,883 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#include "PS_AI_ConvReplay_Subsystem.h"
|
||||||
|
#include "PS_AI_ConvReplay_AgentTag.h"
|
||||||
|
#include "PS_AI_ConvAgent_ElevenLabsComponent.h"
|
||||||
|
#include "ReplaySystemBPLibrary.h"
|
||||||
|
|
||||||
|
#include "VoiceModule.h"
|
||||||
|
#include "Interfaces/VoiceCodec.h"
|
||||||
|
|
||||||
|
#include "Sound/SoundWaveProcedural.h"
|
||||||
|
#include "Components/AudioComponent.h"
|
||||||
|
#include "Kismet/GameplayStatics.h"
|
||||||
|
|
||||||
|
#include "GameFramework/Pawn.h"
|
||||||
|
#include "GameFramework/PlayerState.h"
|
||||||
|
#include "GameFramework/Actor.h"
|
||||||
|
#include "Engine/World.h"
|
||||||
|
#include "EngineUtils.h"
|
||||||
|
|
||||||
|
#include "Stats/Stats.h"
|
||||||
|
#include "Serialization/MemoryWriter.h"
|
||||||
|
#include "Serialization/MemoryReader.h"
|
||||||
|
#include "HAL/PlatformFileManager.h"
|
||||||
|
#include "HAL/IConsoleManager.h"
|
||||||
|
#include "GenericPlatform/GenericPlatformFile.h"
|
||||||
|
#include "Misc/FileHelper.h"
|
||||||
|
#include "Misc/Paths.h"
|
||||||
|
|
||||||
|
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvReplay, Log, All);
|
||||||
|
|
||||||
|
// Live-tunable extra gain on replayed player mic audio (on top of auto-normalization).
|
||||||
|
// e.g. console: ps.convreplay.PlayerGain 1.5
|
||||||
|
static TAutoConsoleVariable<float> CVarPlayerGain(
|
||||||
|
TEXT("ps.convreplay.PlayerGain"),
|
||||||
|
1.0f,
|
||||||
|
TEXT("Extra gain multiplier applied to replayed player mic audio (on top of auto-normalization). Default 1.0."),
|
||||||
|
ECVF_Default);
|
||||||
|
|
||||||
|
namespace PS_AI_ConvReplay_Audio
|
||||||
|
{
|
||||||
|
static constexpr int32 SampleRate = 16000;
|
||||||
|
static constexpr int32 Channels = 1;
|
||||||
|
static constexpr int32 BytesPerFrame = 640; // 320 samples * 2 bytes (20ms @16k mono)
|
||||||
|
static constexpr int32 FramesPerPacket = 40; // matches ConvAgent's live path, under the 76-frame cap
|
||||||
|
static constexpr int32 PacketPCMBytes = BytesPerFrame * FramesPerPacket; // 25600
|
||||||
|
static constexpr int32 WorkBufferBytes = 8 * 1024;
|
||||||
|
static constexpr int32 DecodeBufferBytes = 64 * 1024;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// USubsystem
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::Initialize(FSubsystemCollectionBase& Collection)
|
||||||
|
{
|
||||||
|
Super::Initialize(Collection);
|
||||||
|
bRecording = false;
|
||||||
|
bPlaying = false;
|
||||||
|
LastReplayTime = -1.0;
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::Deinitialize()
|
||||||
|
{
|
||||||
|
if (bRecording) EndRecordingSession();
|
||||||
|
if (bPlaying) EndReplaySession();
|
||||||
|
|
||||||
|
if (WriteHandle)
|
||||||
|
{
|
||||||
|
delete WriteHandle;
|
||||||
|
WriteHandle = nullptr;
|
||||||
|
}
|
||||||
|
Encoder.Reset();
|
||||||
|
Decoder.Reset();
|
||||||
|
|
||||||
|
Super::Deinitialize();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool UPS_AI_ConvReplay_Subsystem::DoesSupportWorldType(const EWorldType::Type WorldType) const
|
||||||
|
{
|
||||||
|
return WorldType == EWorldType::Game || WorldType == EWorldType::PIE;
|
||||||
|
}
|
||||||
|
|
||||||
|
TStatId UPS_AI_ConvReplay_Subsystem::GetStatId() const
|
||||||
|
{
|
||||||
|
RETURN_QUICK_DECLARE_CYCLE_STAT(UPS_AI_ConvReplay_Subsystem, STATGROUP_Tickables);
|
||||||
|
}
|
||||||
|
|
||||||
|
double UPS_AI_ConvReplay_Subsystem::NowDemoSeconds() const
|
||||||
|
{
|
||||||
|
return static_cast<double>(UReplaySystemBPLibrary::GetCurrentReplayTime(GetWorld()));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Tick — drive record / replay based on the current demo state
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::Tick(float DeltaTime)
|
||||||
|
{
|
||||||
|
UWorld* World = GetWorld();
|
||||||
|
if (!World) return;
|
||||||
|
|
||||||
|
const bool bRecNow = UReplaySystemBPLibrary::IsRecordingReplay(World);
|
||||||
|
const bool bPlayNow = UReplaySystemBPLibrary::IsPlayingReplay(World);
|
||||||
|
|
||||||
|
if (bRecNow && !bRecording) BeginRecordingSession();
|
||||||
|
else if (!bRecNow && bRecording) EndRecordingSession();
|
||||||
|
|
||||||
|
if (bPlayNow && !bPlaying) BeginReplaySession();
|
||||||
|
else if (!bPlayNow && bPlaying) EndReplaySession();
|
||||||
|
|
||||||
|
if (bRecording) TickRecording();
|
||||||
|
if (bPlaying) TickReplay();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Agent discovery
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::RebuildTagList()
|
||||||
|
{
|
||||||
|
Tags.Reset();
|
||||||
|
UWorld* World = GetWorld();
|
||||||
|
if (!World) return;
|
||||||
|
|
||||||
|
for (TActorIterator<AActor> It(World); It; ++It)
|
||||||
|
{
|
||||||
|
AActor* Actor = *It;
|
||||||
|
if (!Actor || !Actor->FindComponentByClass<UPS_AI_ConvAgent_ElevenLabsComponent>())
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
UPS_AI_ConvReplay_AgentTag* Tag = Actor->FindComponentByClass<UPS_AI_ConvReplay_AgentTag>();
|
||||||
|
if (!Tag)
|
||||||
|
{
|
||||||
|
// Auto-provision a tag (id = actor name) so agents work with zero setup.
|
||||||
|
Tag = NewObject<UPS_AI_ConvReplay_AgentTag>(Actor);
|
||||||
|
if (Tag)
|
||||||
|
{
|
||||||
|
Tag->RegisterComponent();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (Tag)
|
||||||
|
{
|
||||||
|
Tags.AddUnique(Tag);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
UPS_AI_ConvReplay_AgentTag* UPS_AI_ConvReplay_Subsystem::FindTagById(const FString& Id)
|
||||||
|
{
|
||||||
|
for (UPS_AI_ConvReplay_AgentTag* Tag : Tags)
|
||||||
|
{
|
||||||
|
if (Tag && Tag->GetResolvedId() == Id)
|
||||||
|
{
|
||||||
|
return Tag;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Lazy rescan in case agents spawned after the session started.
|
||||||
|
RebuildTagList();
|
||||||
|
for (UPS_AI_ConvReplay_AgentTag* Tag : Tags)
|
||||||
|
{
|
||||||
|
if (Tag && Tag->GetResolvedId() == Id)
|
||||||
|
{
|
||||||
|
return Tag;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
UPS_AI_ConvReplay_AgentTag* UPS_AI_ConvReplay_Subsystem::FindAgentForRecord(const FString& Id, const FVector& Pos)
|
||||||
|
{
|
||||||
|
// Exact id first (FindTagById also (re)populates Tags on a miss).
|
||||||
|
if (UPS_AI_ConvReplay_AgentTag* Exact = FindTagById(Id))
|
||||||
|
{
|
||||||
|
return Exact;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback: nearest agent to the recorded utterance position. Runtime-spawned
|
||||||
|
// agents get a different actor-name suffix on replay, so the id won't match —
|
||||||
|
// but the demo replays their transform, so the speaker is at (about) the same
|
||||||
|
// world position when this record fires.
|
||||||
|
UPS_AI_ConvReplay_AgentTag* Best = nullptr;
|
||||||
|
double BestDistSq = TNumericLimits<double>::Max();
|
||||||
|
for (UPS_AI_ConvReplay_AgentTag* Tag : Tags)
|
||||||
|
{
|
||||||
|
if (!Tag) continue;
|
||||||
|
if (const AActor* Owner = Tag->GetOwner())
|
||||||
|
{
|
||||||
|
const double D = FVector::DistSquared(Owner->GetActorLocation(), Pos);
|
||||||
|
if (D < BestDistSq)
|
||||||
|
{
|
||||||
|
BestDistSq = D;
|
||||||
|
Best = Tag;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (Best)
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvReplay, Verbose,
|
||||||
|
TEXT("Agent id '%s' not found by name; matched nearest agent '%s' (%.0f cm) to recorded position."),
|
||||||
|
*Id, *Best->GetResolvedId(), FMath::Sqrt(BestDistSq));
|
||||||
|
}
|
||||||
|
return Best;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Recording session
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::BeginRecordingSession()
|
||||||
|
{
|
||||||
|
bRecording = true;
|
||||||
|
AgentAccum.Reset();
|
||||||
|
PlayerAccum.Reset();
|
||||||
|
|
||||||
|
const FString Path = GetSidecarPath();
|
||||||
|
if (Path.IsEmpty())
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvReplay, Warning,
|
||||||
|
TEXT("Recording started but no active replay name — conversation track disabled."));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
IPlatformFile& PF = FPlatformFileManager::Get().GetPlatformFile();
|
||||||
|
PF.CreateDirectoryTree(*FPaths::GetPath(Path));
|
||||||
|
if (WriteHandle) { delete WriteHandle; WriteHandle = nullptr; }
|
||||||
|
WriteHandle = PF.OpenWrite(*Path, /*bAppend=*/false, /*bAllowRead=*/true);
|
||||||
|
|
||||||
|
if (!WriteHandle)
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvReplay, Warning, TEXT("Could not open sidecar for write: %s"), *Path);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvReplay, Log, TEXT("Recording conversation track → %s"), *Path);
|
||||||
|
}
|
||||||
|
|
||||||
|
RebuildTagList();
|
||||||
|
for (UPS_AI_ConvReplay_AgentTag* Tag : Tags)
|
||||||
|
{
|
||||||
|
if (Tag) Tag->BeginRecording(this);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::EndRecordingSession()
|
||||||
|
{
|
||||||
|
// Flush any player segment still open.
|
||||||
|
TArray<TWeakObjectPtr<APawn>> Pending;
|
||||||
|
PlayerAccum.GetKeys(Pending);
|
||||||
|
for (const TWeakObjectPtr<APawn>& P : Pending)
|
||||||
|
{
|
||||||
|
FlushPlayerSegment(P.Get());
|
||||||
|
}
|
||||||
|
|
||||||
|
for (UPS_AI_ConvReplay_AgentTag* Tag : Tags)
|
||||||
|
{
|
||||||
|
if (Tag) Tag->EndRecording();
|
||||||
|
}
|
||||||
|
|
||||||
|
AgentAccum.Reset();
|
||||||
|
PlayerAccum.Reset();
|
||||||
|
|
||||||
|
if (WriteHandle)
|
||||||
|
{
|
||||||
|
WriteHandle->Flush();
|
||||||
|
delete WriteHandle;
|
||||||
|
WriteHandle = nullptr;
|
||||||
|
}
|
||||||
|
bRecording = false;
|
||||||
|
UE_LOG(LogPS_AI_ConvReplay, Log, TEXT("Conversation track recording stopped."));
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::TickRecording()
|
||||||
|
{
|
||||||
|
// Flush player segments that have gone silent.
|
||||||
|
const double Now = NowDemoSeconds();
|
||||||
|
TArray<TWeakObjectPtr<APawn>> ToFlush;
|
||||||
|
for (TPair<TWeakObjectPtr<APawn>, FPlayerAccum>& Pair : PlayerAccum)
|
||||||
|
{
|
||||||
|
FPlayerAccum& A = Pair.Value;
|
||||||
|
if (A.bActive && A.PCM.Num() > 0 && (Now - A.LastVoiceTime) > PlayerSilenceFlushSec)
|
||||||
|
{
|
||||||
|
ToFlush.Add(Pair.Key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const TWeakObjectPtr<APawn>& P : ToFlush)
|
||||||
|
{
|
||||||
|
FlushPlayerSegment(P.Get());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Record callbacks ─────────────────────────────────────────────────────────
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::RecordAgentStarted(UPS_AI_ConvReplay_AgentTag* Tag)
|
||||||
|
{
|
||||||
|
if (!bRecording || !Tag) return;
|
||||||
|
FAgentAccum& A = AgentAccum.FindOrAdd(Tag);
|
||||||
|
A.StartTime = NowDemoSeconds();
|
||||||
|
A.PCM.Reset();
|
||||||
|
A.bOpen = true;
|
||||||
|
if (AActor* Owner = Tag->GetOwner())
|
||||||
|
{
|
||||||
|
A.Pos = Owner->GetActorLocation();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::RecordAgentAudio(UPS_AI_ConvReplay_AgentTag* Tag, const TArray<uint8>& PCM)
|
||||||
|
{
|
||||||
|
if (!bRecording || !Tag || PCM.Num() == 0) return;
|
||||||
|
FAgentAccum& A = AgentAccum.FindOrAdd(Tag);
|
||||||
|
if (!A.bOpen)
|
||||||
|
{
|
||||||
|
// Audio before an explicit start — open a segment defensively.
|
||||||
|
A.StartTime = NowDemoSeconds();
|
||||||
|
A.bOpen = true;
|
||||||
|
if (AActor* Owner = Tag->GetOwner()) A.Pos = Owner->GetActorLocation();
|
||||||
|
}
|
||||||
|
A.PCM.Append(PCM);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::RecordAgentText(UPS_AI_ConvReplay_AgentTag* Tag, const FString& Text)
|
||||||
|
{
|
||||||
|
if (!bRecording || !Tag) return;
|
||||||
|
AgentAccum.FindOrAdd(Tag).Text = Text;
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::RecordAgentStopped(UPS_AI_ConvReplay_AgentTag* Tag)
|
||||||
|
{
|
||||||
|
if (!bRecording || !Tag) return;
|
||||||
|
FAgentAccum* A = AgentAccum.Find(Tag);
|
||||||
|
if (!A || !A->bOpen || A->PCM.Num() == 0)
|
||||||
|
{
|
||||||
|
if (A) { A->bOpen = false; A->PCM.Reset(); A->Text.Reset(); }
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
FPS_AI_ConvReplay_Record R;
|
||||||
|
R.Kind = static_cast<uint8>(EPS_AI_ConvReplay_Kind::AgentSpeech);
|
||||||
|
R.TimeSec = A->StartTime;
|
||||||
|
R.DurationSec = static_cast<double>(A->PCM.Num() / 2) / PS_AI_ConvReplay_Audio::SampleRate;
|
||||||
|
R.Id = Tag->GetResolvedId();
|
||||||
|
R.Speaker = 0;
|
||||||
|
R.Text = A->Text;
|
||||||
|
R.Position = A->Pos;
|
||||||
|
EncodeUtterance(A->PCM, R.Audio);
|
||||||
|
WriteRecord(R);
|
||||||
|
|
||||||
|
A->bOpen = false;
|
||||||
|
A->PCM.Reset();
|
||||||
|
A->Text.Reset();
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::RecordAction(UPS_AI_ConvReplay_AgentTag* Tag, const FString& ActionName)
|
||||||
|
{
|
||||||
|
if (!bRecording || !Tag) return;
|
||||||
|
FPS_AI_ConvReplay_Record R;
|
||||||
|
R.Kind = static_cast<uint8>(EPS_AI_ConvReplay_Kind::Action);
|
||||||
|
R.TimeSec = NowDemoSeconds();
|
||||||
|
R.Id = Tag->GetResolvedId();
|
||||||
|
R.Speaker = 0;
|
||||||
|
R.Text = ActionName;
|
||||||
|
if (const AActor* Owner = Tag->GetOwner()) R.Position = Owner->GetActorLocation();
|
||||||
|
WriteRecord(R);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::RecordToolCall(UPS_AI_ConvReplay_AgentTag* Tag,
|
||||||
|
const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall)
|
||||||
|
{
|
||||||
|
if (!bRecording || !Tag) return;
|
||||||
|
FPS_AI_ConvReplay_Record R;
|
||||||
|
R.Kind = static_cast<uint8>(EPS_AI_ConvReplay_Kind::ToolCall);
|
||||||
|
R.TimeSec = NowDemoSeconds();
|
||||||
|
R.Id = Tag->GetResolvedId();
|
||||||
|
R.Speaker = 0;
|
||||||
|
R.Text = ToolCall.ToolName;
|
||||||
|
R.Params = ToolCall.Parameters;
|
||||||
|
R.Params.Add(TEXT("__toolCallId"), ToolCall.ToolCallId); // stashed so we can rebuild the struct on replay
|
||||||
|
if (const AActor* Owner = Tag->GetOwner()) R.Position = Owner->GetActorLocation();
|
||||||
|
WriteRecord(R);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::RecordUserTranscript(UPS_AI_ConvReplay_AgentTag* Tag, const FString& Text)
|
||||||
|
{
|
||||||
|
if (!bRecording || !Tag || Text.IsEmpty()) return;
|
||||||
|
FPS_AI_ConvReplay_Record R;
|
||||||
|
R.Kind = static_cast<uint8>(EPS_AI_ConvReplay_Kind::PlayerText);
|
||||||
|
R.TimeSec = NowDemoSeconds();
|
||||||
|
R.Speaker = 1;
|
||||||
|
R.Text = Text;
|
||||||
|
if (UPS_AI_ConvAgent_ElevenLabsComponent* Comp = Tag->GetAgentComponent())
|
||||||
|
{
|
||||||
|
if (APawn* SP = Comp->NetActiveSpeakerPawn)
|
||||||
|
{
|
||||||
|
R.Id = ResolvePlayerId(SP);
|
||||||
|
R.Position = SP->GetActorLocation();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
WriteRecord(R);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::RecordPlayerAudio(APawn* SpeakerPawn, const TArray<uint8>& PCM)
|
||||||
|
{
|
||||||
|
if (!bRecording || !SpeakerPawn || PCM.Num() == 0) return;
|
||||||
|
|
||||||
|
const bool bVoiced = IsVoiced(PCM);
|
||||||
|
FPlayerAccum& A = PlayerAccum.FindOrAdd(SpeakerPawn);
|
||||||
|
const double Now = NowDemoSeconds();
|
||||||
|
|
||||||
|
if (!A.bActive)
|
||||||
|
{
|
||||||
|
if (!bVoiced) return; // don't open a segment on ambient/silence
|
||||||
|
A.bActive = true;
|
||||||
|
A.StartTime = Now;
|
||||||
|
A.LastVoiceTime = Now;
|
||||||
|
A.Pos = SpeakerPawn->GetActorLocation();
|
||||||
|
A.Id = ResolvePlayerId(SpeakerPawn);
|
||||||
|
A.PCM.Reset();
|
||||||
|
}
|
||||||
|
|
||||||
|
A.PCM.Append(PCM);
|
||||||
|
if (bVoiced)
|
||||||
|
{
|
||||||
|
A.LastVoiceTime = Now;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::FlushPlayerSegment(APawn* Pawn)
|
||||||
|
{
|
||||||
|
FPlayerAccum* A = PlayerAccum.Find(Pawn);
|
||||||
|
if (!A) return;
|
||||||
|
if (A->bActive && A->PCM.Num() > 0)
|
||||||
|
{
|
||||||
|
FPS_AI_ConvReplay_Record R;
|
||||||
|
R.Kind = static_cast<uint8>(EPS_AI_ConvReplay_Kind::PlayerSpeech);
|
||||||
|
R.TimeSec = A->StartTime;
|
||||||
|
R.DurationSec = static_cast<double>(A->PCM.Num() / 2) / PS_AI_ConvReplay_Audio::SampleRate;
|
||||||
|
R.Id = A->Id;
|
||||||
|
R.Speaker = 1;
|
||||||
|
R.Position = A->Pos;
|
||||||
|
EncodeUtterance(A->PCM, R.Audio);
|
||||||
|
WriteRecord(R);
|
||||||
|
}
|
||||||
|
A->bActive = false;
|
||||||
|
A->PCM.Reset();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Replay session
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::BeginReplaySession()
|
||||||
|
{
|
||||||
|
bPlaying = true;
|
||||||
|
LastReplayTime = -1.0;
|
||||||
|
PlayerAudioPool.Reset();
|
||||||
|
RebuildTagList();
|
||||||
|
LoadRecords();
|
||||||
|
UE_LOG(LogPS_AI_ConvReplay, Log, TEXT("Replaying conversation track: %d records."), Records.Num());
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::EndReplaySession()
|
||||||
|
{
|
||||||
|
for (UAudioComponent* AC : PlayerAudioPool)
|
||||||
|
{
|
||||||
|
if (AC) AC->Stop();
|
||||||
|
}
|
||||||
|
PlayerAudioPool.Reset();
|
||||||
|
Records.Reset();
|
||||||
|
bPlaying = false;
|
||||||
|
LastReplayTime = -1.0;
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::TickReplay()
|
||||||
|
{
|
||||||
|
UWorld* World = GetWorld();
|
||||||
|
if (!World) return;
|
||||||
|
|
||||||
|
const double Now = static_cast<double>(UReplaySystemBPLibrary::GetCurrentReplayTime(World));
|
||||||
|
|
||||||
|
// Scrub detection: a jump backward, or a large forward jump, resets playback.
|
||||||
|
if (LastReplayTime >= 0.0)
|
||||||
|
{
|
||||||
|
const double Delta = Now - LastReplayTime;
|
||||||
|
if (Delta < -static_cast<double>(ScrubBackThreshold) || Delta > static_cast<double>(ScrubFwdThreshold))
|
||||||
|
{
|
||||||
|
ResetPlaybackFrom(Now);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
LastReplayTime = Now;
|
||||||
|
|
||||||
|
const float Speed = UReplaySystemBPLibrary::GetPlaybackSpeed(World);
|
||||||
|
const bool bAudioAllowed = FMath::IsNearlyEqual(Speed, 1.0f, NormalSpeedTolerance);
|
||||||
|
DispatchDueRecords(Now, bAudioAllowed);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::DispatchDueRecords(double NowSec, bool bAudioAllowed)
|
||||||
|
{
|
||||||
|
for (FPS_AI_ConvReplay_Record& R : Records)
|
||||||
|
{
|
||||||
|
if (R.bPlayed || R.TimeSec > NowSec)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
R.bPlayed = true;
|
||||||
|
|
||||||
|
UE_LOG(LogPS_AI_ConvReplay, Log,
|
||||||
|
TEXT("Dispatch kind=%d id='%s' t=%.2f (now=%.2f) audio=%dB speed-allowed=%d"),
|
||||||
|
R.Kind, *R.Id, R.TimeSec, NowSec, R.Audio.Num(), bAudioAllowed ? 1 : 0);
|
||||||
|
|
||||||
|
OnReplayLinePlayed.Broadcast(
|
||||||
|
FName(*R.Id),
|
||||||
|
R.Speaker == 1 ? TEXT("user") : TEXT("agent"),
|
||||||
|
R.Text,
|
||||||
|
static_cast<float>(R.TimeSec));
|
||||||
|
|
||||||
|
switch (static_cast<EPS_AI_ConvReplay_Kind>(R.Kind))
|
||||||
|
{
|
||||||
|
case EPS_AI_ConvReplay_Kind::AgentSpeech:
|
||||||
|
if (bAudioAllowed && R.Audio.Num() > 0)
|
||||||
|
{
|
||||||
|
UPS_AI_ConvReplay_AgentTag* Tag = FindAgentForRecord(R.Id, R.Position);
|
||||||
|
UPS_AI_ConvAgent_ElevenLabsComponent* Comp = Tag ? Tag->GetAgentComponent() : nullptr;
|
||||||
|
TArray<uint8> PCM;
|
||||||
|
const bool bDecoded = Comp && DecodeUtterance(R.Audio, PCM);
|
||||||
|
if (bDecoded)
|
||||||
|
{
|
||||||
|
Comp->PlayExternalAudioPCM(PCM);
|
||||||
|
UE_LOG(LogPS_AI_ConvReplay, Log,
|
||||||
|
TEXT(" agent speech → '%s' : played %d PCM bytes."),
|
||||||
|
*Tag->GetResolvedId(), PCM.Num());
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvReplay, Warning,
|
||||||
|
TEXT(" agent speech NOT played (agent=%s decoded=%d) — %d agent tags known."),
|
||||||
|
Comp ? TEXT("found") : TEXT("NOT FOUND"), PCM.Num(), Tags.Num());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case EPS_AI_ConvReplay_Kind::PlayerSpeech:
|
||||||
|
if (bAudioAllowed && R.Audio.Num() > 0)
|
||||||
|
{
|
||||||
|
TArray<uint8> PCM;
|
||||||
|
if (DecodeUtterance(R.Audio, PCM))
|
||||||
|
{
|
||||||
|
PlayPlayerAudioAtLocation(R.Position, PCM);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case EPS_AI_ConvReplay_Kind::Action:
|
||||||
|
if (UPS_AI_ConvReplay_AgentTag* Tag = FindAgentForRecord(R.Id, R.Position))
|
||||||
|
{
|
||||||
|
if (UPS_AI_ConvAgent_ElevenLabsComponent* Comp = Tag->GetAgentComponent())
|
||||||
|
{
|
||||||
|
Comp->TriggerReplayedAction(R.Text);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case EPS_AI_ConvReplay_Kind::ToolCall:
|
||||||
|
if (UPS_AI_ConvReplay_AgentTag* Tag = FindAgentForRecord(R.Id, R.Position))
|
||||||
|
{
|
||||||
|
if (UPS_AI_ConvAgent_ElevenLabsComponent* Comp = Tag->GetAgentComponent())
|
||||||
|
{
|
||||||
|
FPS_AI_ConvAgent_ClientToolCall_ElevenLabs TC;
|
||||||
|
TC.ToolName = R.Text;
|
||||||
|
TC.Parameters = R.Params;
|
||||||
|
if (FString* Cid = TC.Parameters.Find(TEXT("__toolCallId")))
|
||||||
|
{
|
||||||
|
TC.ToolCallId = *Cid;
|
||||||
|
TC.Parameters.Remove(TEXT("__toolCallId"));
|
||||||
|
}
|
||||||
|
Comp->TriggerReplayedClientToolCall(TC);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case EPS_AI_ConvReplay_Kind::PlayerText:
|
||||||
|
default:
|
||||||
|
break; // subtitle only (already broadcast above)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::ResetPlaybackFrom(double NowSec)
|
||||||
|
{
|
||||||
|
for (FPS_AI_ConvReplay_Record& R : Records)
|
||||||
|
{
|
||||||
|
// Records at/behind the new time are considered already elapsed (skipped),
|
||||||
|
// records ahead of it will fire as time advances.
|
||||||
|
R.bPlayed = (R.TimeSec <= NowSec);
|
||||||
|
}
|
||||||
|
for (UPS_AI_ConvReplay_AgentTag* Tag : Tags)
|
||||||
|
{
|
||||||
|
if (Tag)
|
||||||
|
{
|
||||||
|
if (UPS_AI_ConvAgent_ElevenLabsComponent* Comp = Tag->GetAgentComponent())
|
||||||
|
{
|
||||||
|
Comp->StopExternalAudio();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (UAudioComponent* AC : PlayerAudioPool)
|
||||||
|
{
|
||||||
|
if (AC) AC->Stop();
|
||||||
|
}
|
||||||
|
PlayerAudioPool.Reset();
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::PlayPlayerAudioAtLocation(const FVector& Loc, const TArray<uint8>& PCM)
|
||||||
|
{
|
||||||
|
UWorld* World = GetWorld();
|
||||||
|
if (!World || PCM.Num() == 0) return;
|
||||||
|
|
||||||
|
// Drop finished/dead components from the pool.
|
||||||
|
PlayerAudioPool.RemoveAll([](const TObjectPtr<UAudioComponent>& AC)
|
||||||
|
{
|
||||||
|
return AC == nullptr || !AC->IsPlaying();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Mic capture is often quiet — normalize this utterance to a consistent level.
|
||||||
|
TArray<uint8> Boosted = PCM;
|
||||||
|
NormalizePlayerPcm(Boosted);
|
||||||
|
|
||||||
|
USoundWaveProcedural* Wave = NewObject<USoundWaveProcedural>(this);
|
||||||
|
Wave->SetSampleRate(PS_AI_ConvReplay_Audio::SampleRate);
|
||||||
|
Wave->NumChannels = PS_AI_ConvReplay_Audio::Channels;
|
||||||
|
Wave->Duration = static_cast<float>(Boosted.Num() / 2) / PS_AI_ConvReplay_Audio::SampleRate;
|
||||||
|
Wave->SoundGroup = SOUNDGROUP_Voice;
|
||||||
|
Wave->bLooping = false;
|
||||||
|
Wave->QueueAudio(Boosted.GetData(), Boosted.Num());
|
||||||
|
|
||||||
|
UAudioComponent* AC = UGameplayStatics::SpawnSoundAtLocation(World, Wave, Loc);
|
||||||
|
if (AC)
|
||||||
|
{
|
||||||
|
PlayerAudioPool.Add(AC);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Sidecar IO
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
FString UPS_AI_ConvReplay_Subsystem::GetSidecarPath() const
|
||||||
|
{
|
||||||
|
UWorld* World = GetWorld();
|
||||||
|
if (!World) return FString();
|
||||||
|
|
||||||
|
FString Name = UReplaySystemBPLibrary::GetActiveReplayName(World);
|
||||||
|
if (Name.IsEmpty() || Name == TEXT("None"))
|
||||||
|
{
|
||||||
|
return FString();
|
||||||
|
}
|
||||||
|
|
||||||
|
FString Dir = UReplaySystemBPLibrary::GetReplaySavePath(World);
|
||||||
|
if (Dir.IsEmpty())
|
||||||
|
{
|
||||||
|
Dir = FPaths::Combine(FPaths::ProjectSavedDir(), TEXT("Demos"));
|
||||||
|
}
|
||||||
|
return FPaths::Combine(Dir, Name + TEXT(".convreplay"));
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::WriteRecord(FPS_AI_ConvReplay_Record& Rec)
|
||||||
|
{
|
||||||
|
if (!WriteHandle) return;
|
||||||
|
TArray<uint8> Blob;
|
||||||
|
FMemoryWriter Writer(Blob, /*bIsPersistent=*/true);
|
||||||
|
Writer << Rec;
|
||||||
|
WriteHandle->Write(Blob.GetData(), Blob.Num());
|
||||||
|
WriteHandle->Flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::LoadRecords()
|
||||||
|
{
|
||||||
|
Records.Reset();
|
||||||
|
const FString Path = GetSidecarPath();
|
||||||
|
if (Path.IsEmpty()) return;
|
||||||
|
|
||||||
|
TArray<uint8> Data;
|
||||||
|
if (!FFileHelper::LoadFileToArray(Data, *Path))
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvReplay, Log, TEXT("No conversation track found at %s."), *Path);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
FMemoryReader Reader(Data, /*bIsPersistent=*/true);
|
||||||
|
while (!Reader.AtEnd())
|
||||||
|
{
|
||||||
|
FPS_AI_ConvReplay_Record R;
|
||||||
|
Reader << R;
|
||||||
|
Records.Add(MoveTemp(R));
|
||||||
|
}
|
||||||
|
|
||||||
|
Records.Sort([](const FPS_AI_ConvReplay_Record& A, const FPS_AI_ConvReplay_Record& B)
|
||||||
|
{
|
||||||
|
return A.TimeSec < B.TimeSec;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Opus codec (own instances)
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::EnsureCodec()
|
||||||
|
{
|
||||||
|
if (Encoder.IsValid() && Decoder.IsValid()) return;
|
||||||
|
if (!FVoiceModule::IsAvailable())
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvReplay, Warning,
|
||||||
|
TEXT("FVoiceModule not available — conversation audio cannot be (de)compressed. Check [Voice] bEnabled=true."));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
FVoiceModule& VM = FVoiceModule::Get();
|
||||||
|
if (!Encoder.IsValid())
|
||||||
|
{
|
||||||
|
Encoder = VM.CreateVoiceEncoder(PS_AI_ConvReplay_Audio::SampleRate,
|
||||||
|
PS_AI_ConvReplay_Audio::Channels, EAudioEncodeHint::VoiceEncode_Voice);
|
||||||
|
}
|
||||||
|
if (!Decoder.IsValid())
|
||||||
|
{
|
||||||
|
Decoder = VM.CreateVoiceDecoder(PS_AI_ConvReplay_Audio::SampleRate,
|
||||||
|
PS_AI_ConvReplay_Audio::Channels);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool UPS_AI_ConvReplay_Subsystem::EncodeUtterance(const TArray<uint8>& PCM, TArray<uint8>& OutPackets)
|
||||||
|
{
|
||||||
|
OutPackets.Reset();
|
||||||
|
EnsureCodec();
|
||||||
|
if (!Encoder.IsValid() || PCM.Num() == 0) return false;
|
||||||
|
|
||||||
|
using namespace PS_AI_ConvReplay_Audio;
|
||||||
|
|
||||||
|
Encoder->Reset(); // make this utterance self-contained (independent of the previous one)
|
||||||
|
|
||||||
|
// Pad to a whole number of Opus frames (the encoder only consumes full frames).
|
||||||
|
TArray<uint8> In = PCM;
|
||||||
|
const int32 Rem = In.Num() % BytesPerFrame;
|
||||||
|
if (Rem != 0)
|
||||||
|
{
|
||||||
|
In.AddZeroed(BytesPerFrame - Rem);
|
||||||
|
}
|
||||||
|
|
||||||
|
TArray<uint8> Work;
|
||||||
|
Work.SetNumUninitialized(WorkBufferBytes);
|
||||||
|
|
||||||
|
FMemoryWriter Writer(OutPackets, /*bIsPersistent=*/true);
|
||||||
|
int32 Offset = 0;
|
||||||
|
while (Offset < In.Num())
|
||||||
|
{
|
||||||
|
const int32 Block = FMath::Min(PacketPCMBytes, In.Num() - Offset);
|
||||||
|
uint32 CompSize = static_cast<uint32>(Work.Num());
|
||||||
|
Encoder->Encode(In.GetData() + Offset, static_cast<uint32>(Block), Work.GetData(), CompSize);
|
||||||
|
if (CompSize > 0)
|
||||||
|
{
|
||||||
|
int32 Len = static_cast<int32>(CompSize);
|
||||||
|
Writer << Len;
|
||||||
|
Writer.Serialize(Work.GetData(), Len);
|
||||||
|
}
|
||||||
|
Offset += Block;
|
||||||
|
}
|
||||||
|
return OutPackets.Num() > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool UPS_AI_ConvReplay_Subsystem::DecodeUtterance(const TArray<uint8>& Packets, TArray<uint8>& OutPCM)
|
||||||
|
{
|
||||||
|
OutPCM.Reset();
|
||||||
|
EnsureCodec();
|
||||||
|
if (!Decoder.IsValid() || Packets.Num() == 0) return false;
|
||||||
|
|
||||||
|
using namespace PS_AI_ConvReplay_Audio;
|
||||||
|
|
||||||
|
Decoder->Reset();
|
||||||
|
|
||||||
|
TArray<uint8> Frame;
|
||||||
|
Frame.SetNumUninitialized(DecodeBufferBytes);
|
||||||
|
|
||||||
|
FMemoryReader Reader(Packets, /*bIsPersistent=*/true);
|
||||||
|
while (!Reader.AtEnd())
|
||||||
|
{
|
||||||
|
int32 Len = 0;
|
||||||
|
Reader << Len;
|
||||||
|
if (Len <= 0 || Reader.Tell() + Len > Packets.Num())
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
TArray<uint8> Packet;
|
||||||
|
Packet.SetNumUninitialized(Len);
|
||||||
|
Reader.Serialize(Packet.GetData(), Len);
|
||||||
|
|
||||||
|
uint32 OutSize = static_cast<uint32>(Frame.Num());
|
||||||
|
Decoder->Decode(Packet.GetData(), static_cast<uint32>(Len), Frame.GetData(), OutSize);
|
||||||
|
if (OutSize > 0)
|
||||||
|
{
|
||||||
|
OutPCM.Append(Frame.GetData(), OutSize);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return OutPCM.Num() > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Helpers
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
bool UPS_AI_ConvReplay_Subsystem::IsVoiced(const TArray<uint8>& PCM16)
|
||||||
|
{
|
||||||
|
const int32 NumSamples = PCM16.Num() / static_cast<int32>(sizeof(int16));
|
||||||
|
if (NumSamples <= 0) return false;
|
||||||
|
const int16* S = reinterpret_cast<const int16*>(PCM16.GetData());
|
||||||
|
double SumSq = 0.0;
|
||||||
|
for (int32 i = 0; i < NumSamples; ++i)
|
||||||
|
{
|
||||||
|
const double v = S[i] / 32768.0;
|
||||||
|
SumSq += v * v;
|
||||||
|
}
|
||||||
|
const double Rms = FMath::Sqrt(SumSq / NumSamples);
|
||||||
|
return Rms >= VoicedRmsThreshold;
|
||||||
|
}
|
||||||
|
|
||||||
|
FString UPS_AI_ConvReplay_Subsystem::ResolvePlayerId(APawn* Pawn)
|
||||||
|
{
|
||||||
|
// Player audio is replayed at its recorded world position, so this id is only a
|
||||||
|
// human-readable label for subtitles — the player name (or pawn name) is enough,
|
||||||
|
// and avoids pulling in the online-subsystem net-id types.
|
||||||
|
if (!Pawn) return TEXT("player");
|
||||||
|
if (APlayerState* PS = Pawn->GetPlayerState())
|
||||||
|
{
|
||||||
|
const FString PlayerName = PS->GetPlayerName();
|
||||||
|
if (!PlayerName.IsEmpty())
|
||||||
|
{
|
||||||
|
return PlayerName;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Pawn->GetName();
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_Subsystem::NormalizePlayerPcm(TArray<uint8>& Pcm16)
|
||||||
|
{
|
||||||
|
const int32 N = Pcm16.Num() / static_cast<int32>(sizeof(int16));
|
||||||
|
if (N <= 0) return;
|
||||||
|
|
||||||
|
int16* S = reinterpret_cast<int16*>(Pcm16.GetData());
|
||||||
|
|
||||||
|
// Measure loudness (RMS), peak, and how many samples are already at/near full-scale
|
||||||
|
// (a signature of clipping AT CAPTURE — which no playback processing can undo).
|
||||||
|
double SumSq = 0.0;
|
||||||
|
int32 Peak = 0;
|
||||||
|
int32 ClippedIn = 0;
|
||||||
|
for (int32 i = 0; i < N; ++i)
|
||||||
|
{
|
||||||
|
const int32 A = FMath::Abs(static_cast<int32>(S[i]));
|
||||||
|
Peak = FMath::Max(Peak, A);
|
||||||
|
if (A >= 32000) ++ClippedIn;
|
||||||
|
const double v = S[i] / 32768.0;
|
||||||
|
SumSq += v * v;
|
||||||
|
}
|
||||||
|
if (Peak == 0) return;
|
||||||
|
const double Rms = FMath::Sqrt(SumSq / N);
|
||||||
|
|
||||||
|
// Bidirectional AGC: bring each utterance to a consistent target RMS — BOOST quiet
|
||||||
|
// ones and ATTENUATE loud ones — so speaking loud no longer runs hot. Then the live CVar.
|
||||||
|
double Gain = (Rms > 1e-5) ? (PlayerTargetRms / Rms) : 1.0;
|
||||||
|
Gain = FMath::Clamp(Gain, static_cast<double>(PlayerMinGain), static_cast<double>(PlayerMaxGain));
|
||||||
|
Gain *= FMath::Max(0.0f, CVarPlayerGain.GetValueOnGameThread());
|
||||||
|
|
||||||
|
// Soft-knee limiter on the output: samples above the threshold are smoothly
|
||||||
|
// compressed toward full-scale (tanh) instead of hard-clipped, so residual peaks
|
||||||
|
// (or a hot CVar) don't produce harsh saturation.
|
||||||
|
const double T = static_cast<double>(PlayerSoftClipThreshold);
|
||||||
|
int32 ClippedOut = 0;
|
||||||
|
for (int32 i = 0; i < N; ++i)
|
||||||
|
{
|
||||||
|
double x = (S[i] / 32768.0) * Gain;
|
||||||
|
const double a = FMath::Abs(x);
|
||||||
|
if (a > T)
|
||||||
|
{
|
||||||
|
const double s = (x < 0.0) ? -1.0 : 1.0;
|
||||||
|
x = s * (T + (1.0 - T) * FMath::Tanh((a - T) / (1.0 - T)));
|
||||||
|
++ClippedOut;
|
||||||
|
}
|
||||||
|
const int32 v = FMath::RoundToInt(x * 32767.0);
|
||||||
|
S[i] = static_cast<int16>(FMath::Clamp(v, -32768, 32767));
|
||||||
|
}
|
||||||
|
|
||||||
|
UE_LOG(LogPS_AI_ConvReplay, Log,
|
||||||
|
TEXT("Player utterance: rms=%.3f peak=%d/32767 clipped-in=%d/%d gain=%.2f soft-limited=%d"),
|
||||||
|
Rms, Peak, ClippedIn, N, Gain, ClippedOut);
|
||||||
|
}
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "CoreMinimal.h"
|
||||||
|
#include "Components/ActorComponent.h"
|
||||||
|
// Pulls in the ElevenLabs component + FPS_AI_ConvAgent_ClientToolCall_ElevenLabs /
|
||||||
|
// FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs (needed for the UFUNCTION handler signatures).
|
||||||
|
#include "PS_AI_ConvAgent_ElevenLabsComponent.h"
|
||||||
|
#include "PS_AI_ConvReplay_AgentTag.generated.h"
|
||||||
|
|
||||||
|
class UPS_AI_ConvReplay_Subsystem;
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Per-agent bridge component.
|
||||||
|
//
|
||||||
|
// Add one to each conversational agent to give it a stable, replay-safe identity
|
||||||
|
// (AgentReplayId). During recording it binds the sibling ElevenLabs component's
|
||||||
|
// delegates and forwards every conversation event to the subsystem, tagged with
|
||||||
|
// this agent's identity. During replay the subsystem looks the agent up by id to
|
||||||
|
// route recorded audio / reactions back to it.
|
||||||
|
//
|
||||||
|
// If you don't add one, the subsystem auto-provisions a tag on every agent at
|
||||||
|
// record/replay start, using the owner actor name as the id (fine for
|
||||||
|
// placed-in-level agents; add an explicit tag for runtime-spawned ones).
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
UCLASS(ClassGroup = "ASTERION|PS_AI_ConvReplay", meta = (BlueprintSpawnableComponent),
|
||||||
|
DisplayName = "PS AI ConvReplay - Agent Tag")
|
||||||
|
class PS_AI_CONVREPLAY_API UPS_AI_ConvReplay_AgentTag : public UActorComponent
|
||||||
|
{
|
||||||
|
GENERATED_BODY()
|
||||||
|
|
||||||
|
public:
|
||||||
|
UPS_AI_ConvReplay_AgentTag();
|
||||||
|
|
||||||
|
/** Stable id used to match recorded audio/reactions back to this agent on replay.
|
||||||
|
* Leave as None to fall back to the owner actor name (stable for placed agents). */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvReplay")
|
||||||
|
FName AgentReplayId;
|
||||||
|
|
||||||
|
/** Resolved identity string (AgentReplayId if set, else the owner actor name). */
|
||||||
|
FString GetResolvedId() const;
|
||||||
|
|
||||||
|
/** The sibling ElevenLabs component on the owner actor (may be null). */
|
||||||
|
UPS_AI_ConvAgent_ElevenLabsComponent* GetAgentComponent() const;
|
||||||
|
|
||||||
|
/** Bind the sibling component's delegates and start forwarding events. Called by the subsystem. */
|
||||||
|
void BeginRecording(UPS_AI_ConvReplay_Subsystem* InSubsystem);
|
||||||
|
|
||||||
|
/** Unbind delegates and stop forwarding. Called by the subsystem. */
|
||||||
|
void EndRecording();
|
||||||
|
|
||||||
|
private:
|
||||||
|
UPROPERTY(Transient) TObjectPtr<UPS_AI_ConvReplay_Subsystem> Subsystem;
|
||||||
|
UPROPERTY(Transient) TObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> Agent;
|
||||||
|
|
||||||
|
FDelegateHandle AudioDataHandle;
|
||||||
|
bool bBound = false;
|
||||||
|
|
||||||
|
// Dynamic-delegate handlers (must be UFUNCTION and match the delegate signatures).
|
||||||
|
UFUNCTION() void HandleStarted();
|
||||||
|
UFUNCTION() void HandleStopped();
|
||||||
|
UFUNCTION() void HandleActionRequested(const FString& ActionName);
|
||||||
|
UFUNCTION() void HandleClientToolCall(const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall);
|
||||||
|
UFUNCTION() void HandleTextResponse(const FString& Text);
|
||||||
|
UFUNCTION() void HandleTranscript(const FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs& Segment);
|
||||||
|
UFUNCTION() void HandlePlayerAudio(APawn* SpeakerPawn, const TArray<uint8>& PCM);
|
||||||
|
|
||||||
|
// Non-dynamic (raw C++ multicast) handler for agent PCM.
|
||||||
|
void HandleAgentAudioData(const TArray<uint8>& PCM);
|
||||||
|
};
|
||||||
@@ -0,0 +1,161 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "CoreMinimal.h"
|
||||||
|
#include "Subsystems/WorldSubsystem.h"
|
||||||
|
#include "PS_AI_ConvReplay_Types.h"
|
||||||
|
#include "PS_AI_ConvReplay_Subsystem.generated.h"
|
||||||
|
|
||||||
|
class UPS_AI_ConvReplay_AgentTag;
|
||||||
|
class UPS_AI_ConvAgent_ElevenLabsComponent;
|
||||||
|
class UAudioComponent;
|
||||||
|
class APawn;
|
||||||
|
class IVoiceEncoder;
|
||||||
|
class IVoiceDecoder;
|
||||||
|
struct FPS_AI_ConvAgent_ClientToolCall_ElevenLabs;
|
||||||
|
|
||||||
|
/** Fired on the replay client when a recorded line's timestamp is reached.
|
||||||
|
* Use it to drive subtitles / a conversation log. Audio playback is automatic. */
|
||||||
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE_FourParams(FPS_AI_ConvReplay_OnReplayLinePlayed,
|
||||||
|
FName, AgentId, const FString&, Speaker, const FString&, Text, float, TimeSec);
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// World subsystem that records the conversation audio track while a replay is
|
||||||
|
// being recorded, and plays it back while a replay is being played.
|
||||||
|
//
|
||||||
|
// • Record: taps every agent's ElevenLabs delegates (via AgentTag), accumulates
|
||||||
|
// PCM per speaker, Opus-encodes each utterance and appends it to an external
|
||||||
|
// sidecar file (<ReplayName>.convreplay) next to the .replay — nothing heavy
|
||||||
|
// goes into the demo stream. Also records AI reactions (perform_action / custom
|
||||||
|
// tools) and player transcripts.
|
||||||
|
// • Replay: loads the sidecar, and each tick fires the records whose timestamp is
|
||||||
|
// reached — agent audio through the agent's own voice (drives lip-sync + body
|
||||||
|
// expression), player audio spatialized at the recorded location, reactions
|
||||||
|
// re-broadcast. Gaze / emotion / facial expression replay for free (replicated).
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
UCLASS()
|
||||||
|
class PS_AI_CONVREPLAY_API UPS_AI_ConvReplay_Subsystem : public UTickableWorldSubsystem
|
||||||
|
{
|
||||||
|
GENERATED_BODY()
|
||||||
|
|
||||||
|
public:
|
||||||
|
// USubsystem / UWorldSubsystem
|
||||||
|
virtual void Initialize(FSubsystemCollectionBase& Collection) override;
|
||||||
|
virtual void Deinitialize() override;
|
||||||
|
virtual bool DoesSupportWorldType(const EWorldType::Type WorldType) const override;
|
||||||
|
|
||||||
|
// FTickableGameObject
|
||||||
|
virtual void Tick(float DeltaTime) override;
|
||||||
|
virtual TStatId GetStatId() const override;
|
||||||
|
|
||||||
|
/** Fired on the replay client for each recorded line as its timestamp is reached. */
|
||||||
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvReplay")
|
||||||
|
FPS_AI_ConvReplay_OnReplayLinePlayed OnReplayLinePlayed;
|
||||||
|
|
||||||
|
// ── Record callbacks (invoked by AgentTag on the recording machine) ──────
|
||||||
|
void RecordAgentStarted(UPS_AI_ConvReplay_AgentTag* Tag);
|
||||||
|
void RecordAgentAudio(UPS_AI_ConvReplay_AgentTag* Tag, const TArray<uint8>& PCM);
|
||||||
|
void RecordAgentText(UPS_AI_ConvReplay_AgentTag* Tag, const FString& Text);
|
||||||
|
void RecordAgentStopped(UPS_AI_ConvReplay_AgentTag* Tag);
|
||||||
|
void RecordAction(UPS_AI_ConvReplay_AgentTag* Tag, const FString& ActionName);
|
||||||
|
void RecordToolCall(UPS_AI_ConvReplay_AgentTag* Tag, const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall);
|
||||||
|
void RecordUserTranscript(UPS_AI_ConvReplay_AgentTag* Tag, const FString& Text);
|
||||||
|
void RecordPlayerAudio(APawn* SpeakerPawn, const TArray<uint8>& PCM);
|
||||||
|
|
||||||
|
private:
|
||||||
|
// Session transitions.
|
||||||
|
void BeginRecordingSession();
|
||||||
|
void EndRecordingSession();
|
||||||
|
void BeginReplaySession();
|
||||||
|
void EndReplaySession();
|
||||||
|
|
||||||
|
void TickRecording();
|
||||||
|
void TickReplay();
|
||||||
|
|
||||||
|
// Agent discovery / lookup.
|
||||||
|
void RebuildTagList();
|
||||||
|
UPS_AI_ConvReplay_AgentTag* FindTagById(const FString& Id);
|
||||||
|
// Robust match for a recorded agent record: exact id first (placed agents /
|
||||||
|
// explicit AgentReplayId), else the nearest agent to the recorded position
|
||||||
|
// (handles runtime-spawned agents whose actor names differ on replay).
|
||||||
|
UPS_AI_ConvReplay_AgentTag* FindAgentForRecord(const FString& Id, const FVector& Pos);
|
||||||
|
|
||||||
|
// Sidecar IO.
|
||||||
|
FString GetSidecarPath() const;
|
||||||
|
void WriteRecord(FPS_AI_ConvReplay_Record& Rec);
|
||||||
|
void LoadRecords();
|
||||||
|
|
||||||
|
// Opus (own instances — not shared with ConvAgent's live encoder).
|
||||||
|
void EnsureCodec();
|
||||||
|
bool EncodeUtterance(const TArray<uint8>& PCM, TArray<uint8>& OutPackets);
|
||||||
|
bool DecodeUtterance(const TArray<uint8>& Packets, TArray<uint8>& OutPCM);
|
||||||
|
|
||||||
|
// Replay dispatch.
|
||||||
|
void DispatchDueRecords(double NowSec, bool bAudioAllowed);
|
||||||
|
void ResetPlaybackFrom(double NowSec);
|
||||||
|
void PlayPlayerAudioAtLocation(const FVector& Loc, const TArray<uint8>& PCM);
|
||||||
|
|
||||||
|
// Helpers.
|
||||||
|
double NowDemoSeconds() const;
|
||||||
|
void FlushPlayerSegment(APawn* Pawn);
|
||||||
|
static bool IsVoiced(const TArray<uint8>& PCM16);
|
||||||
|
static FString ResolvePlayerId(APawn* Pawn);
|
||||||
|
// Boost quiet mic capture: normalize a replayed player utterance toward a target
|
||||||
|
// RMS (peak-safe), then apply the ps.convreplay.PlayerGain CVar. In-place on int16 PCM.
|
||||||
|
static void NormalizePlayerPcm(TArray<uint8>& Pcm16);
|
||||||
|
|
||||||
|
// ── State ────────────────────────────────────────────────────────────────
|
||||||
|
bool bRecording = false;
|
||||||
|
bool bPlaying = false;
|
||||||
|
double LastReplayTime = -1.0;
|
||||||
|
|
||||||
|
IFileHandle* WriteHandle = nullptr;
|
||||||
|
|
||||||
|
TSharedPtr<IVoiceEncoder> Encoder;
|
||||||
|
TSharedPtr<IVoiceDecoder> Decoder;
|
||||||
|
|
||||||
|
UPROPERTY(Transient)
|
||||||
|
TArray<TObjectPtr<UPS_AI_ConvReplay_AgentTag>> Tags;
|
||||||
|
|
||||||
|
// Per-agent recording accumulator.
|
||||||
|
struct FAgentAccum
|
||||||
|
{
|
||||||
|
double StartTime = 0.0;
|
||||||
|
FVector Pos = FVector::ZeroVector;
|
||||||
|
FString Text;
|
||||||
|
TArray<uint8> PCM;
|
||||||
|
bool bOpen = false;
|
||||||
|
};
|
||||||
|
TMap<TWeakObjectPtr<UPS_AI_ConvReplay_AgentTag>, FAgentAccum> AgentAccum;
|
||||||
|
|
||||||
|
// Per-player recording accumulator (segmented by silence).
|
||||||
|
struct FPlayerAccum
|
||||||
|
{
|
||||||
|
double StartTime = 0.0;
|
||||||
|
double LastVoiceTime = 0.0;
|
||||||
|
FVector Pos = FVector::ZeroVector;
|
||||||
|
FString Id;
|
||||||
|
TArray<uint8> PCM;
|
||||||
|
bool bActive = false;
|
||||||
|
};
|
||||||
|
TMap<TWeakObjectPtr<APawn>, FPlayerAccum> PlayerAccum;
|
||||||
|
|
||||||
|
// Loaded conversation track (replay).
|
||||||
|
TArray<FPS_AI_ConvReplay_Record> Records;
|
||||||
|
|
||||||
|
// Transient one-shot components used to play back player voices during replay.
|
||||||
|
UPROPERTY(Transient)
|
||||||
|
TArray<TObjectPtr<UAudioComponent>> PlayerAudioPool;
|
||||||
|
|
||||||
|
// Tunables.
|
||||||
|
static constexpr float PlayerSilenceFlushSec = 0.6f;
|
||||||
|
static constexpr float VoicedRmsThreshold = 0.02f;
|
||||||
|
static constexpr float ScrubBackThreshold = 0.05f; // seconds jumped back => scrub
|
||||||
|
static constexpr float ScrubFwdThreshold = 1.5f; // seconds jumped forward => scrub / fast-seek
|
||||||
|
static constexpr float NormalSpeedTolerance = 0.35f; // only play audio within this of 1x
|
||||||
|
static constexpr float PlayerTargetRms = 0.12f; // target loudness for replayed player mic (−1..1 scale)
|
||||||
|
static constexpr float PlayerMinGain = 0.1f; // allow attenuating LOUD utterances (bidirectional AGC)
|
||||||
|
static constexpr float PlayerMaxGain = 12.0f; // cap so near-silence isn't blown up
|
||||||
|
static constexpr float PlayerSoftClipThreshold = 0.8f; // above this (|x|), peaks are soft-limited, not hard-clipped
|
||||||
|
};
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "CoreMinimal.h"
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// One entry in the conversation sidecar track (<ReplayName>.convreplay).
|
||||||
|
//
|
||||||
|
// The sidecar is an append-only sequence of these records, each serialized
|
||||||
|
// back-to-back with FMemoryWriter (self-delimiting via FString/TArray length
|
||||||
|
// prefixes) so it can be streamed while recording and read in one pass at replay.
|
||||||
|
// Audio is stored as a sequence of length-prefixed Opus packets (see the
|
||||||
|
// subsystem's EncodeUtterance/DecodeUtterance), NOT inside the demo metadata.
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
enum class EPS_AI_ConvReplay_Kind : uint8
|
||||||
|
{
|
||||||
|
AgentSpeech = 0, // an agent utterance: Audio + Text, played through the agent's voice
|
||||||
|
PlayerSpeech = 1, // a player utterance: Audio, played spatialized at Position
|
||||||
|
Action = 2, // a perform_action reaction: Text = action name
|
||||||
|
ToolCall = 3, // a custom client tool call: Text = tool name, Params = parameters
|
||||||
|
PlayerText = 4, // a player transcript marker (subtitle only, no audio)
|
||||||
|
};
|
||||||
|
|
||||||
|
struct FPS_AI_ConvReplay_Record
|
||||||
|
{
|
||||||
|
/** EPS_AI_ConvReplay_Kind. */
|
||||||
|
uint8 Kind = 0;
|
||||||
|
|
||||||
|
/** Start time in demo seconds (captured via GetCurrentReplayTime at record time). */
|
||||||
|
double TimeSec = 0.0;
|
||||||
|
|
||||||
|
/** Audio duration in seconds (0 for non-audio records). */
|
||||||
|
double DurationSec = 0.0;
|
||||||
|
|
||||||
|
/** Agent id (resolved from AgentTag / actor name) or player id. */
|
||||||
|
FString Id;
|
||||||
|
|
||||||
|
/** 0 = agent, 1 = user. */
|
||||||
|
uint8 Speaker = 0;
|
||||||
|
|
||||||
|
/** Transcript, or action name, or tool name depending on Kind. */
|
||||||
|
FString Text;
|
||||||
|
|
||||||
|
/** World location of the speaker at utterance start (used to spatialize player audio). */
|
||||||
|
FVector Position = FVector::ZeroVector;
|
||||||
|
|
||||||
|
/** Tool-call parameters (empty otherwise). */
|
||||||
|
TMap<FString, FString> Params;
|
||||||
|
|
||||||
|
/** Sequence of length-prefixed Opus packets (empty for non-audio records). */
|
||||||
|
TArray<uint8> Audio;
|
||||||
|
|
||||||
|
/** Runtime-only: set once this record has been dispatched during playback. NOT serialized. */
|
||||||
|
bool bPlayed = false;
|
||||||
|
|
||||||
|
friend FArchive& operator<<(FArchive& Ar, FPS_AI_ConvReplay_Record& R)
|
||||||
|
{
|
||||||
|
Ar << R.Kind;
|
||||||
|
Ar << R.TimeSec;
|
||||||
|
Ar << R.DurationSec;
|
||||||
|
Ar << R.Id;
|
||||||
|
Ar << R.Speaker;
|
||||||
|
Ar << R.Text;
|
||||||
|
Ar << R.Position;
|
||||||
|
Ar << R.Params;
|
||||||
|
Ar << R.Audio;
|
||||||
|
return Ar;
|
||||||
|
}
|
||||||
|
};
|
||||||
Reference in New Issue
Block a user