WIP multi agent
This commit is contained in:
parent
f24c9355eb
commit
88a824897e
@ -2,9 +2,11 @@
|
||||
|
||||
#include "PS_AI_ConvAgent_ElevenLabsComponent.h"
|
||||
#include "PS_AI_ConvAgent_MicrophoneCaptureComponent.h"
|
||||
#include "PS_AI_ConvAgent_InteractionSubsystem.h"
|
||||
#include "PS_AI_ConvAgent.h"
|
||||
|
||||
#include "Components/AudioComponent.h"
|
||||
#include "Sound/SoundAttenuation.h"
|
||||
#include "Sound/SoundWaveProcedural.h"
|
||||
#include "GameFramework/Actor.h"
|
||||
|
||||
@ -28,10 +30,28 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::BeginPlay()
|
||||
{
|
||||
Super::BeginPlay();
|
||||
InitAudioPlayback();
|
||||
|
||||
// Auto-register with the interaction subsystem so InteractionComponents can discover us.
|
||||
if (UWorld* World = GetWorld())
|
||||
{
|
||||
if (UPS_AI_ConvAgent_InteractionSubsystem* Sub = World->GetSubsystem<UPS_AI_ConvAgent_InteractionSubsystem>())
|
||||
{
|
||||
Sub->RegisterAgent(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void UPS_AI_ConvAgent_ElevenLabsComponent::EndPlay(const EEndPlayReason::Type EndPlayReason)
|
||||
{
|
||||
// Unregister from the interaction subsystem before tearing down.
|
||||
if (UWorld* World = GetWorld())
|
||||
{
|
||||
if (UPS_AI_ConvAgent_InteractionSubsystem* Sub = World->GetSubsystem<UPS_AI_ConvAgent_InteractionSubsystem>())
|
||||
{
|
||||
Sub->UnregisterAgent(this);
|
||||
}
|
||||
}
|
||||
|
||||
EndConversation();
|
||||
Super::EndPlay(EndPlayReason);
|
||||
}
|
||||
@ -256,39 +276,45 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StartListening()
|
||||
WebSocketProxy->SendUserTurnStart();
|
||||
}
|
||||
|
||||
// Find the microphone component on our owner actor, or create one.
|
||||
UPS_AI_ConvAgent_MicrophoneCaptureComponent* Mic =
|
||||
GetOwner()->FindComponentByClass<UPS_AI_ConvAgent_MicrophoneCaptureComponent>();
|
||||
// External mic mode: the InteractionComponent on the pawn manages the mic and
|
||||
// feeds audio via FeedExternalAudio(). We only manage turn state here.
|
||||
if (!bExternalMicManagement)
|
||||
{
|
||||
// Find the microphone component on our owner actor, or create one.
|
||||
UPS_AI_ConvAgent_MicrophoneCaptureComponent* Mic =
|
||||
GetOwner()->FindComponentByClass<UPS_AI_ConvAgent_MicrophoneCaptureComponent>();
|
||||
|
||||
if (!Mic)
|
||||
{
|
||||
Mic = NewObject<UPS_AI_ConvAgent_MicrophoneCaptureComponent>(GetOwner(),
|
||||
TEXT("PS_AI_ConvAgent_Microphone"));
|
||||
Mic->RegisterComponent();
|
||||
}
|
||||
if (!Mic)
|
||||
{
|
||||
Mic = NewObject<UPS_AI_ConvAgent_MicrophoneCaptureComponent>(GetOwner(),
|
||||
TEXT("PS_AI_ConvAgent_Microphone"));
|
||||
Mic->RegisterComponent();
|
||||
}
|
||||
|
||||
// Always remove existing binding first to prevent duplicate delegates stacking
|
||||
// up if StartListening is called more than once without a matching StopListening.
|
||||
Mic->OnAudioCaptured.RemoveAll(this);
|
||||
Mic->OnAudioCaptured.AddUObject(this,
|
||||
&UPS_AI_ConvAgent_ElevenLabsComponent::OnMicrophoneDataCaptured);
|
||||
// Echo suppression: point the mic at our atomic bAgentSpeaking flag so it skips
|
||||
// capture entirely (before resampling) while the agent is speaking.
|
||||
// In Server VAD + interruption mode, disable echo suppression so the server
|
||||
// receives the user's voice even during agent playback — the server's own VAD
|
||||
// handles echo filtering and interruption detection.
|
||||
if (TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server && bAllowInterruption)
|
||||
{
|
||||
Mic->EchoSuppressFlag = nullptr;
|
||||
// Always remove existing binding first to prevent duplicate delegates stacking
|
||||
// up if StartListening is called more than once without a matching StopListening.
|
||||
Mic->OnAudioCaptured.RemoveAll(this);
|
||||
Mic->OnAudioCaptured.AddUObject(this,
|
||||
&UPS_AI_ConvAgent_ElevenLabsComponent::OnMicrophoneDataCaptured);
|
||||
// Echo suppression: point the mic at our atomic bAgentSpeaking flag so it skips
|
||||
// capture entirely (before resampling) while the agent is speaking.
|
||||
// In Server VAD + interruption mode, disable echo suppression so the server
|
||||
// receives the user's voice even during agent playback — the server's own VAD
|
||||
// handles echo filtering and interruption detection.
|
||||
if (TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server && bAllowInterruption)
|
||||
{
|
||||
Mic->EchoSuppressFlag = nullptr;
|
||||
}
|
||||
else
|
||||
{
|
||||
Mic->EchoSuppressFlag = &bAgentSpeaking;
|
||||
}
|
||||
Mic->StartCapture();
|
||||
}
|
||||
else
|
||||
{
|
||||
Mic->EchoSuppressFlag = &bAgentSpeaking;
|
||||
}
|
||||
Mic->StartCapture();
|
||||
|
||||
const double T = TurnStartTime - SessionStartTime;
|
||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log, TEXT("[T+%.2fs] [Turn %d] Mic opened — user speaking."), T, TurnIndex);
|
||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log, TEXT("[T+%.2fs] [Turn %d] Mic opened%s — user speaking."),
|
||||
T, TurnIndex, bExternalMicManagement ? TEXT(" (external)") : TEXT(""));
|
||||
}
|
||||
|
||||
void UPS_AI_ConvAgent_ElevenLabsComponent::StopListening()
|
||||
@ -296,11 +322,15 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StopListening()
|
||||
if (!bIsListening) return;
|
||||
bIsListening = false;
|
||||
|
||||
if (UPS_AI_ConvAgent_MicrophoneCaptureComponent* Mic =
|
||||
GetOwner() ? GetOwner()->FindComponentByClass<UPS_AI_ConvAgent_MicrophoneCaptureComponent>() : nullptr)
|
||||
// External mic mode: mic is managed by the InteractionComponent, not us.
|
||||
if (!bExternalMicManagement)
|
||||
{
|
||||
Mic->StopCapture();
|
||||
Mic->OnAudioCaptured.RemoveAll(this);
|
||||
if (UPS_AI_ConvAgent_MicrophoneCaptureComponent* Mic =
|
||||
GetOwner() ? GetOwner()->FindComponentByClass<UPS_AI_ConvAgent_MicrophoneCaptureComponent>() : nullptr)
|
||||
{
|
||||
Mic->StopCapture();
|
||||
Mic->OnAudioCaptured.RemoveAll(this);
|
||||
}
|
||||
}
|
||||
|
||||
// Flush any partially-accumulated mic audio before signalling end-of-turn.
|
||||
@ -383,6 +413,30 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::InterruptAgent()
|
||||
StopAgentAudio();
|
||||
}
|
||||
|
||||
void UPS_AI_ConvAgent_ElevenLabsComponent::FeedExternalAudio(const TArray<float>& FloatPCM)
|
||||
{
|
||||
// Same logic as OnMicrophoneDataCaptured but called from an external source
|
||||
// (e.g. InteractionComponent on the pawn) instead of a local mic component.
|
||||
if (!IsConnected() || !bIsListening) return;
|
||||
|
||||
// Echo suppression: skip sending mic audio while the agent is speaking.
|
||||
if (bAgentSpeaking && !(TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server && bAllowInterruption))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
TArray<uint8> PCMBytes = FloatPCMToInt16Bytes(FloatPCM);
|
||||
|
||||
FScopeLock Lock(&MicSendLock);
|
||||
MicAccumulationBuffer.Append(PCMBytes);
|
||||
|
||||
if (MicAccumulationBuffer.Num() >= GetMicChunkMinBytes())
|
||||
{
|
||||
WebSocketProxy->SendAudioChunk(MicAccumulationBuffer);
|
||||
MicAccumulationBuffer.Reset();
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// State queries
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@ -647,6 +701,15 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::InitAudioPlayback()
|
||||
AudioPlaybackComponent->bAutoActivate = false;
|
||||
AudioPlaybackComponent->SetSound(ProceduralSoundWave);
|
||||
|
||||
// Spatialization: if an attenuation asset is set, enable 3D audio so the
|
||||
// agent's voice attenuates with distance and pans spatially for the listener.
|
||||
if (SoundAttenuation)
|
||||
{
|
||||
AudioPlaybackComponent->AttenuationSettings = SoundAttenuation;
|
||||
AudioPlaybackComponent->bOverrideAttenuation = false;
|
||||
AudioPlaybackComponent->bAllowSpatialization = true;
|
||||
}
|
||||
|
||||
// When the procedural sound wave needs more audio data, pull from our queue.
|
||||
ProceduralSoundWave->OnSoundWaveProceduralUnderflow =
|
||||
FOnSoundWaveProceduralUnderflow::CreateUObject(
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
#include "PS_AI_ConvAgent_ElevenLabsComponent.h"
|
||||
#include "PS_AI_ConvAgent_EmotionPoseMap.h"
|
||||
#include "Animation/AnimSequence.h"
|
||||
#include "Animation/AnimData/IAnimationDataModel.h"
|
||||
#include "Animation/AnimCurveTypes.h"
|
||||
|
||||
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_FacialExpr, Log, All);
|
||||
|
||||
@ -156,10 +156,8 @@ TMap<FName, float> UPS_AI_ConvAgent_FacialExpressionComponent::EvaluateAnimCurve
|
||||
TMap<FName, float> CurveValues;
|
||||
if (!AnimSeq) return CurveValues;
|
||||
|
||||
const IAnimationDataModel* DataModel = AnimSeq->GetDataModel();
|
||||
if (!DataModel) return CurveValues;
|
||||
|
||||
const TArray<FFloatCurve>& FloatCurves = DataModel->GetFloatCurves();
|
||||
// Use runtime GetCurveData() — GetDataModel() is editor-only in UE 5.5.
|
||||
const TArray<FFloatCurve>& FloatCurves = AnimSeq->GetCurveData().FloatCurves;
|
||||
for (const FFloatCurve& Curve : FloatCurves)
|
||||
{
|
||||
const float Value = Curve.FloatCurve.Eval(Time);
|
||||
|
||||
@ -0,0 +1,264 @@
|
||||
// Copyright ASTERION. All Rights Reserved.
|
||||
|
||||
#include "PS_AI_ConvAgent_InteractionComponent.h"
|
||||
#include "PS_AI_ConvAgent_InteractionSubsystem.h"
|
||||
#include "PS_AI_ConvAgent_ElevenLabsComponent.h"
|
||||
#include "PS_AI_ConvAgent_MicrophoneCaptureComponent.h"
|
||||
|
||||
#include "GameFramework/Pawn.h"
|
||||
#include "GameFramework/PlayerController.h"
|
||||
#include "Camera/PlayerCameraManager.h"
|
||||
|
||||
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_Select, Log, All);
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Constructor
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
UPS_AI_ConvAgent_InteractionComponent::UPS_AI_ConvAgent_InteractionComponent()
|
||||
{
|
||||
PrimaryComponentTick.bCanEverTick = true;
|
||||
PrimaryComponentTick.TickInterval = 1.0f / 30.0f; // 30 Hz is enough for selection evaluation.
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Lifecycle
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
void UPS_AI_ConvAgent_InteractionComponent::BeginPlay()
|
||||
{
|
||||
Super::BeginPlay();
|
||||
|
||||
// Create mic capture component on the pawn.
|
||||
AActor* Owner = GetOwner();
|
||||
if (!Owner) return;
|
||||
|
||||
MicComponent = Owner->FindComponentByClass<UPS_AI_ConvAgent_MicrophoneCaptureComponent>();
|
||||
if (!MicComponent)
|
||||
{
|
||||
MicComponent = NewObject<UPS_AI_ConvAgent_MicrophoneCaptureComponent>(
|
||||
Owner, TEXT("PS_AI_ConvAgent_InteractionMic"));
|
||||
MicComponent->RegisterComponent();
|
||||
}
|
||||
|
||||
// Bind mic audio callback.
|
||||
MicComponent->OnAudioCaptured.AddUObject(this,
|
||||
&UPS_AI_ConvAgent_InteractionComponent::OnMicAudioCaptured);
|
||||
|
||||
UE_LOG(LogPS_AI_ConvAgent_Select, Log, TEXT("InteractionComponent initialized on %s."), *Owner->GetName());
|
||||
}
|
||||
|
||||
void UPS_AI_ConvAgent_InteractionComponent::EndPlay(const EEndPlayReason::Type EndPlayReason)
|
||||
{
|
||||
if (MicComponent)
|
||||
{
|
||||
MicComponent->StopCapture();
|
||||
MicComponent->OnAudioCaptured.RemoveAll(this);
|
||||
}
|
||||
|
||||
// Fire deselection event for cleanup.
|
||||
if (UPS_AI_ConvAgent_ElevenLabsComponent* Agent = SelectedAgent.Get())
|
||||
{
|
||||
SelectedAgent.Reset();
|
||||
OnAgentDeselected.Broadcast(Agent);
|
||||
}
|
||||
|
||||
Super::EndPlay(EndPlayReason);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Tick — agent selection
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
void UPS_AI_ConvAgent_InteractionComponent::TickComponent(float DeltaTime, ELevelTick TickType,
|
||||
FActorComponentTickFunction* ThisTickFunction)
|
||||
{
|
||||
Super::TickComponent(DeltaTime, TickType, ThisTickFunction);
|
||||
|
||||
UPS_AI_ConvAgent_ElevenLabsComponent* BestAgent = EvaluateBestAgent();
|
||||
|
||||
// Check if selection changed.
|
||||
UPS_AI_ConvAgent_ElevenLabsComponent* CurrentAgent = SelectedAgent.Get();
|
||||
if (BestAgent != CurrentAgent)
|
||||
{
|
||||
SetSelectedAgent(BestAgent);
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Selection evaluation
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
UPS_AI_ConvAgent_ElevenLabsComponent* UPS_AI_ConvAgent_InteractionComponent::EvaluateBestAgent() const
|
||||
{
|
||||
UWorld* World = GetWorld();
|
||||
if (!World) return nullptr;
|
||||
|
||||
UPS_AI_ConvAgent_InteractionSubsystem* Subsystem = World->GetSubsystem<UPS_AI_ConvAgent_InteractionSubsystem>();
|
||||
if (!Subsystem) return nullptr;
|
||||
|
||||
TArray<UPS_AI_ConvAgent_ElevenLabsComponent*> Agents = Subsystem->GetRegisteredAgents();
|
||||
if (Agents.Num() == 0) return nullptr;
|
||||
|
||||
FVector ViewLocation;
|
||||
FVector ViewDirection;
|
||||
GetPawnViewPoint(ViewLocation, ViewDirection);
|
||||
|
||||
const float MaxDistSq = MaxInteractionDistance * MaxInteractionDistance;
|
||||
const float CosViewCone = FMath::Cos(FMath::DegreesToRadians(ViewConeHalfAngle));
|
||||
const float CosStickyAngle = FMath::Cos(FMath::DegreesToRadians(SelectionStickyAngle));
|
||||
|
||||
UPS_AI_ConvAgent_ElevenLabsComponent* BestCandidate = nullptr;
|
||||
float BestAngle = MAX_FLT; // Smallest angle = closest to center of view.
|
||||
|
||||
UPS_AI_ConvAgent_ElevenLabsComponent* CurrentAgent = SelectedAgent.Get();
|
||||
|
||||
for (UPS_AI_ConvAgent_ElevenLabsComponent* Agent : Agents)
|
||||
{
|
||||
AActor* AgentActor = Agent->GetOwner();
|
||||
if (!AgentActor) continue;
|
||||
|
||||
const FVector AgentLocation = AgentActor->GetActorLocation();
|
||||
const FVector ToAgent = AgentLocation - ViewLocation;
|
||||
const float DistSq = ToAgent.SizeSquared();
|
||||
|
||||
// Distance check.
|
||||
if (DistSq > MaxDistSq) continue;
|
||||
|
||||
float AngleDeg = 0.0f;
|
||||
if (bRequireLookAt)
|
||||
{
|
||||
const FVector DirToAgent = ToAgent.GetSafeNormal();
|
||||
const float Dot = FVector::DotProduct(ViewDirection, DirToAgent);
|
||||
AngleDeg = FMath::RadiansToDegrees(FMath::Acos(FMath::Clamp(Dot, -1.0f, 1.0f)));
|
||||
|
||||
// Use wider sticky angle for the currently selected agent.
|
||||
const float ConeThreshold = (Agent == CurrentAgent) ? SelectionStickyAngle : ViewConeHalfAngle;
|
||||
if (AngleDeg > ConeThreshold) continue;
|
||||
}
|
||||
else
|
||||
{
|
||||
// When not requiring look-at, use distance as the ranking metric.
|
||||
// Convert distance to a comparable "angle" value (smaller distance = smaller value).
|
||||
AngleDeg = FMath::Sqrt(DistSq);
|
||||
}
|
||||
|
||||
if (bDebug && DebugVerbosity >= 2)
|
||||
{
|
||||
UE_LOG(LogPS_AI_ConvAgent_Select, Log,
|
||||
TEXT(" Candidate: %s | Dist=%.0f | Angle=%.1f%s"),
|
||||
*AgentActor->GetName(),
|
||||
FMath::Sqrt(DistSq),
|
||||
AngleDeg,
|
||||
(Agent == CurrentAgent) ? TEXT(" (current)") : TEXT(""));
|
||||
}
|
||||
|
||||
if (AngleDeg < BestAngle)
|
||||
{
|
||||
BestAngle = AngleDeg;
|
||||
BestCandidate = Agent;
|
||||
}
|
||||
}
|
||||
|
||||
return BestCandidate;
|
||||
}
|
||||
|
||||
void UPS_AI_ConvAgent_InteractionComponent::SetSelectedAgent(UPS_AI_ConvAgent_ElevenLabsComponent* NewAgent)
|
||||
{
|
||||
UPS_AI_ConvAgent_ElevenLabsComponent* OldAgent = SelectedAgent.Get();
|
||||
|
||||
// Deselect old agent.
|
||||
if (OldAgent)
|
||||
{
|
||||
SelectedAgent.Reset();
|
||||
|
||||
if (bDebug)
|
||||
{
|
||||
UE_LOG(LogPS_AI_ConvAgent_Select, Log, TEXT("Agent deselected: %s"),
|
||||
OldAgent->GetOwner() ? *OldAgent->GetOwner()->GetName() : TEXT("(null)"));
|
||||
}
|
||||
|
||||
OnAgentDeselected.Broadcast(OldAgent);
|
||||
}
|
||||
|
||||
// Select new agent.
|
||||
if (NewAgent)
|
||||
{
|
||||
SelectedAgent = NewAgent;
|
||||
|
||||
if (bDebug)
|
||||
{
|
||||
UE_LOG(LogPS_AI_ConvAgent_Select, Log, TEXT("Agent selected: %s"),
|
||||
NewAgent->GetOwner() ? *NewAgent->GetOwner()->GetName() : TEXT("(null)"));
|
||||
}
|
||||
|
||||
// Ensure mic is capturing so we can route audio to the new agent.
|
||||
if (MicComponent && !MicComponent->IsCapturing())
|
||||
{
|
||||
MicComponent->StartCapture();
|
||||
}
|
||||
|
||||
OnAgentSelected.Broadcast(NewAgent);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (bDebug)
|
||||
{
|
||||
UE_LOG(LogPS_AI_ConvAgent_Select, Log, TEXT("No agent in range."));
|
||||
}
|
||||
|
||||
OnNoAgentInRange.Broadcast();
|
||||
}
|
||||
}
|
||||
|
||||
void UPS_AI_ConvAgent_InteractionComponent::GetPawnViewPoint(FVector& OutLocation, FVector& OutDirection) const
|
||||
{
|
||||
AActor* Owner = GetOwner();
|
||||
if (!Owner)
|
||||
{
|
||||
OutLocation = FVector::ZeroVector;
|
||||
OutDirection = FVector::ForwardVector;
|
||||
return;
|
||||
}
|
||||
|
||||
// Try to get the player controller's camera view (most accurate for first/third person).
|
||||
if (APawn* Pawn = Cast<APawn>(Owner))
|
||||
{
|
||||
if (APlayerController* PC = Cast<APlayerController>(Pawn->GetController()))
|
||||
{
|
||||
FRotator ViewRotation;
|
||||
PC->GetPlayerViewPoint(OutLocation, ViewRotation);
|
||||
OutDirection = ViewRotation.Vector();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: use actor location and forward.
|
||||
OutLocation = Owner->GetActorLocation();
|
||||
OutDirection = Owner->GetActorForwardVector();
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Blueprint API
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
UPS_AI_ConvAgent_ElevenLabsComponent* UPS_AI_ConvAgent_InteractionComponent::GetSelectedAgent() const
|
||||
{
|
||||
return SelectedAgent.Get();
|
||||
}
|
||||
|
||||
void UPS_AI_ConvAgent_InteractionComponent::ForceSelectAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Agent)
|
||||
{
|
||||
SetSelectedAgent(Agent);
|
||||
}
|
||||
|
||||
void UPS_AI_ConvAgent_InteractionComponent::ClearSelection()
|
||||
{
|
||||
SetSelectedAgent(nullptr);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Mic routing
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
void UPS_AI_ConvAgent_InteractionComponent::OnMicAudioCaptured(const TArray<float>& FloatPCM)
|
||||
{
|
||||
UPS_AI_ConvAgent_ElevenLabsComponent* Agent = SelectedAgent.Get();
|
||||
if (!Agent) return;
|
||||
|
||||
Agent->FeedExternalAudio(FloatPCM);
|
||||
}
|
||||
@ -0,0 +1,67 @@
|
||||
// Copyright ASTERION. All Rights Reserved.
|
||||
|
||||
#include "PS_AI_ConvAgent_InteractionSubsystem.h"
|
||||
#include "PS_AI_ConvAgent_ElevenLabsComponent.h"
|
||||
|
||||
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_Interaction, Log, All);
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Registration
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
void UPS_AI_ConvAgent_InteractionSubsystem::RegisterAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Agent)
|
||||
{
|
||||
if (!Agent) return;
|
||||
|
||||
// Avoid duplicates.
|
||||
for (const TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent>& Existing : RegisteredAgents)
|
||||
{
|
||||
if (Existing.Get() == Agent) return;
|
||||
}
|
||||
|
||||
RegisteredAgents.Add(Agent);
|
||||
UE_LOG(LogPS_AI_ConvAgent_Interaction, Log, TEXT("Agent registered: %s (%d total)"),
|
||||
*Agent->GetOwner()->GetName(), RegisteredAgents.Num());
|
||||
}
|
||||
|
||||
void UPS_AI_ConvAgent_InteractionSubsystem::UnregisterAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Agent)
|
||||
{
|
||||
if (!Agent) return;
|
||||
|
||||
RegisteredAgents.RemoveAll([Agent](const TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent>& Weak)
|
||||
{
|
||||
return !Weak.IsValid() || Weak.Get() == Agent;
|
||||
});
|
||||
|
||||
UE_LOG(LogPS_AI_ConvAgent_Interaction, Log, TEXT("Agent unregistered: %s (%d remaining)"),
|
||||
Agent->GetOwner() ? *Agent->GetOwner()->GetName() : TEXT("(destroyed)"),
|
||||
RegisteredAgents.Num());
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Queries
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
TArray<UPS_AI_ConvAgent_ElevenLabsComponent*> UPS_AI_ConvAgent_InteractionSubsystem::GetRegisteredAgents() const
|
||||
{
|
||||
TArray<UPS_AI_ConvAgent_ElevenLabsComponent*> Result;
|
||||
Result.Reserve(RegisteredAgents.Num());
|
||||
|
||||
for (const TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent>& Weak : RegisteredAgents)
|
||||
{
|
||||
if (UPS_AI_ConvAgent_ElevenLabsComponent* Agent = Weak.Get())
|
||||
{
|
||||
Result.Add(Agent);
|
||||
}
|
||||
}
|
||||
|
||||
return Result;
|
||||
}
|
||||
|
||||
int32 UPS_AI_ConvAgent_InteractionSubsystem::GetNumRegisteredAgents() const
|
||||
{
|
||||
int32 Count = 0;
|
||||
for (const TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent>& Weak : RegisteredAgents)
|
||||
{
|
||||
if (Weak.IsValid()) ++Count;
|
||||
}
|
||||
return Count;
|
||||
}
|
||||
@ -8,7 +8,7 @@
|
||||
#include "Engine/SkeletalMesh.h"
|
||||
#include "Animation/MorphTarget.h"
|
||||
#include "Animation/AnimSequence.h"
|
||||
#include "Animation/AnimData/IAnimationDataModel.h"
|
||||
#include "Animation/AnimCurveTypes.h"
|
||||
#include "GameFramework/Actor.h"
|
||||
|
||||
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_LipSync, Log, All);
|
||||
@ -444,17 +444,9 @@ void UPS_AI_ConvAgent_LipSyncComponent::ExtractPoseCurves(const FName& VisemeNam
|
||||
{
|
||||
if (!AnimSeq) return;
|
||||
|
||||
const IAnimationDataModel* DataModel = AnimSeq->GetDataModel();
|
||||
if (!DataModel)
|
||||
{
|
||||
UE_LOG(LogPS_AI_ConvAgent_LipSync, Warning,
|
||||
TEXT("Pose '%s' (%s): No DataModel available — skipping."),
|
||||
*VisemeName.ToString(), *AnimSeq->GetName());
|
||||
return;
|
||||
}
|
||||
|
||||
// Use runtime GetCurveData() — GetDataModel() is editor-only in UE 5.5.
|
||||
TMap<FName, float> CurveValues;
|
||||
const TArray<FFloatCurve>& FloatCurves = DataModel->GetFloatCurves();
|
||||
const TArray<FFloatCurve>& FloatCurves = AnimSeq->GetCurveData().FloatCurves;
|
||||
|
||||
for (const FFloatCurve& Curve : FloatCurves)
|
||||
{
|
||||
|
||||
@ -11,6 +11,7 @@
|
||||
#include "PS_AI_ConvAgent_ElevenLabsComponent.generated.h"
|
||||
|
||||
class UAudioComponent;
|
||||
class USoundAttenuation;
|
||||
class UPS_AI_ConvAgent_MicrophoneCaptureComponent;
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@ -168,6 +169,26 @@ public:
|
||||
ToolTip = "Seconds to wait for a server response after the user stops speaking.\nFires OnAgentResponseTimeout if exceeded. Normal latency is 0.1-0.8s.\nSet to 0 to disable. Default: 10s."))
|
||||
float ResponseTimeoutSeconds = 10.0f;
|
||||
|
||||
// ── Multi-agent / external mic ───────────────────────────────────────────
|
||||
|
||||
/** When true, StartListening/StopListening manage the turn state but do NOT
|
||||
* create or control a local microphone component. Audio is instead fed via
|
||||
* FeedExternalAudio() from an external source (e.g. InteractionComponent on the pawn).
|
||||
* Use this when a centralized mic on the player pawn routes audio to agents. */
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs",
|
||||
meta = (ToolTip = "External mic mode: turn management only, no local mic.\nAudio is fed via FeedExternalAudio() from an external source."))
|
||||
bool bExternalMicManagement = false;
|
||||
|
||||
// ── Audio spatialization ─────────────────────────────────────────────────
|
||||
|
||||
/** Optional sound attenuation settings for spatializing the agent's voice.
|
||||
* When set, the agent's audio playback will be spatialized in 3D — volume
|
||||
* and panning change based on distance and direction from the listener.
|
||||
* Leave null for non-spatialized (2D) audio (default, same as before). */
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs|Audio",
|
||||
meta = (ToolTip = "Sound attenuation for 3D spatialized audio.\nLeave null for non-spatialized (2D) playback."))
|
||||
USoundAttenuation* SoundAttenuation = nullptr;
|
||||
|
||||
// ── Debug ────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Enable debug logging for this component.
|
||||
@ -300,6 +321,15 @@ public:
|
||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
||||
void InterruptAgent();
|
||||
|
||||
/**
|
||||
* Feed microphone audio from an external source (e.g. InteractionComponent on the pawn).
|
||||
* Use this instead of the local mic when bExternalMicManagement is true.
|
||||
* The component must be connected and listening (StartListening called) for audio to be sent.
|
||||
* @param FloatPCM Float32 samples, 16000 Hz mono (same format as MicrophoneCaptureComponent output).
|
||||
*/
|
||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
||||
void FeedExternalAudio(const TArray<float>& FloatPCM);
|
||||
|
||||
// ── State queries ─────────────────────────────────────────────────────────
|
||||
|
||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|ElevenLabs")
|
||||
|
||||
@ -0,0 +1,163 @@
|
||||
// Copyright ASTERION. All Rights Reserved.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "CoreMinimal.h"
|
||||
#include "Components/ActorComponent.h"
|
||||
#include "PS_AI_ConvAgent_InteractionComponent.generated.h"
|
||||
|
||||
class UPS_AI_ConvAgent_ElevenLabsComponent;
|
||||
class UPS_AI_ConvAgent_MicrophoneCaptureComponent;
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Delegates
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Fired when a new agent is selected (enters range + view cone). */
|
||||
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnConvAgentSelected,
|
||||
UPS_AI_ConvAgent_ElevenLabsComponent*, Agent);
|
||||
|
||||
/** Fired when the previously selected agent loses selection. */
|
||||
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnConvAgentDeselected,
|
||||
UPS_AI_ConvAgent_ElevenLabsComponent*, Agent);
|
||||
|
||||
/** Fired when no agent is within interaction range/view. */
|
||||
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnNoConvAgentInRange);
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// UPS_AI_ConvAgent_InteractionComponent
|
||||
//
|
||||
// Place on the player pawn to enable multi-agent interaction.
|
||||
// Evaluates all registered agents each tick, selects the best candidate
|
||||
// based on distance and view direction (frustum cone), and routes the
|
||||
// pawn's microphone audio to the selected agent.
|
||||
//
|
||||
// Only one agent can be selected at a time. Selection is sticky (hysteresis)
|
||||
// to prevent flickering when the player looks between agents.
|
||||
//
|
||||
// Workflow:
|
||||
// 1. Add this component to your player pawn Blueprint.
|
||||
// 2. Configure MaxInteractionDistance, ViewConeHalfAngle, etc.
|
||||
// 3. Bind to OnAgentSelected / OnAgentDeselected to update posture,
|
||||
// UI, and other agent-specific logic from Blueprint.
|
||||
// 4. Each agent's ElevenLabsComponent should have bExternalMicManagement = true.
|
||||
// 5. Agents manage their own WebSocket connections independently.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
UCLASS(ClassGroup = "PS AI ConvAgent", meta = (BlueprintSpawnableComponent),
|
||||
DisplayName = "PS AI ConvAgent Interaction")
|
||||
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_InteractionComponent : public UActorComponent
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
public:
|
||||
UPS_AI_ConvAgent_InteractionComponent();
|
||||
|
||||
// ── Configuration ─────────────────────────────────────────────────────────
|
||||
|
||||
/** Maximum distance (cm) at which an agent can be selected. Agents beyond this are ignored. */
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction",
|
||||
meta = (ClampMin = "0",
|
||||
ToolTip = "Max distance in cm to interact with an agent.\nAgents beyond this distance are never selected."))
|
||||
float MaxInteractionDistance = 300.0f;
|
||||
|
||||
/** Half-angle (degrees) of the view cone used for agent selection.
|
||||
* The cone is centered on the pawn's view direction (camera or control rotation).
|
||||
* Agents outside this cone are not considered unless they are the current sticky selection. */
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction",
|
||||
meta = (ClampMin = "0", ClampMax = "180",
|
||||
ToolTip = "Half-angle of the view cone (degrees).\nAgents outside this cone are not selected.\n45 = 90-degree total FOV."))
|
||||
float ViewConeHalfAngle = 45.0f;
|
||||
|
||||
/** Wider half-angle (degrees) used for the currently selected agent.
|
||||
* This provides hysteresis so the player can look slightly away from
|
||||
* the current agent without losing selection (prevents flickering). */
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction",
|
||||
meta = (ClampMin = "0", ClampMax = "180",
|
||||
ToolTip = "Sticky cone half-angle for the current agent (degrees).\nMust be >= ViewConeHalfAngle for hysteresis to work.\n60 = agent stays selected until 120-degree total cone is exceeded."))
|
||||
float SelectionStickyAngle = 60.0f;
|
||||
|
||||
/** When false, only distance matters for selection (no view cone check).
|
||||
* The closest agent within MaxInteractionDistance is always selected. */
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction",
|
||||
meta = (ToolTip = "Require the player to look at an agent to select it.\nWhen false, the closest agent within range is always selected."))
|
||||
bool bRequireLookAt = true;
|
||||
|
||||
// ── Debug ────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Enable debug logging for this component. */
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Debug")
|
||||
bool bDebug = false;
|
||||
|
||||
/** Verbosity level when bDebug is true.
|
||||
* 0 = selection changes only, 1 = candidate evaluation, 2 = per-frame data. */
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Debug",
|
||||
meta = (ClampMin = "0", ClampMax = "3", EditCondition = "bDebug"))
|
||||
int32 DebugVerbosity = 1;
|
||||
|
||||
// ── Events ────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Fired when a new agent enters selection. Use this to set posture targets, show UI, etc. */
|
||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|Interaction|Events",
|
||||
meta = (ToolTip = "Fires when a new agent is selected.\nSet posture targets, update UI, etc."))
|
||||
FOnConvAgentSelected OnAgentSelected;
|
||||
|
||||
/** Fired when the previously selected agent loses selection. */
|
||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|Interaction|Events",
|
||||
meta = (ToolTip = "Fires when the current agent is deselected.\nClear posture targets, hide UI, etc."))
|
||||
FOnConvAgentDeselected OnAgentDeselected;
|
||||
|
||||
/** Fired when no agent is within range or view cone. */
|
||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|Interaction|Events",
|
||||
meta = (ToolTip = "Fires when no agent meets the distance/view criteria."))
|
||||
FOnNoConvAgentInRange OnNoAgentInRange;
|
||||
|
||||
// ── Blueprint API ─────────────────────────────────────────────────────────
|
||||
|
||||
/** Get the currently selected agent (null if none). */
|
||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|Interaction")
|
||||
UPS_AI_ConvAgent_ElevenLabsComponent* GetSelectedAgent() const;
|
||||
|
||||
/** Manually select a specific agent, bypassing distance/view checks.
|
||||
* Pass null to clear the selection. Automatic selection resumes next tick. */
|
||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
|
||||
void ForceSelectAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Agent);
|
||||
|
||||
/** Clear the current selection. Automatic selection resumes next tick. */
|
||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
|
||||
void ClearSelection();
|
||||
|
||||
// ── UActorComponent overrides ────────────────────────────────────────────
|
||||
virtual void BeginPlay() override;
|
||||
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
|
||||
virtual void TickComponent(float DeltaTime, ELevelTick TickType,
|
||||
FActorComponentTickFunction* ThisTickFunction) override;
|
||||
|
||||
private:
|
||||
// ── Selection logic ──────────────────────────────────────────────────────
|
||||
|
||||
/** Evaluate all registered agents, return the best candidate (or null). */
|
||||
UPS_AI_ConvAgent_ElevenLabsComponent* EvaluateBestAgent() const;
|
||||
|
||||
/** Apply a new selection — fire events, reroute mic. */
|
||||
void SetSelectedAgent(UPS_AI_ConvAgent_ElevenLabsComponent* NewAgent);
|
||||
|
||||
/** Get the pawn's view location and direction (uses camera or control rotation). */
|
||||
void GetPawnViewPoint(FVector& OutLocation, FVector& OutDirection) const;
|
||||
|
||||
// ── Mic routing ──────────────────────────────────────────────────────────
|
||||
|
||||
/** Forward captured mic audio to the currently selected agent. */
|
||||
void OnMicAudioCaptured(const TArray<float>& FloatPCM);
|
||||
|
||||
// ── State ────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Currently selected agent (weak pointer for safety). */
|
||||
TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> SelectedAgent;
|
||||
|
||||
/** Microphone capture component (created on the pawn in BeginPlay). */
|
||||
UPROPERTY()
|
||||
UPS_AI_ConvAgent_MicrophoneCaptureComponent* MicComponent = nullptr;
|
||||
|
||||
/** True while ForceSelectAgent is active (suppresses automatic re-evaluation for one frame). */
|
||||
bool bForceSelectionActive = false;
|
||||
};
|
||||
@ -0,0 +1,42 @@
|
||||
// Copyright ASTERION. All Rights Reserved.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "CoreMinimal.h"
|
||||
#include "Subsystems/WorldSubsystem.h"
|
||||
#include "PS_AI_ConvAgent_InteractionSubsystem.generated.h"
|
||||
|
||||
class UPS_AI_ConvAgent_ElevenLabsComponent;
|
||||
|
||||
/**
|
||||
* World subsystem that maintains a registry of all conversational agent
|
||||
* components in the current world. Agents auto-register on BeginPlay
|
||||
* and unregister on EndPlay.
|
||||
*
|
||||
* Used by UPS_AI_ConvAgent_InteractionComponent (on the player pawn)
|
||||
* to discover agents for distance/frustum-based selection.
|
||||
*/
|
||||
UCLASS()
|
||||
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_InteractionSubsystem : public UWorldSubsystem
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
public:
|
||||
/** Register an agent component. Called automatically from ElevenLabsComponent::BeginPlay. */
|
||||
void RegisterAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Agent);
|
||||
|
||||
/** Unregister an agent component. Called automatically from ElevenLabsComponent::EndPlay. */
|
||||
void UnregisterAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Agent);
|
||||
|
||||
/** Get all currently registered agent components (weak pointers, may contain stale entries that are cleaned on access). */
|
||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
|
||||
TArray<UPS_AI_ConvAgent_ElevenLabsComponent*> GetRegisteredAgents() const;
|
||||
|
||||
/** Get the number of registered agents. */
|
||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|Interaction")
|
||||
int32 GetNumRegisteredAgents() const;
|
||||
|
||||
private:
|
||||
/** Internal registry. Uses weak pointers so destroyed actors don't cause dangling references. */
|
||||
TArray<TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent>> RegisteredAgents;
|
||||
};
|
||||
Loading…
x
Reference in New Issue
Block a user