diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp index a9d04c1..20b0435 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp @@ -2,9 +2,11 @@ #include "PS_AI_ConvAgent_ElevenLabsComponent.h" #include "PS_AI_ConvAgent_MicrophoneCaptureComponent.h" +#include "PS_AI_ConvAgent_InteractionSubsystem.h" #include "PS_AI_ConvAgent.h" #include "Components/AudioComponent.h" +#include "Sound/SoundAttenuation.h" #include "Sound/SoundWaveProcedural.h" #include "GameFramework/Actor.h" @@ -28,10 +30,28 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::BeginPlay() { Super::BeginPlay(); InitAudioPlayback(); + + // Auto-register with the interaction subsystem so InteractionComponents can discover us. + if (UWorld* World = GetWorld()) + { + if (UPS_AI_ConvAgent_InteractionSubsystem* Sub = World->GetSubsystem()) + { + Sub->RegisterAgent(this); + } + } } void UPS_AI_ConvAgent_ElevenLabsComponent::EndPlay(const EEndPlayReason::Type EndPlayReason) { + // Unregister from the interaction subsystem before tearing down. + if (UWorld* World = GetWorld()) + { + if (UPS_AI_ConvAgent_InteractionSubsystem* Sub = World->GetSubsystem()) + { + Sub->UnregisterAgent(this); + } + } + EndConversation(); Super::EndPlay(EndPlayReason); } @@ -256,39 +276,45 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StartListening() WebSocketProxy->SendUserTurnStart(); } - // Find the microphone component on our owner actor, or create one. - UPS_AI_ConvAgent_MicrophoneCaptureComponent* Mic = - GetOwner()->FindComponentByClass(); + // External mic mode: the InteractionComponent on the pawn manages the mic and + // feeds audio via FeedExternalAudio(). We only manage turn state here. + if (!bExternalMicManagement) + { + // Find the microphone component on our owner actor, or create one. + UPS_AI_ConvAgent_MicrophoneCaptureComponent* Mic = + GetOwner()->FindComponentByClass(); - if (!Mic) - { - Mic = NewObject(GetOwner(), - TEXT("PS_AI_ConvAgent_Microphone")); - Mic->RegisterComponent(); - } + if (!Mic) + { + Mic = NewObject(GetOwner(), + TEXT("PS_AI_ConvAgent_Microphone")); + Mic->RegisterComponent(); + } - // Always remove existing binding first to prevent duplicate delegates stacking - // up if StartListening is called more than once without a matching StopListening. - Mic->OnAudioCaptured.RemoveAll(this); - Mic->OnAudioCaptured.AddUObject(this, - &UPS_AI_ConvAgent_ElevenLabsComponent::OnMicrophoneDataCaptured); - // Echo suppression: point the mic at our atomic bAgentSpeaking flag so it skips - // capture entirely (before resampling) while the agent is speaking. - // In Server VAD + interruption mode, disable echo suppression so the server - // receives the user's voice even during agent playback — the server's own VAD - // handles echo filtering and interruption detection. - if (TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server && bAllowInterruption) - { - Mic->EchoSuppressFlag = nullptr; + // Always remove existing binding first to prevent duplicate delegates stacking + // up if StartListening is called more than once without a matching StopListening. + Mic->OnAudioCaptured.RemoveAll(this); + Mic->OnAudioCaptured.AddUObject(this, + &UPS_AI_ConvAgent_ElevenLabsComponent::OnMicrophoneDataCaptured); + // Echo suppression: point the mic at our atomic bAgentSpeaking flag so it skips + // capture entirely (before resampling) while the agent is speaking. + // In Server VAD + interruption mode, disable echo suppression so the server + // receives the user's voice even during agent playback — the server's own VAD + // handles echo filtering and interruption detection. + if (TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server && bAllowInterruption) + { + Mic->EchoSuppressFlag = nullptr; + } + else + { + Mic->EchoSuppressFlag = &bAgentSpeaking; + } + Mic->StartCapture(); } - else - { - Mic->EchoSuppressFlag = &bAgentSpeaking; - } - Mic->StartCapture(); const double T = TurnStartTime - SessionStartTime; - UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log, TEXT("[T+%.2fs] [Turn %d] Mic opened — user speaking."), T, TurnIndex); + UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log, TEXT("[T+%.2fs] [Turn %d] Mic opened%s — user speaking."), + T, TurnIndex, bExternalMicManagement ? TEXT(" (external)") : TEXT("")); } void UPS_AI_ConvAgent_ElevenLabsComponent::StopListening() @@ -296,11 +322,15 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StopListening() if (!bIsListening) return; bIsListening = false; - if (UPS_AI_ConvAgent_MicrophoneCaptureComponent* Mic = - GetOwner() ? GetOwner()->FindComponentByClass() : nullptr) + // External mic mode: mic is managed by the InteractionComponent, not us. + if (!bExternalMicManagement) { - Mic->StopCapture(); - Mic->OnAudioCaptured.RemoveAll(this); + if (UPS_AI_ConvAgent_MicrophoneCaptureComponent* Mic = + GetOwner() ? GetOwner()->FindComponentByClass() : nullptr) + { + Mic->StopCapture(); + Mic->OnAudioCaptured.RemoveAll(this); + } } // Flush any partially-accumulated mic audio before signalling end-of-turn. @@ -383,6 +413,30 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::InterruptAgent() StopAgentAudio(); } +void UPS_AI_ConvAgent_ElevenLabsComponent::FeedExternalAudio(const TArray& FloatPCM) +{ + // Same logic as OnMicrophoneDataCaptured but called from an external source + // (e.g. InteractionComponent on the pawn) instead of a local mic component. + if (!IsConnected() || !bIsListening) return; + + // Echo suppression: skip sending mic audio while the agent is speaking. + if (bAgentSpeaking && !(TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server && bAllowInterruption)) + { + return; + } + + TArray PCMBytes = FloatPCMToInt16Bytes(FloatPCM); + + FScopeLock Lock(&MicSendLock); + MicAccumulationBuffer.Append(PCMBytes); + + if (MicAccumulationBuffer.Num() >= GetMicChunkMinBytes()) + { + WebSocketProxy->SendAudioChunk(MicAccumulationBuffer); + MicAccumulationBuffer.Reset(); + } +} + // ───────────────────────────────────────────────────────────────────────────── // State queries // ───────────────────────────────────────────────────────────────────────────── @@ -647,6 +701,15 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::InitAudioPlayback() AudioPlaybackComponent->bAutoActivate = false; AudioPlaybackComponent->SetSound(ProceduralSoundWave); + // Spatialization: if an attenuation asset is set, enable 3D audio so the + // agent's voice attenuates with distance and pans spatially for the listener. + if (SoundAttenuation) + { + AudioPlaybackComponent->AttenuationSettings = SoundAttenuation; + AudioPlaybackComponent->bOverrideAttenuation = false; + AudioPlaybackComponent->bAllowSpatialization = true; + } + // When the procedural sound wave needs more audio data, pull from our queue. ProceduralSoundWave->OnSoundWaveProceduralUnderflow = FOnSoundWaveProceduralUnderflow::CreateUObject( diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_FacialExpressionComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_FacialExpressionComponent.cpp index cb74bce..65f01aa 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_FacialExpressionComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_FacialExpressionComponent.cpp @@ -4,7 +4,7 @@ #include "PS_AI_ConvAgent_ElevenLabsComponent.h" #include "PS_AI_ConvAgent_EmotionPoseMap.h" #include "Animation/AnimSequence.h" -#include "Animation/AnimData/IAnimationDataModel.h" +#include "Animation/AnimCurveTypes.h" DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_FacialExpr, Log, All); @@ -156,10 +156,8 @@ TMap UPS_AI_ConvAgent_FacialExpressionComponent::EvaluateAnimCurve TMap CurveValues; if (!AnimSeq) return CurveValues; - const IAnimationDataModel* DataModel = AnimSeq->GetDataModel(); - if (!DataModel) return CurveValues; - - const TArray& FloatCurves = DataModel->GetFloatCurves(); + // Use runtime GetCurveData() — GetDataModel() is editor-only in UE 5.5. + const TArray& FloatCurves = AnimSeq->GetCurveData().FloatCurves; for (const FFloatCurve& Curve : FloatCurves) { const float Value = Curve.FloatCurve.Eval(Time); diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp new file mode 100644 index 0000000..7e423d2 --- /dev/null +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp @@ -0,0 +1,264 @@ +// Copyright ASTERION. All Rights Reserved. + +#include "PS_AI_ConvAgent_InteractionComponent.h" +#include "PS_AI_ConvAgent_InteractionSubsystem.h" +#include "PS_AI_ConvAgent_ElevenLabsComponent.h" +#include "PS_AI_ConvAgent_MicrophoneCaptureComponent.h" + +#include "GameFramework/Pawn.h" +#include "GameFramework/PlayerController.h" +#include "Camera/PlayerCameraManager.h" + +DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_Select, Log, All); + +// ───────────────────────────────────────────────────────────────────────────── +// Constructor +// ───────────────────────────────────────────────────────────────────────────── +UPS_AI_ConvAgent_InteractionComponent::UPS_AI_ConvAgent_InteractionComponent() +{ + PrimaryComponentTick.bCanEverTick = true; + PrimaryComponentTick.TickInterval = 1.0f / 30.0f; // 30 Hz is enough for selection evaluation. +} + +// ───────────────────────────────────────────────────────────────────────────── +// Lifecycle +// ───────────────────────────────────────────────────────────────────────────── +void UPS_AI_ConvAgent_InteractionComponent::BeginPlay() +{ + Super::BeginPlay(); + + // Create mic capture component on the pawn. + AActor* Owner = GetOwner(); + if (!Owner) return; + + MicComponent = Owner->FindComponentByClass(); + if (!MicComponent) + { + MicComponent = NewObject( + Owner, TEXT("PS_AI_ConvAgent_InteractionMic")); + MicComponent->RegisterComponent(); + } + + // Bind mic audio callback. + MicComponent->OnAudioCaptured.AddUObject(this, + &UPS_AI_ConvAgent_InteractionComponent::OnMicAudioCaptured); + + UE_LOG(LogPS_AI_ConvAgent_Select, Log, TEXT("InteractionComponent initialized on %s."), *Owner->GetName()); +} + +void UPS_AI_ConvAgent_InteractionComponent::EndPlay(const EEndPlayReason::Type EndPlayReason) +{ + if (MicComponent) + { + MicComponent->StopCapture(); + MicComponent->OnAudioCaptured.RemoveAll(this); + } + + // Fire deselection event for cleanup. + if (UPS_AI_ConvAgent_ElevenLabsComponent* Agent = SelectedAgent.Get()) + { + SelectedAgent.Reset(); + OnAgentDeselected.Broadcast(Agent); + } + + Super::EndPlay(EndPlayReason); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Tick — agent selection +// ───────────────────────────────────────────────────────────────────────────── +void UPS_AI_ConvAgent_InteractionComponent::TickComponent(float DeltaTime, ELevelTick TickType, + FActorComponentTickFunction* ThisTickFunction) +{ + Super::TickComponent(DeltaTime, TickType, ThisTickFunction); + + UPS_AI_ConvAgent_ElevenLabsComponent* BestAgent = EvaluateBestAgent(); + + // Check if selection changed. + UPS_AI_ConvAgent_ElevenLabsComponent* CurrentAgent = SelectedAgent.Get(); + if (BestAgent != CurrentAgent) + { + SetSelectedAgent(BestAgent); + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Selection evaluation +// ───────────────────────────────────────────────────────────────────────────── +UPS_AI_ConvAgent_ElevenLabsComponent* UPS_AI_ConvAgent_InteractionComponent::EvaluateBestAgent() const +{ + UWorld* World = GetWorld(); + if (!World) return nullptr; + + UPS_AI_ConvAgent_InteractionSubsystem* Subsystem = World->GetSubsystem(); + if (!Subsystem) return nullptr; + + TArray Agents = Subsystem->GetRegisteredAgents(); + if (Agents.Num() == 0) return nullptr; + + FVector ViewLocation; + FVector ViewDirection; + GetPawnViewPoint(ViewLocation, ViewDirection); + + const float MaxDistSq = MaxInteractionDistance * MaxInteractionDistance; + const float CosViewCone = FMath::Cos(FMath::DegreesToRadians(ViewConeHalfAngle)); + const float CosStickyAngle = FMath::Cos(FMath::DegreesToRadians(SelectionStickyAngle)); + + UPS_AI_ConvAgent_ElevenLabsComponent* BestCandidate = nullptr; + float BestAngle = MAX_FLT; // Smallest angle = closest to center of view. + + UPS_AI_ConvAgent_ElevenLabsComponent* CurrentAgent = SelectedAgent.Get(); + + for (UPS_AI_ConvAgent_ElevenLabsComponent* Agent : Agents) + { + AActor* AgentActor = Agent->GetOwner(); + if (!AgentActor) continue; + + const FVector AgentLocation = AgentActor->GetActorLocation(); + const FVector ToAgent = AgentLocation - ViewLocation; + const float DistSq = ToAgent.SizeSquared(); + + // Distance check. + if (DistSq > MaxDistSq) continue; + + float AngleDeg = 0.0f; + if (bRequireLookAt) + { + const FVector DirToAgent = ToAgent.GetSafeNormal(); + const float Dot = FVector::DotProduct(ViewDirection, DirToAgent); + AngleDeg = FMath::RadiansToDegrees(FMath::Acos(FMath::Clamp(Dot, -1.0f, 1.0f))); + + // Use wider sticky angle for the currently selected agent. + const float ConeThreshold = (Agent == CurrentAgent) ? SelectionStickyAngle : ViewConeHalfAngle; + if (AngleDeg > ConeThreshold) continue; + } + else + { + // When not requiring look-at, use distance as the ranking metric. + // Convert distance to a comparable "angle" value (smaller distance = smaller value). + AngleDeg = FMath::Sqrt(DistSq); + } + + if (bDebug && DebugVerbosity >= 2) + { + UE_LOG(LogPS_AI_ConvAgent_Select, Log, + TEXT(" Candidate: %s | Dist=%.0f | Angle=%.1f%s"), + *AgentActor->GetName(), + FMath::Sqrt(DistSq), + AngleDeg, + (Agent == CurrentAgent) ? TEXT(" (current)") : TEXT("")); + } + + if (AngleDeg < BestAngle) + { + BestAngle = AngleDeg; + BestCandidate = Agent; + } + } + + return BestCandidate; +} + +void UPS_AI_ConvAgent_InteractionComponent::SetSelectedAgent(UPS_AI_ConvAgent_ElevenLabsComponent* NewAgent) +{ + UPS_AI_ConvAgent_ElevenLabsComponent* OldAgent = SelectedAgent.Get(); + + // Deselect old agent. + if (OldAgent) + { + SelectedAgent.Reset(); + + if (bDebug) + { + UE_LOG(LogPS_AI_ConvAgent_Select, Log, TEXT("Agent deselected: %s"), + OldAgent->GetOwner() ? *OldAgent->GetOwner()->GetName() : TEXT("(null)")); + } + + OnAgentDeselected.Broadcast(OldAgent); + } + + // Select new agent. + if (NewAgent) + { + SelectedAgent = NewAgent; + + if (bDebug) + { + UE_LOG(LogPS_AI_ConvAgent_Select, Log, TEXT("Agent selected: %s"), + NewAgent->GetOwner() ? *NewAgent->GetOwner()->GetName() : TEXT("(null)")); + } + + // Ensure mic is capturing so we can route audio to the new agent. + if (MicComponent && !MicComponent->IsCapturing()) + { + MicComponent->StartCapture(); + } + + OnAgentSelected.Broadcast(NewAgent); + } + else + { + if (bDebug) + { + UE_LOG(LogPS_AI_ConvAgent_Select, Log, TEXT("No agent in range.")); + } + + OnNoAgentInRange.Broadcast(); + } +} + +void UPS_AI_ConvAgent_InteractionComponent::GetPawnViewPoint(FVector& OutLocation, FVector& OutDirection) const +{ + AActor* Owner = GetOwner(); + if (!Owner) + { + OutLocation = FVector::ZeroVector; + OutDirection = FVector::ForwardVector; + return; + } + + // Try to get the player controller's camera view (most accurate for first/third person). + if (APawn* Pawn = Cast(Owner)) + { + if (APlayerController* PC = Cast(Pawn->GetController())) + { + FRotator ViewRotation; + PC->GetPlayerViewPoint(OutLocation, ViewRotation); + OutDirection = ViewRotation.Vector(); + return; + } + } + + // Fallback: use actor location and forward. + OutLocation = Owner->GetActorLocation(); + OutDirection = Owner->GetActorForwardVector(); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Blueprint API +// ───────────────────────────────────────────────────────────────────────────── +UPS_AI_ConvAgent_ElevenLabsComponent* UPS_AI_ConvAgent_InteractionComponent::GetSelectedAgent() const +{ + return SelectedAgent.Get(); +} + +void UPS_AI_ConvAgent_InteractionComponent::ForceSelectAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Agent) +{ + SetSelectedAgent(Agent); +} + +void UPS_AI_ConvAgent_InteractionComponent::ClearSelection() +{ + SetSelectedAgent(nullptr); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Mic routing +// ───────────────────────────────────────────────────────────────────────────── +void UPS_AI_ConvAgent_InteractionComponent::OnMicAudioCaptured(const TArray& FloatPCM) +{ + UPS_AI_ConvAgent_ElevenLabsComponent* Agent = SelectedAgent.Get(); + if (!Agent) return; + + Agent->FeedExternalAudio(FloatPCM); +} diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionSubsystem.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionSubsystem.cpp new file mode 100644 index 0000000..5d2e279 --- /dev/null +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionSubsystem.cpp @@ -0,0 +1,67 @@ +// Copyright ASTERION. All Rights Reserved. + +#include "PS_AI_ConvAgent_InteractionSubsystem.h" +#include "PS_AI_ConvAgent_ElevenLabsComponent.h" + +DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_Interaction, Log, All); + +// ───────────────────────────────────────────────────────────────────────────── +// Registration +// ───────────────────────────────────────────────────────────────────────────── +void UPS_AI_ConvAgent_InteractionSubsystem::RegisterAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Agent) +{ + if (!Agent) return; + + // Avoid duplicates. + for (const TWeakObjectPtr& Existing : RegisteredAgents) + { + if (Existing.Get() == Agent) return; + } + + RegisteredAgents.Add(Agent); + UE_LOG(LogPS_AI_ConvAgent_Interaction, Log, TEXT("Agent registered: %s (%d total)"), + *Agent->GetOwner()->GetName(), RegisteredAgents.Num()); +} + +void UPS_AI_ConvAgent_InteractionSubsystem::UnregisterAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Agent) +{ + if (!Agent) return; + + RegisteredAgents.RemoveAll([Agent](const TWeakObjectPtr& Weak) + { + return !Weak.IsValid() || Weak.Get() == Agent; + }); + + UE_LOG(LogPS_AI_ConvAgent_Interaction, Log, TEXT("Agent unregistered: %s (%d remaining)"), + Agent->GetOwner() ? *Agent->GetOwner()->GetName() : TEXT("(destroyed)"), + RegisteredAgents.Num()); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Queries +// ───────────────────────────────────────────────────────────────────────────── +TArray UPS_AI_ConvAgent_InteractionSubsystem::GetRegisteredAgents() const +{ + TArray Result; + Result.Reserve(RegisteredAgents.Num()); + + for (const TWeakObjectPtr& Weak : RegisteredAgents) + { + if (UPS_AI_ConvAgent_ElevenLabsComponent* Agent = Weak.Get()) + { + Result.Add(Agent); + } + } + + return Result; +} + +int32 UPS_AI_ConvAgent_InteractionSubsystem::GetNumRegisteredAgents() const +{ + int32 Count = 0; + for (const TWeakObjectPtr& Weak : RegisteredAgents) + { + if (Weak.IsValid()) ++Count; + } + return Count; +} diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_LipSyncComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_LipSyncComponent.cpp index 8577764..3a166d4 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_LipSyncComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_LipSyncComponent.cpp @@ -8,7 +8,7 @@ #include "Engine/SkeletalMesh.h" #include "Animation/MorphTarget.h" #include "Animation/AnimSequence.h" -#include "Animation/AnimData/IAnimationDataModel.h" +#include "Animation/AnimCurveTypes.h" #include "GameFramework/Actor.h" DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_LipSync, Log, All); @@ -444,17 +444,9 @@ void UPS_AI_ConvAgent_LipSyncComponent::ExtractPoseCurves(const FName& VisemeNam { if (!AnimSeq) return; - const IAnimationDataModel* DataModel = AnimSeq->GetDataModel(); - if (!DataModel) - { - UE_LOG(LogPS_AI_ConvAgent_LipSync, Warning, - TEXT("Pose '%s' (%s): No DataModel available — skipping."), - *VisemeName.ToString(), *AnimSeq->GetName()); - return; - } - + // Use runtime GetCurveData() — GetDataModel() is editor-only in UE 5.5. TMap CurveValues; - const TArray& FloatCurves = DataModel->GetFloatCurves(); + const TArray& FloatCurves = AnimSeq->GetCurveData().FloatCurves; for (const FFloatCurve& Curve : FloatCurves) { diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h index a6535d4..1757a78 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h @@ -11,6 +11,7 @@ #include "PS_AI_ConvAgent_ElevenLabsComponent.generated.h" class UAudioComponent; +class USoundAttenuation; class UPS_AI_ConvAgent_MicrophoneCaptureComponent; // ───────────────────────────────────────────────────────────────────────────── @@ -168,6 +169,26 @@ public: ToolTip = "Seconds to wait for a server response after the user stops speaking.\nFires OnAgentResponseTimeout if exceeded. Normal latency is 0.1-0.8s.\nSet to 0 to disable. Default: 10s.")) float ResponseTimeoutSeconds = 10.0f; + // ── Multi-agent / external mic ─────────────────────────────────────────── + + /** When true, StartListening/StopListening manage the turn state but do NOT + * create or control a local microphone component. Audio is instead fed via + * FeedExternalAudio() from an external source (e.g. InteractionComponent on the pawn). + * Use this when a centralized mic on the player pawn routes audio to agents. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs", + meta = (ToolTip = "External mic mode: turn management only, no local mic.\nAudio is fed via FeedExternalAudio() from an external source.")) + bool bExternalMicManagement = false; + + // ── Audio spatialization ───────────────────────────────────────────────── + + /** Optional sound attenuation settings for spatializing the agent's voice. + * When set, the agent's audio playback will be spatialized in 3D — volume + * and panning change based on distance and direction from the listener. + * Leave null for non-spatialized (2D) audio (default, same as before). */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs|Audio", + meta = (ToolTip = "Sound attenuation for 3D spatialized audio.\nLeave null for non-spatialized (2D) playback.")) + USoundAttenuation* SoundAttenuation = nullptr; + // ── Debug ──────────────────────────────────────────────────────────────── /** Enable debug logging for this component. @@ -300,6 +321,15 @@ public: UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs") void InterruptAgent(); + /** + * Feed microphone audio from an external source (e.g. InteractionComponent on the pawn). + * Use this instead of the local mic when bExternalMicManagement is true. + * The component must be connected and listening (StartListening called) for audio to be sent. + * @param FloatPCM Float32 samples, 16000 Hz mono (same format as MicrophoneCaptureComponent output). + */ + UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs") + void FeedExternalAudio(const TArray& FloatPCM); + // ── State queries ───────────────────────────────────────────────────────── UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|ElevenLabs") diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionComponent.h new file mode 100644 index 0000000..68a706d --- /dev/null +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionComponent.h @@ -0,0 +1,163 @@ +// Copyright ASTERION. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Components/ActorComponent.h" +#include "PS_AI_ConvAgent_InteractionComponent.generated.h" + +class UPS_AI_ConvAgent_ElevenLabsComponent; +class UPS_AI_ConvAgent_MicrophoneCaptureComponent; + +// ───────────────────────────────────────────────────────────────────────────── +// Delegates +// ───────────────────────────────────────────────────────────────────────────── + +/** Fired when a new agent is selected (enters range + view cone). */ +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnConvAgentSelected, + UPS_AI_ConvAgent_ElevenLabsComponent*, Agent); + +/** Fired when the previously selected agent loses selection. */ +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnConvAgentDeselected, + UPS_AI_ConvAgent_ElevenLabsComponent*, Agent); + +/** Fired when no agent is within interaction range/view. */ +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnNoConvAgentInRange); + +// ───────────────────────────────────────────────────────────────────────────── +// UPS_AI_ConvAgent_InteractionComponent +// +// Place on the player pawn to enable multi-agent interaction. +// Evaluates all registered agents each tick, selects the best candidate +// based on distance and view direction (frustum cone), and routes the +// pawn's microphone audio to the selected agent. +// +// Only one agent can be selected at a time. Selection is sticky (hysteresis) +// to prevent flickering when the player looks between agents. +// +// Workflow: +// 1. Add this component to your player pawn Blueprint. +// 2. Configure MaxInteractionDistance, ViewConeHalfAngle, etc. +// 3. Bind to OnAgentSelected / OnAgentDeselected to update posture, +// UI, and other agent-specific logic from Blueprint. +// 4. Each agent's ElevenLabsComponent should have bExternalMicManagement = true. +// 5. Agents manage their own WebSocket connections independently. +// ───────────────────────────────────────────────────────────────────────────── +UCLASS(ClassGroup = "PS AI ConvAgent", meta = (BlueprintSpawnableComponent), + DisplayName = "PS AI ConvAgent Interaction") +class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_InteractionComponent : public UActorComponent +{ + GENERATED_BODY() + +public: + UPS_AI_ConvAgent_InteractionComponent(); + + // ── Configuration ───────────────────────────────────────────────────────── + + /** Maximum distance (cm) at which an agent can be selected. Agents beyond this are ignored. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction", + meta = (ClampMin = "0", + ToolTip = "Max distance in cm to interact with an agent.\nAgents beyond this distance are never selected.")) + float MaxInteractionDistance = 300.0f; + + /** Half-angle (degrees) of the view cone used for agent selection. + * The cone is centered on the pawn's view direction (camera or control rotation). + * Agents outside this cone are not considered unless they are the current sticky selection. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction", + meta = (ClampMin = "0", ClampMax = "180", + ToolTip = "Half-angle of the view cone (degrees).\nAgents outside this cone are not selected.\n45 = 90-degree total FOV.")) + float ViewConeHalfAngle = 45.0f; + + /** Wider half-angle (degrees) used for the currently selected agent. + * This provides hysteresis so the player can look slightly away from + * the current agent without losing selection (prevents flickering). */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction", + meta = (ClampMin = "0", ClampMax = "180", + ToolTip = "Sticky cone half-angle for the current agent (degrees).\nMust be >= ViewConeHalfAngle for hysteresis to work.\n60 = agent stays selected until 120-degree total cone is exceeded.")) + float SelectionStickyAngle = 60.0f; + + /** When false, only distance matters for selection (no view cone check). + * The closest agent within MaxInteractionDistance is always selected. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction", + meta = (ToolTip = "Require the player to look at an agent to select it.\nWhen false, the closest agent within range is always selected.")) + bool bRequireLookAt = true; + + // ── Debug ──────────────────────────────────────────────────────────────── + + /** Enable debug logging for this component. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Debug") + bool bDebug = false; + + /** Verbosity level when bDebug is true. + * 0 = selection changes only, 1 = candidate evaluation, 2 = per-frame data. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Debug", + meta = (ClampMin = "0", ClampMax = "3", EditCondition = "bDebug")) + int32 DebugVerbosity = 1; + + // ── Events ──────────────────────────────────────────────────────────────── + + /** Fired when a new agent enters selection. Use this to set posture targets, show UI, etc. */ + UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|Interaction|Events", + meta = (ToolTip = "Fires when a new agent is selected.\nSet posture targets, update UI, etc.")) + FOnConvAgentSelected OnAgentSelected; + + /** Fired when the previously selected agent loses selection. */ + UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|Interaction|Events", + meta = (ToolTip = "Fires when the current agent is deselected.\nClear posture targets, hide UI, etc.")) + FOnConvAgentDeselected OnAgentDeselected; + + /** Fired when no agent is within range or view cone. */ + UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|Interaction|Events", + meta = (ToolTip = "Fires when no agent meets the distance/view criteria.")) + FOnNoConvAgentInRange OnNoAgentInRange; + + // ── Blueprint API ───────────────────────────────────────────────────────── + + /** Get the currently selected agent (null if none). */ + UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|Interaction") + UPS_AI_ConvAgent_ElevenLabsComponent* GetSelectedAgent() const; + + /** Manually select a specific agent, bypassing distance/view checks. + * Pass null to clear the selection. Automatic selection resumes next tick. */ + UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction") + void ForceSelectAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Agent); + + /** Clear the current selection. Automatic selection resumes next tick. */ + UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction") + void ClearSelection(); + + // ── UActorComponent overrides ──────────────────────────────────────────── + virtual void BeginPlay() override; + virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override; + virtual void TickComponent(float DeltaTime, ELevelTick TickType, + FActorComponentTickFunction* ThisTickFunction) override; + +private: + // ── Selection logic ────────────────────────────────────────────────────── + + /** Evaluate all registered agents, return the best candidate (or null). */ + UPS_AI_ConvAgent_ElevenLabsComponent* EvaluateBestAgent() const; + + /** Apply a new selection — fire events, reroute mic. */ + void SetSelectedAgent(UPS_AI_ConvAgent_ElevenLabsComponent* NewAgent); + + /** Get the pawn's view location and direction (uses camera or control rotation). */ + void GetPawnViewPoint(FVector& OutLocation, FVector& OutDirection) const; + + // ── Mic routing ────────────────────────────────────────────────────────── + + /** Forward captured mic audio to the currently selected agent. */ + void OnMicAudioCaptured(const TArray& FloatPCM); + + // ── State ──────────────────────────────────────────────────────────────── + + /** Currently selected agent (weak pointer for safety). */ + TWeakObjectPtr SelectedAgent; + + /** Microphone capture component (created on the pawn in BeginPlay). */ + UPROPERTY() + UPS_AI_ConvAgent_MicrophoneCaptureComponent* MicComponent = nullptr; + + /** True while ForceSelectAgent is active (suppresses automatic re-evaluation for one frame). */ + bool bForceSelectionActive = false; +}; diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionSubsystem.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionSubsystem.h new file mode 100644 index 0000000..486812f --- /dev/null +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionSubsystem.h @@ -0,0 +1,42 @@ +// Copyright ASTERION. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Subsystems/WorldSubsystem.h" +#include "PS_AI_ConvAgent_InteractionSubsystem.generated.h" + +class UPS_AI_ConvAgent_ElevenLabsComponent; + +/** + * World subsystem that maintains a registry of all conversational agent + * components in the current world. Agents auto-register on BeginPlay + * and unregister on EndPlay. + * + * Used by UPS_AI_ConvAgent_InteractionComponent (on the player pawn) + * to discover agents for distance/frustum-based selection. + */ +UCLASS() +class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_InteractionSubsystem : public UWorldSubsystem +{ + GENERATED_BODY() + +public: + /** Register an agent component. Called automatically from ElevenLabsComponent::BeginPlay. */ + void RegisterAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Agent); + + /** Unregister an agent component. Called automatically from ElevenLabsComponent::EndPlay. */ + void UnregisterAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Agent); + + /** Get all currently registered agent components (weak pointers, may contain stale entries that are cleaned on access). */ + UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction") + TArray GetRegisteredAgents() const; + + /** Get the number of registered agents. */ + UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|Interaction") + int32 GetNumRegisteredAgents() const; + +private: + /** Internal registry. Uses weak pointers so destroyed actors don't cause dangling references. */ + TArray> RegisteredAgents; +};