diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.dll b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.dll index beab3b9..9fbf803 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.dll and b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.dll differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.exp b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.exp index 8271cd2..6b073f7 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.exp and b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.exp differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_0.exe b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_0.exe deleted file mode 100644 index eb924d3..0000000 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_0.exe and /dev/null differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_0.exp b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_0.exp index ceb6481..c3e412a 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_0.exp and b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_0.exp differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_0.pdb b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_0.pdb index d647cf9..7a5796d 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_0.pdb and b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_0.pdb differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_1.exe b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_1.exe index d847e7a..1ea4d84 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_1.exe and b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_1.exe differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_1.pdb b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_1.pdb index f904688..c838668 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_1.pdb and b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_1.pdb differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_2.exe b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_2.exe index c86ce86..fc1ed44 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_2.exe and b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_2.exe differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_2.pdb b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_2.pdb index 563cce3..ab76d8f 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_2.pdb and b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.patch_2.pdb differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.pdb b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.pdb index 5b99053..ee73809 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.pdb and b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_Behavior.pdb differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_BehaviorEditor.dll b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_BehaviorEditor.dll index b870f6b..5e25371 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_BehaviorEditor.dll and b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_BehaviorEditor.dll differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_BehaviorEditor.exp b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_BehaviorEditor.exp index 4f83f9f..996d269 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_BehaviorEditor.exp and b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_BehaviorEditor.exp differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_BehaviorEditor.patch_0.pdb b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_BehaviorEditor.patch_0.pdb index 12073ee..2c57992 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_BehaviorEditor.patch_0.pdb and b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_BehaviorEditor.patch_0.pdb differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_BehaviorEditor.pdb b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_BehaviorEditor.pdb index 12b31a3..2a8fd2d 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_BehaviorEditor.pdb and b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Binaries/Win64/UnrealEditor-PS_AI_BehaviorEditor.pdb differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Source/PS_AI_Behavior/Private/PS_AI_Behavior_AIController.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Source/PS_AI_Behavior/Private/PS_AI_Behavior_AIController.cpp index c602414..ff81b07 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Source/PS_AI_Behavior/Private/PS_AI_Behavior_AIController.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Source/PS_AI_Behavior/Private/PS_AI_Behavior_AIController.cpp @@ -100,6 +100,9 @@ void APS_AI_Behavior_AIController::OnUnPossess() ClearGazeTarget(); CachedGazeComponent = nullptr; CachedConvAgentComponent = nullptr; + CachedMicComponent = nullptr; + bConversationPaused = false; + bUserHasSpokenInConversation = false; ProximityGazeTarget = nullptr; PersonalityComp = nullptr; @@ -199,6 +202,12 @@ void APS_AI_Behavior_AIController::SetupBlackboard() PreferCoverEntry.EntryName = PS_AI_Behavior_BB::PreferCover; PreferCoverEntry.KeyType = NewObject(BlackboardAsset); BlackboardAsset->Keys.Add(PreferCoverEntry); + + // ConversationPaused (bool: NPC paused during active conversation) + FBlackboardEntry ConvPausedEntry; + ConvPausedEntry.EntryName = PS_AI_Behavior_BB::ConversationPaused; + ConvPausedEntry.KeyType = NewObject(BlackboardAsset); + BlackboardAsset->Keys.Add(ConvPausedEntry); } UBlackboardComponent* RawBBComp = nullptr; @@ -536,12 +545,29 @@ void APS_AI_Behavior_AIController::TryBindGazeComponent() } } + // Cache MicrophoneCaptureComponent for local VAD + // The mic lives on the PLAYER's Pawn, not the NPC — find it on the first player + static UClass* MicClass = nullptr; + if (!MicClass) + { + MicClass = LoadClass(nullptr, + TEXT("/Script/PS_AI_ConvAgent.PS_AI_ConvAgent_MicrophoneCaptureComponent")); + } + if (MicClass) + { + MicProp_bIsUserSpeaking = CastField( + MicClass->FindPropertyByName(TEXT("bIsUserSpeaking"))); + // Don't cache the component here — resolve dynamically in UpdateGazeTarget + // because the player Pawn may not exist yet at OnPossess time + } + UE_LOG(LogPS_AI_Behavior, Log, - TEXT("[%s] Gaze bridge bound: TargetActor=%s, BodyTracking=%s, Conversation=%s"), + TEXT("[%s] Gaze bridge bound: TargetActor=%s, BodyTracking=%s, Conversation=%s, VAD=%s"), *GetName(), GazeProp_TargetActor ? TEXT("OK") : TEXT("MISS"), GazeProp_bEnableBodyTracking ? TEXT("OK") : TEXT("MISS"), - ConvProp_bNetIsConversing ? TEXT("OK") : TEXT("N/A")); + ConvProp_bNetIsConversing ? TEXT("OK") : TEXT("N/A"), + MicProp_bIsUserSpeaking ? TEXT("OK") : TEXT("N/A")); } void APS_AI_Behavior_AIController::SetGazeTarget(AActor* Target, bool bEnableBody) @@ -628,13 +654,117 @@ void APS_AI_Behavior_AIController::UpdateGazeTarget(float DeltaSeconds) } } - // ── Priority 2: Conversation active → do NOT touch gaze ───────── - // TODO: Add movement pause when local VAD detects user speech (not just connection) - if (IsConversationActiveOnPawn()) + // ── Priority 2: Conversation active → manage movement pause via local VAD { - ProximityGazeTarget = nullptr; - ProximityGazeDuration = 0.0f; - return; + const bool bConversing = IsConversationActiveOnPawn(); + + + // Check local VAD: is the user actually speaking? + // The mic is on the PLAYER's Pawn — find it dynamically + bool bUserSpeaking = false; + if (MicProp_bIsUserSpeaking) + { + // Try cached mic first + UActorComponent* MicComp = CachedMicComponent.Get(); + if (!MicComp) + { + // Find mic on any player pawn + static UClass* MicClass = LoadClass(nullptr, + TEXT("/Script/PS_AI_ConvAgent.PS_AI_ConvAgent_MicrophoneCaptureComponent")); + if (MicClass) + { + for (FConstPlayerControllerIterator It = GetWorld()->GetPlayerControllerIterator(); It; ++It) + { + if (APlayerController* PC = It->Get()) + { + if (APawn* PlayerPawn = PC->GetPawn()) + { + MicComp = PlayerPawn->FindComponentByClass(MicClass); + if (MicComp) + { + CachedMicComponent = MicComp; + break; + } + } + } + } + } + } + + if (MicComp) + { + bUserSpeaking = MicProp_bIsUserSpeaking->GetPropertyValue_InContainer(MicComp); + } + } + + // Once user has spoken during this conversation, stay paused until conversation ends + if (bConversing && bUserSpeaking) + { + bUserHasSpokenInConversation = true; + } + + if (bConversing && bUserHasSpokenInConversation) + { + // Set BB flag to block movement branches via decorator + if (!bConversationPaused) + { + bConversationPaused = true; + + if (Blackboard) + { + Blackboard->SetValueAsBool(PS_AI_Behavior_BB::ConversationPaused, true); + } + + StopMovement(); + + APawn* ConvPawn = GetPawn(); + if (ConvPawn) + { + auto* Spline = ConvPawn->FindComponentByClass(); + if (Spline) + { + Spline->PauseFollowing(); + } + } + + UE_LOG(LogPS_AI_Behavior, Log, TEXT("[%s] User speaking — conversation pause."), *GetName()); + } + + ProximityGazeTarget = nullptr; + ProximityGazeDuration = 0.0f; + return; // Gaze managed by ConvAgent + } + + if (bConversing) + { + // Connected but user hasn't spoken yet — don't touch gaze, don't pause + ProximityGazeTarget = nullptr; + ProximityGazeDuration = 0.0f; + return; + } + + // Conversation ended — clear BB flag, resume movement + if (bConversationPaused) + { + bConversationPaused = false; + bUserHasSpokenInConversation = false; + + if (Blackboard) + { + Blackboard->SetValueAsBool(PS_AI_Behavior_BB::ConversationPaused, false); + } + + APawn* ConvPawn = GetPawn(); + if (ConvPawn) + { + if (auto* Spline = ConvPawn->FindComponentByClass()) + { + Spline->ResumeFollowing(); + } + } + + UE_LOG(LogPS_AI_Behavior, Log, TEXT("[%s] Conversation ended — resumed."), *GetName()); + } } // ── Priority 3/4: Proximity gaze ──────────────────────────────── @@ -750,3 +880,4 @@ void APS_AI_Behavior_AIController::UpdateGazeTarget(float DeltaSeconds) } #endif } + diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Source/PS_AI_Behavior/Public/PS_AI_Behavior_AIController.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Source/PS_AI_Behavior/Public/PS_AI_Behavior_AIController.h index 33791bb..5a072b2 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Source/PS_AI_Behavior/Public/PS_AI_Behavior_AIController.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Source/PS_AI_Behavior/Public/PS_AI_Behavior_AIController.h @@ -162,6 +162,14 @@ private: TWeakObjectPtr CachedConvAgentComponent; FBoolProperty* ConvProp_bNetIsConversing = nullptr; + // Mic VAD cache (local voice activity detection) + TWeakObjectPtr CachedMicComponent; + FBoolProperty* MicProp_bIsUserSpeaking = nullptr; + + // Conversation pause state + bool bConversationPaused = false; + bool bUserHasSpokenInConversation = false; + // Proximity gaze state TWeakObjectPtr ProximityGazeTarget; float ProximityGazeDuration = 0.0f; diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Source/PS_AI_Behavior/Public/PS_AI_Behavior_Definitions.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Source/PS_AI_Behavior/Public/PS_AI_Behavior_Definitions.h index 1b044c7..f8aa571 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Source/PS_AI_Behavior/Public/PS_AI_Behavior_Definitions.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Source/PS_AI_Behavior/Public/PS_AI_Behavior_Definitions.h @@ -188,4 +188,5 @@ namespace PS_AI_Behavior_BB inline const FName LastKnownTargetPosition = TEXT("LastKnownTargetPosition"); inline const FName ThreatPawnName = TEXT("ThreatPawnName"); // Debug: name of the owning Pawn behind ThreatActor inline const FName PreferCover = TEXT("PreferCover"); // Bool: personality-driven cover preference cycle + inline const FName ConversationPaused = TEXT("ConversationPaused"); // Bool: NPC paused during active conversation } diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.dll b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.dll index c7eae63..e4d817f 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.dll and b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.dll differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.exp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.exp index b51d938..125ac61 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.exp and b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.exp differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.patch_0.exe b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.patch_0.exe deleted file mode 100644 index 441ea93..0000000 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.patch_0.exe and /dev/null differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.patch_0.exp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.patch_0.exp index c393b5e..6fe87fa 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.patch_0.exp and b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.patch_0.exp differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.patch_0.pdb b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.patch_0.pdb index b795a7f..819a33a 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.patch_0.pdb and b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.patch_0.pdb differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.pdb b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.pdb index 0ceb729..4ead606 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.pdb and b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgent.pdb differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgentEditor.dll b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgentEditor.dll index f823ce8..211aa16 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgentEditor.dll and b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgentEditor.dll differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgentEditor.exp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgentEditor.exp index 3ff1fde..8b6aa83 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgentEditor.exp and b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgentEditor.exp differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgentEditor.patch_0.pdb b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgentEditor.patch_0.pdb index dbd86b5..d80a317 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgentEditor.patch_0.pdb and b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgentEditor.patch_0.pdb differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgentEditor.pdb b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgentEditor.pdb index 481ca00..ffd5e89 100644 Binary files a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgentEditor.pdb and b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Binaries/Win64/UnrealEditor-PS_AI_ConvAgentEditor.pdb differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_MicrophoneCaptureComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_MicrophoneCaptureComponent.cpp index dff17c1..9de8b5b 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_MicrophoneCaptureComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_MicrophoneCaptureComponent.cpp @@ -198,6 +198,54 @@ void UPS_AI_ConvAgent_MicrophoneCaptureComponent::OnAudioGenerate( PeakLevel.store(Peak, std::memory_order_relaxed); } + // ── Local Voice Activity Detection ────────────────────────────────── + // Runs on the audio thread for minimal latency. Uses hysteresis to + // prevent flickering. Dispatches state change to game thread. + { + const float Rms = CurrentRMS.load(std::memory_order_relaxed); + const float BufferDuration = static_cast(NumFrames) / FMath::Max(1, InSampleRate); + const bool bAboveThreshold = (Rms > VoiceActivityThreshold); + + if (bAboveThreshold) + { + VoiceSilenceAccumulator.store(0.0f, std::memory_order_relaxed); + const float NewOnset = VoiceOnsetAccumulator.load(std::memory_order_relaxed) + BufferDuration; + VoiceOnsetAccumulator.store(NewOnset, std::memory_order_relaxed); + + if (!bIsUserSpeaking && NewOnset >= VoiceOnsetTime) + { + bIsUserSpeaking = true; + TWeakObjectPtr WeakSelf(this); + AsyncTask(ENamedThreads::GameThread, [WeakSelf]() + { + if (auto* Self = WeakSelf.Get()) + { + Self->OnUserVoiceActivityChanged.Broadcast(true); + } + }); + } + } + else + { + VoiceOnsetAccumulator.store(0.0f, std::memory_order_relaxed); + const float NewSilence = VoiceSilenceAccumulator.load(std::memory_order_relaxed) + BufferDuration; + VoiceSilenceAccumulator.store(NewSilence, std::memory_order_relaxed); + + if (bIsUserSpeaking && NewSilence >= VoiceSilenceTime) + { + bIsUserSpeaking = false; + TWeakObjectPtr WeakSelf(this); + AsyncTask(ENamedThreads::GameThread, [WeakSelf]() + { + if (auto* Self = WeakSelf.Get()) + { + Self->OnUserVoiceActivityChanged.Broadcast(false); + } + }); + } + } + } + // Resample + downmix to 16000 Hz mono. TArray Resampled = ResampleTo16000(FloatAudio, NumFrames, InNumChannels, InSampleRate); diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_MicrophoneCaptureComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_MicrophoneCaptureComponent.h index 4d9544b..68a4d02 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_MicrophoneCaptureComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_MicrophoneCaptureComponent.h @@ -11,6 +11,9 @@ // Delivers captured float PCM samples (16000 Hz mono, resampled from device rate). DECLARE_MULTICAST_DELEGATE_OneParam(FOnPS_AI_ConvAgent_AudioCaptured, const TArray& /*FloatPCM*/); +// Fired when local voice activity changes (user starts/stops speaking). +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnPS_AI_ConvAgent_VoiceActivityChanged, bool, bIsSpeaking); + /** * Lightweight microphone capture component. * Captures from the default audio input device, resamples to 16000 Hz mono, @@ -46,6 +49,31 @@ public: * Set by the agent component for echo suppression (skip mic while agent speaks). */ std::atomic* EchoSuppressFlag = nullptr; + // ── Local Voice Activity Detection ────────────────────────────────── + + /** RMS threshold above which audio is considered voice. Adjust based on mic sensitivity and environment noise. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Voice Activity", + meta = (ClampMin = "0.001", ClampMax = "0.2")) + float VoiceActivityThreshold = 0.015f; + + /** Time (seconds) RMS must stay above threshold before declaring speech onset. Prevents false triggers from clicks/bumps. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Voice Activity", + meta = (ClampMin = "0.01", ClampMax = "1.0")) + float VoiceOnsetTime = 0.1f; + + /** Time (seconds) RMS must stay below threshold before declaring speech ended. Prevents cutting off between words. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Voice Activity", + meta = (ClampMin = "0.1", ClampMax = "3.0")) + float VoiceSilenceTime = 0.5f; + + /** Whether the user is currently speaking (local VAD, independent of ElevenLabs server). */ + UPROPERTY(BlueprintReadOnly, Category = "PS AI ConvAgent|Voice Activity") + bool bIsUserSpeaking = false; + + /** Fired when voice activity changes. Use to pause NPC movement, trigger animations, etc. */ + UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|Voice Activity") + FOnPS_AI_ConvAgent_VoiceActivityChanged OnUserVoiceActivityChanged; + // ── Debug ──────────────────────────────────────────────────────────────── /** Enable debug logging for this component. @@ -99,6 +127,10 @@ private: std::atomic CurrentRMS{0.0f}; std::atomic PeakLevel{0.0f}; + // VAD accumulators (written from audio callback thread) + std::atomic VoiceOnsetAccumulator{0.0f}; + std::atomic VoiceSilenceAccumulator{0.0f}; + // Device name cached on StartCapture for HUD display. FString CachedDeviceName; };