From 9321e21a3bc138f2bdc61cc24d518a27d5779624 Mon Sep 17 00:00:00 2001 From: "j.foucher" Date: Thu, 5 Mar 2026 13:33:33 +0100 Subject: [PATCH] Debug HUD: fix flicker, add CVars, mic VU meter, reuse existing mic component - Fix DisplayTime=0.0f causing flicker on all debug HUDs (now 1.0f) - Add per-component CVars (ps.ai.ConvAgent.Debug.*) for console debug toggle - Add MicrophoneCapture debug HUD with VU meter (RMS/peak/dB bar) - InteractionComponent reuses existing MicrophoneCaptureComponent on pawn instead of always creating a new one Co-Authored-By: Claude Opus 4.6 --- ...S_AI_ConvAgent_BodyExpressionComponent.cpp | 17 +++- .../PS_AI_ConvAgent_ElevenLabsComponent.cpp | 91 +++++++++++++++++ ...AI_ConvAgent_FacialExpressionComponent.cpp | 65 ++++++++++++ .../Private/PS_AI_ConvAgent_GazeComponent.cpp | 60 +++++++++++ .../PS_AI_ConvAgent_InteractionComponent.cpp | 96 +++++++++++++++++- .../PS_AI_ConvAgent_LipSyncComponent.cpp | 89 +++++++++++++++++ ...I_ConvAgent_MicrophoneCaptureComponent.cpp | 99 ++++++++++++++++++- .../PS_AI_ConvAgent_ElevenLabsComponent.h | 3 + ...S_AI_ConvAgent_FacialExpressionComponent.h | 3 + .../Public/PS_AI_ConvAgent_GazeComponent.h | 3 + .../PS_AI_ConvAgent_InteractionComponent.h | 3 + .../Public/PS_AI_ConvAgent_LipSyncComponent.h | 3 + ..._AI_ConvAgent_MicrophoneCaptureComponent.h | 10 ++ 13 files changed, 534 insertions(+), 8 deletions(-) diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_BodyExpressionComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_BodyExpressionComponent.cpp index 2a3ed1f..df0b918 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_BodyExpressionComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_BodyExpressionComponent.cpp @@ -8,6 +8,12 @@ DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_BodyExpr, Log, All); +static TAutoConsoleVariable CVarDebugBodyExpr( + TEXT("ps.ai.ConvAgent.Debug.BodyExpr"), + -1, + TEXT("Debug HUD for BodyExpression. -1=use property, 0=off, 1-3=verbosity."), + ECVF_Default); + // ───────────────────────────────────────────────────────────────────────────── // Construction // ───────────────────────────────────────────────────────────────────────────── @@ -485,9 +491,14 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::TickComponent( } // ── On-screen debug HUD ─────────────────────────────────────────────── - if (bDebug && DebugVerbosity >= 1) { - DrawDebugHUD(); + const int32 CVarVal = CVarDebugBodyExpr.GetValueOnGameThread(); + const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug; + const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity; + if (bShowDebug && EffectiveVerbosity >= 1) + { + DrawDebugHUD(); + } } } @@ -523,7 +534,7 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::DrawDebugHUD() const // Use key offset to avoid colliding with other debug messages // Keys 2000-2010 reserved for BodyExpression const int32 BaseKey = 2000; - const float DisplayTime = 0.0f; // Refresh every frame + const float DisplayTime = 1.0f; const FColor MainColor = FColor::Cyan; const FColor WarnColor = FColor::Yellow; diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp index dc4db4e..803cb0c 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp @@ -20,6 +20,12 @@ DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_ElevenLabs, Log, All); +static TAutoConsoleVariable CVarDebugElevenLabs( + TEXT("ps.ai.ConvAgent.Debug.ElevenLabs"), + -1, + TEXT("Debug HUD for ElevenLabs. -1=use property, 0=off, 1-3=verbosity."), + ECVF_Default); + // ───────────────────────────────────────────────────────────────────────────── // Constructor // ───────────────────────────────────────────────────────────────────────────── @@ -258,6 +264,17 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::TickComponent(float DeltaTime, ELevel MulticastAgentStoppedSpeaking(); } } + + // On-screen debug display. + { + const int32 CVarVal = CVarDebugElevenLabs.GetValueOnGameThread(); + const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug; + const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity; + if (bShowDebug && EffectiveVerbosity >= 1) + { + DrawDebugHUD(); + } + } } // ───────────────────────────────────────────────────────────────────────────── @@ -2028,3 +2045,77 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ApplyConversationGaze() } } } + +// ───────────────────────────────────────────────────────────────────────────── +// On-screen debug display +// ───────────────────────────────────────────────────────────────────────────── + +void UPS_AI_ConvAgent_ElevenLabsComponent::DrawDebugHUD() const +{ + if (!GEngine) return; + + const int32 BaseKey = 2040; + const float DisplayTime = 1.0f; + const FColor MainColor = FColor::Cyan; + const FColor WarnColor = FColor::Yellow; + const FColor GoodColor = FColor::Green; + + const bool bConnected = IsConnected(); + + GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, + bConnected ? GoodColor : FColor::Red, + FString::Printf(TEXT("=== ELEVENLABS: %s ==="), + bConnected ? TEXT("CONNECTED") : TEXT("DISCONNECTED"))); + + // Session info + FString ModeStr = (TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server) + ? TEXT("ServerVAD") : TEXT("ClientPTT"); + GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor, + FString::Printf(TEXT(" Session: %s Turn: %d Mode: %s"), + bPersistentSession ? TEXT("persistent") : TEXT("ephemeral"), + TurnIndex, *ModeStr)); + + // State flags + const bool bListening = bIsListening.load(); + const bool bSpeaking = bAgentSpeaking.load(); + const bool bGenerating = bAgentGenerating.load(); + GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, MainColor, + FString::Printf(TEXT(" Listening: %s Speaking: %s Generating: %s"), + bListening ? TEXT("YES") : TEXT("NO"), + bSpeaking ? TEXT("YES") : TEXT("NO"), + bGenerating ? TEXT("YES") : TEXT("NO"))); + + const bool bWaiting = bWaitingForAgentResponse.load(); + GEngine->AddOnScreenDebugMessage(BaseKey + 3, DisplayTime, + (bPreBuffering || bWaiting) ? WarnColor : MainColor, + FString::Printf(TEXT(" PreBuffer: %s WaitResponse: %s"), + bPreBuffering ? TEXT("YES") : TEXT("NO"), + bWaiting ? TEXT("YES") : TEXT("NO"))); + + // Emotion + GEngine->AddOnScreenDebugMessage(BaseKey + 4, DisplayTime, MainColor, + FString::Printf(TEXT(" Emotion: %s (%s)"), + *UEnum::GetDisplayValueAsText(CurrentEmotion).ToString(), + *UEnum::GetDisplayValueAsText(CurrentEmotionIntensity).ToString())); + + // Audio queue (read without lock for debug display — minor race is acceptable) + const int32 QueueBytes = FMath::Max(0, AudioQueue.Num() - AudioQueueReadOffset); + GEngine->AddOnScreenDebugMessage(BaseKey + 5, DisplayTime, MainColor, + FString::Printf(TEXT(" AudioQueue: %d bytes SilentTicks: %d"), + QueueBytes, SilentTickCount)); + + // Timing + const double Now = FPlatformTime::Seconds(); + const float SessionSec = (SessionStartTime > 0.0) ? static_cast(Now - SessionStartTime) : 0.0f; + const float TurnSec = (TurnStartTime > 0.0) ? static_cast(Now - TurnStartTime) : 0.0f; + GEngine->AddOnScreenDebugMessage(BaseKey + 6, DisplayTime, MainColor, + FString::Printf(TEXT(" Timing: session=%.1fs turn=%.1fs"), + SessionSec, TurnSec)); + + // Reconnection + GEngine->AddOnScreenDebugMessage(BaseKey + 7, DisplayTime, + bWantsReconnect ? FColor::Red : MainColor, + FString::Printf(TEXT(" Reconnect: %d/%d attempts%s"), + ReconnectAttemptCount, MaxReconnectAttempts, + bWantsReconnect ? TEXT(" (ACTIVE)") : TEXT(""))); +} diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_FacialExpressionComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_FacialExpressionComponent.cpp index 2b36d8c..b30abf6 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_FacialExpressionComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_FacialExpressionComponent.cpp @@ -5,9 +5,16 @@ #include "PS_AI_ConvAgent_EmotionPoseMap.h" #include "Animation/AnimSequence.h" #include "Animation/AnimCurveTypes.h" +#include "Engine/Engine.h" DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_FacialExpr, Log, All); +static TAutoConsoleVariable CVarDebugFacialExpr( + TEXT("ps.ai.ConvAgent.Debug.FacialExpr"), + -1, + TEXT("Debug HUD for FacialExpression. -1=use property, 0=off, 1-3=verbosity."), + ECVF_Default); + // ───────────────────────────────────────────────────────────────────────────── // Construction // ───────────────────────────────────────────────────────────────────────────── @@ -398,6 +405,64 @@ void UPS_AI_ConvAgent_FacialExpressionComponent::TickComponent( FScopeLock Lock(&EmotionCurveLock); CurrentEmotionCurves = MoveTemp(NewCurves); } + + // ── On-screen debug HUD ─────────────────────────────────────────────── + { + const int32 CVarVal = CVarDebugFacialExpr.GetValueOnGameThread(); + const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug; + const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity; + if (bShowDebug && EffectiveVerbosity >= 1) + { + DrawDebugHUD(); + } + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// On-screen debug display +// ───────────────────────────────────────────────────────────────────────────── + +void UPS_AI_ConvAgent_FacialExpressionComponent::DrawDebugHUD() const +{ + if (!GEngine) return; + + const int32 BaseKey = 2010; + const float DisplayTime = 1.0f; + const FColor MainColor = FColor::Cyan; + const FColor WarnColor = FColor::Yellow; + + // State label + FString StateStr = bActive ? TEXT("ACTIVE") : TEXT("INACTIVE"); + + GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, MainColor, + FString::Printf(TEXT("=== FACIAL EXPR: %s ==="), *StateStr)); + + GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor, + FString::Printf(TEXT(" ActivationAlpha: %.3f (target: %s)"), + CurrentActiveAlpha, bActive ? TEXT("1") : TEXT("0"))); + + FString ActiveName = ActiveAnim ? ActiveAnim->GetName() : TEXT("(none)"); + GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, MainColor, + FString::Printf(TEXT(" Active: %s t=%.2f"), *ActiveName, ActivePlaybackTime)); + + FString PrevName = PrevAnim ? PrevAnim->GetName() : TEXT("---"); + GEngine->AddOnScreenDebugMessage(BaseKey + 3, DisplayTime, + CrossfadeAlpha < 1.0f ? WarnColor : MainColor, + FString::Printf(TEXT(" Crossfade: %.3f Prev: %s"), + CrossfadeAlpha, *PrevName)); + + GEngine->AddOnScreenDebugMessage(BaseKey + 4, DisplayTime, MainColor, + FString::Printf(TEXT(" Emotion: %s (%s)"), + *UEnum::GetDisplayValueAsText(ActiveEmotion).ToString(), + *UEnum::GetDisplayValueAsText(ActiveEmotionIntensity).ToString())); + + int32 CurveCount = 0; + { + FScopeLock Lock(&EmotionCurveLock); + CurveCount = CurrentEmotionCurves.Num(); + } + GEngine->AddOnScreenDebugMessage(BaseKey + 5, DisplayTime, MainColor, + FString::Printf(TEXT(" Curves: %d active"), CurveCount)); } // ───────────────────────────────────────────────────────────────────────────── diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_GazeComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_GazeComponent.cpp index ded02e0..7993aea 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_GazeComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_GazeComponent.cpp @@ -8,9 +8,16 @@ #include "GameFramework/Pawn.h" #include "Math/UnrealMathUtility.h" #include "DrawDebugHelpers.h" +#include "Engine/Engine.h" DEFINE_LOG_CATEGORY(LogPS_AI_ConvAgent_Gaze); +static TAutoConsoleVariable CVarDebugGaze( + TEXT("ps.ai.ConvAgent.Debug.Gaze"), + -1, + TEXT("Debug HUD for Gaze. -1=use property, 0=off, 1-3=verbosity."), + ECVF_Default); + // ── ARKit eye curve names ──────────────────────────────────────────────────── static const FName EyeLookUpLeft(TEXT("eyeLookUpLeft")); static const FName EyeLookDownLeft(TEXT("eyeLookDownLeft")); @@ -646,4 +653,57 @@ void UPS_AI_ConvAgent_GazeComponent::TickComponent( Owner->GetActorRotation().Yaw); } } + + // ── On-screen debug HUD ─────────────────────────────────────────────── + { + const int32 CVarVal = CVarDebugGaze.GetValueOnGameThread(); + const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug; + const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity; + if (bShowDebug && EffectiveVerbosity >= 1) + { + DrawDebugHUD(); + } + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// On-screen debug display +// ───────────────────────────────────────────────────────────────────────────── + +void UPS_AI_ConvAgent_GazeComponent::DrawDebugHUD() const +{ + if (!GEngine) return; + + const int32 BaseKey = 2020; + const float DisplayTime = 1.0f; + const FColor MainColor = FColor::Cyan; + + FString StateStr = bActive ? TEXT("ACTIVE") : TEXT("INACTIVE"); + + GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, MainColor, + FString::Printf(TEXT("=== GAZE: %s ==="), *StateStr)); + + FString TargetName = TargetActor ? TargetActor->GetName() : TEXT("(none)"); + GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor, + FString::Printf(TEXT(" Target: %s BodyTrack: %s"), + *TargetName, bEnableBodyTracking ? TEXT("ON") : TEXT("OFF"))); + + GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, MainColor, + FString::Printf(TEXT(" ActivationAlpha: %.3f"), CurrentActiveAlpha)); + + GEngine->AddOnScreenDebugMessage(BaseKey + 3, DisplayTime, MainColor, + FString::Printf(TEXT(" Head: Yaw=%.1f Pitch=%.1f (target: %.1f / %.1f)"), + CurrentHeadYaw, CurrentHeadPitch, TargetHeadYaw, TargetHeadPitch)); + + GEngine->AddOnScreenDebugMessage(BaseKey + 4, DisplayTime, MainColor, + FString::Printf(TEXT(" Eyes: Yaw=%.1f Pitch=%.1f"), + CurrentEyeYaw, CurrentEyePitch)); + + GEngine->AddOnScreenDebugMessage(BaseKey + 5, DisplayTime, MainColor, + FString::Printf(TEXT(" Body: SmoothedYaw=%.1f TargetYaw=%.1f"), + SmoothedBodyYaw, TargetBodyWorldYaw)); + + GEngine->AddOnScreenDebugMessage(BaseKey + 6, DisplayTime, MainColor, + FString::Printf(TEXT(" Compensation: Head=%.2f Eye=%.2f Body=%.2f"), + HeadAnimationCompensation, EyeAnimationCompensation, BodyDriftCompensation)); } diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp index f56adbf..2f121ec 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp @@ -14,6 +14,12 @@ DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_Select, Log, All); +static TAutoConsoleVariable CVarDebugInteraction( + TEXT("ps.ai.ConvAgent.Debug.Interaction"), + -1, + TEXT("Debug HUD for Interaction. -1=use property, 0=off, 1-3=verbosity."), + ECVF_Default); + // ───────────────────────────────────────────────────────────────────────────── // Constructor // ───────────────────────────────────────────────────────────────────────────── @@ -110,10 +116,16 @@ void UPS_AI_ConvAgent_InteractionComponent::TickComponent(float DeltaTime, ELeve return; } - // ── This is the locally controlled pawn — create the mic component ── - MicComponent = NewObject( - GetOwner(), TEXT("PS_AI_ConvAgent_Mic_Interaction")); - MicComponent->RegisterComponent(); + // ── This is the locally controlled pawn — find or create mic component ── + MicComponent = GetOwner()->FindComponentByClass(); + if (!MicComponent) + { + MicComponent = NewObject( + GetOwner(), TEXT("PS_AI_ConvAgent_Mic_Interaction")); + MicComponent->bDebug = bDebug; + MicComponent->DebugVerbosity = DebugVerbosity; + MicComponent->RegisterComponent(); + } MicComponent->OnAudioCaptured.AddUObject(this, &UPS_AI_ConvAgent_InteractionComponent::OnMicAudioCaptured); @@ -136,6 +148,17 @@ void UPS_AI_ConvAgent_InteractionComponent::TickComponent(float DeltaTime, ELeve { SetSelectedAgent(BestAgent); } + + // ── On-screen debug HUD ─────────────────────────────────────────────── + { + const int32 CVarVal = CVarDebugInteraction.GetValueOnGameThread(); + const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug; + const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity; + if (bShowDebug && EffectiveVerbosity >= 1) + { + DrawDebugHUD(); + } + } } // ───────────────────────────────────────────────────────────────────────────── @@ -609,6 +632,71 @@ void UPS_AI_ConvAgent_InteractionComponent::OnMicAudioCaptured(const TArrayFeedExternalAudio(FloatPCM); } +// ───────────────────────────────────────────────────────────────────────────── +// On-screen debug display +// ───────────────────────────────────────────────────────────────────────────── + +void UPS_AI_ConvAgent_InteractionComponent::DrawDebugHUD() const +{ + if (!GEngine) return; + + const int32 BaseKey = 2060; + const float DisplayTime = 1.0f; + const FColor MainColor = FColor::Cyan; + + GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, MainColor, + TEXT("=== INTERACTION ===")); + + UPS_AI_ConvAgent_ElevenLabsComponent* Agent = SelectedAgent.Get(); + if (Agent) + { + FString AgentName = Agent->GetOwner() ? Agent->GetOwner()->GetName() : TEXT("(?)"); + FString ConvState = Agent->bNetIsConversing ? TEXT("conversing") : TEXT("selected"); + + // Compute distance and angle to selected agent + FVector ViewLoc, ViewDir; + GetPawnViewPoint(ViewLoc, ViewDir); + + float Dist = 0.0f; + float Angle = 0.0f; + if (Agent->GetOwner()) + { + FVector AgentLoc = Agent->GetOwner()->GetActorLocation() + + FVector(0.0f, 0.0f, AgentEyeLevelOffset); + FVector ToAgent = AgentLoc - ViewLoc; + Dist = ToAgent.Size(); + FVector DirToAgent = ToAgent.GetSafeNormal(); + Angle = FMath::RadiansToDegrees( + FMath::Acos(FMath::Clamp(FVector::DotProduct(ViewDir, DirToAgent), -1.0f, 1.0f))); + } + + GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor, + FString::Printf(TEXT(" Selected: %s (%s)"), *AgentName, *ConvState)); + + GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, MainColor, + FString::Printf(TEXT(" Distance: %.0fcm Angle: %.1f deg"), + Dist, Angle)); + } + else + { + GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor, + TEXT(" Selected: (none)")); + + GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, MainColor, + TEXT(" Distance: --- Angle: ---")); + } + + GEngine->AddOnScreenDebugMessage(BaseKey + 3, DisplayTime, MainColor, + FString::Printf(TEXT(" AutoStart: %s AutoGaze: %s AutoListen: %s"), + bAutoStartConversation ? TEXT("ON") : TEXT("OFF"), + bAutoManageGaze ? TEXT("ON") : TEXT("OFF"), + bAutoManageListening ? TEXT("ON") : TEXT("OFF"))); + + GEngine->AddOnScreenDebugMessage(BaseKey + 4, DisplayTime, MainColor, + FString::Printf(TEXT(" Mic: %s"), + (MicComponent != nullptr) ? TEXT("initialized") : TEXT("not initialized"))); +} + // ───────────────────────────────────────────────────────────────────────────── // Replication // ───────────────────────────────────────────────────────────────────────────── diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_LipSyncComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_LipSyncComponent.cpp index 0d8c489..4a17a8a 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_LipSyncComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_LipSyncComponent.cpp @@ -13,6 +13,12 @@ DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_LipSync, Log, All); +static TAutoConsoleVariable CVarDebugLipSync( + TEXT("ps.ai.ConvAgent.Debug.LipSync"), + -1, + TEXT("Debug HUD for LipSync. -1=use property, 0=off, 1-3=verbosity."), + ECVF_Default); + // ───────────────────────────────────────────────────────────────────────────── // Static data // ───────────────────────────────────────────────────────────────────────────── @@ -1114,6 +1120,17 @@ void UPS_AI_ConvAgent_LipSyncComponent::TickComponent(float DeltaTime, ELevelTic { OnVisemesReady.Broadcast(); } + + // ── On-screen debug HUD ─────────────────────────────────────────────── + { + const int32 CVarVal = CVarDebugLipSync.GetValueOnGameThread(); + const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug; + const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity; + if (bShowDebug && EffectiveVerbosity >= 1) + { + DrawDebugHUD(); + } + } } // ───────────────────────────────────────────────────────────────────────────── @@ -2575,6 +2592,78 @@ void UPS_AI_ConvAgent_LipSyncComponent::ApplyMorphTargets() } } +// ───────────────────────────────────────────────────────────────────────────── +// On-screen debug display +// ───────────────────────────────────────────────────────────────────────────── + +void UPS_AI_ConvAgent_LipSyncComponent::DrawDebugHUD() const +{ + if (!GEngine) return; + + const int32 BaseKey = 2030; + const float DisplayTime = 1.0f; + const FColor MainColor = FColor::Cyan; + const FColor WarnColor = FColor::Yellow; + + FString StateStr = bActive ? TEXT("ACTIVE") : TEXT("INACTIVE"); + + GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, MainColor, + FString::Printf(TEXT("=== LIP SYNC: %s ==="), *StateStr)); + + GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, + bIsSpeaking ? FColor::Green : MainColor, + FString::Printf(TEXT(" Speaking: %s SpeechBlend: %.3f"), + bIsSpeaking ? TEXT("YES") : TEXT("NO"), SpeechBlendAlpha)); + + GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, MainColor, + FString::Printf(TEXT(" ActivationAlpha: %.3f"), CurrentActiveAlpha)); + + GEngine->AddOnScreenDebugMessage(BaseKey + 3, DisplayTime, MainColor, + FString::Printf(TEXT(" Amplitude: %.3f Envelope: %.3f"), + CurrentAmplitude, AudioEnvelopeValue)); + + GEngine->AddOnScreenDebugMessage(BaseKey + 4, DisplayTime, MainColor, + FString::Printf(TEXT(" Queue: %d frames Playback: %.3fs"), + VisemeQueue.Num(), PlaybackTimer)); + + GEngine->AddOnScreenDebugMessage(BaseKey + 5, DisplayTime, + bVisemeTimelineActive ? WarnColor : MainColor, + FString::Printf(TEXT(" Timeline: %s cursor=%.2fs"), + bVisemeTimelineActive ? TEXT("ACTIVE") : TEXT("OFF"), + VisemeTimelineCursor)); + + // Top 3 visemes by weight + FString TopVisemes; + { + TArray> Sorted; + for (const auto& Pair : SmoothedVisemes) + { + if (Pair.Value > 0.01f) + { + Sorted.Add(TPair(Pair.Key, Pair.Value)); + } + } + Sorted.Sort([](const TPair& A, const TPair& B) + { + return A.Value > B.Value; + }); + for (int32 i = 0; i < FMath::Min(3, Sorted.Num()); ++i) + { + if (i > 0) TopVisemes += TEXT(", "); + TopVisemes += FString::Printf(TEXT("%s=%.2f"), + *Sorted[i].Key.ToString(), Sorted[i].Value); + } + if (TopVisemes.IsEmpty()) TopVisemes = TEXT("---"); + } + GEngine->AddOnScreenDebugMessage(BaseKey + 6, DisplayTime, MainColor, + FString::Printf(TEXT(" Top visemes: %s"), *TopVisemes)); + + GEngine->AddOnScreenDebugMessage(BaseKey + 7, DisplayTime, MainColor, + FString::Printf(TEXT(" Mode: %s PoseMap: %s"), + bUseCurveMode ? TEXT("Curves") : TEXT("MorphTargets"), + PoseMap ? TEXT("YES") : TEXT("NO"))); +} + // ───────────────────────────────────────────────────────────────────────────── // ARKit → MetaHuman curve name conversion // ───────────────────────────────────────────────────────────────────────────── diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_MicrophoneCaptureComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_MicrophoneCaptureComponent.cpp index 399e2d9..cc4af06 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_MicrophoneCaptureComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_MicrophoneCaptureComponent.cpp @@ -5,15 +5,23 @@ #include "AudioCaptureCore.h" #include "Async/Async.h" +#include "Engine/Engine.h" DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_Mic, Log, All); +static TAutoConsoleVariable CVarDebugMic( + TEXT("ps.ai.ConvAgent.Debug.Mic"), + -1, + TEXT("Debug HUD for Microphone. -1=use property, 0=off, 1-3=verbosity."), + ECVF_Default); + // ───────────────────────────────────────────────────────────────────────────── // Constructor // ───────────────────────────────────────────────────────────────────────────── UPS_AI_ConvAgent_MicrophoneCaptureComponent::UPS_AI_ConvAgent_MicrophoneCaptureComponent() { - PrimaryComponentTick.bCanEverTick = false; + PrimaryComponentTick.bCanEverTick = true; + PrimaryComponentTick.TickInterval = 1.0f / 15.0f; // 15 Hz — enough for debug HUD. } // ───────────────────────────────────────────────────────────────────────────── @@ -25,6 +33,78 @@ void UPS_AI_ConvAgent_MicrophoneCaptureComponent::EndPlay(const EEndPlayReason:: Super::EndPlay(EndPlayReason); } +// ───────────────────────────────────────────────────────────────────────────── +// Tick +// ───────────────────────────────────────────────────────────────────────────── +void UPS_AI_ConvAgent_MicrophoneCaptureComponent::TickComponent( + float DeltaTime, ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction) +{ + Super::TickComponent(DeltaTime, TickType, ThisTickFunction); + + // On-screen debug HUD + const int32 CVarVal = CVarDebugMic.GetValueOnGameThread(); + const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug; + const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity; + if (bShowDebug && EffectiveVerbosity >= 1) + { + DrawDebugHUD(); + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// On-screen debug display +// ───────────────────────────────────────────────────────────────────────────── +void UPS_AI_ConvAgent_MicrophoneCaptureComponent::DrawDebugHUD() const +{ + if (!GEngine) return; + + const int32 BaseKey = 2050; + const float DisplayTime = 1.0f; + const FColor MainColor = FColor::Cyan; + + const bool bCapt = bCapturing.load(); + const bool bEchoSuppressed = EchoSuppressFlag && EchoSuppressFlag->load(std::memory_order_relaxed); + + GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, + bCapt ? FColor::Green : FColor::Red, + FString::Printf(TEXT("=== MIC: %s ==="), + bCapt ? TEXT("CAPTURING") : TEXT("STOPPED"))); + + GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor, + FString::Printf(TEXT(" Device: %s Rate: %d Ch: %d"), + CachedDeviceName.IsEmpty() ? TEXT("(none)") : *CachedDeviceName, + DeviceSampleRate, DeviceChannels)); + + GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, + bEchoSuppressed ? FColor::Yellow : MainColor, + FString::Printf(TEXT(" EchoSuppress: %s VolMul: %.2f"), + bEchoSuppressed ? TEXT("YES") : TEXT("NO"), VolumeMultiplier)); + + // VU meter + const float RMS = CurrentRMS.load(std::memory_order_relaxed); + const float Peak = PeakLevel.load(std::memory_order_relaxed); + const float dB = (RMS > 1e-6f) ? 20.0f * FMath::LogX(10.0f, RMS) : -60.0f; + + // Build text bar: 30 chars wide, mapped from -60dB to 0dB. + const int32 BarWidth = 30; + const float NormLevel = FMath::Clamp((dB + 60.0f) / 60.0f, 0.0f, 1.0f); + const int32 FilledChars = FMath::RoundToInt(NormLevel * BarWidth); + + FString Bar; + for (int32 i = 0; i < BarWidth; i++) + { + Bar += (i < FilledChars) ? TEXT("|") : TEXT(" "); + } + + FColor VUColor = MainColor; + if (dB > -6.0f) VUColor = FColor::Red; + else if (dB > -20.0f) VUColor = FColor::Green; + + GEngine->AddOnScreenDebugMessage(BaseKey + 3, DisplayTime, VUColor, + FString::Printf(TEXT(" VU [%s] %.1fdB peak=%.3f"), + *Bar, dB, Peak)); +} + // ───────────────────────────────────────────────────────────────────────────── // Capture control // ───────────────────────────────────────────────────────────────────────────── @@ -57,6 +137,7 @@ void UPS_AI_ConvAgent_MicrophoneCaptureComponent::StartCapture() { DeviceSampleRate = DeviceInfo.PreferredSampleRate; DeviceChannels = DeviceInfo.InputChannels; + CachedDeviceName = DeviceInfo.DeviceName; if (bDebug) { UE_LOG(LogPS_AI_ConvAgent_Mic, Log, TEXT("Capture device: %s | Rate=%d | Channels=%d"), @@ -99,6 +180,22 @@ void UPS_AI_ConvAgent_MicrophoneCaptureComponent::OnAudioGenerate( // Device sends float32 interleaved samples; cast from the void* API. const float* FloatAudio = static_cast(InAudio); + // Compute RMS from raw input for VU meter (before resample, cheap). + { + float SumSq = 0.0f; + float Peak = 0.0f; + const int32 TotalSamples = NumFrames * InNumChannels; + for (int32 i = 0; i < TotalSamples; i++) + { + const float S = FloatAudio[i]; + SumSq += S * S; + const float AbsS = FMath::Abs(S); + if (AbsS > Peak) Peak = AbsS; + } + CurrentRMS.store(FMath::Sqrt(SumSq / FMath::Max(1, TotalSamples)), std::memory_order_relaxed); + PeakLevel.store(Peak, std::memory_order_relaxed); + } + // Resample + downmix to 16000 Hz mono. TArray Resampled = ResampleTo16000(FloatAudio, NumFrames, InNumChannels, InSampleRate); diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h index 1ab4887..12d66f3 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h @@ -675,4 +675,7 @@ private: * Called on the server when bNetIsConversing / NetConversatingPawn change, * because OnRep_ConversationState never fires on the Authority. */ void ApplyConversationGaze(); + + /** Draw on-screen debug info (called from TickComponent when bDebug). */ + void DrawDebugHUD() const; }; diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_FacialExpressionComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_FacialExpressionComponent.h index ccf5c67..5b6d63f 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_FacialExpressionComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_FacialExpressionComponent.h @@ -146,6 +146,9 @@ private: /** Evaluate all FloatCurves from an AnimSequence at a given time. */ TMap EvaluateAnimCurves(UAnimSequence* AnimSeq, float Time) const; + /** Draw on-screen debug info (called from TickComponent when bDebug). */ + void DrawDebugHUD() const; + // ── Animation playback state ───────────────────────────────────────────── /** Currently playing emotion AnimSequence (looping). */ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_GazeComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_GazeComponent.h index ee0de0e..fc89b04 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_GazeComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_GazeComponent.h @@ -337,6 +337,9 @@ private: /** Map eye yaw/pitch angles to 8 ARKit eye curves. */ void UpdateEyeCurves(float EyeYaw, float EyePitch); + /** Draw on-screen debug info (called from TickComponent when bDebug). */ + void DrawDebugHUD() const; + // ── Smoothed current values (head + eyes, body is actor yaw) ──────────── /** Current blend alpha (0 = fully inactive/passthrough, 1 = fully active). */ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionComponent.h index 1e63c64..dc41522 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionComponent.h @@ -260,6 +260,9 @@ private: /** Clear the agent's GazeComponent target (detach). */ void DetachGazeTarget(TWeakObjectPtr Agent); + /** Draw on-screen debug info (called from TickComponent when bDebug). */ + void DrawDebugHUD() const; + // ── Mic routing ────────────────────────────────────────────────────────── /** Forward captured mic audio to the currently selected agent. */ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_LipSyncComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_LipSyncComponent.h index 85997b1..6fe3807 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_LipSyncComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_LipSyncComponent.h @@ -252,6 +252,9 @@ private: /** Sample the spectrum magnitude across a frequency range. */ float GetBandEnergy(float LowFreq, float HighFreq, int32 NumSamples = 8) const; + /** Draw on-screen debug info (called from TickComponent when bDebug). */ + void DrawDebugHUD() const; + // ── State ───────────────────────────────────────────────────────────────── TUniquePtr SpectrumAnalyzer; diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_MicrophoneCaptureComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_MicrophoneCaptureComponent.h index a3f2bdb..4d9544b 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_MicrophoneCaptureComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_MicrophoneCaptureComponent.h @@ -73,9 +73,12 @@ public: // ───────────────────────────────────────────────────────────────────────── // UActorComponent overrides // ───────────────────────────────────────────────────────────────────────── + virtual void TickComponent(float DeltaTime, ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction) override; virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override; private: + /** Draw on-screen debug info (called from TickComponent when debug is active). */ + void DrawDebugHUD() const; /** Called by the audio capture callback on a background thread. Raw void* per UE 5.3+ API. */ void OnAudioGenerate(const void* InAudio, int32 NumFrames, int32 InNumChannels, int32 InSampleRate, double StreamTime, bool bOverflow); @@ -91,4 +94,11 @@ private: // Device sample rate discovered on StartCapture int32 DeviceSampleRate = 44100; int32 DeviceChannels = 1; + + // RMS level for VU meter (written from audio callback, read on game thread). + std::atomic CurrentRMS{0.0f}; + std::atomic PeakLevel{0.0f}; + + // Device name cached on StartCapture for HUD display. + FString CachedDeviceName; };