Compare commits

..

No commits in common. "e5f40c65ec189843a2b785598c59b1639fcf6ba8" and "8bb4371a741628f68f5d1a887dc9890c1b7c863b" have entirely different histories.

19 changed files with 256 additions and 2241 deletions

View File

@ -90,7 +90,7 @@ FontDPI=72
; ── Generic classes: ElevenLabs → PS_AI_ConvAgent ── ; ── Generic classes: ElevenLabs → PS_AI_ConvAgent ──
+ClassRedirects=(OldName="ElevenLabsLipSyncComponent", NewName="PS_AI_ConvAgent_LipSyncComponent") +ClassRedirects=(OldName="ElevenLabsLipSyncComponent", NewName="PS_AI_ConvAgent_LipSyncComponent")
+ClassRedirects=(OldName="ElevenLabsFacialExpressionComponent", NewName="PS_AI_ConvAgent_FacialExpressionComponent") +ClassRedirects=(OldName="ElevenLabsFacialExpressionComponent", NewName="PS_AI_ConvAgent_FacialExpressionComponent")
+ClassRedirects=(OldName="ElevenLabsPostureComponent", NewName="PS_AI_ConvAgent_GazeComponent") +ClassRedirects=(OldName="ElevenLabsPostureComponent", NewName="PS_AI_ConvAgent_PostureComponent")
+ClassRedirects=(OldName="ElevenLabsMicrophoneCaptureComponent", NewName="PS_AI_ConvAgent_MicrophoneCaptureComponent") +ClassRedirects=(OldName="ElevenLabsMicrophoneCaptureComponent", NewName="PS_AI_ConvAgent_MicrophoneCaptureComponent")
+ClassRedirects=(OldName="ElevenLabsLipSyncPoseMap", NewName="PS_AI_ConvAgent_LipSyncPoseMap") +ClassRedirects=(OldName="ElevenLabsLipSyncPoseMap", NewName="PS_AI_ConvAgent_LipSyncPoseMap")
+ClassRedirects=(OldName="ElevenLabsEmotionPoseMap", NewName="PS_AI_ConvAgent_EmotionPoseMap") +ClassRedirects=(OldName="ElevenLabsEmotionPoseMap", NewName="PS_AI_ConvAgent_EmotionPoseMap")
@ -98,7 +98,7 @@ FontDPI=72
; ── Generic classes: PS_AI_Agent → PS_AI_ConvAgent (intermediate rename) ── ; ── Generic classes: PS_AI_Agent → PS_AI_ConvAgent (intermediate rename) ──
+ClassRedirects=(OldName="PS_AI_Agent_LipSyncComponent", NewName="PS_AI_ConvAgent_LipSyncComponent") +ClassRedirects=(OldName="PS_AI_Agent_LipSyncComponent", NewName="PS_AI_ConvAgent_LipSyncComponent")
+ClassRedirects=(OldName="PS_AI_Agent_FacialExpressionComponent", NewName="PS_AI_ConvAgent_FacialExpressionComponent") +ClassRedirects=(OldName="PS_AI_Agent_FacialExpressionComponent", NewName="PS_AI_ConvAgent_FacialExpressionComponent")
+ClassRedirects=(OldName="PS_AI_Agent_PostureComponent", NewName="PS_AI_ConvAgent_GazeComponent") +ClassRedirects=(OldName="PS_AI_Agent_PostureComponent", NewName="PS_AI_ConvAgent_PostureComponent")
+ClassRedirects=(OldName="PS_AI_Agent_MicrophoneCaptureComponent", NewName="PS_AI_ConvAgent_MicrophoneCaptureComponent") +ClassRedirects=(OldName="PS_AI_Agent_MicrophoneCaptureComponent", NewName="PS_AI_ConvAgent_MicrophoneCaptureComponent")
+ClassRedirects=(OldName="PS_AI_Agent_LipSyncPoseMap", NewName="PS_AI_ConvAgent_LipSyncPoseMap") +ClassRedirects=(OldName="PS_AI_Agent_LipSyncPoseMap", NewName="PS_AI_ConvAgent_LipSyncPoseMap")
+ClassRedirects=(OldName="PS_AI_Agent_EmotionPoseMap", NewName="PS_AI_ConvAgent_EmotionPoseMap") +ClassRedirects=(OldName="PS_AI_Agent_EmotionPoseMap", NewName="PS_AI_ConvAgent_EmotionPoseMap")
@ -114,23 +114,18 @@ FontDPI=72
; ── AnimNode structs ── ; ── AnimNode structs ──
+StructRedirects=(OldName="AnimNode_ElevenLabsLipSync", NewName="AnimNode_PS_AI_ConvAgent_LipSync") +StructRedirects=(OldName="AnimNode_ElevenLabsLipSync", NewName="AnimNode_PS_AI_ConvAgent_LipSync")
+StructRedirects=(OldName="AnimNode_ElevenLabsFacialExpression", NewName="AnimNode_PS_AI_ConvAgent_FacialExpression") +StructRedirects=(OldName="AnimNode_ElevenLabsFacialExpression", NewName="AnimNode_PS_AI_ConvAgent_FacialExpression")
+StructRedirects=(OldName="AnimNode_ElevenLabsPosture", NewName="AnimNode_PS_AI_ConvAgent_Gaze") +StructRedirects=(OldName="AnimNode_ElevenLabsPosture", NewName="AnimNode_PS_AI_ConvAgent_Posture")
+StructRedirects=(OldName="AnimNode_PS_AI_Agent_LipSync", NewName="AnimNode_PS_AI_ConvAgent_LipSync") +StructRedirects=(OldName="AnimNode_PS_AI_Agent_LipSync", NewName="AnimNode_PS_AI_ConvAgent_LipSync")
+StructRedirects=(OldName="AnimNode_PS_AI_Agent_FacialExpression", NewName="AnimNode_PS_AI_ConvAgent_FacialExpression") +StructRedirects=(OldName="AnimNode_PS_AI_Agent_FacialExpression", NewName="AnimNode_PS_AI_ConvAgent_FacialExpression")
+StructRedirects=(OldName="AnimNode_PS_AI_Agent_Posture", NewName="AnimNode_PS_AI_ConvAgent_Gaze") +StructRedirects=(OldName="AnimNode_PS_AI_Agent_Posture", NewName="AnimNode_PS_AI_ConvAgent_Posture")
; ── AnimGraphNode classes ── ; ── AnimGraphNode classes ──
+ClassRedirects=(OldName="AnimGraphNode_ElevenLabsLipSync", NewName="AnimGraphNode_PS_AI_ConvAgent_LipSync") +ClassRedirects=(OldName="AnimGraphNode_ElevenLabsLipSync", NewName="AnimGraphNode_PS_AI_ConvAgent_LipSync")
+ClassRedirects=(OldName="AnimGraphNode_ElevenLabsFacialExpression", NewName="AnimGraphNode_PS_AI_ConvAgent_FacialExpression") +ClassRedirects=(OldName="AnimGraphNode_ElevenLabsFacialExpression", NewName="AnimGraphNode_PS_AI_ConvAgent_FacialExpression")
+ClassRedirects=(OldName="AnimGraphNode_ElevenLabsPosture", NewName="AnimGraphNode_PS_AI_ConvAgent_Gaze") +ClassRedirects=(OldName="AnimGraphNode_ElevenLabsPosture", NewName="AnimGraphNode_PS_AI_ConvAgent_Posture")
+ClassRedirects=(OldName="AnimGraphNode_PS_AI_Agent_LipSync", NewName="AnimGraphNode_PS_AI_ConvAgent_LipSync") +ClassRedirects=(OldName="AnimGraphNode_PS_AI_Agent_LipSync", NewName="AnimGraphNode_PS_AI_ConvAgent_LipSync")
+ClassRedirects=(OldName="AnimGraphNode_PS_AI_Agent_FacialExpression", NewName="AnimGraphNode_PS_AI_ConvAgent_FacialExpression") +ClassRedirects=(OldName="AnimGraphNode_PS_AI_Agent_FacialExpression", NewName="AnimGraphNode_PS_AI_ConvAgent_FacialExpression")
+ClassRedirects=(OldName="AnimGraphNode_PS_AI_Agent_Posture", NewName="AnimGraphNode_PS_AI_ConvAgent_Gaze") +ClassRedirects=(OldName="AnimGraphNode_PS_AI_Agent_Posture", NewName="AnimGraphNode_PS_AI_ConvAgent_Posture")
; ── Posture → Gaze rename ──
+ClassRedirects=(OldName="PS_AI_ConvAgent_PostureComponent", NewName="PS_AI_ConvAgent_GazeComponent")
+StructRedirects=(OldName="AnimNode_PS_AI_ConvAgent_Posture", NewName="AnimNode_PS_AI_ConvAgent_Gaze")
+ClassRedirects=(OldName="AnimGraphNode_PS_AI_ConvAgent_Posture", NewName="AnimGraphNode_PS_AI_ConvAgent_Gaze")
; ── Factory classes ── ; ── Factory classes ──
+ClassRedirects=(OldName="ElevenLabsLipSyncPoseMapFactory", NewName="PS_AI_ConvAgent_LipSyncPoseMapFactory") +ClassRedirects=(OldName="ElevenLabsLipSyncPoseMapFactory", NewName="PS_AI_ConvAgent_LipSyncPoseMapFactory")

File diff suppressed because it is too large Load Diff

View File

@ -127,10 +127,7 @@ void FAnimNode_PS_AI_ConvAgent_BodyExpression::Evaluate_AnyThread(FPoseContext&
const bool bHavePrevPose = bCrossfading && PrevPose.GetNumBones() > 0; const bool bHavePrevPose = bCrossfading && PrevPose.GetNumBones() > 0;
// ── Per-bone blend (override mode) ────────────────────────────────────── // ── Per-bone blend ──────────────────────────────────────────────────────
// Lerp each bone from the upstream pose toward the expression pose,
// weighted by the bone mask and activation alpha.
// ExcludeBones (default: neck_01) prevents conflicts with Gaze/Posture.
for (int32 CompactIdx = 0; CompactIdx < NumBones; ++CompactIdx) for (int32 CompactIdx = 0; CompactIdx < NumBones; ++CompactIdx)
{ {
if (BoneMask[CompactIdx] <= 0.0f) if (BoneMask[CompactIdx] <= 0.0f)
@ -139,19 +136,18 @@ void FAnimNode_PS_AI_ConvAgent_BodyExpression::Evaluate_AnyThread(FPoseContext&
const FCompactPoseBoneIndex BoneIdx(CompactIdx); const FCompactPoseBoneIndex BoneIdx(CompactIdx);
const float W = BoneMask[CompactIdx] * FinalWeight; const float W = BoneMask[CompactIdx] * FinalWeight;
// Compute the expression pose (handle crossfade between prev and active)
FTransform ExpressionPose;
if (bHavePrevPose) if (bHavePrevPose)
{ {
ExpressionPose.Blend(PrevPose[BoneIdx], ActivePose[BoneIdx], CachedSnapshot.CrossfadeAlpha); // Crossfade between previous and active emotion poses
FTransform BlendedEmotion;
BlendedEmotion.Blend(PrevPose[BoneIdx], ActivePose[BoneIdx], CachedSnapshot.CrossfadeAlpha);
Output.Pose[BoneIdx].BlendWith(BlendedEmotion, W);
} }
else else
{ {
ExpressionPose = ActivePose[BoneIdx]; // Direct blend with active emotion pose
Output.Pose[BoneIdx].BlendWith(ActivePose[BoneIdx], W);
} }
// Override: replace (lerp) upstream toward expression pose
Output.Pose[BoneIdx].BlendWith(ExpressionPose, W);
} }
} }
@ -173,22 +169,22 @@ void FAnimNode_PS_AI_ConvAgent_BodyExpression::BuildBoneMask(const FBoneContaine
BoneMask.SetNumZeroed(NumBones); BoneMask.SetNumZeroed(NumBones);
bBoneMaskValid = false; bBoneMaskValid = false;
const FReferenceSkeleton& RefSkel = RequiredBones.GetReferenceSkeleton(); // Full body mode: all bones get weight 1.0
const TArray<FBoneIndexType>& BoneIndices = RequiredBones.GetBoneIndicesArray();
// ── Step 1: Build initial mask (include) ──────────────────────────────
if (!bUpperBodyOnly) if (!bUpperBodyOnly)
{ {
// Full body mode: all bones get weight 1.0
for (int32 i = 0; i < NumBones; ++i) for (int32 i = 0; i < NumBones; ++i)
{ {
BoneMask[i] = 1.0f; BoneMask[i] = 1.0f;
} }
bBoneMaskValid = (NumBones > 0);
UE_LOG(LogPS_AI_ConvAgent_BodyExprAnimNode, Log,
TEXT("Bone mask built: FULL BODY (%d bones)."), NumBones);
return;
} }
else
{
// Upper body mode: only BlendRootBone descendants // Upper body mode: only BlendRootBone descendants
const FReferenceSkeleton& RefSkel = RequiredBones.GetReferenceSkeleton();
const int32 RootMeshIdx = RefSkel.FindBoneIndex(BlendRootBone); const int32 RootMeshIdx = RefSkel.FindBoneIndex(BlendRootBone);
if (RootMeshIdx == INDEX_NONE) if (RootMeshIdx == INDEX_NONE)
@ -199,6 +195,9 @@ void FAnimNode_PS_AI_ConvAgent_BodyExpression::BuildBoneMask(const FBoneContaine
return; return;
} }
const TArray<FBoneIndexType>& BoneIndices = RequiredBones.GetBoneIndicesArray();
int32 MaskedCount = 0;
for (int32 CompactIdx = 0; CompactIdx < NumBones; ++CompactIdx) for (int32 CompactIdx = 0; CompactIdx < NumBones; ++CompactIdx)
{ {
const int32 MeshIdx = static_cast<int32>(BoneIndices[CompactIdx]); const int32 MeshIdx = static_cast<int32>(BoneIndices[CompactIdx]);
@ -211,73 +210,18 @@ void FAnimNode_PS_AI_ConvAgent_BodyExpression::BuildBoneMask(const FBoneContaine
if (Current == RootMeshIdx) if (Current == RootMeshIdx)
{ {
BoneMask[CompactIdx] = 1.0f; BoneMask[CompactIdx] = 1.0f;
break;
}
Current = RefSkel.GetParentIndex(Current);
}
}
}
// ── Step 2: Exclude bone subtrees ────────────────────────────────────
// For each exclude bone, zero out it and all its descendants.
// This prevents conflicts with Gaze/Posture on neck/head bones.
int32 ExcludedCount = 0;
if (ExcludeBones.Num() > 0)
{
// Collect mesh indices for all exclude roots
TArray<int32> ExcludeRootIndices;
for (const FName& BoneName : ExcludeBones)
{
const int32 ExclIdx = RefSkel.FindBoneIndex(BoneName);
if (ExclIdx != INDEX_NONE)
{
ExcludeRootIndices.Add(ExclIdx);
}
else
{
UE_LOG(LogPS_AI_ConvAgent_BodyExprAnimNode, Warning,
TEXT("ExcludeBone '%s' not found in skeleton — ignored."),
*BoneName.ToString());
}
}
// Zero out any bone whose parent chain includes an exclude root
for (int32 CompactIdx = 0; CompactIdx < NumBones; ++CompactIdx)
{
if (BoneMask[CompactIdx] <= 0.0f)
continue; // Already excluded or not included
const int32 MeshIdx = static_cast<int32>(BoneIndices[CompactIdx]);
int32 Current = MeshIdx;
while (Current != INDEX_NONE)
{
if (ExcludeRootIndices.Contains(Current))
{
BoneMask[CompactIdx] = 0.0f;
++ExcludedCount;
break;
}
Current = RefSkel.GetParentIndex(Current);
}
}
}
// ── Count final active bones ─────────────────────────────────────────
int32 MaskedCount = 0;
for (int32 i = 0; i < NumBones; ++i)
{
if (BoneMask[i] > 0.0f)
++MaskedCount; ++MaskedCount;
break;
}
Current = RefSkel.GetParentIndex(Current);
}
} }
bBoneMaskValid = (MaskedCount > 0); bBoneMaskValid = (MaskedCount > 0);
UE_LOG(LogPS_AI_ConvAgent_BodyExprAnimNode, Log, UE_LOG(LogPS_AI_ConvAgent_BodyExprAnimNode, Log,
TEXT("Bone mask built: %d/%d active bones (root='%s', excluded=%d from %d subtrees)."), TEXT("Bone mask built: %d/%d bones from root '%s'."),
MaskedCount, NumBones, *BlendRootBone.ToString(), MaskedCount, NumBones, *BlendRootBone.ToString());
ExcludedCount, ExcludeBones.Num());
} }
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────

View File

@ -89,15 +89,17 @@ void FAnimNode_PS_AI_ConvAgent_LipSync::Evaluate_AnyThread(FPoseContext& Output)
// ── Inject lip sync curves into the pose output ────────────────────── // ── Inject lip sync curves into the pose output ──────────────────────
// //
// While lip sync is producing curves: write them all (including 0s) // IMPORTANT: Always write ALL curves that lip sync has ever touched,
// and track every curve name in KnownCurveNames. // including at 0.0. If we skip near-zero curves, upstream animation
// values (idle expressions, breathing, etc.) leak through and cause
// visible pops when lip sync curves cross the threshold.
// //
// When lip sync goes silent (CachedCurves empty): release immediately. // Strategy:
// The component's SpeechBlendAlpha already handles smooth fade-out, // - While lip sync is producing curves: write them all (including 0s)
// so by the time CachedCurves becomes empty, values have already // and track every curve name in KnownCurveNames.
// decayed to near-zero. Releasing immediately allows the upstream // - After lip sync goes silent: keep writing 0s for a grace period
// FacialExpressionComponent's emotion curves (including mouth) to // (30 frames ≈ 0.5s) so the upstream can blend back in smoothly
// flow through without being overwritten by zeros. // via the component's activation alpha, then release.
if (CachedCurves.Num() > 0) if (CachedCurves.Num() > 0)
{ {
@ -111,9 +113,7 @@ void FAnimNode_PS_AI_ConvAgent_LipSync::Evaluate_AnyThread(FPoseContext& Output)
} }
// Zero any known curves NOT in the current frame // Zero any known curves NOT in the current frame
// (e.g. a blendshape that was active last frame but decayed away). // (e.g. a blendshape that was active last frame but decayed away)
// This prevents upstream animation values from leaking through
// during active speech when a curve temporarily goes to zero.
for (const FName& Name : KnownCurveNames) for (const FName& Name : KnownCurveNames)
{ {
if (!CachedCurves.Contains(Name)) if (!CachedCurves.Contains(Name))
@ -124,13 +124,25 @@ void FAnimNode_PS_AI_ConvAgent_LipSync::Evaluate_AnyThread(FPoseContext& Output)
} }
else if (KnownCurveNames.Num() > 0) else if (KnownCurveNames.Num() > 0)
{ {
// Lip sync went silent — release immediately. // Lip sync went silent — keep zeroing known curves for a grace
// The smooth fade-out was handled at the component level // period so upstream values don't pop in abruptly.
// (SpeechBlendAlpha), so upstream emotion curves can take over. ++FramesSinceLastActive;
if (FramesSinceLastActive < 30)
{
for (const FName& Name : KnownCurveNames)
{
Output.Curve.Set(Name, 0.0f);
}
}
else
{
// Grace period over — release curves, let upstream through
KnownCurveNames.Reset(); KnownCurveNames.Reset();
FramesSinceLastActive = 0; FramesSinceLastActive = 0;
} }
} }
}
void FAnimNode_PS_AI_ConvAgent_LipSync::GatherDebugData(FNodeDebugData& DebugData) void FAnimNode_PS_AI_ConvAgent_LipSync::GatherDebugData(FNodeDebugData& DebugData)
{ {

View File

@ -4,16 +4,9 @@
#include "PS_AI_ConvAgent_ElevenLabsComponent.h" #include "PS_AI_ConvAgent_ElevenLabsComponent.h"
#include "PS_AI_ConvAgent_BodyPoseMap.h" #include "PS_AI_ConvAgent_BodyPoseMap.h"
#include "Animation/AnimSequence.h" #include "Animation/AnimSequence.h"
#include "Engine/Engine.h"
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_BodyExpr, Log, All); DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_BodyExpr, Log, All);
static TAutoConsoleVariable<int32> CVarDebugBodyExpr(
TEXT("ps.ai.ConvAgent.Debug.BodyExpr"),
-1,
TEXT("Debug HUD for BodyExpression. -1=use property, 0=off, 1-3=verbosity."),
ECVF_Default);
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
// Construction // Construction
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@ -50,7 +43,7 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::BeginPlay()
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationConnected); this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationConnected);
Agent->OnAgentDisconnected.AddDynamic( Agent->OnAgentDisconnected.AddDynamic(
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationDisconnected); this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationDisconnected);
Agent->OnAudioPlaybackStarted.AddDynamic( Agent->OnAgentStartedSpeaking.AddDynamic(
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStarted); this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStarted);
Agent->OnAgentStoppedSpeaking.AddDynamic( Agent->OnAgentStoppedSpeaking.AddDynamic(
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStopped); this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStopped);
@ -103,7 +96,7 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::EndPlay(const EEndPlayReason::Typ
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationConnected); this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationConnected);
AgentComponent->OnAgentDisconnected.RemoveDynamic( AgentComponent->OnAgentDisconnected.RemoveDynamic(
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationDisconnected); this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationDisconnected);
AgentComponent->OnAudioPlaybackStarted.RemoveDynamic( AgentComponent->OnAgentStartedSpeaking.RemoveDynamic(
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStarted); this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStarted);
AgentComponent->OnAgentStoppedSpeaking.RemoveDynamic( AgentComponent->OnAgentStoppedSpeaking.RemoveDynamic(
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStopped); this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStopped);
@ -126,12 +119,6 @@ const TArray<TObjectPtr<UAnimSequence>>* UPS_AI_ConvAgent_BodyExpressionComponen
if (!BodyPoseMap) return nullptr; if (!BodyPoseMap) return nullptr;
const FPS_AI_ConvAgent_BodyAnimList* AnimList = BodyPoseMap->BodyPoses.Find(Emotion); const FPS_AI_ConvAgent_BodyAnimList* AnimList = BodyPoseMap->BodyPoses.Find(Emotion);
// Fallback to Neutral if the requested emotion has no entry in the data asset
if (!AnimList && Emotion != EPS_AI_ConvAgent_Emotion::Neutral)
{
AnimList = BodyPoseMap->BodyPoses.Find(EPS_AI_ConvAgent_Emotion::Neutral);
}
if (!AnimList) return nullptr; if (!AnimList) return nullptr;
if (!bSpeaking) if (!bSpeaking)
@ -191,15 +178,15 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::SwitchToNewAnim(UAnimSequence* Ne
if (!bForce && NewAnim == ActiveAnim) return; if (!bForce && NewAnim == ActiveAnim) return;
if (!NewAnim) return; if (!NewAnim) return;
// Always start a fresh crossfade from whatever is currently active. // Current active becomes previous for crossfade
// If a crossfade was in progress, the old PrevAnim is lost, but the
// transition FROM the current ActiveAnim (at its current time) to the
// new anim will always be smooth and predictable.
PrevAnim = ActiveAnim; PrevAnim = ActiveAnim;
PrevPlaybackTime = ActivePlaybackTime; PrevPlaybackTime = ActivePlaybackTime;
// New anim starts from the beginning
ActiveAnim = NewAnim; ActiveAnim = NewAnim;
ActivePlaybackTime = 0.0f; ActivePlaybackTime = 0.0f;
// Begin crossfade
CrossfadeAlpha = 0.0f; CrossfadeAlpha = 0.0f;
if (bDebug && DebugVerbosity >= 1) if (bDebug && DebugVerbosity >= 1)
@ -234,8 +221,6 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationConnected(
{ {
bActive = true; bActive = true;
bIsSpeaking = false; bIsSpeaking = false;
LastEventName = TEXT("Connected");
LastEventWorldTime = GetWorld() ? GetWorld()->GetTimeSeconds() : 0.0f;
// Start with an idle anim // Start with an idle anim
PickAndSwitchAnim(); PickAndSwitchAnim();
@ -252,8 +237,6 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationDisconnected(
{ {
bActive = false; bActive = false;
bIsSpeaking = false; bIsSpeaking = false;
LastEventName = TEXT("Disconnected");
LastEventWorldTime = GetWorld() ? GetWorld()->GetTimeSeconds() : 0.0f;
if (bDebug) if (bDebug)
{ {
@ -269,8 +252,6 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationDisconnected(
void UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStarted() void UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStarted()
{ {
bIsSpeaking = true; bIsSpeaking = true;
LastEventName = TEXT("SpeakStart");
LastEventWorldTime = GetWorld() ? GetWorld()->GetTimeSeconds() : 0.0f;
// Crossfade from idle anim to a speaking anim // Crossfade from idle anim to a speaking anim
PickAndSwitchAnim(); PickAndSwitchAnim();
@ -285,8 +266,6 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStarted()
void UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStopped() void UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStopped()
{ {
bIsSpeaking = false; bIsSpeaking = false;
LastEventName = TEXT("SpeakStop");
LastEventWorldTime = GetWorld() ? GetWorld()->GetTimeSeconds() : 0.0f;
// Crossfade from speaking anim to an idle anim // Crossfade from speaking anim to an idle anim
PickAndSwitchAnim(); PickAndSwitchAnim();
@ -301,8 +280,6 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStopped()
void UPS_AI_ConvAgent_BodyExpressionComponent::OnInterrupted() void UPS_AI_ConvAgent_BodyExpressionComponent::OnInterrupted()
{ {
bIsSpeaking = false; bIsSpeaking = false;
LastEventName = TEXT("Interrupted");
LastEventWorldTime = GetWorld() ? GetWorld()->GetTimeSeconds() : 0.0f;
// Crossfade to idle anim // Crossfade to idle anim
PickAndSwitchAnim(); PickAndSwitchAnim();
@ -326,9 +303,6 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::OnEmotionChanged(
ActiveEmotion = Emotion; ActiveEmotion = Emotion;
ActiveEmotionIntensity = Intensity; ActiveEmotionIntensity = Intensity;
LastEventName = FString::Printf(TEXT("Emotion:%s"),
*UEnum::GetDisplayValueAsText(Emotion).ToString());
LastEventWorldTime = GetWorld() ? GetWorld()->GetTimeSeconds() : 0.0f;
// Pick a new anim from the appropriate list for the new emotion // Pick a new anim from the appropriate list for the new emotion
PickAndSwitchAnim(); PickAndSwitchAnim();
@ -369,7 +343,7 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::TickComponent(
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationConnected); this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationConnected);
Agent->OnAgentDisconnected.AddDynamic( Agent->OnAgentDisconnected.AddDynamic(
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationDisconnected); this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationDisconnected);
Agent->OnAudioPlaybackStarted.AddDynamic( Agent->OnAgentStartedSpeaking.AddDynamic(
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStarted); this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStarted);
Agent->OnAgentStoppedSpeaking.AddDynamic( Agent->OnAgentStoppedSpeaking.AddDynamic(
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStopped); this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStopped);
@ -384,12 +358,9 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::TickComponent(
const float TargetAlpha = bActive ? 1.0f : 0.0f; const float TargetAlpha = bActive ? 1.0f : 0.0f;
if (!FMath::IsNearlyEqual(CurrentActiveAlpha, TargetAlpha, 0.001f)) if (!FMath::IsNearlyEqual(CurrentActiveAlpha, TargetAlpha, 0.001f))
{ {
// Exponential ease-out: fast start, gradual approach to target. const float BlendSpeed = 1.0f / FMath::Max(ActivationBlendDuration, 0.01f);
// Factor of 3 compensates for FInterpTo's exponential decay CurrentActiveAlpha = FMath::FInterpConstantTo(
// reaching ~95% in ActivationBlendDuration seconds. CurrentActiveAlpha, TargetAlpha, DeltaTime, BlendSpeed);
const float InterpSpeed = 3.0f / FMath::Max(ActivationBlendDuration, 0.01f);
CurrentActiveAlpha = FMath::FInterpTo(
CurrentActiveAlpha, TargetAlpha, DeltaTime, InterpSpeed);
} }
else else
{ {
@ -445,8 +416,6 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::TickComponent(
if (NewAnim) if (NewAnim)
{ {
SwitchToNewAnim(NewAnim, true); SwitchToNewAnim(NewAnim, true);
LastEventName = TEXT("AutoCycle");
LastEventWorldTime = GetWorld() ? GetWorld()->GetTimeSeconds() : 0.0f;
if (bDebug && DebugVerbosity >= 2) if (bDebug && DebugVerbosity >= 2)
{ {
@ -462,16 +431,12 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::TickComponent(
if (CrossfadeAlpha < 1.0f) if (CrossfadeAlpha < 1.0f)
{ {
// Exponential ease-out: fast start, gradual approach to 1.0. const float BlendSpeed = 1.0f / FMath::Max(0.05f, EmotionBlendDuration);
// Factor of 3 compensates for FInterpTo's exponential decay CrossfadeAlpha = FMath::Min(1.0f, CrossfadeAlpha + DeltaTime * BlendSpeed);
// reaching ~95% in EmotionBlendDuration seconds.
const float InterpSpeed = 3.0f / FMath::Max(0.05f, EmotionBlendDuration);
CrossfadeAlpha = FMath::FInterpTo(CrossfadeAlpha, 1.0f, DeltaTime, InterpSpeed);
// Snap to 1.0 when close enough, release previous anim // Crossfade complete — release previous anim
if (CrossfadeAlpha > 0.999f) if (CrossfadeAlpha >= 1.0f)
{ {
CrossfadeAlpha = 1.0f;
PrevAnim = nullptr; PrevAnim = nullptr;
PrevPlaybackTime = 0.0f; PrevPlaybackTime = 0.0f;
} }
@ -484,82 +449,8 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::TickComponent(
CurrentSnapshot.PrevAnim = PrevAnim; CurrentSnapshot.PrevAnim = PrevAnim;
CurrentSnapshot.ActiveTime = ActivePlaybackTime; CurrentSnapshot.ActiveTime = ActivePlaybackTime;
CurrentSnapshot.PrevTime = PrevPlaybackTime; CurrentSnapshot.PrevTime = PrevPlaybackTime;
// FInterpTo already provides exponential easing — pass alpha directly.
CurrentSnapshot.CrossfadeAlpha = CrossfadeAlpha; CurrentSnapshot.CrossfadeAlpha = CrossfadeAlpha;
CurrentSnapshot.ActivationAlpha = CurrentActiveAlpha; CurrentSnapshot.ActivationAlpha = CurrentActiveAlpha;
CurrentSnapshot.BlendWeight = BlendWeight; CurrentSnapshot.BlendWeight = BlendWeight;
} }
// ── On-screen debug HUD ───────────────────────────────────────────────
{
const int32 CVarVal = CVarDebugBodyExpr.GetValueOnGameThread();
const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug;
const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity;
if (bShowDebug && EffectiveVerbosity >= 1)
{
DrawDebugHUD();
}
}
}
// ─────────────────────────────────────────────────────────────────────────────
// On-screen debug display
// ─────────────────────────────────────────────────────────────────────────────
void UPS_AI_ConvAgent_BodyExpressionComponent::DrawDebugHUD() const
{
if (!GEngine) return;
const float WorldTime = GetWorld() ? GetWorld()->GetTimeSeconds() : 0.0f;
const float EventAge = WorldTime - LastEventWorldTime;
// Active anim name
FString ActiveName = ActiveAnim ? ActiveAnim->GetName() : TEXT("(none)");
FString PrevName = PrevAnim ? PrevAnim->GetName() : TEXT("---");
// State label
FString StateStr;
if (!bActive)
StateStr = TEXT("INACTIVE");
else if (bIsSpeaking)
StateStr = TEXT("SPEAKING");
else
StateStr = TEXT("IDLE");
// Event label with age
FString EventStr = LastEventName.IsEmpty()
? TEXT("---")
: FString::Printf(TEXT("%s (%.1fs ago)"), *LastEventName, EventAge);
// Use key offset to avoid colliding with other debug messages
// Keys 2000-2010 reserved for BodyExpression
const int32 BaseKey = 2000;
const float DisplayTime = 1.0f;
const FColor MainColor = FColor::Cyan;
const FColor WarnColor = FColor::Yellow;
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, MainColor,
FString::Printf(TEXT("=== BODY EXPR: %s ==="), *StateStr));
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor,
FString::Printf(TEXT(" ActivationAlpha: %.3f (target: %s)"),
CurrentActiveAlpha, bActive ? TEXT("1") : TEXT("0")));
GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, MainColor,
FString::Printf(TEXT(" Active: %s t=%.2f"), *ActiveName, ActivePlaybackTime));
GEngine->AddOnScreenDebugMessage(BaseKey + 3, DisplayTime,
CrossfadeAlpha < 1.0f ? WarnColor : MainColor,
FString::Printf(TEXT(" Crossfade: %.3f Prev: %s"),
CrossfadeAlpha, *PrevName));
GEngine->AddOnScreenDebugMessage(BaseKey + 4, DisplayTime, MainColor,
FString::Printf(TEXT(" Emotion: %s (%s) Weight: %.2f"),
*UEnum::GetDisplayValueAsText(ActiveEmotion).ToString(),
*UEnum::GetDisplayValueAsText(ActiveEmotionIntensity).ToString(),
BlendWeight));
GEngine->AddOnScreenDebugMessage(BaseKey + 5, DisplayTime,
EventAge < 1.0f ? FColor::Green : MainColor,
FString::Printf(TEXT(" LastEvent: %s"), *EventStr));
} }

View File

@ -20,12 +20,6 @@
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_ElevenLabs, Log, All); DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_ElevenLabs, Log, All);
static TAutoConsoleVariable<int32> CVarDebugElevenLabs(
TEXT("ps.ai.ConvAgent.Debug.ElevenLabs"),
-1,
TEXT("Debug HUD for ElevenLabs. -1=use property, 0=off, 1-3=verbosity."),
ECVF_Default);
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
// Constructor // Constructor
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@ -151,16 +145,10 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::TickComponent(float DeltaTime, ELevel
TEXT("[T+%.2fs] [Turn %d] Pre-buffer timeout (%dms). Starting playback."), TEXT("[T+%.2fs] [Turn %d] Pre-buffer timeout (%dms). Starting playback."),
Tpb, LastClosedTurnIndex, AudioPreBufferMs); Tpb, LastClosedTurnIndex, AudioPreBufferMs);
} }
// Only start playback if the agent is still speaking.
// If silence detection already set bAgentSpeaking=false, this is stale.
if (bAgentSpeaking)
{
if (AudioPlaybackComponent && !AudioPlaybackComponent->IsPlaying()) if (AudioPlaybackComponent && !AudioPlaybackComponent->IsPlaying())
{ {
AudioPlaybackComponent->Play(); AudioPlaybackComponent->Play();
} }
OnAudioPlaybackStarted.Broadcast();
}
} }
} }
@ -235,7 +223,6 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::TickComponent(float DeltaTime, ELevel
{ {
bHardTimeoutFired = bHardTimeout && !bAgentResponseReceived; bHardTimeoutFired = bHardTimeout && !bAgentResponseReceived;
bAgentSpeaking = false; bAgentSpeaking = false;
bPreBuffering = false; // Cancel pending pre-buffer to prevent stale OnAudioPlaybackStarted.
bAgentResponseReceived = false; bAgentResponseReceived = false;
SilentTickCount = 0; SilentTickCount = 0;
bShouldBroadcastStopped = true; bShouldBroadcastStopped = true;
@ -264,17 +251,6 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::TickComponent(float DeltaTime, ELevel
MulticastAgentStoppedSpeaking(); MulticastAgentStoppedSpeaking();
} }
} }
// On-screen debug display.
{
const int32 CVarVal = CVarDebugElevenLabs.GetValueOnGameThread();
const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug;
const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity;
if (bShowDebug && EffectiveVerbosity >= 1)
{
DrawDebugHUD();
}
}
} }
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@ -391,17 +367,9 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::EndConversation()
{ {
bIntentionalDisconnect = true; bIntentionalDisconnect = true;
WebSocketProxy->Disconnect(); WebSocketProxy->Disconnect();
// OnClosed callback will fire OnAgentDisconnected.
WebSocketProxy = nullptr; WebSocketProxy = nullptr;
} }
} }
else
{
// Persistent mode: WebSocket stays alive but the interaction is over.
// Broadcast OnAgentDisconnected so expression components deactivate
// (body, facial, etc.). The WebSocket OnClosed never fires here.
OnAgentDisconnected.Broadcast(1000, TEXT("EndConversation (persistent)"));
}
// Reset replicated state so other players can talk to this NPC. // Reset replicated state so other players can talk to this NPC.
bNetIsConversing = false; bNetIsConversing = false;
@ -1365,14 +1333,10 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::EnqueueAgentAudio(const TArray<uint8>
Tpb2, LastClosedTurnIndex, AudioPreBufferMs); Tpb2, LastClosedTurnIndex, AudioPreBufferMs);
} }
} }
else else if (AudioPlaybackComponent && !AudioPlaybackComponent->IsPlaying())
{
if (AudioPlaybackComponent && !AudioPlaybackComponent->IsPlaying())
{ {
AudioPlaybackComponent->Play(); AudioPlaybackComponent->Play();
} }
OnAudioPlaybackStarted.Broadcast();
}
} }
else if (bPreBuffering) else if (bPreBuffering)
{ {
@ -1397,7 +1361,6 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::EnqueueAgentAudio(const TArray<uint8>
{ {
AudioPlaybackComponent->Play(); AudioPlaybackComponent->Play();
} }
OnAudioPlaybackStarted.Broadcast();
} }
SilentTickCount = 0; SilentTickCount = 0;
} }
@ -2045,77 +2008,3 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ApplyConversationGaze()
} }
} }
} }
// ─────────────────────────────────────────────────────────────────────────────
// On-screen debug display
// ─────────────────────────────────────────────────────────────────────────────
void UPS_AI_ConvAgent_ElevenLabsComponent::DrawDebugHUD() const
{
if (!GEngine) return;
const int32 BaseKey = 2040;
const float DisplayTime = 1.0f;
const FColor MainColor = FColor::Cyan;
const FColor WarnColor = FColor::Yellow;
const FColor GoodColor = FColor::Green;
const bool bConnected = IsConnected();
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime,
bConnected ? GoodColor : FColor::Red,
FString::Printf(TEXT("=== ELEVENLABS: %s ==="),
bConnected ? TEXT("CONNECTED") : TEXT("DISCONNECTED")));
// Session info
FString ModeStr = (TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server)
? TEXT("ServerVAD") : TEXT("ClientPTT");
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor,
FString::Printf(TEXT(" Session: %s Turn: %d Mode: %s"),
bPersistentSession ? TEXT("persistent") : TEXT("ephemeral"),
TurnIndex, *ModeStr));
// State flags
const bool bListening = bIsListening.load();
const bool bSpeaking = bAgentSpeaking.load();
const bool bGenerating = bAgentGenerating.load();
GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, MainColor,
FString::Printf(TEXT(" Listening: %s Speaking: %s Generating: %s"),
bListening ? TEXT("YES") : TEXT("NO"),
bSpeaking ? TEXT("YES") : TEXT("NO"),
bGenerating ? TEXT("YES") : TEXT("NO")));
const bool bWaiting = bWaitingForAgentResponse.load();
GEngine->AddOnScreenDebugMessage(BaseKey + 3, DisplayTime,
(bPreBuffering || bWaiting) ? WarnColor : MainColor,
FString::Printf(TEXT(" PreBuffer: %s WaitResponse: %s"),
bPreBuffering ? TEXT("YES") : TEXT("NO"),
bWaiting ? TEXT("YES") : TEXT("NO")));
// Emotion
GEngine->AddOnScreenDebugMessage(BaseKey + 4, DisplayTime, MainColor,
FString::Printf(TEXT(" Emotion: %s (%s)"),
*UEnum::GetDisplayValueAsText(CurrentEmotion).ToString(),
*UEnum::GetDisplayValueAsText(CurrentEmotionIntensity).ToString()));
// Audio queue (read without lock for debug display — minor race is acceptable)
const int32 QueueBytes = FMath::Max(0, AudioQueue.Num() - AudioQueueReadOffset);
GEngine->AddOnScreenDebugMessage(BaseKey + 5, DisplayTime, MainColor,
FString::Printf(TEXT(" AudioQueue: %d bytes SilentTicks: %d"),
QueueBytes, SilentTickCount));
// Timing
const double Now = FPlatformTime::Seconds();
const float SessionSec = (SessionStartTime > 0.0) ? static_cast<float>(Now - SessionStartTime) : 0.0f;
const float TurnSec = (TurnStartTime > 0.0) ? static_cast<float>(Now - TurnStartTime) : 0.0f;
GEngine->AddOnScreenDebugMessage(BaseKey + 6, DisplayTime, MainColor,
FString::Printf(TEXT(" Timing: session=%.1fs turn=%.1fs"),
SessionSec, TurnSec));
// Reconnection
GEngine->AddOnScreenDebugMessage(BaseKey + 7, DisplayTime,
bWantsReconnect ? FColor::Red : MainColor,
FString::Printf(TEXT(" Reconnect: %d/%d attempts%s"),
ReconnectAttemptCount, MaxReconnectAttempts,
bWantsReconnect ? TEXT(" (ACTIVE)") : TEXT("")));
}

View File

@ -5,16 +5,9 @@
#include "PS_AI_ConvAgent_EmotionPoseMap.h" #include "PS_AI_ConvAgent_EmotionPoseMap.h"
#include "Animation/AnimSequence.h" #include "Animation/AnimSequence.h"
#include "Animation/AnimCurveTypes.h" #include "Animation/AnimCurveTypes.h"
#include "Engine/Engine.h"
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_FacialExpr, Log, All); DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_FacialExpr, Log, All);
static TAutoConsoleVariable<int32> CVarDebugFacialExpr(
TEXT("ps.ai.ConvAgent.Debug.FacialExpr"),
-1,
TEXT("Debug HUD for FacialExpression. -1=use property, 0=off, 1-3=verbosity."),
ECVF_Default);
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
// Construction // Construction
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@ -143,12 +136,6 @@ UAnimSequence* UPS_AI_ConvAgent_FacialExpressionComponent::FindAnimForEmotion(
if (!EmotionPoseMap) return nullptr; if (!EmotionPoseMap) return nullptr;
const FPS_AI_ConvAgent_EmotionPoseSet* PoseSet = EmotionPoseMap->EmotionPoses.Find(Emotion); const FPS_AI_ConvAgent_EmotionPoseSet* PoseSet = EmotionPoseMap->EmotionPoses.Find(Emotion);
// Fallback to Neutral if the requested emotion has no entry in the data asset
if (!PoseSet && Emotion != EPS_AI_ConvAgent_Emotion::Neutral)
{
PoseSet = EmotionPoseMap->EmotionPoses.Find(EPS_AI_ConvAgent_Emotion::Neutral);
}
if (!PoseSet) return nullptr; if (!PoseSet) return nullptr;
// Direct match // Direct match
@ -295,10 +282,9 @@ void UPS_AI_ConvAgent_FacialExpressionComponent::TickComponent(
const float TargetAlpha = bActive ? 1.0f : 0.0f; const float TargetAlpha = bActive ? 1.0f : 0.0f;
if (!FMath::IsNearlyEqual(CurrentActiveAlpha, TargetAlpha, 0.001f)) if (!FMath::IsNearlyEqual(CurrentActiveAlpha, TargetAlpha, 0.001f))
{ {
// Exponential ease-out: fast start, gradual approach to target. const float BlendSpeed = 1.0f / FMath::Max(ActivationBlendDuration, 0.01f);
const float InterpSpeed = 3.0f / FMath::Max(ActivationBlendDuration, 0.01f); CurrentActiveAlpha = FMath::FInterpConstantTo(
CurrentActiveAlpha = FMath::FInterpTo( CurrentActiveAlpha, TargetAlpha, DeltaTime, BlendSpeed);
CurrentActiveAlpha, TargetAlpha, DeltaTime, InterpSpeed);
} }
else else
{ {
@ -370,16 +356,13 @@ void UPS_AI_ConvAgent_FacialExpressionComponent::TickComponent(
for (const auto& P : ActiveCurves) AllCurves.Add(P.Key); for (const auto& P : ActiveCurves) AllCurves.Add(P.Key);
for (const auto& P : PrevCurves) AllCurves.Add(P.Key); for (const auto& P : PrevCurves) AllCurves.Add(P.Key);
// Apply SmoothStep for ease-in-out crossfade (raw alpha is linear)
const float SmoothedCrossfade = FMath::SmoothStep(0.0f, 1.0f, CrossfadeAlpha);
for (const FName& CurveName : AllCurves) for (const FName& CurveName : AllCurves)
{ {
const float PrevVal = PrevCurves.Contains(CurveName) const float PrevVal = PrevCurves.Contains(CurveName)
? PrevCurves[CurveName] : 0.0f; ? PrevCurves[CurveName] : 0.0f;
const float ActiveVal = ActiveCurves.Contains(CurveName) const float ActiveVal = ActiveCurves.Contains(CurveName)
? ActiveCurves[CurveName] : 0.0f; ? ActiveCurves[CurveName] : 0.0f;
const float Blended = FMath::Lerp(PrevVal, ActiveVal, SmoothedCrossfade); const float Blended = FMath::Lerp(PrevVal, ActiveVal, CrossfadeAlpha);
// Always include the curve even at 0 — the AnimNode needs // Always include the curve even at 0 — the AnimNode needs
// to see it to block upstream values from popping through. // to see it to block upstream values from popping through.
@ -405,64 +388,6 @@ void UPS_AI_ConvAgent_FacialExpressionComponent::TickComponent(
FScopeLock Lock(&EmotionCurveLock); FScopeLock Lock(&EmotionCurveLock);
CurrentEmotionCurves = MoveTemp(NewCurves); CurrentEmotionCurves = MoveTemp(NewCurves);
} }
// ── On-screen debug HUD ───────────────────────────────────────────────
{
const int32 CVarVal = CVarDebugFacialExpr.GetValueOnGameThread();
const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug;
const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity;
if (bShowDebug && EffectiveVerbosity >= 1)
{
DrawDebugHUD();
}
}
}
// ─────────────────────────────────────────────────────────────────────────────
// On-screen debug display
// ─────────────────────────────────────────────────────────────────────────────
void UPS_AI_ConvAgent_FacialExpressionComponent::DrawDebugHUD() const
{
if (!GEngine) return;
const int32 BaseKey = 2010;
const float DisplayTime = 1.0f;
const FColor MainColor = FColor::Cyan;
const FColor WarnColor = FColor::Yellow;
// State label
FString StateStr = bActive ? TEXT("ACTIVE") : TEXT("INACTIVE");
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, MainColor,
FString::Printf(TEXT("=== FACIAL EXPR: %s ==="), *StateStr));
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor,
FString::Printf(TEXT(" ActivationAlpha: %.3f (target: %s)"),
CurrentActiveAlpha, bActive ? TEXT("1") : TEXT("0")));
FString ActiveName = ActiveAnim ? ActiveAnim->GetName() : TEXT("(none)");
GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, MainColor,
FString::Printf(TEXT(" Active: %s t=%.2f"), *ActiveName, ActivePlaybackTime));
FString PrevName = PrevAnim ? PrevAnim->GetName() : TEXT("---");
GEngine->AddOnScreenDebugMessage(BaseKey + 3, DisplayTime,
CrossfadeAlpha < 1.0f ? WarnColor : MainColor,
FString::Printf(TEXT(" Crossfade: %.3f Prev: %s"),
CrossfadeAlpha, *PrevName));
GEngine->AddOnScreenDebugMessage(BaseKey + 4, DisplayTime, MainColor,
FString::Printf(TEXT(" Emotion: %s (%s)"),
*UEnum::GetDisplayValueAsText(ActiveEmotion).ToString(),
*UEnum::GetDisplayValueAsText(ActiveEmotionIntensity).ToString()));
int32 CurveCount = 0;
{
FScopeLock Lock(&EmotionCurveLock);
CurveCount = CurrentEmotionCurves.Num();
}
GEngine->AddOnScreenDebugMessage(BaseKey + 5, DisplayTime, MainColor,
FString::Printf(TEXT(" Curves: %d active"), CurveCount));
} }
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────

View File

@ -3,21 +3,12 @@
#include "PS_AI_ConvAgent_GazeComponent.h" #include "PS_AI_ConvAgent_GazeComponent.h"
#include "PS_AI_ConvAgent_ElevenLabsComponent.h" #include "PS_AI_ConvAgent_ElevenLabsComponent.h"
#include "Components/SkeletalMeshComponent.h" #include "Components/SkeletalMeshComponent.h"
#include "Camera/CameraComponent.h"
#include "GameFramework/Actor.h" #include "GameFramework/Actor.h"
#include "GameFramework/Pawn.h"
#include "Math/UnrealMathUtility.h" #include "Math/UnrealMathUtility.h"
#include "DrawDebugHelpers.h" #include "DrawDebugHelpers.h"
#include "Engine/Engine.h"
DEFINE_LOG_CATEGORY(LogPS_AI_ConvAgent_Gaze); DEFINE_LOG_CATEGORY(LogPS_AI_ConvAgent_Gaze);
static TAutoConsoleVariable<int32> CVarDebugGaze(
TEXT("ps.ai.ConvAgent.Debug.Gaze"),
-1,
TEXT("Debug HUD for Gaze. -1=use property, 0=off, 1-3=verbosity."),
ECVF_Default);
// ── ARKit eye curve names ──────────────────────────────────────────────────── // ── ARKit eye curve names ────────────────────────────────────────────────────
static const FName EyeLookUpLeft(TEXT("eyeLookUpLeft")); static const FName EyeLookUpLeft(TEXT("eyeLookUpLeft"));
static const FName EyeLookDownLeft(TEXT("eyeLookDownLeft")); static const FName EyeLookDownLeft(TEXT("eyeLookDownLeft"));
@ -47,8 +38,7 @@ static const FName TargetHeadBone(TEXT("head"));
* bAutoTargetEyes = true: * bAutoTargetEyes = true:
* 1. Try eye bones (FACIAL_L_Eye / FACIAL_R_Eye midpoint) on the target's Face mesh. * 1. Try eye bones (FACIAL_L_Eye / FACIAL_R_Eye midpoint) on the target's Face mesh.
* 2. Fallback to "head" bone on any skeletal mesh. * 2. Fallback to "head" bone on any skeletal mesh.
* 3. Fallback to CameraComponent location (first-person pawn). * 3. Fallback to TargetActor origin + (0, 0, FallbackEyeHeight).
* 4. Fallback to TargetActor origin + (0, 0, FallbackEyeHeight) for non-camera actors.
* *
* bAutoTargetEyes = false: * bAutoTargetEyes = false:
* Always returns TargetActor origin + TargetOffset. * Always returns TargetActor origin + TargetOffset.
@ -86,15 +76,7 @@ static FVector ResolveTargetPosition(const AActor* Target, bool bAutoEyes,
} }
} }
// Fallback: CameraComponent — the canonical eye position for // No skeleton — use FallbackEyeHeight
// first-person pawns and any actor with an active camera.
if (const UCameraComponent* Cam =
const_cast<AActor*>(Target)->FindComponentByClass<UCameraComponent>())
{
return Cam->GetComponentLocation();
}
// Final fallback: actor origin + height offset
return Target->GetActorLocation() + FVector(0.0f, 0.0f, FallbackHeight); return Target->GetActorLocation() + FVector(0.0f, 0.0f, FallbackHeight);
} }
@ -353,10 +335,9 @@ void UPS_AI_ConvAgent_GazeComponent::TickComponent(
const float TargetAlpha = bActive ? 1.0f : 0.0f; const float TargetAlpha = bActive ? 1.0f : 0.0f;
if (!FMath::IsNearlyEqual(CurrentActiveAlpha, TargetAlpha, 0.001f)) if (!FMath::IsNearlyEqual(CurrentActiveAlpha, TargetAlpha, 0.001f))
{ {
// Exponential ease-out: fast start, gradual approach to target. const float BlendSpeed = 1.0f / FMath::Max(ActivationBlendDuration, 0.01f);
const float InterpSpeed = 3.0f / FMath::Max(ActivationBlendDuration, 0.01f); CurrentActiveAlpha = FMath::FInterpConstantTo(
CurrentActiveAlpha = FMath::FInterpTo( CurrentActiveAlpha, TargetAlpha, SafeDeltaTime, BlendSpeed);
CurrentActiveAlpha, TargetAlpha, SafeDeltaTime, InterpSpeed);
} }
else else
{ {
@ -653,57 +634,4 @@ void UPS_AI_ConvAgent_GazeComponent::TickComponent(
Owner->GetActorRotation().Yaw); Owner->GetActorRotation().Yaw);
} }
} }
// ── On-screen debug HUD ───────────────────────────────────────────────
{
const int32 CVarVal = CVarDebugGaze.GetValueOnGameThread();
const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug;
const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity;
if (bShowDebug && EffectiveVerbosity >= 1)
{
DrawDebugHUD();
}
}
}
// ─────────────────────────────────────────────────────────────────────────────
// On-screen debug display
// ─────────────────────────────────────────────────────────────────────────────
void UPS_AI_ConvAgent_GazeComponent::DrawDebugHUD() const
{
if (!GEngine) return;
const int32 BaseKey = 2020;
const float DisplayTime = 1.0f;
const FColor MainColor = FColor::Cyan;
FString StateStr = bActive ? TEXT("ACTIVE") : TEXT("INACTIVE");
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, MainColor,
FString::Printf(TEXT("=== GAZE: %s ==="), *StateStr));
FString TargetName = TargetActor ? TargetActor->GetName() : TEXT("(none)");
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor,
FString::Printf(TEXT(" Target: %s BodyTrack: %s"),
*TargetName, bEnableBodyTracking ? TEXT("ON") : TEXT("OFF")));
GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, MainColor,
FString::Printf(TEXT(" ActivationAlpha: %.3f"), CurrentActiveAlpha));
GEngine->AddOnScreenDebugMessage(BaseKey + 3, DisplayTime, MainColor,
FString::Printf(TEXT(" Head: Yaw=%.1f Pitch=%.1f (target: %.1f / %.1f)"),
CurrentHeadYaw, CurrentHeadPitch, TargetHeadYaw, TargetHeadPitch));
GEngine->AddOnScreenDebugMessage(BaseKey + 4, DisplayTime, MainColor,
FString::Printf(TEXT(" Eyes: Yaw=%.1f Pitch=%.1f"),
CurrentEyeYaw, CurrentEyePitch));
GEngine->AddOnScreenDebugMessage(BaseKey + 5, DisplayTime, MainColor,
FString::Printf(TEXT(" Body: SmoothedYaw=%.1f TargetYaw=%.1f"),
SmoothedBodyYaw, TargetBodyWorldYaw));
GEngine->AddOnScreenDebugMessage(BaseKey + 6, DisplayTime, MainColor,
FString::Printf(TEXT(" Compensation: Head=%.2f Eye=%.2f Body=%.2f"),
HeadAnimationCompensation, EyeAnimationCompensation, BodyDriftCompensation));
} }

View File

@ -14,12 +14,6 @@
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_Select, Log, All); DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_Select, Log, All);
static TAutoConsoleVariable<int32> CVarDebugInteraction(
TEXT("ps.ai.ConvAgent.Debug.Interaction"),
-1,
TEXT("Debug HUD for Interaction. -1=use property, 0=off, 1-3=verbosity."),
ECVF_Default);
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
// Constructor // Constructor
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@ -116,16 +110,10 @@ void UPS_AI_ConvAgent_InteractionComponent::TickComponent(float DeltaTime, ELeve
return; return;
} }
// ── This is the locally controlled pawn — find or create mic component ── // ── This is the locally controlled pawn — create the mic component ──
MicComponent = GetOwner()->FindComponentByClass<UPS_AI_ConvAgent_MicrophoneCaptureComponent>();
if (!MicComponent)
{
MicComponent = NewObject<UPS_AI_ConvAgent_MicrophoneCaptureComponent>( MicComponent = NewObject<UPS_AI_ConvAgent_MicrophoneCaptureComponent>(
GetOwner(), TEXT("PS_AI_ConvAgent_Mic_Interaction")); GetOwner(), TEXT("PS_AI_ConvAgent_Mic_Interaction"));
MicComponent->bDebug = bDebug;
MicComponent->DebugVerbosity = DebugVerbosity;
MicComponent->RegisterComponent(); MicComponent->RegisterComponent();
}
MicComponent->OnAudioCaptured.AddUObject(this, MicComponent->OnAudioCaptured.AddUObject(this,
&UPS_AI_ConvAgent_InteractionComponent::OnMicAudioCaptured); &UPS_AI_ConvAgent_InteractionComponent::OnMicAudioCaptured);
@ -148,17 +136,6 @@ void UPS_AI_ConvAgent_InteractionComponent::TickComponent(float DeltaTime, ELeve
{ {
SetSelectedAgent(BestAgent); SetSelectedAgent(BestAgent);
} }
// ── On-screen debug HUD ───────────────────────────────────────────────
{
const int32 CVarVal = CVarDebugInteraction.GetValueOnGameThread();
const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug;
const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity;
if (bShowDebug && EffectiveVerbosity >= 1)
{
DrawDebugHUD();
}
}
} }
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@ -188,26 +165,6 @@ UPS_AI_ConvAgent_ElevenLabsComponent* UPS_AI_ConvAgent_InteractionComponent::Eva
UPS_AI_ConvAgent_ElevenLabsComponent* CurrentAgent = SelectedAgent.Get(); UPS_AI_ConvAgent_ElevenLabsComponent* CurrentAgent = SelectedAgent.Get();
// ── Conversation lock ──────────────────────────────────────────────
// While we're actively conversing with an agent, keep it selected as
// long as it's within interaction distance — ignore the view cone.
// This prevents deselect/reselect flicker when the player turns quickly
// (which would cause spurious OnAgentConnected re-broadcasts in
// persistent session mode).
if (CurrentAgent && CurrentAgent->bNetIsConversing)
{
if (AActor* AgentActor = CurrentAgent->GetOwner())
{
const FVector AgentLoc = AgentActor->GetActorLocation()
+ FVector(0.0f, 0.0f, AgentEyeLevelOffset);
const float DistSq = (AgentLoc - ViewLocation).SizeSquared();
if (DistSq <= MaxDistSq)
{
return CurrentAgent; // Keep conversing agent selected.
}
}
}
// Get local player's pawn for occupied-NPC check. // Get local player's pawn for occupied-NPC check.
// Use pawn (replicated to ALL clients) instead of PlayerController // Use pawn (replicated to ALL clients) instead of PlayerController
// (only replicated to owning client due to bOnlyRelevantToOwner=true). // (only replicated to owning client due to bOnlyRelevantToOwner=true).
@ -519,37 +476,6 @@ void UPS_AI_ConvAgent_InteractionComponent::StartConversationWithSelectedAgent()
} }
} }
void UPS_AI_ConvAgent_InteractionComponent::SendTextToSelectedAgent(const FString& Text)
{
UPS_AI_ConvAgent_ElevenLabsComponent* Agent = SelectedAgent.Get();
if (!Agent)
{
if (bDebug)
{
UE_LOG(LogPS_AI_ConvAgent_Select, Warning,
TEXT("SendTextToSelectedAgent: no agent selected."));
}
return;
}
// Route through relay on clients (can't call Server RPCs on NPC actors).
if (GetOwnerRole() == ROLE_Authority || (GetWorld() && GetWorld()->GetNetMode() == NM_Standalone))
{
Agent->SendTextMessage(Text);
}
else
{
ServerRelaySendText(Agent->GetOwner(), Text);
}
if (bDebug)
{
UE_LOG(LogPS_AI_ConvAgent_Select, Log,
TEXT("SendTextToSelectedAgent: \"%s\" → %s"),
*Text, Agent->GetOwner() ? *Agent->GetOwner()->GetName() : TEXT("(null)"));
}
}
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
// Gaze helpers // Gaze helpers
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@ -632,71 +558,6 @@ void UPS_AI_ConvAgent_InteractionComponent::OnMicAudioCaptured(const TArray<floa
Agent->FeedExternalAudio(FloatPCM); Agent->FeedExternalAudio(FloatPCM);
} }
// ─────────────────────────────────────────────────────────────────────────────
// On-screen debug display
// ─────────────────────────────────────────────────────────────────────────────
void UPS_AI_ConvAgent_InteractionComponent::DrawDebugHUD() const
{
if (!GEngine) return;
const int32 BaseKey = 2060;
const float DisplayTime = 1.0f;
const FColor MainColor = FColor::Cyan;
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, MainColor,
TEXT("=== INTERACTION ==="));
UPS_AI_ConvAgent_ElevenLabsComponent* Agent = SelectedAgent.Get();
if (Agent)
{
FString AgentName = Agent->GetOwner() ? Agent->GetOwner()->GetName() : TEXT("(?)");
FString ConvState = Agent->bNetIsConversing ? TEXT("conversing") : TEXT("selected");
// Compute distance and angle to selected agent
FVector ViewLoc, ViewDir;
GetPawnViewPoint(ViewLoc, ViewDir);
float Dist = 0.0f;
float Angle = 0.0f;
if (Agent->GetOwner())
{
FVector AgentLoc = Agent->GetOwner()->GetActorLocation()
+ FVector(0.0f, 0.0f, AgentEyeLevelOffset);
FVector ToAgent = AgentLoc - ViewLoc;
Dist = ToAgent.Size();
FVector DirToAgent = ToAgent.GetSafeNormal();
Angle = FMath::RadiansToDegrees(
FMath::Acos(FMath::Clamp(FVector::DotProduct(ViewDir, DirToAgent), -1.0f, 1.0f)));
}
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor,
FString::Printf(TEXT(" Selected: %s (%s)"), *AgentName, *ConvState));
GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, MainColor,
FString::Printf(TEXT(" Distance: %.0fcm Angle: %.1f deg"),
Dist, Angle));
}
else
{
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor,
TEXT(" Selected: (none)"));
GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, MainColor,
TEXT(" Distance: --- Angle: ---"));
}
GEngine->AddOnScreenDebugMessage(BaseKey + 3, DisplayTime, MainColor,
FString::Printf(TEXT(" AutoStart: %s AutoGaze: %s AutoListen: %s"),
bAutoStartConversation ? TEXT("ON") : TEXT("OFF"),
bAutoManageGaze ? TEXT("ON") : TEXT("OFF"),
bAutoManageListening ? TEXT("ON") : TEXT("OFF")));
GEngine->AddOnScreenDebugMessage(BaseKey + 4, DisplayTime, MainColor,
FString::Printf(TEXT(" Mic: %s"),
(MicComponent != nullptr) ? TEXT("initialized") : TEXT("not initialized")));
}
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
// Replication // Replication
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────

View File

@ -13,12 +13,6 @@
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_LipSync, Log, All); DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_LipSync, Log, All);
static TAutoConsoleVariable<int32> CVarDebugLipSync(
TEXT("ps.ai.ConvAgent.Debug.LipSync"),
-1,
TEXT("Debug HUD for LipSync. -1=use property, 0=off, 1-3=verbosity."),
ECVF_Default);
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
// Static data // Static data
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@ -708,10 +702,9 @@ void UPS_AI_ConvAgent_LipSyncComponent::TickComponent(float DeltaTime, ELevelTic
const float TargetAlpha = bActive ? 1.0f : 0.0f; const float TargetAlpha = bActive ? 1.0f : 0.0f;
if (!FMath::IsNearlyEqual(CurrentActiveAlpha, TargetAlpha, 0.001f)) if (!FMath::IsNearlyEqual(CurrentActiveAlpha, TargetAlpha, 0.001f))
{ {
// Exponential ease-out: fast start, gradual approach to target. const float BlendSpeed = 1.0f / FMath::Max(ActivationBlendDuration, 0.01f);
const float InterpSpeed = 3.0f / FMath::Max(ActivationBlendDuration, 0.01f); CurrentActiveAlpha = FMath::FInterpConstantTo(
CurrentActiveAlpha = FMath::FInterpTo( CurrentActiveAlpha, TargetAlpha, DeltaTime, BlendSpeed);
CurrentActiveAlpha, TargetAlpha, DeltaTime, InterpSpeed);
} }
else else
{ {
@ -719,24 +712,6 @@ void UPS_AI_ConvAgent_LipSyncComponent::TickComponent(float DeltaTime, ELevelTic
} }
} }
// ── Smooth speech blend ────────────────────────────────────────────────
// Fades lip sync curves in/out when speech starts/stops.
// When not speaking, lip sync releases all mouth curves so emotion
// facial expression (from FacialExpressionComponent) controls the mouth.
{
const float TargetSpeech = bIsSpeaking ? 1.0f : 0.0f;
if (!FMath::IsNearlyEqual(SpeechBlendAlpha, TargetSpeech, 0.001f))
{
const float SpeechSpeed = 3.0f / FMath::Max(SpeechBlendDuration, 0.01f);
SpeechBlendAlpha = FMath::FInterpTo(
SpeechBlendAlpha, TargetSpeech, DeltaTime, SpeechSpeed);
}
else
{
SpeechBlendAlpha = TargetSpeech;
}
}
// ── Lazy binding: in packaged builds, BeginPlay may run before the ──────── // ── Lazy binding: in packaged builds, BeginPlay may run before the ────────
// ElevenLabsComponent is fully initialized. Retry discovery until bound. // ElevenLabsComponent is fully initialized. Retry discovery until bound.
if (!AgentComponent.IsValid()) if (!AgentComponent.IsValid())
@ -883,12 +858,6 @@ void UPS_AI_ConvAgent_LipSyncComponent::TickComponent(float DeltaTime, ELevelTic
{ {
bVisemeTimelineActive = false; bVisemeTimelineActive = false;
// Fallback: mark as not speaking when timeline ends
if (bIsSpeaking && AudioEnvelopeValue < 0.01f)
{
bIsSpeaking = false;
}
if (AccumulatedText.Len() > 0 && bTextVisemesApplied && bFullTextReceived) if (AccumulatedText.Len() > 0 && bTextVisemesApplied && bFullTextReceived)
{ {
AccumulatedText.Reset(); AccumulatedText.Reset();
@ -995,14 +964,6 @@ void UPS_AI_ConvAgent_LipSyncComponent::TickComponent(float DeltaTime, ELevelTic
TargetVisemes.FindOrAdd(FName("sil")) = 1.0f; TargetVisemes.FindOrAdd(FName("sil")) = 1.0f;
PlaybackTimer = 0.0f; PlaybackTimer = 0.0f;
// Fallback: if the queue has been dry and envelope is near zero,
// mark as not speaking. OnAgentStopped should have already done this,
// but this handles edge cases where the event doesn't fire.
if (bIsSpeaking && AudioEnvelopeValue < 0.01f)
{
bIsSpeaking = false;
}
if (AccumulatedText.Len() > 0 && bTextVisemesApplied && bFullTextReceived) if (AccumulatedText.Len() > 0 && bTextVisemesApplied && bFullTextReceived)
{ {
AccumulatedText.Reset(); AccumulatedText.Reset();
@ -1091,24 +1052,6 @@ void UPS_AI_ConvAgent_LipSyncComponent::TickComponent(float DeltaTime, ELevelTic
} }
} }
// ── Apply speech blend alpha ─────────────────────────────────────────
// When the agent is NOT speaking, fade out all lip sync curves so the
// FacialExpressionComponent's emotion curves (including mouth expressions)
// can flow through unobstructed. Without this, lip sync zeros on mouth
// curves (via the AnimNode → mh_arkit_mapping_pose) would overwrite the
// emotion mouth curves even during silence.
if (SpeechBlendAlpha < 0.001f)
{
CurrentBlendshapes.Reset();
}
else if (SpeechBlendAlpha < 0.999f)
{
for (auto& Pair : CurrentBlendshapes)
{
Pair.Value *= SpeechBlendAlpha;
}
}
// Auto-apply morph targets if a target mesh is set // Auto-apply morph targets if a target mesh is set
if (TargetMesh) if (TargetMesh)
{ {
@ -1120,17 +1063,6 @@ void UPS_AI_ConvAgent_LipSyncComponent::TickComponent(float DeltaTime, ELevelTic
{ {
OnVisemesReady.Broadcast(); OnVisemesReady.Broadcast();
} }
// ── On-screen debug HUD ───────────────────────────────────────────────
{
const int32 CVarVal = CVarDebugLipSync.GetValueOnGameThread();
const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug;
const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity;
if (bShowDebug && EffectiveVerbosity >= 1)
{
DrawDebugHUD();
}
}
} }
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@ -1158,10 +1090,6 @@ void UPS_AI_ConvAgent_LipSyncComponent::OnAgentStopped()
VisemeTimeline.Reset(); VisemeTimeline.Reset();
VisemeTimelineCursor = 0.0f; VisemeTimelineCursor = 0.0f;
TotalActiveFramesSeen = 0; TotalActiveFramesSeen = 0;
// Speech ended — SpeechBlendAlpha will smoothly fade out lip sync curves,
// allowing FacialExpression emotion curves to take over the mouth.
bIsSpeaking = false;
} }
void UPS_AI_ConvAgent_LipSyncComponent::ResetToNeutral() void UPS_AI_ConvAgent_LipSyncComponent::ResetToNeutral()
@ -1185,11 +1113,6 @@ void UPS_AI_ConvAgent_LipSyncComponent::ResetToNeutral()
VisemeTimelineCursor = 0.0f; VisemeTimelineCursor = 0.0f;
TotalActiveFramesSeen = 0; TotalActiveFramesSeen = 0;
// Speech interrupted — snap speech blend to 0 immediately
// so emotion curves take over the mouth right away.
bIsSpeaking = false;
SpeechBlendAlpha = 0.0f;
// Snap all visemes to silence immediately (no smoothing delay) // Snap all visemes to silence immediately (no smoothing delay)
for (const FName& Name : VisemeNames) for (const FName& Name : VisemeNames)
{ {
@ -1213,10 +1136,6 @@ void UPS_AI_ConvAgent_LipSyncComponent::OnAudioChunkReceived(const TArray<uint8>
{ {
if (!SpectrumAnalyzer) return; if (!SpectrumAnalyzer) return;
// Mark as speaking — audio is arriving from the agent.
// SpeechBlendAlpha will smoothly fade in lip sync curves.
bIsSpeaking = true;
// Convert int16 PCM to float32 [-1, 1] // Convert int16 PCM to float32 [-1, 1]
const int16* Samples = reinterpret_cast<const int16*>(PCMData.GetData()); const int16* Samples = reinterpret_cast<const int16*>(PCMData.GetData());
const int32 NumSamples = PCMData.Num() / sizeof(int16); const int32 NumSamples = PCMData.Num() / sizeof(int16);
@ -2592,78 +2511,6 @@ void UPS_AI_ConvAgent_LipSyncComponent::ApplyMorphTargets()
} }
} }
// ─────────────────────────────────────────────────────────────────────────────
// On-screen debug display
// ─────────────────────────────────────────────────────────────────────────────
void UPS_AI_ConvAgent_LipSyncComponent::DrawDebugHUD() const
{
if (!GEngine) return;
const int32 BaseKey = 2030;
const float DisplayTime = 1.0f;
const FColor MainColor = FColor::Cyan;
const FColor WarnColor = FColor::Yellow;
FString StateStr = bActive ? TEXT("ACTIVE") : TEXT("INACTIVE");
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, MainColor,
FString::Printf(TEXT("=== LIP SYNC: %s ==="), *StateStr));
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime,
bIsSpeaking ? FColor::Green : MainColor,
FString::Printf(TEXT(" Speaking: %s SpeechBlend: %.3f"),
bIsSpeaking ? TEXT("YES") : TEXT("NO"), SpeechBlendAlpha));
GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, MainColor,
FString::Printf(TEXT(" ActivationAlpha: %.3f"), CurrentActiveAlpha));
GEngine->AddOnScreenDebugMessage(BaseKey + 3, DisplayTime, MainColor,
FString::Printf(TEXT(" Amplitude: %.3f Envelope: %.3f"),
CurrentAmplitude, AudioEnvelopeValue));
GEngine->AddOnScreenDebugMessage(BaseKey + 4, DisplayTime, MainColor,
FString::Printf(TEXT(" Queue: %d frames Playback: %.3fs"),
VisemeQueue.Num(), PlaybackTimer));
GEngine->AddOnScreenDebugMessage(BaseKey + 5, DisplayTime,
bVisemeTimelineActive ? WarnColor : MainColor,
FString::Printf(TEXT(" Timeline: %s cursor=%.2fs"),
bVisemeTimelineActive ? TEXT("ACTIVE") : TEXT("OFF"),
VisemeTimelineCursor));
// Top 3 visemes by weight
FString TopVisemes;
{
TArray<TPair<FName, float>> Sorted;
for (const auto& Pair : SmoothedVisemes)
{
if (Pair.Value > 0.01f)
{
Sorted.Add(TPair<FName, float>(Pair.Key, Pair.Value));
}
}
Sorted.Sort([](const TPair<FName, float>& A, const TPair<FName, float>& B)
{
return A.Value > B.Value;
});
for (int32 i = 0; i < FMath::Min(3, Sorted.Num()); ++i)
{
if (i > 0) TopVisemes += TEXT(", ");
TopVisemes += FString::Printf(TEXT("%s=%.2f"),
*Sorted[i].Key.ToString(), Sorted[i].Value);
}
if (TopVisemes.IsEmpty()) TopVisemes = TEXT("---");
}
GEngine->AddOnScreenDebugMessage(BaseKey + 6, DisplayTime, MainColor,
FString::Printf(TEXT(" Top visemes: %s"), *TopVisemes));
GEngine->AddOnScreenDebugMessage(BaseKey + 7, DisplayTime, MainColor,
FString::Printf(TEXT(" Mode: %s PoseMap: %s"),
bUseCurveMode ? TEXT("Curves") : TEXT("MorphTargets"),
PoseMap ? TEXT("YES") : TEXT("NO")));
}
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
// ARKit → MetaHuman curve name conversion // ARKit → MetaHuman curve name conversion
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────

View File

@ -5,23 +5,15 @@
#include "AudioCaptureCore.h" #include "AudioCaptureCore.h"
#include "Async/Async.h" #include "Async/Async.h"
#include "Engine/Engine.h"
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_Mic, Log, All); DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_Mic, Log, All);
static TAutoConsoleVariable<int32> CVarDebugMic(
TEXT("ps.ai.ConvAgent.Debug.Mic"),
-1,
TEXT("Debug HUD for Microphone. -1=use property, 0=off, 1-3=verbosity."),
ECVF_Default);
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
// Constructor // Constructor
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
UPS_AI_ConvAgent_MicrophoneCaptureComponent::UPS_AI_ConvAgent_MicrophoneCaptureComponent() UPS_AI_ConvAgent_MicrophoneCaptureComponent::UPS_AI_ConvAgent_MicrophoneCaptureComponent()
{ {
PrimaryComponentTick.bCanEverTick = true; PrimaryComponentTick.bCanEverTick = false;
PrimaryComponentTick.TickInterval = 1.0f / 15.0f; // 15 Hz — enough for debug HUD.
} }
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@ -33,78 +25,6 @@ void UPS_AI_ConvAgent_MicrophoneCaptureComponent::EndPlay(const EEndPlayReason::
Super::EndPlay(EndPlayReason); Super::EndPlay(EndPlayReason);
} }
// ─────────────────────────────────────────────────────────────────────────────
// Tick
// ─────────────────────────────────────────────────────────────────────────────
void UPS_AI_ConvAgent_MicrophoneCaptureComponent::TickComponent(
float DeltaTime, ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction)
{
Super::TickComponent(DeltaTime, TickType, ThisTickFunction);
// On-screen debug HUD
const int32 CVarVal = CVarDebugMic.GetValueOnGameThread();
const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug;
const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity;
if (bShowDebug && EffectiveVerbosity >= 1)
{
DrawDebugHUD();
}
}
// ─────────────────────────────────────────────────────────────────────────────
// On-screen debug display
// ─────────────────────────────────────────────────────────────────────────────
void UPS_AI_ConvAgent_MicrophoneCaptureComponent::DrawDebugHUD() const
{
if (!GEngine) return;
const int32 BaseKey = 2050;
const float DisplayTime = 1.0f;
const FColor MainColor = FColor::Cyan;
const bool bCapt = bCapturing.load();
const bool bEchoSuppressed = EchoSuppressFlag && EchoSuppressFlag->load(std::memory_order_relaxed);
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime,
bCapt ? FColor::Green : FColor::Red,
FString::Printf(TEXT("=== MIC: %s ==="),
bCapt ? TEXT("CAPTURING") : TEXT("STOPPED")));
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor,
FString::Printf(TEXT(" Device: %s Rate: %d Ch: %d"),
CachedDeviceName.IsEmpty() ? TEXT("(none)") : *CachedDeviceName,
DeviceSampleRate, DeviceChannels));
GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime,
bEchoSuppressed ? FColor::Yellow : MainColor,
FString::Printf(TEXT(" EchoSuppress: %s VolMul: %.2f"),
bEchoSuppressed ? TEXT("YES") : TEXT("NO"), VolumeMultiplier));
// VU meter
const float RMS = CurrentRMS.load(std::memory_order_relaxed);
const float Peak = PeakLevel.load(std::memory_order_relaxed);
const float dB = (RMS > 1e-6f) ? 20.0f * FMath::LogX(10.0f, RMS) : -60.0f;
// Build text bar: 30 chars wide, mapped from -60dB to 0dB.
const int32 BarWidth = 30;
const float NormLevel = FMath::Clamp((dB + 60.0f) / 60.0f, 0.0f, 1.0f);
const int32 FilledChars = FMath::RoundToInt(NormLevel * BarWidth);
FString Bar;
for (int32 i = 0; i < BarWidth; i++)
{
Bar += (i < FilledChars) ? TEXT("|") : TEXT(" ");
}
FColor VUColor = MainColor;
if (dB > -6.0f) VUColor = FColor::Red;
else if (dB > -20.0f) VUColor = FColor::Green;
GEngine->AddOnScreenDebugMessage(BaseKey + 3, DisplayTime, VUColor,
FString::Printf(TEXT(" VU [%s] %.1fdB peak=%.3f"),
*Bar, dB, Peak));
}
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
// Capture control // Capture control
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@ -137,7 +57,6 @@ void UPS_AI_ConvAgent_MicrophoneCaptureComponent::StartCapture()
{ {
DeviceSampleRate = DeviceInfo.PreferredSampleRate; DeviceSampleRate = DeviceInfo.PreferredSampleRate;
DeviceChannels = DeviceInfo.InputChannels; DeviceChannels = DeviceInfo.InputChannels;
CachedDeviceName = DeviceInfo.DeviceName;
if (bDebug) if (bDebug)
{ {
UE_LOG(LogPS_AI_ConvAgent_Mic, Log, TEXT("Capture device: %s | Rate=%d | Channels=%d"), UE_LOG(LogPS_AI_ConvAgent_Mic, Log, TEXT("Capture device: %s | Rate=%d | Channels=%d"),
@ -180,22 +99,6 @@ void UPS_AI_ConvAgent_MicrophoneCaptureComponent::OnAudioGenerate(
// Device sends float32 interleaved samples; cast from the void* API. // Device sends float32 interleaved samples; cast from the void* API.
const float* FloatAudio = static_cast<const float*>(InAudio); const float* FloatAudio = static_cast<const float*>(InAudio);
// Compute RMS from raw input for VU meter (before resample, cheap).
{
float SumSq = 0.0f;
float Peak = 0.0f;
const int32 TotalSamples = NumFrames * InNumChannels;
for (int32 i = 0; i < TotalSamples; i++)
{
const float S = FloatAudio[i];
SumSq += S * S;
const float AbsS = FMath::Abs(S);
if (AbsS > Peak) Peak = AbsS;
}
CurrentRMS.store(FMath::Sqrt(SumSq / FMath::Max(1, TotalSamples)), std::memory_order_relaxed);
PeakLevel.store(Peak, std::memory_order_relaxed);
}
// Resample + downmix to 16000 Hz mono. // Resample + downmix to 16000 Hz mono.
TArray<float> Resampled = ResampleTo16000(FloatAudio, NumFrames, InNumChannels, InSampleRate); TArray<float> Resampled = ResampleTo16000(FloatAudio, NumFrames, InNumChannels, InSampleRate);

View File

@ -16,13 +16,10 @@ class UAnimSequence;
* AnimSequences from the PS_AI_ConvAgent_BodyExpressionComponent and blends * AnimSequences from the PS_AI_ConvAgent_BodyExpressionComponent and blends
* them per-bone onto the upstream pose (idle, locomotion). * them per-bone onto the upstream pose (idle, locomotion).
* *
* The node uses Override mode: it replaces (lerps) the upstream pose toward * Two modes:
* the expression pose. Use ExcludeBones (default: neck_01) to prevent * - bUpperBodyOnly = true (default): only bones at and below BlendRootBone
* conflicts with Gaze/Posture on neck/head bones. * are blended; lower body passes through from the upstream pose.
* * - bUpperBodyOnly = false: the emotion pose is applied to the entire skeleton.
* Region modes:
* - bUpperBodyOnly = true (default): only bones at and below BlendRootBone.
* - bUpperBodyOnly = false: full skeleton.
* *
* Graph layout: * Graph layout:
* [Upstream body anims (idle, locomotion)] [PS AI ConvAgent Body Expression] [Output] * [Upstream body anims (idle, locomotion)] [PS AI ConvAgent Body Expression] [Output]
@ -53,13 +50,6 @@ struct PS_AI_CONVAGENT_API FAnimNode_PS_AI_ConvAgent_BodyExpression : public FAn
ToolTip = "Root bone for upper body blend.\nAll descendants of this bone are blended with the emotion pose.\nDefault: spine_02 (arms, spine, neck, head).")) ToolTip = "Root bone for upper body blend.\nAll descendants of this bone are blended with the emotion pose.\nDefault: spine_02 (arms, spine, neck, head)."))
FName BlendRootBone = FName(TEXT("spine_02")); FName BlendRootBone = FName(TEXT("spine_02"));
/** Bones to EXCLUDE from body expression blending (each bone and its entire subtree).
* Prevents conflicts with Gaze/Posture on neck/head bones.
* Default: neck_01 (excludes neck + head, leaving them to Gaze/Posture). */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Settings",
meta = (ToolTip = "Exclude these bones and their subtrees from blending.\nDefault: neck_01 prevents conflicts with Gaze/Posture on neck/head."))
TArray<FName> ExcludeBones = { FName(TEXT("neck_01")) };
// ── FAnimNode_Base interface ────────────────────────────────────────────── // ── FAnimNode_Base interface ──────────────────────────────────────────────
virtual void Initialize_AnyThread(const FAnimationInitializeContext& Context) override; virtual void Initialize_AnyThread(const FAnimationInitializeContext& Context) override;

View File

@ -86,7 +86,7 @@ public:
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|BodyExpression", UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|BodyExpression",
meta = (ClampMin = "0.1", ClampMax = "3.0", meta = (ClampMin = "0.1", ClampMax = "3.0",
ToolTip = "How long (seconds) to crossfade between animations.\n0.5 = snappy, 1.5 = smooth.")) ToolTip = "How long (seconds) to crossfade between animations.\n0.5 = snappy, 1.5 = smooth."))
float EmotionBlendDuration = 1.0f; float EmotionBlendDuration = 0.5f;
/** Overall blend weight for body expressions. 1.0 = full, 0.5 = subtle. */ /** Overall blend weight for body expressions. 1.0 = full, 0.5 = subtle. */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|BodyExpression", UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|BodyExpression",
@ -232,15 +232,4 @@ private:
/** Cached reference to the agent component on the same Actor. */ /** Cached reference to the agent component on the same Actor. */
TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> AgentComponent; TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> AgentComponent;
// ── Debug event tracking ────────────────────────────────────────────────
/** Last event name (for on-screen debug display). */
FString LastEventName;
/** World time when the last event fired. */
float LastEventWorldTime = 0.0f;
/** Draw on-screen debug info (called from TickComponent when bDebug). */
void DrawDebugHUD() const;
}; };

View File

@ -39,14 +39,6 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAgentStartedSpeaking);
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAgentStoppedSpeaking); DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAgentStoppedSpeaking);
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAgentInterrupted); DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAgentInterrupted);
/**
* Fired when audio playback actually starts AFTER any pre-buffering delay.
* Unlike OnAgentStartedSpeaking (which fires at the first audio chunk arrival),
* this fires when the AudioComponent calls Play(), meaning the audio is now audible.
* Use this when you need animation/behaviour synced with audible speech.
*/
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAudioPlaybackStarted);
/** /**
* Fired when the server sends its first agent_chat_response_part i.e. the moment * Fired when the server sends its first agent_chat_response_part i.e. the moment
* the LLM starts generating, well before audio arrives. * the LLM starts generating, well before audio arrives.
@ -261,13 +253,6 @@ public:
meta = (ToolTip = "Fires when the agent starts speaking (first audio chunk). Use for lip-sync or UI feedback.")) meta = (ToolTip = "Fires when the agent starts speaking (first audio chunk). Use for lip-sync or UI feedback."))
FOnAgentStartedSpeaking OnAgentStartedSpeaking; FOnAgentStartedSpeaking OnAgentStartedSpeaking;
/** Fired when audio playback actually starts — AFTER any pre-buffering delay.
* Unlike OnAgentStartedSpeaking (first chunk arrival), this fires when audio is audible.
* Use this for body/gesture animations that should be synced with audible speech. */
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
meta = (ToolTip = "Fires when audio playback actually starts (after pre-buffering).\nSynced with audible speech. Use for body animations."))
FOnAudioPlaybackStarted OnAudioPlaybackStarted;
/** Fired when the agent finishes playing all audio. Use this to re-open the microphone (in Server VAD mode without interruption) or update UI. */ /** Fired when the agent finishes playing all audio. Use this to re-open the microphone (in Server VAD mode without interruption) or update UI. */
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events", UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
meta = (ToolTip = "Fires when the agent finishes speaking. Use to re-open the mic or update UI.")) meta = (ToolTip = "Fires when the agent finishes speaking. Use to re-open the mic or update UI."))
@ -675,7 +660,4 @@ private:
* Called on the server when bNetIsConversing / NetConversatingPawn change, * Called on the server when bNetIsConversing / NetConversatingPawn change,
* because OnRep_ConversationState never fires on the Authority. */ * because OnRep_ConversationState never fires on the Authority. */
void ApplyConversationGaze(); void ApplyConversationGaze();
/** Draw on-screen debug info (called from TickComponent when bDebug). */
void DrawDebugHUD() const;
}; };

View File

@ -146,9 +146,6 @@ private:
/** Evaluate all FloatCurves from an AnimSequence at a given time. */ /** Evaluate all FloatCurves from an AnimSequence at a given time. */
TMap<FName, float> EvaluateAnimCurves(UAnimSequence* AnimSeq, float Time) const; TMap<FName, float> EvaluateAnimCurves(UAnimSequence* AnimSeq, float Time) const;
/** Draw on-screen debug info (called from TickComponent when bDebug). */
void DrawDebugHUD() const;
// ── Animation playback state ───────────────────────────────────────────── // ── Animation playback state ─────────────────────────────────────────────
/** Currently playing emotion AnimSequence (looping). */ /** Currently playing emotion AnimSequence (looping). */

View File

@ -95,18 +95,17 @@ public:
/** Automatically aim at the target's eye bones (MetaHuman FACIAL_L_Eye / FACIAL_R_Eye). /** Automatically aim at the target's eye bones (MetaHuman FACIAL_L_Eye / FACIAL_R_Eye).
* When enabled, TargetOffset is ignored and the agent looks at the midpoint * When enabled, TargetOffset is ignored and the agent looks at the midpoint
* between the target pawn's eye bones. * between the target pawn's eye bones.
* Fallback chain: eye bones head bone PawnViewLocation FallbackEyeHeight. */ * Fallback chain: eye bones head bone ActorOrigin + (0,0,FallbackEyeHeight). */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze", UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
meta = (ToolTip = "Auto-target the pawn's eye bones for eye contact.\nFallback: eye bones > head bone > PawnViewLocation > FallbackEyeHeight.")) meta = (ToolTip = "Auto-target the pawn's eye bones for eye contact.\nFallback: eye bones > head bone > FallbackEyeHeight."))
bool bAutoTargetEyes = true; bool bAutoTargetEyes = true;
/** Height offset (cm) from the target actor's origin when no eye/head bones are found /** Height offset (cm) from the target actor's origin when no eye/head bones are found.
* AND the target is not a Pawn. For Pawns without skeleton, GetPawnViewLocation() * Used as fallback when bAutoTargetEyes is true but the target has no skeleton
* is used instead (accounts for BaseEyeHeight automatically). * (e.g. first-person pawn, simple actor). 160 eye height for a standing human. */
* Only applies to non-Pawn actors (props, triggers, etc.). */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze", UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
meta = (EditCondition = "bAutoTargetEyes", ClampMin = "0", meta = (EditCondition = "bAutoTargetEyes", ClampMin = "0",
ToolTip = "Height offset (cm) for non-Pawn targets without skeleton.\nPawns use BaseEyeHeight automatically.\nOnly used as last-resort fallback for non-Pawn actors.")) ToolTip = "Height offset (cm) when no eye/head bones exist on the target.\n160 = standing human eye level.\nOnly used as last-resort fallback."))
float FallbackEyeHeight = 160.0f; float FallbackEyeHeight = 160.0f;
/** Offset from the target actor's origin to aim at. /** Offset from the target actor's origin to aim at.
@ -337,9 +336,6 @@ private:
/** Map eye yaw/pitch angles to 8 ARKit eye curves. */ /** Map eye yaw/pitch angles to 8 ARKit eye curves. */
void UpdateEyeCurves(float EyeYaw, float EyePitch); void UpdateEyeCurves(float EyeYaw, float EyePitch);
/** Draw on-screen debug info (called from TickComponent when bDebug). */
void DrawDebugHUD() const;
// ── Smoothed current values (head + eyes, body is actor yaw) ──────────── // ── Smoothed current values (head + eyes, body is actor yaw) ────────────
/** Current blend alpha (0 = fully inactive/passthrough, 1 = fully active). */ /** Current blend alpha (0 = fully inactive/passthrough, 1 = fully active). */

View File

@ -184,13 +184,6 @@ public:
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction") UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
void StartConversationWithSelectedAgent(); void StartConversationWithSelectedAgent();
/** Send a text message to the currently selected agent.
* The agent responds with audio and text, just as if it heard the player speak.
* Handles network relay automatically (works on both server and client).
* Does nothing if no agent is selected or not connected. */
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
void SendTextToSelectedAgent(const FString& Text);
/** Clear the current selection. Automatic selection resumes next tick. */ /** Clear the current selection. Automatic selection resumes next tick. */
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction") UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
void ClearSelection(); void ClearSelection();
@ -260,9 +253,6 @@ private:
/** Clear the agent's GazeComponent target (detach). */ /** Clear the agent's GazeComponent target (detach). */
void DetachGazeTarget(TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> Agent); void DetachGazeTarget(TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> Agent);
/** Draw on-screen debug info (called from TickComponent when bDebug). */
void DrawDebugHUD() const;
// ── Mic routing ────────────────────────────────────────────────────────── // ── Mic routing ──────────────────────────────────────────────────────────
/** Forward captured mic audio to the currently selected agent. */ /** Forward captured mic audio to the currently selected agent. */

View File

@ -89,17 +89,6 @@ public:
ToolTip = "Smoothing speed for viseme transitions.\n35 = smooth and soft, 50 = balanced, 65 = sharp and responsive.")) ToolTip = "Smoothing speed for viseme transitions.\n35 = smooth and soft, 50 = balanced, 65 = sharp and responsive."))
float SmoothingSpeed = 50.0f; float SmoothingSpeed = 50.0f;
// ── Speech Blend ────────────────────────────────────────────────────────
/** How long (seconds) to blend lip sync curves in/out when speech starts/stops.
* When the agent is NOT speaking, lip sync releases all mouth curves so the
* FacialExpressionComponent's emotion curves (including mouth) can show through.
* When the agent starts speaking, lip sync smoothly takes over mouth curves. */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|LipSync",
meta = (ClampMin = "0.05", ClampMax = "1.0",
ToolTip = "Blend duration when speech starts/stops.\nDuring silence, emotion facial curves control the mouth.\nDuring speech, lip sync takes over."))
float SpeechBlendDuration = 0.15f;
// ── Emotion Expression Blend ───────────────────────────────────────────── // ── Emotion Expression Blend ─────────────────────────────────────────────
/** How much facial emotion (from PS_AI_ConvAgent_FacialExpressionComponent) bleeds through /** How much facial emotion (from PS_AI_ConvAgent_FacialExpressionComponent) bleeds through
@ -174,15 +163,6 @@ public:
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|LipSync") UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|LipSync")
TMap<FName, float> GetCurrentBlendshapes() const { return CurrentBlendshapes; } TMap<FName, float> GetCurrentBlendshapes() const { return CurrentBlendshapes; }
/** True when the agent is currently producing speech audio.
* When false, lip sync releases mouth curves to let emotion curves through. */
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|LipSync")
bool IsSpeaking() const { return bIsSpeaking; }
/** Current speech blend alpha (0 = silent/emotion mouth, 1 = lip sync mouth).
* Smooth transition between silence and speech states. */
float GetSpeechBlendAlpha() const { return SpeechBlendAlpha; }
// ── UActorComponent overrides ───────────────────────────────────────────── // ── UActorComponent overrides ─────────────────────────────────────────────
virtual void BeginPlay() override; virtual void BeginPlay() override;
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override; virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
@ -252,9 +232,6 @@ private:
/** Sample the spectrum magnitude across a frequency range. */ /** Sample the spectrum magnitude across a frequency range. */
float GetBandEnergy(float LowFreq, float HighFreq, int32 NumSamples = 8) const; float GetBandEnergy(float LowFreq, float HighFreq, int32 NumSamples = 8) const;
/** Draw on-screen debug info (called from TickComponent when bDebug). */
void DrawDebugHUD() const;
// ── State ───────────────────────────────────────────────────────────────── // ── State ─────────────────────────────────────────────────────────────────
TUniquePtr<Audio::FSpectrumAnalyzer> SpectrumAnalyzer; TUniquePtr<Audio::FSpectrumAnalyzer> SpectrumAnalyzer;
@ -281,14 +258,6 @@ private:
// Current blend alpha (0 = fully inactive/passthrough, 1 = fully active). // Current blend alpha (0 = fully inactive/passthrough, 1 = fully active).
float CurrentActiveAlpha = 1.0f; float CurrentActiveAlpha = 1.0f;
// True when the agent is currently producing speech audio.
// Set true in OnAudioChunkReceived, false in OnAgentStopped/OnAgentInterrupted.
bool bIsSpeaking = false;
// Smooth blend alpha for speech state: 0 = not speaking (emotion controls mouth),
// 1 = speaking (lip sync controls mouth). Interpolated each tick.
float SpeechBlendAlpha = 0.0f;
// MetaHuman mode: Face mesh has no morph targets, use animation curves instead. // MetaHuman mode: Face mesh has no morph targets, use animation curves instead.
// Set automatically in BeginPlay when TargetMesh has 0 morph targets. // Set automatically in BeginPlay when TargetMesh has 0 morph targets.
bool bUseCurveMode = false; bool bUseCurveMode = false;

View File

@ -73,12 +73,9 @@ public:
// ───────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────
// UActorComponent overrides // UActorComponent overrides
// ───────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────
virtual void TickComponent(float DeltaTime, ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction) override;
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override; virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
private: private:
/** Draw on-screen debug info (called from TickComponent when debug is active). */
void DrawDebugHUD() const;
/** Called by the audio capture callback on a background thread. Raw void* per UE 5.3+ API. */ /** Called by the audio capture callback on a background thread. Raw void* per UE 5.3+ API. */
void OnAudioGenerate(const void* InAudio, int32 NumFrames, void OnAudioGenerate(const void* InAudio, int32 NumFrames,
int32 InNumChannels, int32 InSampleRate, double StreamTime, bool bOverflow); int32 InNumChannels, int32 InSampleRate, double StreamTime, bool bOverflow);
@ -94,11 +91,4 @@ private:
// Device sample rate discovered on StartCapture // Device sample rate discovered on StartCapture
int32 DeviceSampleRate = 44100; int32 DeviceSampleRate = 44100;
int32 DeviceChannels = 1; int32 DeviceChannels = 1;
// RMS level for VU meter (written from audio callback, read on game thread).
std::atomic<float> CurrentRMS{0.0f};
std::atomic<float> PeakLevel{0.0f};
// Device name cached on StartCapture for HUD display.
FString CachedDeviceName;
}; };