diff --git a/Unreal/PS_AI_Agent/Content/Demo_VoiceOnly.umap b/Unreal/PS_AI_Agent/Content/Demo_VoiceOnly.umap index 6935a08..754a1cf 100644 Binary files a/Unreal/PS_AI_Agent/Content/Demo_VoiceOnly.umap and b/Unreal/PS_AI_Agent/Content/Demo_VoiceOnly.umap differ diff --git a/Unreal/PS_AI_Agent/Content/MetaHumans/Animations/BodyBP.uasset b/Unreal/PS_AI_Agent/Content/MetaHumans/Animations/BodyBP.uasset index 1d832f2..d39b0d4 100644 Binary files a/Unreal/PS_AI_Agent/Content/MetaHumans/Animations/BodyBP.uasset and b/Unreal/PS_AI_Agent/Content/MetaHumans/Animations/BodyBP.uasset differ diff --git a/Unreal/PS_AI_Agent/Content/MetaHumans/Common/Face/Face_AnimBP.uasset b/Unreal/PS_AI_Agent/Content/MetaHumans/Common/Face/Face_AnimBP.uasset index 869c950..b8b4676 100644 Binary files a/Unreal/PS_AI_Agent/Content/MetaHumans/Common/Face/Face_AnimBP.uasset and b/Unreal/PS_AI_Agent/Content/MetaHumans/Common/Face/Face_AnimBP.uasset differ diff --git a/Unreal/PS_AI_Agent/Content/MetaHumans/Taro/BP_Taro.uasset b/Unreal/PS_AI_Agent/Content/MetaHumans/Taro/BP_Taro.uasset index 897431d..0630cdd 100644 Binary files a/Unreal/PS_AI_Agent/Content/MetaHumans/Taro/BP_Taro.uasset and b/Unreal/PS_AI_Agent/Content/MetaHumans/Taro/BP_Taro.uasset differ diff --git a/Unreal/PS_AI_Agent/Content/NewElevenLabsEmotionPoseMap.uasset b/Unreal/PS_AI_Agent/Content/NewElevenLabsEmotionPoseMap.uasset index 5504676..6200916 100644 Binary files a/Unreal/PS_AI_Agent/Content/NewElevenLabsEmotionPoseMap.uasset and b/Unreal/PS_AI_Agent/Content/NewElevenLabsEmotionPoseMap.uasset differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/AnimNode_ElevenLabsPosture.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/AnimNode_ElevenLabsPosture.cpp index 5bc3a9a..70ee384 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/AnimNode_ElevenLabsPosture.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/AnimNode_ElevenLabsPosture.cpp @@ -8,6 +8,46 @@ DEFINE_LOG_CATEGORY_STATIC(LogElevenLabsPostureAnimNode, Log, All); +// ───────────────────────────────────────────────────────────────────────────── +// ARKit → MetaHuman CTRL eye curve mapping. +// +// MetaHuman's post-process AnimBP reads CTRL curves in its own naming: +// CTRL_expressions_eyeLook{Direction}{L/R} +// +// Key differences from ARKit naming: +// - Eye suffix: "Left"/"Right" → abbreviated "L"/"R" +// - Horizontal: "In"/"Out" (relative to nose) → "Left"/"Right" (absolute) +// · Left eye "In" = toward nose = looking Right → eyeLookRightL +// · Left eye "Out" = away from nose = looking Left → eyeLookLeftL +// · Right eye "In" = toward nose = looking Left → eyeLookLeftR +// · Right eye "Out" = away from nose = looking Right → eyeLookRightR +// ───────────────────────────────────────────────────────────────────────────── +static const TMap& GetARKitToCTRLEyeMap() +{ + static const TMap Map = []() + { + TMap M; + M.Reserve(8); + + // Vertical: just abbreviate Left→L, Right→R + M.Add(FName(TEXT("eyeLookUpLeft")), FName(TEXT("CTRL_expressions_eyeLookUpL"))); + M.Add(FName(TEXT("eyeLookDownLeft")), FName(TEXT("CTRL_expressions_eyeLookDownL"))); + M.Add(FName(TEXT("eyeLookUpRight")), FName(TEXT("CTRL_expressions_eyeLookUpR"))); + M.Add(FName(TEXT("eyeLookDownRight")), FName(TEXT("CTRL_expressions_eyeLookDownR"))); + + // Horizontal: In/Out (relative) → Left/Right (absolute) + // Left eye: In = toward nose = Right direction + M.Add(FName(TEXT("eyeLookInLeft")), FName(TEXT("CTRL_expressions_eyeLookRightL"))); + M.Add(FName(TEXT("eyeLookOutLeft")), FName(TEXT("CTRL_expressions_eyeLookLeftL"))); + // Right eye: In = toward nose = Left direction + M.Add(FName(TEXT("eyeLookInRight")), FName(TEXT("CTRL_expressions_eyeLookLeftR"))); + M.Add(FName(TEXT("eyeLookOutRight")), FName(TEXT("CTRL_expressions_eyeLookRightR"))); + + return M; + }(); + return Map; +} + // ───────────────────────────────────────────────────────────────────────────── // DIAGNOSTIC: Set to 1 to enable axis test mode. // This overrides head rotation with a 20° test value that cycles through @@ -17,6 +57,22 @@ DEFINE_LOG_CATEGORY_STATIC(LogElevenLabsPostureAnimNode, Log, All); // ───────────────────────────────────────────────────────────────────────────── #define ELEVENLABS_AXIS_DIAGNOSTIC 0 +// ───────────────────────────────────────────────────────────────────────────── +// EYE DIAGNOSTIC: Tests which pipeline actually drives the MetaHuman eyes. +// +// 0 = Production behavior (normal) +// 1 = Force CTRL_expressions_eyeLookUpLeft = 1.0 (tests Control Rig path) +// 2 = Force ARKit eyeLookUpLeft = 1.0 (tests mh_arkit path) +// 3 = Force FACIAL_L_Eye bone rotation 25° up (tests direct bone path) +// +// Set to 1, 2, or 3, rebuild, and watch the LEFT eye: +// - If it looks UP → that pipeline works for eye gaze on this MetaHuman. +// - If it doesn't move → that pipeline does NOT drive eye gaze. +// +// Test all three to know exactly which mechanism controls eyes. +// ───────────────────────────────────────────────────────────────────────────── +#define ELEVENLABS_EYE_DIAGNOSTIC 0 + // ───────────────────────────────────────────────────────────────────────────── // FAnimNode_Base interface // ───────────────────────────────────────────────────────────────────────────── @@ -29,7 +85,8 @@ void FAnimNode_ElevenLabsPosture::Initialize_AnyThread(const FAnimationInitializ PostureComponent.Reset(); CachedEyeCurves.Reset(); CachedHeadRotation = FQuat::Identity; - CachedAnimationCompensation = 1.0f; + CachedHeadCompensation = 1.0f; + CachedEyeCompensation = 1.0f; HeadBoneIndex = FCompactPoseBoneIndex(INDEX_NONE); HeadRefPoseRotation = FQuat::Identity; ChainBoneNames.Reset(); @@ -67,12 +124,13 @@ void FAnimNode_ElevenLabsPosture::Initialize_AnyThread(const FAnimationInitializ } UE_LOG(LogElevenLabsPostureAnimNode, Log, - TEXT("ElevenLabs Posture AnimNode: Owner=%s Mesh=%s Head=%s Eyes=%s Chain=%d AnimComp=%.1f"), + TEXT("ElevenLabs Posture AnimNode: Owner=%s Mesh=%s Head=%s Eyes=%s Chain=%d HeadComp=%.1f EyeComp=%.1f"), *Owner->GetName(), *SkelMesh->GetName(), bApplyHeadRotation ? TEXT("ON") : TEXT("OFF"), bApplyEyeCurves ? TEXT("ON") : TEXT("OFF"), ChainBoneNames.Num(), - Comp->GetAnimationCompensation()); + Comp->GetHeadAnimationCompensation(), + Comp->GetEyeAnimationCompensation()); } else { @@ -92,6 +150,10 @@ void FAnimNode_ElevenLabsPosture::CacheBones_AnyThread(const FAnimationCacheBone // Reset all bone indices and ref pose caches HeadBoneIndex = FCompactPoseBoneIndex(INDEX_NONE); HeadRefPoseRotation = FQuat::Identity; + LeftEyeBoneIndex = FCompactPoseBoneIndex(INDEX_NONE); + RightEyeBoneIndex = FCompactPoseBoneIndex(INDEX_NONE); + LeftEyeRefPoseRotation = FQuat::Identity; + RightEyeRefPoseRotation = FQuat::Identity; ChainBoneIndices.Reset(); ChainRefPoseRotations.Reset(); @@ -169,6 +231,44 @@ void FAnimNode_ElevenLabsPosture::CacheBones_AnyThread(const FAnimationCacheBone } } } + + // ── Resolve eye bones for animation compensation ───────────────── + // mh_arkit_mapping_pose is ADDITIVE: it adds ARKit→CTRL eye + // deformations on TOP of the base pose's eye bone rotations. + // If the source animation rotates FACIAL_L_Eye / FACIAL_R_Eye, + // those rotations persist regardless of our curve overrides. + // We compensate them the same way as head bones. + { + static const FName DefaultLeftEyeBone(TEXT("FACIAL_L_Eye")); + static const FName DefaultRightEyeBone(TEXT("FACIAL_R_Eye")); + + auto ResolveEyeBone = [&](const FName& BoneName, + FCompactPoseBoneIndex& OutIndex, FQuat& OutRefRot) + { + const int32 MeshIdx = RefSkeleton.FindBoneIndex(BoneName); + if (MeshIdx != INDEX_NONE) + { + OutIndex = RequiredBones.MakeCompactPoseIndex( + FMeshPoseBoneIndex(MeshIdx)); + OutRefRot = (MeshIdx < RefPose.Num()) + ? RefPose[MeshIdx].GetRotation() + : FQuat::Identity; + + UE_LOG(LogElevenLabsPostureAnimNode, Log, + TEXT("Eye bone '%s' resolved to index %d."), + *BoneName.ToString(), OutIndex.GetInt()); + } + else + { + UE_LOG(LogElevenLabsPostureAnimNode, Log, + TEXT("Eye bone '%s' not found in skeleton (OK for Body AnimBP)."), + *BoneName.ToString()); + } + }; + + ResolveEyeBone(DefaultLeftEyeBone, LeftEyeBoneIndex, LeftEyeRefPoseRotation); + ResolveEyeBone(DefaultRightEyeBone, RightEyeBoneIndex, RightEyeRefPoseRotation); + } } } @@ -186,7 +286,8 @@ void FAnimNode_ElevenLabsPosture::Update_AnyThread(const FAnimationUpdateContext { CachedEyeCurves = PostureComponent->GetCurrentEyeCurves(); CachedHeadRotation = PostureComponent->GetCurrentHeadRotation(); - CachedAnimationCompensation = PostureComponent->GetAnimationCompensation(); + CachedHeadCompensation = PostureComponent->GetHeadAnimationCompensation(); + CachedEyeCompensation = PostureComponent->GetEyeAnimationCompensation(); } } @@ -238,17 +339,219 @@ void FAnimNode_ElevenLabsPosture::Evaluate_AnyThread(FPoseContext& Output) // Evaluate the upstream pose (pass-through) BasePose.Evaluate(Output); - // ── 1. Inject eye gaze curves (8 ARKit eye look curves) ────────────────── + // ── Periodic diagnostic (runs for EVERY instance, before any early return) ─ +#if !UE_BUILD_SHIPPING + if (++EvalDebugFrameCounter % 300 == 1) // ~every 5 seconds at 60fps + { + const TCHAR* NodeRole = bApplyHeadRotation + ? (bApplyEyeCurves ? TEXT("Body+Eyes") : TEXT("Body")) + : (bApplyEyeCurves ? TEXT("Face/Eyes") : TEXT("NONE?!")); + + const bool bHasEyeBones = (LeftEyeBoneIndex.GetInt() != INDEX_NONE); + + UE_LOG(LogElevenLabsPostureAnimNode, Warning, + TEXT("[%s] Posture Evaluate: HeadComp=%.2f EyeComp=%.2f Valid=%s HeadRot=(%s) Eyes=%d Chain=%d EyeBones=%s"), + NodeRole, + CachedHeadCompensation, + CachedEyeCompensation, + PostureComponent.IsValid() ? TEXT("YES") : TEXT("NO"), + *CachedHeadRotation.Rotator().ToCompactString(), + CachedEyeCurves.Num(), + ChainBoneIndices.Num(), + bHasEyeBones ? TEXT("YES") : TEXT("NO")); + + // Log first chain bone's anim vs ref delta (to see if compensation changes anything) + if (bApplyHeadRotation && ChainBoneIndices.Num() > 0 + && ChainBoneIndices[0].GetInt() != INDEX_NONE + && ChainBoneIndices[0].GetInt() < Output.Pose.GetNumBones()) + { + const FQuat AnimRot = Output.Pose[ChainBoneIndices[0]].GetRotation(); + const FQuat RefRot = ChainRefPoseRotations[0]; + const FQuat Delta = AnimRot * RefRot.Inverse(); + const FRotator DeltaRot = Delta.Rotator(); + + UE_LOG(LogElevenLabsPostureAnimNode, Warning, + TEXT(" Chain[0] '%s' AnimDelta from RefPose: Y=%.2f P=%.2f R=%.2f (this gets removed at Comp=1)"), + *ChainBoneNames[0].ToString(), + DeltaRot.Yaw, DeltaRot.Pitch, DeltaRot.Roll); + } + } +#endif + + // ── 1. Inject eye gaze curves ───────────────────────────────────────────── if (bApplyEyeCurves) { - for (const auto& Pair : CachedEyeCurves) +#if ELEVENLABS_EYE_DIAGNOSTIC + // ── EYE PIPELINE DIAGNOSTIC ────────────────────────────────────── + // Forces extreme eye values through ONE pipeline at a time. + // Watch the LEFT eye: if it looks UP, that pipeline drives gaze. { - Output.Curve.Set(Pair.Key, Pair.Value); + static int32 EyeDiagLogCounter = 0; + +#if ELEVENLABS_EYE_DIAGNOSTIC == 1 + // MODE 1: CTRL curves → tests CORRECT MetaHuman CTRL naming + // Real format: CTRL_expressions_eyeLook{Dir}{L/R} (NOT eyeLookUpLeft!) + static const FName ForceCTRL(TEXT("CTRL_expressions_eyeLookUpL")); + Output.Curve.Set(ForceCTRL, 1.0f); + // Zero ARKit to isolate + static const FName ZeroARKit(TEXT("eyeLookUpLeft")); + Output.Curve.Set(ZeroARKit, 0.0f); + // Reset eye bone to ref pose to isolate + if (LeftEyeBoneIndex.GetInt() != INDEX_NONE + && LeftEyeBoneIndex.GetInt() < Output.Pose.GetNumBones()) + { + Output.Pose[LeftEyeBoneIndex].SetRotation(LeftEyeRefPoseRotation); + } + if (++EyeDiagLogCounter % 300 == 1) + { + UE_LOG(LogElevenLabsPostureAnimNode, Warning, + TEXT("[EYE DIAG MODE 1] Forcing CTRL_expressions_eyeLookUpL=1.0 | Left eye should look UP if Control Rig reads CTRL curves")); + } + +#elif ELEVENLABS_EYE_DIAGNOSTIC == 2 + // MODE 2: ARKit curves → tests if mh_arkit_mapping_pose drives eyes + static const FName ForceARKit(TEXT("eyeLookUpLeft")); + Output.Curve.Set(ForceARKit, 1.0f); + // Zero CTRL to isolate + static const FName ZeroCTRL(TEXT("CTRL_expressions_eyeLookUpLeft")); + Output.Curve.Set(ZeroCTRL, 0.0f); + // Reset eye bone to ref pose to isolate + if (LeftEyeBoneIndex.GetInt() != INDEX_NONE + && LeftEyeBoneIndex.GetInt() < Output.Pose.GetNumBones()) + { + Output.Pose[LeftEyeBoneIndex].SetRotation(LeftEyeRefPoseRotation); + } + if (++EyeDiagLogCounter % 300 == 1) + { + UE_LOG(LogElevenLabsPostureAnimNode, Warning, + TEXT("[EYE DIAG MODE 2] Forcing ARKit eyeLookUpLeft=1.0 | Left eye should look UP if mh_arkit_mapping_pose drives eyes")); + } + +#elif ELEVENLABS_EYE_DIAGNOSTIC == 3 + // MODE 3: Direct bone rotation → tests if AnimBP bone output drives eyes + if (LeftEyeBoneIndex.GetInt() != INDEX_NONE + && LeftEyeBoneIndex.GetInt() < Output.Pose.GetNumBones()) + { + // Rotate left eye 25° up (pitch) from ref pose + const FQuat LookUp = FRotator(-25.0f, 0.0f, 0.0f).Quaternion(); + Output.Pose[LeftEyeBoneIndex].SetRotation( + (LookUp * LeftEyeRefPoseRotation).GetNormalized()); + } + // Zero curves to isolate + static const FName ZeroARKit(TEXT("eyeLookUpLeft")); + static const FName ZeroCTRL(TEXT("CTRL_expressions_eyeLookUpLeft")); + Output.Curve.Set(ZeroARKit, 0.0f); + Output.Curve.Set(ZeroCTRL, 0.0f); + if (++EyeDiagLogCounter % 300 == 1) + { + UE_LOG(LogElevenLabsPostureAnimNode, Warning, + TEXT("[EYE DIAG MODE 3] Forcing FACIAL_L_Eye bone -25° pitch | Left eye should look UP if bone rotation drives eyes")); + } +#endif } + +#else // ELEVENLABS_EYE_DIAGNOSTIC == 0 — PRODUCTION + // ── Production eye gaze injection ──────────────────────────────── + // + // Smooth blend between animation and posture eye direction using + // AnimationCompensation (Comp): + // + // Comp=0.0 → 100% animation (pure passthrough, nothing touched) + // Comp=0.5 → 50% animation + 50% posture + // Comp=1.0 → 100% posture (eyes frozen on target) + // + // How it works: + // (a) Eye bone compensation: Slerp FACIAL_L/R_Eye bone rotation + // toward ref pose proportional to Comp. (non-Control-Rig path) + // (b) CTRL curve blend: Read animation's CTRL_expressions_eyeLook* + // via Curve.Get(), Lerp with posture value, Curve.Set() result. + // (c) Zero ARKit eye curves to prevent mh_arkit_mapping_pose from + // overwriting the blended CTRL values. + // + const float Comp = CachedEyeCompensation; + + if (Comp > 0.001f) + { + // (a) Eye bone compensation — remove animation's FACIAL_L/R_Eye + // rotation proportional to Comp (already smooth via Slerp) + if (LeftEyeBoneIndex.GetInt() != INDEX_NONE + && LeftEyeBoneIndex.GetInt() < Output.Pose.GetNumBones()) + { + FTransform& LEyeTransform = Output.Pose[LeftEyeBoneIndex]; + LEyeTransform.SetRotation( + ComputeCompensatedBoneRot( + LEyeTransform.GetRotation(), + LeftEyeRefPoseRotation, + Comp).GetNormalized()); + } + if (RightEyeBoneIndex.GetInt() != INDEX_NONE + && RightEyeBoneIndex.GetInt() < Output.Pose.GetNumBones()) + { + FTransform& REyeTransform = Output.Pose[RightEyeBoneIndex]; + REyeTransform.SetRotation( + ComputeCompensatedBoneRot( + REyeTransform.GetRotation(), + RightEyeRefPoseRotation, + Comp).GetNormalized()); + } + + // (b) Blend CTRL eye curves: read animation's value, lerp with posture + { + const auto& CTRLMap = GetARKitToCTRLEyeMap(); + for (const auto& Pair : CachedEyeCurves) + { + if (const FName* CTRLName = CTRLMap.Find(Pair.Key)) + { + // Read the animation's current CTRL value (0.0 if not set) + const float AnimValue = Output.Curve.Get(*CTRLName); + const float PostureValue = Pair.Value; + // Comp=0 → AnimValue, Comp=1 → PostureValue + const float BlendedValue = FMath::Lerp(AnimValue, PostureValue, Comp); + Output.Curve.Set(*CTRLName, BlendedValue); + } + } + } + + // (c) Zero ARKit eye curves to prevent mh_arkit_mapping_pose + // from overwriting our carefully blended CTRL values. + // mh_arkit converts ARKit→CTRL additively; zeroing means + // it adds nothing for eyes, preserving our blend. + for (const auto& Pair : CachedEyeCurves) + { + Output.Curve.Set(Pair.Key, 0.0f); + } + } + + // Eye diagnostic logging +#if !UE_BUILD_SHIPPING + if (EvalDebugFrameCounter % 300 == 1 && CachedEyeCurves.Num() > 0) + { + UE_LOG(LogElevenLabsPostureAnimNode, Warning, + TEXT(" Eyes: Comp=%.2f → %s (anim weight=%.0f%%, posture weight=%.0f%%)"), + Comp, + Comp > 0.001f ? TEXT("BLEND") : TEXT("PASSTHROUGH"), + (1.0f - Comp) * 100.0f, + Comp * 100.0f); + } +#endif + +#endif // ELEVENLABS_EYE_DIAGNOSTIC } // ── 2. Apply head bone rotation ───────────────────────────────────────── - if (!bApplyHeadRotation || CachedHeadRotation.Equals(FQuat::Identity, 0.001f)) + if (!bApplyHeadRotation) + { + return; + } + + // IMPORTANT: Even when posture is near-zero (head looking straight at target), + // we still need to run compensation to REMOVE the animation's head contribution. + // Only skip if BOTH posture is identity AND compensation is inactive (pure additive). + // Bug fix: the old check `CachedHeadRotation.Equals(Identity)` would early-return + // even at Comp=1.0, letting the animation's head movement play through unchecked. + const bool bHasPosture = !CachedHeadRotation.Equals(FQuat::Identity, 0.001f); + const bool bHasCompensation = CachedHeadCompensation > 0.001f; + if (!bHasPosture && !bHasCompensation) { return; } @@ -316,7 +619,7 @@ void FAnimNode_ElevenLabsPosture::Evaluate_AnyThread(FPoseContext& Output) // Compensate animation: blend out the animation's head contribution const FQuat AnimBoneRot = BoneTransform.GetRotation(); const FQuat CompensatedRot = ComputeCompensatedBoneRot( - AnimBoneRot, ChainRefPoseRotations[i], CachedAnimationCompensation); + AnimBoneRot, ChainRefPoseRotations[i], CachedHeadCompensation); // Fractional posture rotation for this bone. // Enforce shortest-path to prevent sign-flip jumps in Slerp. @@ -346,7 +649,7 @@ void FAnimNode_ElevenLabsPosture::Evaluate_AnyThread(FPoseContext& Output) // Compensate animation const FQuat AnimBoneRot = HeadTransform.GetRotation(); const FQuat CompensatedRot = ComputeCompensatedBoneRot( - AnimBoneRot, HeadRefPoseRotation, CachedAnimationCompensation); + AnimBoneRot, HeadRefPoseRotation, CachedHeadCompensation); // Apply posture on compensated rotation. // Enforce shortest-path to prevent sign-flip jumps. @@ -368,11 +671,12 @@ void FAnimNode_ElevenLabsPosture::GatherDebugData(FNodeDebugData& DebugData) { const FRotator DebugRot = CachedHeadRotation.Rotator(); FString DebugLine = FString::Printf( - TEXT("ElevenLabs Posture (eyes: %d curves, head: Y=%.1f P=%.1f, chain: %d bones, comp: %.1f)"), + TEXT("ElevenLabs Posture (eyes: %d curves, head: Y=%.1f P=%.1f, chain: %d bones, headComp: %.1f, eyeComp: %.1f)"), CachedEyeCurves.Num(), DebugRot.Yaw, DebugRot.Pitch, ChainBoneIndices.Num(), - CachedAnimationCompensation); + CachedHeadCompensation, + CachedEyeCompensation); DebugData.AddDebugItem(DebugLine); BasePose.GatherDebugData(DebugData); } diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsPostureComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsPostureComponent.cpp index e5e1305..7948177 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsPostureComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsPostureComponent.cpp @@ -84,6 +84,16 @@ void UElevenLabsPostureComponent::UpdateEyeCurves(float EyeYaw, float EyePitch) { CurrentEyeCurves.Reset(); + // Always emit ALL 8 eye curves, with inactive directions at 0. + // This is critical for AnimationCompensation: when Comp > 0, the AnimNode + // uses Output.Curve.Set() which overwrites. If we only emit the active + // directions (e.g. eyeLookUpLeft), the opposing curves (eyeLookDownLeft) + // from the emotion/base animation leak through uncleared. + // Emitting all 8 ensures posture fully controls eye direction at Comp=1. + + float LookOutL = 0.0f, LookInL = 0.0f, LookOutR = 0.0f, LookInR = 0.0f; + float LookUpL = 0.0f, LookDownL = 0.0f, LookUpR = 0.0f, LookDownR = 0.0f; + // Horizontal: positive yaw = looking right // Normalized by the fixed ARKit physical range, NOT MaxEyeHorizontal // (which only controls the cascade threshold). @@ -91,30 +101,39 @@ void UElevenLabsPostureComponent::UpdateEyeCurves(float EyeYaw, float EyePitch) { // Looking right: left eye looks outward, right eye looks inward (nasal) const float Value = FMath::Clamp(EyeYaw / ARKitEyeRangeHorizontal, 0.0f, 1.0f); - CurrentEyeCurves.Add(EyeLookOutLeft, Value); - CurrentEyeCurves.Add(EyeLookInRight, Value); + LookOutL = Value; + LookInR = Value; } else if (EyeYaw < 0.0f) { // Looking left: left eye looks inward (nasal), right eye looks outward const float Value = FMath::Clamp(-EyeYaw / ARKitEyeRangeHorizontal, 0.0f, 1.0f); - CurrentEyeCurves.Add(EyeLookInLeft, Value); - CurrentEyeCurves.Add(EyeLookOutRight, Value); + LookInL = Value; + LookOutR = Value; } // Vertical: positive pitch = looking up if (EyePitch > 0.0f) { const float Value = FMath::Clamp(EyePitch / ARKitEyeRangeVertical, 0.0f, 1.0f); - CurrentEyeCurves.Add(EyeLookUpLeft, Value); - CurrentEyeCurves.Add(EyeLookUpRight, Value); + LookUpL = Value; + LookUpR = Value; } else if (EyePitch < 0.0f) { const float Value = FMath::Clamp(-EyePitch / ARKitEyeRangeVertical, 0.0f, 1.0f); - CurrentEyeCurves.Add(EyeLookDownLeft, Value); - CurrentEyeCurves.Add(EyeLookDownRight, Value); + LookDownL = Value; + LookDownR = Value; } + + CurrentEyeCurves.Add(EyeLookOutLeft, LookOutL); + CurrentEyeCurves.Add(EyeLookInLeft, LookInL); + CurrentEyeCurves.Add(EyeLookOutRight, LookOutR); + CurrentEyeCurves.Add(EyeLookInRight, LookInR); + CurrentEyeCurves.Add(EyeLookUpLeft, LookUpL); + CurrentEyeCurves.Add(EyeLookDownLeft, LookDownL); + CurrentEyeCurves.Add(EyeLookUpRight, LookUpR); + CurrentEyeCurves.Add(EyeLookDownRight, LookDownR); } // ───────────────────────────────────────────────────────────────────────────── diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/AnimNode_ElevenLabsPosture.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/AnimNode_ElevenLabsPosture.h index f89cdc2..7abb87b 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/AnimNode_ElevenLabsPosture.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/AnimNode_ElevenLabsPosture.h @@ -93,9 +93,31 @@ private: /** Reference pose rotation for fallback head bone. */ FQuat HeadRefPoseRotation = FQuat::Identity; - // ── Animation compensation ────────────────────────────────────────────── + // ── Eye bone compensation (Face AnimBP) ──────────────────────────────── + // + // mh_arkit_mapping_pose is ADDITIVE: it adds ARKit→CTRL deformations on + // top of the base pose. If the base pose's eye bones (FACIAL_L_Eye, + // FACIAL_R_Eye) already carry animation rotation, that persists even + // when we override the ARKit/CTRL eye curves. Same fix as head: + // blend out the animation's eye bone contribution before the additive. - /** How much posture overrides the animation's head rotation (0=additive, 1=override). + FCompactPoseBoneIndex LeftEyeBoneIndex = FCompactPoseBoneIndex(INDEX_NONE); + FCompactPoseBoneIndex RightEyeBoneIndex = FCompactPoseBoneIndex(INDEX_NONE); + FQuat LeftEyeRefPoseRotation = FQuat::Identity; + FQuat RightEyeRefPoseRotation = FQuat::Identity; + + // ── Animation compensation (separate head and eye) ───────────────────── + + /** How much posture overrides the animation's head/neck rotation (0=additive, 1=override). * Cached from the component during Update. */ - float CachedAnimationCompensation = 1.0f; + float CachedHeadCompensation = 1.0f; + + /** How much posture overrides the animation's eye gaze (0=additive, 1=override). + * Cached from the component during Update. */ + float CachedEyeCompensation = 1.0f; + +#if !UE_BUILD_SHIPPING + /** Frame counter for periodic diagnostic logging in Evaluate. */ + int32 EvalDebugFrameCounter = 0; +#endif }; diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsPostureComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsPostureComponent.h index 2496998..6ed8eb1 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsPostureComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsPostureComponent.h @@ -131,14 +131,27 @@ public: float ReturnToNeutralSpeed = 3.0f; // ── Animation compensation ────────────────────────────────────────────── + // + // Two independent controls to balance animation vs. posture per layer: + // 1.0 = full override — posture replaces animation entirely. + // 0.0 = pure additive — posture stacks on top of animation. - /** How much the look-at overrides the animation's head rotation. - * 1.0 = full override — head always points at target regardless of animation. - * 0.0 = pure additive — posture stacks on top of animation (old behavior). + /** How much posture overrides the animation's head/neck rotation. + * 1.0 = head always points at target regardless of animation. + * 0.0 = posture is additive on top of animation (old behavior). * Default: 1.0 for conversational AI (always look at who you talk to). */ UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ElevenLabs|Posture", meta = (ClampMin = "0", ClampMax = "1")) - float AnimationCompensation = 1.0f; + float HeadAnimationCompensation = 1.0f; + + /** How much posture overrides the animation's eye gaze. + * 1.0 = eyes frozen on posture target, animation's eye movement removed. + * 0.0 = animation's eyes play through, posture is additive. + * Intermediate (e.g. 0.5) = smooth 50/50 blend. + * Default: 1.0 for conversational AI (always look at who you talk to). */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ElevenLabs|Posture", + meta = (ClampMin = "0", ClampMax = "1")) + float EyeAnimationCompensation = 1.0f; // ── Forward offset ────────────────────────────────────────────────────── @@ -192,8 +205,11 @@ public: /** Get the neck bone chain (used by AnimNode to resolve bone indices). */ const TArray& GetNeckBoneChain() const { return NeckBoneChain; } - /** Get animation compensation factor (0 = additive, 1 = full override). */ - float GetAnimationCompensation() const { return AnimationCompensation; } + /** Get head animation compensation factor (0 = additive, 1 = full override). */ + float GetHeadAnimationCompensation() const { return HeadAnimationCompensation; } + + /** Get eye animation compensation factor (0 = additive, 1 = full override). */ + float GetEyeAnimationCompensation() const { return EyeAnimationCompensation; } // ── UActorComponent overrides ──────────────────────────────────────────── virtual void BeginPlay() override;