diff --git a/Unreal/PS_AI_Agent/Content/MetaHumans/Animations/Convai_MetaHuman_FaceAnim.uasset b/Unreal/PS_AI_Agent/Content/MetaHumans/Animations/Convai_MetaHuman_FaceAnim.uasset deleted file mode 100644 index df4d844..0000000 Binary files a/Unreal/PS_AI_Agent/Content/MetaHumans/Animations/Convai_MetaHuman_FaceAnim.uasset and /dev/null differ diff --git a/Unreal/PS_AI_Agent/Content/MetaHumans/Taro/BP_Taro.uasset b/Unreal/PS_AI_Agent/Content/MetaHumans/Taro/BP_Taro.uasset index 3c0ae22..897431d 100644 Binary files a/Unreal/PS_AI_Agent/Content/MetaHumans/Taro/BP_Taro.uasset and b/Unreal/PS_AI_Agent/Content/MetaHumans/Taro/BP_Taro.uasset differ diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/AnimNode_ElevenLabsPosture.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/AnimNode_ElevenLabsPosture.cpp index d550a5a..5bc3a9a 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/AnimNode_ElevenLabsPosture.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/AnimNode_ElevenLabsPosture.cpp @@ -29,10 +29,13 @@ void FAnimNode_ElevenLabsPosture::Initialize_AnyThread(const FAnimationInitializ PostureComponent.Reset(); CachedEyeCurves.Reset(); CachedHeadRotation = FQuat::Identity; + CachedAnimationCompensation = 1.0f; HeadBoneIndex = FCompactPoseBoneIndex(INDEX_NONE); + HeadRefPoseRotation = FQuat::Identity; ChainBoneNames.Reset(); ChainBoneWeights.Reset(); ChainBoneIndices.Reset(); + ChainRefPoseRotations.Reset(); if (const FAnimInstanceProxy* Proxy = Context.AnimInstanceProxy) { @@ -64,11 +67,12 @@ void FAnimNode_ElevenLabsPosture::Initialize_AnyThread(const FAnimationInitializ } UE_LOG(LogElevenLabsPostureAnimNode, Log, - TEXT("ElevenLabs Posture AnimNode: Owner=%s Mesh=%s Head=%s Eyes=%s Chain=%d"), + TEXT("ElevenLabs Posture AnimNode: Owner=%s Mesh=%s Head=%s Eyes=%s Chain=%d AnimComp=%.1f"), *Owner->GetName(), *SkelMesh->GetName(), bApplyHeadRotation ? TEXT("ON") : TEXT("OFF"), bApplyEyeCurves ? TEXT("ON") : TEXT("OFF"), - ChainBoneNames.Num()); + ChainBoneNames.Num(), + Comp->GetAnimationCompensation()); } else { @@ -85,19 +89,23 @@ void FAnimNode_ElevenLabsPosture::CacheBones_AnyThread(const FAnimationCacheBone { BasePose.CacheBones(Context); - // Reset all bone indices + // Reset all bone indices and ref pose caches HeadBoneIndex = FCompactPoseBoneIndex(INDEX_NONE); + HeadRefPoseRotation = FQuat::Identity; ChainBoneIndices.Reset(); + ChainRefPoseRotations.Reset(); if (const FAnimInstanceProxy* Proxy = Context.AnimInstanceProxy) { const FBoneContainer& RequiredBones = Proxy->GetRequiredBones(); const FReferenceSkeleton& RefSkeleton = RequiredBones.GetReferenceSkeleton(); + const TArray& RefPose = RefSkeleton.GetRefBonePose(); // ── Resolve neck bone chain ────────────────────────────────────── if (ChainBoneNames.Num() > 0) { ChainBoneIndices.Reserve(ChainBoneNames.Num()); + ChainRefPoseRotations.Reserve(ChainBoneNames.Num()); for (int32 i = 0; i < ChainBoneNames.Num(); ++i) { const int32 MeshIndex = RefSkeleton.FindBoneIndex(ChainBoneNames[i]); @@ -107,6 +115,12 @@ void FAnimNode_ElevenLabsPosture::CacheBones_AnyThread(const FAnimationCacheBone RequiredBones.MakeCompactPoseIndex(FMeshPoseBoneIndex(MeshIndex)); ChainBoneIndices.Add(CompactIdx); + // Cache reference pose rotation for animation compensation + const FQuat RefRot = (MeshIndex < RefPose.Num()) + ? RefPose[MeshIndex].GetRotation() + : FQuat::Identity; + ChainRefPoseRotations.Add(RefRot); + UE_LOG(LogElevenLabsPostureAnimNode, Log, TEXT(" Chain bone [%d] '%s' → index %d (weight=%.2f)"), i, *ChainBoneNames[i].ToString(), CompactIdx.GetInt(), @@ -114,8 +128,8 @@ void FAnimNode_ElevenLabsPosture::CacheBones_AnyThread(const FAnimationCacheBone } else { - // Bone not found — placeholder to keep arrays parallel ChainBoneIndices.Add(FCompactPoseBoneIndex(INDEX_NONE)); + ChainRefPoseRotations.Add(FQuat::Identity); UE_LOG(LogElevenLabsPostureAnimNode, Warning, TEXT(" Chain bone [%d] '%s' NOT FOUND in skeleton!"), i, *ChainBoneNames[i].ToString()); @@ -132,6 +146,11 @@ void FAnimNode_ElevenLabsPosture::CacheBones_AnyThread(const FAnimationCacheBone HeadBoneIndex = RequiredBones.MakeCompactPoseIndex( FMeshPoseBoneIndex(MeshIndex)); + // Cache reference pose rotation + HeadRefPoseRotation = (MeshIndex < RefPose.Num()) + ? RefPose[MeshIndex].GetRotation() + : FQuat::Identity; + UE_LOG(LogElevenLabsPostureAnimNode, Log, TEXT("Head bone '%s' resolved to index %d."), *HeadBoneName.ToString(), HeadBoneIndex.GetInt()); @@ -158,16 +177,62 @@ void FAnimNode_ElevenLabsPosture::Update_AnyThread(const FAnimationUpdateContext BasePose.Update(Context); // Cache posture data from the component (game thread safe copy). - CachedEyeCurves.Reset(); - CachedHeadRotation = FQuat::Identity; - + // IMPORTANT: Do NOT reset CachedHeadRotation to Identity when the + // component is momentarily invalid (GC pause, re-registration, etc.). + // Resetting would cause a 1-2 frame flash where posture is skipped + // and the head snaps to the raw animation pose then back. + // Instead, keep the last valid cached values as a hold-over. if (PostureComponent.IsValid()) { CachedEyeCurves = PostureComponent->GetCurrentEyeCurves(); CachedHeadRotation = PostureComponent->GetCurrentHeadRotation(); + CachedAnimationCompensation = PostureComponent->GetAnimationCompensation(); } } +// ───────────────────────────────────────────────────────────────────────────── +// Helper: Ensure quaternion is in the same hemisphere as Reference to prevent +// sign flips. q and -q represent the same rotation, but Slerp will take +// different arcs depending on the sign. Without this check, a frame-to-frame +// sign flip can cause a sudden 1-frame rotation jump. +// ───────────────────────────────────────────────────────────────────────────── +static FQuat EnforceShortestPath(const FQuat& Q, const FQuat& Reference) +{ + return ((Q | Reference) < 0.0f) ? -Q : Q; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Helper: Compute compensated bone rotation. +// +// AnimBoneRot = current rotation from animation +// RefPoseRot = static reference pose rotation +// Compensation = 0 → keep animation (additive), 1 → remove animation (override) +// +// Returns a bone rotation where the animation's contribution has been +// blended out proportionally to the compensation factor. +// ───────────────────────────────────────────────────────────────────────────── +static FQuat ComputeCompensatedBoneRot( + const FQuat& AnimBoneRot, + const FQuat& RefPoseRot, + float Compensation) +{ + if (Compensation < 0.001f) + { + return AnimBoneRot; // Pure additive — keep full animation + } + + // Extract what the animation adds beyond the reference pose + const FQuat AnimContrib = AnimBoneRot * RefPoseRot.Inverse(); + + // Enforce shortest-path before Slerp to prevent sign-flip jumps + const FQuat SafeContrib = EnforceShortestPath(AnimContrib, FQuat::Identity); + + // Blend toward identity (removing the animation's contribution) + const FQuat CompensatedContrib = FQuat::Slerp(SafeContrib, FQuat::Identity, Compensation); + + return (CompensatedContrib * RefPoseRot).GetNormalized(); +} + void FAnimNode_ElevenLabsPosture::Evaluate_AnyThread(FPoseContext& Output) { // Evaluate the upstream pose (pass-through) @@ -231,6 +296,11 @@ void FAnimNode_ElevenLabsPosture::Evaluate_AnyThread(FPoseContext& Output) // tilt axis. This prevents parasitic ear-to-shoulder tilt that occurred // when a single CleanRotation (derived from the tip bone) was applied // to bones with different local orientations. + // + // Animation compensation: before applying posture, we blend out the + // animation's contribution proportionally to AnimationCompensation. + // Comp=1.0 → posture replaces animation (head always faces target). + // Comp=0.0 → posture is additive on top of animation (old behavior). for (int32 i = 0; i < ChainBoneIndices.Num(); ++i) { @@ -241,17 +311,24 @@ void FAnimNode_ElevenLabsPosture::Evaluate_AnyThread(FPoseContext& Output) continue; } - // Fractional rotation for this bone - const FQuat FractionalRot = FQuat::Slerp( - FQuat::Identity, CachedHeadRotation, ChainBoneWeights[i]); - - // Compose with THIS bone's own rotation FTransform& BoneTransform = Output.Pose[BoneIdx]; - const FQuat BoneRot = BoneTransform.GetRotation(); - const FQuat Combined = FractionalRot * BoneRot; + + // Compensate animation: blend out the animation's head contribution + const FQuat AnimBoneRot = BoneTransform.GetRotation(); + const FQuat CompensatedRot = ComputeCompensatedBoneRot( + AnimBoneRot, ChainRefPoseRotations[i], CachedAnimationCompensation); + + // Fractional posture rotation for this bone. + // Enforce shortest-path to prevent sign-flip jumps in Slerp. + const FQuat SafeHeadRot = EnforceShortestPath(CachedHeadRotation, FQuat::Identity); + const FQuat FractionalRot = FQuat::Slerp( + FQuat::Identity, SafeHeadRot, ChainBoneWeights[i]); + + // Compose with compensated bone rotation + const FQuat Combined = FractionalRot * CompensatedRot; // Swing-twist on THIS bone's tilt axis (removes roll/tilt) - const FVector BoneTiltAxis = BoneRot.RotateVector(FVector::RightVector); + const FVector BoneTiltAxis = CompensatedRot.RotateVector(FVector::RightVector); FQuat Swing, TiltTwist; Combined.ToSwingTwist(BoneTiltAxis, Swing, TiltTwist); @@ -260,16 +337,24 @@ void FAnimNode_ElevenLabsPosture::Evaluate_AnyThread(FPoseContext& Output) } else { - // ── Fallback: single head bone (original behavior) ────────────── + // ── Fallback: single head bone ────────────────────────────────── if (HeadBoneIndex.GetInt() != INDEX_NONE && HeadBoneIndex.GetInt() < Output.Pose.GetNumBones()) { FTransform& HeadTransform = Output.Pose[HeadBoneIndex]; - const FQuat BoneRot = HeadTransform.GetRotation(); - const FQuat Combined = CachedHeadRotation * BoneRot; + + // Compensate animation + const FQuat AnimBoneRot = HeadTransform.GetRotation(); + const FQuat CompensatedRot = ComputeCompensatedBoneRot( + AnimBoneRot, HeadRefPoseRotation, CachedAnimationCompensation); + + // Apply posture on compensated rotation. + // Enforce shortest-path to prevent sign-flip jumps. + const FQuat SafeHeadRot = EnforceShortestPath(CachedHeadRotation, CompensatedRot); + const FQuat Combined = SafeHeadRot * CompensatedRot; // Remove ear-to-shoulder tilt using the bone's actual tilt axis - const FVector BoneTiltAxis = BoneRot.RotateVector(FVector::RightVector); + const FVector BoneTiltAxis = CompensatedRot.RotateVector(FVector::RightVector); FQuat Swing, TiltTwist; Combined.ToSwingTwist(BoneTiltAxis, Swing, TiltTwist); @@ -283,10 +368,11 @@ void FAnimNode_ElevenLabsPosture::GatherDebugData(FNodeDebugData& DebugData) { const FRotator DebugRot = CachedHeadRotation.Rotator(); FString DebugLine = FString::Printf( - TEXT("ElevenLabs Posture (eyes: %d curves, head: Y=%.1f P=%.1f, chain: %d bones)"), + TEXT("ElevenLabs Posture (eyes: %d curves, head: Y=%.1f P=%.1f, chain: %d bones, comp: %.1f)"), CachedEyeCurves.Num(), DebugRot.Yaw, DebugRot.Pitch, - ChainBoneIndices.Num()); + ChainBoneIndices.Num(), + CachedAnimationCompensation); DebugData.AddDebugItem(DebugLine); BasePose.GatherDebugData(DebugData); } diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsLipSyncComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsLipSyncComponent.cpp index 0330da8..ee4d3ef 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsLipSyncComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsLipSyncComponent.cpp @@ -1,6 +1,7 @@ // Copyright ASTERION. All Rights Reserved. #include "ElevenLabsLipSyncComponent.h" +#include "ElevenLabsFacialExpressionComponent.h" #include "ElevenLabsLipSyncPoseMap.h" #include "ElevenLabsConversationalAgentComponent.h" #include "Components/SkeletalMeshComponent.h" @@ -22,6 +23,23 @@ const TArray UElevenLabsLipSyncComponent::VisemeNames = { FName("aa"), FName("E"), FName("ih"), FName("oh"), FName("ou") }; +// ───────────────────────────────────────────────────────────────────────────── +// Expression curve names (ARKit) — facial expression, NOT lip shape. +// These curves carry emotional expression (smile, frown, sneer, etc.) and are +// blended additively with emotion data during speech. Shape curves (jaw, +// tongue, lip shape) remain 100% lip sync. +// ───────────────────────────────────────────────────────────────────────────── +static const TSet ExpressionCurveNames = { + FName("mouthSmileLeft"), FName("mouthSmileRight"), + FName("mouthFrownLeft"), FName("mouthFrownRight"), + FName("mouthDimpleLeft"), FName("mouthDimpleRight"), + FName("mouthStretchLeft"), FName("mouthStretchRight"), + FName("mouthPressLeft"), FName("mouthPressRight"), + FName("noseSneerLeft"), FName("noseSneerRight"), + FName("cheekPuff"), + FName("cheekSquintLeft"), FName("cheekSquintRight"), +}; + // OVR Viseme → ARKit blendshape mapping. // Each viseme activates a combination of ARKit morph targets with specific weights. // These values are tuned for MetaHuman faces and can be adjusted per project. @@ -251,6 +269,15 @@ void UElevenLabsLipSyncComponent::BeginPlay() *Owner->GetName()); } + // Cache the facial expression component for emotion-aware blending. + CachedFacialExprComp = Owner->FindComponentByClass(); + if (CachedFacialExprComp.IsValid()) + { + UE_LOG(LogElevenLabsLipSync, Log, + TEXT("Facial expression component found — emotion blending enabled (blend=%.2f)."), + EmotionExpressionBlend); + } + // Auto-detect TargetMesh if not set manually. // Priority: 1) component named "Face", 2) any mesh with morph targets, 3) first mesh if (!TargetMesh) @@ -2252,6 +2279,40 @@ void UElevenLabsLipSyncComponent::MapVisemesToBlendshapes() } } + // ── Emotion expression blending ────────────────────────────────────── + // For expression curves (smile, frown, sneer, etc.), additively blend + // the current facial emotion so the character speaks with appropriate + // expression. Shape curves (jaw, tongue, lip geometry) remain pure + // lip sync — only expression curves receive emotion contribution. + if (EmotionExpressionBlend > 0.001f && CachedFacialExprComp.IsValid()) + { + const TMap& EmotionCurves = + CachedFacialExprComp->GetCurrentEmotionCurves(); + + if (EmotionCurves.Num() > 0) + { + for (const FName& ExprARKitName : ExpressionCurveNames) + { + // Convert ARKit name → CTRL_expressions_* to match emotion curve naming + const FName CTRLName = ARKitToMetaHumanCurveName(ExprARKitName); + const float* EmotionValue = EmotionCurves.Find(CTRLName); + if (!EmotionValue || FMath::Abs(*EmotionValue) < 0.001f) + { + continue; + } + + // Add emotion contribution to the appropriate key. + // Non-pose mode: blendshape keys are ARKit names. + // Pose mode with CTRL naming: blendshape keys are CTRL_expressions_*. + const FName& BlendshapeKey = + (bUsePoseMapping && bPosesUseCTRLNaming) ? CTRLName : ExprARKitName; + + float& BS = CurrentBlendshapes.FindOrAdd(BlendshapeKey); + BS += (*EmotionValue) * EmotionExpressionBlend; + } + } + } + // Clamp all values. Use wider range for pose data (CTRL curves can exceed 1.0). const float MaxClamp = bUsePoseMapping ? 2.0f : 1.0f; for (auto& Pair : CurrentBlendshapes) diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsPostureComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsPostureComponent.cpp index 5e84326..e5e1305 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsPostureComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsPostureComponent.cpp @@ -140,6 +140,12 @@ void UElevenLabsPostureComponent::TickComponent( if (!Owner) return; + // Clamp DeltaTime to prevent massive jumps during GC pauses, level + // loading, or frame stutters. 50ms cap ≈ 20 FPS floor; any gap + // larger than that is smoothed out over multiple frames instead of + // snapping in one frame. + const float SafeDeltaTime = FMath::Min(DeltaTime, 0.05f); + if (TargetActor) { // ── 1. Compute target position and eye origin ────────────────────── @@ -175,7 +181,7 @@ void UElevenLabsPostureComponent::TickComponent( Owner->GetActorRotation().Yaw, TargetBodyWorldYaw); if (FMath::Abs(BodyDelta) > 0.1f) { - const float BodyStep = FMath::FInterpTo(0.0f, BodyDelta, DeltaTime, BodyInterpSpeed); + const float BodyStep = FMath::FInterpTo(0.0f, BodyDelta, SafeDeltaTime, BodyInterpSpeed); Owner->AddActorWorldRotation(FRotator(0.0f, BodyStep, 0.0f)); } @@ -204,12 +210,14 @@ void UElevenLabsPostureComponent::TickComponent( const float DeltaFromBodyTarget = FMath::FindDeltaAngleDegrees( BodyTargetFacing, TargetWorldYaw); + bool bBodyOverflowed = false; if (FMath::Abs(DeltaFromBodyTarget) > MaxHeadYaw + MaxEyeHorizontal) { // Body realigns to face target TargetBodyWorldYaw = TargetWorldYaw - MeshForwardYawOffset; // Head returns to ~0° since body will face target directly TargetHeadYaw = 0.0f; + bBodyOverflowed = true; } // ── 6. HEAD: realign when eyes overflow (check against body TARGET) ── @@ -219,19 +227,28 @@ void UElevenLabsPostureComponent::TickComponent( // the head from overcompensating during body interpolation — // otherwise the head turns to track while body catches up, then // snaps back when body arrives (two-step animation artifact). + // + // GUARD: Skip eye overflow check when body just overflowed in this + // same frame. Body overflow already set TargetHeadYaw=0 (head will + // recenter as body turns). Allowing eye overflow to also fire would + // snap TargetHeadYaw to a second value in the same frame, causing + // a visible 1-2 frame jerk as FInterpTo chases two targets. - const float HeadDeltaYaw = FMath::FindDeltaAngleDegrees( - TargetBodyWorldYaw + MeshForwardYawOffset, TargetWorldYaw); - - const float EyeDeltaYaw = HeadDeltaYaw - TargetHeadYaw; - - if (FMath::Abs(EyeDeltaYaw) > MaxEyeHorizontal) + if (!bBodyOverflowed) { - TargetHeadYaw = FMath::Clamp(HeadDeltaYaw, -MaxHeadYaw, MaxHeadYaw); + const float HeadDeltaYaw = FMath::FindDeltaAngleDegrees( + TargetBodyWorldYaw + MeshForwardYawOffset, TargetWorldYaw); + + const float EyeDeltaYaw = HeadDeltaYaw - TargetHeadYaw; + + if (FMath::Abs(EyeDeltaYaw) > MaxEyeHorizontal) + { + TargetHeadYaw = FMath::Clamp(HeadDeltaYaw, -MaxHeadYaw, MaxHeadYaw); + } } // Head smoothly interpolates toward its persistent target - CurrentHeadYaw = FMath::FInterpTo(CurrentHeadYaw, TargetHeadYaw, DeltaTime, HeadInterpSpeed); + CurrentHeadYaw = FMath::FInterpTo(CurrentHeadYaw, TargetHeadYaw, SafeDeltaTime, HeadInterpSpeed); // Eyes = remaining gap (during transients, eyes may sit at MaxEye while // the head catches up — ARKit normalization keeps visual deflection small) @@ -248,7 +265,7 @@ void UElevenLabsPostureComponent::TickComponent( TargetHeadPitch = FMath::Clamp(TargetPitch, -MaxHeadPitch, MaxHeadPitch); } - CurrentHeadPitch = FMath::FInterpTo(CurrentHeadPitch, TargetHeadPitch, DeltaTime, HeadInterpSpeed); + CurrentHeadPitch = FMath::FInterpTo(CurrentHeadPitch, TargetHeadPitch, SafeDeltaTime, HeadInterpSpeed); // Eyes = remaining pitch gap CurrentEyePitch = FMath::Clamp(TargetPitch - CurrentHeadPitch, -MaxEyeVertical, MaxEyeVertical); @@ -263,17 +280,17 @@ void UElevenLabsPostureComponent::TickComponent( Owner->GetActorRotation().Yaw, TargetBodyWorldYaw); if (FMath::Abs(NeutralDelta) > 0.1f) { - const float NeutralStep = FMath::FInterpTo(0.0f, NeutralDelta, DeltaTime, ReturnToNeutralSpeed); + const float NeutralStep = FMath::FInterpTo(0.0f, NeutralDelta, SafeDeltaTime, ReturnToNeutralSpeed); Owner->AddActorWorldRotation(FRotator(0.0f, NeutralStep, 0.0f)); } // Head + Eyes: return to center TargetHeadYaw = 0.0f; TargetHeadPitch = 0.0f; - CurrentHeadYaw = FMath::FInterpTo(CurrentHeadYaw, 0.0f, DeltaTime, ReturnToNeutralSpeed); - CurrentHeadPitch = FMath::FInterpTo(CurrentHeadPitch, 0.0f, DeltaTime, ReturnToNeutralSpeed); - CurrentEyeYaw = FMath::FInterpTo(CurrentEyeYaw, 0.0f, DeltaTime, ReturnToNeutralSpeed); - CurrentEyePitch = FMath::FInterpTo(CurrentEyePitch, 0.0f, DeltaTime, ReturnToNeutralSpeed); + CurrentHeadYaw = FMath::FInterpTo(CurrentHeadYaw, 0.0f, SafeDeltaTime, ReturnToNeutralSpeed); + CurrentHeadPitch = FMath::FInterpTo(CurrentHeadPitch, 0.0f, SafeDeltaTime, ReturnToNeutralSpeed); + CurrentEyeYaw = FMath::FInterpTo(CurrentEyeYaw, 0.0f, SafeDeltaTime, ReturnToNeutralSpeed); + CurrentEyePitch = FMath::FInterpTo(CurrentEyePitch, 0.0f, SafeDeltaTime, ReturnToNeutralSpeed); } // ── 6. Output for AnimNode (thread-safe write) ──────────────────────── @@ -289,7 +306,7 @@ void UElevenLabsPostureComponent::TickComponent( // bone's reference rotation (which also contributes Y-coupling). const FQuat NodQuat(FVector::UpVector, FMath::DegreesToRadians(-CurrentHeadPitch)); const FQuat TurnQuat(FVector::ForwardVector, FMath::DegreesToRadians(CurrentHeadYaw)); - CurrentHeadRotation = TurnQuat * NodQuat; + CurrentHeadRotation = (TurnQuat * NodQuat).GetNormalized(); // Eye yaw is negated to match ARKit curve direction convention. UpdateEyeCurves(-CurrentEyeYaw, CurrentEyePitch); diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/AnimNode_ElevenLabsPosture.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/AnimNode_ElevenLabsPosture.h index 69490b1..f89cdc2 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/AnimNode_ElevenLabsPosture.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/AnimNode_ElevenLabsPosture.h @@ -78,6 +78,10 @@ private: /** Rotation weights, parallel to ChainBoneNames. */ TArray ChainBoneWeights; + /** Reference pose rotations, parallel to ChainBoneIndices. + * Cached from the skeleton at CacheBones for animation compensation. */ + TArray ChainRefPoseRotations; + // ── Fallback single-bone (when chain is empty) ────────────────────────── /** Resolved head bone index in the skeleton (fallback). */ @@ -85,4 +89,13 @@ private: /** Head bone name cached from the component at initialization (fallback). */ FName HeadBoneName; + + /** Reference pose rotation for fallback head bone. */ + FQuat HeadRefPoseRotation = FQuat::Identity; + + // ── Animation compensation ────────────────────────────────────────────── + + /** How much posture overrides the animation's head rotation (0=additive, 1=override). + * Cached from the component during Update. */ + float CachedAnimationCompensation = 1.0f; }; diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsLipSyncComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsLipSyncComponent.h index a324697..5979a23 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsLipSyncComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsLipSyncComponent.h @@ -8,6 +8,7 @@ #include "ElevenLabsLipSyncComponent.generated.h" class UElevenLabsConversationalAgentComponent; +class UElevenLabsFacialExpressionComponent; class UElevenLabsLipSyncPoseMap; class USkeletalMeshComponent; @@ -75,6 +76,19 @@ public: ToolTip = "Smoothing speed for viseme transitions.\n35 = smooth and soft, 50 = balanced, 65 = sharp and responsive.")) float SmoothingSpeed = 50.0f; + // ── Emotion Expression Blend ───────────────────────────────────────────── + + /** How much facial emotion (from ElevenLabsFacialExpressionComponent) bleeds through + * during speech on expression curves (smile, frown, sneer, dimple, etc.). + * Shape curves (jaw, tongue, lip shape) are always 100% lip sync. + * 0.0 = lip sync completely overrides expression curves (old behavior). + * 1.0 = emotion fully additive on expression curves during speech. + * 0.5 = balanced blend (recommended). */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ElevenLabs|LipSync", + meta = (ClampMin = "0.0", ClampMax = "1.0", + ToolTip = "Emotion bleed-through during speech.\n0 = pure lip sync, 0.5 = balanced, 1 = full emotion on expression curves.\nOnly affects expression curves (smile, frown, etc.), not lip shape.")) + float EmotionExpressionBlend = 0.5f; + // ── Audio Envelope ────────────────────────────────────────────────────── /** Envelope attack time in milliseconds. @@ -274,6 +288,9 @@ private: TWeakObjectPtr AgentComponent; FDelegateHandle AudioDataHandle; + // Cached reference to the facial expression component for emotion blending + TWeakObjectPtr CachedFacialExprComp; + // ── Pose-extracted curve data ──────────────────────────────────────────── /** Instance-level mapping extracted from pose AnimSequences at BeginPlay. diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsPostureComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsPostureComponent.h index 757c73e..2496998 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsPostureComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsPostureComponent.h @@ -192,6 +192,9 @@ public: /** Get the neck bone chain (used by AnimNode to resolve bone indices). */ const TArray& GetNeckBoneChain() const { return NeckBoneChain; } + /** Get animation compensation factor (0 = additive, 1 = full override). */ + float GetAnimationCompensation() const { return AnimationCompensation; } + // ── UActorComponent overrides ──────────────────────────────────────────── virtual void BeginPlay() override; virtual void TickComponent(float DeltaTime, ELevelTick TickType,