Posture: animation compensation, emotion lip sync, head jump fixes
Feature 1 — AnimationCompensation: posture overrides head/neck orientation while body animations still play. Extracts animation contribution vs reference pose, Slerps toward identity proportionally to compensation. Feature 2 — Emotion-aware lip sync: expression curves (smile, frown, sneer) blend additively with facial emotion during speech. Shape curves (jaw, tongue) remain 100% lip sync. EmotionExpressionBlend property controls blend ratio (default 0.5). Bug fixes — Head rotation 1-2 frame jump prevention: - Keep last cached rotation when PostureComponent is momentarily invalid - Normalize composed quaternion (TurnQuat * NodQuat) - Clamp DeltaTime to 50ms to prevent FInterpTo spikes during GC - Guard against double cascade overflow (body + eye) in same frame - EnforceShortestPath dot-product check before all Slerp operations Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
d2e904144a
commit
9d281c2e03
Binary file not shown.
Binary file not shown.
@ -29,10 +29,13 @@ void FAnimNode_ElevenLabsPosture::Initialize_AnyThread(const FAnimationInitializ
|
||||
PostureComponent.Reset();
|
||||
CachedEyeCurves.Reset();
|
||||
CachedHeadRotation = FQuat::Identity;
|
||||
CachedAnimationCompensation = 1.0f;
|
||||
HeadBoneIndex = FCompactPoseBoneIndex(INDEX_NONE);
|
||||
HeadRefPoseRotation = FQuat::Identity;
|
||||
ChainBoneNames.Reset();
|
||||
ChainBoneWeights.Reset();
|
||||
ChainBoneIndices.Reset();
|
||||
ChainRefPoseRotations.Reset();
|
||||
|
||||
if (const FAnimInstanceProxy* Proxy = Context.AnimInstanceProxy)
|
||||
{
|
||||
@ -64,11 +67,12 @@ void FAnimNode_ElevenLabsPosture::Initialize_AnyThread(const FAnimationInitializ
|
||||
}
|
||||
|
||||
UE_LOG(LogElevenLabsPostureAnimNode, Log,
|
||||
TEXT("ElevenLabs Posture AnimNode: Owner=%s Mesh=%s Head=%s Eyes=%s Chain=%d"),
|
||||
TEXT("ElevenLabs Posture AnimNode: Owner=%s Mesh=%s Head=%s Eyes=%s Chain=%d AnimComp=%.1f"),
|
||||
*Owner->GetName(), *SkelMesh->GetName(),
|
||||
bApplyHeadRotation ? TEXT("ON") : TEXT("OFF"),
|
||||
bApplyEyeCurves ? TEXT("ON") : TEXT("OFF"),
|
||||
ChainBoneNames.Num());
|
||||
ChainBoneNames.Num(),
|
||||
Comp->GetAnimationCompensation());
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -85,19 +89,23 @@ void FAnimNode_ElevenLabsPosture::CacheBones_AnyThread(const FAnimationCacheBone
|
||||
{
|
||||
BasePose.CacheBones(Context);
|
||||
|
||||
// Reset all bone indices
|
||||
// Reset all bone indices and ref pose caches
|
||||
HeadBoneIndex = FCompactPoseBoneIndex(INDEX_NONE);
|
||||
HeadRefPoseRotation = FQuat::Identity;
|
||||
ChainBoneIndices.Reset();
|
||||
ChainRefPoseRotations.Reset();
|
||||
|
||||
if (const FAnimInstanceProxy* Proxy = Context.AnimInstanceProxy)
|
||||
{
|
||||
const FBoneContainer& RequiredBones = Proxy->GetRequiredBones();
|
||||
const FReferenceSkeleton& RefSkeleton = RequiredBones.GetReferenceSkeleton();
|
||||
const TArray<FTransform>& RefPose = RefSkeleton.GetRefBonePose();
|
||||
|
||||
// ── Resolve neck bone chain ──────────────────────────────────────
|
||||
if (ChainBoneNames.Num() > 0)
|
||||
{
|
||||
ChainBoneIndices.Reserve(ChainBoneNames.Num());
|
||||
ChainRefPoseRotations.Reserve(ChainBoneNames.Num());
|
||||
for (int32 i = 0; i < ChainBoneNames.Num(); ++i)
|
||||
{
|
||||
const int32 MeshIndex = RefSkeleton.FindBoneIndex(ChainBoneNames[i]);
|
||||
@ -107,6 +115,12 @@ void FAnimNode_ElevenLabsPosture::CacheBones_AnyThread(const FAnimationCacheBone
|
||||
RequiredBones.MakeCompactPoseIndex(FMeshPoseBoneIndex(MeshIndex));
|
||||
ChainBoneIndices.Add(CompactIdx);
|
||||
|
||||
// Cache reference pose rotation for animation compensation
|
||||
const FQuat RefRot = (MeshIndex < RefPose.Num())
|
||||
? RefPose[MeshIndex].GetRotation()
|
||||
: FQuat::Identity;
|
||||
ChainRefPoseRotations.Add(RefRot);
|
||||
|
||||
UE_LOG(LogElevenLabsPostureAnimNode, Log,
|
||||
TEXT(" Chain bone [%d] '%s' → index %d (weight=%.2f)"),
|
||||
i, *ChainBoneNames[i].ToString(), CompactIdx.GetInt(),
|
||||
@ -114,8 +128,8 @@ void FAnimNode_ElevenLabsPosture::CacheBones_AnyThread(const FAnimationCacheBone
|
||||
}
|
||||
else
|
||||
{
|
||||
// Bone not found — placeholder to keep arrays parallel
|
||||
ChainBoneIndices.Add(FCompactPoseBoneIndex(INDEX_NONE));
|
||||
ChainRefPoseRotations.Add(FQuat::Identity);
|
||||
UE_LOG(LogElevenLabsPostureAnimNode, Warning,
|
||||
TEXT(" Chain bone [%d] '%s' NOT FOUND in skeleton!"),
|
||||
i, *ChainBoneNames[i].ToString());
|
||||
@ -132,6 +146,11 @@ void FAnimNode_ElevenLabsPosture::CacheBones_AnyThread(const FAnimationCacheBone
|
||||
HeadBoneIndex = RequiredBones.MakeCompactPoseIndex(
|
||||
FMeshPoseBoneIndex(MeshIndex));
|
||||
|
||||
// Cache reference pose rotation
|
||||
HeadRefPoseRotation = (MeshIndex < RefPose.Num())
|
||||
? RefPose[MeshIndex].GetRotation()
|
||||
: FQuat::Identity;
|
||||
|
||||
UE_LOG(LogElevenLabsPostureAnimNode, Log,
|
||||
TEXT("Head bone '%s' resolved to index %d."),
|
||||
*HeadBoneName.ToString(), HeadBoneIndex.GetInt());
|
||||
@ -158,16 +177,62 @@ void FAnimNode_ElevenLabsPosture::Update_AnyThread(const FAnimationUpdateContext
|
||||
BasePose.Update(Context);
|
||||
|
||||
// Cache posture data from the component (game thread safe copy).
|
||||
CachedEyeCurves.Reset();
|
||||
CachedHeadRotation = FQuat::Identity;
|
||||
|
||||
// IMPORTANT: Do NOT reset CachedHeadRotation to Identity when the
|
||||
// component is momentarily invalid (GC pause, re-registration, etc.).
|
||||
// Resetting would cause a 1-2 frame flash where posture is skipped
|
||||
// and the head snaps to the raw animation pose then back.
|
||||
// Instead, keep the last valid cached values as a hold-over.
|
||||
if (PostureComponent.IsValid())
|
||||
{
|
||||
CachedEyeCurves = PostureComponent->GetCurrentEyeCurves();
|
||||
CachedHeadRotation = PostureComponent->GetCurrentHeadRotation();
|
||||
CachedAnimationCompensation = PostureComponent->GetAnimationCompensation();
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Helper: Ensure quaternion is in the same hemisphere as Reference to prevent
|
||||
// sign flips. q and -q represent the same rotation, but Slerp will take
|
||||
// different arcs depending on the sign. Without this check, a frame-to-frame
|
||||
// sign flip can cause a sudden 1-frame rotation jump.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
static FQuat EnforceShortestPath(const FQuat& Q, const FQuat& Reference)
|
||||
{
|
||||
return ((Q | Reference) < 0.0f) ? -Q : Q;
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Helper: Compute compensated bone rotation.
|
||||
//
|
||||
// AnimBoneRot = current rotation from animation
|
||||
// RefPoseRot = static reference pose rotation
|
||||
// Compensation = 0 → keep animation (additive), 1 → remove animation (override)
|
||||
//
|
||||
// Returns a bone rotation where the animation's contribution has been
|
||||
// blended out proportionally to the compensation factor.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
static FQuat ComputeCompensatedBoneRot(
|
||||
const FQuat& AnimBoneRot,
|
||||
const FQuat& RefPoseRot,
|
||||
float Compensation)
|
||||
{
|
||||
if (Compensation < 0.001f)
|
||||
{
|
||||
return AnimBoneRot; // Pure additive — keep full animation
|
||||
}
|
||||
|
||||
// Extract what the animation adds beyond the reference pose
|
||||
const FQuat AnimContrib = AnimBoneRot * RefPoseRot.Inverse();
|
||||
|
||||
// Enforce shortest-path before Slerp to prevent sign-flip jumps
|
||||
const FQuat SafeContrib = EnforceShortestPath(AnimContrib, FQuat::Identity);
|
||||
|
||||
// Blend toward identity (removing the animation's contribution)
|
||||
const FQuat CompensatedContrib = FQuat::Slerp(SafeContrib, FQuat::Identity, Compensation);
|
||||
|
||||
return (CompensatedContrib * RefPoseRot).GetNormalized();
|
||||
}
|
||||
|
||||
void FAnimNode_ElevenLabsPosture::Evaluate_AnyThread(FPoseContext& Output)
|
||||
{
|
||||
// Evaluate the upstream pose (pass-through)
|
||||
@ -231,6 +296,11 @@ void FAnimNode_ElevenLabsPosture::Evaluate_AnyThread(FPoseContext& Output)
|
||||
// tilt axis. This prevents parasitic ear-to-shoulder tilt that occurred
|
||||
// when a single CleanRotation (derived from the tip bone) was applied
|
||||
// to bones with different local orientations.
|
||||
//
|
||||
// Animation compensation: before applying posture, we blend out the
|
||||
// animation's contribution proportionally to AnimationCompensation.
|
||||
// Comp=1.0 → posture replaces animation (head always faces target).
|
||||
// Comp=0.0 → posture is additive on top of animation (old behavior).
|
||||
|
||||
for (int32 i = 0; i < ChainBoneIndices.Num(); ++i)
|
||||
{
|
||||
@ -241,17 +311,24 @@ void FAnimNode_ElevenLabsPosture::Evaluate_AnyThread(FPoseContext& Output)
|
||||
continue;
|
||||
}
|
||||
|
||||
// Fractional rotation for this bone
|
||||
const FQuat FractionalRot = FQuat::Slerp(
|
||||
FQuat::Identity, CachedHeadRotation, ChainBoneWeights[i]);
|
||||
|
||||
// Compose with THIS bone's own rotation
|
||||
FTransform& BoneTransform = Output.Pose[BoneIdx];
|
||||
const FQuat BoneRot = BoneTransform.GetRotation();
|
||||
const FQuat Combined = FractionalRot * BoneRot;
|
||||
|
||||
// Compensate animation: blend out the animation's head contribution
|
||||
const FQuat AnimBoneRot = BoneTransform.GetRotation();
|
||||
const FQuat CompensatedRot = ComputeCompensatedBoneRot(
|
||||
AnimBoneRot, ChainRefPoseRotations[i], CachedAnimationCompensation);
|
||||
|
||||
// Fractional posture rotation for this bone.
|
||||
// Enforce shortest-path to prevent sign-flip jumps in Slerp.
|
||||
const FQuat SafeHeadRot = EnforceShortestPath(CachedHeadRotation, FQuat::Identity);
|
||||
const FQuat FractionalRot = FQuat::Slerp(
|
||||
FQuat::Identity, SafeHeadRot, ChainBoneWeights[i]);
|
||||
|
||||
// Compose with compensated bone rotation
|
||||
const FQuat Combined = FractionalRot * CompensatedRot;
|
||||
|
||||
// Swing-twist on THIS bone's tilt axis (removes roll/tilt)
|
||||
const FVector BoneTiltAxis = BoneRot.RotateVector(FVector::RightVector);
|
||||
const FVector BoneTiltAxis = CompensatedRot.RotateVector(FVector::RightVector);
|
||||
FQuat Swing, TiltTwist;
|
||||
Combined.ToSwingTwist(BoneTiltAxis, Swing, TiltTwist);
|
||||
|
||||
@ -260,16 +337,24 @@ void FAnimNode_ElevenLabsPosture::Evaluate_AnyThread(FPoseContext& Output)
|
||||
}
|
||||
else
|
||||
{
|
||||
// ── Fallback: single head bone (original behavior) ──────────────
|
||||
// ── Fallback: single head bone ──────────────────────────────────
|
||||
if (HeadBoneIndex.GetInt() != INDEX_NONE
|
||||
&& HeadBoneIndex.GetInt() < Output.Pose.GetNumBones())
|
||||
{
|
||||
FTransform& HeadTransform = Output.Pose[HeadBoneIndex];
|
||||
const FQuat BoneRot = HeadTransform.GetRotation();
|
||||
const FQuat Combined = CachedHeadRotation * BoneRot;
|
||||
|
||||
// Compensate animation
|
||||
const FQuat AnimBoneRot = HeadTransform.GetRotation();
|
||||
const FQuat CompensatedRot = ComputeCompensatedBoneRot(
|
||||
AnimBoneRot, HeadRefPoseRotation, CachedAnimationCompensation);
|
||||
|
||||
// Apply posture on compensated rotation.
|
||||
// Enforce shortest-path to prevent sign-flip jumps.
|
||||
const FQuat SafeHeadRot = EnforceShortestPath(CachedHeadRotation, CompensatedRot);
|
||||
const FQuat Combined = SafeHeadRot * CompensatedRot;
|
||||
|
||||
// Remove ear-to-shoulder tilt using the bone's actual tilt axis
|
||||
const FVector BoneTiltAxis = BoneRot.RotateVector(FVector::RightVector);
|
||||
const FVector BoneTiltAxis = CompensatedRot.RotateVector(FVector::RightVector);
|
||||
FQuat Swing, TiltTwist;
|
||||
Combined.ToSwingTwist(BoneTiltAxis, Swing, TiltTwist);
|
||||
|
||||
@ -283,10 +368,11 @@ void FAnimNode_ElevenLabsPosture::GatherDebugData(FNodeDebugData& DebugData)
|
||||
{
|
||||
const FRotator DebugRot = CachedHeadRotation.Rotator();
|
||||
FString DebugLine = FString::Printf(
|
||||
TEXT("ElevenLabs Posture (eyes: %d curves, head: Y=%.1f P=%.1f, chain: %d bones)"),
|
||||
TEXT("ElevenLabs Posture (eyes: %d curves, head: Y=%.1f P=%.1f, chain: %d bones, comp: %.1f)"),
|
||||
CachedEyeCurves.Num(),
|
||||
DebugRot.Yaw, DebugRot.Pitch,
|
||||
ChainBoneIndices.Num());
|
||||
ChainBoneIndices.Num(),
|
||||
CachedAnimationCompensation);
|
||||
DebugData.AddDebugItem(DebugLine);
|
||||
BasePose.GatherDebugData(DebugData);
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
// Copyright ASTERION. All Rights Reserved.
|
||||
|
||||
#include "ElevenLabsLipSyncComponent.h"
|
||||
#include "ElevenLabsFacialExpressionComponent.h"
|
||||
#include "ElevenLabsLipSyncPoseMap.h"
|
||||
#include "ElevenLabsConversationalAgentComponent.h"
|
||||
#include "Components/SkeletalMeshComponent.h"
|
||||
@ -22,6 +23,23 @@ const TArray<FName> UElevenLabsLipSyncComponent::VisemeNames = {
|
||||
FName("aa"), FName("E"), FName("ih"), FName("oh"), FName("ou")
|
||||
};
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Expression curve names (ARKit) — facial expression, NOT lip shape.
|
||||
// These curves carry emotional expression (smile, frown, sneer, etc.) and are
|
||||
// blended additively with emotion data during speech. Shape curves (jaw,
|
||||
// tongue, lip shape) remain 100% lip sync.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
static const TSet<FName> ExpressionCurveNames = {
|
||||
FName("mouthSmileLeft"), FName("mouthSmileRight"),
|
||||
FName("mouthFrownLeft"), FName("mouthFrownRight"),
|
||||
FName("mouthDimpleLeft"), FName("mouthDimpleRight"),
|
||||
FName("mouthStretchLeft"), FName("mouthStretchRight"),
|
||||
FName("mouthPressLeft"), FName("mouthPressRight"),
|
||||
FName("noseSneerLeft"), FName("noseSneerRight"),
|
||||
FName("cheekPuff"),
|
||||
FName("cheekSquintLeft"), FName("cheekSquintRight"),
|
||||
};
|
||||
|
||||
// OVR Viseme → ARKit blendshape mapping.
|
||||
// Each viseme activates a combination of ARKit morph targets with specific weights.
|
||||
// These values are tuned for MetaHuman faces and can be adjusted per project.
|
||||
@ -251,6 +269,15 @@ void UElevenLabsLipSyncComponent::BeginPlay()
|
||||
*Owner->GetName());
|
||||
}
|
||||
|
||||
// Cache the facial expression component for emotion-aware blending.
|
||||
CachedFacialExprComp = Owner->FindComponentByClass<UElevenLabsFacialExpressionComponent>();
|
||||
if (CachedFacialExprComp.IsValid())
|
||||
{
|
||||
UE_LOG(LogElevenLabsLipSync, Log,
|
||||
TEXT("Facial expression component found — emotion blending enabled (blend=%.2f)."),
|
||||
EmotionExpressionBlend);
|
||||
}
|
||||
|
||||
// Auto-detect TargetMesh if not set manually.
|
||||
// Priority: 1) component named "Face", 2) any mesh with morph targets, 3) first mesh
|
||||
if (!TargetMesh)
|
||||
@ -2252,6 +2279,40 @@ void UElevenLabsLipSyncComponent::MapVisemesToBlendshapes()
|
||||
}
|
||||
}
|
||||
|
||||
// ── Emotion expression blending ──────────────────────────────────────
|
||||
// For expression curves (smile, frown, sneer, etc.), additively blend
|
||||
// the current facial emotion so the character speaks with appropriate
|
||||
// expression. Shape curves (jaw, tongue, lip geometry) remain pure
|
||||
// lip sync — only expression curves receive emotion contribution.
|
||||
if (EmotionExpressionBlend > 0.001f && CachedFacialExprComp.IsValid())
|
||||
{
|
||||
const TMap<FName, float>& EmotionCurves =
|
||||
CachedFacialExprComp->GetCurrentEmotionCurves();
|
||||
|
||||
if (EmotionCurves.Num() > 0)
|
||||
{
|
||||
for (const FName& ExprARKitName : ExpressionCurveNames)
|
||||
{
|
||||
// Convert ARKit name → CTRL_expressions_* to match emotion curve naming
|
||||
const FName CTRLName = ARKitToMetaHumanCurveName(ExprARKitName);
|
||||
const float* EmotionValue = EmotionCurves.Find(CTRLName);
|
||||
if (!EmotionValue || FMath::Abs(*EmotionValue) < 0.001f)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// Add emotion contribution to the appropriate key.
|
||||
// Non-pose mode: blendshape keys are ARKit names.
|
||||
// Pose mode with CTRL naming: blendshape keys are CTRL_expressions_*.
|
||||
const FName& BlendshapeKey =
|
||||
(bUsePoseMapping && bPosesUseCTRLNaming) ? CTRLName : ExprARKitName;
|
||||
|
||||
float& BS = CurrentBlendshapes.FindOrAdd(BlendshapeKey);
|
||||
BS += (*EmotionValue) * EmotionExpressionBlend;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Clamp all values. Use wider range for pose data (CTRL curves can exceed 1.0).
|
||||
const float MaxClamp = bUsePoseMapping ? 2.0f : 1.0f;
|
||||
for (auto& Pair : CurrentBlendshapes)
|
||||
|
||||
@ -140,6 +140,12 @@ void UElevenLabsPostureComponent::TickComponent(
|
||||
if (!Owner)
|
||||
return;
|
||||
|
||||
// Clamp DeltaTime to prevent massive jumps during GC pauses, level
|
||||
// loading, or frame stutters. 50ms cap ≈ 20 FPS floor; any gap
|
||||
// larger than that is smoothed out over multiple frames instead of
|
||||
// snapping in one frame.
|
||||
const float SafeDeltaTime = FMath::Min(DeltaTime, 0.05f);
|
||||
|
||||
if (TargetActor)
|
||||
{
|
||||
// ── 1. Compute target position and eye origin ──────────────────────
|
||||
@ -175,7 +181,7 @@ void UElevenLabsPostureComponent::TickComponent(
|
||||
Owner->GetActorRotation().Yaw, TargetBodyWorldYaw);
|
||||
if (FMath::Abs(BodyDelta) > 0.1f)
|
||||
{
|
||||
const float BodyStep = FMath::FInterpTo(0.0f, BodyDelta, DeltaTime, BodyInterpSpeed);
|
||||
const float BodyStep = FMath::FInterpTo(0.0f, BodyDelta, SafeDeltaTime, BodyInterpSpeed);
|
||||
Owner->AddActorWorldRotation(FRotator(0.0f, BodyStep, 0.0f));
|
||||
}
|
||||
|
||||
@ -204,12 +210,14 @@ void UElevenLabsPostureComponent::TickComponent(
|
||||
const float DeltaFromBodyTarget = FMath::FindDeltaAngleDegrees(
|
||||
BodyTargetFacing, TargetWorldYaw);
|
||||
|
||||
bool bBodyOverflowed = false;
|
||||
if (FMath::Abs(DeltaFromBodyTarget) > MaxHeadYaw + MaxEyeHorizontal)
|
||||
{
|
||||
// Body realigns to face target
|
||||
TargetBodyWorldYaw = TargetWorldYaw - MeshForwardYawOffset;
|
||||
// Head returns to ~0° since body will face target directly
|
||||
TargetHeadYaw = 0.0f;
|
||||
bBodyOverflowed = true;
|
||||
}
|
||||
|
||||
// ── 6. HEAD: realign when eyes overflow (check against body TARGET) ──
|
||||
@ -219,19 +227,28 @@ void UElevenLabsPostureComponent::TickComponent(
|
||||
// the head from overcompensating during body interpolation —
|
||||
// otherwise the head turns to track while body catches up, then
|
||||
// snaps back when body arrives (two-step animation artifact).
|
||||
//
|
||||
// GUARD: Skip eye overflow check when body just overflowed in this
|
||||
// same frame. Body overflow already set TargetHeadYaw=0 (head will
|
||||
// recenter as body turns). Allowing eye overflow to also fire would
|
||||
// snap TargetHeadYaw to a second value in the same frame, causing
|
||||
// a visible 1-2 frame jerk as FInterpTo chases two targets.
|
||||
|
||||
const float HeadDeltaYaw = FMath::FindDeltaAngleDegrees(
|
||||
TargetBodyWorldYaw + MeshForwardYawOffset, TargetWorldYaw);
|
||||
|
||||
const float EyeDeltaYaw = HeadDeltaYaw - TargetHeadYaw;
|
||||
|
||||
if (FMath::Abs(EyeDeltaYaw) > MaxEyeHorizontal)
|
||||
if (!bBodyOverflowed)
|
||||
{
|
||||
TargetHeadYaw = FMath::Clamp(HeadDeltaYaw, -MaxHeadYaw, MaxHeadYaw);
|
||||
const float HeadDeltaYaw = FMath::FindDeltaAngleDegrees(
|
||||
TargetBodyWorldYaw + MeshForwardYawOffset, TargetWorldYaw);
|
||||
|
||||
const float EyeDeltaYaw = HeadDeltaYaw - TargetHeadYaw;
|
||||
|
||||
if (FMath::Abs(EyeDeltaYaw) > MaxEyeHorizontal)
|
||||
{
|
||||
TargetHeadYaw = FMath::Clamp(HeadDeltaYaw, -MaxHeadYaw, MaxHeadYaw);
|
||||
}
|
||||
}
|
||||
|
||||
// Head smoothly interpolates toward its persistent target
|
||||
CurrentHeadYaw = FMath::FInterpTo(CurrentHeadYaw, TargetHeadYaw, DeltaTime, HeadInterpSpeed);
|
||||
CurrentHeadYaw = FMath::FInterpTo(CurrentHeadYaw, TargetHeadYaw, SafeDeltaTime, HeadInterpSpeed);
|
||||
|
||||
// Eyes = remaining gap (during transients, eyes may sit at MaxEye while
|
||||
// the head catches up — ARKit normalization keeps visual deflection small)
|
||||
@ -248,7 +265,7 @@ void UElevenLabsPostureComponent::TickComponent(
|
||||
TargetHeadPitch = FMath::Clamp(TargetPitch, -MaxHeadPitch, MaxHeadPitch);
|
||||
}
|
||||
|
||||
CurrentHeadPitch = FMath::FInterpTo(CurrentHeadPitch, TargetHeadPitch, DeltaTime, HeadInterpSpeed);
|
||||
CurrentHeadPitch = FMath::FInterpTo(CurrentHeadPitch, TargetHeadPitch, SafeDeltaTime, HeadInterpSpeed);
|
||||
|
||||
// Eyes = remaining pitch gap
|
||||
CurrentEyePitch = FMath::Clamp(TargetPitch - CurrentHeadPitch, -MaxEyeVertical, MaxEyeVertical);
|
||||
@ -263,17 +280,17 @@ void UElevenLabsPostureComponent::TickComponent(
|
||||
Owner->GetActorRotation().Yaw, TargetBodyWorldYaw);
|
||||
if (FMath::Abs(NeutralDelta) > 0.1f)
|
||||
{
|
||||
const float NeutralStep = FMath::FInterpTo(0.0f, NeutralDelta, DeltaTime, ReturnToNeutralSpeed);
|
||||
const float NeutralStep = FMath::FInterpTo(0.0f, NeutralDelta, SafeDeltaTime, ReturnToNeutralSpeed);
|
||||
Owner->AddActorWorldRotation(FRotator(0.0f, NeutralStep, 0.0f));
|
||||
}
|
||||
|
||||
// Head + Eyes: return to center
|
||||
TargetHeadYaw = 0.0f;
|
||||
TargetHeadPitch = 0.0f;
|
||||
CurrentHeadYaw = FMath::FInterpTo(CurrentHeadYaw, 0.0f, DeltaTime, ReturnToNeutralSpeed);
|
||||
CurrentHeadPitch = FMath::FInterpTo(CurrentHeadPitch, 0.0f, DeltaTime, ReturnToNeutralSpeed);
|
||||
CurrentEyeYaw = FMath::FInterpTo(CurrentEyeYaw, 0.0f, DeltaTime, ReturnToNeutralSpeed);
|
||||
CurrentEyePitch = FMath::FInterpTo(CurrentEyePitch, 0.0f, DeltaTime, ReturnToNeutralSpeed);
|
||||
CurrentHeadYaw = FMath::FInterpTo(CurrentHeadYaw, 0.0f, SafeDeltaTime, ReturnToNeutralSpeed);
|
||||
CurrentHeadPitch = FMath::FInterpTo(CurrentHeadPitch, 0.0f, SafeDeltaTime, ReturnToNeutralSpeed);
|
||||
CurrentEyeYaw = FMath::FInterpTo(CurrentEyeYaw, 0.0f, SafeDeltaTime, ReturnToNeutralSpeed);
|
||||
CurrentEyePitch = FMath::FInterpTo(CurrentEyePitch, 0.0f, SafeDeltaTime, ReturnToNeutralSpeed);
|
||||
}
|
||||
|
||||
// ── 6. Output for AnimNode (thread-safe write) ────────────────────────
|
||||
@ -289,7 +306,7 @@ void UElevenLabsPostureComponent::TickComponent(
|
||||
// bone's reference rotation (which also contributes Y-coupling).
|
||||
const FQuat NodQuat(FVector::UpVector, FMath::DegreesToRadians(-CurrentHeadPitch));
|
||||
const FQuat TurnQuat(FVector::ForwardVector, FMath::DegreesToRadians(CurrentHeadYaw));
|
||||
CurrentHeadRotation = TurnQuat * NodQuat;
|
||||
CurrentHeadRotation = (TurnQuat * NodQuat).GetNormalized();
|
||||
|
||||
// Eye yaw is negated to match ARKit curve direction convention.
|
||||
UpdateEyeCurves(-CurrentEyeYaw, CurrentEyePitch);
|
||||
|
||||
@ -78,6 +78,10 @@ private:
|
||||
/** Rotation weights, parallel to ChainBoneNames. */
|
||||
TArray<float> ChainBoneWeights;
|
||||
|
||||
/** Reference pose rotations, parallel to ChainBoneIndices.
|
||||
* Cached from the skeleton at CacheBones for animation compensation. */
|
||||
TArray<FQuat> ChainRefPoseRotations;
|
||||
|
||||
// ── Fallback single-bone (when chain is empty) ──────────────────────────
|
||||
|
||||
/** Resolved head bone index in the skeleton (fallback). */
|
||||
@ -85,4 +89,13 @@ private:
|
||||
|
||||
/** Head bone name cached from the component at initialization (fallback). */
|
||||
FName HeadBoneName;
|
||||
|
||||
/** Reference pose rotation for fallback head bone. */
|
||||
FQuat HeadRefPoseRotation = FQuat::Identity;
|
||||
|
||||
// ── Animation compensation ──────────────────────────────────────────────
|
||||
|
||||
/** How much posture overrides the animation's head rotation (0=additive, 1=override).
|
||||
* Cached from the component during Update. */
|
||||
float CachedAnimationCompensation = 1.0f;
|
||||
};
|
||||
|
||||
@ -8,6 +8,7 @@
|
||||
#include "ElevenLabsLipSyncComponent.generated.h"
|
||||
|
||||
class UElevenLabsConversationalAgentComponent;
|
||||
class UElevenLabsFacialExpressionComponent;
|
||||
class UElevenLabsLipSyncPoseMap;
|
||||
class USkeletalMeshComponent;
|
||||
|
||||
@ -75,6 +76,19 @@ public:
|
||||
ToolTip = "Smoothing speed for viseme transitions.\n35 = smooth and soft, 50 = balanced, 65 = sharp and responsive."))
|
||||
float SmoothingSpeed = 50.0f;
|
||||
|
||||
// ── Emotion Expression Blend ─────────────────────────────────────────────
|
||||
|
||||
/** How much facial emotion (from ElevenLabsFacialExpressionComponent) bleeds through
|
||||
* during speech on expression curves (smile, frown, sneer, dimple, etc.).
|
||||
* Shape curves (jaw, tongue, lip shape) are always 100% lip sync.
|
||||
* 0.0 = lip sync completely overrides expression curves (old behavior).
|
||||
* 1.0 = emotion fully additive on expression curves during speech.
|
||||
* 0.5 = balanced blend (recommended). */
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ElevenLabs|LipSync",
|
||||
meta = (ClampMin = "0.0", ClampMax = "1.0",
|
||||
ToolTip = "Emotion bleed-through during speech.\n0 = pure lip sync, 0.5 = balanced, 1 = full emotion on expression curves.\nOnly affects expression curves (smile, frown, etc.), not lip shape."))
|
||||
float EmotionExpressionBlend = 0.5f;
|
||||
|
||||
// ── Audio Envelope ──────────────────────────────────────────────────────
|
||||
|
||||
/** Envelope attack time in milliseconds.
|
||||
@ -274,6 +288,9 @@ private:
|
||||
TWeakObjectPtr<UElevenLabsConversationalAgentComponent> AgentComponent;
|
||||
FDelegateHandle AudioDataHandle;
|
||||
|
||||
// Cached reference to the facial expression component for emotion blending
|
||||
TWeakObjectPtr<UElevenLabsFacialExpressionComponent> CachedFacialExprComp;
|
||||
|
||||
// ── Pose-extracted curve data ────────────────────────────────────────────
|
||||
|
||||
/** Instance-level mapping extracted from pose AnimSequences at BeginPlay.
|
||||
|
||||
@ -192,6 +192,9 @@ public:
|
||||
/** Get the neck bone chain (used by AnimNode to resolve bone indices). */
|
||||
const TArray<FElevenLabsNeckBoneEntry>& GetNeckBoneChain() const { return NeckBoneChain; }
|
||||
|
||||
/** Get animation compensation factor (0 = additive, 1 = full override). */
|
||||
float GetAnimationCompensation() const { return AnimationCompensation; }
|
||||
|
||||
// ── UActorComponent overrides ────────────────────────────────────────────
|
||||
virtual void BeginPlay() override;
|
||||
virtual void TickComponent(float DeltaTime, ELevelTick TickType,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user