Posture: animation compensation, emotion lip sync, head jump fixes
Feature 1 — AnimationCompensation: posture overrides head/neck orientation while body animations still play. Extracts animation contribution vs reference pose, Slerps toward identity proportionally to compensation. Feature 2 — Emotion-aware lip sync: expression curves (smile, frown, sneer) blend additively with facial emotion during speech. Shape curves (jaw, tongue) remain 100% lip sync. EmotionExpressionBlend property controls blend ratio (default 0.5). Bug fixes — Head rotation 1-2 frame jump prevention: - Keep last cached rotation when PostureComponent is momentarily invalid - Normalize composed quaternion (TurnQuat * NodQuat) - Clamp DeltaTime to 50ms to prevent FInterpTo spikes during GC - Guard against double cascade overflow (body + eye) in same frame - EnforceShortestPath dot-product check before all Slerp operations Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
d2e904144a
commit
9d281c2e03
Binary file not shown.
Binary file not shown.
@ -29,10 +29,13 @@ void FAnimNode_ElevenLabsPosture::Initialize_AnyThread(const FAnimationInitializ
|
|||||||
PostureComponent.Reset();
|
PostureComponent.Reset();
|
||||||
CachedEyeCurves.Reset();
|
CachedEyeCurves.Reset();
|
||||||
CachedHeadRotation = FQuat::Identity;
|
CachedHeadRotation = FQuat::Identity;
|
||||||
|
CachedAnimationCompensation = 1.0f;
|
||||||
HeadBoneIndex = FCompactPoseBoneIndex(INDEX_NONE);
|
HeadBoneIndex = FCompactPoseBoneIndex(INDEX_NONE);
|
||||||
|
HeadRefPoseRotation = FQuat::Identity;
|
||||||
ChainBoneNames.Reset();
|
ChainBoneNames.Reset();
|
||||||
ChainBoneWeights.Reset();
|
ChainBoneWeights.Reset();
|
||||||
ChainBoneIndices.Reset();
|
ChainBoneIndices.Reset();
|
||||||
|
ChainRefPoseRotations.Reset();
|
||||||
|
|
||||||
if (const FAnimInstanceProxy* Proxy = Context.AnimInstanceProxy)
|
if (const FAnimInstanceProxy* Proxy = Context.AnimInstanceProxy)
|
||||||
{
|
{
|
||||||
@ -64,11 +67,12 @@ void FAnimNode_ElevenLabsPosture::Initialize_AnyThread(const FAnimationInitializ
|
|||||||
}
|
}
|
||||||
|
|
||||||
UE_LOG(LogElevenLabsPostureAnimNode, Log,
|
UE_LOG(LogElevenLabsPostureAnimNode, Log,
|
||||||
TEXT("ElevenLabs Posture AnimNode: Owner=%s Mesh=%s Head=%s Eyes=%s Chain=%d"),
|
TEXT("ElevenLabs Posture AnimNode: Owner=%s Mesh=%s Head=%s Eyes=%s Chain=%d AnimComp=%.1f"),
|
||||||
*Owner->GetName(), *SkelMesh->GetName(),
|
*Owner->GetName(), *SkelMesh->GetName(),
|
||||||
bApplyHeadRotation ? TEXT("ON") : TEXT("OFF"),
|
bApplyHeadRotation ? TEXT("ON") : TEXT("OFF"),
|
||||||
bApplyEyeCurves ? TEXT("ON") : TEXT("OFF"),
|
bApplyEyeCurves ? TEXT("ON") : TEXT("OFF"),
|
||||||
ChainBoneNames.Num());
|
ChainBoneNames.Num(),
|
||||||
|
Comp->GetAnimationCompensation());
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -85,19 +89,23 @@ void FAnimNode_ElevenLabsPosture::CacheBones_AnyThread(const FAnimationCacheBone
|
|||||||
{
|
{
|
||||||
BasePose.CacheBones(Context);
|
BasePose.CacheBones(Context);
|
||||||
|
|
||||||
// Reset all bone indices
|
// Reset all bone indices and ref pose caches
|
||||||
HeadBoneIndex = FCompactPoseBoneIndex(INDEX_NONE);
|
HeadBoneIndex = FCompactPoseBoneIndex(INDEX_NONE);
|
||||||
|
HeadRefPoseRotation = FQuat::Identity;
|
||||||
ChainBoneIndices.Reset();
|
ChainBoneIndices.Reset();
|
||||||
|
ChainRefPoseRotations.Reset();
|
||||||
|
|
||||||
if (const FAnimInstanceProxy* Proxy = Context.AnimInstanceProxy)
|
if (const FAnimInstanceProxy* Proxy = Context.AnimInstanceProxy)
|
||||||
{
|
{
|
||||||
const FBoneContainer& RequiredBones = Proxy->GetRequiredBones();
|
const FBoneContainer& RequiredBones = Proxy->GetRequiredBones();
|
||||||
const FReferenceSkeleton& RefSkeleton = RequiredBones.GetReferenceSkeleton();
|
const FReferenceSkeleton& RefSkeleton = RequiredBones.GetReferenceSkeleton();
|
||||||
|
const TArray<FTransform>& RefPose = RefSkeleton.GetRefBonePose();
|
||||||
|
|
||||||
// ── Resolve neck bone chain ──────────────────────────────────────
|
// ── Resolve neck bone chain ──────────────────────────────────────
|
||||||
if (ChainBoneNames.Num() > 0)
|
if (ChainBoneNames.Num() > 0)
|
||||||
{
|
{
|
||||||
ChainBoneIndices.Reserve(ChainBoneNames.Num());
|
ChainBoneIndices.Reserve(ChainBoneNames.Num());
|
||||||
|
ChainRefPoseRotations.Reserve(ChainBoneNames.Num());
|
||||||
for (int32 i = 0; i < ChainBoneNames.Num(); ++i)
|
for (int32 i = 0; i < ChainBoneNames.Num(); ++i)
|
||||||
{
|
{
|
||||||
const int32 MeshIndex = RefSkeleton.FindBoneIndex(ChainBoneNames[i]);
|
const int32 MeshIndex = RefSkeleton.FindBoneIndex(ChainBoneNames[i]);
|
||||||
@ -107,6 +115,12 @@ void FAnimNode_ElevenLabsPosture::CacheBones_AnyThread(const FAnimationCacheBone
|
|||||||
RequiredBones.MakeCompactPoseIndex(FMeshPoseBoneIndex(MeshIndex));
|
RequiredBones.MakeCompactPoseIndex(FMeshPoseBoneIndex(MeshIndex));
|
||||||
ChainBoneIndices.Add(CompactIdx);
|
ChainBoneIndices.Add(CompactIdx);
|
||||||
|
|
||||||
|
// Cache reference pose rotation for animation compensation
|
||||||
|
const FQuat RefRot = (MeshIndex < RefPose.Num())
|
||||||
|
? RefPose[MeshIndex].GetRotation()
|
||||||
|
: FQuat::Identity;
|
||||||
|
ChainRefPoseRotations.Add(RefRot);
|
||||||
|
|
||||||
UE_LOG(LogElevenLabsPostureAnimNode, Log,
|
UE_LOG(LogElevenLabsPostureAnimNode, Log,
|
||||||
TEXT(" Chain bone [%d] '%s' → index %d (weight=%.2f)"),
|
TEXT(" Chain bone [%d] '%s' → index %d (weight=%.2f)"),
|
||||||
i, *ChainBoneNames[i].ToString(), CompactIdx.GetInt(),
|
i, *ChainBoneNames[i].ToString(), CompactIdx.GetInt(),
|
||||||
@ -114,8 +128,8 @@ void FAnimNode_ElevenLabsPosture::CacheBones_AnyThread(const FAnimationCacheBone
|
|||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
// Bone not found — placeholder to keep arrays parallel
|
|
||||||
ChainBoneIndices.Add(FCompactPoseBoneIndex(INDEX_NONE));
|
ChainBoneIndices.Add(FCompactPoseBoneIndex(INDEX_NONE));
|
||||||
|
ChainRefPoseRotations.Add(FQuat::Identity);
|
||||||
UE_LOG(LogElevenLabsPostureAnimNode, Warning,
|
UE_LOG(LogElevenLabsPostureAnimNode, Warning,
|
||||||
TEXT(" Chain bone [%d] '%s' NOT FOUND in skeleton!"),
|
TEXT(" Chain bone [%d] '%s' NOT FOUND in skeleton!"),
|
||||||
i, *ChainBoneNames[i].ToString());
|
i, *ChainBoneNames[i].ToString());
|
||||||
@ -132,6 +146,11 @@ void FAnimNode_ElevenLabsPosture::CacheBones_AnyThread(const FAnimationCacheBone
|
|||||||
HeadBoneIndex = RequiredBones.MakeCompactPoseIndex(
|
HeadBoneIndex = RequiredBones.MakeCompactPoseIndex(
|
||||||
FMeshPoseBoneIndex(MeshIndex));
|
FMeshPoseBoneIndex(MeshIndex));
|
||||||
|
|
||||||
|
// Cache reference pose rotation
|
||||||
|
HeadRefPoseRotation = (MeshIndex < RefPose.Num())
|
||||||
|
? RefPose[MeshIndex].GetRotation()
|
||||||
|
: FQuat::Identity;
|
||||||
|
|
||||||
UE_LOG(LogElevenLabsPostureAnimNode, Log,
|
UE_LOG(LogElevenLabsPostureAnimNode, Log,
|
||||||
TEXT("Head bone '%s' resolved to index %d."),
|
TEXT("Head bone '%s' resolved to index %d."),
|
||||||
*HeadBoneName.ToString(), HeadBoneIndex.GetInt());
|
*HeadBoneName.ToString(), HeadBoneIndex.GetInt());
|
||||||
@ -158,16 +177,62 @@ void FAnimNode_ElevenLabsPosture::Update_AnyThread(const FAnimationUpdateContext
|
|||||||
BasePose.Update(Context);
|
BasePose.Update(Context);
|
||||||
|
|
||||||
// Cache posture data from the component (game thread safe copy).
|
// Cache posture data from the component (game thread safe copy).
|
||||||
CachedEyeCurves.Reset();
|
// IMPORTANT: Do NOT reset CachedHeadRotation to Identity when the
|
||||||
CachedHeadRotation = FQuat::Identity;
|
// component is momentarily invalid (GC pause, re-registration, etc.).
|
||||||
|
// Resetting would cause a 1-2 frame flash where posture is skipped
|
||||||
|
// and the head snaps to the raw animation pose then back.
|
||||||
|
// Instead, keep the last valid cached values as a hold-over.
|
||||||
if (PostureComponent.IsValid())
|
if (PostureComponent.IsValid())
|
||||||
{
|
{
|
||||||
CachedEyeCurves = PostureComponent->GetCurrentEyeCurves();
|
CachedEyeCurves = PostureComponent->GetCurrentEyeCurves();
|
||||||
CachedHeadRotation = PostureComponent->GetCurrentHeadRotation();
|
CachedHeadRotation = PostureComponent->GetCurrentHeadRotation();
|
||||||
|
CachedAnimationCompensation = PostureComponent->GetAnimationCompensation();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Helper: Ensure quaternion is in the same hemisphere as Reference to prevent
|
||||||
|
// sign flips. q and -q represent the same rotation, but Slerp will take
|
||||||
|
// different arcs depending on the sign. Without this check, a frame-to-frame
|
||||||
|
// sign flip can cause a sudden 1-frame rotation jump.
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
static FQuat EnforceShortestPath(const FQuat& Q, const FQuat& Reference)
|
||||||
|
{
|
||||||
|
return ((Q | Reference) < 0.0f) ? -Q : Q;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Helper: Compute compensated bone rotation.
|
||||||
|
//
|
||||||
|
// AnimBoneRot = current rotation from animation
|
||||||
|
// RefPoseRot = static reference pose rotation
|
||||||
|
// Compensation = 0 → keep animation (additive), 1 → remove animation (override)
|
||||||
|
//
|
||||||
|
// Returns a bone rotation where the animation's contribution has been
|
||||||
|
// blended out proportionally to the compensation factor.
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
static FQuat ComputeCompensatedBoneRot(
|
||||||
|
const FQuat& AnimBoneRot,
|
||||||
|
const FQuat& RefPoseRot,
|
||||||
|
float Compensation)
|
||||||
|
{
|
||||||
|
if (Compensation < 0.001f)
|
||||||
|
{
|
||||||
|
return AnimBoneRot; // Pure additive — keep full animation
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract what the animation adds beyond the reference pose
|
||||||
|
const FQuat AnimContrib = AnimBoneRot * RefPoseRot.Inverse();
|
||||||
|
|
||||||
|
// Enforce shortest-path before Slerp to prevent sign-flip jumps
|
||||||
|
const FQuat SafeContrib = EnforceShortestPath(AnimContrib, FQuat::Identity);
|
||||||
|
|
||||||
|
// Blend toward identity (removing the animation's contribution)
|
||||||
|
const FQuat CompensatedContrib = FQuat::Slerp(SafeContrib, FQuat::Identity, Compensation);
|
||||||
|
|
||||||
|
return (CompensatedContrib * RefPoseRot).GetNormalized();
|
||||||
|
}
|
||||||
|
|
||||||
void FAnimNode_ElevenLabsPosture::Evaluate_AnyThread(FPoseContext& Output)
|
void FAnimNode_ElevenLabsPosture::Evaluate_AnyThread(FPoseContext& Output)
|
||||||
{
|
{
|
||||||
// Evaluate the upstream pose (pass-through)
|
// Evaluate the upstream pose (pass-through)
|
||||||
@ -231,6 +296,11 @@ void FAnimNode_ElevenLabsPosture::Evaluate_AnyThread(FPoseContext& Output)
|
|||||||
// tilt axis. This prevents parasitic ear-to-shoulder tilt that occurred
|
// tilt axis. This prevents parasitic ear-to-shoulder tilt that occurred
|
||||||
// when a single CleanRotation (derived from the tip bone) was applied
|
// when a single CleanRotation (derived from the tip bone) was applied
|
||||||
// to bones with different local orientations.
|
// to bones with different local orientations.
|
||||||
|
//
|
||||||
|
// Animation compensation: before applying posture, we blend out the
|
||||||
|
// animation's contribution proportionally to AnimationCompensation.
|
||||||
|
// Comp=1.0 → posture replaces animation (head always faces target).
|
||||||
|
// Comp=0.0 → posture is additive on top of animation (old behavior).
|
||||||
|
|
||||||
for (int32 i = 0; i < ChainBoneIndices.Num(); ++i)
|
for (int32 i = 0; i < ChainBoneIndices.Num(); ++i)
|
||||||
{
|
{
|
||||||
@ -241,17 +311,24 @@ void FAnimNode_ElevenLabsPosture::Evaluate_AnyThread(FPoseContext& Output)
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fractional rotation for this bone
|
|
||||||
const FQuat FractionalRot = FQuat::Slerp(
|
|
||||||
FQuat::Identity, CachedHeadRotation, ChainBoneWeights[i]);
|
|
||||||
|
|
||||||
// Compose with THIS bone's own rotation
|
|
||||||
FTransform& BoneTransform = Output.Pose[BoneIdx];
|
FTransform& BoneTransform = Output.Pose[BoneIdx];
|
||||||
const FQuat BoneRot = BoneTransform.GetRotation();
|
|
||||||
const FQuat Combined = FractionalRot * BoneRot;
|
// Compensate animation: blend out the animation's head contribution
|
||||||
|
const FQuat AnimBoneRot = BoneTransform.GetRotation();
|
||||||
|
const FQuat CompensatedRot = ComputeCompensatedBoneRot(
|
||||||
|
AnimBoneRot, ChainRefPoseRotations[i], CachedAnimationCompensation);
|
||||||
|
|
||||||
|
// Fractional posture rotation for this bone.
|
||||||
|
// Enforce shortest-path to prevent sign-flip jumps in Slerp.
|
||||||
|
const FQuat SafeHeadRot = EnforceShortestPath(CachedHeadRotation, FQuat::Identity);
|
||||||
|
const FQuat FractionalRot = FQuat::Slerp(
|
||||||
|
FQuat::Identity, SafeHeadRot, ChainBoneWeights[i]);
|
||||||
|
|
||||||
|
// Compose with compensated bone rotation
|
||||||
|
const FQuat Combined = FractionalRot * CompensatedRot;
|
||||||
|
|
||||||
// Swing-twist on THIS bone's tilt axis (removes roll/tilt)
|
// Swing-twist on THIS bone's tilt axis (removes roll/tilt)
|
||||||
const FVector BoneTiltAxis = BoneRot.RotateVector(FVector::RightVector);
|
const FVector BoneTiltAxis = CompensatedRot.RotateVector(FVector::RightVector);
|
||||||
FQuat Swing, TiltTwist;
|
FQuat Swing, TiltTwist;
|
||||||
Combined.ToSwingTwist(BoneTiltAxis, Swing, TiltTwist);
|
Combined.ToSwingTwist(BoneTiltAxis, Swing, TiltTwist);
|
||||||
|
|
||||||
@ -260,16 +337,24 @@ void FAnimNode_ElevenLabsPosture::Evaluate_AnyThread(FPoseContext& Output)
|
|||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
// ── Fallback: single head bone (original behavior) ──────────────
|
// ── Fallback: single head bone ──────────────────────────────────
|
||||||
if (HeadBoneIndex.GetInt() != INDEX_NONE
|
if (HeadBoneIndex.GetInt() != INDEX_NONE
|
||||||
&& HeadBoneIndex.GetInt() < Output.Pose.GetNumBones())
|
&& HeadBoneIndex.GetInt() < Output.Pose.GetNumBones())
|
||||||
{
|
{
|
||||||
FTransform& HeadTransform = Output.Pose[HeadBoneIndex];
|
FTransform& HeadTransform = Output.Pose[HeadBoneIndex];
|
||||||
const FQuat BoneRot = HeadTransform.GetRotation();
|
|
||||||
const FQuat Combined = CachedHeadRotation * BoneRot;
|
// Compensate animation
|
||||||
|
const FQuat AnimBoneRot = HeadTransform.GetRotation();
|
||||||
|
const FQuat CompensatedRot = ComputeCompensatedBoneRot(
|
||||||
|
AnimBoneRot, HeadRefPoseRotation, CachedAnimationCompensation);
|
||||||
|
|
||||||
|
// Apply posture on compensated rotation.
|
||||||
|
// Enforce shortest-path to prevent sign-flip jumps.
|
||||||
|
const FQuat SafeHeadRot = EnforceShortestPath(CachedHeadRotation, CompensatedRot);
|
||||||
|
const FQuat Combined = SafeHeadRot * CompensatedRot;
|
||||||
|
|
||||||
// Remove ear-to-shoulder tilt using the bone's actual tilt axis
|
// Remove ear-to-shoulder tilt using the bone's actual tilt axis
|
||||||
const FVector BoneTiltAxis = BoneRot.RotateVector(FVector::RightVector);
|
const FVector BoneTiltAxis = CompensatedRot.RotateVector(FVector::RightVector);
|
||||||
FQuat Swing, TiltTwist;
|
FQuat Swing, TiltTwist;
|
||||||
Combined.ToSwingTwist(BoneTiltAxis, Swing, TiltTwist);
|
Combined.ToSwingTwist(BoneTiltAxis, Swing, TiltTwist);
|
||||||
|
|
||||||
@ -283,10 +368,11 @@ void FAnimNode_ElevenLabsPosture::GatherDebugData(FNodeDebugData& DebugData)
|
|||||||
{
|
{
|
||||||
const FRotator DebugRot = CachedHeadRotation.Rotator();
|
const FRotator DebugRot = CachedHeadRotation.Rotator();
|
||||||
FString DebugLine = FString::Printf(
|
FString DebugLine = FString::Printf(
|
||||||
TEXT("ElevenLabs Posture (eyes: %d curves, head: Y=%.1f P=%.1f, chain: %d bones)"),
|
TEXT("ElevenLabs Posture (eyes: %d curves, head: Y=%.1f P=%.1f, chain: %d bones, comp: %.1f)"),
|
||||||
CachedEyeCurves.Num(),
|
CachedEyeCurves.Num(),
|
||||||
DebugRot.Yaw, DebugRot.Pitch,
|
DebugRot.Yaw, DebugRot.Pitch,
|
||||||
ChainBoneIndices.Num());
|
ChainBoneIndices.Num(),
|
||||||
|
CachedAnimationCompensation);
|
||||||
DebugData.AddDebugItem(DebugLine);
|
DebugData.AddDebugItem(DebugLine);
|
||||||
BasePose.GatherDebugData(DebugData);
|
BasePose.GatherDebugData(DebugData);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
// Copyright ASTERION. All Rights Reserved.
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
#include "ElevenLabsLipSyncComponent.h"
|
#include "ElevenLabsLipSyncComponent.h"
|
||||||
|
#include "ElevenLabsFacialExpressionComponent.h"
|
||||||
#include "ElevenLabsLipSyncPoseMap.h"
|
#include "ElevenLabsLipSyncPoseMap.h"
|
||||||
#include "ElevenLabsConversationalAgentComponent.h"
|
#include "ElevenLabsConversationalAgentComponent.h"
|
||||||
#include "Components/SkeletalMeshComponent.h"
|
#include "Components/SkeletalMeshComponent.h"
|
||||||
@ -22,6 +23,23 @@ const TArray<FName> UElevenLabsLipSyncComponent::VisemeNames = {
|
|||||||
FName("aa"), FName("E"), FName("ih"), FName("oh"), FName("ou")
|
FName("aa"), FName("E"), FName("ih"), FName("oh"), FName("ou")
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Expression curve names (ARKit) — facial expression, NOT lip shape.
|
||||||
|
// These curves carry emotional expression (smile, frown, sneer, etc.) and are
|
||||||
|
// blended additively with emotion data during speech. Shape curves (jaw,
|
||||||
|
// tongue, lip shape) remain 100% lip sync.
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
static const TSet<FName> ExpressionCurveNames = {
|
||||||
|
FName("mouthSmileLeft"), FName("mouthSmileRight"),
|
||||||
|
FName("mouthFrownLeft"), FName("mouthFrownRight"),
|
||||||
|
FName("mouthDimpleLeft"), FName("mouthDimpleRight"),
|
||||||
|
FName("mouthStretchLeft"), FName("mouthStretchRight"),
|
||||||
|
FName("mouthPressLeft"), FName("mouthPressRight"),
|
||||||
|
FName("noseSneerLeft"), FName("noseSneerRight"),
|
||||||
|
FName("cheekPuff"),
|
||||||
|
FName("cheekSquintLeft"), FName("cheekSquintRight"),
|
||||||
|
};
|
||||||
|
|
||||||
// OVR Viseme → ARKit blendshape mapping.
|
// OVR Viseme → ARKit blendshape mapping.
|
||||||
// Each viseme activates a combination of ARKit morph targets with specific weights.
|
// Each viseme activates a combination of ARKit morph targets with specific weights.
|
||||||
// These values are tuned for MetaHuman faces and can be adjusted per project.
|
// These values are tuned for MetaHuman faces and can be adjusted per project.
|
||||||
@ -251,6 +269,15 @@ void UElevenLabsLipSyncComponent::BeginPlay()
|
|||||||
*Owner->GetName());
|
*Owner->GetName());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Cache the facial expression component for emotion-aware blending.
|
||||||
|
CachedFacialExprComp = Owner->FindComponentByClass<UElevenLabsFacialExpressionComponent>();
|
||||||
|
if (CachedFacialExprComp.IsValid())
|
||||||
|
{
|
||||||
|
UE_LOG(LogElevenLabsLipSync, Log,
|
||||||
|
TEXT("Facial expression component found — emotion blending enabled (blend=%.2f)."),
|
||||||
|
EmotionExpressionBlend);
|
||||||
|
}
|
||||||
|
|
||||||
// Auto-detect TargetMesh if not set manually.
|
// Auto-detect TargetMesh if not set manually.
|
||||||
// Priority: 1) component named "Face", 2) any mesh with morph targets, 3) first mesh
|
// Priority: 1) component named "Face", 2) any mesh with morph targets, 3) first mesh
|
||||||
if (!TargetMesh)
|
if (!TargetMesh)
|
||||||
@ -2252,6 +2279,40 @@ void UElevenLabsLipSyncComponent::MapVisemesToBlendshapes()
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Emotion expression blending ──────────────────────────────────────
|
||||||
|
// For expression curves (smile, frown, sneer, etc.), additively blend
|
||||||
|
// the current facial emotion so the character speaks with appropriate
|
||||||
|
// expression. Shape curves (jaw, tongue, lip geometry) remain pure
|
||||||
|
// lip sync — only expression curves receive emotion contribution.
|
||||||
|
if (EmotionExpressionBlend > 0.001f && CachedFacialExprComp.IsValid())
|
||||||
|
{
|
||||||
|
const TMap<FName, float>& EmotionCurves =
|
||||||
|
CachedFacialExprComp->GetCurrentEmotionCurves();
|
||||||
|
|
||||||
|
if (EmotionCurves.Num() > 0)
|
||||||
|
{
|
||||||
|
for (const FName& ExprARKitName : ExpressionCurveNames)
|
||||||
|
{
|
||||||
|
// Convert ARKit name → CTRL_expressions_* to match emotion curve naming
|
||||||
|
const FName CTRLName = ARKitToMetaHumanCurveName(ExprARKitName);
|
||||||
|
const float* EmotionValue = EmotionCurves.Find(CTRLName);
|
||||||
|
if (!EmotionValue || FMath::Abs(*EmotionValue) < 0.001f)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add emotion contribution to the appropriate key.
|
||||||
|
// Non-pose mode: blendshape keys are ARKit names.
|
||||||
|
// Pose mode with CTRL naming: blendshape keys are CTRL_expressions_*.
|
||||||
|
const FName& BlendshapeKey =
|
||||||
|
(bUsePoseMapping && bPosesUseCTRLNaming) ? CTRLName : ExprARKitName;
|
||||||
|
|
||||||
|
float& BS = CurrentBlendshapes.FindOrAdd(BlendshapeKey);
|
||||||
|
BS += (*EmotionValue) * EmotionExpressionBlend;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Clamp all values. Use wider range for pose data (CTRL curves can exceed 1.0).
|
// Clamp all values. Use wider range for pose data (CTRL curves can exceed 1.0).
|
||||||
const float MaxClamp = bUsePoseMapping ? 2.0f : 1.0f;
|
const float MaxClamp = bUsePoseMapping ? 2.0f : 1.0f;
|
||||||
for (auto& Pair : CurrentBlendshapes)
|
for (auto& Pair : CurrentBlendshapes)
|
||||||
|
|||||||
@ -140,6 +140,12 @@ void UElevenLabsPostureComponent::TickComponent(
|
|||||||
if (!Owner)
|
if (!Owner)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
// Clamp DeltaTime to prevent massive jumps during GC pauses, level
|
||||||
|
// loading, or frame stutters. 50ms cap ≈ 20 FPS floor; any gap
|
||||||
|
// larger than that is smoothed out over multiple frames instead of
|
||||||
|
// snapping in one frame.
|
||||||
|
const float SafeDeltaTime = FMath::Min(DeltaTime, 0.05f);
|
||||||
|
|
||||||
if (TargetActor)
|
if (TargetActor)
|
||||||
{
|
{
|
||||||
// ── 1. Compute target position and eye origin ──────────────────────
|
// ── 1. Compute target position and eye origin ──────────────────────
|
||||||
@ -175,7 +181,7 @@ void UElevenLabsPostureComponent::TickComponent(
|
|||||||
Owner->GetActorRotation().Yaw, TargetBodyWorldYaw);
|
Owner->GetActorRotation().Yaw, TargetBodyWorldYaw);
|
||||||
if (FMath::Abs(BodyDelta) > 0.1f)
|
if (FMath::Abs(BodyDelta) > 0.1f)
|
||||||
{
|
{
|
||||||
const float BodyStep = FMath::FInterpTo(0.0f, BodyDelta, DeltaTime, BodyInterpSpeed);
|
const float BodyStep = FMath::FInterpTo(0.0f, BodyDelta, SafeDeltaTime, BodyInterpSpeed);
|
||||||
Owner->AddActorWorldRotation(FRotator(0.0f, BodyStep, 0.0f));
|
Owner->AddActorWorldRotation(FRotator(0.0f, BodyStep, 0.0f));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -204,12 +210,14 @@ void UElevenLabsPostureComponent::TickComponent(
|
|||||||
const float DeltaFromBodyTarget = FMath::FindDeltaAngleDegrees(
|
const float DeltaFromBodyTarget = FMath::FindDeltaAngleDegrees(
|
||||||
BodyTargetFacing, TargetWorldYaw);
|
BodyTargetFacing, TargetWorldYaw);
|
||||||
|
|
||||||
|
bool bBodyOverflowed = false;
|
||||||
if (FMath::Abs(DeltaFromBodyTarget) > MaxHeadYaw + MaxEyeHorizontal)
|
if (FMath::Abs(DeltaFromBodyTarget) > MaxHeadYaw + MaxEyeHorizontal)
|
||||||
{
|
{
|
||||||
// Body realigns to face target
|
// Body realigns to face target
|
||||||
TargetBodyWorldYaw = TargetWorldYaw - MeshForwardYawOffset;
|
TargetBodyWorldYaw = TargetWorldYaw - MeshForwardYawOffset;
|
||||||
// Head returns to ~0° since body will face target directly
|
// Head returns to ~0° since body will face target directly
|
||||||
TargetHeadYaw = 0.0f;
|
TargetHeadYaw = 0.0f;
|
||||||
|
bBodyOverflowed = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── 6. HEAD: realign when eyes overflow (check against body TARGET) ──
|
// ── 6. HEAD: realign when eyes overflow (check against body TARGET) ──
|
||||||
@ -219,7 +227,15 @@ void UElevenLabsPostureComponent::TickComponent(
|
|||||||
// the head from overcompensating during body interpolation —
|
// the head from overcompensating during body interpolation —
|
||||||
// otherwise the head turns to track while body catches up, then
|
// otherwise the head turns to track while body catches up, then
|
||||||
// snaps back when body arrives (two-step animation artifact).
|
// snaps back when body arrives (two-step animation artifact).
|
||||||
|
//
|
||||||
|
// GUARD: Skip eye overflow check when body just overflowed in this
|
||||||
|
// same frame. Body overflow already set TargetHeadYaw=0 (head will
|
||||||
|
// recenter as body turns). Allowing eye overflow to also fire would
|
||||||
|
// snap TargetHeadYaw to a second value in the same frame, causing
|
||||||
|
// a visible 1-2 frame jerk as FInterpTo chases two targets.
|
||||||
|
|
||||||
|
if (!bBodyOverflowed)
|
||||||
|
{
|
||||||
const float HeadDeltaYaw = FMath::FindDeltaAngleDegrees(
|
const float HeadDeltaYaw = FMath::FindDeltaAngleDegrees(
|
||||||
TargetBodyWorldYaw + MeshForwardYawOffset, TargetWorldYaw);
|
TargetBodyWorldYaw + MeshForwardYawOffset, TargetWorldYaw);
|
||||||
|
|
||||||
@ -229,9 +245,10 @@ void UElevenLabsPostureComponent::TickComponent(
|
|||||||
{
|
{
|
||||||
TargetHeadYaw = FMath::Clamp(HeadDeltaYaw, -MaxHeadYaw, MaxHeadYaw);
|
TargetHeadYaw = FMath::Clamp(HeadDeltaYaw, -MaxHeadYaw, MaxHeadYaw);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Head smoothly interpolates toward its persistent target
|
// Head smoothly interpolates toward its persistent target
|
||||||
CurrentHeadYaw = FMath::FInterpTo(CurrentHeadYaw, TargetHeadYaw, DeltaTime, HeadInterpSpeed);
|
CurrentHeadYaw = FMath::FInterpTo(CurrentHeadYaw, TargetHeadYaw, SafeDeltaTime, HeadInterpSpeed);
|
||||||
|
|
||||||
// Eyes = remaining gap (during transients, eyes may sit at MaxEye while
|
// Eyes = remaining gap (during transients, eyes may sit at MaxEye while
|
||||||
// the head catches up — ARKit normalization keeps visual deflection small)
|
// the head catches up — ARKit normalization keeps visual deflection small)
|
||||||
@ -248,7 +265,7 @@ void UElevenLabsPostureComponent::TickComponent(
|
|||||||
TargetHeadPitch = FMath::Clamp(TargetPitch, -MaxHeadPitch, MaxHeadPitch);
|
TargetHeadPitch = FMath::Clamp(TargetPitch, -MaxHeadPitch, MaxHeadPitch);
|
||||||
}
|
}
|
||||||
|
|
||||||
CurrentHeadPitch = FMath::FInterpTo(CurrentHeadPitch, TargetHeadPitch, DeltaTime, HeadInterpSpeed);
|
CurrentHeadPitch = FMath::FInterpTo(CurrentHeadPitch, TargetHeadPitch, SafeDeltaTime, HeadInterpSpeed);
|
||||||
|
|
||||||
// Eyes = remaining pitch gap
|
// Eyes = remaining pitch gap
|
||||||
CurrentEyePitch = FMath::Clamp(TargetPitch - CurrentHeadPitch, -MaxEyeVertical, MaxEyeVertical);
|
CurrentEyePitch = FMath::Clamp(TargetPitch - CurrentHeadPitch, -MaxEyeVertical, MaxEyeVertical);
|
||||||
@ -263,17 +280,17 @@ void UElevenLabsPostureComponent::TickComponent(
|
|||||||
Owner->GetActorRotation().Yaw, TargetBodyWorldYaw);
|
Owner->GetActorRotation().Yaw, TargetBodyWorldYaw);
|
||||||
if (FMath::Abs(NeutralDelta) > 0.1f)
|
if (FMath::Abs(NeutralDelta) > 0.1f)
|
||||||
{
|
{
|
||||||
const float NeutralStep = FMath::FInterpTo(0.0f, NeutralDelta, DeltaTime, ReturnToNeutralSpeed);
|
const float NeutralStep = FMath::FInterpTo(0.0f, NeutralDelta, SafeDeltaTime, ReturnToNeutralSpeed);
|
||||||
Owner->AddActorWorldRotation(FRotator(0.0f, NeutralStep, 0.0f));
|
Owner->AddActorWorldRotation(FRotator(0.0f, NeutralStep, 0.0f));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Head + Eyes: return to center
|
// Head + Eyes: return to center
|
||||||
TargetHeadYaw = 0.0f;
|
TargetHeadYaw = 0.0f;
|
||||||
TargetHeadPitch = 0.0f;
|
TargetHeadPitch = 0.0f;
|
||||||
CurrentHeadYaw = FMath::FInterpTo(CurrentHeadYaw, 0.0f, DeltaTime, ReturnToNeutralSpeed);
|
CurrentHeadYaw = FMath::FInterpTo(CurrentHeadYaw, 0.0f, SafeDeltaTime, ReturnToNeutralSpeed);
|
||||||
CurrentHeadPitch = FMath::FInterpTo(CurrentHeadPitch, 0.0f, DeltaTime, ReturnToNeutralSpeed);
|
CurrentHeadPitch = FMath::FInterpTo(CurrentHeadPitch, 0.0f, SafeDeltaTime, ReturnToNeutralSpeed);
|
||||||
CurrentEyeYaw = FMath::FInterpTo(CurrentEyeYaw, 0.0f, DeltaTime, ReturnToNeutralSpeed);
|
CurrentEyeYaw = FMath::FInterpTo(CurrentEyeYaw, 0.0f, SafeDeltaTime, ReturnToNeutralSpeed);
|
||||||
CurrentEyePitch = FMath::FInterpTo(CurrentEyePitch, 0.0f, DeltaTime, ReturnToNeutralSpeed);
|
CurrentEyePitch = FMath::FInterpTo(CurrentEyePitch, 0.0f, SafeDeltaTime, ReturnToNeutralSpeed);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── 6. Output for AnimNode (thread-safe write) ────────────────────────
|
// ── 6. Output for AnimNode (thread-safe write) ────────────────────────
|
||||||
@ -289,7 +306,7 @@ void UElevenLabsPostureComponent::TickComponent(
|
|||||||
// bone's reference rotation (which also contributes Y-coupling).
|
// bone's reference rotation (which also contributes Y-coupling).
|
||||||
const FQuat NodQuat(FVector::UpVector, FMath::DegreesToRadians(-CurrentHeadPitch));
|
const FQuat NodQuat(FVector::UpVector, FMath::DegreesToRadians(-CurrentHeadPitch));
|
||||||
const FQuat TurnQuat(FVector::ForwardVector, FMath::DegreesToRadians(CurrentHeadYaw));
|
const FQuat TurnQuat(FVector::ForwardVector, FMath::DegreesToRadians(CurrentHeadYaw));
|
||||||
CurrentHeadRotation = TurnQuat * NodQuat;
|
CurrentHeadRotation = (TurnQuat * NodQuat).GetNormalized();
|
||||||
|
|
||||||
// Eye yaw is negated to match ARKit curve direction convention.
|
// Eye yaw is negated to match ARKit curve direction convention.
|
||||||
UpdateEyeCurves(-CurrentEyeYaw, CurrentEyePitch);
|
UpdateEyeCurves(-CurrentEyeYaw, CurrentEyePitch);
|
||||||
|
|||||||
@ -78,6 +78,10 @@ private:
|
|||||||
/** Rotation weights, parallel to ChainBoneNames. */
|
/** Rotation weights, parallel to ChainBoneNames. */
|
||||||
TArray<float> ChainBoneWeights;
|
TArray<float> ChainBoneWeights;
|
||||||
|
|
||||||
|
/** Reference pose rotations, parallel to ChainBoneIndices.
|
||||||
|
* Cached from the skeleton at CacheBones for animation compensation. */
|
||||||
|
TArray<FQuat> ChainRefPoseRotations;
|
||||||
|
|
||||||
// ── Fallback single-bone (when chain is empty) ──────────────────────────
|
// ── Fallback single-bone (when chain is empty) ──────────────────────────
|
||||||
|
|
||||||
/** Resolved head bone index in the skeleton (fallback). */
|
/** Resolved head bone index in the skeleton (fallback). */
|
||||||
@ -85,4 +89,13 @@ private:
|
|||||||
|
|
||||||
/** Head bone name cached from the component at initialization (fallback). */
|
/** Head bone name cached from the component at initialization (fallback). */
|
||||||
FName HeadBoneName;
|
FName HeadBoneName;
|
||||||
|
|
||||||
|
/** Reference pose rotation for fallback head bone. */
|
||||||
|
FQuat HeadRefPoseRotation = FQuat::Identity;
|
||||||
|
|
||||||
|
// ── Animation compensation ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
/** How much posture overrides the animation's head rotation (0=additive, 1=override).
|
||||||
|
* Cached from the component during Update. */
|
||||||
|
float CachedAnimationCompensation = 1.0f;
|
||||||
};
|
};
|
||||||
|
|||||||
@ -8,6 +8,7 @@
|
|||||||
#include "ElevenLabsLipSyncComponent.generated.h"
|
#include "ElevenLabsLipSyncComponent.generated.h"
|
||||||
|
|
||||||
class UElevenLabsConversationalAgentComponent;
|
class UElevenLabsConversationalAgentComponent;
|
||||||
|
class UElevenLabsFacialExpressionComponent;
|
||||||
class UElevenLabsLipSyncPoseMap;
|
class UElevenLabsLipSyncPoseMap;
|
||||||
class USkeletalMeshComponent;
|
class USkeletalMeshComponent;
|
||||||
|
|
||||||
@ -75,6 +76,19 @@ public:
|
|||||||
ToolTip = "Smoothing speed for viseme transitions.\n35 = smooth and soft, 50 = balanced, 65 = sharp and responsive."))
|
ToolTip = "Smoothing speed for viseme transitions.\n35 = smooth and soft, 50 = balanced, 65 = sharp and responsive."))
|
||||||
float SmoothingSpeed = 50.0f;
|
float SmoothingSpeed = 50.0f;
|
||||||
|
|
||||||
|
// ── Emotion Expression Blend ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
/** How much facial emotion (from ElevenLabsFacialExpressionComponent) bleeds through
|
||||||
|
* during speech on expression curves (smile, frown, sneer, dimple, etc.).
|
||||||
|
* Shape curves (jaw, tongue, lip shape) are always 100% lip sync.
|
||||||
|
* 0.0 = lip sync completely overrides expression curves (old behavior).
|
||||||
|
* 1.0 = emotion fully additive on expression curves during speech.
|
||||||
|
* 0.5 = balanced blend (recommended). */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ElevenLabs|LipSync",
|
||||||
|
meta = (ClampMin = "0.0", ClampMax = "1.0",
|
||||||
|
ToolTip = "Emotion bleed-through during speech.\n0 = pure lip sync, 0.5 = balanced, 1 = full emotion on expression curves.\nOnly affects expression curves (smile, frown, etc.), not lip shape."))
|
||||||
|
float EmotionExpressionBlend = 0.5f;
|
||||||
|
|
||||||
// ── Audio Envelope ──────────────────────────────────────────────────────
|
// ── Audio Envelope ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Envelope attack time in milliseconds.
|
/** Envelope attack time in milliseconds.
|
||||||
@ -274,6 +288,9 @@ private:
|
|||||||
TWeakObjectPtr<UElevenLabsConversationalAgentComponent> AgentComponent;
|
TWeakObjectPtr<UElevenLabsConversationalAgentComponent> AgentComponent;
|
||||||
FDelegateHandle AudioDataHandle;
|
FDelegateHandle AudioDataHandle;
|
||||||
|
|
||||||
|
// Cached reference to the facial expression component for emotion blending
|
||||||
|
TWeakObjectPtr<UElevenLabsFacialExpressionComponent> CachedFacialExprComp;
|
||||||
|
|
||||||
// ── Pose-extracted curve data ────────────────────────────────────────────
|
// ── Pose-extracted curve data ────────────────────────────────────────────
|
||||||
|
|
||||||
/** Instance-level mapping extracted from pose AnimSequences at BeginPlay.
|
/** Instance-level mapping extracted from pose AnimSequences at BeginPlay.
|
||||||
|
|||||||
@ -192,6 +192,9 @@ public:
|
|||||||
/** Get the neck bone chain (used by AnimNode to resolve bone indices). */
|
/** Get the neck bone chain (used by AnimNode to resolve bone indices). */
|
||||||
const TArray<FElevenLabsNeckBoneEntry>& GetNeckBoneChain() const { return NeckBoneChain; }
|
const TArray<FElevenLabsNeckBoneEntry>& GetNeckBoneChain() const { return NeckBoneChain; }
|
||||||
|
|
||||||
|
/** Get animation compensation factor (0 = additive, 1 = full override). */
|
||||||
|
float GetAnimationCompensation() const { return AnimationCompensation; }
|
||||||
|
|
||||||
// ── UActorComponent overrides ────────────────────────────────────────────
|
// ── UActorComponent overrides ────────────────────────────────────────────
|
||||||
virtual void BeginPlay() override;
|
virtual void BeginPlay() override;
|
||||||
virtual void TickComponent(float DeltaTime, ELevelTick TickType,
|
virtual void TickComponent(float DeltaTime, ELevelTick TickType,
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user