j.foucher/PS_AI_Agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix thread-safety crash and body rotation saccades in network play

Browse Source 
- FacialExpressionComponent: add FCriticalSection around emotion curves
  to prevent race between TickComponent (game thread) and anim worker
  thread — root cause of EXCEPTION_ACCESS_VIOLATION at Evaluate_AnyThread
- Remove deprecated FBlendedCurve::IsValid() guards (UE 5.5: always true)
- Body tracking: replace AddActorWorldRotation() with animation-only
  pelvis bone rotation via AnimNode — eliminates replication tug-of-war
  that caused client-side saccades when server overwrote local rotation

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
j.foucher 2026-03-03 10:55:59 +01:00
parent 45ee0c6f7d
commit 677f08e936
8 changed files with 123 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/AnimNode_PS_AI_ConvAgent_FacialExpression.cpp
View File

@ -89,12 +89,9 @@ void FAnimNode_PS_AI_ConvAgent_FacialExpression::Evaluate_AnyThread(FPoseContext
// covering eyes, eyebrows, cheeks, nose, and mouth mood.
// The downstream Lip Sync node will override mouth-area curves
// during speech, while non-mouth emotion curves pass through.
if (Output.Curve.IsValid())
for (const auto& Pair : CachedEmotionCurves)
{
for (const auto& Pair : CachedEmotionCurves)
{
Output.Curve.Set(Pair.Key, Pair.Value);
}
Output.Curve.Set(Pair.Key, Pair.Value);
}
}

9
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/AnimNode_PS_AI_ConvAgent_LipSync.cpp
View File

@ -89,14 +89,11 @@ void FAnimNode_PS_AI_ConvAgent_LipSync::Evaluate_AnyThread(FPoseContext& Output)
// Skip near-zero values so that the upstream Facial Expression node's
// emotion curves (eyes, brows, mouth mood) pass through during silence.
// During speech, active lip sync curves override emotion's mouth curves.
if (Output.Curve.IsValid())
for (const auto& Pair : CachedCurves)
{
for (const auto& Pair : CachedCurves)
if (FMath::Abs(Pair.Value) > 0.01f)
{
if (FMath::Abs(Pair.Value) > 0.01f)
{
Output.Curve.Set(Pair.Key, Pair.Value);
}
Output.Curve.Set(Pair.Key, Pair.Value);
}
}
}

56
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/AnimNode_PS_AI_ConvAgent_Posture.cpp
View File

@ -144,6 +144,8 @@ void FAnimNode_PS_AI_ConvAgent_Posture::CacheBones_AnyThread(const FAnimationCac
RightEyeBoneIndex = FCompactPoseBoneIndex(INDEX_NONE);
LeftEyeRefPoseRotation = FQuat::Identity;
RightEyeRefPoseRotation = FQuat::Identity;
BodyBoneIndex = FCompactPoseBoneIndex(INDEX_NONE);
BodyBoneRefPoseRotation = FQuat::Identity;
ChainBoneIndices.Reset();
ChainRefPoseRotations.Reset();
@ -247,6 +249,22 @@ void FAnimNode_PS_AI_ConvAgent_Posture::CacheBones_AnyThread(const FAnimationCac
ResolveEyeBone(DefaultRightEyeBone, RightEyeBoneIndex, RightEyeRefPoseRotation);
}
// ── Resolve body bone for body yaw rotation ─────────────────────
// The body yaw offset is applied to "pelvis" (or first skeleton bone)
// to rotate the entire skeleton without modifying actor rotation.
{
static const FName DefaultBodyBone(TEXT("pelvis"));
const int32 MeshIdx = RefSkeleton.FindBoneIndex(DefaultBodyBone);
if (MeshIdx != INDEX_NONE)
{
BodyBoneIndex = RequiredBones.MakeCompactPoseIndex(
FMeshPoseBoneIndex(MeshIdx));
BodyBoneRefPoseRotation = (MeshIdx < RefPose.Num())
? RefPose[MeshIdx].GetRotation()
: FQuat::Identity;
}
}
// ── Resolve ancestor chain for body drift compensation ────────────
// Walk from the parent of the first neck/head bone up to root.
// This lets us measure how much the spine/torso animation has
@ -328,6 +346,7 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Update_AnyThread(const FAnimationUpdateC
CachedHeadCompensation = PostureComponent->GetHeadAnimationCompensation();
CachedEyeCompensation = PostureComponent->GetEyeAnimationCompensation();
CachedBodyDriftCompensation = PostureComponent->GetBodyDriftCompensation();
CachedBodyYawOffset = PostureComponent->GetBodyYawOffset();
}
}
@ -379,11 +398,6 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
// Evaluate the upstream pose (pass-through)
BasePose.Evaluate(Output);
// Guard: in packaged+network builds the curve container may not be
// initialized yet (skeleton not fully loaded). All Output.Curve access
// must be gated on this flag to avoid null-pointer crashes.
const bool bCurveValid = Output.Curve.IsValid();
// ── Periodic diagnostic (runs for EVERY instance, before any early return) ─
#if !UE_BUILD_SHIPPING
if (++EvalDebugFrameCounter % 300 == 1) // ~every 5 seconds at 60fps
@ -437,7 +451,6 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
#if ELEVENLABS_EYE_DIAGNOSTIC == 1
// MODE 1: CTRL curves → tests CORRECT MetaHuman CTRL naming
// Real format: CTRL_expressions_eyeLook{Dir}{L/R} (NOT eyeLookUpLeft!)
if (bCurveValid)
{
static const FName ForceCTRL(TEXT("CTRL_expressions_eyeLookUpL"));
Output.Curve.Set(ForceCTRL, 1.0f);
@ -459,7 +472,6 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
#elif ELEVENLABS_EYE_DIAGNOSTIC == 2
// MODE 2: ARKit curves → tests if mh_arkit_mapping_pose drives eyes
if (bCurveValid)
{
static const FName ForceARKit(TEXT("eyeLookUpLeft"));
Output.Curve.Set(ForceARKit, 1.0f);
@ -490,7 +502,6 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
(LookUp * LeftEyeRefPoseRotation).GetNormalized());
}
// Zero curves to isolate
if (bCurveValid)
{
static const FName ZeroARKit(TEXT("eyeLookUpLeft"));
static const FName ZeroCTRL(TEXT("CTRL_expressions_eyeLookUpLeft"));
@ -551,7 +562,6 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
}
// (b) Blend CTRL eye curves: read animation's value, lerp with posture
if (bCurveValid)
{
const auto& CTRLMap = GetARKitToCTRLEyeMap();
for (const auto& Pair : CachedEyeCurves)
@ -566,15 +576,15 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
Output.Curve.Set(*CTRLName, BlendedValue);
}
}
}
// (c) Zero ARKit eye curves to prevent mh_arkit_mapping_pose
// from overwriting our carefully blended CTRL values.
// mh_arkit converts ARKit→CTRL additively; zeroing means
// it adds nothing for eyes, preserving our blend.
for (const auto& Pair : CachedEyeCurves)
{
Output.Curve.Set(Pair.Key, 0.0f);
}
// (c) Zero ARKit eye curves to prevent mh_arkit_mapping_pose
// from overwriting our carefully blended CTRL values.
// mh_arkit converts ARKit→CTRL additively; zeroing means
// it adds nothing for eyes, preserving our blend.
for (const auto& Pair : CachedEyeCurves)
{
Output.Curve.Set(Pair.Key, 0.0f);
}
}
@ -600,6 +610,18 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
return;
}
// ── Apply body yaw rotation to pelvis bone ─────────────────────────────
// This replaces the old AddActorWorldRotation() approach — the actor rotation
// is never modified; the body turn is purely animation-driven (local, no replication).
if (FMath::Abs(CachedBodyYawOffset) > 0.1f
&& BodyBoneIndex.GetInt() != INDEX_NONE
&& BodyBoneIndex.GetInt() < Output.Pose.GetNumBones())
{
const FQuat BodyYawQuat(FVector::UpVector, FMath::DegreesToRadians(CachedBodyYawOffset));
FTransform& BodyTransform = Output.Pose[BodyBoneIndex];
BodyTransform.SetRotation((BodyYawQuat * BodyTransform.GetRotation()).GetNormalized());
}
// IMPORTANT: Even when posture is near-zero (head looking straight at target),
// we still need to run compensation to REMOVE the animation's head contribution.
// Only skip if BOTH posture is identity AND compensation is inactive (pure additive).

19
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_FacialExpressionComponent.cpp
View File

@ -335,13 +335,17 @@ void UPS_AI_ConvAgent_FacialExpressionComponent::TickComponent(
}
// ── Evaluate curves from playing animations ─────────────────────────────
// Compute into a local map first, then swap under lock to avoid
// racing with the AnimNode worker thread reading GetCurrentEmotionCurves().
TMap<FName, float> NewCurves;
TMap<FName, float> ActiveCurves = EvaluateAnimCurves(ActiveAnim, ActivePlaybackTime);
if (CrossfadeAlpha >= 1.0f)
{
// No crossfade — use active curves directly
CurrentEmotionCurves = MoveTemp(ActiveCurves);
NewCurves = MoveTemp(ActiveCurves);
}
else
{
@ -349,7 +353,6 @@ void UPS_AI_ConvAgent_FacialExpressionComponent::TickComponent(
TMap<FName, float> PrevCurves = EvaluateAnimCurves(PrevAnim, PrevPlaybackTime);
// Collect all curve names from both anims
CurrentEmotionCurves.Reset();
TSet<FName> AllCurves;
for (const auto& P : ActiveCurves) AllCurves.Add(P.Key);
for (const auto& P : PrevCurves) AllCurves.Add(P.Key);
@ -364,7 +367,7 @@ void UPS_AI_ConvAgent_FacialExpressionComponent::TickComponent(
if (FMath::Abs(Blended) > 0.001f)
{
CurrentEmotionCurves.Add(CurveName, Blended);
NewCurves.Add(CurveName, Blended);
}
}
}
@ -372,15 +375,21 @@ void UPS_AI_ConvAgent_FacialExpressionComponent::TickComponent(
// ── Apply activation alpha to output curves ──────────────────────────
if (CurrentActiveAlpha < 0.001f)
{
CurrentEmotionCurves.Reset();
NewCurves.Reset();
}
else if (CurrentActiveAlpha < 0.999f)
{
for (auto& Pair : CurrentEmotionCurves)
for (auto& Pair : NewCurves)
{
Pair.Value *= CurrentActiveAlpha;
}
}
// Swap under lock — the AnimNode reads CurrentEmotionCurves from a worker thread
{
FScopeLock Lock(&EmotionCurveLock);
CurrentEmotionCurves = MoveTemp(NewCurves);
}
}
// ─────────────────────────────────────────────────────────────────────────────

31
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_PostureComponent.cpp
View File

@ -99,6 +99,7 @@ void UPS_AI_ConvAgent_PostureComponent::BeginPlay()
// Apply the mesh forward offset so "neutral" aligns with where the face points.
OriginalActorYaw = Owner->GetActorRotation().Yaw + MeshForwardYawOffset;
TargetBodyWorldYaw = Owner->GetActorRotation().Yaw;
CurrentBodyWorldYaw = Owner->GetActorRotation().Yaw;
if (bDebug)
{
@ -179,6 +180,7 @@ void UPS_AI_ConvAgent_PostureComponent::ResetBodyTarget()
if (AActor* Owner = GetOwner())
{
TargetBodyWorldYaw = Owner->GetActorRotation().Yaw;
CurrentBodyWorldYaw = Owner->GetActorRotation().Yaw;
}
}
@ -333,19 +335,14 @@ void UPS_AI_ConvAgent_PostureComponent::TickComponent(
if (bEnableBodyTracking)
{
const float BodyDelta = FMath::FindDeltaAngleDegrees(
Owner->GetActorRotation().Yaw, TargetBodyWorldYaw);
CurrentBodyWorldYaw, TargetBodyWorldYaw);
if (FMath::Abs(BodyDelta) > 0.1f)
{
const float BodyStep = FMath::FInterpTo(0.0f, BodyDelta, SafeDeltaTime, BodyInterpSpeed);
// Only modify actor rotation on the authority (server/standalone).
// On clients, the rotation arrives via replication — calling
// AddActorWorldRotation here would fight with replicated updates,
// causing visible stuttering as the network periodically snaps
// the rotation back to the server's value.
if (Owner->HasAuthority())
{
Owner->AddActorWorldRotation(FRotator(0.0f, BodyStep, 0.0f));
}
// Track body facing internally instead of modifying the actor rotation.
// The AnimNode applies the delta as a bone rotation — purely local,
// no actor rotation change, no replication conflict.
CurrentBodyWorldYaw += BodyStep;
}
}
@ -354,7 +351,7 @@ void UPS_AI_ConvAgent_PostureComponent::TickComponent(
float DeltaYaw = 0.0f;
if (!HorizontalDir.IsNearlyZero(1.0f))
{
const float CurrentFacingYaw = Owner->GetActorRotation().Yaw + MeshForwardYawOffset;
const float CurrentFacingYaw = CurrentBodyWorldYaw + MeshForwardYawOffset;
DeltaYaw = FMath::FindDeltaAngleDegrees(CurrentFacingYaw, TargetWorldYaw);
}
@ -467,6 +464,11 @@ void UPS_AI_ConvAgent_PostureComponent::TickComponent(
// Eye yaw is negated to match ARKit curve direction convention.
UpdateEyeCurves(-CurrentEyeYaw, CurrentEyePitch);
// Body yaw offset = how much the body has turned relative to the actor's rotation.
// The AnimNode applies this as a bone rotation (pelvis) — purely local.
CachedBodyYawOffset = FMath::FindDeltaAngleDegrees(
Owner->GetActorRotation().Yaw, CurrentBodyWorldYaw) * CurrentActiveAlpha;
}
// ── Debug gaze lines ────────────────────────────────────────────────────
@ -539,21 +541,22 @@ void UPS_AI_ConvAgent_PostureComponent::TickComponent(
DebugFrameCounter++;
if (DebugFrameCounter % 120 == 0)
{
const float FacingYaw = Owner->GetActorRotation().Yaw + MeshForwardYawOffset;
const float FacingYaw = CurrentBodyWorldYaw + MeshForwardYawOffset;
const FVector TP = TargetActor->GetActorLocation() + TargetOffset;
const FVector Dir = TP - Owner->GetActorLocation();
const float TgtYaw = FVector(Dir.X, Dir.Y, 0.0f).Rotation().Yaw;
const float Delta = FMath::FindDeltaAngleDegrees(FacingYaw, TgtYaw);
UE_LOG(LogPS_AI_ConvAgent_Posture, Log,
TEXT("Posture [%s -> %s]: Delta=%.1f | Head=%.1f/%.1f | Eyes=%.1f/%.1f | Body: enabled=%s TargetYaw=%.1f ActorYaw=%.1f"),
TEXT("Posture [%s -> %s]: Delta=%.1f | Head=%.1f/%.1f | Eyes=%.1f/%.1f | Body: enabled=%s TargetYaw=%.1f BodyYaw=%.1f (offset=%.1f)"),
*Owner->GetName(), *TargetActor->GetName(),
Delta,
CurrentHeadYaw, CurrentHeadPitch,
CurrentEyeYaw, CurrentEyePitch,
bEnableBodyTracking ? TEXT("Y") : TEXT("N"),
TargetBodyWorldYaw,
Owner->GetActorRotation().Yaw);
CurrentBodyWorldYaw,
CachedBodyYawOffset);
}
}
}

14
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/AnimNode_PS_AI_ConvAgent_Posture.h
View File

@ -139,6 +139,20 @@ private:
* Cached from the component during Update. */
float CachedBodyDriftCompensation = 0.0f;
// ── Body yaw (animation-only body tracking) ──────────────────────────
/** Bone to apply body yaw rotation to. Rotates the entire skeleton
* visually without modifying the actor rotation (no replication conflict).
* Default "pelvis" works for MetaHuman and most humanoid skeletons. */
FCompactPoseBoneIndex BodyBoneIndex = FCompactPoseBoneIndex(INDEX_NONE);
/** Reference pose rotation for the body bone. */
FQuat BodyBoneRefPoseRotation = FQuat::Identity;
/** Body yaw offset in degrees (relative to actor rotation).
* Cached from the component during Update. */
float CachedBodyYawOffset = 0.0f;
#if !UE_BUILD_SHIPPING
/** Frame counter for periodic diagnostic logging in Evaluate. */
int32 EvalDebugFrameCounter = 0;

17
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_FacialExpressionComponent.h
View File

@ -3,6 +3,7 @@
#pragma once
#include "CoreMinimal.h"
#include "HAL/CriticalSection.h"
#include "Components/ActorComponent.h"
#include "PS_AI_ConvAgent_Definitions.h"
#include "PS_AI_ConvAgent_FacialExpressionComponent.generated.h"
@ -83,9 +84,14 @@ public:
// ── Getters ───────────────────────────────────────────────────────────────
/** Get the current emotion curves evaluated from the playing AnimSequence. */
/** Get the current emotion curves evaluated from the playing AnimSequence.
* Returns a COPY — safe to call from any thread (AnimNode worker thread). */
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|FacialExpression")
const TMap<FName, float>& GetCurrentEmotionCurves() const { return CurrentEmotionCurves; }
TMap<FName, float> GetCurrentEmotionCurves() const
{
FScopeLock Lock(&EmotionCurveLock);
return CurrentEmotionCurves;
}
/** Get the active emotion. */
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|FacialExpression")
@ -159,9 +165,14 @@ private:
// ── Curve output ─────────────────────────────────────────────────────────
/** Current blended emotion curves (evaluated each tick from playing anims). */
/** Current blended emotion curves (evaluated each tick from playing anims).
* Protected by EmotionCurveLock — written in TickComponent (game thread),
* read by AnimNode via GetCurrentEmotionCurves() (worker thread). */
TMap<FName, float> CurrentEmotionCurves;
/** Lock protecting CurrentEmotionCurves from concurrent game/anim thread access. */
mutable FCriticalSection EmotionCurveLock;
/** Current blend alpha (0 = fully inactive/passthrough, 1 = fully active). */
float CurrentActiveAlpha = 1.0f;

21
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_PostureComponent.h
View File

@ -273,6 +273,15 @@ public:
* Scaled by activation alpha for smooth passthrough when inactive. */
float GetBodyDriftCompensation() const { return BodyDriftCompensation * CurrentActiveAlpha; }
/** Get body yaw offset in degrees (relative to actor's rotation).
* Applied by the AnimNode as a bone rotation — purely local, no replication.
* Thread-safe copy, blended by activation alpha. */
float GetBodyYawOffset() const
{
FScopeLock Lock(&PostureDataLock);
return CachedBodyYawOffset;
}
/** Reset the persistent body yaw target to the actor's current facing.
* Call this when re-attaching a posture target so body tracking starts
* fresh instead of chasing a stale yaw from the previous interaction. */
@ -316,11 +325,17 @@ private:
float TargetHeadYaw = 0.0f;
float TargetHeadPitch = 0.0f;
/** Persistent body target — world yaw the actor should face.
/** Persistent body target — world yaw the body should face.
* Only updated when head+eyes can't reach the target (overflow).
* Same sticky pattern as TargetHeadYaw but for the body layer. */
float TargetBodyWorldYaw = 0.0f;
/** Interpolated actual body facing in world space.
* Replaces the old AddActorWorldRotation() approach — the actor rotation
* is never modified; instead this tracks the virtual body facing and the
* AnimNode applies the delta as a bone rotation (100% local, no replication). */
float CurrentBodyWorldYaw = 0.0f;
/** Original actor yaw at BeginPlay (for neutral return when TargetActor is null). */
float OriginalActorYaw = 0.0f;
@ -338,6 +353,10 @@ private:
/** Head bone rotation offset as quaternion (no Euler round-trip). */
FQuat CurrentHeadRotation = FQuat::Identity;
/** Body yaw offset in degrees (relative to actor rotation).
* Computed in TickComponent, read by AnimNode via GetBodyYawOffset(). */
float CachedBodyYawOffset = 0.0f;
/** Cached skeletal mesh component on the owning actor (Body). */
TWeakObjectPtr<USkeletalMeshComponent> CachedMesh;