v3.0.1: Packaged build fixes, conversation-triggered body tracking, diagnostic cleanup

Packaged build fixes:
- Use EvaluateCurveData() for emotion and lip sync curves (works with
  compressed/cooked data instead of raw FloatCurves)
- Add FMemMark wrapper for game-thread curve evaluation (FBlendedCurve
  uses TMemStackAllocator)
- Lazy binding in AnimNodes and LipSyncComponent for packaged build
  component initialization order
- SetIsReplicatedByDefault(true) instead of SetIsReplicated(true)
- Load AudioCapture module explicitly in plugin StartupModule
- Bundle cacert.pem for SSL in packaged builds
- Add DirectoriesToAlwaysStageAsNonUFS for certificates
- Add EmotionPoseMap and LipSyncPoseMap .cpp implementations

Body tracking:
- Body tracking now activates on conversation start (HandleAgentResponseStarted)
  instead of on selection, creating a natural notice→engage two-step:
  eyes+head track on selection, body turns when agent starts responding
- SendTextMessage also enables body tracking for text input

Cleanup:
- Remove all temporary [DIAG] and [BODY] debug logs
- Gate PostureComponent periodic debug log behind bDebug flag
- Remove obsolete editor-time curve caches (CachedCurveData, RebuildCurveCache,
  FPS_AI_ConvAgent_CachedEmotionCurves, FPS_AI_ConvAgent_CachedPoseCurves)
  from EmotionPoseMap and LipSyncPoseMap — no longer needed since
  EvaluateCurveData() reads compressed curves directly at runtime

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
j.foucher 2026-03-01 12:41:03 +01:00
parent 21298e01b0
commit 350558ba50
20 changed files with 3547 additions and 90 deletions

View File

@ -1,3 +1,6 @@
[/Script/EngineSettings.GeneralProjectSettings]
ProjectID=C89AEFD7484597308B8E6EB1C7AE0965
[/Script/UnrealEd.ProjectPackagingSettings]
+DirectoriesToAlwaysStageAsNonUFS=(Path="Certificates")

View File

@ -36,6 +36,10 @@
"Linux",
"Android"
]
},
{
"Name": "AudioCapture",
"Enabled": true
}
]
}

File diff suppressed because it is too large Load Diff

View File

@ -57,6 +57,30 @@ void FAnimNode_PS_AI_ConvAgent_FacialExpression::Update_AnyThread(const FAnimati
{
BasePose.Update(Context);
// Lazy lookup: in packaged builds, Initialize_AnyThread may run before
// components are created. Retry discovery until found.
if (!FacialExpressionComponent.IsValid())
{
if (const FAnimInstanceProxy* Proxy = Context.AnimInstanceProxy)
{
if (const USkeletalMeshComponent* SkelMesh = Proxy->GetSkelMeshComponent())
{
if (AActor* Owner = SkelMesh->GetOwner())
{
UPS_AI_ConvAgent_FacialExpressionComponent* Comp =
Owner->FindComponentByClass<UPS_AI_ConvAgent_FacialExpressionComponent>();
if (Comp)
{
FacialExpressionComponent = Comp;
UE_LOG(LogPS_AI_ConvAgent_FacialExprAnimNode, Log,
TEXT("PS AI ConvAgent Facial Expression AnimNode (late) bound to component on %s."),
*Owner->GetName());
}
}
}
}
}
// Cache emotion curves from the facial expression component.
// GetCurrentEmotionCurves() returns CTRL_expressions_* curves
// extracted from emotion pose AnimSequences, already smoothly blended.

View File

@ -57,6 +57,30 @@ void FAnimNode_PS_AI_ConvAgent_LipSync::Update_AnyThread(const FAnimationUpdateC
{
BasePose.Update(Context);
// Lazy lookup: in packaged builds, Initialize_AnyThread may run before
// components are created. Retry discovery until found.
if (!LipSyncComponent.IsValid())
{
if (const FAnimInstanceProxy* Proxy = Context.AnimInstanceProxy)
{
if (const USkeletalMeshComponent* SkelMesh = Proxy->GetSkelMeshComponent())
{
if (AActor* Owner = SkelMesh->GetOwner())
{
UPS_AI_ConvAgent_LipSyncComponent* Comp =
Owner->FindComponentByClass<UPS_AI_ConvAgent_LipSyncComponent>();
if (Comp)
{
LipSyncComponent = Comp;
UE_LOG(LogPS_AI_ConvAgent_LipSyncAnimNode, Log,
TEXT("PS AI ConvAgent Lip Sync AnimNode (late) bound to component on %s."),
*Owner->GetName());
}
}
}
}
}
// Cache ARKit blendshape data from the lip sync component.
// GetCurrentBlendshapes() returns ARKit names (jawOpen, mouthFunnel, etc.).
// These are injected as-is into the pose curves; the downstream

View File

@ -196,7 +196,7 @@ void FAnimNode_PS_AI_ConvAgent_Posture::CacheBones_AnyThread(const FAnimationCac
{
ChainBoneIndices.Add(FCompactPoseBoneIndex(INDEX_NONE));
ChainRefPoseRotations.Add(FQuat::Identity);
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose,
TEXT(" Chain bone [%d] '%s' NOT FOUND in skeleton!"),
i, *ChainBoneNames[i].ToString());
}
@ -223,14 +223,14 @@ void FAnimNode_PS_AI_ConvAgent_Posture::CacheBones_AnyThread(const FAnimationCac
}
else
{
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose,
TEXT("Head bone '%s' NOT FOUND in skeleton. Available bones:"),
*HeadBoneName.ToString());
const int32 NumBones = FMath::Min(RefSkeleton.GetNum(), 10);
for (int32 i = 0; i < NumBones; ++i)
{
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose,
TEXT(" [%d] %s"), i, *RefSkeleton.GetBoneName(i).ToString());
}
}
@ -324,6 +324,30 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Update_AnyThread(const FAnimationUpdateC
{
BasePose.Update(Context);
// Lazy lookup: in packaged builds, Initialize_AnyThread may run before
// components are created. Retry discovery until found.
if (!PostureComponent.IsValid())
{
if (const FAnimInstanceProxy* Proxy = Context.AnimInstanceProxy)
{
if (const USkeletalMeshComponent* SkelMesh = Proxy->GetSkelMeshComponent())
{
if (AActor* Owner = SkelMesh->GetOwner())
{
UPS_AI_ConvAgent_PostureComponent* Comp =
Owner->FindComponentByClass<UPS_AI_ConvAgent_PostureComponent>();
if (Comp)
{
PostureComponent = Comp;
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Log,
TEXT("PS AI ConvAgent Posture AnimNode (late) bound to component on %s."),
*Owner->GetName());
}
}
}
}
}
// Cache posture data from the component (game thread safe copy).
// IMPORTANT: Do NOT reset CachedHeadRotation to Identity when the
// component is momentarily invalid (GC pause, re-registration, etc.).
@ -398,7 +422,7 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
const bool bHasEyeBones = (LeftEyeBoneIndex.GetInt() != INDEX_NONE);
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose,
TEXT("[%s] Posture Evaluate: HeadComp=%.2f EyeComp=%.2f DriftComp=%.2f Valid=%s HeadRot=(%s) Eyes=%d Chain=%d Ancestors=%d"),
NodeRole,
CachedHeadCompensation,
@ -420,7 +444,7 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
const FQuat Delta = AnimRot * RefRot.Inverse();
const FRotator DeltaRot = Delta.Rotator();
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose,
TEXT(" Chain[0] '%s' AnimDelta from RefPose: Y=%.2f P=%.2f R=%.2f (this gets removed at Comp=1)"),
*ChainBoneNames[0].ToString(),
DeltaRot.Yaw, DeltaRot.Pitch, DeltaRot.Roll);
@ -454,7 +478,7 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
}
if (++EyeDiagLogCounter % 300 == 1)
{
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose,
TEXT("[EYE DIAG MODE 1] Forcing CTRL_expressions_eyeLookUpL=1.0 | Left eye should look UP if Control Rig reads CTRL curves"));
}
@ -473,7 +497,7 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
}
if (++EyeDiagLogCounter % 300 == 1)
{
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose,
TEXT("[EYE DIAG MODE 2] Forcing ARKit eyeLookUpLeft=1.0 | Left eye should look UP if mh_arkit_mapping_pose drives eyes"));
}
@ -494,7 +518,7 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
Output.Curve.Set(ZeroCTRL, 0.0f);
if (++EyeDiagLogCounter % 300 == 1)
{
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose,
TEXT("[EYE DIAG MODE 3] Forcing FACIAL_L_Eye bone -25° pitch | Left eye should look UP if bone rotation drives eyes"));
}
#endif
@ -576,7 +600,7 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
#if !UE_BUILD_SHIPPING
if (EvalDebugFrameCounter % 300 == 1 && CachedEyeCurves.Num() > 0)
{
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose,
TEXT(" Eyes: Comp=%.2f → %s (anim weight=%.0f%%, posture weight=%.0f%%)"),
Comp,
Comp > 0.001f ? TEXT("BLEND") : TEXT("PASSTHROUGH"),
@ -633,7 +657,7 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
if (++DiagLogCounter % 90 == 0)
{
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose,
TEXT("DIAG Phase %d: %s | Timer=%.1f"), Phase, PhaseName, DiagTimer);
}
}
@ -695,7 +719,7 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
if (EvalDebugFrameCounter % 300 == 1)
{
const FRotator CorrRot = FullCorrection.Rotator();
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose,
TEXT(" DriftCorrection: Y=%.1f P=%.1f R=%.1f | Comp=%.2f"),
CorrRot.Yaw, CorrRot.Pitch, CorrRot.Roll,
CachedBodyDriftCompensation);

View File

@ -4,6 +4,11 @@
#include "Developer/Settings/Public/ISettingsModule.h"
#include "UObject/UObjectGlobals.h"
#include "UObject/Package.h"
#include "Misc/Paths.h"
#include "HAL/PlatformFileManager.h"
#include "Interfaces/IPluginManager.h"
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent, Log, All);
IMPLEMENT_MODULE(FPS_AI_ConvAgentModule, PS_AI_ConvAgent)
@ -22,6 +27,54 @@ void FPS_AI_ConvAgentModule::StartupModule()
LOCTEXT("SettingsDescription", "Configure the PS AI ConvAgent - ElevenLabs plugin"),
Settings);
}
// Ensure the AudioCapture module is loaded (registers WASAPI factory).
// In packaged builds, the module may be linked but not started unless
// explicitly loaded — without it, microphone capture is silent.
if (!FModuleManager::Get().IsModuleLoaded("AudioCapture"))
{
FModuleManager::Get().LoadModule("AudioCapture");
UE_LOG(LogPS_AI_ConvAgent, Log, TEXT("Loaded AudioCapture module for microphone support."));
}
// Ensure SSL certificates are available for packaged builds.
// UE5 expects cacert.pem in [Project]/Content/Certificates/.
// The plugin ships a copy in its Resources/Certificates/ folder.
// If the project doesn't have one, auto-copy it.
EnsureSSLCertificates();
}
void FPS_AI_ConvAgentModule::EnsureSSLCertificates()
{
const FString ProjectCertPath = FPaths::ProjectContentDir() / TEXT("Certificates") / TEXT("cacert.pem");
if (FPlatformFileManager::Get().GetPlatformFile().FileExists(*ProjectCertPath))
{
UE_LOG(LogPS_AI_ConvAgent, Log, TEXT("SSL cacert.pem found at: %s"), *ProjectCertPath);
return;
}
// Try to auto-copy from the plugin's Resources directory.
TSharedPtr<IPlugin> Plugin = IPluginManager::Get().FindPlugin(TEXT("PS_AI_ConvAgent"));
if (Plugin.IsValid())
{
const FString PluginCertPath = Plugin->GetBaseDir()
/ TEXT("Resources") / TEXT("Certificates") / TEXT("cacert.pem");
if (FPlatformFileManager::Get().GetPlatformFile().FileExists(*PluginCertPath))
{
FPlatformFileManager::Get().GetPlatformFile().CreateDirectoryTree(
*(FPaths::ProjectContentDir() / TEXT("Certificates")));
if (FPlatformFileManager::Get().GetPlatformFile().CopyFile(*ProjectCertPath, *PluginCertPath))
{
UE_LOG(LogPS_AI_ConvAgent, Log, TEXT("Copied SSL cacert.pem from plugin to: %s"), *ProjectCertPath);
return;
}
}
}
UE_LOG(LogPS_AI_ConvAgent, Warning, TEXT("SSL cacert.pem not found at: %s. "
"WebSocket wss:// connections may fail in packaged builds. "
"Copy cacert.pem to [Project]/Content/Certificates/."), *ProjectCertPath);
}
void FPS_AI_ConvAgentModule::ShutdownModule()

View File

@ -27,7 +27,7 @@ UPS_AI_ConvAgent_ElevenLabsComponent::UPS_AI_ConvAgent_ElevenLabsComponent()
PrimaryComponentTick.TickInterval = 1.0f / 60.0f;
// Enable network replication for this component.
SetIsReplicated(true);
SetIsReplicatedByDefault(true);
}
// ─────────────────────────────────────────────────────────────────────────────
@ -374,17 +374,6 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StartListening()
Mic->StartCapture();
}
// Enable body tracking on the sibling PostureComponent (if present).
// Voice input counts as conversation engagement, same as text.
if (AActor* OwnerActor = GetOwner())
{
if (UPS_AI_ConvAgent_PostureComponent* Posture =
OwnerActor->FindComponentByClass<UPS_AI_ConvAgent_PostureComponent>())
{
Posture->bEnableBodyTracking = true;
}
}
const double T = TurnStartTime - SessionStartTime;
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log, TEXT("[T+%.2fs] [Turn %d] Mic opened%s — user speaking."),
T, TurnIndex, bExternalMicManagement ? TEXT(" (external)") : TEXT(""));
@ -568,6 +557,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleConnected(const FPS_AI_ConvAgen
{
StartListening();
}
}
void UPS_AI_ConvAgent_ElevenLabsComponent::HandleDisconnected(int32 StatusCode, const FString& Reason)
@ -695,6 +685,18 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAgentResponseStarted()
bAgentGenerating = true;
bWaitingForAgentResponse = false; // Server is generating — response timeout cancelled.
// Enable body tracking: the agent is responding, so conversation has started.
// Body tracking was deliberately NOT enabled on selection (only eyes+head)
// so the agent "notices" the player first and turns its body only when engaging.
if (AActor* OwnerActor = GetOwner())
{
if (UPS_AI_ConvAgent_PostureComponent* Posture =
OwnerActor->FindComponentByClass<UPS_AI_ConvAgent_PostureComponent>())
{
Posture->bEnableBodyTracking = true;
}
}
const double Now = FPlatformTime::Seconds();
const double T = Now - SessionStartTime;
const double LatencyFromTurnEnd = TurnEndTime > 0.0 ? Now - TurnEndTime : 0.0;

View File

@ -0,0 +1,3 @@
// Copyright ASTERION. All Rights Reserved.
#include "PS_AI_ConvAgent_EmotionPoseMap.h"

View File

@ -156,15 +156,23 @@ TMap<FName, float> UPS_AI_ConvAgent_FacialExpressionComponent::EvaluateAnimCurve
TMap<FName, float> CurveValues;
if (!AnimSeq) return CurveValues;
// Use runtime GetCurveData() — GetDataModel() is editor-only in UE 5.5.
const TArray<FFloatCurve>& FloatCurves = AnimSeq->GetCurveData().FloatCurves;
for (const FFloatCurve& Curve : FloatCurves)
// Use UAnimSequence::EvaluateCurveData() — works with both raw (editor)
// and compressed (cooked/packaged) curve data.
// FBlendedCurve uses TMemStackAllocator which requires an active FMemMark.
// We're on the game thread (TickComponent), not in the anim evaluation pipeline,
// so we must set up the mark manually.
{
const float Value = Curve.FloatCurve.Eval(Time);
if (FMath::Abs(Value) > 0.001f)
FMemMark Mark(FMemStack::Get());
FBlendedCurve BlendedCurve;
AnimSeq->EvaluateCurveData(BlendedCurve, Time);
BlendedCurve.ForEachElement([&CurveValues](const UE::Anim::FCurveElement& Element)
{
CurveValues.Add(Curve.GetName(), Value);
}
if (FMath::Abs(Element.Value) > 0.001f)
{
CurveValues.Add(Element.Name, Element.Value);
}
});
}
return CurveValues;
@ -212,6 +220,25 @@ void UPS_AI_ConvAgent_FacialExpressionComponent::TickComponent(
{
Super::TickComponent(DeltaTime, TickType, ThisTickFunction);
// ── Lazy binding: in packaged builds, BeginPlay may run before the
// ElevenLabsComponent is fully initialized. Retry discovery until bound.
if (!AgentComponent.IsValid())
{
if (AActor* Owner = GetOwner())
{
auto* Agent = Owner->FindComponentByClass<UPS_AI_ConvAgent_ElevenLabsComponent>();
if (Agent)
{
AgentComponent = Agent;
Agent->OnAgentEmotionChanged.AddDynamic(
this, &UPS_AI_ConvAgent_FacialExpressionComponent::OnEmotionChanged);
UE_LOG(LogPS_AI_ConvAgent_FacialExpr, Log,
TEXT("Facial expression (late) bound to agent component on %s."),
*Owner->GetName());
}
}
}
// Nothing to play
if (!ActiveAnim && !PrevAnim)
return;

View File

@ -264,15 +264,8 @@ void UPS_AI_ConvAgent_InteractionComponent::SetSelectedAgent(UPS_AI_ConvAgent_El
MicComponent->StartCapture();
}
// ── Listening: start ─────────────────────────────────────────────
// Body tracking is enabled by ElevenLabsComponent itself (in StartListening
// and SendTextMessage) so it works for both voice and text input.
if (bAutoManageListening)
{
NewAgent->StartListening();
}
// ── Posture: attach ──────────────────────────────────────────────
// ── Posture: attach (eyes+head only — body tracking is enabled later
// by ElevenLabsComponent when the agent starts responding) ──
if (bAutoManagePosture && World)
{
// Cancel any pending detach — agent came back before detach fired.
@ -292,6 +285,15 @@ void UPS_AI_ConvAgent_InteractionComponent::SetSelectedAgent(UPS_AI_ConvAgent_El
}
}
// ── Listening: start ─────────────────────────────────────────────
// Opens the mic but does NOT enable body tracking. Body tracking
// is enabled later by HandleAgentResponseStarted (agent starts
// responding) or SendTextMessage (explicit text engagement).
if (bAutoManageListening)
{
NewAgent->StartListening();
}
OnAgentSelected.Broadcast(NewAgent);
}
else

View File

@ -444,55 +444,59 @@ void UPS_AI_ConvAgent_LipSyncComponent::ExtractPoseCurves(const FName& VisemeNam
{
if (!AnimSeq) return;
// Use runtime GetCurveData() — GetDataModel() is editor-only in UE 5.5.
TMap<FName, float> CurveValues;
const TArray<FFloatCurve>& FloatCurves = AnimSeq->GetCurveData().FloatCurves;
for (const FFloatCurve& Curve : FloatCurves)
// Use UAnimSequence::EvaluateCurveData() — works with both raw (editor)
// and compressed (cooked/packaged) curve data. Extract at frame 0.
// FBlendedCurve uses TMemStackAllocator which requires an active FMemMark.
// We're on the game thread (BeginPlay), not in the anim evaluation pipeline,
// so we must set up the mark manually.
{
const FName CurveName = Curve.GetName();
const float Value = Curve.FloatCurve.Eval(0.0f);
FMemMark Mark(FMemStack::Get());
FBlendedCurve BlendedCurve;
AnimSeq->EvaluateCurveData(BlendedCurve, 0.0f);
// Skip curves with near-zero values — not part of this pose's expression
if (FMath::Abs(Value) < 0.001f) continue;
CurveValues.Add(CurveName, Value);
// Auto-detect naming convention from the very first non-zero curve we encounter
if (!bPosesUseCTRLNaming && PoseExtractedCurveMap.Num() == 0 && CurveValues.Num() == 1)
BlendedCurve.ForEachElement([&CurveValues](const UE::Anim::FCurveElement& Element)
{
bPosesUseCTRLNaming = CurveName.ToString().StartsWith(TEXT("CTRL_"));
if (FMath::Abs(Element.Value) >= 0.001f)
{
CurveValues.Add(Element.Name, Element.Value);
}
});
}
if (bDebug)
{
UE_LOG(LogPS_AI_ConvAgent_LipSync, Log,
TEXT("Pose '%s' (%s): Extracted %d curves via EvaluateCurveData."),
*VisemeName.ToString(), *AnimSeq->GetName(), CurveValues.Num());
}
// Auto-detect naming convention from the first non-zero curve
if (!bPosesUseCTRLNaming && PoseExtractedCurveMap.Num() == 0 && CurveValues.Num() > 0)
{
for (const auto& Pair : CurveValues)
{
bPosesUseCTRLNaming = Pair.Key.ToString().StartsWith(TEXT("CTRL_"));
if (bDebug)
{
UE_LOG(LogPS_AI_ConvAgent_LipSync, Log,
TEXT("Pose curve naming detected: %s (from curve '%s')"),
bPosesUseCTRLNaming ? TEXT("CTRL_expressions_*") : TEXT("ARKit / other"),
*CurveName.ToString());
*Pair.Key.ToString());
}
break;
}
}
if (CurveValues.Num() > 0)
{
PoseExtractedCurveMap.Add(VisemeName, MoveTemp(CurveValues));
if (bDebug)
{
UE_LOG(LogPS_AI_ConvAgent_LipSync, Log,
TEXT("Pose '%s' (%s): Extracted %d non-zero curves."),
*VisemeName.ToString(), *AnimSeq->GetName(),
PoseExtractedCurveMap[VisemeName].Num());
}
}
else
{
// Still add an empty map so we know this viseme was assigned (silence pose)
// Empty map: silence pose or no data available
PoseExtractedCurveMap.Add(VisemeName, TMap<FName, float>());
if (bDebug)
{
UE_LOG(LogPS_AI_ConvAgent_LipSync, Log,
TEXT("Pose '%s' (%s): All curves are zero — neutral/silence pose."),
*VisemeName.ToString(), *AnimSeq->GetName());
}
}
}
@ -503,11 +507,8 @@ void UPS_AI_ConvAgent_LipSyncComponent::InitializePoseMappings()
if (!PoseMap)
{
if (bDebug)
{
UE_LOG(LogPS_AI_ConvAgent_LipSync, Log,
TEXT("No PoseMap assigned — using hardcoded ARKit mapping."));
}
UE_LOG(LogPS_AI_ConvAgent_LipSync, Warning,
TEXT("InitializePoseMappings: PoseMap is NULL — using hardcoded ARKit mapping."));
return;
}
@ -597,6 +598,22 @@ void UPS_AI_ConvAgent_LipSyncComponent::InitializePoseMappings()
TEXT("No phoneme pose AnimSequences assigned — using hardcoded ARKit mapping."));
}
if (bDebug)
{
int32 TotalCurves = 0;
int32 EmptyPoses = 0;
for (const auto& Entry : PoseExtractedCurveMap)
{
TotalCurves += Entry.Value.Num();
if (Entry.Value.Num() == 0) ++EmptyPoses;
}
UE_LOG(LogPS_AI_ConvAgent_LipSync, Log,
TEXT("InitializePoseMappings: PoseMap=%s, Assigned=%d, "
"PoseEntries=%d (empty=%d), TotalCurves=%d, CTRL=%s"),
PoseMap ? *PoseMap->GetName() : TEXT("NULL"),
AssignedCount, PoseExtractedCurveMap.Num(), EmptyPoses,
TotalCurves, bPosesUseCTRLNaming ? TEXT("YES") : TEXT("NO"));
}
}
void UPS_AI_ConvAgent_LipSyncComponent::EndPlay(const EEndPlayReason::Type EndPlayReason)
@ -633,6 +650,45 @@ void UPS_AI_ConvAgent_LipSyncComponent::TickComponent(float DeltaTime, ELevelTic
{
Super::TickComponent(DeltaTime, TickType, ThisTickFunction);
// ── Lazy binding: in packaged builds, BeginPlay may run before the ────────
// ElevenLabsComponent is fully initialized. Retry discovery until bound.
if (!AgentComponent.IsValid())
{
if (AActor* Owner = GetOwner())
{
UPS_AI_ConvAgent_ElevenLabsComponent* Agent =
Owner->FindComponentByClass<UPS_AI_ConvAgent_ElevenLabsComponent>();
if (Agent)
{
AgentComponent = Agent;
AudioDataHandle = Agent->OnAgentAudioData.AddUObject(
this, &UPS_AI_ConvAgent_LipSyncComponent::OnAudioChunkReceived);
Agent->OnAgentPartialResponse.AddDynamic(
this, &UPS_AI_ConvAgent_LipSyncComponent::OnPartialTextReceived);
Agent->OnAgentTextResponse.AddDynamic(
this, &UPS_AI_ConvAgent_LipSyncComponent::OnTextResponseReceived);
Agent->OnAgentInterrupted.AddDynamic(
this, &UPS_AI_ConvAgent_LipSyncComponent::OnAgentInterrupted);
Agent->OnAgentStoppedSpeaking.AddDynamic(
this, &UPS_AI_ConvAgent_LipSyncComponent::OnAgentStopped);
Agent->bEnableAgentPartialResponse = true;
UE_LOG(LogPS_AI_ConvAgent_LipSync, Log,
TEXT("Lip sync (late) bound to agent component on %s."),
*Owner->GetName());
}
}
}
// Also retry caching the facial expression component if it wasn't found initially.
if (!CachedFacialExprComp.IsValid())
{
if (AActor* Owner = GetOwner())
{
CachedFacialExprComp = Owner->FindComponentByClass<UPS_AI_ConvAgent_FacialExpressionComponent>();
}
}
// ── Consume queued viseme analysis frames at the FFT window rate ─────────
// Each 512-sample FFT window at 16kHz = 32ms of audio.
// We consume one queued frame every 32ms to match the original audio timing.
@ -1017,13 +1073,6 @@ void UPS_AI_ConvAgent_LipSyncComponent::OnAudioChunkReceived(const TArray<uint8>
const int16* Samples = reinterpret_cast<const int16*>(PCMData.GetData());
const int32 NumSamples = PCMData.Num() / sizeof(int16);
static bool bFirstChunkLogged = false;
if (!bFirstChunkLogged)
{
UE_LOG(LogPS_AI_ConvAgent_LipSync, Verbose, TEXT("First audio chunk received: %d bytes (%d samples)"), PCMData.Num(), NumSamples);
bFirstChunkLogged = true;
}
FloatBuffer.Reset(NumSamples);
for (int32 i = 0; i < NumSamples; ++i)
{

View File

@ -0,0 +1,3 @@
// Copyright ASTERION. All Rights Reserved.
#include "PS_AI_ConvAgent_LipSyncPoseMap.h"

View File

@ -436,12 +436,11 @@ void UPS_AI_ConvAgent_PostureComponent::TickComponent(
}
}
// ── Debug (every ~2 seconds) ─────────────────────────────────────────
#if !UE_BUILD_SHIPPING
DebugFrameCounter++;
if (DebugFrameCounter % 120 == 0)
// ── Debug (every ~2 seconds, only when bDebug is on) ────────────────
if (bDebug && TargetActor)
{
if (TargetActor)
DebugFrameCounter++;
if (DebugFrameCounter % 120 == 0)
{
const float FacingYaw = Owner->GetActorRotation().Yaw + MeshForwardYawOffset;
const FVector TP = TargetActor->GetActorLocation() + TargetOffset;
@ -450,13 +449,14 @@ void UPS_AI_ConvAgent_PostureComponent::TickComponent(
const float Delta = FMath::FindDeltaAngleDegrees(FacingYaw, TgtYaw);
UE_LOG(LogPS_AI_ConvAgent_Posture, Log,
TEXT("Posture [%s -> %s]: Delta=%.1f | Head=%.1f/%.1f | Eyes=%.1f/%.1f | EyeGap=%.1f"),
TEXT("Posture [%s -> %s]: Delta=%.1f | Head=%.1f/%.1f | Eyes=%.1f/%.1f | Body: enabled=%s TargetYaw=%.1f ActorYaw=%.1f"),
*Owner->GetName(), *TargetActor->GetName(),
Delta,
CurrentHeadYaw, CurrentHeadPitch,
CurrentEyeYaw, CurrentEyePitch,
Delta - CurrentHeadYaw);
bEnableBodyTracking ? TEXT("Y") : TEXT("N"),
TargetBodyWorldYaw,
Owner->GetActorRotation().Yaw);
}
}
#endif
}

View File

@ -96,4 +96,7 @@ public:
private:
UPS_AI_ConvAgent_Settings_ElevenLabs* Settings = nullptr;
/** Copy SSL cacert.pem from plugin Resources to project Content if missing. */
void EnsureSSLCertificates();
};

View File

@ -45,6 +45,9 @@ struct PS_AI_CONVAGENT_API FPS_AI_ConvAgent_EmotionPoseSet
* The component plays the AnimSequence in real-time (looping) to drive
* emotion-based facial expressions (eyes, eyebrows, cheeks, mouth mood).
* Lip sync overrides the mouth-area curves on top.
*
* Curve data is read at runtime via UAnimSequence::EvaluateCurveData()
* which works with both raw (editor) and compressed (cooked) curves.
*/
UCLASS(BlueprintType, Blueprintable, DisplayName = "PS AI ConvAgent Emotion Pose Map")
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_EmotionPoseMap : public UPrimaryDataAsset
@ -52,6 +55,8 @@ class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_EmotionPoseMap : public UPrimaryDataA
GENERATED_BODY()
public:
// ── Emotion Poses ─────────────────────────────────────────────────────────
/** Map of emotions to their AnimSequence sets (Normal / Medium / Extreme).
* Add entries for each emotion your agent uses (Joy, Sadness, Anger, Surprise, Fear, Disgust).
* Neutral is recommended it plays by default at startup (blinking, breathing). */

View File

@ -17,9 +17,11 @@ class UAnimSequence;
* assign your MHF_* AnimSequences once, then reference this single asset
* on every MetaHuman's PS AI ConvAgent Lip Sync component.
*
* The component extracts curve data from each pose at BeginPlay and uses it
* to drive lip sync replacing the hardcoded ARKit blendshape mapping with
* artist-crafted poses that coordinate dozens of facial curves.
* The component extracts curve data from each pose at BeginPlay using
* UAnimSequence::EvaluateCurveData() (works with both raw and compressed
* curves in cooked builds) and uses it to drive lip sync replacing the
* hardcoded ARKit blendshape mapping with artist-crafted poses that
* coordinate dozens of facial curves.
*/
UCLASS(BlueprintType, Blueprintable, DisplayName = "PS AI ConvAgent Lip Sync Pose Map")
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_LipSyncPoseMap : public UPrimaryDataAsset
@ -27,6 +29,7 @@ class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_LipSyncPoseMap : public UPrimaryDataA
GENERATED_BODY()
public:
// ── Phoneme Poses (15 OVR visemes) ───────────────────────────────────────
/** Silence / neutral pose. Mouth at rest. (OVR viseme: sil) */