Compare commits
4 Commits
8bb4371a74
...
e5f40c65ec
| Author | SHA1 | Date | |
|---|---|---|---|
| e5f40c65ec | |||
| 9321e21a3b | |||
| fb641d5aa4 | |||
| 2e96e3c766 |
@ -90,7 +90,7 @@ FontDPI=72
|
|||||||
; ── Generic classes: ElevenLabs → PS_AI_ConvAgent ──
|
; ── Generic classes: ElevenLabs → PS_AI_ConvAgent ──
|
||||||
+ClassRedirects=(OldName="ElevenLabsLipSyncComponent", NewName="PS_AI_ConvAgent_LipSyncComponent")
|
+ClassRedirects=(OldName="ElevenLabsLipSyncComponent", NewName="PS_AI_ConvAgent_LipSyncComponent")
|
||||||
+ClassRedirects=(OldName="ElevenLabsFacialExpressionComponent", NewName="PS_AI_ConvAgent_FacialExpressionComponent")
|
+ClassRedirects=(OldName="ElevenLabsFacialExpressionComponent", NewName="PS_AI_ConvAgent_FacialExpressionComponent")
|
||||||
+ClassRedirects=(OldName="ElevenLabsPostureComponent", NewName="PS_AI_ConvAgent_PostureComponent")
|
+ClassRedirects=(OldName="ElevenLabsPostureComponent", NewName="PS_AI_ConvAgent_GazeComponent")
|
||||||
+ClassRedirects=(OldName="ElevenLabsMicrophoneCaptureComponent", NewName="PS_AI_ConvAgent_MicrophoneCaptureComponent")
|
+ClassRedirects=(OldName="ElevenLabsMicrophoneCaptureComponent", NewName="PS_AI_ConvAgent_MicrophoneCaptureComponent")
|
||||||
+ClassRedirects=(OldName="ElevenLabsLipSyncPoseMap", NewName="PS_AI_ConvAgent_LipSyncPoseMap")
|
+ClassRedirects=(OldName="ElevenLabsLipSyncPoseMap", NewName="PS_AI_ConvAgent_LipSyncPoseMap")
|
||||||
+ClassRedirects=(OldName="ElevenLabsEmotionPoseMap", NewName="PS_AI_ConvAgent_EmotionPoseMap")
|
+ClassRedirects=(OldName="ElevenLabsEmotionPoseMap", NewName="PS_AI_ConvAgent_EmotionPoseMap")
|
||||||
@ -98,7 +98,7 @@ FontDPI=72
|
|||||||
; ── Generic classes: PS_AI_Agent → PS_AI_ConvAgent (intermediate rename) ──
|
; ── Generic classes: PS_AI_Agent → PS_AI_ConvAgent (intermediate rename) ──
|
||||||
+ClassRedirects=(OldName="PS_AI_Agent_LipSyncComponent", NewName="PS_AI_ConvAgent_LipSyncComponent")
|
+ClassRedirects=(OldName="PS_AI_Agent_LipSyncComponent", NewName="PS_AI_ConvAgent_LipSyncComponent")
|
||||||
+ClassRedirects=(OldName="PS_AI_Agent_FacialExpressionComponent", NewName="PS_AI_ConvAgent_FacialExpressionComponent")
|
+ClassRedirects=(OldName="PS_AI_Agent_FacialExpressionComponent", NewName="PS_AI_ConvAgent_FacialExpressionComponent")
|
||||||
+ClassRedirects=(OldName="PS_AI_Agent_PostureComponent", NewName="PS_AI_ConvAgent_PostureComponent")
|
+ClassRedirects=(OldName="PS_AI_Agent_PostureComponent", NewName="PS_AI_ConvAgent_GazeComponent")
|
||||||
+ClassRedirects=(OldName="PS_AI_Agent_MicrophoneCaptureComponent", NewName="PS_AI_ConvAgent_MicrophoneCaptureComponent")
|
+ClassRedirects=(OldName="PS_AI_Agent_MicrophoneCaptureComponent", NewName="PS_AI_ConvAgent_MicrophoneCaptureComponent")
|
||||||
+ClassRedirects=(OldName="PS_AI_Agent_LipSyncPoseMap", NewName="PS_AI_ConvAgent_LipSyncPoseMap")
|
+ClassRedirects=(OldName="PS_AI_Agent_LipSyncPoseMap", NewName="PS_AI_ConvAgent_LipSyncPoseMap")
|
||||||
+ClassRedirects=(OldName="PS_AI_Agent_EmotionPoseMap", NewName="PS_AI_ConvAgent_EmotionPoseMap")
|
+ClassRedirects=(OldName="PS_AI_Agent_EmotionPoseMap", NewName="PS_AI_ConvAgent_EmotionPoseMap")
|
||||||
@ -114,18 +114,23 @@ FontDPI=72
|
|||||||
; ── AnimNode structs ──
|
; ── AnimNode structs ──
|
||||||
+StructRedirects=(OldName="AnimNode_ElevenLabsLipSync", NewName="AnimNode_PS_AI_ConvAgent_LipSync")
|
+StructRedirects=(OldName="AnimNode_ElevenLabsLipSync", NewName="AnimNode_PS_AI_ConvAgent_LipSync")
|
||||||
+StructRedirects=(OldName="AnimNode_ElevenLabsFacialExpression", NewName="AnimNode_PS_AI_ConvAgent_FacialExpression")
|
+StructRedirects=(OldName="AnimNode_ElevenLabsFacialExpression", NewName="AnimNode_PS_AI_ConvAgent_FacialExpression")
|
||||||
+StructRedirects=(OldName="AnimNode_ElevenLabsPosture", NewName="AnimNode_PS_AI_ConvAgent_Posture")
|
+StructRedirects=(OldName="AnimNode_ElevenLabsPosture", NewName="AnimNode_PS_AI_ConvAgent_Gaze")
|
||||||
+StructRedirects=(OldName="AnimNode_PS_AI_Agent_LipSync", NewName="AnimNode_PS_AI_ConvAgent_LipSync")
|
+StructRedirects=(OldName="AnimNode_PS_AI_Agent_LipSync", NewName="AnimNode_PS_AI_ConvAgent_LipSync")
|
||||||
+StructRedirects=(OldName="AnimNode_PS_AI_Agent_FacialExpression", NewName="AnimNode_PS_AI_ConvAgent_FacialExpression")
|
+StructRedirects=(OldName="AnimNode_PS_AI_Agent_FacialExpression", NewName="AnimNode_PS_AI_ConvAgent_FacialExpression")
|
||||||
+StructRedirects=(OldName="AnimNode_PS_AI_Agent_Posture", NewName="AnimNode_PS_AI_ConvAgent_Posture")
|
+StructRedirects=(OldName="AnimNode_PS_AI_Agent_Posture", NewName="AnimNode_PS_AI_ConvAgent_Gaze")
|
||||||
|
|
||||||
; ── AnimGraphNode classes ──
|
; ── AnimGraphNode classes ──
|
||||||
+ClassRedirects=(OldName="AnimGraphNode_ElevenLabsLipSync", NewName="AnimGraphNode_PS_AI_ConvAgent_LipSync")
|
+ClassRedirects=(OldName="AnimGraphNode_ElevenLabsLipSync", NewName="AnimGraphNode_PS_AI_ConvAgent_LipSync")
|
||||||
+ClassRedirects=(OldName="AnimGraphNode_ElevenLabsFacialExpression", NewName="AnimGraphNode_PS_AI_ConvAgent_FacialExpression")
|
+ClassRedirects=(OldName="AnimGraphNode_ElevenLabsFacialExpression", NewName="AnimGraphNode_PS_AI_ConvAgent_FacialExpression")
|
||||||
+ClassRedirects=(OldName="AnimGraphNode_ElevenLabsPosture", NewName="AnimGraphNode_PS_AI_ConvAgent_Posture")
|
+ClassRedirects=(OldName="AnimGraphNode_ElevenLabsPosture", NewName="AnimGraphNode_PS_AI_ConvAgent_Gaze")
|
||||||
+ClassRedirects=(OldName="AnimGraphNode_PS_AI_Agent_LipSync", NewName="AnimGraphNode_PS_AI_ConvAgent_LipSync")
|
+ClassRedirects=(OldName="AnimGraphNode_PS_AI_Agent_LipSync", NewName="AnimGraphNode_PS_AI_ConvAgent_LipSync")
|
||||||
+ClassRedirects=(OldName="AnimGraphNode_PS_AI_Agent_FacialExpression", NewName="AnimGraphNode_PS_AI_ConvAgent_FacialExpression")
|
+ClassRedirects=(OldName="AnimGraphNode_PS_AI_Agent_FacialExpression", NewName="AnimGraphNode_PS_AI_ConvAgent_FacialExpression")
|
||||||
+ClassRedirects=(OldName="AnimGraphNode_PS_AI_Agent_Posture", NewName="AnimGraphNode_PS_AI_ConvAgent_Posture")
|
+ClassRedirects=(OldName="AnimGraphNode_PS_AI_Agent_Posture", NewName="AnimGraphNode_PS_AI_ConvAgent_Gaze")
|
||||||
|
|
||||||
|
; ── Posture → Gaze rename ──
|
||||||
|
+ClassRedirects=(OldName="PS_AI_ConvAgent_PostureComponent", NewName="PS_AI_ConvAgent_GazeComponent")
|
||||||
|
+StructRedirects=(OldName="AnimNode_PS_AI_ConvAgent_Posture", NewName="AnimNode_PS_AI_ConvAgent_Gaze")
|
||||||
|
+ClassRedirects=(OldName="AnimGraphNode_PS_AI_ConvAgent_Posture", NewName="AnimGraphNode_PS_AI_ConvAgent_Gaze")
|
||||||
|
|
||||||
; ── Factory classes ──
|
; ── Factory classes ──
|
||||||
+ClassRedirects=(OldName="ElevenLabsLipSyncPoseMapFactory", NewName="PS_AI_ConvAgent_LipSyncPoseMapFactory")
|
+ClassRedirects=(OldName="ElevenLabsLipSyncPoseMapFactory", NewName="PS_AI_ConvAgent_LipSyncPoseMapFactory")
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@ -127,7 +127,10 @@ void FAnimNode_PS_AI_ConvAgent_BodyExpression::Evaluate_AnyThread(FPoseContext&
|
|||||||
|
|
||||||
const bool bHavePrevPose = bCrossfading && PrevPose.GetNumBones() > 0;
|
const bool bHavePrevPose = bCrossfading && PrevPose.GetNumBones() > 0;
|
||||||
|
|
||||||
// ── Per-bone blend ──────────────────────────────────────────────────────
|
// ── Per-bone blend (override mode) ──────────────────────────────────────
|
||||||
|
// Lerp each bone from the upstream pose toward the expression pose,
|
||||||
|
// weighted by the bone mask and activation alpha.
|
||||||
|
// ExcludeBones (default: neck_01) prevents conflicts with Gaze/Posture.
|
||||||
for (int32 CompactIdx = 0; CompactIdx < NumBones; ++CompactIdx)
|
for (int32 CompactIdx = 0; CompactIdx < NumBones; ++CompactIdx)
|
||||||
{
|
{
|
||||||
if (BoneMask[CompactIdx] <= 0.0f)
|
if (BoneMask[CompactIdx] <= 0.0f)
|
||||||
@ -136,18 +139,19 @@ void FAnimNode_PS_AI_ConvAgent_BodyExpression::Evaluate_AnyThread(FPoseContext&
|
|||||||
const FCompactPoseBoneIndex BoneIdx(CompactIdx);
|
const FCompactPoseBoneIndex BoneIdx(CompactIdx);
|
||||||
const float W = BoneMask[CompactIdx] * FinalWeight;
|
const float W = BoneMask[CompactIdx] * FinalWeight;
|
||||||
|
|
||||||
|
// Compute the expression pose (handle crossfade between prev and active)
|
||||||
|
FTransform ExpressionPose;
|
||||||
if (bHavePrevPose)
|
if (bHavePrevPose)
|
||||||
{
|
{
|
||||||
// Crossfade between previous and active emotion poses
|
ExpressionPose.Blend(PrevPose[BoneIdx], ActivePose[BoneIdx], CachedSnapshot.CrossfadeAlpha);
|
||||||
FTransform BlendedEmotion;
|
|
||||||
BlendedEmotion.Blend(PrevPose[BoneIdx], ActivePose[BoneIdx], CachedSnapshot.CrossfadeAlpha);
|
|
||||||
Output.Pose[BoneIdx].BlendWith(BlendedEmotion, W);
|
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
// Direct blend with active emotion pose
|
ExpressionPose = ActivePose[BoneIdx];
|
||||||
Output.Pose[BoneIdx].BlendWith(ActivePose[BoneIdx], W);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Override: replace (lerp) upstream toward expression pose
|
||||||
|
Output.Pose[BoneIdx].BlendWith(ExpressionPose, W);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -169,59 +173,111 @@ void FAnimNode_PS_AI_ConvAgent_BodyExpression::BuildBoneMask(const FBoneContaine
|
|||||||
BoneMask.SetNumZeroed(NumBones);
|
BoneMask.SetNumZeroed(NumBones);
|
||||||
bBoneMaskValid = false;
|
bBoneMaskValid = false;
|
||||||
|
|
||||||
// Full body mode: all bones get weight 1.0
|
const FReferenceSkeleton& RefSkel = RequiredBones.GetReferenceSkeleton();
|
||||||
|
const TArray<FBoneIndexType>& BoneIndices = RequiredBones.GetBoneIndicesArray();
|
||||||
|
|
||||||
|
// ── Step 1: Build initial mask (include) ──────────────────────────────
|
||||||
|
|
||||||
if (!bUpperBodyOnly)
|
if (!bUpperBodyOnly)
|
||||||
{
|
{
|
||||||
|
// Full body mode: all bones get weight 1.0
|
||||||
for (int32 i = 0; i < NumBones; ++i)
|
for (int32 i = 0; i < NumBones; ++i)
|
||||||
{
|
{
|
||||||
BoneMask[i] = 1.0f;
|
BoneMask[i] = 1.0f;
|
||||||
}
|
}
|
||||||
bBoneMaskValid = (NumBones > 0);
|
|
||||||
|
|
||||||
UE_LOG(LogPS_AI_ConvAgent_BodyExprAnimNode, Log,
|
|
||||||
TEXT("Bone mask built: FULL BODY (%d bones)."), NumBones);
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
else
|
||||||
// Upper body mode: only BlendRootBone descendants
|
|
||||||
const FReferenceSkeleton& RefSkel = RequiredBones.GetReferenceSkeleton();
|
|
||||||
const int32 RootMeshIdx = RefSkel.FindBoneIndex(BlendRootBone);
|
|
||||||
|
|
||||||
if (RootMeshIdx == INDEX_NONE)
|
|
||||||
{
|
{
|
||||||
UE_LOG(LogPS_AI_ConvAgent_BodyExprAnimNode, Warning,
|
// Upper body mode: only BlendRootBone descendants
|
||||||
TEXT("BlendRootBone '%s' not found in skeleton. Body expression disabled."),
|
const int32 RootMeshIdx = RefSkel.FindBoneIndex(BlendRootBone);
|
||||||
*BlendRootBone.ToString());
|
|
||||||
return;
|
if (RootMeshIdx == INDEX_NONE)
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_BodyExprAnimNode, Warning,
|
||||||
|
TEXT("BlendRootBone '%s' not found in skeleton. Body expression disabled."),
|
||||||
|
*BlendRootBone.ToString());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int32 CompactIdx = 0; CompactIdx < NumBones; ++CompactIdx)
|
||||||
|
{
|
||||||
|
const int32 MeshIdx = static_cast<int32>(BoneIndices[CompactIdx]);
|
||||||
|
|
||||||
|
// Walk up the parent chain: if BlendRootBone is an ancestor
|
||||||
|
// (or is this bone itself), mark it for blending.
|
||||||
|
int32 Current = MeshIdx;
|
||||||
|
while (Current != INDEX_NONE)
|
||||||
|
{
|
||||||
|
if (Current == RootMeshIdx)
|
||||||
|
{
|
||||||
|
BoneMask[CompactIdx] = 1.0f;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Current = RefSkel.GetParentIndex(Current);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const TArray<FBoneIndexType>& BoneIndices = RequiredBones.GetBoneIndicesArray();
|
// ── Step 2: Exclude bone subtrees ────────────────────────────────────
|
||||||
|
// For each exclude bone, zero out it and all its descendants.
|
||||||
|
// This prevents conflicts with Gaze/Posture on neck/head bones.
|
||||||
|
|
||||||
|
int32 ExcludedCount = 0;
|
||||||
|
if (ExcludeBones.Num() > 0)
|
||||||
|
{
|
||||||
|
// Collect mesh indices for all exclude roots
|
||||||
|
TArray<int32> ExcludeRootIndices;
|
||||||
|
for (const FName& BoneName : ExcludeBones)
|
||||||
|
{
|
||||||
|
const int32 ExclIdx = RefSkel.FindBoneIndex(BoneName);
|
||||||
|
if (ExclIdx != INDEX_NONE)
|
||||||
|
{
|
||||||
|
ExcludeRootIndices.Add(ExclIdx);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_BodyExprAnimNode, Warning,
|
||||||
|
TEXT("ExcludeBone '%s' not found in skeleton — ignored."),
|
||||||
|
*BoneName.ToString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Zero out any bone whose parent chain includes an exclude root
|
||||||
|
for (int32 CompactIdx = 0; CompactIdx < NumBones; ++CompactIdx)
|
||||||
|
{
|
||||||
|
if (BoneMask[CompactIdx] <= 0.0f)
|
||||||
|
continue; // Already excluded or not included
|
||||||
|
|
||||||
|
const int32 MeshIdx = static_cast<int32>(BoneIndices[CompactIdx]);
|
||||||
|
int32 Current = MeshIdx;
|
||||||
|
while (Current != INDEX_NONE)
|
||||||
|
{
|
||||||
|
if (ExcludeRootIndices.Contains(Current))
|
||||||
|
{
|
||||||
|
BoneMask[CompactIdx] = 0.0f;
|
||||||
|
++ExcludedCount;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Current = RefSkel.GetParentIndex(Current);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Count final active bones ─────────────────────────────────────────
|
||||||
|
|
||||||
int32 MaskedCount = 0;
|
int32 MaskedCount = 0;
|
||||||
for (int32 CompactIdx = 0; CompactIdx < NumBones; ++CompactIdx)
|
for (int32 i = 0; i < NumBones; ++i)
|
||||||
{
|
{
|
||||||
const int32 MeshIdx = static_cast<int32>(BoneIndices[CompactIdx]);
|
if (BoneMask[i] > 0.0f)
|
||||||
|
++MaskedCount;
|
||||||
// Walk up the parent chain: if BlendRootBone is an ancestor
|
|
||||||
// (or is this bone itself), mark it for blending.
|
|
||||||
int32 Current = MeshIdx;
|
|
||||||
while (Current != INDEX_NONE)
|
|
||||||
{
|
|
||||||
if (Current == RootMeshIdx)
|
|
||||||
{
|
|
||||||
BoneMask[CompactIdx] = 1.0f;
|
|
||||||
++MaskedCount;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Current = RefSkel.GetParentIndex(Current);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
bBoneMaskValid = (MaskedCount > 0);
|
bBoneMaskValid = (MaskedCount > 0);
|
||||||
|
|
||||||
UE_LOG(LogPS_AI_ConvAgent_BodyExprAnimNode, Log,
|
UE_LOG(LogPS_AI_ConvAgent_BodyExprAnimNode, Log,
|
||||||
TEXT("Bone mask built: %d/%d bones from root '%s'."),
|
TEXT("Bone mask built: %d/%d active bones (root='%s', excluded=%d from %d subtrees)."),
|
||||||
MaskedCount, NumBones, *BlendRootBone.ToString());
|
MaskedCount, NumBones, *BlendRootBone.ToString(),
|
||||||
|
ExcludedCount, ExcludeBones.Num());
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|||||||
@ -89,17 +89,15 @@ void FAnimNode_PS_AI_ConvAgent_LipSync::Evaluate_AnyThread(FPoseContext& Output)
|
|||||||
|
|
||||||
// ── Inject lip sync curves into the pose output ──────────────────────
|
// ── Inject lip sync curves into the pose output ──────────────────────
|
||||||
//
|
//
|
||||||
// IMPORTANT: Always write ALL curves that lip sync has ever touched,
|
// While lip sync is producing curves: write them all (including 0s)
|
||||||
// including at 0.0. If we skip near-zero curves, upstream animation
|
// and track every curve name in KnownCurveNames.
|
||||||
// values (idle expressions, breathing, etc.) leak through and cause
|
|
||||||
// visible pops when lip sync curves cross the threshold.
|
|
||||||
//
|
//
|
||||||
// Strategy:
|
// When lip sync goes silent (CachedCurves empty): release immediately.
|
||||||
// - While lip sync is producing curves: write them all (including 0s)
|
// The component's SpeechBlendAlpha already handles smooth fade-out,
|
||||||
// and track every curve name in KnownCurveNames.
|
// so by the time CachedCurves becomes empty, values have already
|
||||||
// - After lip sync goes silent: keep writing 0s for a grace period
|
// decayed to near-zero. Releasing immediately allows the upstream
|
||||||
// (30 frames ≈ 0.5s) so the upstream can blend back in smoothly
|
// FacialExpressionComponent's emotion curves (including mouth) to
|
||||||
// via the component's activation alpha, then release.
|
// flow through without being overwritten by zeros.
|
||||||
|
|
||||||
if (CachedCurves.Num() > 0)
|
if (CachedCurves.Num() > 0)
|
||||||
{
|
{
|
||||||
@ -113,7 +111,9 @@ void FAnimNode_PS_AI_ConvAgent_LipSync::Evaluate_AnyThread(FPoseContext& Output)
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Zero any known curves NOT in the current frame
|
// Zero any known curves NOT in the current frame
|
||||||
// (e.g. a blendshape that was active last frame but decayed away)
|
// (e.g. a blendshape that was active last frame but decayed away).
|
||||||
|
// This prevents upstream animation values from leaking through
|
||||||
|
// during active speech when a curve temporarily goes to zero.
|
||||||
for (const FName& Name : KnownCurveNames)
|
for (const FName& Name : KnownCurveNames)
|
||||||
{
|
{
|
||||||
if (!CachedCurves.Contains(Name))
|
if (!CachedCurves.Contains(Name))
|
||||||
@ -124,23 +124,11 @@ void FAnimNode_PS_AI_ConvAgent_LipSync::Evaluate_AnyThread(FPoseContext& Output)
|
|||||||
}
|
}
|
||||||
else if (KnownCurveNames.Num() > 0)
|
else if (KnownCurveNames.Num() > 0)
|
||||||
{
|
{
|
||||||
// Lip sync went silent — keep zeroing known curves for a grace
|
// Lip sync went silent — release immediately.
|
||||||
// period so upstream values don't pop in abruptly.
|
// The smooth fade-out was handled at the component level
|
||||||
++FramesSinceLastActive;
|
// (SpeechBlendAlpha), so upstream emotion curves can take over.
|
||||||
|
KnownCurveNames.Reset();
|
||||||
if (FramesSinceLastActive < 30)
|
FramesSinceLastActive = 0;
|
||||||
{
|
|
||||||
for (const FName& Name : KnownCurveNames)
|
|
||||||
{
|
|
||||||
Output.Curve.Set(Name, 0.0f);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
// Grace period over — release curves, let upstream through
|
|
||||||
KnownCurveNames.Reset();
|
|
||||||
FramesSinceLastActive = 0;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -4,9 +4,16 @@
|
|||||||
#include "PS_AI_ConvAgent_ElevenLabsComponent.h"
|
#include "PS_AI_ConvAgent_ElevenLabsComponent.h"
|
||||||
#include "PS_AI_ConvAgent_BodyPoseMap.h"
|
#include "PS_AI_ConvAgent_BodyPoseMap.h"
|
||||||
#include "Animation/AnimSequence.h"
|
#include "Animation/AnimSequence.h"
|
||||||
|
#include "Engine/Engine.h"
|
||||||
|
|
||||||
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_BodyExpr, Log, All);
|
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_BodyExpr, Log, All);
|
||||||
|
|
||||||
|
static TAutoConsoleVariable<int32> CVarDebugBodyExpr(
|
||||||
|
TEXT("ps.ai.ConvAgent.Debug.BodyExpr"),
|
||||||
|
-1,
|
||||||
|
TEXT("Debug HUD for BodyExpression. -1=use property, 0=off, 1-3=verbosity."),
|
||||||
|
ECVF_Default);
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
// Construction
|
// Construction
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@ -43,7 +50,7 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::BeginPlay()
|
|||||||
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationConnected);
|
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationConnected);
|
||||||
Agent->OnAgentDisconnected.AddDynamic(
|
Agent->OnAgentDisconnected.AddDynamic(
|
||||||
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationDisconnected);
|
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationDisconnected);
|
||||||
Agent->OnAgentStartedSpeaking.AddDynamic(
|
Agent->OnAudioPlaybackStarted.AddDynamic(
|
||||||
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStarted);
|
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStarted);
|
||||||
Agent->OnAgentStoppedSpeaking.AddDynamic(
|
Agent->OnAgentStoppedSpeaking.AddDynamic(
|
||||||
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStopped);
|
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStopped);
|
||||||
@ -96,7 +103,7 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::EndPlay(const EEndPlayReason::Typ
|
|||||||
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationConnected);
|
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationConnected);
|
||||||
AgentComponent->OnAgentDisconnected.RemoveDynamic(
|
AgentComponent->OnAgentDisconnected.RemoveDynamic(
|
||||||
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationDisconnected);
|
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationDisconnected);
|
||||||
AgentComponent->OnAgentStartedSpeaking.RemoveDynamic(
|
AgentComponent->OnAudioPlaybackStarted.RemoveDynamic(
|
||||||
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStarted);
|
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStarted);
|
||||||
AgentComponent->OnAgentStoppedSpeaking.RemoveDynamic(
|
AgentComponent->OnAgentStoppedSpeaking.RemoveDynamic(
|
||||||
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStopped);
|
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStopped);
|
||||||
@ -119,6 +126,12 @@ const TArray<TObjectPtr<UAnimSequence>>* UPS_AI_ConvAgent_BodyExpressionComponen
|
|||||||
if (!BodyPoseMap) return nullptr;
|
if (!BodyPoseMap) return nullptr;
|
||||||
|
|
||||||
const FPS_AI_ConvAgent_BodyAnimList* AnimList = BodyPoseMap->BodyPoses.Find(Emotion);
|
const FPS_AI_ConvAgent_BodyAnimList* AnimList = BodyPoseMap->BodyPoses.Find(Emotion);
|
||||||
|
|
||||||
|
// Fallback to Neutral if the requested emotion has no entry in the data asset
|
||||||
|
if (!AnimList && Emotion != EPS_AI_ConvAgent_Emotion::Neutral)
|
||||||
|
{
|
||||||
|
AnimList = BodyPoseMap->BodyPoses.Find(EPS_AI_ConvAgent_Emotion::Neutral);
|
||||||
|
}
|
||||||
if (!AnimList) return nullptr;
|
if (!AnimList) return nullptr;
|
||||||
|
|
||||||
if (!bSpeaking)
|
if (!bSpeaking)
|
||||||
@ -178,15 +191,15 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::SwitchToNewAnim(UAnimSequence* Ne
|
|||||||
if (!bForce && NewAnim == ActiveAnim) return;
|
if (!bForce && NewAnim == ActiveAnim) return;
|
||||||
if (!NewAnim) return;
|
if (!NewAnim) return;
|
||||||
|
|
||||||
// Current active becomes previous for crossfade
|
// Always start a fresh crossfade from whatever is currently active.
|
||||||
|
// If a crossfade was in progress, the old PrevAnim is lost, but the
|
||||||
|
// transition FROM the current ActiveAnim (at its current time) to the
|
||||||
|
// new anim will always be smooth and predictable.
|
||||||
PrevAnim = ActiveAnim;
|
PrevAnim = ActiveAnim;
|
||||||
PrevPlaybackTime = ActivePlaybackTime;
|
PrevPlaybackTime = ActivePlaybackTime;
|
||||||
|
|
||||||
// New anim starts from the beginning
|
|
||||||
ActiveAnim = NewAnim;
|
ActiveAnim = NewAnim;
|
||||||
ActivePlaybackTime = 0.0f;
|
ActivePlaybackTime = 0.0f;
|
||||||
|
|
||||||
// Begin crossfade
|
|
||||||
CrossfadeAlpha = 0.0f;
|
CrossfadeAlpha = 0.0f;
|
||||||
|
|
||||||
if (bDebug && DebugVerbosity >= 1)
|
if (bDebug && DebugVerbosity >= 1)
|
||||||
@ -221,6 +234,8 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationConnected(
|
|||||||
{
|
{
|
||||||
bActive = true;
|
bActive = true;
|
||||||
bIsSpeaking = false;
|
bIsSpeaking = false;
|
||||||
|
LastEventName = TEXT("Connected");
|
||||||
|
LastEventWorldTime = GetWorld() ? GetWorld()->GetTimeSeconds() : 0.0f;
|
||||||
|
|
||||||
// Start with an idle anim
|
// Start with an idle anim
|
||||||
PickAndSwitchAnim();
|
PickAndSwitchAnim();
|
||||||
@ -237,6 +252,8 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationDisconnected(
|
|||||||
{
|
{
|
||||||
bActive = false;
|
bActive = false;
|
||||||
bIsSpeaking = false;
|
bIsSpeaking = false;
|
||||||
|
LastEventName = TEXT("Disconnected");
|
||||||
|
LastEventWorldTime = GetWorld() ? GetWorld()->GetTimeSeconds() : 0.0f;
|
||||||
|
|
||||||
if (bDebug)
|
if (bDebug)
|
||||||
{
|
{
|
||||||
@ -252,6 +269,8 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationDisconnected(
|
|||||||
void UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStarted()
|
void UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStarted()
|
||||||
{
|
{
|
||||||
bIsSpeaking = true;
|
bIsSpeaking = true;
|
||||||
|
LastEventName = TEXT("SpeakStart");
|
||||||
|
LastEventWorldTime = GetWorld() ? GetWorld()->GetTimeSeconds() : 0.0f;
|
||||||
|
|
||||||
// Crossfade from idle anim to a speaking anim
|
// Crossfade from idle anim to a speaking anim
|
||||||
PickAndSwitchAnim();
|
PickAndSwitchAnim();
|
||||||
@ -266,6 +285,8 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStarted()
|
|||||||
void UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStopped()
|
void UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStopped()
|
||||||
{
|
{
|
||||||
bIsSpeaking = false;
|
bIsSpeaking = false;
|
||||||
|
LastEventName = TEXT("SpeakStop");
|
||||||
|
LastEventWorldTime = GetWorld() ? GetWorld()->GetTimeSeconds() : 0.0f;
|
||||||
|
|
||||||
// Crossfade from speaking anim to an idle anim
|
// Crossfade from speaking anim to an idle anim
|
||||||
PickAndSwitchAnim();
|
PickAndSwitchAnim();
|
||||||
@ -280,6 +301,8 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStopped()
|
|||||||
void UPS_AI_ConvAgent_BodyExpressionComponent::OnInterrupted()
|
void UPS_AI_ConvAgent_BodyExpressionComponent::OnInterrupted()
|
||||||
{
|
{
|
||||||
bIsSpeaking = false;
|
bIsSpeaking = false;
|
||||||
|
LastEventName = TEXT("Interrupted");
|
||||||
|
LastEventWorldTime = GetWorld() ? GetWorld()->GetTimeSeconds() : 0.0f;
|
||||||
|
|
||||||
// Crossfade to idle anim
|
// Crossfade to idle anim
|
||||||
PickAndSwitchAnim();
|
PickAndSwitchAnim();
|
||||||
@ -303,6 +326,9 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::OnEmotionChanged(
|
|||||||
|
|
||||||
ActiveEmotion = Emotion;
|
ActiveEmotion = Emotion;
|
||||||
ActiveEmotionIntensity = Intensity;
|
ActiveEmotionIntensity = Intensity;
|
||||||
|
LastEventName = FString::Printf(TEXT("Emotion:%s"),
|
||||||
|
*UEnum::GetDisplayValueAsText(Emotion).ToString());
|
||||||
|
LastEventWorldTime = GetWorld() ? GetWorld()->GetTimeSeconds() : 0.0f;
|
||||||
|
|
||||||
// Pick a new anim from the appropriate list for the new emotion
|
// Pick a new anim from the appropriate list for the new emotion
|
||||||
PickAndSwitchAnim();
|
PickAndSwitchAnim();
|
||||||
@ -343,7 +369,7 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::TickComponent(
|
|||||||
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationConnected);
|
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationConnected);
|
||||||
Agent->OnAgentDisconnected.AddDynamic(
|
Agent->OnAgentDisconnected.AddDynamic(
|
||||||
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationDisconnected);
|
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnConversationDisconnected);
|
||||||
Agent->OnAgentStartedSpeaking.AddDynamic(
|
Agent->OnAudioPlaybackStarted.AddDynamic(
|
||||||
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStarted);
|
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStarted);
|
||||||
Agent->OnAgentStoppedSpeaking.AddDynamic(
|
Agent->OnAgentStoppedSpeaking.AddDynamic(
|
||||||
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStopped);
|
this, &UPS_AI_ConvAgent_BodyExpressionComponent::OnSpeakingStopped);
|
||||||
@ -358,9 +384,12 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::TickComponent(
|
|||||||
const float TargetAlpha = bActive ? 1.0f : 0.0f;
|
const float TargetAlpha = bActive ? 1.0f : 0.0f;
|
||||||
if (!FMath::IsNearlyEqual(CurrentActiveAlpha, TargetAlpha, 0.001f))
|
if (!FMath::IsNearlyEqual(CurrentActiveAlpha, TargetAlpha, 0.001f))
|
||||||
{
|
{
|
||||||
const float BlendSpeed = 1.0f / FMath::Max(ActivationBlendDuration, 0.01f);
|
// Exponential ease-out: fast start, gradual approach to target.
|
||||||
CurrentActiveAlpha = FMath::FInterpConstantTo(
|
// Factor of 3 compensates for FInterpTo's exponential decay
|
||||||
CurrentActiveAlpha, TargetAlpha, DeltaTime, BlendSpeed);
|
// reaching ~95% in ActivationBlendDuration seconds.
|
||||||
|
const float InterpSpeed = 3.0f / FMath::Max(ActivationBlendDuration, 0.01f);
|
||||||
|
CurrentActiveAlpha = FMath::FInterpTo(
|
||||||
|
CurrentActiveAlpha, TargetAlpha, DeltaTime, InterpSpeed);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -416,6 +445,8 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::TickComponent(
|
|||||||
if (NewAnim)
|
if (NewAnim)
|
||||||
{
|
{
|
||||||
SwitchToNewAnim(NewAnim, true);
|
SwitchToNewAnim(NewAnim, true);
|
||||||
|
LastEventName = TEXT("AutoCycle");
|
||||||
|
LastEventWorldTime = GetWorld() ? GetWorld()->GetTimeSeconds() : 0.0f;
|
||||||
|
|
||||||
if (bDebug && DebugVerbosity >= 2)
|
if (bDebug && DebugVerbosity >= 2)
|
||||||
{
|
{
|
||||||
@ -431,12 +462,16 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::TickComponent(
|
|||||||
|
|
||||||
if (CrossfadeAlpha < 1.0f)
|
if (CrossfadeAlpha < 1.0f)
|
||||||
{
|
{
|
||||||
const float BlendSpeed = 1.0f / FMath::Max(0.05f, EmotionBlendDuration);
|
// Exponential ease-out: fast start, gradual approach to 1.0.
|
||||||
CrossfadeAlpha = FMath::Min(1.0f, CrossfadeAlpha + DeltaTime * BlendSpeed);
|
// Factor of 3 compensates for FInterpTo's exponential decay
|
||||||
|
// reaching ~95% in EmotionBlendDuration seconds.
|
||||||
|
const float InterpSpeed = 3.0f / FMath::Max(0.05f, EmotionBlendDuration);
|
||||||
|
CrossfadeAlpha = FMath::FInterpTo(CrossfadeAlpha, 1.0f, DeltaTime, InterpSpeed);
|
||||||
|
|
||||||
// Crossfade complete — release previous anim
|
// Snap to 1.0 when close enough, release previous anim
|
||||||
if (CrossfadeAlpha >= 1.0f)
|
if (CrossfadeAlpha > 0.999f)
|
||||||
{
|
{
|
||||||
|
CrossfadeAlpha = 1.0f;
|
||||||
PrevAnim = nullptr;
|
PrevAnim = nullptr;
|
||||||
PrevPlaybackTime = 0.0f;
|
PrevPlaybackTime = 0.0f;
|
||||||
}
|
}
|
||||||
@ -449,8 +484,82 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::TickComponent(
|
|||||||
CurrentSnapshot.PrevAnim = PrevAnim;
|
CurrentSnapshot.PrevAnim = PrevAnim;
|
||||||
CurrentSnapshot.ActiveTime = ActivePlaybackTime;
|
CurrentSnapshot.ActiveTime = ActivePlaybackTime;
|
||||||
CurrentSnapshot.PrevTime = PrevPlaybackTime;
|
CurrentSnapshot.PrevTime = PrevPlaybackTime;
|
||||||
|
// FInterpTo already provides exponential easing — pass alpha directly.
|
||||||
CurrentSnapshot.CrossfadeAlpha = CrossfadeAlpha;
|
CurrentSnapshot.CrossfadeAlpha = CrossfadeAlpha;
|
||||||
CurrentSnapshot.ActivationAlpha = CurrentActiveAlpha;
|
CurrentSnapshot.ActivationAlpha = CurrentActiveAlpha;
|
||||||
CurrentSnapshot.BlendWeight = BlendWeight;
|
CurrentSnapshot.BlendWeight = BlendWeight;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── On-screen debug HUD ───────────────────────────────────────────────
|
||||||
|
{
|
||||||
|
const int32 CVarVal = CVarDebugBodyExpr.GetValueOnGameThread();
|
||||||
|
const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug;
|
||||||
|
const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity;
|
||||||
|
if (bShowDebug && EffectiveVerbosity >= 1)
|
||||||
|
{
|
||||||
|
DrawDebugHUD();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// On-screen debug display
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_BodyExpressionComponent::DrawDebugHUD() const
|
||||||
|
{
|
||||||
|
if (!GEngine) return;
|
||||||
|
|
||||||
|
const float WorldTime = GetWorld() ? GetWorld()->GetTimeSeconds() : 0.0f;
|
||||||
|
const float EventAge = WorldTime - LastEventWorldTime;
|
||||||
|
|
||||||
|
// Active anim name
|
||||||
|
FString ActiveName = ActiveAnim ? ActiveAnim->GetName() : TEXT("(none)");
|
||||||
|
FString PrevName = PrevAnim ? PrevAnim->GetName() : TEXT("---");
|
||||||
|
|
||||||
|
// State label
|
||||||
|
FString StateStr;
|
||||||
|
if (!bActive)
|
||||||
|
StateStr = TEXT("INACTIVE");
|
||||||
|
else if (bIsSpeaking)
|
||||||
|
StateStr = TEXT("SPEAKING");
|
||||||
|
else
|
||||||
|
StateStr = TEXT("IDLE");
|
||||||
|
|
||||||
|
// Event label with age
|
||||||
|
FString EventStr = LastEventName.IsEmpty()
|
||||||
|
? TEXT("---")
|
||||||
|
: FString::Printf(TEXT("%s (%.1fs ago)"), *LastEventName, EventAge);
|
||||||
|
|
||||||
|
// Use key offset to avoid colliding with other debug messages
|
||||||
|
// Keys 2000-2010 reserved for BodyExpression
|
||||||
|
const int32 BaseKey = 2000;
|
||||||
|
const float DisplayTime = 1.0f;
|
||||||
|
const FColor MainColor = FColor::Cyan;
|
||||||
|
const FColor WarnColor = FColor::Yellow;
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT("=== BODY EXPR: %s ==="), *StateStr));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" ActivationAlpha: %.3f (target: %s)"),
|
||||||
|
CurrentActiveAlpha, bActive ? TEXT("1") : TEXT("0")));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Active: %s t=%.2f"), *ActiveName, ActivePlaybackTime));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 3, DisplayTime,
|
||||||
|
CrossfadeAlpha < 1.0f ? WarnColor : MainColor,
|
||||||
|
FString::Printf(TEXT(" Crossfade: %.3f Prev: %s"),
|
||||||
|
CrossfadeAlpha, *PrevName));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 4, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Emotion: %s (%s) Weight: %.2f"),
|
||||||
|
*UEnum::GetDisplayValueAsText(ActiveEmotion).ToString(),
|
||||||
|
*UEnum::GetDisplayValueAsText(ActiveEmotionIntensity).ToString(),
|
||||||
|
BlendWeight));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 5, DisplayTime,
|
||||||
|
EventAge < 1.0f ? FColor::Green : MainColor,
|
||||||
|
FString::Printf(TEXT(" LastEvent: %s"), *EventStr));
|
||||||
}
|
}
|
||||||
|
|||||||
@ -20,6 +20,12 @@
|
|||||||
|
|
||||||
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_ElevenLabs, Log, All);
|
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_ElevenLabs, Log, All);
|
||||||
|
|
||||||
|
static TAutoConsoleVariable<int32> CVarDebugElevenLabs(
|
||||||
|
TEXT("ps.ai.ConvAgent.Debug.ElevenLabs"),
|
||||||
|
-1,
|
||||||
|
TEXT("Debug HUD for ElevenLabs. -1=use property, 0=off, 1-3=verbosity."),
|
||||||
|
ECVF_Default);
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
// Constructor
|
// Constructor
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@ -145,9 +151,15 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::TickComponent(float DeltaTime, ELevel
|
|||||||
TEXT("[T+%.2fs] [Turn %d] Pre-buffer timeout (%dms). Starting playback."),
|
TEXT("[T+%.2fs] [Turn %d] Pre-buffer timeout (%dms). Starting playback."),
|
||||||
Tpb, LastClosedTurnIndex, AudioPreBufferMs);
|
Tpb, LastClosedTurnIndex, AudioPreBufferMs);
|
||||||
}
|
}
|
||||||
if (AudioPlaybackComponent && !AudioPlaybackComponent->IsPlaying())
|
// Only start playback if the agent is still speaking.
|
||||||
|
// If silence detection already set bAgentSpeaking=false, this is stale.
|
||||||
|
if (bAgentSpeaking)
|
||||||
{
|
{
|
||||||
AudioPlaybackComponent->Play();
|
if (AudioPlaybackComponent && !AudioPlaybackComponent->IsPlaying())
|
||||||
|
{
|
||||||
|
AudioPlaybackComponent->Play();
|
||||||
|
}
|
||||||
|
OnAudioPlaybackStarted.Broadcast();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -223,6 +235,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::TickComponent(float DeltaTime, ELevel
|
|||||||
{
|
{
|
||||||
bHardTimeoutFired = bHardTimeout && !bAgentResponseReceived;
|
bHardTimeoutFired = bHardTimeout && !bAgentResponseReceived;
|
||||||
bAgentSpeaking = false;
|
bAgentSpeaking = false;
|
||||||
|
bPreBuffering = false; // Cancel pending pre-buffer to prevent stale OnAudioPlaybackStarted.
|
||||||
bAgentResponseReceived = false;
|
bAgentResponseReceived = false;
|
||||||
SilentTickCount = 0;
|
SilentTickCount = 0;
|
||||||
bShouldBroadcastStopped = true;
|
bShouldBroadcastStopped = true;
|
||||||
@ -251,6 +264,17 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::TickComponent(float DeltaTime, ELevel
|
|||||||
MulticastAgentStoppedSpeaking();
|
MulticastAgentStoppedSpeaking();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// On-screen debug display.
|
||||||
|
{
|
||||||
|
const int32 CVarVal = CVarDebugElevenLabs.GetValueOnGameThread();
|
||||||
|
const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug;
|
||||||
|
const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity;
|
||||||
|
if (bShowDebug && EffectiveVerbosity >= 1)
|
||||||
|
{
|
||||||
|
DrawDebugHUD();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@ -367,9 +391,17 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::EndConversation()
|
|||||||
{
|
{
|
||||||
bIntentionalDisconnect = true;
|
bIntentionalDisconnect = true;
|
||||||
WebSocketProxy->Disconnect();
|
WebSocketProxy->Disconnect();
|
||||||
|
// OnClosed callback will fire OnAgentDisconnected.
|
||||||
WebSocketProxy = nullptr;
|
WebSocketProxy = nullptr;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// Persistent mode: WebSocket stays alive but the interaction is over.
|
||||||
|
// Broadcast OnAgentDisconnected so expression components deactivate
|
||||||
|
// (body, facial, etc.). The WebSocket OnClosed never fires here.
|
||||||
|
OnAgentDisconnected.Broadcast(1000, TEXT("EndConversation (persistent)"));
|
||||||
|
}
|
||||||
|
|
||||||
// Reset replicated state so other players can talk to this NPC.
|
// Reset replicated state so other players can talk to this NPC.
|
||||||
bNetIsConversing = false;
|
bNetIsConversing = false;
|
||||||
@ -1333,9 +1365,13 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::EnqueueAgentAudio(const TArray<uint8>
|
|||||||
Tpb2, LastClosedTurnIndex, AudioPreBufferMs);
|
Tpb2, LastClosedTurnIndex, AudioPreBufferMs);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else if (AudioPlaybackComponent && !AudioPlaybackComponent->IsPlaying())
|
else
|
||||||
{
|
{
|
||||||
AudioPlaybackComponent->Play();
|
if (AudioPlaybackComponent && !AudioPlaybackComponent->IsPlaying())
|
||||||
|
{
|
||||||
|
AudioPlaybackComponent->Play();
|
||||||
|
}
|
||||||
|
OnAudioPlaybackStarted.Broadcast();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else if (bPreBuffering)
|
else if (bPreBuffering)
|
||||||
@ -1361,6 +1397,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::EnqueueAgentAudio(const TArray<uint8>
|
|||||||
{
|
{
|
||||||
AudioPlaybackComponent->Play();
|
AudioPlaybackComponent->Play();
|
||||||
}
|
}
|
||||||
|
OnAudioPlaybackStarted.Broadcast();
|
||||||
}
|
}
|
||||||
SilentTickCount = 0;
|
SilentTickCount = 0;
|
||||||
}
|
}
|
||||||
@ -2008,3 +2045,77 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ApplyConversationGaze()
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// On-screen debug display
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_ElevenLabsComponent::DrawDebugHUD() const
|
||||||
|
{
|
||||||
|
if (!GEngine) return;
|
||||||
|
|
||||||
|
const int32 BaseKey = 2040;
|
||||||
|
const float DisplayTime = 1.0f;
|
||||||
|
const FColor MainColor = FColor::Cyan;
|
||||||
|
const FColor WarnColor = FColor::Yellow;
|
||||||
|
const FColor GoodColor = FColor::Green;
|
||||||
|
|
||||||
|
const bool bConnected = IsConnected();
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime,
|
||||||
|
bConnected ? GoodColor : FColor::Red,
|
||||||
|
FString::Printf(TEXT("=== ELEVENLABS: %s ==="),
|
||||||
|
bConnected ? TEXT("CONNECTED") : TEXT("DISCONNECTED")));
|
||||||
|
|
||||||
|
// Session info
|
||||||
|
FString ModeStr = (TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server)
|
||||||
|
? TEXT("ServerVAD") : TEXT("ClientPTT");
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Session: %s Turn: %d Mode: %s"),
|
||||||
|
bPersistentSession ? TEXT("persistent") : TEXT("ephemeral"),
|
||||||
|
TurnIndex, *ModeStr));
|
||||||
|
|
||||||
|
// State flags
|
||||||
|
const bool bListening = bIsListening.load();
|
||||||
|
const bool bSpeaking = bAgentSpeaking.load();
|
||||||
|
const bool bGenerating = bAgentGenerating.load();
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Listening: %s Speaking: %s Generating: %s"),
|
||||||
|
bListening ? TEXT("YES") : TEXT("NO"),
|
||||||
|
bSpeaking ? TEXT("YES") : TEXT("NO"),
|
||||||
|
bGenerating ? TEXT("YES") : TEXT("NO")));
|
||||||
|
|
||||||
|
const bool bWaiting = bWaitingForAgentResponse.load();
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 3, DisplayTime,
|
||||||
|
(bPreBuffering || bWaiting) ? WarnColor : MainColor,
|
||||||
|
FString::Printf(TEXT(" PreBuffer: %s WaitResponse: %s"),
|
||||||
|
bPreBuffering ? TEXT("YES") : TEXT("NO"),
|
||||||
|
bWaiting ? TEXT("YES") : TEXT("NO")));
|
||||||
|
|
||||||
|
// Emotion
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 4, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Emotion: %s (%s)"),
|
||||||
|
*UEnum::GetDisplayValueAsText(CurrentEmotion).ToString(),
|
||||||
|
*UEnum::GetDisplayValueAsText(CurrentEmotionIntensity).ToString()));
|
||||||
|
|
||||||
|
// Audio queue (read without lock for debug display — minor race is acceptable)
|
||||||
|
const int32 QueueBytes = FMath::Max(0, AudioQueue.Num() - AudioQueueReadOffset);
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 5, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" AudioQueue: %d bytes SilentTicks: %d"),
|
||||||
|
QueueBytes, SilentTickCount));
|
||||||
|
|
||||||
|
// Timing
|
||||||
|
const double Now = FPlatformTime::Seconds();
|
||||||
|
const float SessionSec = (SessionStartTime > 0.0) ? static_cast<float>(Now - SessionStartTime) : 0.0f;
|
||||||
|
const float TurnSec = (TurnStartTime > 0.0) ? static_cast<float>(Now - TurnStartTime) : 0.0f;
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 6, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Timing: session=%.1fs turn=%.1fs"),
|
||||||
|
SessionSec, TurnSec));
|
||||||
|
|
||||||
|
// Reconnection
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 7, DisplayTime,
|
||||||
|
bWantsReconnect ? FColor::Red : MainColor,
|
||||||
|
FString::Printf(TEXT(" Reconnect: %d/%d attempts%s"),
|
||||||
|
ReconnectAttemptCount, MaxReconnectAttempts,
|
||||||
|
bWantsReconnect ? TEXT(" (ACTIVE)") : TEXT("")));
|
||||||
|
}
|
||||||
|
|||||||
@ -5,9 +5,16 @@
|
|||||||
#include "PS_AI_ConvAgent_EmotionPoseMap.h"
|
#include "PS_AI_ConvAgent_EmotionPoseMap.h"
|
||||||
#include "Animation/AnimSequence.h"
|
#include "Animation/AnimSequence.h"
|
||||||
#include "Animation/AnimCurveTypes.h"
|
#include "Animation/AnimCurveTypes.h"
|
||||||
|
#include "Engine/Engine.h"
|
||||||
|
|
||||||
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_FacialExpr, Log, All);
|
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_FacialExpr, Log, All);
|
||||||
|
|
||||||
|
static TAutoConsoleVariable<int32> CVarDebugFacialExpr(
|
||||||
|
TEXT("ps.ai.ConvAgent.Debug.FacialExpr"),
|
||||||
|
-1,
|
||||||
|
TEXT("Debug HUD for FacialExpression. -1=use property, 0=off, 1-3=verbosity."),
|
||||||
|
ECVF_Default);
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
// Construction
|
// Construction
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@ -136,6 +143,12 @@ UAnimSequence* UPS_AI_ConvAgent_FacialExpressionComponent::FindAnimForEmotion(
|
|||||||
if (!EmotionPoseMap) return nullptr;
|
if (!EmotionPoseMap) return nullptr;
|
||||||
|
|
||||||
const FPS_AI_ConvAgent_EmotionPoseSet* PoseSet = EmotionPoseMap->EmotionPoses.Find(Emotion);
|
const FPS_AI_ConvAgent_EmotionPoseSet* PoseSet = EmotionPoseMap->EmotionPoses.Find(Emotion);
|
||||||
|
|
||||||
|
// Fallback to Neutral if the requested emotion has no entry in the data asset
|
||||||
|
if (!PoseSet && Emotion != EPS_AI_ConvAgent_Emotion::Neutral)
|
||||||
|
{
|
||||||
|
PoseSet = EmotionPoseMap->EmotionPoses.Find(EPS_AI_ConvAgent_Emotion::Neutral);
|
||||||
|
}
|
||||||
if (!PoseSet) return nullptr;
|
if (!PoseSet) return nullptr;
|
||||||
|
|
||||||
// Direct match
|
// Direct match
|
||||||
@ -282,9 +295,10 @@ void UPS_AI_ConvAgent_FacialExpressionComponent::TickComponent(
|
|||||||
const float TargetAlpha = bActive ? 1.0f : 0.0f;
|
const float TargetAlpha = bActive ? 1.0f : 0.0f;
|
||||||
if (!FMath::IsNearlyEqual(CurrentActiveAlpha, TargetAlpha, 0.001f))
|
if (!FMath::IsNearlyEqual(CurrentActiveAlpha, TargetAlpha, 0.001f))
|
||||||
{
|
{
|
||||||
const float BlendSpeed = 1.0f / FMath::Max(ActivationBlendDuration, 0.01f);
|
// Exponential ease-out: fast start, gradual approach to target.
|
||||||
CurrentActiveAlpha = FMath::FInterpConstantTo(
|
const float InterpSpeed = 3.0f / FMath::Max(ActivationBlendDuration, 0.01f);
|
||||||
CurrentActiveAlpha, TargetAlpha, DeltaTime, BlendSpeed);
|
CurrentActiveAlpha = FMath::FInterpTo(
|
||||||
|
CurrentActiveAlpha, TargetAlpha, DeltaTime, InterpSpeed);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -356,13 +370,16 @@ void UPS_AI_ConvAgent_FacialExpressionComponent::TickComponent(
|
|||||||
for (const auto& P : ActiveCurves) AllCurves.Add(P.Key);
|
for (const auto& P : ActiveCurves) AllCurves.Add(P.Key);
|
||||||
for (const auto& P : PrevCurves) AllCurves.Add(P.Key);
|
for (const auto& P : PrevCurves) AllCurves.Add(P.Key);
|
||||||
|
|
||||||
|
// Apply SmoothStep for ease-in-out crossfade (raw alpha is linear)
|
||||||
|
const float SmoothedCrossfade = FMath::SmoothStep(0.0f, 1.0f, CrossfadeAlpha);
|
||||||
|
|
||||||
for (const FName& CurveName : AllCurves)
|
for (const FName& CurveName : AllCurves)
|
||||||
{
|
{
|
||||||
const float PrevVal = PrevCurves.Contains(CurveName)
|
const float PrevVal = PrevCurves.Contains(CurveName)
|
||||||
? PrevCurves[CurveName] : 0.0f;
|
? PrevCurves[CurveName] : 0.0f;
|
||||||
const float ActiveVal = ActiveCurves.Contains(CurveName)
|
const float ActiveVal = ActiveCurves.Contains(CurveName)
|
||||||
? ActiveCurves[CurveName] : 0.0f;
|
? ActiveCurves[CurveName] : 0.0f;
|
||||||
const float Blended = FMath::Lerp(PrevVal, ActiveVal, CrossfadeAlpha);
|
const float Blended = FMath::Lerp(PrevVal, ActiveVal, SmoothedCrossfade);
|
||||||
|
|
||||||
// Always include the curve even at 0 — the AnimNode needs
|
// Always include the curve even at 0 — the AnimNode needs
|
||||||
// to see it to block upstream values from popping through.
|
// to see it to block upstream values from popping through.
|
||||||
@ -388,6 +405,64 @@ void UPS_AI_ConvAgent_FacialExpressionComponent::TickComponent(
|
|||||||
FScopeLock Lock(&EmotionCurveLock);
|
FScopeLock Lock(&EmotionCurveLock);
|
||||||
CurrentEmotionCurves = MoveTemp(NewCurves);
|
CurrentEmotionCurves = MoveTemp(NewCurves);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── On-screen debug HUD ───────────────────────────────────────────────
|
||||||
|
{
|
||||||
|
const int32 CVarVal = CVarDebugFacialExpr.GetValueOnGameThread();
|
||||||
|
const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug;
|
||||||
|
const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity;
|
||||||
|
if (bShowDebug && EffectiveVerbosity >= 1)
|
||||||
|
{
|
||||||
|
DrawDebugHUD();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// On-screen debug display
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_FacialExpressionComponent::DrawDebugHUD() const
|
||||||
|
{
|
||||||
|
if (!GEngine) return;
|
||||||
|
|
||||||
|
const int32 BaseKey = 2010;
|
||||||
|
const float DisplayTime = 1.0f;
|
||||||
|
const FColor MainColor = FColor::Cyan;
|
||||||
|
const FColor WarnColor = FColor::Yellow;
|
||||||
|
|
||||||
|
// State label
|
||||||
|
FString StateStr = bActive ? TEXT("ACTIVE") : TEXT("INACTIVE");
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT("=== FACIAL EXPR: %s ==="), *StateStr));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" ActivationAlpha: %.3f (target: %s)"),
|
||||||
|
CurrentActiveAlpha, bActive ? TEXT("1") : TEXT("0")));
|
||||||
|
|
||||||
|
FString ActiveName = ActiveAnim ? ActiveAnim->GetName() : TEXT("(none)");
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Active: %s t=%.2f"), *ActiveName, ActivePlaybackTime));
|
||||||
|
|
||||||
|
FString PrevName = PrevAnim ? PrevAnim->GetName() : TEXT("---");
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 3, DisplayTime,
|
||||||
|
CrossfadeAlpha < 1.0f ? WarnColor : MainColor,
|
||||||
|
FString::Printf(TEXT(" Crossfade: %.3f Prev: %s"),
|
||||||
|
CrossfadeAlpha, *PrevName));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 4, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Emotion: %s (%s)"),
|
||||||
|
*UEnum::GetDisplayValueAsText(ActiveEmotion).ToString(),
|
||||||
|
*UEnum::GetDisplayValueAsText(ActiveEmotionIntensity).ToString()));
|
||||||
|
|
||||||
|
int32 CurveCount = 0;
|
||||||
|
{
|
||||||
|
FScopeLock Lock(&EmotionCurveLock);
|
||||||
|
CurveCount = CurrentEmotionCurves.Num();
|
||||||
|
}
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 5, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Curves: %d active"), CurveCount));
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|||||||
@ -3,12 +3,21 @@
|
|||||||
#include "PS_AI_ConvAgent_GazeComponent.h"
|
#include "PS_AI_ConvAgent_GazeComponent.h"
|
||||||
#include "PS_AI_ConvAgent_ElevenLabsComponent.h"
|
#include "PS_AI_ConvAgent_ElevenLabsComponent.h"
|
||||||
#include "Components/SkeletalMeshComponent.h"
|
#include "Components/SkeletalMeshComponent.h"
|
||||||
|
#include "Camera/CameraComponent.h"
|
||||||
#include "GameFramework/Actor.h"
|
#include "GameFramework/Actor.h"
|
||||||
|
#include "GameFramework/Pawn.h"
|
||||||
#include "Math/UnrealMathUtility.h"
|
#include "Math/UnrealMathUtility.h"
|
||||||
#include "DrawDebugHelpers.h"
|
#include "DrawDebugHelpers.h"
|
||||||
|
#include "Engine/Engine.h"
|
||||||
|
|
||||||
DEFINE_LOG_CATEGORY(LogPS_AI_ConvAgent_Gaze);
|
DEFINE_LOG_CATEGORY(LogPS_AI_ConvAgent_Gaze);
|
||||||
|
|
||||||
|
static TAutoConsoleVariable<int32> CVarDebugGaze(
|
||||||
|
TEXT("ps.ai.ConvAgent.Debug.Gaze"),
|
||||||
|
-1,
|
||||||
|
TEXT("Debug HUD for Gaze. -1=use property, 0=off, 1-3=verbosity."),
|
||||||
|
ECVF_Default);
|
||||||
|
|
||||||
// ── ARKit eye curve names ────────────────────────────────────────────────────
|
// ── ARKit eye curve names ────────────────────────────────────────────────────
|
||||||
static const FName EyeLookUpLeft(TEXT("eyeLookUpLeft"));
|
static const FName EyeLookUpLeft(TEXT("eyeLookUpLeft"));
|
||||||
static const FName EyeLookDownLeft(TEXT("eyeLookDownLeft"));
|
static const FName EyeLookDownLeft(TEXT("eyeLookDownLeft"));
|
||||||
@ -38,7 +47,8 @@ static const FName TargetHeadBone(TEXT("head"));
|
|||||||
* bAutoTargetEyes = true:
|
* bAutoTargetEyes = true:
|
||||||
* 1. Try eye bones (FACIAL_L_Eye / FACIAL_R_Eye midpoint) on the target's Face mesh.
|
* 1. Try eye bones (FACIAL_L_Eye / FACIAL_R_Eye midpoint) on the target's Face mesh.
|
||||||
* 2. Fallback to "head" bone on any skeletal mesh.
|
* 2. Fallback to "head" bone on any skeletal mesh.
|
||||||
* 3. Fallback to TargetActor origin + (0, 0, FallbackEyeHeight).
|
* 3. Fallback to CameraComponent location (first-person pawn).
|
||||||
|
* 4. Fallback to TargetActor origin + (0, 0, FallbackEyeHeight) for non-camera actors.
|
||||||
*
|
*
|
||||||
* bAutoTargetEyes = false:
|
* bAutoTargetEyes = false:
|
||||||
* Always returns TargetActor origin + TargetOffset.
|
* Always returns TargetActor origin + TargetOffset.
|
||||||
@ -76,7 +86,15 @@ static FVector ResolveTargetPosition(const AActor* Target, bool bAutoEyes,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// No skeleton — use FallbackEyeHeight
|
// Fallback: CameraComponent — the canonical eye position for
|
||||||
|
// first-person pawns and any actor with an active camera.
|
||||||
|
if (const UCameraComponent* Cam =
|
||||||
|
const_cast<AActor*>(Target)->FindComponentByClass<UCameraComponent>())
|
||||||
|
{
|
||||||
|
return Cam->GetComponentLocation();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Final fallback: actor origin + height offset
|
||||||
return Target->GetActorLocation() + FVector(0.0f, 0.0f, FallbackHeight);
|
return Target->GetActorLocation() + FVector(0.0f, 0.0f, FallbackHeight);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -335,9 +353,10 @@ void UPS_AI_ConvAgent_GazeComponent::TickComponent(
|
|||||||
const float TargetAlpha = bActive ? 1.0f : 0.0f;
|
const float TargetAlpha = bActive ? 1.0f : 0.0f;
|
||||||
if (!FMath::IsNearlyEqual(CurrentActiveAlpha, TargetAlpha, 0.001f))
|
if (!FMath::IsNearlyEqual(CurrentActiveAlpha, TargetAlpha, 0.001f))
|
||||||
{
|
{
|
||||||
const float BlendSpeed = 1.0f / FMath::Max(ActivationBlendDuration, 0.01f);
|
// Exponential ease-out: fast start, gradual approach to target.
|
||||||
CurrentActiveAlpha = FMath::FInterpConstantTo(
|
const float InterpSpeed = 3.0f / FMath::Max(ActivationBlendDuration, 0.01f);
|
||||||
CurrentActiveAlpha, TargetAlpha, SafeDeltaTime, BlendSpeed);
|
CurrentActiveAlpha = FMath::FInterpTo(
|
||||||
|
CurrentActiveAlpha, TargetAlpha, SafeDeltaTime, InterpSpeed);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -634,4 +653,57 @@ void UPS_AI_ConvAgent_GazeComponent::TickComponent(
|
|||||||
Owner->GetActorRotation().Yaw);
|
Owner->GetActorRotation().Yaw);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── On-screen debug HUD ───────────────────────────────────────────────
|
||||||
|
{
|
||||||
|
const int32 CVarVal = CVarDebugGaze.GetValueOnGameThread();
|
||||||
|
const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug;
|
||||||
|
const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity;
|
||||||
|
if (bShowDebug && EffectiveVerbosity >= 1)
|
||||||
|
{
|
||||||
|
DrawDebugHUD();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// On-screen debug display
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_GazeComponent::DrawDebugHUD() const
|
||||||
|
{
|
||||||
|
if (!GEngine) return;
|
||||||
|
|
||||||
|
const int32 BaseKey = 2020;
|
||||||
|
const float DisplayTime = 1.0f;
|
||||||
|
const FColor MainColor = FColor::Cyan;
|
||||||
|
|
||||||
|
FString StateStr = bActive ? TEXT("ACTIVE") : TEXT("INACTIVE");
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT("=== GAZE: %s ==="), *StateStr));
|
||||||
|
|
||||||
|
FString TargetName = TargetActor ? TargetActor->GetName() : TEXT("(none)");
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Target: %s BodyTrack: %s"),
|
||||||
|
*TargetName, bEnableBodyTracking ? TEXT("ON") : TEXT("OFF")));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" ActivationAlpha: %.3f"), CurrentActiveAlpha));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 3, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Head: Yaw=%.1f Pitch=%.1f (target: %.1f / %.1f)"),
|
||||||
|
CurrentHeadYaw, CurrentHeadPitch, TargetHeadYaw, TargetHeadPitch));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 4, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Eyes: Yaw=%.1f Pitch=%.1f"),
|
||||||
|
CurrentEyeYaw, CurrentEyePitch));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 5, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Body: SmoothedYaw=%.1f TargetYaw=%.1f"),
|
||||||
|
SmoothedBodyYaw, TargetBodyWorldYaw));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 6, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Compensation: Head=%.2f Eye=%.2f Body=%.2f"),
|
||||||
|
HeadAnimationCompensation, EyeAnimationCompensation, BodyDriftCompensation));
|
||||||
}
|
}
|
||||||
|
|||||||
@ -14,6 +14,12 @@
|
|||||||
|
|
||||||
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_Select, Log, All);
|
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_Select, Log, All);
|
||||||
|
|
||||||
|
static TAutoConsoleVariable<int32> CVarDebugInteraction(
|
||||||
|
TEXT("ps.ai.ConvAgent.Debug.Interaction"),
|
||||||
|
-1,
|
||||||
|
TEXT("Debug HUD for Interaction. -1=use property, 0=off, 1-3=verbosity."),
|
||||||
|
ECVF_Default);
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
// Constructor
|
// Constructor
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@ -110,10 +116,16 @@ void UPS_AI_ConvAgent_InteractionComponent::TickComponent(float DeltaTime, ELeve
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── This is the locally controlled pawn — create the mic component ──
|
// ── This is the locally controlled pawn — find or create mic component ──
|
||||||
MicComponent = NewObject<UPS_AI_ConvAgent_MicrophoneCaptureComponent>(
|
MicComponent = GetOwner()->FindComponentByClass<UPS_AI_ConvAgent_MicrophoneCaptureComponent>();
|
||||||
GetOwner(), TEXT("PS_AI_ConvAgent_Mic_Interaction"));
|
if (!MicComponent)
|
||||||
MicComponent->RegisterComponent();
|
{
|
||||||
|
MicComponent = NewObject<UPS_AI_ConvAgent_MicrophoneCaptureComponent>(
|
||||||
|
GetOwner(), TEXT("PS_AI_ConvAgent_Mic_Interaction"));
|
||||||
|
MicComponent->bDebug = bDebug;
|
||||||
|
MicComponent->DebugVerbosity = DebugVerbosity;
|
||||||
|
MicComponent->RegisterComponent();
|
||||||
|
}
|
||||||
MicComponent->OnAudioCaptured.AddUObject(this,
|
MicComponent->OnAudioCaptured.AddUObject(this,
|
||||||
&UPS_AI_ConvAgent_InteractionComponent::OnMicAudioCaptured);
|
&UPS_AI_ConvAgent_InteractionComponent::OnMicAudioCaptured);
|
||||||
|
|
||||||
@ -136,6 +148,17 @@ void UPS_AI_ConvAgent_InteractionComponent::TickComponent(float DeltaTime, ELeve
|
|||||||
{
|
{
|
||||||
SetSelectedAgent(BestAgent);
|
SetSelectedAgent(BestAgent);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── On-screen debug HUD ───────────────────────────────────────────────
|
||||||
|
{
|
||||||
|
const int32 CVarVal = CVarDebugInteraction.GetValueOnGameThread();
|
||||||
|
const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug;
|
||||||
|
const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity;
|
||||||
|
if (bShowDebug && EffectiveVerbosity >= 1)
|
||||||
|
{
|
||||||
|
DrawDebugHUD();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@ -165,6 +188,26 @@ UPS_AI_ConvAgent_ElevenLabsComponent* UPS_AI_ConvAgent_InteractionComponent::Eva
|
|||||||
|
|
||||||
UPS_AI_ConvAgent_ElevenLabsComponent* CurrentAgent = SelectedAgent.Get();
|
UPS_AI_ConvAgent_ElevenLabsComponent* CurrentAgent = SelectedAgent.Get();
|
||||||
|
|
||||||
|
// ── Conversation lock ──────────────────────────────────────────────
|
||||||
|
// While we're actively conversing with an agent, keep it selected as
|
||||||
|
// long as it's within interaction distance — ignore the view cone.
|
||||||
|
// This prevents deselect/reselect flicker when the player turns quickly
|
||||||
|
// (which would cause spurious OnAgentConnected re-broadcasts in
|
||||||
|
// persistent session mode).
|
||||||
|
if (CurrentAgent && CurrentAgent->bNetIsConversing)
|
||||||
|
{
|
||||||
|
if (AActor* AgentActor = CurrentAgent->GetOwner())
|
||||||
|
{
|
||||||
|
const FVector AgentLoc = AgentActor->GetActorLocation()
|
||||||
|
+ FVector(0.0f, 0.0f, AgentEyeLevelOffset);
|
||||||
|
const float DistSq = (AgentLoc - ViewLocation).SizeSquared();
|
||||||
|
if (DistSq <= MaxDistSq)
|
||||||
|
{
|
||||||
|
return CurrentAgent; // Keep conversing agent selected.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Get local player's pawn for occupied-NPC check.
|
// Get local player's pawn for occupied-NPC check.
|
||||||
// Use pawn (replicated to ALL clients) instead of PlayerController
|
// Use pawn (replicated to ALL clients) instead of PlayerController
|
||||||
// (only replicated to owning client due to bOnlyRelevantToOwner=true).
|
// (only replicated to owning client due to bOnlyRelevantToOwner=true).
|
||||||
@ -476,6 +519,37 @@ void UPS_AI_ConvAgent_InteractionComponent::StartConversationWithSelectedAgent()
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_InteractionComponent::SendTextToSelectedAgent(const FString& Text)
|
||||||
|
{
|
||||||
|
UPS_AI_ConvAgent_ElevenLabsComponent* Agent = SelectedAgent.Get();
|
||||||
|
if (!Agent)
|
||||||
|
{
|
||||||
|
if (bDebug)
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Select, Warning,
|
||||||
|
TEXT("SendTextToSelectedAgent: no agent selected."));
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Route through relay on clients (can't call Server RPCs on NPC actors).
|
||||||
|
if (GetOwnerRole() == ROLE_Authority || (GetWorld() && GetWorld()->GetNetMode() == NM_Standalone))
|
||||||
|
{
|
||||||
|
Agent->SendTextMessage(Text);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
ServerRelaySendText(Agent->GetOwner(), Text);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (bDebug)
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Select, Log,
|
||||||
|
TEXT("SendTextToSelectedAgent: \"%s\" → %s"),
|
||||||
|
*Text, Agent->GetOwner() ? *Agent->GetOwner()->GetName() : TEXT("(null)"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
// Gaze helpers
|
// Gaze helpers
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@ -558,6 +632,71 @@ void UPS_AI_ConvAgent_InteractionComponent::OnMicAudioCaptured(const TArray<floa
|
|||||||
Agent->FeedExternalAudio(FloatPCM);
|
Agent->FeedExternalAudio(FloatPCM);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// On-screen debug display
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_InteractionComponent::DrawDebugHUD() const
|
||||||
|
{
|
||||||
|
if (!GEngine) return;
|
||||||
|
|
||||||
|
const int32 BaseKey = 2060;
|
||||||
|
const float DisplayTime = 1.0f;
|
||||||
|
const FColor MainColor = FColor::Cyan;
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, MainColor,
|
||||||
|
TEXT("=== INTERACTION ==="));
|
||||||
|
|
||||||
|
UPS_AI_ConvAgent_ElevenLabsComponent* Agent = SelectedAgent.Get();
|
||||||
|
if (Agent)
|
||||||
|
{
|
||||||
|
FString AgentName = Agent->GetOwner() ? Agent->GetOwner()->GetName() : TEXT("(?)");
|
||||||
|
FString ConvState = Agent->bNetIsConversing ? TEXT("conversing") : TEXT("selected");
|
||||||
|
|
||||||
|
// Compute distance and angle to selected agent
|
||||||
|
FVector ViewLoc, ViewDir;
|
||||||
|
GetPawnViewPoint(ViewLoc, ViewDir);
|
||||||
|
|
||||||
|
float Dist = 0.0f;
|
||||||
|
float Angle = 0.0f;
|
||||||
|
if (Agent->GetOwner())
|
||||||
|
{
|
||||||
|
FVector AgentLoc = Agent->GetOwner()->GetActorLocation()
|
||||||
|
+ FVector(0.0f, 0.0f, AgentEyeLevelOffset);
|
||||||
|
FVector ToAgent = AgentLoc - ViewLoc;
|
||||||
|
Dist = ToAgent.Size();
|
||||||
|
FVector DirToAgent = ToAgent.GetSafeNormal();
|
||||||
|
Angle = FMath::RadiansToDegrees(
|
||||||
|
FMath::Acos(FMath::Clamp(FVector::DotProduct(ViewDir, DirToAgent), -1.0f, 1.0f)));
|
||||||
|
}
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Selected: %s (%s)"), *AgentName, *ConvState));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Distance: %.0fcm Angle: %.1f deg"),
|
||||||
|
Dist, Angle));
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor,
|
||||||
|
TEXT(" Selected: (none)"));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, MainColor,
|
||||||
|
TEXT(" Distance: --- Angle: ---"));
|
||||||
|
}
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 3, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" AutoStart: %s AutoGaze: %s AutoListen: %s"),
|
||||||
|
bAutoStartConversation ? TEXT("ON") : TEXT("OFF"),
|
||||||
|
bAutoManageGaze ? TEXT("ON") : TEXT("OFF"),
|
||||||
|
bAutoManageListening ? TEXT("ON") : TEXT("OFF")));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 4, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Mic: %s"),
|
||||||
|
(MicComponent != nullptr) ? TEXT("initialized") : TEXT("not initialized")));
|
||||||
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
// Replication
|
// Replication
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|||||||
@ -13,6 +13,12 @@
|
|||||||
|
|
||||||
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_LipSync, Log, All);
|
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_LipSync, Log, All);
|
||||||
|
|
||||||
|
static TAutoConsoleVariable<int32> CVarDebugLipSync(
|
||||||
|
TEXT("ps.ai.ConvAgent.Debug.LipSync"),
|
||||||
|
-1,
|
||||||
|
TEXT("Debug HUD for LipSync. -1=use property, 0=off, 1-3=verbosity."),
|
||||||
|
ECVF_Default);
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
// Static data
|
// Static data
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@ -702,9 +708,10 @@ void UPS_AI_ConvAgent_LipSyncComponent::TickComponent(float DeltaTime, ELevelTic
|
|||||||
const float TargetAlpha = bActive ? 1.0f : 0.0f;
|
const float TargetAlpha = bActive ? 1.0f : 0.0f;
|
||||||
if (!FMath::IsNearlyEqual(CurrentActiveAlpha, TargetAlpha, 0.001f))
|
if (!FMath::IsNearlyEqual(CurrentActiveAlpha, TargetAlpha, 0.001f))
|
||||||
{
|
{
|
||||||
const float BlendSpeed = 1.0f / FMath::Max(ActivationBlendDuration, 0.01f);
|
// Exponential ease-out: fast start, gradual approach to target.
|
||||||
CurrentActiveAlpha = FMath::FInterpConstantTo(
|
const float InterpSpeed = 3.0f / FMath::Max(ActivationBlendDuration, 0.01f);
|
||||||
CurrentActiveAlpha, TargetAlpha, DeltaTime, BlendSpeed);
|
CurrentActiveAlpha = FMath::FInterpTo(
|
||||||
|
CurrentActiveAlpha, TargetAlpha, DeltaTime, InterpSpeed);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -712,6 +719,24 @@ void UPS_AI_ConvAgent_LipSyncComponent::TickComponent(float DeltaTime, ELevelTic
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Smooth speech blend ────────────────────────────────────────────────
|
||||||
|
// Fades lip sync curves in/out when speech starts/stops.
|
||||||
|
// When not speaking, lip sync releases all mouth curves so emotion
|
||||||
|
// facial expression (from FacialExpressionComponent) controls the mouth.
|
||||||
|
{
|
||||||
|
const float TargetSpeech = bIsSpeaking ? 1.0f : 0.0f;
|
||||||
|
if (!FMath::IsNearlyEqual(SpeechBlendAlpha, TargetSpeech, 0.001f))
|
||||||
|
{
|
||||||
|
const float SpeechSpeed = 3.0f / FMath::Max(SpeechBlendDuration, 0.01f);
|
||||||
|
SpeechBlendAlpha = FMath::FInterpTo(
|
||||||
|
SpeechBlendAlpha, TargetSpeech, DeltaTime, SpeechSpeed);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
SpeechBlendAlpha = TargetSpeech;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ── Lazy binding: in packaged builds, BeginPlay may run before the ────────
|
// ── Lazy binding: in packaged builds, BeginPlay may run before the ────────
|
||||||
// ElevenLabsComponent is fully initialized. Retry discovery until bound.
|
// ElevenLabsComponent is fully initialized. Retry discovery until bound.
|
||||||
if (!AgentComponent.IsValid())
|
if (!AgentComponent.IsValid())
|
||||||
@ -858,6 +883,12 @@ void UPS_AI_ConvAgent_LipSyncComponent::TickComponent(float DeltaTime, ELevelTic
|
|||||||
{
|
{
|
||||||
bVisemeTimelineActive = false;
|
bVisemeTimelineActive = false;
|
||||||
|
|
||||||
|
// Fallback: mark as not speaking when timeline ends
|
||||||
|
if (bIsSpeaking && AudioEnvelopeValue < 0.01f)
|
||||||
|
{
|
||||||
|
bIsSpeaking = false;
|
||||||
|
}
|
||||||
|
|
||||||
if (AccumulatedText.Len() > 0 && bTextVisemesApplied && bFullTextReceived)
|
if (AccumulatedText.Len() > 0 && bTextVisemesApplied && bFullTextReceived)
|
||||||
{
|
{
|
||||||
AccumulatedText.Reset();
|
AccumulatedText.Reset();
|
||||||
@ -964,6 +995,14 @@ void UPS_AI_ConvAgent_LipSyncComponent::TickComponent(float DeltaTime, ELevelTic
|
|||||||
TargetVisemes.FindOrAdd(FName("sil")) = 1.0f;
|
TargetVisemes.FindOrAdd(FName("sil")) = 1.0f;
|
||||||
PlaybackTimer = 0.0f;
|
PlaybackTimer = 0.0f;
|
||||||
|
|
||||||
|
// Fallback: if the queue has been dry and envelope is near zero,
|
||||||
|
// mark as not speaking. OnAgentStopped should have already done this,
|
||||||
|
// but this handles edge cases where the event doesn't fire.
|
||||||
|
if (bIsSpeaking && AudioEnvelopeValue < 0.01f)
|
||||||
|
{
|
||||||
|
bIsSpeaking = false;
|
||||||
|
}
|
||||||
|
|
||||||
if (AccumulatedText.Len() > 0 && bTextVisemesApplied && bFullTextReceived)
|
if (AccumulatedText.Len() > 0 && bTextVisemesApplied && bFullTextReceived)
|
||||||
{
|
{
|
||||||
AccumulatedText.Reset();
|
AccumulatedText.Reset();
|
||||||
@ -1052,6 +1091,24 @@ void UPS_AI_ConvAgent_LipSyncComponent::TickComponent(float DeltaTime, ELevelTic
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Apply speech blend alpha ─────────────────────────────────────────
|
||||||
|
// When the agent is NOT speaking, fade out all lip sync curves so the
|
||||||
|
// FacialExpressionComponent's emotion curves (including mouth expressions)
|
||||||
|
// can flow through unobstructed. Without this, lip sync zeros on mouth
|
||||||
|
// curves (via the AnimNode → mh_arkit_mapping_pose) would overwrite the
|
||||||
|
// emotion mouth curves even during silence.
|
||||||
|
if (SpeechBlendAlpha < 0.001f)
|
||||||
|
{
|
||||||
|
CurrentBlendshapes.Reset();
|
||||||
|
}
|
||||||
|
else if (SpeechBlendAlpha < 0.999f)
|
||||||
|
{
|
||||||
|
for (auto& Pair : CurrentBlendshapes)
|
||||||
|
{
|
||||||
|
Pair.Value *= SpeechBlendAlpha;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Auto-apply morph targets if a target mesh is set
|
// Auto-apply morph targets if a target mesh is set
|
||||||
if (TargetMesh)
|
if (TargetMesh)
|
||||||
{
|
{
|
||||||
@ -1063,6 +1120,17 @@ void UPS_AI_ConvAgent_LipSyncComponent::TickComponent(float DeltaTime, ELevelTic
|
|||||||
{
|
{
|
||||||
OnVisemesReady.Broadcast();
|
OnVisemesReady.Broadcast();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── On-screen debug HUD ───────────────────────────────────────────────
|
||||||
|
{
|
||||||
|
const int32 CVarVal = CVarDebugLipSync.GetValueOnGameThread();
|
||||||
|
const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug;
|
||||||
|
const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity;
|
||||||
|
if (bShowDebug && EffectiveVerbosity >= 1)
|
||||||
|
{
|
||||||
|
DrawDebugHUD();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@ -1090,6 +1158,10 @@ void UPS_AI_ConvAgent_LipSyncComponent::OnAgentStopped()
|
|||||||
VisemeTimeline.Reset();
|
VisemeTimeline.Reset();
|
||||||
VisemeTimelineCursor = 0.0f;
|
VisemeTimelineCursor = 0.0f;
|
||||||
TotalActiveFramesSeen = 0;
|
TotalActiveFramesSeen = 0;
|
||||||
|
|
||||||
|
// Speech ended — SpeechBlendAlpha will smoothly fade out lip sync curves,
|
||||||
|
// allowing FacialExpression emotion curves to take over the mouth.
|
||||||
|
bIsSpeaking = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
void UPS_AI_ConvAgent_LipSyncComponent::ResetToNeutral()
|
void UPS_AI_ConvAgent_LipSyncComponent::ResetToNeutral()
|
||||||
@ -1113,6 +1185,11 @@ void UPS_AI_ConvAgent_LipSyncComponent::ResetToNeutral()
|
|||||||
VisemeTimelineCursor = 0.0f;
|
VisemeTimelineCursor = 0.0f;
|
||||||
TotalActiveFramesSeen = 0;
|
TotalActiveFramesSeen = 0;
|
||||||
|
|
||||||
|
// Speech interrupted — snap speech blend to 0 immediately
|
||||||
|
// so emotion curves take over the mouth right away.
|
||||||
|
bIsSpeaking = false;
|
||||||
|
SpeechBlendAlpha = 0.0f;
|
||||||
|
|
||||||
// Snap all visemes to silence immediately (no smoothing delay)
|
// Snap all visemes to silence immediately (no smoothing delay)
|
||||||
for (const FName& Name : VisemeNames)
|
for (const FName& Name : VisemeNames)
|
||||||
{
|
{
|
||||||
@ -1136,6 +1213,10 @@ void UPS_AI_ConvAgent_LipSyncComponent::OnAudioChunkReceived(const TArray<uint8>
|
|||||||
{
|
{
|
||||||
if (!SpectrumAnalyzer) return;
|
if (!SpectrumAnalyzer) return;
|
||||||
|
|
||||||
|
// Mark as speaking — audio is arriving from the agent.
|
||||||
|
// SpeechBlendAlpha will smoothly fade in lip sync curves.
|
||||||
|
bIsSpeaking = true;
|
||||||
|
|
||||||
// Convert int16 PCM to float32 [-1, 1]
|
// Convert int16 PCM to float32 [-1, 1]
|
||||||
const int16* Samples = reinterpret_cast<const int16*>(PCMData.GetData());
|
const int16* Samples = reinterpret_cast<const int16*>(PCMData.GetData());
|
||||||
const int32 NumSamples = PCMData.Num() / sizeof(int16);
|
const int32 NumSamples = PCMData.Num() / sizeof(int16);
|
||||||
@ -2511,6 +2592,78 @@ void UPS_AI_ConvAgent_LipSyncComponent::ApplyMorphTargets()
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// On-screen debug display
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_LipSyncComponent::DrawDebugHUD() const
|
||||||
|
{
|
||||||
|
if (!GEngine) return;
|
||||||
|
|
||||||
|
const int32 BaseKey = 2030;
|
||||||
|
const float DisplayTime = 1.0f;
|
||||||
|
const FColor MainColor = FColor::Cyan;
|
||||||
|
const FColor WarnColor = FColor::Yellow;
|
||||||
|
|
||||||
|
FString StateStr = bActive ? TEXT("ACTIVE") : TEXT("INACTIVE");
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT("=== LIP SYNC: %s ==="), *StateStr));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime,
|
||||||
|
bIsSpeaking ? FColor::Green : MainColor,
|
||||||
|
FString::Printf(TEXT(" Speaking: %s SpeechBlend: %.3f"),
|
||||||
|
bIsSpeaking ? TEXT("YES") : TEXT("NO"), SpeechBlendAlpha));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" ActivationAlpha: %.3f"), CurrentActiveAlpha));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 3, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Amplitude: %.3f Envelope: %.3f"),
|
||||||
|
CurrentAmplitude, AudioEnvelopeValue));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 4, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Queue: %d frames Playback: %.3fs"),
|
||||||
|
VisemeQueue.Num(), PlaybackTimer));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 5, DisplayTime,
|
||||||
|
bVisemeTimelineActive ? WarnColor : MainColor,
|
||||||
|
FString::Printf(TEXT(" Timeline: %s cursor=%.2fs"),
|
||||||
|
bVisemeTimelineActive ? TEXT("ACTIVE") : TEXT("OFF"),
|
||||||
|
VisemeTimelineCursor));
|
||||||
|
|
||||||
|
// Top 3 visemes by weight
|
||||||
|
FString TopVisemes;
|
||||||
|
{
|
||||||
|
TArray<TPair<FName, float>> Sorted;
|
||||||
|
for (const auto& Pair : SmoothedVisemes)
|
||||||
|
{
|
||||||
|
if (Pair.Value > 0.01f)
|
||||||
|
{
|
||||||
|
Sorted.Add(TPair<FName, float>(Pair.Key, Pair.Value));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Sorted.Sort([](const TPair<FName, float>& A, const TPair<FName, float>& B)
|
||||||
|
{
|
||||||
|
return A.Value > B.Value;
|
||||||
|
});
|
||||||
|
for (int32 i = 0; i < FMath::Min(3, Sorted.Num()); ++i)
|
||||||
|
{
|
||||||
|
if (i > 0) TopVisemes += TEXT(", ");
|
||||||
|
TopVisemes += FString::Printf(TEXT("%s=%.2f"),
|
||||||
|
*Sorted[i].Key.ToString(), Sorted[i].Value);
|
||||||
|
}
|
||||||
|
if (TopVisemes.IsEmpty()) TopVisemes = TEXT("---");
|
||||||
|
}
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 6, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Top visemes: %s"), *TopVisemes));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 7, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Mode: %s PoseMap: %s"),
|
||||||
|
bUseCurveMode ? TEXT("Curves") : TEXT("MorphTargets"),
|
||||||
|
PoseMap ? TEXT("YES") : TEXT("NO")));
|
||||||
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
// ARKit → MetaHuman curve name conversion
|
// ARKit → MetaHuman curve name conversion
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|||||||
@ -5,15 +5,23 @@
|
|||||||
|
|
||||||
#include "AudioCaptureCore.h"
|
#include "AudioCaptureCore.h"
|
||||||
#include "Async/Async.h"
|
#include "Async/Async.h"
|
||||||
|
#include "Engine/Engine.h"
|
||||||
|
|
||||||
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_Mic, Log, All);
|
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_Mic, Log, All);
|
||||||
|
|
||||||
|
static TAutoConsoleVariable<int32> CVarDebugMic(
|
||||||
|
TEXT("ps.ai.ConvAgent.Debug.Mic"),
|
||||||
|
-1,
|
||||||
|
TEXT("Debug HUD for Microphone. -1=use property, 0=off, 1-3=verbosity."),
|
||||||
|
ECVF_Default);
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
// Constructor
|
// Constructor
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
UPS_AI_ConvAgent_MicrophoneCaptureComponent::UPS_AI_ConvAgent_MicrophoneCaptureComponent()
|
UPS_AI_ConvAgent_MicrophoneCaptureComponent::UPS_AI_ConvAgent_MicrophoneCaptureComponent()
|
||||||
{
|
{
|
||||||
PrimaryComponentTick.bCanEverTick = false;
|
PrimaryComponentTick.bCanEverTick = true;
|
||||||
|
PrimaryComponentTick.TickInterval = 1.0f / 15.0f; // 15 Hz — enough for debug HUD.
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@ -25,6 +33,78 @@ void UPS_AI_ConvAgent_MicrophoneCaptureComponent::EndPlay(const EEndPlayReason::
|
|||||||
Super::EndPlay(EndPlayReason);
|
Super::EndPlay(EndPlayReason);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Tick
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
void UPS_AI_ConvAgent_MicrophoneCaptureComponent::TickComponent(
|
||||||
|
float DeltaTime, ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction)
|
||||||
|
{
|
||||||
|
Super::TickComponent(DeltaTime, TickType, ThisTickFunction);
|
||||||
|
|
||||||
|
// On-screen debug HUD
|
||||||
|
const int32 CVarVal = CVarDebugMic.GetValueOnGameThread();
|
||||||
|
const bool bShowDebug = (CVarVal >= 0) ? (CVarVal > 0) : bDebug;
|
||||||
|
const int32 EffectiveVerbosity = (CVarVal > 0) ? CVarVal : DebugVerbosity;
|
||||||
|
if (bShowDebug && EffectiveVerbosity >= 1)
|
||||||
|
{
|
||||||
|
DrawDebugHUD();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// On-screen debug display
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
void UPS_AI_ConvAgent_MicrophoneCaptureComponent::DrawDebugHUD() const
|
||||||
|
{
|
||||||
|
if (!GEngine) return;
|
||||||
|
|
||||||
|
const int32 BaseKey = 2050;
|
||||||
|
const float DisplayTime = 1.0f;
|
||||||
|
const FColor MainColor = FColor::Cyan;
|
||||||
|
|
||||||
|
const bool bCapt = bCapturing.load();
|
||||||
|
const bool bEchoSuppressed = EchoSuppressFlag && EchoSuppressFlag->load(std::memory_order_relaxed);
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime,
|
||||||
|
bCapt ? FColor::Green : FColor::Red,
|
||||||
|
FString::Printf(TEXT("=== MIC: %s ==="),
|
||||||
|
bCapt ? TEXT("CAPTURING") : TEXT("STOPPED")));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor,
|
||||||
|
FString::Printf(TEXT(" Device: %s Rate: %d Ch: %d"),
|
||||||
|
CachedDeviceName.IsEmpty() ? TEXT("(none)") : *CachedDeviceName,
|
||||||
|
DeviceSampleRate, DeviceChannels));
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 2, DisplayTime,
|
||||||
|
bEchoSuppressed ? FColor::Yellow : MainColor,
|
||||||
|
FString::Printf(TEXT(" EchoSuppress: %s VolMul: %.2f"),
|
||||||
|
bEchoSuppressed ? TEXT("YES") : TEXT("NO"), VolumeMultiplier));
|
||||||
|
|
||||||
|
// VU meter
|
||||||
|
const float RMS = CurrentRMS.load(std::memory_order_relaxed);
|
||||||
|
const float Peak = PeakLevel.load(std::memory_order_relaxed);
|
||||||
|
const float dB = (RMS > 1e-6f) ? 20.0f * FMath::LogX(10.0f, RMS) : -60.0f;
|
||||||
|
|
||||||
|
// Build text bar: 30 chars wide, mapped from -60dB to 0dB.
|
||||||
|
const int32 BarWidth = 30;
|
||||||
|
const float NormLevel = FMath::Clamp((dB + 60.0f) / 60.0f, 0.0f, 1.0f);
|
||||||
|
const int32 FilledChars = FMath::RoundToInt(NormLevel * BarWidth);
|
||||||
|
|
||||||
|
FString Bar;
|
||||||
|
for (int32 i = 0; i < BarWidth; i++)
|
||||||
|
{
|
||||||
|
Bar += (i < FilledChars) ? TEXT("|") : TEXT(" ");
|
||||||
|
}
|
||||||
|
|
||||||
|
FColor VUColor = MainColor;
|
||||||
|
if (dB > -6.0f) VUColor = FColor::Red;
|
||||||
|
else if (dB > -20.0f) VUColor = FColor::Green;
|
||||||
|
|
||||||
|
GEngine->AddOnScreenDebugMessage(BaseKey + 3, DisplayTime, VUColor,
|
||||||
|
FString::Printf(TEXT(" VU [%s] %.1fdB peak=%.3f"),
|
||||||
|
*Bar, dB, Peak));
|
||||||
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
// Capture control
|
// Capture control
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@ -57,6 +137,7 @@ void UPS_AI_ConvAgent_MicrophoneCaptureComponent::StartCapture()
|
|||||||
{
|
{
|
||||||
DeviceSampleRate = DeviceInfo.PreferredSampleRate;
|
DeviceSampleRate = DeviceInfo.PreferredSampleRate;
|
||||||
DeviceChannels = DeviceInfo.InputChannels;
|
DeviceChannels = DeviceInfo.InputChannels;
|
||||||
|
CachedDeviceName = DeviceInfo.DeviceName;
|
||||||
if (bDebug)
|
if (bDebug)
|
||||||
{
|
{
|
||||||
UE_LOG(LogPS_AI_ConvAgent_Mic, Log, TEXT("Capture device: %s | Rate=%d | Channels=%d"),
|
UE_LOG(LogPS_AI_ConvAgent_Mic, Log, TEXT("Capture device: %s | Rate=%d | Channels=%d"),
|
||||||
@ -99,6 +180,22 @@ void UPS_AI_ConvAgent_MicrophoneCaptureComponent::OnAudioGenerate(
|
|||||||
// Device sends float32 interleaved samples; cast from the void* API.
|
// Device sends float32 interleaved samples; cast from the void* API.
|
||||||
const float* FloatAudio = static_cast<const float*>(InAudio);
|
const float* FloatAudio = static_cast<const float*>(InAudio);
|
||||||
|
|
||||||
|
// Compute RMS from raw input for VU meter (before resample, cheap).
|
||||||
|
{
|
||||||
|
float SumSq = 0.0f;
|
||||||
|
float Peak = 0.0f;
|
||||||
|
const int32 TotalSamples = NumFrames * InNumChannels;
|
||||||
|
for (int32 i = 0; i < TotalSamples; i++)
|
||||||
|
{
|
||||||
|
const float S = FloatAudio[i];
|
||||||
|
SumSq += S * S;
|
||||||
|
const float AbsS = FMath::Abs(S);
|
||||||
|
if (AbsS > Peak) Peak = AbsS;
|
||||||
|
}
|
||||||
|
CurrentRMS.store(FMath::Sqrt(SumSq / FMath::Max(1, TotalSamples)), std::memory_order_relaxed);
|
||||||
|
PeakLevel.store(Peak, std::memory_order_relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
// Resample + downmix to 16000 Hz mono.
|
// Resample + downmix to 16000 Hz mono.
|
||||||
TArray<float> Resampled = ResampleTo16000(FloatAudio, NumFrames, InNumChannels, InSampleRate);
|
TArray<float> Resampled = ResampleTo16000(FloatAudio, NumFrames, InNumChannels, InSampleRate);
|
||||||
|
|
||||||
|
|||||||
@ -16,10 +16,13 @@ class UAnimSequence;
|
|||||||
* AnimSequences from the PS_AI_ConvAgent_BodyExpressionComponent and blends
|
* AnimSequences from the PS_AI_ConvAgent_BodyExpressionComponent and blends
|
||||||
* them per-bone onto the upstream pose (idle, locomotion).
|
* them per-bone onto the upstream pose (idle, locomotion).
|
||||||
*
|
*
|
||||||
* Two modes:
|
* The node uses Override mode: it replaces (lerps) the upstream pose toward
|
||||||
* - bUpperBodyOnly = true (default): only bones at and below BlendRootBone
|
* the expression pose. Use ExcludeBones (default: neck_01) to prevent
|
||||||
* are blended; lower body passes through from the upstream pose.
|
* conflicts with Gaze/Posture on neck/head bones.
|
||||||
* - bUpperBodyOnly = false: the emotion pose is applied to the entire skeleton.
|
*
|
||||||
|
* Region modes:
|
||||||
|
* - bUpperBodyOnly = true (default): only bones at and below BlendRootBone.
|
||||||
|
* - bUpperBodyOnly = false: full skeleton.
|
||||||
*
|
*
|
||||||
* Graph layout:
|
* Graph layout:
|
||||||
* [Upstream body anims (idle, locomotion)] → [PS AI ConvAgent Body Expression] → [Output]
|
* [Upstream body anims (idle, locomotion)] → [PS AI ConvAgent Body Expression] → [Output]
|
||||||
@ -50,6 +53,13 @@ struct PS_AI_CONVAGENT_API FAnimNode_PS_AI_ConvAgent_BodyExpression : public FAn
|
|||||||
ToolTip = "Root bone for upper body blend.\nAll descendants of this bone are blended with the emotion pose.\nDefault: spine_02 (arms, spine, neck, head)."))
|
ToolTip = "Root bone for upper body blend.\nAll descendants of this bone are blended with the emotion pose.\nDefault: spine_02 (arms, spine, neck, head)."))
|
||||||
FName BlendRootBone = FName(TEXT("spine_02"));
|
FName BlendRootBone = FName(TEXT("spine_02"));
|
||||||
|
|
||||||
|
/** Bones to EXCLUDE from body expression blending (each bone and its entire subtree).
|
||||||
|
* Prevents conflicts with Gaze/Posture on neck/head bones.
|
||||||
|
* Default: neck_01 (excludes neck + head, leaving them to Gaze/Posture). */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Settings",
|
||||||
|
meta = (ToolTip = "Exclude these bones and their subtrees from blending.\nDefault: neck_01 prevents conflicts with Gaze/Posture on neck/head."))
|
||||||
|
TArray<FName> ExcludeBones = { FName(TEXT("neck_01")) };
|
||||||
|
|
||||||
// ── FAnimNode_Base interface ──────────────────────────────────────────────
|
// ── FAnimNode_Base interface ──────────────────────────────────────────────
|
||||||
|
|
||||||
virtual void Initialize_AnyThread(const FAnimationInitializeContext& Context) override;
|
virtual void Initialize_AnyThread(const FAnimationInitializeContext& Context) override;
|
||||||
|
|||||||
@ -86,7 +86,7 @@ public:
|
|||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|BodyExpression",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|BodyExpression",
|
||||||
meta = (ClampMin = "0.1", ClampMax = "3.0",
|
meta = (ClampMin = "0.1", ClampMax = "3.0",
|
||||||
ToolTip = "How long (seconds) to crossfade between animations.\n0.5 = snappy, 1.5 = smooth."))
|
ToolTip = "How long (seconds) to crossfade between animations.\n0.5 = snappy, 1.5 = smooth."))
|
||||||
float EmotionBlendDuration = 0.5f;
|
float EmotionBlendDuration = 1.0f;
|
||||||
|
|
||||||
/** Overall blend weight for body expressions. 1.0 = full, 0.5 = subtle. */
|
/** Overall blend weight for body expressions. 1.0 = full, 0.5 = subtle. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|BodyExpression",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|BodyExpression",
|
||||||
@ -232,4 +232,15 @@ private:
|
|||||||
|
|
||||||
/** Cached reference to the agent component on the same Actor. */
|
/** Cached reference to the agent component on the same Actor. */
|
||||||
TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> AgentComponent;
|
TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> AgentComponent;
|
||||||
|
|
||||||
|
// ── Debug event tracking ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Last event name (for on-screen debug display). */
|
||||||
|
FString LastEventName;
|
||||||
|
|
||||||
|
/** World time when the last event fired. */
|
||||||
|
float LastEventWorldTime = 0.0f;
|
||||||
|
|
||||||
|
/** Draw on-screen debug info (called from TickComponent when bDebug). */
|
||||||
|
void DrawDebugHUD() const;
|
||||||
};
|
};
|
||||||
|
|||||||
@ -39,6 +39,14 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAgentStartedSpeaking);
|
|||||||
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAgentStoppedSpeaking);
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAgentStoppedSpeaking);
|
||||||
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAgentInterrupted);
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAgentInterrupted);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fired when audio playback actually starts — AFTER any pre-buffering delay.
|
||||||
|
* Unlike OnAgentStartedSpeaking (which fires at the first audio chunk arrival),
|
||||||
|
* this fires when the AudioComponent calls Play(), meaning the audio is now audible.
|
||||||
|
* Use this when you need animation/behaviour synced with audible speech.
|
||||||
|
*/
|
||||||
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAudioPlaybackStarted);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Fired when the server sends its first agent_chat_response_part — i.e. the moment
|
* Fired when the server sends its first agent_chat_response_part — i.e. the moment
|
||||||
* the LLM starts generating, well before audio arrives.
|
* the LLM starts generating, well before audio arrives.
|
||||||
@ -253,6 +261,13 @@ public:
|
|||||||
meta = (ToolTip = "Fires when the agent starts speaking (first audio chunk). Use for lip-sync or UI feedback."))
|
meta = (ToolTip = "Fires when the agent starts speaking (first audio chunk). Use for lip-sync or UI feedback."))
|
||||||
FOnAgentStartedSpeaking OnAgentStartedSpeaking;
|
FOnAgentStartedSpeaking OnAgentStartedSpeaking;
|
||||||
|
|
||||||
|
/** Fired when audio playback actually starts — AFTER any pre-buffering delay.
|
||||||
|
* Unlike OnAgentStartedSpeaking (first chunk arrival), this fires when audio is audible.
|
||||||
|
* Use this for body/gesture animations that should be synced with audible speech. */
|
||||||
|
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
||||||
|
meta = (ToolTip = "Fires when audio playback actually starts (after pre-buffering).\nSynced with audible speech. Use for body animations."))
|
||||||
|
FOnAudioPlaybackStarted OnAudioPlaybackStarted;
|
||||||
|
|
||||||
/** Fired when the agent finishes playing all audio. Use this to re-open the microphone (in Server VAD mode without interruption) or update UI. */
|
/** Fired when the agent finishes playing all audio. Use this to re-open the microphone (in Server VAD mode without interruption) or update UI. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Fires when the agent finishes speaking. Use to re-open the mic or update UI."))
|
meta = (ToolTip = "Fires when the agent finishes speaking. Use to re-open the mic or update UI."))
|
||||||
@ -660,4 +675,7 @@ private:
|
|||||||
* Called on the server when bNetIsConversing / NetConversatingPawn change,
|
* Called on the server when bNetIsConversing / NetConversatingPawn change,
|
||||||
* because OnRep_ConversationState never fires on the Authority. */
|
* because OnRep_ConversationState never fires on the Authority. */
|
||||||
void ApplyConversationGaze();
|
void ApplyConversationGaze();
|
||||||
|
|
||||||
|
/** Draw on-screen debug info (called from TickComponent when bDebug). */
|
||||||
|
void DrawDebugHUD() const;
|
||||||
};
|
};
|
||||||
|
|||||||
@ -146,6 +146,9 @@ private:
|
|||||||
/** Evaluate all FloatCurves from an AnimSequence at a given time. */
|
/** Evaluate all FloatCurves from an AnimSequence at a given time. */
|
||||||
TMap<FName, float> EvaluateAnimCurves(UAnimSequence* AnimSeq, float Time) const;
|
TMap<FName, float> EvaluateAnimCurves(UAnimSequence* AnimSeq, float Time) const;
|
||||||
|
|
||||||
|
/** Draw on-screen debug info (called from TickComponent when bDebug). */
|
||||||
|
void DrawDebugHUD() const;
|
||||||
|
|
||||||
// ── Animation playback state ─────────────────────────────────────────────
|
// ── Animation playback state ─────────────────────────────────────────────
|
||||||
|
|
||||||
/** Currently playing emotion AnimSequence (looping). */
|
/** Currently playing emotion AnimSequence (looping). */
|
||||||
|
|||||||
@ -95,17 +95,18 @@ public:
|
|||||||
/** Automatically aim at the target's eye bones (MetaHuman FACIAL_L_Eye / FACIAL_R_Eye).
|
/** Automatically aim at the target's eye bones (MetaHuman FACIAL_L_Eye / FACIAL_R_Eye).
|
||||||
* When enabled, TargetOffset is ignored and the agent looks at the midpoint
|
* When enabled, TargetOffset is ignored and the agent looks at the midpoint
|
||||||
* between the target pawn's eye bones.
|
* between the target pawn's eye bones.
|
||||||
* Fallback chain: eye bones → head bone → ActorOrigin + (0,0,FallbackEyeHeight). */
|
* Fallback chain: eye bones → head bone → PawnViewLocation → FallbackEyeHeight. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
||||||
meta = (ToolTip = "Auto-target the pawn's eye bones for eye contact.\nFallback: eye bones > head bone > FallbackEyeHeight."))
|
meta = (ToolTip = "Auto-target the pawn's eye bones for eye contact.\nFallback: eye bones > head bone > PawnViewLocation > FallbackEyeHeight."))
|
||||||
bool bAutoTargetEyes = true;
|
bool bAutoTargetEyes = true;
|
||||||
|
|
||||||
/** Height offset (cm) from the target actor's origin when no eye/head bones are found.
|
/** Height offset (cm) from the target actor's origin when no eye/head bones are found
|
||||||
* Used as fallback when bAutoTargetEyes is true but the target has no skeleton
|
* AND the target is not a Pawn. For Pawns without skeleton, GetPawnViewLocation()
|
||||||
* (e.g. first-person pawn, simple actor). 160 ≈ eye height for a standing human. */
|
* is used instead (accounts for BaseEyeHeight automatically).
|
||||||
|
* Only applies to non-Pawn actors (props, triggers, etc.). */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
||||||
meta = (EditCondition = "bAutoTargetEyes", ClampMin = "0",
|
meta = (EditCondition = "bAutoTargetEyes", ClampMin = "0",
|
||||||
ToolTip = "Height offset (cm) when no eye/head bones exist on the target.\n160 = standing human eye level.\nOnly used as last-resort fallback."))
|
ToolTip = "Height offset (cm) for non-Pawn targets without skeleton.\nPawns use BaseEyeHeight automatically.\nOnly used as last-resort fallback for non-Pawn actors."))
|
||||||
float FallbackEyeHeight = 160.0f;
|
float FallbackEyeHeight = 160.0f;
|
||||||
|
|
||||||
/** Offset from the target actor's origin to aim at.
|
/** Offset from the target actor's origin to aim at.
|
||||||
@ -336,6 +337,9 @@ private:
|
|||||||
/** Map eye yaw/pitch angles to 8 ARKit eye curves. */
|
/** Map eye yaw/pitch angles to 8 ARKit eye curves. */
|
||||||
void UpdateEyeCurves(float EyeYaw, float EyePitch);
|
void UpdateEyeCurves(float EyeYaw, float EyePitch);
|
||||||
|
|
||||||
|
/** Draw on-screen debug info (called from TickComponent when bDebug). */
|
||||||
|
void DrawDebugHUD() const;
|
||||||
|
|
||||||
// ── Smoothed current values (head + eyes, body is actor yaw) ────────────
|
// ── Smoothed current values (head + eyes, body is actor yaw) ────────────
|
||||||
|
|
||||||
/** Current blend alpha (0 = fully inactive/passthrough, 1 = fully active). */
|
/** Current blend alpha (0 = fully inactive/passthrough, 1 = fully active). */
|
||||||
|
|||||||
@ -184,6 +184,13 @@ public:
|
|||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
|
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
|
||||||
void StartConversationWithSelectedAgent();
|
void StartConversationWithSelectedAgent();
|
||||||
|
|
||||||
|
/** Send a text message to the currently selected agent.
|
||||||
|
* The agent responds with audio and text, just as if it heard the player speak.
|
||||||
|
* Handles network relay automatically (works on both server and client).
|
||||||
|
* Does nothing if no agent is selected or not connected. */
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
|
||||||
|
void SendTextToSelectedAgent(const FString& Text);
|
||||||
|
|
||||||
/** Clear the current selection. Automatic selection resumes next tick. */
|
/** Clear the current selection. Automatic selection resumes next tick. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
|
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
|
||||||
void ClearSelection();
|
void ClearSelection();
|
||||||
@ -253,6 +260,9 @@ private:
|
|||||||
/** Clear the agent's GazeComponent target (detach). */
|
/** Clear the agent's GazeComponent target (detach). */
|
||||||
void DetachGazeTarget(TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> Agent);
|
void DetachGazeTarget(TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> Agent);
|
||||||
|
|
||||||
|
/** Draw on-screen debug info (called from TickComponent when bDebug). */
|
||||||
|
void DrawDebugHUD() const;
|
||||||
|
|
||||||
// ── Mic routing ──────────────────────────────────────────────────────────
|
// ── Mic routing ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Forward captured mic audio to the currently selected agent. */
|
/** Forward captured mic audio to the currently selected agent. */
|
||||||
|
|||||||
@ -89,6 +89,17 @@ public:
|
|||||||
ToolTip = "Smoothing speed for viseme transitions.\n35 = smooth and soft, 50 = balanced, 65 = sharp and responsive."))
|
ToolTip = "Smoothing speed for viseme transitions.\n35 = smooth and soft, 50 = balanced, 65 = sharp and responsive."))
|
||||||
float SmoothingSpeed = 50.0f;
|
float SmoothingSpeed = 50.0f;
|
||||||
|
|
||||||
|
// ── Speech Blend ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/** How long (seconds) to blend lip sync curves in/out when speech starts/stops.
|
||||||
|
* When the agent is NOT speaking, lip sync releases all mouth curves so the
|
||||||
|
* FacialExpressionComponent's emotion curves (including mouth) can show through.
|
||||||
|
* When the agent starts speaking, lip sync smoothly takes over mouth curves. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|LipSync",
|
||||||
|
meta = (ClampMin = "0.05", ClampMax = "1.0",
|
||||||
|
ToolTip = "Blend duration when speech starts/stops.\nDuring silence, emotion facial curves control the mouth.\nDuring speech, lip sync takes over."))
|
||||||
|
float SpeechBlendDuration = 0.15f;
|
||||||
|
|
||||||
// ── Emotion Expression Blend ─────────────────────────────────────────────
|
// ── Emotion Expression Blend ─────────────────────────────────────────────
|
||||||
|
|
||||||
/** How much facial emotion (from PS_AI_ConvAgent_FacialExpressionComponent) bleeds through
|
/** How much facial emotion (from PS_AI_ConvAgent_FacialExpressionComponent) bleeds through
|
||||||
@ -163,6 +174,15 @@ public:
|
|||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|LipSync")
|
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|LipSync")
|
||||||
TMap<FName, float> GetCurrentBlendshapes() const { return CurrentBlendshapes; }
|
TMap<FName, float> GetCurrentBlendshapes() const { return CurrentBlendshapes; }
|
||||||
|
|
||||||
|
/** True when the agent is currently producing speech audio.
|
||||||
|
* When false, lip sync releases mouth curves to let emotion curves through. */
|
||||||
|
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|LipSync")
|
||||||
|
bool IsSpeaking() const { return bIsSpeaking; }
|
||||||
|
|
||||||
|
/** Current speech blend alpha (0 = silent/emotion mouth, 1 = lip sync mouth).
|
||||||
|
* Smooth transition between silence and speech states. */
|
||||||
|
float GetSpeechBlendAlpha() const { return SpeechBlendAlpha; }
|
||||||
|
|
||||||
// ── UActorComponent overrides ─────────────────────────────────────────────
|
// ── UActorComponent overrides ─────────────────────────────────────────────
|
||||||
virtual void BeginPlay() override;
|
virtual void BeginPlay() override;
|
||||||
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
|
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
|
||||||
@ -232,6 +252,9 @@ private:
|
|||||||
/** Sample the spectrum magnitude across a frequency range. */
|
/** Sample the spectrum magnitude across a frequency range. */
|
||||||
float GetBandEnergy(float LowFreq, float HighFreq, int32 NumSamples = 8) const;
|
float GetBandEnergy(float LowFreq, float HighFreq, int32 NumSamples = 8) const;
|
||||||
|
|
||||||
|
/** Draw on-screen debug info (called from TickComponent when bDebug). */
|
||||||
|
void DrawDebugHUD() const;
|
||||||
|
|
||||||
// ── State ─────────────────────────────────────────────────────────────────
|
// ── State ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
TUniquePtr<Audio::FSpectrumAnalyzer> SpectrumAnalyzer;
|
TUniquePtr<Audio::FSpectrumAnalyzer> SpectrumAnalyzer;
|
||||||
@ -258,6 +281,14 @@ private:
|
|||||||
// Current blend alpha (0 = fully inactive/passthrough, 1 = fully active).
|
// Current blend alpha (0 = fully inactive/passthrough, 1 = fully active).
|
||||||
float CurrentActiveAlpha = 1.0f;
|
float CurrentActiveAlpha = 1.0f;
|
||||||
|
|
||||||
|
// True when the agent is currently producing speech audio.
|
||||||
|
// Set true in OnAudioChunkReceived, false in OnAgentStopped/OnAgentInterrupted.
|
||||||
|
bool bIsSpeaking = false;
|
||||||
|
|
||||||
|
// Smooth blend alpha for speech state: 0 = not speaking (emotion controls mouth),
|
||||||
|
// 1 = speaking (lip sync controls mouth). Interpolated each tick.
|
||||||
|
float SpeechBlendAlpha = 0.0f;
|
||||||
|
|
||||||
// MetaHuman mode: Face mesh has no morph targets, use animation curves instead.
|
// MetaHuman mode: Face mesh has no morph targets, use animation curves instead.
|
||||||
// Set automatically in BeginPlay when TargetMesh has 0 morph targets.
|
// Set automatically in BeginPlay when TargetMesh has 0 morph targets.
|
||||||
bool bUseCurveMode = false;
|
bool bUseCurveMode = false;
|
||||||
|
|||||||
@ -73,9 +73,12 @@ public:
|
|||||||
// ─────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────
|
||||||
// UActorComponent overrides
|
// UActorComponent overrides
|
||||||
// ─────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────
|
||||||
|
virtual void TickComponent(float DeltaTime, ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction) override;
|
||||||
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
|
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
/** Draw on-screen debug info (called from TickComponent when debug is active). */
|
||||||
|
void DrawDebugHUD() const;
|
||||||
/** Called by the audio capture callback on a background thread. Raw void* per UE 5.3+ API. */
|
/** Called by the audio capture callback on a background thread. Raw void* per UE 5.3+ API. */
|
||||||
void OnAudioGenerate(const void* InAudio, int32 NumFrames,
|
void OnAudioGenerate(const void* InAudio, int32 NumFrames,
|
||||||
int32 InNumChannels, int32 InSampleRate, double StreamTime, bool bOverflow);
|
int32 InNumChannels, int32 InSampleRate, double StreamTime, bool bOverflow);
|
||||||
@ -91,4 +94,11 @@ private:
|
|||||||
// Device sample rate discovered on StartCapture
|
// Device sample rate discovered on StartCapture
|
||||||
int32 DeviceSampleRate = 44100;
|
int32 DeviceSampleRate = 44100;
|
||||||
int32 DeviceChannels = 1;
|
int32 DeviceChannels = 1;
|
||||||
|
|
||||||
|
// RMS level for VU meter (written from audio callback, read on game thread).
|
||||||
|
std::atomic<float> CurrentRMS{0.0f};
|
||||||
|
std::atomic<float> PeakLevel{0.0f};
|
||||||
|
|
||||||
|
// Device name cached on StartCapture for HUD display.
|
||||||
|
FString CachedDeviceName;
|
||||||
};
|
};
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user