j.foucher/PS_AI_Agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: j.foucher:5323da6908ed5bc4f267597242f7682becea2469
Branches Tags
j.foucher:main
..
compare: j.foucher:453450d7ebefc185df9e7f41c73b8f0cce8ddf03
Branches Tags
j.foucher:main

No commits in common. "5323da6908ed5bc4f267597242f7682becea2469" and "453450d7ebefc185df9e7f41c73b8f0cce8ddf03" have entirely different histories.
5323da6908 ... 453450d7eb

26 changed files with 102 additions and 7422 deletions
Show Stats Expand all files Collapse all files

2
.gitignore vendored
View File

@ -20,5 +20,3 @@ Unreal/PS_AI_Agent/*.suo
# Documentation generator script (dev tool, output .pptx is committed instead) # Documentation generator script (dev tool, output .pptx is committed instead)
generate_pptx.py generate_pptx.py
~$PS_AI_Agent_ElevenLabs_Documentation.pptx ~$PS_AI_Agent_ElevenLabs_Documentation.pptx
Unreal/PS_AI_Agent/Build/
Unreal/PS_AI_Agent/Builds/

3
Unreal/PS_AI_Agent/Config/DefaultGame.ini
View File

@ -1,6 +1,3 @@
[/Script/EngineSettings.GeneralProjectSettings] [/Script/EngineSettings.GeneralProjectSettings]
ProjectID=C89AEFD7484597308B8E6EB1C7AE0965 ProjectID=C89AEFD7484597308B8E6EB1C7AE0965
[/Script/UnrealEd.ProjectPackagingSettings]
+DirectoriesToAlwaysStageAsNonUFS=(Path="Certificates")

3228
Unreal/PS_AI_Agent/Content/Certificates/cacert.pem
View File

File diff suppressed because it is too large Load Diff

BIN
Unreal/PS_AI_Agent/Content/MetaHumans/Taro/BP_Taro.uasset
View File

Binary file not shown.

25
Unreal/PS_AI_Agent/Host.bat
View File

@ -1,25 +0,0 @@
@echo off
REM ============================================================================
REM HOST — Lance le jeu en Listen Server
REM Ce PC est le serveur + premier joueur
REM Les autres PC utilisent Join.bat pour se connecter
REM ============================================================================
echo.
echo ========================================
echo PS_AI_Agent — HOST (Listen Server)
echo ========================================
echo.
REM Detect IP address for display
for /f "tokens=2 delims=:" %%a in ('ipconfig ^| findstr /C:"IPv4"') do (
set IP=%%a
goto :found
)
:found
echo Les autres joueurs doivent entrer cette IP: %IP%
echo.
echo Appuyer sur une touche pour lancer...
pause >nul
start "" "%~dp0PS_AI_Agent\Binaries\Win64\PS_AI_Agent.exe" /PS_AI_ConvAgent/Demo_Metahuman?listen -log -windowed -ResX=1920 -ResY=1080 -WinX=0 -WinY=0

25
Unreal/PS_AI_Agent/Join.bat
View File

@ -1,25 +0,0 @@
@echo off
REM ============================================================================
REM JOIN — Se connecter a un Host sur le LAN
REM Entrer l'IP affichee par le Host
REM ============================================================================
echo.
echo ========================================
echo PS_AI_Agent — JOIN (Client)
echo ========================================
echo.
set /p HOST_IP="Entrer l'IP du Host: "
if "%HOST_IP%"=="" (
echo Aucune IP entree. Abandon.
pause
exit /b 1
)
echo.
echo Connexion a %HOST_IP% ...
echo.
start "" "%~dp0PS_AI_Agent\Binaries\Win64\PS_AI_Agent.exe" %HOST_IP% -log -windowed -ResX=1920 -ResY=1080 -WinX=0 -WinY=0

4
Unreal/PS_AI_Agent/PS_AI_Agent.uproject
View File

@ -36,10 +36,6 @@
"Linux", "Linux",
"Android" "Android"
] ]
},
{
"Name": "AudioCapture",
"Enabled": true
} }
] ]
} }

56
Unreal/PS_AI_Agent/Package.bat
View File

@ -1,56 +0,0 @@
@echo off
REM ============================================================================
REM Package PS_AI_Agent — Development build for LAN testing
REM Output: Builds\Windows\
REM ============================================================================
set UE_ROOT=C:\Program Files\Epic Games\UE_5.5
set PROJECT=C:\ASTERION\GIT\PS_AI_Agent\Unreal\PS_AI_Agent\PS_AI_Agent.uproject
set OUTPUT=C:\ASTERION\GIT\PS_AI_Agent\Unreal\PS_AI_Agent\Builds
echo.
echo ========================================
echo Packaging PS_AI_Agent (Development)
echo ========================================
echo.
echo Output: %OUTPUT%\Windows\
echo.
"%UE_ROOT%\Engine\Build\BatchFiles\RunUAT.bat" BuildCookRun ^
-project="%PROJECT%" ^
-noP4 ^
-platform=Win64 ^
-clientconfig=Development ^
-build ^
-cook ^
-stage ^
-pak ^
-archive ^
-archivedirectory="%OUTPUT%" ^
-utf8output
if %ERRORLEVEL% NEQ 0 (
echo.
echo ========================================
echo BUILD FAILED
echo ========================================
pause
exit /b 1
)
echo.
echo ========================================
echo BUILD SUCCESSFUL
echo ========================================
echo.
echo Copier le dossier Builds\Windows\ sur chaque PC.
echo Utiliser Host.bat et Join.bat pour lancer.
echo.
REM Copy Host/Join scripts into the build output
copy /Y "%~dp0Host.bat" "%OUTPUT%\Windows\"
copy /Y "%~dp0Join.bat" "%OUTPUT%\Windows\"
echo Scripts Host.bat et Join.bat copies dans Builds\Windows\
echo.
pause

BIN
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Content/Demo_Metahuman.umap
View File

Binary file not shown.

BIN
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Content/PS_AI_ConvAgent_LipSyncPoseMap.uasset
View File

Binary file not shown.

3228
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Resources/Certificates/cacert.pem
View File

File diff suppressed because it is too large Load Diff

24
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/AnimNode_PS_AI_ConvAgent_FacialExpression.cpp
View File

@ -57,30 +57,6 @@ void FAnimNode_PS_AI_ConvAgent_FacialExpression::Update_AnyThread(const FAnimati
{ {
BasePose.Update(Context); BasePose.Update(Context);
// Lazy lookup: in packaged builds, Initialize_AnyThread may run before
// components are created. Retry discovery until found.
if (!FacialExpressionComponent.IsValid())
{
if (const FAnimInstanceProxy* Proxy = Context.AnimInstanceProxy)
{
if (const USkeletalMeshComponent* SkelMesh = Proxy->GetSkelMeshComponent())
{
if (AActor* Owner = SkelMesh->GetOwner())
{
UPS_AI_ConvAgent_FacialExpressionComponent* Comp =
Owner->FindComponentByClass<UPS_AI_ConvAgent_FacialExpressionComponent>();
if (Comp)
{
FacialExpressionComponent = Comp;
UE_LOG(LogPS_AI_ConvAgent_FacialExprAnimNode, Log,
TEXT("PS AI ConvAgent Facial Expression AnimNode (late) bound to component on %s."),
*Owner->GetName());
}
}
}
}
}
// Cache emotion curves from the facial expression component. // Cache emotion curves from the facial expression component.
// GetCurrentEmotionCurves() returns CTRL_expressions_* curves // GetCurrentEmotionCurves() returns CTRL_expressions_* curves
// extracted from emotion pose AnimSequences, already smoothly blended. // extracted from emotion pose AnimSequences, already smoothly blended.

24
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/AnimNode_PS_AI_ConvAgent_LipSync.cpp
View File

@ -57,30 +57,6 @@ void FAnimNode_PS_AI_ConvAgent_LipSync::Update_AnyThread(const FAnimationUpdateC
{ {
BasePose.Update(Context); BasePose.Update(Context);
// Lazy lookup: in packaged builds, Initialize_AnyThread may run before
// components are created. Retry discovery until found.
if (!LipSyncComponent.IsValid())
{
if (const FAnimInstanceProxy* Proxy = Context.AnimInstanceProxy)
{
if (const USkeletalMeshComponent* SkelMesh = Proxy->GetSkelMeshComponent())
{
if (AActor* Owner = SkelMesh->GetOwner())
{
UPS_AI_ConvAgent_LipSyncComponent* Comp =
Owner->FindComponentByClass<UPS_AI_ConvAgent_LipSyncComponent>();
if (Comp)
{
LipSyncComponent = Comp;
UE_LOG(LogPS_AI_ConvAgent_LipSyncAnimNode, Log,
TEXT("PS AI ConvAgent Lip Sync AnimNode (late) bound to component on %s."),
*Owner->GetName());
}
}
}
}
}
// Cache ARKit blendshape data from the lip sync component. // Cache ARKit blendshape data from the lip sync component.
// GetCurrentBlendshapes() returns ARKit names (jawOpen, mouthFunnel, etc.). // GetCurrentBlendshapes() returns ARKit names (jawOpen, mouthFunnel, etc.).
// These are injected as-is into the pose curves; the downstream // These are injected as-is into the pose curves; the downstream

46
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/AnimNode_PS_AI_ConvAgent_Posture.cpp
View File

@ -196,7 +196,7 @@ void FAnimNode_PS_AI_ConvAgent_Posture::CacheBones_AnyThread(const FAnimationCac
{ {
ChainBoneIndices.Add(FCompactPoseBoneIndex(INDEX_NONE)); ChainBoneIndices.Add(FCompactPoseBoneIndex(INDEX_NONE));
ChainRefPoseRotations.Add(FQuat::Identity); ChainRefPoseRotations.Add(FQuat::Identity);
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose, UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
TEXT(" Chain bone [%d] '%s' NOT FOUND in skeleton!"), TEXT(" Chain bone [%d] '%s' NOT FOUND in skeleton!"),
i, *ChainBoneNames[i].ToString()); i, *ChainBoneNames[i].ToString());
} }
@ -223,14 +223,14 @@ void FAnimNode_PS_AI_ConvAgent_Posture::CacheBones_AnyThread(const FAnimationCac
} }
else else
{ {
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose, UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
TEXT("Head bone '%s' NOT FOUND in skeleton. Available bones:"), TEXT("Head bone '%s' NOT FOUND in skeleton. Available bones:"),
*HeadBoneName.ToString()); *HeadBoneName.ToString());
const int32 NumBones = FMath::Min(RefSkeleton.GetNum(), 10); const int32 NumBones = FMath::Min(RefSkeleton.GetNum(), 10);
for (int32 i = 0; i < NumBones; ++i) for (int32 i = 0; i < NumBones; ++i)
{ {
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose, UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
TEXT(" [%d] %s"), i, *RefSkeleton.GetBoneName(i).ToString()); TEXT(" [%d] %s"), i, *RefSkeleton.GetBoneName(i).ToString());
} }
} }
@ -324,30 +324,6 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Update_AnyThread(const FAnimationUpdateC
{ {
BasePose.Update(Context); BasePose.Update(Context);
// Lazy lookup: in packaged builds, Initialize_AnyThread may run before
// components are created. Retry discovery until found.
if (!PostureComponent.IsValid())
{
if (const FAnimInstanceProxy* Proxy = Context.AnimInstanceProxy)
{
if (const USkeletalMeshComponent* SkelMesh = Proxy->GetSkelMeshComponent())
{
if (AActor* Owner = SkelMesh->GetOwner())
{
UPS_AI_ConvAgent_PostureComponent* Comp =
Owner->FindComponentByClass<UPS_AI_ConvAgent_PostureComponent>();
if (Comp)
{
PostureComponent = Comp;
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Log,
TEXT("PS AI ConvAgent Posture AnimNode (late) bound to component on %s."),
*Owner->GetName());
}
}
}
}
}
// Cache posture data from the component (game thread safe copy). // Cache posture data from the component (game thread safe copy).
// IMPORTANT: Do NOT reset CachedHeadRotation to Identity when the // IMPORTANT: Do NOT reset CachedHeadRotation to Identity when the
// component is momentarily invalid (GC pause, re-registration, etc.). // component is momentarily invalid (GC pause, re-registration, etc.).
@ -422,7 +398,7 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
const bool bHasEyeBones = (LeftEyeBoneIndex.GetInt() != INDEX_NONE); const bool bHasEyeBones = (LeftEyeBoneIndex.GetInt() != INDEX_NONE);
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose, UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
TEXT("[%s] Posture Evaluate: HeadComp=%.2f EyeComp=%.2f DriftComp=%.2f Valid=%s HeadRot=(%s) Eyes=%d Chain=%d Ancestors=%d"), TEXT("[%s] Posture Evaluate: HeadComp=%.2f EyeComp=%.2f DriftComp=%.2f Valid=%s HeadRot=(%s) Eyes=%d Chain=%d Ancestors=%d"),
NodeRole, NodeRole,
CachedHeadCompensation, CachedHeadCompensation,
@ -444,7 +420,7 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
const FQuat Delta = AnimRot * RefRot.Inverse(); const FQuat Delta = AnimRot * RefRot.Inverse();
const FRotator DeltaRot = Delta.Rotator(); const FRotator DeltaRot = Delta.Rotator();
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose, UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
TEXT(" Chain[0] '%s' AnimDelta from RefPose: Y=%.2f P=%.2f R=%.2f (this gets removed at Comp=1)"), TEXT(" Chain[0] '%s' AnimDelta from RefPose: Y=%.2f P=%.2f R=%.2f (this gets removed at Comp=1)"),
*ChainBoneNames[0].ToString(), *ChainBoneNames[0].ToString(),
DeltaRot.Yaw, DeltaRot.Pitch, DeltaRot.Roll); DeltaRot.Yaw, DeltaRot.Pitch, DeltaRot.Roll);
@ -478,7 +454,7 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
} }
if (++EyeDiagLogCounter % 300 == 1) if (++EyeDiagLogCounter % 300 == 1)
{ {
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose, UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
TEXT("[EYE DIAG MODE 1] Forcing CTRL_expressions_eyeLookUpL=1.0 | Left eye should look UP if Control Rig reads CTRL curves")); TEXT("[EYE DIAG MODE 1] Forcing CTRL_expressions_eyeLookUpL=1.0 | Left eye should look UP if Control Rig reads CTRL curves"));
} }
@ -497,7 +473,7 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
} }
if (++EyeDiagLogCounter % 300 == 1) if (++EyeDiagLogCounter % 300 == 1)
{ {
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose, UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
TEXT("[EYE DIAG MODE 2] Forcing ARKit eyeLookUpLeft=1.0 | Left eye should look UP if mh_arkit_mapping_pose drives eyes")); TEXT("[EYE DIAG MODE 2] Forcing ARKit eyeLookUpLeft=1.0 | Left eye should look UP if mh_arkit_mapping_pose drives eyes"));
} }
@ -518,7 +494,7 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
Output.Curve.Set(ZeroCTRL, 0.0f); Output.Curve.Set(ZeroCTRL, 0.0f);
if (++EyeDiagLogCounter % 300 == 1) if (++EyeDiagLogCounter % 300 == 1)
{ {
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose, UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
TEXT("[EYE DIAG MODE 3] Forcing FACIAL_L_Eye bone -25° pitch | Left eye should look UP if bone rotation drives eyes")); TEXT("[EYE DIAG MODE 3] Forcing FACIAL_L_Eye bone -25° pitch | Left eye should look UP if bone rotation drives eyes"));
} }
#endif #endif
@ -600,7 +576,7 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
#if !UE_BUILD_SHIPPING #if !UE_BUILD_SHIPPING
if (EvalDebugFrameCounter % 300 == 1 && CachedEyeCurves.Num() > 0) if (EvalDebugFrameCounter % 300 == 1 && CachedEyeCurves.Num() > 0)
{ {
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose, UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
TEXT(" Eyes: Comp=%.2f → %s (anim weight=%.0f%%, posture weight=%.0f%%)"), TEXT(" Eyes: Comp=%.2f → %s (anim weight=%.0f%%, posture weight=%.0f%%)"),
Comp, Comp,
Comp > 0.001f ? TEXT("BLEND") : TEXT("PASSTHROUGH"), Comp > 0.001f ? TEXT("BLEND") : TEXT("PASSTHROUGH"),
@ -657,7 +633,7 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
if (++DiagLogCounter % 90 == 0) if (++DiagLogCounter % 90 == 0)
{ {
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose, UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
TEXT("DIAG Phase %d: %s | Timer=%.1f"), Phase, PhaseName, DiagTimer); TEXT("DIAG Phase %d: %s | Timer=%.1f"), Phase, PhaseName, DiagTimer);
} }
} }
@ -719,7 +695,7 @@ void FAnimNode_PS_AI_ConvAgent_Posture::Evaluate_AnyThread(FPoseContext& Output)
if (EvalDebugFrameCounter % 300 == 1) if (EvalDebugFrameCounter % 300 == 1)
{ {
const FRotator CorrRot = FullCorrection.Rotator(); const FRotator CorrRot = FullCorrection.Rotator();
UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Verbose, UE_LOG(LogPS_AI_ConvAgent_PostureAnimNode, Warning,
TEXT(" DriftCorrection: Y=%.1f P=%.1f R=%.1f | Comp=%.2f"), TEXT(" DriftCorrection: Y=%.1f P=%.1f R=%.1f | Comp=%.2f"),
CorrRot.Yaw, CorrRot.Pitch, CorrRot.Roll, CorrRot.Yaw, CorrRot.Pitch, CorrRot.Roll,
CachedBodyDriftCompensation); CachedBodyDriftCompensation);

53
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent.cpp
View File

@ -4,11 +4,6 @@
#include "Developer/Settings/Public/ISettingsModule.h" #include "Developer/Settings/Public/ISettingsModule.h"
#include "UObject/UObjectGlobals.h" #include "UObject/UObjectGlobals.h"
#include "UObject/Package.h" #include "UObject/Package.h"
#include "Misc/Paths.h"
#include "HAL/PlatformFileManager.h"
#include "Interfaces/IPluginManager.h"
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent, Log, All);
IMPLEMENT_MODULE(FPS_AI_ConvAgentModule, PS_AI_ConvAgent) IMPLEMENT_MODULE(FPS_AI_ConvAgentModule, PS_AI_ConvAgent)
@ -27,54 +22,6 @@ void FPS_AI_ConvAgentModule::StartupModule()
LOCTEXT("SettingsDescription", "Configure the PS AI ConvAgent - ElevenLabs plugin"), LOCTEXT("SettingsDescription", "Configure the PS AI ConvAgent - ElevenLabs plugin"),
Settings); Settings);
} }
// Ensure the AudioCapture module is loaded (registers WASAPI factory).
// In packaged builds, the module may be linked but not started unless
// explicitly loaded — without it, microphone capture is silent.
if (!FModuleManager::Get().IsModuleLoaded("AudioCapture"))
{
FModuleManager::Get().LoadModule("AudioCapture");
UE_LOG(LogPS_AI_ConvAgent, Log, TEXT("Loaded AudioCapture module for microphone support."));
}
// Ensure SSL certificates are available for packaged builds.
// UE5 expects cacert.pem in [Project]/Content/Certificates/.
// The plugin ships a copy in its Resources/Certificates/ folder.
// If the project doesn't have one, auto-copy it.
EnsureSSLCertificates();
}
void FPS_AI_ConvAgentModule::EnsureSSLCertificates()
{
const FString ProjectCertPath = FPaths::ProjectContentDir() / TEXT("Certificates") / TEXT("cacert.pem");
if (FPlatformFileManager::Get().GetPlatformFile().FileExists(*ProjectCertPath))
{
UE_LOG(LogPS_AI_ConvAgent, Log, TEXT("SSL cacert.pem found at: %s"), *ProjectCertPath);
return;
}
// Try to auto-copy from the plugin's Resources directory.
TSharedPtr<IPlugin> Plugin = IPluginManager::Get().FindPlugin(TEXT("PS_AI_ConvAgent"));
if (Plugin.IsValid())
{
const FString PluginCertPath = Plugin->GetBaseDir()
/ TEXT("Resources") / TEXT("Certificates") / TEXT("cacert.pem");
if (FPlatformFileManager::Get().GetPlatformFile().FileExists(*PluginCertPath))
{
FPlatformFileManager::Get().GetPlatformFile().CreateDirectoryTree(
*(FPaths::ProjectContentDir() / TEXT("Certificates")));
if (FPlatformFileManager::Get().GetPlatformFile().CopyFile(*ProjectCertPath, *PluginCertPath))
{
UE_LOG(LogPS_AI_ConvAgent, Log, TEXT("Copied SSL cacert.pem from plugin to: %s"), *ProjectCertPath);
return;
}
}
}
UE_LOG(LogPS_AI_ConvAgent, Warning, TEXT("SSL cacert.pem not found at: %s. "
"WebSocket wss:// connections may fail in packaged builds. "
"Copy cacert.pem to [Project]/Content/Certificates/."), *ProjectCertPath);
} }
void FPS_AI_ConvAgentModule::ShutdownModule() void FPS_AI_ConvAgentModule::ShutdownModule()

446
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp
View File

@ -10,9 +10,6 @@
#include "Sound/SoundAttenuation.h" #include "Sound/SoundAttenuation.h"
#include "Sound/SoundWaveProcedural.h" #include "Sound/SoundWaveProcedural.h"
#include "GameFramework/Actor.h" #include "GameFramework/Actor.h"
#include "GameFramework/PlayerController.h"
#include "Net/UnrealNetwork.h"
#include "VoiceModule.h"
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_ElevenLabs, Log, All); DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_ElevenLabs, Log, All);
@ -25,9 +22,6 @@ UPS_AI_ConvAgent_ElevenLabsComponent::UPS_AI_ConvAgent_ElevenLabsComponent()
// Tick is used only to detect silence (agent stopped speaking). // Tick is used only to detect silence (agent stopped speaking).
// Disable if not needed for perf. // Disable if not needed for perf.
PrimaryComponentTick.TickInterval = 1.0f / 60.0f; PrimaryComponentTick.TickInterval = 1.0f / 60.0f;
// Enable network replication for this component.
SetIsReplicatedByDefault(true);
} }
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@ -37,7 +31,6 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::BeginPlay()
{ {
Super::BeginPlay(); Super::BeginPlay();
InitAudioPlayback(); InitAudioPlayback();
InitOpusCodec();
// Auto-register with the interaction subsystem so InteractionComponents can discover us. // Auto-register with the interaction subsystem so InteractionComponents can discover us.
if (UWorld* World = GetWorld()) if (UWorld* World = GetWorld())
@ -127,17 +120,6 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::TickComponent(float DeltaTime, ELevel
} }
} }
// Network: detect if the conversating player disconnected (server only).
if (GetOwnerRole() == ROLE_Authority && bNetIsConversing && NetConversatingPlayer)
{
if (!IsValid(NetConversatingPlayer) || !NetConversatingPlayer->GetPawn())
{
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
TEXT("Conversating player disconnected — releasing NPC."));
ServerReleaseConversation_Implementation();
}
}
// Silence detection. // Silence detection.
// ISSUE-8: broadcast OnAgentStoppedSpeaking OUTSIDE AudioQueueLock. // ISSUE-8: broadcast OnAgentStoppedSpeaking OUTSIDE AudioQueueLock.
// OnProceduralUnderflow (audio thread) also acquires AudioQueueLock — if we broadcast // OnProceduralUnderflow (audio thread) also acquires AudioQueueLock — if we broadcast
@ -190,12 +172,6 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::TickComponent(float DeltaTime, ELevel
Tht, LastClosedTurnIndex); Tht, LastClosedTurnIndex);
} }
OnAgentStoppedSpeaking.Broadcast(); OnAgentStoppedSpeaking.Broadcast();
// Network: notify all clients.
if (GetOwnerRole() == ROLE_Authority)
{
MulticastAgentStoppedSpeaking();
}
} }
} }
@ -203,31 +179,6 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::TickComponent(float DeltaTime, ELevel
// Control // Control
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
void UPS_AI_ConvAgent_ElevenLabsComponent::StartConversation() void UPS_AI_ConvAgent_ElevenLabsComponent::StartConversation()
{
if (GetOwnerRole() == ROLE_Authority)
{
// Server (or standalone): open WebSocket directly.
// In networked mode, also set replicated conversation state.
if (GetWorld() && GetWorld()->GetNetMode() != NM_Standalone)
{
APlayerController* PC = GetWorld()->GetFirstPlayerController();
bNetIsConversing = true;
NetConversatingPlayer = PC;
}
StartConversation_Internal();
}
else
{
// Client: request conversation via Server RPC.
APlayerController* PC = GetWorld() ? GetWorld()->GetFirstPlayerController() : nullptr;
if (PC)
{
ServerRequestConversation(PC);
}
}
}
void UPS_AI_ConvAgent_ElevenLabsComponent::StartConversation_Internal()
{ {
if (!WebSocketProxy) if (!WebSocketProxy)
{ {
@ -263,29 +214,17 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StartConversation_Internal()
void UPS_AI_ConvAgent_ElevenLabsComponent::EndConversation() void UPS_AI_ConvAgent_ElevenLabsComponent::EndConversation()
{ {
if (GetOwnerRole() == ROLE_Authority) StopListening();
{ // ISSUE-4: StopListening() may set bWaitingForAgentResponse=true (normal turn end path).
StopListening(); // Cancel it immediately — there is no response coming because we are ending the session.
// ISSUE-4: StopListening() may set bWaitingForAgentResponse=true (normal turn end path). // Without this, TickComponent could fire OnAgentResponseTimeout after EndConversation().
// Cancel it immediately — there is no response coming because we are ending the session. bWaitingForAgentResponse = false;
// Without this, TickComponent could fire OnAgentResponseTimeout after EndConversation(). StopAgentAudio();
bWaitingForAgentResponse = false;
StopAgentAudio();
if (WebSocketProxy) if (WebSocketProxy)
{
WebSocketProxy->Disconnect();
WebSocketProxy = nullptr;
}
// Reset replicated state so other players can talk to this NPC.
bNetIsConversing = false;
NetConversatingPlayer = nullptr;
}
else
{ {
// Client: request release via Server RPC. WebSocketProxy->Disconnect();
ServerReleaseConversation(); WebSocketProxy = nullptr;
} }
} }
@ -374,6 +313,17 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StartListening()
Mic->StartCapture(); Mic->StartCapture();
} }
// Enable body tracking on the sibling PostureComponent (if present).
// Voice input counts as conversation engagement, same as text.
if (AActor* OwnerActor = GetOwner())
{
if (UPS_AI_ConvAgent_PostureComponent* Posture =
OwnerActor->FindComponentByClass<UPS_AI_ConvAgent_PostureComponent>())
{
Posture->bEnableBodyTracking = true;
}
}
const double T = TurnStartTime - SessionStartTime; const double T = TurnStartTime - SessionStartTime;
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log, TEXT("[T+%.2fs] [Turn %d] Mic opened%s — user speaking."), UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log, TEXT("[T+%.2fs] [Turn %d] Mic opened%s — user speaking."),
T, TurnIndex, bExternalMicManagement ? TEXT(" (external)") : TEXT("")); T, TurnIndex, bExternalMicManagement ? TEXT(" (external)") : TEXT(""));
@ -505,14 +455,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::FeedExternalAudio(const TArray<float>
if (MicAccumulationBuffer.Num() >= GetMicChunkMinBytes()) if (MicAccumulationBuffer.Num() >= GetMicChunkMinBytes())
{ {
if (GetOwnerRole() == ROLE_Authority) WebSocketProxy->SendAudioChunk(MicAccumulationBuffer);
{
if (WebSocketProxy) WebSocketProxy->SendAudioChunk(MicAccumulationBuffer);
}
else
{
ServerSendMicAudio(MicAccumulationBuffer);
}
MicAccumulationBuffer.Reset(); MicAccumulationBuffer.Reset();
} }
} }
@ -542,12 +485,6 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleConnected(const FPS_AI_ConvAgen
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log, TEXT("[T+0.00s] Agent connected. ConversationID=%s"), *Info.ConversationID); UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log, TEXT("[T+0.00s] Agent connected. ConversationID=%s"), *Info.ConversationID);
OnAgentConnected.Broadcast(Info); OnAgentConnected.Broadcast(Info);
// Network: notify the requesting remote client that conversation started.
if (GetOwnerRole() == ROLE_Authority && NetConversatingPlayer)
{
ClientConversationStarted(Info);
}
// In Client turn mode (push-to-talk), the user controls listening manually via // In Client turn mode (push-to-talk), the user controls listening manually via
// StartListening()/StopListening(). Auto-starting would leave the mic open // StartListening()/StopListening(). Auto-starting would leave the mic open
// permanently and interfere with push-to-talk — the T-release StopListening() // permanently and interfere with push-to-talk — the T-release StopListening()
@ -557,7 +494,6 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleConnected(const FPS_AI_ConvAgen
{ {
StartListening(); StartListening();
} }
} }
void UPS_AI_ConvAgent_ElevenLabsComponent::HandleDisconnected(int32 StatusCode, const FString& Reason) void UPS_AI_ConvAgent_ElevenLabsComponent::HandleDisconnected(int32 StatusCode, const FString& Reason)
@ -582,13 +518,6 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleDisconnected(int32 StatusCode,
FScopeLock Lock(&MicSendLock); FScopeLock Lock(&MicSendLock);
MicAccumulationBuffer.Reset(); MicAccumulationBuffer.Reset();
} }
// Reset replicated state on disconnect.
if (GetOwnerRole() == ROLE_Authority)
{
bNetIsConversing = false;
NetConversatingPlayer = nullptr;
}
OnAgentDisconnected.Broadcast(StatusCode, Reason); OnAgentDisconnected.Broadcast(StatusCode, Reason);
} }
@ -616,22 +545,6 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray<uint
QueueBefore, (static_cast<float>(QueueBefore) / 16000.0f) * 1000.0f); QueueBefore, (static_cast<float>(QueueBefore) / 16000.0f) * 1000.0f);
} }
// Network: Opus-compress and broadcast to all clients before local playback.
if (GetOwnerRole() == ROLE_Authority && OpusEncoder.IsValid())
{
uint32 CompressedSize = static_cast<uint32>(OpusWorkBuffer.Num());
int32 Remainder = OpusEncoder->Encode(PCMData.GetData(), PCMData.Num(),
OpusWorkBuffer.GetData(), CompressedSize);
if (CompressedSize > 0)
{
TArray<uint8> CompressedData;
CompressedData.Append(OpusWorkBuffer.GetData(), CompressedSize);
MulticastReceiveAgentAudio(CompressedData);
}
}
// Server local playback (Listen Server is also a client).
EnqueueAgentAudio(PCMData); EnqueueAgentAudio(PCMData);
// Forward raw PCM to any listeners (e.g. LipSync component for spectral analysis). // Forward raw PCM to any listeners (e.g. LipSync component for spectral analysis).
OnAgentAudioData.Broadcast(PCMData); OnAgentAudioData.Broadcast(PCMData);
@ -655,11 +568,6 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAgentResponse(const FString& Re
if (bEnableAgentTextResponse) if (bEnableAgentTextResponse)
{ {
OnAgentTextResponse.Broadcast(ResponseText); OnAgentTextResponse.Broadcast(ResponseText);
// Network: broadcast text to all clients for subtitles.
if (GetOwnerRole() == ROLE_Authority)
{
MulticastAgentTextResponse(ResponseText);
}
} }
} }
@ -668,12 +576,6 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleInterrupted()
bWaitingForAgentResponse = false; // Interrupted — no response expected from previous turn. bWaitingForAgentResponse = false; // Interrupted — no response expected from previous turn.
StopAgentAudio(); StopAgentAudio();
OnAgentInterrupted.Broadcast(); OnAgentInterrupted.Broadcast();
// Network: notify all clients.
if (GetOwnerRole() == ROLE_Authority)
{
MulticastAgentInterrupted();
}
} }
void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAgentResponseStarted() void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAgentResponseStarted()
@ -685,18 +587,6 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAgentResponseStarted()
bAgentGenerating = true; bAgentGenerating = true;
bWaitingForAgentResponse = false; // Server is generating — response timeout cancelled. bWaitingForAgentResponse = false; // Server is generating — response timeout cancelled.
// Enable body tracking: the agent is responding, so conversation has started.
// Body tracking was deliberately NOT enabled on selection (only eyes+head)
// so the agent "notices" the player first and turns its body only when engaging.
if (AActor* OwnerActor = GetOwner())
{
if (UPS_AI_ConvAgent_PostureComponent* Posture =
OwnerActor->FindComponentByClass<UPS_AI_ConvAgent_PostureComponent>())
{
Posture->bEnableBodyTracking = true;
}
}
const double Now = FPlatformTime::Seconds(); const double Now = FPlatformTime::Seconds();
const double T = Now - SessionStartTime; const double T = Now - SessionStartTime;
const double LatencyFromTurnEnd = TurnEndTime > 0.0 ? Now - TurnEndTime : 0.0; const double LatencyFromTurnEnd = TurnEndTime > 0.0 ? Now - TurnEndTime : 0.0;
@ -731,12 +621,6 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAgentResponseStarted()
TEXT("[T+%.2fs] [Turn %d] Agent generating. (%.2fs after turn end)"), TEXT("[T+%.2fs] [Turn %d] Agent generating. (%.2fs after turn end)"),
T, LastClosedTurnIndex, LatencyFromTurnEnd); T, LastClosedTurnIndex, LatencyFromTurnEnd);
OnAgentStartedGenerating.Broadcast(); OnAgentStartedGenerating.Broadcast();
// Network: notify all clients.
if (GetOwnerRole() == ROLE_Authority)
{
MulticastAgentStartedGenerating();
}
} }
void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAgentResponsePart(const FString& PartialText) void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAgentResponsePart(const FString& PartialText)
@ -744,11 +628,6 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAgentResponsePart(const FString
if (bEnableAgentPartialResponse) if (bEnableAgentPartialResponse)
{ {
OnAgentPartialResponse.Broadcast(PartialText); OnAgentPartialResponse.Broadcast(PartialText);
// Network: broadcast partial text to all clients.
if (GetOwnerRole() == ROLE_Authority)
{
MulticastAgentPartialResponse(PartialText);
}
} }
} }
@ -949,12 +828,6 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::EnqueueAgentAudio(const TArray<uint8>
OnAgentStartedSpeaking.Broadcast(); OnAgentStartedSpeaking.Broadcast();
// Network: notify all clients that agent started speaking.
if (GetOwnerRole() == ROLE_Authority)
{
MulticastAgentStartedSpeaking();
}
if (AudioPreBufferMs > 0) if (AudioPreBufferMs > 0)
{ {
// Pre-buffer: accumulate audio before starting playback. // Pre-buffer: accumulate audio before starting playback.
@ -1063,12 +936,6 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StopAgentAudio()
T, LastClosedTurnIndex, AgentSpokeDuration, TotalTurnDuration); T, LastClosedTurnIndex, AgentSpokeDuration, TotalTurnDuration);
OnAgentStoppedSpeaking.Broadcast(); OnAgentStoppedSpeaking.Broadcast();
// Network: notify all clients.
if (GetOwnerRole() == ROLE_Authority)
{
MulticastAgentStoppedSpeaking();
}
} }
} }
@ -1102,14 +969,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::OnMicrophoneDataCaptured(const TArray
if (MicAccumulationBuffer.Num() >= GetMicChunkMinBytes()) if (MicAccumulationBuffer.Num() >= GetMicChunkMinBytes())
{ {
if (GetOwnerRole() == ROLE_Authority) WebSocketProxy->SendAudioChunk(MicAccumulationBuffer);
{
if (WebSocketProxy) WebSocketProxy->SendAudioChunk(MicAccumulationBuffer);
}
else
{
ServerSendMicAudio(MicAccumulationBuffer);
}
MicAccumulationBuffer.Reset(); MicAccumulationBuffer.Reset();
} }
} }
@ -1132,265 +992,3 @@ TArray<uint8> UPS_AI_ConvAgent_ElevenLabsComponent::FloatPCMToInt16Bytes(const T
return Out; return Out;
} }
// ─────────────────────────────────────────────────────────────────────────────
// Network: Replication
// ─────────────────────────────────────────────────────────────────────────────
void UPS_AI_ConvAgent_ElevenLabsComponent::GetLifetimeReplicatedProps(
TArray<FLifetimeProperty>& OutLifetimeProps) const
{
Super::GetLifetimeReplicatedProps(OutLifetimeProps);
DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, bNetIsConversing);
DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, NetConversatingPlayer);
DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, CurrentEmotion);
DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, CurrentEmotionIntensity);
}
void UPS_AI_ConvAgent_ElevenLabsComponent::OnRep_ConversationState()
{
if (!bNetIsConversing)
{
// Conversation ended on server — clean up local playback.
if (bAgentSpeaking)
{
StopAgentAudio();
}
}
}
void UPS_AI_ConvAgent_ElevenLabsComponent::OnRep_Emotion()
{
// Fire the existing delegate so FacialExpressionComponent picks it up on clients.
OnAgentEmotionChanged.Broadcast(CurrentEmotion, CurrentEmotionIntensity);
}
// ─────────────────────────────────────────────────────────────────────────────
// Network: Server RPCs
// ─────────────────────────────────────────────────────────────────────────────
void UPS_AI_ConvAgent_ElevenLabsComponent::ServerRequestConversation_Implementation(
APlayerController* RequestingPlayer)
{
if (bNetIsConversing)
{
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
TEXT("ServerRequestConversation denied — NPC is already in conversation."));
ClientConversationFailed(TEXT("NPC is already in conversation with another player."));
return;
}
bNetIsConversing = true;
NetConversatingPlayer = RequestingPlayer;
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
TEXT("ServerRequestConversation granted for %s."),
*GetNameSafe(RequestingPlayer));
StartConversation_Internal();
}
void UPS_AI_ConvAgent_ElevenLabsComponent::ServerReleaseConversation_Implementation()
{
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log, TEXT("ServerReleaseConversation."));
StopListening();
bWaitingForAgentResponse = false;
StopAgentAudio();
if (WebSocketProxy)
{
WebSocketProxy->Disconnect();
WebSocketProxy = nullptr;
}
bNetIsConversing = false;
NetConversatingPlayer = nullptr;
}
void UPS_AI_ConvAgent_ElevenLabsComponent::ServerSendMicAudio_Implementation(
const TArray<uint8>& PCMBytes)
{
if (WebSocketProxy && WebSocketProxy->IsConnected())
{
WebSocketProxy->SendAudioChunk(PCMBytes);
}
}
void UPS_AI_ConvAgent_ElevenLabsComponent::ServerSendTextMessage_Implementation(
const FString& Text)
{
if (WebSocketProxy && WebSocketProxy->IsConnected())
{
WebSocketProxy->SendTextMessage(Text);
}
}
void UPS_AI_ConvAgent_ElevenLabsComponent::ServerRequestInterrupt_Implementation()
{
InterruptAgent();
}
// ─────────────────────────────────────────────────────────────────────────────
// Network: Client RPCs
// ─────────────────────────────────────────────────────────────────────────────
void UPS_AI_ConvAgent_ElevenLabsComponent::ClientConversationStarted_Implementation(
const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& Info)
{
SessionStartTime = FPlatformTime::Seconds();
TurnIndex = 0;
LastClosedTurnIndex = 0;
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
TEXT("[Client] Conversation started. ConversationID=%s"), *Info.ConversationID);
OnAgentConnected.Broadcast(Info);
// Auto-start listening (same logic as HandleConnected).
if (bAutoStartListening && TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server)
{
StartListening();
}
}
void UPS_AI_ConvAgent_ElevenLabsComponent::ClientConversationFailed_Implementation(
const FString& Reason)
{
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
TEXT("[Client] Conversation request failed: %s"), *Reason);
OnAgentError.Broadcast(Reason);
}
// ─────────────────────────────────────────────────────────────────────────────
// Network: Multicast RPCs
// ─────────────────────────────────────────────────────────────────────────────
void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastReceiveAgentAudio_Implementation(
const TArray<uint8>& OpusData)
{
// Server already handled playback in HandleAudioReceived.
if (GetOwnerRole() == ROLE_Authority) return;
if (!OpusDecoder.IsValid()) return;
// LOD: skip audio if too far (unless this client is the speaker).
const float Dist = GetDistanceToLocalPlayer();
const bool bIsSpeaker = IsLocalPlayerConversating();
if (!bIsSpeaker && AudioLODCullDistance > 0.f && Dist > AudioLODCullDistance) return;
// Decode Opus → PCM.
const uint32 MaxDecompressedSize = 16000 * 2; // 1 second of 16kHz 16-bit mono
TArray<uint8> PCMBuffer;
PCMBuffer.SetNumUninitialized(MaxDecompressedSize);
uint32 DecompressedSize = MaxDecompressedSize;
OpusDecoder->Decode(OpusData.GetData(), OpusData.Num(),
PCMBuffer.GetData(), DecompressedSize);
if (DecompressedSize == 0) return;
PCMBuffer.SetNum(DecompressedSize);
// Local playback.
EnqueueAgentAudio(PCMBuffer);
// Feed lip-sync (within LOD or speaker).
if (bIsSpeaker || LipSyncLODDistance <= 0.f || Dist <= LipSyncLODDistance)
{
OnAgentAudioData.Broadcast(PCMBuffer);
}
}
void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastAgentStartedSpeaking_Implementation()
{
if (GetOwnerRole() == ROLE_Authority) return;
bAgentSpeaking = true;
OnAgentStartedSpeaking.Broadcast();
}
void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastAgentStoppedSpeaking_Implementation()
{
if (GetOwnerRole() == ROLE_Authority) return;
bAgentSpeaking = false;
SilentTickCount = 0;
OnAgentStoppedSpeaking.Broadcast();
}
void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastAgentInterrupted_Implementation()
{
if (GetOwnerRole() == ROLE_Authority) return;
StopAgentAudio();
OnAgentInterrupted.Broadcast();
}
void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastAgentTextResponse_Implementation(
const FString& ResponseText)
{
if (GetOwnerRole() == ROLE_Authority) return;
if (bEnableAgentTextResponse)
{
OnAgentTextResponse.Broadcast(ResponseText);
}
}
void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastAgentPartialResponse_Implementation(
const FString& PartialText)
{
if (GetOwnerRole() == ROLE_Authority) return;
if (bEnableAgentPartialResponse)
{
OnAgentPartialResponse.Broadcast(PartialText);
}
}
void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastAgentStartedGenerating_Implementation()
{
if (GetOwnerRole() == ROLE_Authority) return;
OnAgentStartedGenerating.Broadcast();
}
// ─────────────────────────────────────────────────────────────────────────────
// Network: Opus codec
// ─────────────────────────────────────────────────────────────────────────────
void UPS_AI_ConvAgent_ElevenLabsComponent::InitOpusCodec()
{
if (!FVoiceModule::IsAvailable()) return;
FVoiceModule& VoiceModule = FVoiceModule::Get();
if (GetOwnerRole() == ROLE_Authority)
{
OpusEncoder = VoiceModule.CreateVoiceEncoder(
PS_AI_ConvAgent_Audio_ElevenLabs::SampleRate,
PS_AI_ConvAgent_Audio_ElevenLabs::Channels,
EAudioEncodeHint::VoiceEncode_Voice);
}
OpusDecoder = VoiceModule.CreateVoiceDecoder(
PS_AI_ConvAgent_Audio_ElevenLabs::SampleRate,
PS_AI_ConvAgent_Audio_ElevenLabs::Channels);
OpusWorkBuffer.SetNumUninitialized(8 * 1024); // 8 KB scratch buffer for Opus encode/decode
}
// ─────────────────────────────────────────────────────────────────────────────
// Network: Helpers
// ─────────────────────────────────────────────────────────────────────────────
float UPS_AI_ConvAgent_ElevenLabsComponent::GetDistanceToLocalPlayer() const
{
if (UWorld* World = GetWorld())
{
if (APlayerController* PC = World->GetFirstPlayerController())
{
if (APawn* Pawn = PC->GetPawn())
{
return FVector::Dist(GetOwner()->GetActorLocation(),
Pawn->GetActorLocation());
}
}
}
return MAX_FLT;
}
bool UPS_AI_ConvAgent_ElevenLabsComponent::IsLocalPlayerConversating() const
{
if (UWorld* World = GetWorld())
{
if (APlayerController* PC = World->GetFirstPlayerController())
{
return NetConversatingPlayer == PC;
}
}
return false;
}

3
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_EmotionPoseMap.cpp
View File

@ -1,3 +0,0 @@
// Copyright ASTERION. All Rights Reserved.
#include "PS_AI_ConvAgent_EmotionPoseMap.h"

41
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_FacialExpressionComponent.cpp
View File

@ -156,23 +156,15 @@ TMap<FName, float> UPS_AI_ConvAgent_FacialExpressionComponent::EvaluateAnimCurve
TMap<FName, float> CurveValues; TMap<FName, float> CurveValues;
if (!AnimSeq) return CurveValues; if (!AnimSeq) return CurveValues;
// Use UAnimSequence::EvaluateCurveData() — works with both raw (editor) // Use runtime GetCurveData() — GetDataModel() is editor-only in UE 5.5.
// and compressed (cooked/packaged) curve data. const TArray<FFloatCurve>& FloatCurves = AnimSeq->GetCurveData().FloatCurves;
// FBlendedCurve uses TMemStackAllocator which requires an active FMemMark. for (const FFloatCurve& Curve : FloatCurves)
// We're on the game thread (TickComponent), not in the anim evaluation pipeline,
// so we must set up the mark manually.
{ {
FMemMark Mark(FMemStack::Get()); const float Value = Curve.FloatCurve.Eval(Time);
FBlendedCurve BlendedCurve; if (FMath::Abs(Value) > 0.001f)
AnimSeq->EvaluateCurveData(BlendedCurve, Time);
BlendedCurve.ForEachElement([&CurveValues](const UE::Anim::FCurveElement& Element)
{ {
if (FMath::Abs(Element.Value) > 0.001f) CurveValues.Add(Curve.GetName(), Value);
{ }
CurveValues.Add(Element.Name, Element.Value);
}
});
} }
return CurveValues; return CurveValues;
@ -220,25 +212,6 @@ void UPS_AI_ConvAgent_FacialExpressionComponent::TickComponent(
{ {
Super::TickComponent(DeltaTime, TickType, ThisTickFunction); Super::TickComponent(DeltaTime, TickType, ThisTickFunction);
// ── Lazy binding: in packaged builds, BeginPlay may run before the
// ElevenLabsComponent is fully initialized. Retry discovery until bound.
if (!AgentComponent.IsValid())
{
if (AActor* Owner = GetOwner())
{
auto* Agent = Owner->FindComponentByClass<UPS_AI_ConvAgent_ElevenLabsComponent>();
if (Agent)
{
AgentComponent = Agent;
Agent->OnAgentEmotionChanged.AddDynamic(
this, &UPS_AI_ConvAgent_FacialExpressionComponent::OnEmotionChanged);
UE_LOG(LogPS_AI_ConvAgent_FacialExpr, Log,
TEXT("Facial expression (late) bound to agent component on %s."),
*Owner->GetName());
}
}
}
// Nothing to play // Nothing to play
if (!ActiveAnim && !PrevAnim) if (!ActiveAnim && !PrevAnim)
return; return;

35
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp
View File

@ -128,20 +128,11 @@ UPS_AI_ConvAgent_ElevenLabsComponent* UPS_AI_ConvAgent_InteractionComponent::Eva
UPS_AI_ConvAgent_ElevenLabsComponent* CurrentAgent = SelectedAgent.Get(); UPS_AI_ConvAgent_ElevenLabsComponent* CurrentAgent = SelectedAgent.Get();
// Get local player controller for occupied-NPC check.
APlayerController* LocalPC = World->GetFirstPlayerController();
for (UPS_AI_ConvAgent_ElevenLabsComponent* Agent : Agents) for (UPS_AI_ConvAgent_ElevenLabsComponent* Agent : Agents)
{ {
AActor* AgentActor = Agent->GetOwner(); AActor* AgentActor = Agent->GetOwner();
if (!AgentActor) continue; if (!AgentActor) continue;
// Network: skip agents that are in conversation with a different player.
if (Agent->bNetIsConversing && Agent->NetConversatingPlayer != LocalPC)
{
continue;
}
const FVector AgentLocation = AgentActor->GetActorLocation() + FVector(0.0f, 0.0f, AgentEyeLevelOffset); const FVector AgentLocation = AgentActor->GetActorLocation() + FVector(0.0f, 0.0f, AgentEyeLevelOffset);
const FVector ToAgent = AgentLocation - ViewLocation; const FVector ToAgent = AgentLocation - ViewLocation;
const float DistSq = ToAgent.SizeSquared(); const float DistSq = ToAgent.SizeSquared();
@ -252,20 +243,21 @@ void UPS_AI_ConvAgent_InteractionComponent::SetSelectedAgent(UPS_AI_ConvAgent_El
NewAgent->GetOwner() ? *NewAgent->GetOwner()->GetName() : TEXT("(null)")); NewAgent->GetOwner() ? *NewAgent->GetOwner()->GetName() : TEXT("(null)"));
} }
// Network: auto-start conversation if the agent isn't connected yet.
if (!NewAgent->IsConnected() && !NewAgent->bNetIsConversing)
{
NewAgent->StartConversation();
}
// Ensure mic is capturing so we can route audio to the new agent. // Ensure mic is capturing so we can route audio to the new agent.
if (MicComponent && !MicComponent->IsCapturing()) if (MicComponent && !MicComponent->IsCapturing())
{ {
MicComponent->StartCapture(); MicComponent->StartCapture();
} }
// ── Posture: attach (eyes+head only — body tracking is enabled later // ── Listening: start ─────────────────────────────────────────────
// by ElevenLabsComponent when the agent starts responding) ── // Body tracking is enabled by ElevenLabsComponent itself (in StartListening
// and SendTextMessage) so it works for both voice and text input.
if (bAutoManageListening)
{
NewAgent->StartListening();
}
// ── Posture: attach ──────────────────────────────────────────────
if (bAutoManagePosture && World) if (bAutoManagePosture && World)
{ {
// Cancel any pending detach — agent came back before detach fired. // Cancel any pending detach — agent came back before detach fired.
@ -285,15 +277,6 @@ void UPS_AI_ConvAgent_InteractionComponent::SetSelectedAgent(UPS_AI_ConvAgent_El
} }
} }
// ── Listening: start ─────────────────────────────────────────────
// Opens the mic but does NOT enable body tracking. Body tracking
// is enabled later by HandleAgentResponseStarted (agent starts
// responding) or SendTextMessage (explicit text engagement).
if (bAutoManageListening)
{
NewAgent->StartListening();
}
OnAgentSelected.Broadcast(NewAgent); OnAgentSelected.Broadcast(NewAgent);
} }
else else

129
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_LipSyncComponent.cpp
View File

@ -444,59 +444,55 @@ void UPS_AI_ConvAgent_LipSyncComponent::ExtractPoseCurves(const FName& VisemeNam
{ {
if (!AnimSeq) return; if (!AnimSeq) return;
// Use runtime GetCurveData() — GetDataModel() is editor-only in UE 5.5.
TMap<FName, float> CurveValues; TMap<FName, float> CurveValues;
const TArray<FFloatCurve>& FloatCurves = AnimSeq->GetCurveData().FloatCurves;
// Use UAnimSequence::EvaluateCurveData() — works with both raw (editor) for (const FFloatCurve& Curve : FloatCurves)
// and compressed (cooked/packaged) curve data. Extract at frame 0.
// FBlendedCurve uses TMemStackAllocator which requires an active FMemMark.
// We're on the game thread (BeginPlay), not in the anim evaluation pipeline,
// so we must set up the mark manually.
{ {
FMemMark Mark(FMemStack::Get()); const FName CurveName = Curve.GetName();
FBlendedCurve BlendedCurve; const float Value = Curve.FloatCurve.Eval(0.0f);
AnimSeq->EvaluateCurveData(BlendedCurve, 0.0f);
BlendedCurve.ForEachElement([&CurveValues](const UE::Anim::FCurveElement& Element) // Skip curves with near-zero values — not part of this pose's expression
if (FMath::Abs(Value) < 0.001f) continue;
CurveValues.Add(CurveName, Value);
// Auto-detect naming convention from the very first non-zero curve we encounter
if (!bPosesUseCTRLNaming && PoseExtractedCurveMap.Num() == 0 && CurveValues.Num() == 1)
{ {
if (FMath::Abs(Element.Value) >= 0.001f) bPosesUseCTRLNaming = CurveName.ToString().StartsWith(TEXT("CTRL_"));
{
CurveValues.Add(Element.Name, Element.Value);
}
});
}
if (bDebug)
{
UE_LOG(LogPS_AI_ConvAgent_LipSync, Log,
TEXT("Pose '%s' (%s): Extracted %d curves via EvaluateCurveData."),
*VisemeName.ToString(), *AnimSeq->GetName(), CurveValues.Num());
}
// Auto-detect naming convention from the first non-zero curve
if (!bPosesUseCTRLNaming && PoseExtractedCurveMap.Num() == 0 && CurveValues.Num() > 0)
{
for (const auto& Pair : CurveValues)
{
bPosesUseCTRLNaming = Pair.Key.ToString().StartsWith(TEXT("CTRL_"));
if (bDebug) if (bDebug)
{ {
UE_LOG(LogPS_AI_ConvAgent_LipSync, Log, UE_LOG(LogPS_AI_ConvAgent_LipSync, Log,
TEXT("Pose curve naming detected: %s (from curve '%s')"), TEXT("Pose curve naming detected: %s (from curve '%s')"),
bPosesUseCTRLNaming ? TEXT("CTRL_expressions_*") : TEXT("ARKit / other"), bPosesUseCTRLNaming ? TEXT("CTRL_expressions_*") : TEXT("ARKit / other"),
*Pair.Key.ToString()); *CurveName.ToString());
} }
break;
} }
} }
if (CurveValues.Num() > 0) if (CurveValues.Num() > 0)
{ {
PoseExtractedCurveMap.Add(VisemeName, MoveTemp(CurveValues)); PoseExtractedCurveMap.Add(VisemeName, MoveTemp(CurveValues));
if (bDebug)
{
UE_LOG(LogPS_AI_ConvAgent_LipSync, Log,
TEXT("Pose '%s' (%s): Extracted %d non-zero curves."),
*VisemeName.ToString(), *AnimSeq->GetName(),
PoseExtractedCurveMap[VisemeName].Num());
}
} }
else else
{ {
// Empty map: silence pose or no data available // Still add an empty map so we know this viseme was assigned (silence pose)
PoseExtractedCurveMap.Add(VisemeName, TMap<FName, float>()); PoseExtractedCurveMap.Add(VisemeName, TMap<FName, float>());
if (bDebug)
{
UE_LOG(LogPS_AI_ConvAgent_LipSync, Log,
TEXT("Pose '%s' (%s): All curves are zero — neutral/silence pose."),
*VisemeName.ToString(), *AnimSeq->GetName());
}
} }
} }
@ -507,8 +503,11 @@ void UPS_AI_ConvAgent_LipSyncComponent::InitializePoseMappings()
if (!PoseMap) if (!PoseMap)
{ {
UE_LOG(LogPS_AI_ConvAgent_LipSync, Warning, if (bDebug)
TEXT("InitializePoseMappings: PoseMap is NULL — using hardcoded ARKit mapping.")); {
UE_LOG(LogPS_AI_ConvAgent_LipSync, Log,
TEXT("No PoseMap assigned — using hardcoded ARKit mapping."));
}
return; return;
} }
@ -598,22 +597,6 @@ void UPS_AI_ConvAgent_LipSyncComponent::InitializePoseMappings()
TEXT("No phoneme pose AnimSequences assigned — using hardcoded ARKit mapping.")); TEXT("No phoneme pose AnimSequences assigned — using hardcoded ARKit mapping."));
} }
if (bDebug)
{
int32 TotalCurves = 0;
int32 EmptyPoses = 0;
for (const auto& Entry : PoseExtractedCurveMap)
{
TotalCurves += Entry.Value.Num();
if (Entry.Value.Num() == 0) ++EmptyPoses;
}
UE_LOG(LogPS_AI_ConvAgent_LipSync, Log,
TEXT("InitializePoseMappings: PoseMap=%s, Assigned=%d, "
"PoseEntries=%d (empty=%d), TotalCurves=%d, CTRL=%s"),
PoseMap ? *PoseMap->GetName() : TEXT("NULL"),
AssignedCount, PoseExtractedCurveMap.Num(), EmptyPoses,
TotalCurves, bPosesUseCTRLNaming ? TEXT("YES") : TEXT("NO"));
}
} }
void UPS_AI_ConvAgent_LipSyncComponent::EndPlay(const EEndPlayReason::Type EndPlayReason) void UPS_AI_ConvAgent_LipSyncComponent::EndPlay(const EEndPlayReason::Type EndPlayReason)
@ -650,45 +633,6 @@ void UPS_AI_ConvAgent_LipSyncComponent::TickComponent(float DeltaTime, ELevelTic
{ {
Super::TickComponent(DeltaTime, TickType, ThisTickFunction); Super::TickComponent(DeltaTime, TickType, ThisTickFunction);
// ── Lazy binding: in packaged builds, BeginPlay may run before the ────────
// ElevenLabsComponent is fully initialized. Retry discovery until bound.
if (!AgentComponent.IsValid())
{
if (AActor* Owner = GetOwner())
{
UPS_AI_ConvAgent_ElevenLabsComponent* Agent =
Owner->FindComponentByClass<UPS_AI_ConvAgent_ElevenLabsComponent>();
if (Agent)
{
AgentComponent = Agent;
AudioDataHandle = Agent->OnAgentAudioData.AddUObject(
this, &UPS_AI_ConvAgent_LipSyncComponent::OnAudioChunkReceived);
Agent->OnAgentPartialResponse.AddDynamic(
this, &UPS_AI_ConvAgent_LipSyncComponent::OnPartialTextReceived);
Agent->OnAgentTextResponse.AddDynamic(
this, &UPS_AI_ConvAgent_LipSyncComponent::OnTextResponseReceived);
Agent->OnAgentInterrupted.AddDynamic(
this, &UPS_AI_ConvAgent_LipSyncComponent::OnAgentInterrupted);
Agent->OnAgentStoppedSpeaking.AddDynamic(
this, &UPS_AI_ConvAgent_LipSyncComponent::OnAgentStopped);
Agent->bEnableAgentPartialResponse = true;
UE_LOG(LogPS_AI_ConvAgent_LipSync, Log,
TEXT("Lip sync (late) bound to agent component on %s."),
*Owner->GetName());
}
}
}
// Also retry caching the facial expression component if it wasn't found initially.
if (!CachedFacialExprComp.IsValid())
{
if (AActor* Owner = GetOwner())
{
CachedFacialExprComp = Owner->FindComponentByClass<UPS_AI_ConvAgent_FacialExpressionComponent>();
}
}
// ── Consume queued viseme analysis frames at the FFT window rate ───────── // ── Consume queued viseme analysis frames at the FFT window rate ─────────
// Each 512-sample FFT window at 16kHz = 32ms of audio. // Each 512-sample FFT window at 16kHz = 32ms of audio.
// We consume one queued frame every 32ms to match the original audio timing. // We consume one queued frame every 32ms to match the original audio timing.
@ -1073,6 +1017,13 @@ void UPS_AI_ConvAgent_LipSyncComponent::OnAudioChunkReceived(const TArray<uint8>
const int16* Samples = reinterpret_cast<const int16*>(PCMData.GetData()); const int16* Samples = reinterpret_cast<const int16*>(PCMData.GetData());
const int32 NumSamples = PCMData.Num() / sizeof(int16); const int32 NumSamples = PCMData.Num() / sizeof(int16);
static bool bFirstChunkLogged = false;
if (!bFirstChunkLogged)
{
UE_LOG(LogPS_AI_ConvAgent_LipSync, Verbose, TEXT("First audio chunk received: %d bytes (%d samples)"), PCMData.Num(), NumSamples);
bFirstChunkLogged = true;
}
FloatBuffer.Reset(NumSamples); FloatBuffer.Reset(NumSamples);
for (int32 i = 0; i < NumSamples; ++i) for (int32 i = 0; i < NumSamples; ++i)
{ {

3
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_LipSyncPoseMap.cpp
View File

@ -1,3 +0,0 @@
// Copyright ASTERION. All Rights Reserved.
#include "PS_AI_ConvAgent_LipSyncPoseMap.h"

16
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_PostureComponent.cpp
View File

@ -436,11 +436,12 @@ void UPS_AI_ConvAgent_PostureComponent::TickComponent(
} }
} }
// ── Debug (every ~2 seconds, only when bDebug is on) ──────────────── // ── Debug (every ~2 seconds) ─────────────────────────────────────────
if (bDebug && TargetActor) #if !UE_BUILD_SHIPPING
DebugFrameCounter++;
if (DebugFrameCounter % 120 == 0)
{ {
DebugFrameCounter++; if (TargetActor)
if (DebugFrameCounter % 120 == 0)
{ {
const float FacingYaw = Owner->GetActorRotation().Yaw + MeshForwardYawOffset; const float FacingYaw = Owner->GetActorRotation().Yaw + MeshForwardYawOffset;
const FVector TP = TargetActor->GetActorLocation() + TargetOffset; const FVector TP = TargetActor->GetActorLocation() + TargetOffset;
@ -449,14 +450,13 @@ void UPS_AI_ConvAgent_PostureComponent::TickComponent(
const float Delta = FMath::FindDeltaAngleDegrees(FacingYaw, TgtYaw); const float Delta = FMath::FindDeltaAngleDegrees(FacingYaw, TgtYaw);
UE_LOG(LogPS_AI_ConvAgent_Posture, Log, UE_LOG(LogPS_AI_ConvAgent_Posture, Log,
TEXT("Posture [%s -> %s]: Delta=%.1f | Head=%.1f/%.1f | Eyes=%.1f/%.1f | Body: enabled=%s TargetYaw=%.1f ActorYaw=%.1f"), TEXT("Posture [%s -> %s]: Delta=%.1f | Head=%.1f/%.1f | Eyes=%.1f/%.1f | EyeGap=%.1f"),
*Owner->GetName(), *TargetActor->GetName(), *Owner->GetName(), *TargetActor->GetName(),
Delta, Delta,
CurrentHeadYaw, CurrentHeadPitch, CurrentHeadYaw, CurrentHeadPitch,
CurrentEyeYaw, CurrentEyePitch, CurrentEyeYaw, CurrentEyePitch,
bEnableBodyTracking ? TEXT("Y") : TEXT("N"), Delta - CurrentHeadYaw);
TargetBodyWorldYaw,
Owner->GetActorRotation().Yaw);
} }
} }
#endif
} }

3
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent.h
View File

@ -96,7 +96,4 @@ public:
private: private:
UPS_AI_ConvAgent_Settings_ElevenLabs* Settings = nullptr; UPS_AI_ConvAgent_Settings_ElevenLabs* Settings = nullptr;
/** Copy SSL cacert.pem from plugin Resources to project Content if missing. */
void EnsureSSLCertificates();
}; };

116
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h
View File

@ -7,14 +7,12 @@
#include "PS_AI_ConvAgent_Definitions.h" #include "PS_AI_ConvAgent_Definitions.h"
#include "PS_AI_ConvAgent_WebSocket_ElevenLabsProxy.h" #include "PS_AI_ConvAgent_WebSocket_ElevenLabsProxy.h"
#include "Sound/SoundWaveProcedural.h" #include "Sound/SoundWaveProcedural.h"
#include "Interfaces/VoiceCodec.h"
#include <atomic> #include <atomic>
#include "PS_AI_ConvAgent_ElevenLabsComponent.generated.h" #include "PS_AI_ConvAgent_ElevenLabsComponent.generated.h"
class UAudioComponent; class UAudioComponent;
class USoundAttenuation; class USoundAttenuation;
class UPS_AI_ConvAgent_MicrophoneCaptureComponent; class UPS_AI_ConvAgent_MicrophoneCaptureComponent;
class APlayerController;
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
// Delegates exposed to Blueprint // Delegates exposed to Blueprint
@ -272,11 +270,11 @@ public:
FOnAgentClientToolCall OnAgentClientToolCall; FOnAgentClientToolCall OnAgentClientToolCall;
/** The current emotion of the agent, as set by the "set_emotion" client tool. Defaults to Neutral. */ /** The current emotion of the agent, as set by the "set_emotion" client tool. Defaults to Neutral. */
UPROPERTY(ReplicatedUsing = OnRep_Emotion, BlueprintReadOnly, Category = "PS AI ConvAgent|ElevenLabs") UPROPERTY(BlueprintReadOnly, Category = "PS AI ConvAgent|ElevenLabs")
EPS_AI_ConvAgent_Emotion CurrentEmotion = EPS_AI_ConvAgent_Emotion::Neutral; EPS_AI_ConvAgent_Emotion CurrentEmotion = EPS_AI_ConvAgent_Emotion::Neutral;
/** The current emotion intensity. Defaults to Medium. */ /** The current emotion intensity. Defaults to Medium. */
UPROPERTY(ReplicatedUsing = OnRep_Emotion, BlueprintReadOnly, Category = "PS AI ConvAgent|ElevenLabs") UPROPERTY(BlueprintReadOnly, Category = "PS AI ConvAgent|ElevenLabs")
EPS_AI_ConvAgent_EmotionIntensity CurrentEmotionIntensity = EPS_AI_ConvAgent_EmotionIntensity::Medium; EPS_AI_ConvAgent_EmotionIntensity CurrentEmotionIntensity = EPS_AI_ConvAgent_EmotionIntensity::Medium;
// ── Raw audio data (C++ only, used by LipSync component) ──────────────── // ── Raw audio data (C++ only, used by LipSync component) ────────────────
@ -284,93 +282,6 @@ public:
* Used internally by UPS_AI_ConvAgent_LipSyncComponent for spectral analysis. */ * Used internally by UPS_AI_ConvAgent_LipSyncComponent for spectral analysis. */
FOnAgentAudioData OnAgentAudioData; FOnAgentAudioData OnAgentAudioData;
// ── Network state (replicated) ───────────────────────────────────────────
/** True when a player is currently in conversation with this NPC.
* Replicated to all clients so InteractionComponents can skip occupied NPCs. */
UPROPERTY(ReplicatedUsing = OnRep_ConversationState, BlueprintReadOnly, Category = "PS AI ConvAgent|Network")
bool bNetIsConversing = false;
/** The player controller currently in conversation with this NPC (null if free).
* Replicated so each client knows who is speaking (used for posture target, LOD). */
UPROPERTY(ReplicatedUsing = OnRep_ConversationState, BlueprintReadOnly, Category = "PS AI ConvAgent|Network")
TObjectPtr<APlayerController> NetConversatingPlayer = nullptr;
// ── Network LOD ──────────────────────────────────────────────────────────
/** Distance (cm) beyond which remote clients stop receiving agent audio entirely.
* The speaking player always receives full audio regardless of distance. */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Network|LOD",
meta = (ClampMin = "0", ToolTip = "Distance beyond which audio is culled for non-speaking players. 0 = no cull."))
float AudioLODCullDistance = 3000.f;
/** Distance (cm) beyond which remote clients skip lip-sync / emotion processing.
* Audio still plays (if within AudioLODCullDistance) but without facial animation.
* The speaking player always gets full lip-sync regardless of distance. */
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Network|LOD",
meta = (ClampMin = "0", ToolTip = "Distance beyond which lip-sync is skipped for non-speaking players. 0 = no LOD."))
float LipSyncLODDistance = 1500.f;
// ── Network RPCs ─────────────────────────────────────────────────────────
/** Request exclusive conversation with this NPC. Called by clients; the server
* checks availability and opens the WebSocket connection if the NPC is free. */
UFUNCTION(Server, Reliable)
void ServerRequestConversation(APlayerController* RequestingPlayer);
/** Release this NPC so other players can talk to it. */
UFUNCTION(Server, Reliable)
void ServerReleaseConversation();
/** Stream accumulated mic audio from the speaking client to the server.
* Unreliable: minor packet loss is acceptable for audio streaming. */
UFUNCTION(Server, Unreliable)
void ServerSendMicAudio(const TArray<uint8>& PCMBytes);
/** Send a text message via the server's WebSocket connection. */
UFUNCTION(Server, Reliable)
void ServerSendTextMessage(const FString& Text);
/** Request an agent interruption through the server. */
UFUNCTION(Server, Reliable)
void ServerRequestInterrupt();
/** Broadcast Opus-compressed agent audio to all clients. */
UFUNCTION(NetMulticast, Unreliable)
void MulticastReceiveAgentAudio(const TArray<uint8>& OpusData);
/** Notify all clients that the agent started speaking (first audio chunk). */
UFUNCTION(NetMulticast, Reliable)
void MulticastAgentStartedSpeaking();
/** Notify all clients that the agent stopped speaking. */
UFUNCTION(NetMulticast, Reliable)
void MulticastAgentStoppedSpeaking();
/** Notify all clients that the agent was interrupted. */
UFUNCTION(NetMulticast, Reliable)
void MulticastAgentInterrupted();
/** Broadcast the agent's complete text response (subtitles). */
UFUNCTION(NetMulticast, Reliable)
void MulticastAgentTextResponse(const FString& ResponseText);
/** Broadcast streaming partial text (real-time subtitles). */
UFUNCTION(NetMulticast, Reliable)
void MulticastAgentPartialResponse(const FString& PartialText);
/** Notify all clients that the agent started generating (thinking). */
UFUNCTION(NetMulticast, Reliable)
void MulticastAgentStartedGenerating();
/** Notify the requesting client that conversation started successfully. */
UFUNCTION(Client, Reliable)
void ClientConversationStarted(const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& Info);
/** Notify the requesting client that conversation request was denied. */
UFUNCTION(Client, Reliable)
void ClientConversationFailed(const FString& Reason);
// ── Control ─────────────────────────────────────────────────────────────── // ── Control ───────────────────────────────────────────────────────────────
/** /**
@ -449,16 +360,8 @@ public:
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override; virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
virtual void TickComponent(float DeltaTime, ELevelTick TickType, virtual void TickComponent(float DeltaTime, ELevelTick TickType,
FActorComponentTickFunction* ThisTickFunction) override; FActorComponentTickFunction* ThisTickFunction) override;
virtual void GetLifetimeReplicatedProps(TArray<FLifetimeProperty>& OutLifetimeProps) const override;
private: private:
// ── Network OnRep handlers ───────────────────────────────────────────────
UFUNCTION()
void OnRep_ConversationState();
UFUNCTION()
void OnRep_Emotion();
// ── Internal event handlers ─────────────────────────────────────────────── // ── Internal event handlers ───────────────────────────────────────────────
UFUNCTION() UFUNCTION()
void HandleConnected(const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& Info); void HandleConnected(const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& Info);
@ -595,19 +498,4 @@ private:
/** Compute the minimum bytes from the user-facing MicChunkDurationMs. /** Compute the minimum bytes from the user-facing MicChunkDurationMs.
* Formula: bytes = SampleRate * (ms / 1000) * BytesPerSample = 16000 * ms / 1000 * 2 = 32 * ms */ * Formula: bytes = SampleRate * (ms / 1000) * BytesPerSample = 16000 * ms / 1000 * 2 = 32 * ms */
int32 GetMicChunkMinBytes() const { return MicChunkDurationMs * 32; } int32 GetMicChunkMinBytes() const { return MicChunkDurationMs * 32; }
// ── Opus codec (network audio compression) ───────────────────────────────
TSharedPtr<IVoiceEncoder> OpusEncoder; // Server only
TSharedPtr<IVoiceDecoder> OpusDecoder; // All clients
TArray<uint8> OpusWorkBuffer; // Reusable scratch buffer for encode/decode
void InitOpusCodec();
// ── Network helpers ──────────────────────────────────────────────────────
/** Distance from this NPC to the local player's pawn. Returns MAX_FLT if unavailable. */
float GetDistanceToLocalPlayer() const;
/** True if the local player controller is the one currently in conversation. */
bool IsLocalPlayerConversating() const;
/** Internal: performs the actual WebSocket setup (called by both local and RPC paths). */
void StartConversation_Internal();
}; };

5
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_EmotionPoseMap.h
View File

@ -45,9 +45,6 @@ struct PS_AI_CONVAGENT_API FPS_AI_ConvAgent_EmotionPoseSet
* The component plays the AnimSequence in real-time (looping) to drive * The component plays the AnimSequence in real-time (looping) to drive
* emotion-based facial expressions (eyes, eyebrows, cheeks, mouth mood). * emotion-based facial expressions (eyes, eyebrows, cheeks, mouth mood).
* Lip sync overrides the mouth-area curves on top. * Lip sync overrides the mouth-area curves on top.
*
* Curve data is read at runtime via UAnimSequence::EvaluateCurveData()
* which works with both raw (editor) and compressed (cooked) curves.
*/ */
UCLASS(BlueprintType, Blueprintable, DisplayName = "PS AI ConvAgent Emotion Pose Map") UCLASS(BlueprintType, Blueprintable, DisplayName = "PS AI ConvAgent Emotion Pose Map")
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_EmotionPoseMap : public UPrimaryDataAsset class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_EmotionPoseMap : public UPrimaryDataAsset
@ -55,8 +52,6 @@ class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_EmotionPoseMap : public UPrimaryDataA
GENERATED_BODY() GENERATED_BODY()
public: public:
// ── Emotion Poses ─────────────────────────────────────────────────────────
/** Map of emotions to their AnimSequence sets (Normal / Medium / Extreme). /** Map of emotions to their AnimSequence sets (Normal / Medium / Extreme).
* Add entries for each emotion your agent uses (Joy, Sadness, Anger, Surprise, Fear, Disgust). * Add entries for each emotion your agent uses (Joy, Sadness, Anger, Surprise, Fear, Disgust).
* Neutral is recommended — it plays by default at startup (blinking, breathing). */ * Neutral is recommended — it plays by default at startup (blinking, breathing). */

9
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_LipSyncPoseMap.h
View File

@ -17,11 +17,9 @@ class UAnimSequence;
* assign your MHF_* AnimSequences once, then reference this single asset * assign your MHF_* AnimSequences once, then reference this single asset
* on every MetaHuman's PS AI ConvAgent Lip Sync component. * on every MetaHuman's PS AI ConvAgent Lip Sync component.
* *
* The component extracts curve data from each pose at BeginPlay using * The component extracts curve data from each pose at BeginPlay and uses it
* UAnimSequence::EvaluateCurveData() (works with both raw and compressed * to drive lip sync — replacing the hardcoded ARKit blendshape mapping with
* curves in cooked builds) and uses it to drive lip sync — replacing the * artist-crafted poses that coordinate dozens of facial curves.
* hardcoded ARKit blendshape mapping with artist-crafted poses that
* coordinate dozens of facial curves.
*/ */
UCLASS(BlueprintType, Blueprintable, DisplayName = "PS AI ConvAgent Lip Sync Pose Map") UCLASS(BlueprintType, Blueprintable, DisplayName = "PS AI ConvAgent Lip Sync Pose Map")
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_LipSyncPoseMap : public UPrimaryDataAsset class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_LipSyncPoseMap : public UPrimaryDataAsset
@ -29,7 +27,6 @@ class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_LipSyncPoseMap : public UPrimaryDataA
GENERATED_BODY() GENERATED_BODY()
public: public:
// ── Phoneme Poses (15 OVR visemes) ─────────────────────────────────────── // ── Phoneme Poses (15 OVR visemes) ───────────────────────────────────────
/** Silence / neutral pose. Mouth at rest. (OVR viseme: sil) */ /** Silence / neutral pose. Mouth at rest. (OVR viseme: sil) */