Compare commits
2 Commits
eaa52a5c5f
...
e5a32f5997
| Author | SHA1 | Date | |
|---|---|---|---|
| e5a32f5997 | |||
| aea02abe89 |
@ -0,0 +1,3 @@
|
||||
// Copyright ASTERION. All Rights Reserved.
|
||||
|
||||
#include "PS_AI_ConvAgent_ActionSet_ElevenLabs.h"
|
||||
@ -531,15 +531,15 @@ void UPS_AI_ConvAgent_BodyExpressionComponent::DrawDebugHUD() const
|
||||
? TEXT("---")
|
||||
: FString::Printf(TEXT("%s (%.1fs ago)"), *LastEventName, EventAge);
|
||||
|
||||
// Use key offset to avoid colliding with other debug messages
|
||||
// Keys 2000-2010 reserved for BodyExpression
|
||||
const int32 BaseKey = 2000;
|
||||
// Per-actor key range: stride 150 per actor, offset 0 for BodyExpression
|
||||
const int32 BaseKey = 10000 + (GetOwner()->GetUniqueID() % 500) * 150 + 0;
|
||||
const float DisplayTime = 1.0f;
|
||||
const FColor MainColor = FColor::Cyan;
|
||||
const FColor WarnColor = FColor::Yellow;
|
||||
const FString OwnerName = GetOwner()->GetName();
|
||||
|
||||
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, MainColor,
|
||||
FString::Printf(TEXT("=== BODY EXPR: %s ==="), *StateStr));
|
||||
FString::Printf(TEXT("=== BODY EXPR [%s]: %s ==="), *OwnerName, *StateStr));
|
||||
|
||||
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor,
|
||||
FString::Printf(TEXT(" ActivationAlpha: %.3f (target: %s)"),
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
#include "PS_AI_ConvAgent_MicrophoneCaptureComponent.h"
|
||||
#include "PS_AI_ConvAgent_GazeComponent.h"
|
||||
#include "PS_AI_ConvAgent_FacialExpressionComponent.h"
|
||||
#include "PS_AI_ConvAgent_BodyExpressionComponent.h"
|
||||
#include "PS_AI_ConvAgent_LipSyncComponent.h"
|
||||
#include "PS_AI_ConvAgent_InteractionSubsystem.h"
|
||||
#include "PS_AI_ConvAgent_InteractionComponent.h"
|
||||
@ -100,6 +101,61 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::TickComponent(float DeltaTime, ELevel
|
||||
{
|
||||
Super::TickComponent(DeltaTime, TickType, ThisTickFunction);
|
||||
|
||||
// ── ForceDisableConversation blend-out monitoring ─────────────────────
|
||||
// After ForceDisableConversation(), sub-components are blending their
|
||||
// CurrentActiveAlpha to 0. Once all are at (near) zero, fire OnReadyForAction
|
||||
// so the game can start the physical action.
|
||||
if (bWaitingForBlendOut)
|
||||
{
|
||||
BlendOutElapsedTime += DeltaTime;
|
||||
|
||||
static constexpr float NeutralThreshold = 0.01f;
|
||||
bool bAllNeutral = true;
|
||||
bool bTimedOut = (BlendOutElapsedTime >= BlendOutTimeoutSeconds);
|
||||
|
||||
AActor* Owner = GetOwner();
|
||||
if (Owner)
|
||||
{
|
||||
if (auto* Gaze = Owner->FindComponentByClass<UPS_AI_ConvAgent_GazeComponent>())
|
||||
{
|
||||
if (Gaze->GetActiveAlpha() > NeutralThreshold) bAllNeutral = false;
|
||||
}
|
||||
if (auto* LipSync = Owner->FindComponentByClass<UPS_AI_ConvAgent_LipSyncComponent>())
|
||||
{
|
||||
if (LipSync->GetActiveAlpha() > NeutralThreshold) bAllNeutral = false;
|
||||
}
|
||||
if (auto* FacialExpr = Owner->FindComponentByClass<UPS_AI_ConvAgent_FacialExpressionComponent>())
|
||||
{
|
||||
if (FacialExpr->GetActiveAlpha() > NeutralThreshold) bAllNeutral = false;
|
||||
}
|
||||
if (auto* BodyExpr = Owner->FindComponentByClass<UPS_AI_ConvAgent_BodyExpressionComponent>())
|
||||
{
|
||||
if (BodyExpr->GetActiveAlpha() > NeutralThreshold) bAllNeutral = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (bAllNeutral)
|
||||
{
|
||||
bWaitingForBlendOut = false;
|
||||
BlendOutElapsedTime = 0.0f;
|
||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
||||
TEXT("ForceDisableConversation: all components blended to neutral — firing OnReadyForAction for action '%s'."),
|
||||
*PendingActionName);
|
||||
OnReadyForAction.Broadcast(PendingActionName);
|
||||
}
|
||||
else if (bTimedOut)
|
||||
{
|
||||
// Safety timeout — some component didn't reach alpha 0 in time.
|
||||
// Fire OnReadyForAction anyway to avoid blocking the game action forever.
|
||||
bWaitingForBlendOut = false;
|
||||
BlendOutElapsedTime = 0.0f;
|
||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
|
||||
TEXT("ForceDisableConversation: blend-out timed out after %.1fs — not all components at neutral. Firing OnReadyForAction for action '%s' anyway."),
|
||||
BlendOutTimeoutSeconds, *PendingActionName);
|
||||
OnReadyForAction.Broadcast(PendingActionName);
|
||||
}
|
||||
}
|
||||
|
||||
// Response timeout: if the server hasn't started generating within ResponseTimeoutSeconds
|
||||
// after the user stopped speaking, notify Blueprint so it can react (e.g. show "try again").
|
||||
if (bWaitingForAgentResponse && ResponseTimeoutSeconds > 0.0f && TurnEndTime > 0.0)
|
||||
@ -121,27 +177,27 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::TickComponent(float DeltaTime, ELevel
|
||||
|
||||
// Generating timeout (ISSUE-1): if the server sent agent_chat_response_part
|
||||
// (bAgentGenerating=true) but no audio ever arrived (bAgentSpeaking=false),
|
||||
// force-clear bAgentGenerating after 10s so StartListening() is no longer blocked.
|
||||
// Normal path: first audio chunk → EnqueueAgentAudio → bAgentGenerating=false.
|
||||
// This fallback covers the rare case where TTS produces nothing (e.g. empty response).
|
||||
// force-clear bAgentGenerating after GeneratingTimeoutSeconds so StartListening()
|
||||
// is no longer blocked. Time-based to behave consistently across frame rates.
|
||||
if (bAgentGenerating && !bAgentSpeaking)
|
||||
{
|
||||
if (++GeneratingTickCount >= HardSilenceTimeoutTicks)
|
||||
GeneratingTime += DeltaTime;
|
||||
if (GeneratingTime >= GeneratingTimeoutSeconds)
|
||||
{
|
||||
bAgentGenerating = false;
|
||||
GeneratingTickCount = 0;
|
||||
GeneratingTime = 0.0f;
|
||||
if (bDebug)
|
||||
{
|
||||
const double T = FPlatformTime::Seconds() - SessionStartTime;
|
||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
|
||||
TEXT("[T+%.2fs] [Turn %d] Generating timeout (10s) — no audio arrived. Clearing bAgentGenerating."),
|
||||
T, LastClosedTurnIndex);
|
||||
TEXT("[T+%.2fs] [Turn %d] Generating timeout (%.0fs) — no audio arrived. Clearing bAgentGenerating."),
|
||||
T, LastClosedTurnIndex, GeneratingTimeoutSeconds);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
GeneratingTickCount = 0;
|
||||
GeneratingTime = 0.0f;
|
||||
}
|
||||
|
||||
// Pre-buffer timer: start playback after the pre-buffer period expires.
|
||||
@ -265,19 +321,19 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::TickComponent(float DeltaTime, ELevel
|
||||
FScopeLock Lock(&AudioQueueLock);
|
||||
if (AudioQueue.Num() - AudioQueueReadOffset == 0)
|
||||
{
|
||||
SilentTickCount++;
|
||||
SilentTime += DeltaTime;
|
||||
|
||||
// Wait for agent_response (confirms the full response is done) before
|
||||
// declaring the agent stopped. This prevents premature OnAgentStoppedSpeaking
|
||||
// events when ElevenLabs TTS streams audio in multiple batches with gaps
|
||||
// (e.g. for long responses) — without this guard, the Blueprint's
|
||||
// OnAgentStoppedSpeaking handler reopens the mic mid-response.
|
||||
const bool bResponseConfirmed = bAgentResponseReceived && SilentTickCount >= SilenceThresholdTicks;
|
||||
const bool bResponseConfirmed = bAgentResponseReceived && SilentTime >= SilenceThresholdSeconds;
|
||||
|
||||
// Hard-timeout fallback: if agent_response never arrives (or is very late),
|
||||
// stop after 10s of silence to avoid leaving the state machine stuck.
|
||||
// 10s was chosen to bridge observed inter-batch TTS gaps of up to ~5s.
|
||||
const bool bHardTimeout = SilentTickCount >= HardSilenceTimeoutTicks;
|
||||
// Time-based to behave consistently regardless of frame rate.
|
||||
const bool bHardTimeout = SilentTime >= HardSilenceTimeoutSeconds;
|
||||
|
||||
if (bResponseConfirmed || bHardTimeout)
|
||||
{
|
||||
@ -285,13 +341,13 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::TickComponent(float DeltaTime, ELevel
|
||||
bAgentSpeaking = false;
|
||||
bPreBuffering = false; // Cancel pending pre-buffer to prevent stale OnAudioPlaybackStarted.
|
||||
bAgentResponseReceived = false;
|
||||
SilentTickCount = 0;
|
||||
SilentTime = 0.0f;
|
||||
bShouldBroadcastStopped = true;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
SilentTickCount = 0;
|
||||
SilentTime = 0.0f;
|
||||
}
|
||||
}
|
||||
// Broadcast OUTSIDE the lock — Blueprint handlers can execute for arbitrary time.
|
||||
@ -344,6 +400,13 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::TickComponent(float DeltaTime, ELevel
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
void UPS_AI_ConvAgent_ElevenLabsComponent::StartConversation()
|
||||
{
|
||||
if (bConversationDisabledByAction)
|
||||
{
|
||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
|
||||
TEXT("StartConversation: blocked — conversation disabled by ForceDisableConversation(). Call ForceEnableConversation() first."));
|
||||
return;
|
||||
}
|
||||
|
||||
if (GetOwnerRole() == ROLE_Authority)
|
||||
{
|
||||
// Standalone / listen-server: join via the local player controller.
|
||||
@ -405,6 +468,19 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StartConversation_Internal()
|
||||
WebSocketProxy->bSpeculativeTurn = AgentConfig->bSpeculativeTurn;
|
||||
}
|
||||
|
||||
// Merge dynamic variables: per-agent defaults + global context from Settings.
|
||||
// These resolve {{variable_name}} placeholders in the agent's system prompt.
|
||||
WebSocketProxy->DynamicVariables.Empty();
|
||||
if (AgentConfig)
|
||||
{
|
||||
WebSocketProxy->DynamicVariables = AgentConfig->DefaultDynamicVariables;
|
||||
}
|
||||
const UPS_AI_ConvAgent_Settings_ElevenLabs* Settings = FPS_AI_ConvAgentModule::Get().GetSettings();
|
||||
if (Settings && !Settings->GlobalContextPrompt.IsEmpty())
|
||||
{
|
||||
WebSocketProxy->DynamicVariables.Add(TEXT("global_context"), Settings->GlobalContextPrompt);
|
||||
}
|
||||
|
||||
// Resolve AgentID by priority: AgentConfig > component string > project default.
|
||||
FString ResolvedAgentID = AgentID;
|
||||
if (AgentConfig && !AgentConfig->AgentID.IsEmpty())
|
||||
@ -701,6 +777,104 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::InterruptAgent()
|
||||
StopAgentAudio();
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// ForceDisableConversation / ForceEnableConversation
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
void UPS_AI_ConvAgent_ElevenLabsComponent::ForceDisableConversation(const FString& ActionName)
|
||||
{
|
||||
if (bConversationDisabledByAction)
|
||||
{
|
||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
|
||||
TEXT("ForceDisableConversation: already disabled (pending action: %s)."), *PendingActionName);
|
||||
return;
|
||||
}
|
||||
|
||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
||||
TEXT("ForceDisableConversation: shutting down conversation for action '%s', blending to neutral."), *ActionName);
|
||||
|
||||
PendingActionName = ActionName;
|
||||
|
||||
bConversationDisabledByAction = true;
|
||||
bWaitingForBlendOut = true;
|
||||
BlendOutElapsedTime = 0.0f;
|
||||
|
||||
// 1. End the conversation (closes WebSocket, stops audio, removes pawns).
|
||||
// In persistent session mode EndConversation keeps the WebSocket open,
|
||||
// but it still fires OnAgentDisconnected which deactivates sub-components.
|
||||
EndConversation();
|
||||
|
||||
// 2. Also force-close the WebSocket in persistent mode so the agent is truly idle.
|
||||
// This prevents any stale audio or events from arriving during the action.
|
||||
if (bPersistentSession && WebSocketProxy)
|
||||
{
|
||||
bIntentionalDisconnect = true;
|
||||
WebSocketProxy->Disconnect();
|
||||
WebSocketProxy = nullptr;
|
||||
}
|
||||
|
||||
// 3. Reset emotion to neutral so FacialExpression blends back to idle.
|
||||
if (CurrentEmotion != EPS_AI_ConvAgent_Emotion::Neutral)
|
||||
{
|
||||
CurrentEmotion = EPS_AI_ConvAgent_Emotion::Neutral;
|
||||
CurrentEmotionIntensity = EPS_AI_ConvAgent_EmotionIntensity::Medium;
|
||||
OnAgentEmotionChanged.Broadcast(CurrentEmotion, CurrentEmotionIntensity);
|
||||
}
|
||||
|
||||
// 4. Explicitly force bActive = false on ALL sub-components.
|
||||
// The delegate chain (OnAgentDisconnected) may not fire if no conversation
|
||||
// was active, and InteractionComponent may have independently set
|
||||
// Gaze->bActive = true. This guarantees the interpolation target is 0.
|
||||
AActor* Owner = GetOwner();
|
||||
if (Owner)
|
||||
{
|
||||
if (auto* Gaze = Owner->FindComponentByClass<UPS_AI_ConvAgent_GazeComponent>())
|
||||
{
|
||||
Gaze->bActive = false;
|
||||
Gaze->TargetActor = nullptr;
|
||||
Gaze->bEnableBodyTracking = false;
|
||||
}
|
||||
if (auto* LipSync = Owner->FindComponentByClass<UPS_AI_ConvAgent_LipSyncComponent>())
|
||||
{
|
||||
LipSync->bActive = false;
|
||||
}
|
||||
if (auto* FacialExpr = Owner->FindComponentByClass<UPS_AI_ConvAgent_FacialExpressionComponent>())
|
||||
{
|
||||
FacialExpr->bActive = false;
|
||||
}
|
||||
if (auto* BodyExpr = Owner->FindComponentByClass<UPS_AI_ConvAgent_BodyExpressionComponent>())
|
||||
{
|
||||
BodyExpr->bActive = false;
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Sub-components now have bActive = false. Their CurrentActiveAlpha
|
||||
// will interpolate to 0 over ActivationBlendDuration in their TickComponent.
|
||||
// We monitor completion in our own TickComponent (bWaitingForBlendOut)
|
||||
// with a safety timeout in case any component stalls.
|
||||
}
|
||||
|
||||
void UPS_AI_ConvAgent_ElevenLabsComponent::ForceEnableConversation()
|
||||
{
|
||||
if (!bConversationDisabledByAction)
|
||||
{
|
||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
|
||||
TEXT("ForceEnableConversation: not currently disabled."));
|
||||
return;
|
||||
}
|
||||
|
||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
||||
TEXT("ForceEnableConversation: conversation re-enabled."));
|
||||
|
||||
bConversationDisabledByAction = false;
|
||||
bWaitingForBlendOut = false;
|
||||
BlendOutElapsedTime = 0.0f;
|
||||
|
||||
// The InteractionComponent will naturally re-select this agent
|
||||
// and auto-start on the next tick (if bAutoStartConversation is true
|
||||
// and the player is still looking at it).
|
||||
}
|
||||
|
||||
void UPS_AI_ConvAgent_ElevenLabsComponent::FeedExternalAudio(const TArray<float>& FloatPCM)
|
||||
{
|
||||
// Same logic as OnMicrophoneDataCaptured but called from an external source
|
||||
@ -922,7 +1096,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleDisconnected(int32 StatusCode,
|
||||
bIsListening = false;
|
||||
// bAgentSpeaking / bAgentGenerating / bAgentResponseReceived already cleared by StopAgentAudio.
|
||||
bWaitingForAgentResponse = false;
|
||||
GeneratingTickCount = 0;
|
||||
GeneratingTime = 0.0f;
|
||||
TurnIndex = 0;
|
||||
LastClosedTurnIndex = 0;
|
||||
CurrentEmotion = EPS_AI_ConvAgent_Emotion::Neutral;
|
||||
@ -938,6 +1112,11 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleDisconnected(int32 StatusCode,
|
||||
if (!bIntentionalDisconnect && StatusCode != 1000
|
||||
&& MaxReconnectAttempts > 0 && GetOwnerRole() == ROLE_Authority)
|
||||
{
|
||||
// Clean up stale turn state so the reconnected session starts fresh.
|
||||
// StopAgentAudio() already ran above, but these fields are not reset by it:
|
||||
bWaitingForAgentResponse = false;
|
||||
GeneratingTime = 0.0f;
|
||||
|
||||
bWantsReconnect = true;
|
||||
ReconnectAttemptCount = 0;
|
||||
const double Delay = 1.0; // First attempt after 1 second.
|
||||
@ -1259,6 +1438,30 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleClientToolCall(const FPS_AI_Con
|
||||
false);
|
||||
}
|
||||
}
|
||||
else if (ToolCall.ToolName == TEXT("perform_action"))
|
||||
{
|
||||
// Built-in handler for the "perform_action" tool: parse action name, auto-respond, broadcast.
|
||||
const FString* ActionStr = ToolCall.Parameters.Find(TEXT("action"));
|
||||
FString ActionName = ActionStr ? *ActionStr : TEXT("unknown");
|
||||
|
||||
if (bDebug)
|
||||
{
|
||||
const double T = FPlatformTime::Seconds() - SessionStartTime;
|
||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
||||
TEXT("[T+%.2fs] Agent action requested: %s"), T, *ActionName);
|
||||
}
|
||||
|
||||
OnAgentActionRequested.Broadcast(ActionName);
|
||||
|
||||
// Auto-respond so the agent can continue speaking.
|
||||
if (WebSocketProxy)
|
||||
{
|
||||
WebSocketProxy->SendClientToolResult(
|
||||
ToolCall.ToolCallId,
|
||||
FString::Printf(TEXT("action '%s' executed"), *ActionName),
|
||||
false);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Unknown tool — forward to Blueprint for custom handling.
|
||||
@ -1414,7 +1617,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::EnqueueAgentAudio(const TArray<uint8>
|
||||
bAgentGenerating = false; // Agent is now speaking — generation phase is over.
|
||||
bAgentResponseReceived = false; // Reset: wait for agent_response before allowing StopSpeaking.
|
||||
bQueueWasDry = false;
|
||||
SilentTickCount = 0;
|
||||
SilentTime = 0.0f;
|
||||
// Adaptive pre-buffer: record first chunk timing for inter-chunk gap measurement.
|
||||
TurnFirstChunkTime = FPlatformTime::Seconds();
|
||||
TurnFirstChunkBytes = PCMData.Num();
|
||||
@ -1519,7 +1722,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::EnqueueAgentAudio(const TArray<uint8>
|
||||
}
|
||||
OnAudioPlaybackStarted.Broadcast();
|
||||
}
|
||||
SilentTickCount = 0;
|
||||
SilentTime = 0.0f;
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -1547,7 +1750,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::EnqueueAgentAudio(const TArray<uint8>
|
||||
}
|
||||
}
|
||||
// Reset silence counter — new audio arrived, we're not in a gap anymore
|
||||
SilentTickCount = 0;
|
||||
SilentTime = 0.0f;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1586,7 +1789,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StopAgentAudio()
|
||||
if (bAgentSpeaking)
|
||||
{
|
||||
bAgentSpeaking = false;
|
||||
SilentTickCount = 0;
|
||||
SilentTime = 0.0f;
|
||||
bWasSpeaking = true;
|
||||
Now = FPlatformTime::Seconds();
|
||||
}
|
||||
@ -1896,6 +2099,14 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ServerJoinConversation_Implementation
|
||||
APawn* Pawn = RequestingPlayer ? RequestingPlayer->GetPawn() : nullptr;
|
||||
if (!Pawn) return;
|
||||
|
||||
// Block join while force-disabled (ForceDisableConversation active).
|
||||
if (bConversationDisabledByAction)
|
||||
{
|
||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
||||
TEXT("[NET] ServerJoinConversation: blocked — conversation disabled by ForceDisableConversation()."));
|
||||
return;
|
||||
}
|
||||
|
||||
// Already connected? No-op (idempotent).
|
||||
if (NetConnectedPawns.Contains(Pawn)) return;
|
||||
|
||||
@ -2227,7 +2438,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastAgentStoppedSpeaking_Impleme
|
||||
{
|
||||
if (GetOwnerRole() == ROLE_Authority) return;
|
||||
bAgentSpeaking = false;
|
||||
SilentTickCount = 0;
|
||||
SilentTime = 0.0f;
|
||||
OnAgentStoppedSpeaking.Broadcast();
|
||||
}
|
||||
|
||||
@ -2456,17 +2667,19 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::DrawDebugHUD() const
|
||||
{
|
||||
if (!GEngine) return;
|
||||
|
||||
const int32 BaseKey = 2040;
|
||||
// Per-actor key range: stride 150 per actor, offset 60 for ElevenLabs
|
||||
const int32 BaseKey = 10000 + (GetOwner()->GetUniqueID() % 500) * 150 + 60;
|
||||
const float DisplayTime = 1.0f;
|
||||
const FColor MainColor = FColor::Cyan;
|
||||
const FColor WarnColor = FColor::Yellow;
|
||||
const FColor GoodColor = FColor::Green;
|
||||
const FString OwnerName = GetOwner()->GetName();
|
||||
|
||||
const bool bConnected = IsConnected();
|
||||
|
||||
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime,
|
||||
bConnected ? GoodColor : FColor::Red,
|
||||
FString::Printf(TEXT("=== ELEVENLABS: %s ==="),
|
||||
FString::Printf(TEXT("=== ELEVENLABS [%s]: %s ==="), *OwnerName,
|
||||
bConnected ? TEXT("CONNECTED") : TEXT("DISCONNECTED")));
|
||||
|
||||
// Session info
|
||||
@ -2503,8 +2716,8 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::DrawDebugHUD() const
|
||||
// Audio queue (read without lock for debug display — minor race is acceptable)
|
||||
const int32 QueueBytes = FMath::Max(0, AudioQueue.Num() - AudioQueueReadOffset);
|
||||
GEngine->AddOnScreenDebugMessage(BaseKey + 5, DisplayTime, MainColor,
|
||||
FString::Printf(TEXT(" AudioQueue: %d bytes SilentTicks: %d"),
|
||||
QueueBytes, SilentTickCount));
|
||||
FString::Printf(TEXT(" AudioQueue: %d bytes SilentTime: %.2fs"),
|
||||
QueueBytes, SilentTime));
|
||||
|
||||
// Timing
|
||||
const double Now = FPlatformTime::Seconds();
|
||||
@ -2530,6 +2743,38 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::DrawDebugHUD() const
|
||||
FString::Printf(TEXT(" Reconnect: %d/%d attempts%s"),
|
||||
ReconnectAttemptCount, MaxReconnectAttempts,
|
||||
bWantsReconnect ? TEXT(" (ACTIVE)") : TEXT("")));
|
||||
|
||||
// ForceDisable state
|
||||
if (bConversationDisabledByAction || bWaitingForBlendOut)
|
||||
{
|
||||
const FColor DisableColor = FColor::Orange;
|
||||
GEngine->AddOnScreenDebugMessage(BaseKey + 9, DisplayTime, DisableColor,
|
||||
FString::Printf(TEXT(" FORCE DISABLED — action: '%s' blendOut: %s (%.1fs)"),
|
||||
*PendingActionName,
|
||||
bWaitingForBlendOut ? TEXT("WAITING") : TEXT("DONE"),
|
||||
BlendOutElapsedTime));
|
||||
|
||||
// Show per-component alpha values while blending
|
||||
if (bWaitingForBlendOut)
|
||||
{
|
||||
AActor* Owner = GetOwner();
|
||||
if (Owner)
|
||||
{
|
||||
float GazeA = 0.f, LipA = 0.f, FaceA = 0.f, BodyA = 0.f;
|
||||
if (auto* G = Owner->FindComponentByClass<UPS_AI_ConvAgent_GazeComponent>())
|
||||
GazeA = G->GetActiveAlpha();
|
||||
if (auto* L = Owner->FindComponentByClass<UPS_AI_ConvAgent_LipSyncComponent>())
|
||||
LipA = L->GetActiveAlpha();
|
||||
if (auto* F = Owner->FindComponentByClass<UPS_AI_ConvAgent_FacialExpressionComponent>())
|
||||
FaceA = F->GetActiveAlpha();
|
||||
if (auto* B = Owner->FindComponentByClass<UPS_AI_ConvAgent_BodyExpressionComponent>())
|
||||
BodyA = B->GetActiveAlpha();
|
||||
GEngine->AddOnScreenDebugMessage(BaseKey + 10, DisplayTime, DisableColor,
|
||||
FString::Printf(TEXT(" Alpha — Gaze:%.2f Lip:%.2f Face:%.2f Body:%.2f"),
|
||||
GazeA, LipA, FaceA, BodyA));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@ -2572,8 +2817,8 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::DrawLatencyHUD() const
|
||||
{
|
||||
if (!GEngine) return;
|
||||
|
||||
// Separate BaseKey range so it never collides with DrawDebugHUD
|
||||
const int32 BaseKey = 93700;
|
||||
// Per-actor key range: stride 150 per actor, offset 105 for Latency
|
||||
const int32 BaseKey = 10000 + (GetOwner()->GetUniqueID() % 500) * 150 + 105;
|
||||
const float DisplayTime = 1.0f; // long enough to avoid flicker between ticks
|
||||
|
||||
const FColor TitleColor = FColor::Cyan;
|
||||
|
||||
@ -286,6 +286,15 @@ void UPS_AI_ConvAgent_FacialExpressionComponent::TickComponent(
|
||||
AgentComponent = Agent;
|
||||
Agent->OnAgentEmotionChanged.AddDynamic(
|
||||
this, &UPS_AI_ConvAgent_FacialExpressionComponent::OnEmotionChanged);
|
||||
|
||||
// Bind conversation lifecycle — same as BeginPlay path.
|
||||
// Without these, bActive stays false forever when lazy-bound.
|
||||
Agent->OnAgentConnected.AddDynamic(
|
||||
this, &UPS_AI_ConvAgent_FacialExpressionComponent::OnConversationConnected);
|
||||
Agent->OnAgentDisconnected.AddDynamic(
|
||||
this, &UPS_AI_ConvAgent_FacialExpressionComponent::OnConversationDisconnected);
|
||||
bActive = false;
|
||||
CurrentActiveAlpha = 0.0f;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -426,16 +435,18 @@ void UPS_AI_ConvAgent_FacialExpressionComponent::DrawDebugHUD() const
|
||||
{
|
||||
if (!GEngine) return;
|
||||
|
||||
const int32 BaseKey = 2010;
|
||||
// Per-actor key range: stride 150 per actor, offset 15 for FacialExpression
|
||||
const int32 BaseKey = 10000 + (GetOwner()->GetUniqueID() % 500) * 150 + 15;
|
||||
const float DisplayTime = 1.0f;
|
||||
const FColor MainColor = FColor::Cyan;
|
||||
const FColor WarnColor = FColor::Yellow;
|
||||
const FString OwnerName = GetOwner()->GetName();
|
||||
|
||||
// State label
|
||||
FString StateStr = bActive ? TEXT("ACTIVE") : TEXT("INACTIVE");
|
||||
|
||||
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, MainColor,
|
||||
FString::Printf(TEXT("=== FACIAL EXPR: %s ==="), *StateStr));
|
||||
FString::Printf(TEXT("=== FACIAL EXPR [%s]: %s ==="), *OwnerName, *StateStr));
|
||||
|
||||
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor,
|
||||
FString::Printf(TEXT(" ActivationAlpha: %.3f (target: %s)"),
|
||||
|
||||
@ -685,14 +685,16 @@ void UPS_AI_ConvAgent_GazeComponent::DrawDebugHUD() const
|
||||
{
|
||||
if (!GEngine) return;
|
||||
|
||||
const int32 BaseKey = 2020;
|
||||
// Per-actor key range: stride 150 per actor, offset 30 for Gaze
|
||||
const int32 BaseKey = 10000 + (GetOwner()->GetUniqueID() % 500) * 150 + 30;
|
||||
const float DisplayTime = 1.0f;
|
||||
const FColor MainColor = FColor::Cyan;
|
||||
const FString OwnerName = GetOwner()->GetName();
|
||||
|
||||
FString StateStr = bActive ? TEXT("ACTIVE") : TEXT("INACTIVE");
|
||||
|
||||
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, MainColor,
|
||||
FString::Printf(TEXT("=== GAZE: %s ==="), *StateStr));
|
||||
FString::Printf(TEXT("=== GAZE [%s]: %s ==="), *OwnerName, *StateStr));
|
||||
|
||||
FString TargetName = TargetActor ? TargetActor->GetName() : TEXT("(none)");
|
||||
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor,
|
||||
|
||||
@ -150,6 +150,98 @@ void UPS_AI_ConvAgent_InteractionComponent::TickComponent(float DeltaTime, ELeve
|
||||
SetSelectedAgent(BestAgent);
|
||||
}
|
||||
|
||||
// ── Deferred leave: wait for old agent to finish speaking ────────────
|
||||
// After a mid-conversation switch, the old agent keeps speaking and
|
||||
// looking at the player. Once audio finishes, we leave the conversation
|
||||
// but retain gaze until the player walks out of range.
|
||||
if (PendingLeaveAgent.IsValid())
|
||||
{
|
||||
UPS_AI_ConvAgent_ElevenLabsComponent* Pending = PendingLeaveAgent.Get();
|
||||
if (!Pending)
|
||||
{
|
||||
// Agent was destroyed — clean up.
|
||||
PendingLeaveAgent.Reset();
|
||||
}
|
||||
else
|
||||
{
|
||||
bool bForceLeave = false;
|
||||
|
||||
// Check distance — if the player walked away, force-leave immediately.
|
||||
if (AActor* AgentActor = Pending->GetOwner())
|
||||
{
|
||||
FVector ViewLoc, ViewDir;
|
||||
GetPawnViewPoint(ViewLoc, ViewDir);
|
||||
const float DistSq = FVector::DistSquared(ViewLoc, AgentActor->GetActorLocation());
|
||||
bForceLeave = DistSq > FMath::Square(MaxInteractionDistance);
|
||||
}
|
||||
|
||||
if (bForceLeave)
|
||||
{
|
||||
// Player out of range — force stop everything.
|
||||
ExecuteLeave(Pending);
|
||||
CleanupRetainedGaze(Pending);
|
||||
PendingLeaveAgent.Reset();
|
||||
}
|
||||
else if (!Pending->IsAgentSpeakingOrPending())
|
||||
{
|
||||
// Agent truly finished speaking (not just a TTS inter-batch gap) — leave conversation, retain gaze.
|
||||
ExecuteLeave(Pending);
|
||||
GazeRetainedAgent = Pending;
|
||||
PendingLeaveAgent.Reset();
|
||||
|
||||
// Re-activate passive gaze using configured checkboxes.
|
||||
// ExecuteLeave killed the gaze — re-enable passively.
|
||||
if (bAutoManageGaze)
|
||||
{
|
||||
if (UPS_AI_ConvAgent_GazeComponent* Gaze = FindGazeOnAgent(Pending))
|
||||
{
|
||||
Gaze->TargetActor = GetOwner();
|
||||
Gaze->bActive = bPassiveGazeHeadEyes;
|
||||
Gaze->bEnableBodyTracking = bPassiveGazeBody;
|
||||
}
|
||||
}
|
||||
|
||||
if (bDebug)
|
||||
{
|
||||
UE_LOG(LogPS_AI_ConvAgent_Select, Log,
|
||||
TEXT("Deferred leave completed (agent stopped speaking): %s → retaining gaze (head+eyes)"),
|
||||
Pending->GetOwner() ? *Pending->GetOwner()->GetName() : TEXT("(null)"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Gaze retention: clear gaze when player walks out of range ────────
|
||||
if (GazeRetainedAgent.IsValid())
|
||||
{
|
||||
UPS_AI_ConvAgent_ElevenLabsComponent* Retained = GazeRetainedAgent.Get();
|
||||
if (!Retained)
|
||||
{
|
||||
GazeRetainedAgent.Reset();
|
||||
}
|
||||
else if (AActor* AgentActor = Retained->GetOwner())
|
||||
{
|
||||
FVector ViewLoc, ViewDir;
|
||||
GetPawnViewPoint(ViewLoc, ViewDir);
|
||||
const float DistSq = FVector::DistSquared(ViewLoc, AgentActor->GetActorLocation());
|
||||
if (DistSq > FMath::Square(MaxInteractionDistance))
|
||||
{
|
||||
CleanupRetainedGaze(Retained);
|
||||
GazeRetainedAgent.Reset();
|
||||
|
||||
if (bDebug)
|
||||
{
|
||||
UE_LOG(LogPS_AI_ConvAgent_Select, Log,
|
||||
TEXT("Retained gaze cleared (player out of range): %s"),
|
||||
*AgentActor->GetName());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Passive gaze: nearby agents track the player ─────────────────────
|
||||
UpdatePassiveGaze();
|
||||
|
||||
// ── On-screen debug HUD ───────────────────────────────────────────────
|
||||
{
|
||||
const int32 CVarVal = CVarDebugInteraction.GetValueOnGameThread();
|
||||
@ -162,6 +254,103 @@ void UPS_AI_ConvAgent_InteractionComponent::TickComponent(float DeltaTime, ELeve
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Passive gaze (proximity awareness)
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
void UPS_AI_ConvAgent_InteractionComponent::UpdatePassiveGaze()
|
||||
{
|
||||
// Passive gaze disabled or gaze management off — clean up any active passives.
|
||||
if (!bAutoPassiveGaze || !bAutoManageGaze)
|
||||
{
|
||||
for (auto It = PassiveGazeAgents.CreateIterator(); It; ++It)
|
||||
{
|
||||
if (UPS_AI_ConvAgent_ElevenLabsComponent* Agent = It->Get())
|
||||
{
|
||||
if (UPS_AI_ConvAgent_GazeComponent* Gaze = FindGazeOnAgent(Agent))
|
||||
{
|
||||
if (Gaze->TargetActor == GetOwner())
|
||||
{
|
||||
Gaze->TargetActor = nullptr;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
PassiveGazeAgents.Empty();
|
||||
return;
|
||||
}
|
||||
|
||||
UWorld* World = GetWorld();
|
||||
if (!World) return;
|
||||
|
||||
UPS_AI_ConvAgent_InteractionSubsystem* Subsystem = World->GetSubsystem<UPS_AI_ConvAgent_InteractionSubsystem>();
|
||||
if (!Subsystem) return;
|
||||
|
||||
TArray<UPS_AI_ConvAgent_ElevenLabsComponent*> Agents = Subsystem->GetRegisteredAgents();
|
||||
|
||||
FVector ViewLocation, ViewDirection;
|
||||
GetPawnViewPoint(ViewLocation, ViewDirection);
|
||||
|
||||
const float MaxDistSq = MaxInteractionDistance * MaxInteractionDistance;
|
||||
|
||||
// Build the desired set of passive agents.
|
||||
TSet<TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent>> DesiredPassive;
|
||||
for (UPS_AI_ConvAgent_ElevenLabsComponent* Agent : Agents)
|
||||
{
|
||||
// Skip agents already managed by other systems.
|
||||
if (Agent == SelectedAgent.Get()) continue;
|
||||
if (Agent == GazeRetainedAgent.Get()) continue;
|
||||
if (Agent == PendingLeaveAgent.Get()) continue;
|
||||
if (Agent->IsConversationDisabled()) continue;
|
||||
|
||||
AActor* AgentActor = Agent->GetOwner();
|
||||
if (!AgentActor) continue;
|
||||
|
||||
const float DistSq = FVector::DistSquared(ViewLocation, AgentActor->GetActorLocation());
|
||||
if (DistSq > MaxDistSq) continue;
|
||||
|
||||
DesiredPassive.Add(Agent);
|
||||
}
|
||||
|
||||
// Activate gaze on newly passive agents.
|
||||
for (const TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent>& Weak : DesiredPassive)
|
||||
{
|
||||
if (!PassiveGazeAgents.Contains(Weak))
|
||||
{
|
||||
if (UPS_AI_ConvAgent_ElevenLabsComponent* Agent = Weak.Get())
|
||||
{
|
||||
if (UPS_AI_ConvAgent_GazeComponent* Gaze = FindGazeOnAgent(Agent))
|
||||
{
|
||||
Gaze->TargetActor = GetOwner();
|
||||
Gaze->bActive = bPassiveGazeHeadEyes;
|
||||
Gaze->bEnableBodyTracking = bPassiveGazeBody;
|
||||
}
|
||||
Agent->OnPassiveGazeStarted.Broadcast();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Deactivate gaze on agents that are no longer passive.
|
||||
for (auto It = PassiveGazeAgents.CreateIterator(); It; ++It)
|
||||
{
|
||||
if (!DesiredPassive.Contains(*It))
|
||||
{
|
||||
if (UPS_AI_ConvAgent_ElevenLabsComponent* Agent = It->Get())
|
||||
{
|
||||
if (UPS_AI_ConvAgent_GazeComponent* Gaze = FindGazeOnAgent(Agent))
|
||||
{
|
||||
if (Gaze->TargetActor == GetOwner())
|
||||
{
|
||||
Gaze->TargetActor = nullptr;
|
||||
}
|
||||
}
|
||||
Agent->OnPassiveGazeStopped.Broadcast();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
PassiveGazeAgents = MoveTemp(DesiredPassive);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Selection evaluation
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@ -229,9 +418,27 @@ UPS_AI_ConvAgent_ElevenLabsComponent* UPS_AI_ConvAgent_InteractionComponent::Eva
|
||||
float AngleDeg = 0.0f;
|
||||
if (bRequireLookAt)
|
||||
{
|
||||
const FVector DirToAgent = ToAgent.GetSafeNormal();
|
||||
const float Dot = FVector::DotProduct(ViewDirection, DirToAgent);
|
||||
AngleDeg = FMath::RadiansToDegrees(FMath::Acos(FMath::Clamp(Dot, -1.0f, 1.0f)));
|
||||
// Project both vectors onto the horizontal (XY) plane so that
|
||||
// vertical height difference between the player and the agent
|
||||
// does not inflate the selection angle.
|
||||
FVector ViewDirXY = ViewDirection;
|
||||
ViewDirXY.Z = 0.0f;
|
||||
ViewDirXY = ViewDirXY.GetSafeNormal();
|
||||
|
||||
FVector DirToAgentXY = ToAgent;
|
||||
DirToAgentXY.Z = 0.0f;
|
||||
DirToAgentXY = DirToAgentXY.GetSafeNormal();
|
||||
|
||||
if (ViewDirXY.IsNearlyZero() || DirToAgentXY.IsNearlyZero())
|
||||
{
|
||||
// Degenerate case: looking straight up/down or agent exactly above/below.
|
||||
AngleDeg = 180.0f;
|
||||
}
|
||||
else
|
||||
{
|
||||
const float Dot = FVector::DotProduct(ViewDirXY, DirToAgentXY);
|
||||
AngleDeg = FMath::RadiansToDegrees(FMath::Acos(FMath::Clamp(Dot, -1.0f, 1.0f)));
|
||||
}
|
||||
|
||||
// Use wider sticky angle for the currently selected agent.
|
||||
const float ConeThreshold = (Agent == CurrentAgent) ? SelectionStickyAngle : ViewConeHalfAngle;
|
||||
@ -327,62 +534,113 @@ void UPS_AI_ConvAgent_InteractionComponent::SetSelectedAgent(UPS_AI_ConvAgent_El
|
||||
|
||||
// ── Conversation: leave shared conversation if auto-started ─────
|
||||
// Use Leave instead of End so other players can keep talking to the agent.
|
||||
// When switching agents mid-conversation, we defer the Leave and retain
|
||||
// the gaze so the old agent can finish speaking and keep looking at the
|
||||
// player until they walk out of interaction range.
|
||||
if (bAutoStartConversation && (OldAgent->IsConnected() || OldAgent->bNetIsConversing))
|
||||
{
|
||||
if (GetOwnerRole() == ROLE_Authority || (GetWorld() && GetWorld()->GetNetMode() == NM_Standalone))
|
||||
// If a previous pending leave exists, handle it.
|
||||
if (UPS_AI_ConvAgent_ElevenLabsComponent* PrevPending = PendingLeaveAgent.Get())
|
||||
{
|
||||
APlayerController* PC = nullptr;
|
||||
if (APawn* Pawn = Cast<APawn>(GetOwner()))
|
||||
if (PrevPending == NewAgent)
|
||||
{
|
||||
PC = Cast<APlayerController>(Pawn->GetController());
|
||||
// Player is switching back to the pending agent (A→B→A).
|
||||
// Cancel the deferred leave — don't execute it, the player
|
||||
// is coming back to this agent and the conversation is still alive.
|
||||
PendingLeaveAgent.Reset();
|
||||
|
||||
if (bDebug)
|
||||
{
|
||||
UE_LOG(LogPS_AI_ConvAgent_Select, Log,
|
||||
TEXT(" Cancelled pending leave (switching back to same agent): %s"),
|
||||
PrevPending->GetOwner() ? *PrevPending->GetOwner()->GetName() : TEXT("(null)"));
|
||||
}
|
||||
}
|
||||
if (PC)
|
||||
else
|
||||
{
|
||||
OldAgent->ServerLeaveConversation_Implementation(PC);
|
||||
// Different agent — force-complete the old pending leave.
|
||||
ExecuteLeave(PrevPending);
|
||||
CleanupRetainedGaze(PrevPending);
|
||||
PendingLeaveAgent.Reset();
|
||||
}
|
||||
}
|
||||
// Similarly, clean up any existing retained gaze (unless it's the agent
|
||||
// we're about to re-select — keep gaze alive during the transition).
|
||||
if (UPS_AI_ConvAgent_ElevenLabsComponent* PrevRetained = GazeRetainedAgent.Get())
|
||||
{
|
||||
if (PrevRetained != NewAgent)
|
||||
{
|
||||
CleanupRetainedGaze(PrevRetained);
|
||||
GazeRetainedAgent.Reset();
|
||||
}
|
||||
}
|
||||
|
||||
if (OldAgent->IsAgentSpeakingOrPending())
|
||||
{
|
||||
// Agent is still speaking (or generating, waiting for more audio) — defer the Leave.
|
||||
// Gaze and body tracking stay active so the agent keeps
|
||||
// looking at the player while finishing its sentence.
|
||||
PendingLeaveAgent = OldAgent;
|
||||
|
||||
if (bDebug)
|
||||
{
|
||||
UE_LOG(LogPS_AI_ConvAgent_Select, Log,
|
||||
TEXT(" Deferred leave (agent still speaking): %s"),
|
||||
OldAgent->GetOwner() ? *OldAgent->GetOwner()->GetName() : TEXT("(null)"));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
ServerRelayLeaveConversation(OldAgent->GetOwner());
|
||||
// Agent is silent — leave immediately, but retain gaze
|
||||
// until the player walks out of interaction range.
|
||||
ExecuteLeave(OldAgent);
|
||||
GazeRetainedAgent = OldAgent;
|
||||
|
||||
// Re-activate passive gaze using configured checkboxes.
|
||||
// ExecuteLeave → ServerLeaveConversation → ApplyConversationGaze()
|
||||
// already killed bActive and cleared TargetActor. Override that:
|
||||
// the agent should keep looking at the player passively.
|
||||
if (bAutoManageGaze)
|
||||
{
|
||||
if (UPS_AI_ConvAgent_GazeComponent* Gaze = FindGazeOnAgent(OldAgent))
|
||||
{
|
||||
Gaze->TargetActor = GetOwner();
|
||||
Gaze->bActive = bPassiveGazeHeadEyes;
|
||||
Gaze->bEnableBodyTracking = bPassiveGazeBody;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (bAutoManageListening)
|
||||
{
|
||||
OldAgent->StopListening();
|
||||
}
|
||||
|
||||
// Disable body tracking on deselection — but only if we were the
|
||||
// one who set the TargetActor. The conversation system (OnRep or
|
||||
// server ApplyConversationGaze) may have set TargetActor to a
|
||||
// different player; don't overwrite that.
|
||||
if (bAutoManageGaze)
|
||||
{
|
||||
if (UPS_AI_ConvAgent_GazeComponent* Gaze = FindGazeOnAgent(OldAgent))
|
||||
// No conversation was active — still detach gaze normally.
|
||||
if (bAutoManageGaze)
|
||||
{
|
||||
if (Gaze->TargetActor == GetOwner())
|
||||
if (UPS_AI_ConvAgent_GazeComponent* Gaze = FindGazeOnAgent(OldAgent))
|
||||
{
|
||||
Gaze->bEnableBodyTracking = false;
|
||||
if (Gaze->TargetActor == GetOwner())
|
||||
{
|
||||
Gaze->bEnableBodyTracking = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Gaze: detach ────────────────────────────────────────────────
|
||||
if (bAutoManageGaze && World)
|
||||
{
|
||||
// Cancel any pending gaze attach — agent left before attach fired.
|
||||
World->GetTimerManager().ClearTimer(GazeAttachTimerHandle);
|
||||
|
||||
if (GazeDetachDelay > 0.0f)
|
||||
if (bAutoManageGaze && World)
|
||||
{
|
||||
TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> WeakOld = OldAgent;
|
||||
World->GetTimerManager().SetTimer(GazeDetachTimerHandle,
|
||||
FTimerDelegate::CreateUObject(this,
|
||||
&UPS_AI_ConvAgent_InteractionComponent::DetachGazeTarget, WeakOld),
|
||||
GazeDetachDelay, false);
|
||||
}
|
||||
else
|
||||
{
|
||||
DetachGazeTarget(OldAgent);
|
||||
World->GetTimerManager().ClearTimer(GazeAttachTimerHandle);
|
||||
if (GazeDetachDelay > 0.0f)
|
||||
{
|
||||
TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> WeakOld = OldAgent;
|
||||
World->GetTimerManager().SetTimer(GazeDetachTimerHandle,
|
||||
FTimerDelegate::CreateUObject(this,
|
||||
&UPS_AI_ConvAgent_InteractionComponent::DetachGazeTarget, WeakOld),
|
||||
GazeDetachDelay, false);
|
||||
}
|
||||
else
|
||||
{
|
||||
DetachGazeTarget(OldAgent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -392,6 +650,26 @@ void UPS_AI_ConvAgent_InteractionComponent::SetSelectedAgent(UPS_AI_ConvAgent_El
|
||||
// Select new agent.
|
||||
if (NewAgent)
|
||||
{
|
||||
// If the player re-selects an agent that was pending leave or had retained
|
||||
// gaze, cancel the deferred state — the player is coming back to it.
|
||||
if (PendingLeaveAgent.Get() == NewAgent)
|
||||
{
|
||||
PendingLeaveAgent.Reset();
|
||||
if (bDebug)
|
||||
{
|
||||
UE_LOG(LogPS_AI_ConvAgent_Select, Log,
|
||||
TEXT(" Cancelled pending leave (player re-selected): %s"),
|
||||
NewAgent->GetOwner() ? *NewAgent->GetOwner()->GetName() : TEXT("(null)"));
|
||||
}
|
||||
}
|
||||
if (GazeRetainedAgent.Get() == NewAgent)
|
||||
{
|
||||
GazeRetainedAgent.Reset();
|
||||
}
|
||||
|
||||
// Remove from passive gaze set — now actively managed.
|
||||
PassiveGazeAgents.Remove(NewAgent);
|
||||
|
||||
SelectedAgent = NewAgent;
|
||||
|
||||
if (bDebug)
|
||||
@ -635,6 +913,12 @@ void UPS_AI_ConvAgent_InteractionComponent::AttachGazeTarget(
|
||||
UPS_AI_ConvAgent_ElevenLabsComponent* AgentPtr = Agent.Get();
|
||||
if (!AgentPtr) return;
|
||||
|
||||
// Don't re-activate gaze while conversation is force-disabled.
|
||||
// ForceDisableConversation() explicitly sets bActive = false on sub-components
|
||||
// and monitors blend-out. Re-activating gaze here would prevent OnReadyForAction
|
||||
// from ever firing.
|
||||
if (AgentPtr->IsConversationDisabled()) return;
|
||||
|
||||
if (UPS_AI_ConvAgent_GazeComponent* Gaze = FindGazeOnAgent(AgentPtr))
|
||||
{
|
||||
Gaze->TargetActor = GetOwner();
|
||||
@ -688,6 +972,81 @@ void UPS_AI_ConvAgent_InteractionComponent::DetachGazeTarget(
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Deferred leave & gaze retention
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
void UPS_AI_ConvAgent_InteractionComponent::ExecuteLeave(UPS_AI_ConvAgent_ElevenLabsComponent* Agent)
|
||||
{
|
||||
if (!Agent) return;
|
||||
|
||||
if (GetOwnerRole() == ROLE_Authority || (GetWorld() && GetWorld()->GetNetMode() == NM_Standalone))
|
||||
{
|
||||
APlayerController* PC = nullptr;
|
||||
if (APawn* Pawn = Cast<APawn>(GetOwner()))
|
||||
{
|
||||
PC = Cast<APlayerController>(Pawn->GetController());
|
||||
}
|
||||
if (PC)
|
||||
{
|
||||
Agent->ServerLeaveConversation_Implementation(PC);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
ServerRelayLeaveConversation(Agent->GetOwner());
|
||||
}
|
||||
|
||||
if (bDebug)
|
||||
{
|
||||
UE_LOG(LogPS_AI_ConvAgent_Select, Log, TEXT("ExecuteLeave: %s"),
|
||||
Agent->GetOwner() ? *Agent->GetOwner()->GetName() : TEXT("(null)"));
|
||||
}
|
||||
}
|
||||
|
||||
void UPS_AI_ConvAgent_InteractionComponent::CleanupRetainedGaze(UPS_AI_ConvAgent_ElevenLabsComponent* Agent)
|
||||
{
|
||||
if (!Agent) return;
|
||||
|
||||
// Disable body tracking.
|
||||
if (bAutoManageGaze)
|
||||
{
|
||||
if (UPS_AI_ConvAgent_GazeComponent* Gaze = FindGazeOnAgent(Agent))
|
||||
{
|
||||
if (Gaze->TargetActor == GetOwner())
|
||||
{
|
||||
Gaze->bEnableBodyTracking = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Detach gaze target (with or without delay).
|
||||
if (bAutoManageGaze)
|
||||
{
|
||||
if (UWorld* World = GetWorld())
|
||||
{
|
||||
if (GazeDetachDelay > 0.0f)
|
||||
{
|
||||
TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> WeakAgent = Agent;
|
||||
World->GetTimerManager().SetTimer(GazeDetachTimerHandle,
|
||||
FTimerDelegate::CreateUObject(this,
|
||||
&UPS_AI_ConvAgent_InteractionComponent::DetachGazeTarget, WeakAgent),
|
||||
GazeDetachDelay, false);
|
||||
}
|
||||
else
|
||||
{
|
||||
DetachGazeTarget(Agent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (bDebug)
|
||||
{
|
||||
UE_LOG(LogPS_AI_ConvAgent_Select, Log, TEXT("CleanupRetainedGaze: %s"),
|
||||
Agent->GetOwner() ? *Agent->GetOwner()->GetName() : TEXT("(null)"));
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Mic routing
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@ -707,12 +1066,14 @@ void UPS_AI_ConvAgent_InteractionComponent::DrawDebugHUD() const
|
||||
{
|
||||
if (!GEngine) return;
|
||||
|
||||
const int32 BaseKey = 2060;
|
||||
// Per-actor key range: stride 150 per actor, offset 90 for Interaction
|
||||
const int32 BaseKey = 10000 + (GetOwner()->GetUniqueID() % 500) * 150 + 90;
|
||||
const float DisplayTime = 1.0f;
|
||||
const FColor MainColor = FColor::Cyan;
|
||||
const FString OwnerName = GetOwner()->GetName();
|
||||
|
||||
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, MainColor,
|
||||
TEXT("=== INTERACTION ==="));
|
||||
FString::Printf(TEXT("=== INTERACTION [%s] ==="), *OwnerName));
|
||||
|
||||
UPS_AI_ConvAgent_ElevenLabsComponent* Agent = SelectedAgent.Get();
|
||||
if (Agent)
|
||||
@ -732,9 +1093,21 @@ void UPS_AI_ConvAgent_InteractionComponent::DrawDebugHUD() const
|
||||
+ FVector(0.0f, 0.0f, AgentEyeLevelOffset);
|
||||
FVector ToAgent = AgentLoc - ViewLoc;
|
||||
Dist = ToAgent.Size();
|
||||
FVector DirToAgent = ToAgent.GetSafeNormal();
|
||||
Angle = FMath::RadiansToDegrees(
|
||||
FMath::Acos(FMath::Clamp(FVector::DotProduct(ViewDir, DirToAgent), -1.0f, 1.0f)));
|
||||
|
||||
// Use horizontal-only (XY) angle — consistent with EvaluateBestAgent().
|
||||
FVector ViewDirXY = ViewDir;
|
||||
ViewDirXY.Z = 0.0f;
|
||||
ViewDirXY = ViewDirXY.GetSafeNormal();
|
||||
|
||||
FVector DirToAgentXY = ToAgent;
|
||||
DirToAgentXY.Z = 0.0f;
|
||||
DirToAgentXY = DirToAgentXY.GetSafeNormal();
|
||||
|
||||
if (!ViewDirXY.IsNearlyZero() && !DirToAgentXY.IsNearlyZero())
|
||||
{
|
||||
Angle = FMath::RadiansToDegrees(
|
||||
FMath::Acos(FMath::Clamp(FVector::DotProduct(ViewDirXY, DirToAgentXY), -1.0f, 1.0f)));
|
||||
}
|
||||
}
|
||||
|
||||
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor,
|
||||
|
||||
@ -758,6 +758,16 @@ void UPS_AI_ConvAgent_LipSyncComponent::TickComponent(float DeltaTime, ELevelTic
|
||||
this, &UPS_AI_ConvAgent_LipSyncComponent::OnAgentInterrupted);
|
||||
Agent->OnAgentStoppedSpeaking.AddDynamic(
|
||||
this, &UPS_AI_ConvAgent_LipSyncComponent::OnAgentStopped);
|
||||
|
||||
// Bind conversation lifecycle — same as BeginPlay path.
|
||||
// Without these, bActive stays false forever when lazy-bound.
|
||||
Agent->OnAgentConnected.AddDynamic(
|
||||
this, &UPS_AI_ConvAgent_LipSyncComponent::OnConversationConnected);
|
||||
Agent->OnAgentDisconnected.AddDynamic(
|
||||
this, &UPS_AI_ConvAgent_LipSyncComponent::OnConversationDisconnected);
|
||||
bActive = false;
|
||||
CurrentActiveAlpha = 0.0f;
|
||||
|
||||
Agent->bEnableAgentPartialResponse = true;
|
||||
}
|
||||
}
|
||||
@ -2610,15 +2620,17 @@ void UPS_AI_ConvAgent_LipSyncComponent::DrawDebugHUD() const
|
||||
{
|
||||
if (!GEngine) return;
|
||||
|
||||
const int32 BaseKey = 2030;
|
||||
// Per-actor key range: stride 150 per actor, offset 45 for LipSync
|
||||
const int32 BaseKey = 10000 + (GetOwner()->GetUniqueID() % 500) * 150 + 45;
|
||||
const float DisplayTime = 1.0f;
|
||||
const FColor MainColor = FColor::Cyan;
|
||||
const FColor WarnColor = FColor::Yellow;
|
||||
const FString OwnerName = GetOwner()->GetName();
|
||||
|
||||
FString StateStr = bActive ? TEXT("ACTIVE") : TEXT("INACTIVE");
|
||||
|
||||
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, MainColor,
|
||||
FString::Printf(TEXT("=== LIP SYNC: %s ==="), *StateStr));
|
||||
FString::Printf(TEXT("=== LIP SYNC [%s]: %s ==="), *OwnerName, *StateStr));
|
||||
|
||||
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime,
|
||||
bIsSpeaking ? FColor::Green : MainColor,
|
||||
|
||||
@ -58,16 +58,18 @@ void UPS_AI_ConvAgent_MicrophoneCaptureComponent::DrawDebugHUD() const
|
||||
{
|
||||
if (!GEngine) return;
|
||||
|
||||
const int32 BaseKey = 2050;
|
||||
// Per-actor key range: stride 150 per actor, offset 75 for MicrophoneCapture
|
||||
const int32 BaseKey = 10000 + (GetOwner()->GetUniqueID() % 500) * 150 + 75;
|
||||
const float DisplayTime = 1.0f;
|
||||
const FColor MainColor = FColor::Cyan;
|
||||
const FString OwnerName = GetOwner()->GetName();
|
||||
|
||||
const bool bCapt = bCapturing.load();
|
||||
const bool bEchoSuppressed = EchoSuppressFlag && EchoSuppressFlag->load(std::memory_order_relaxed);
|
||||
|
||||
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime,
|
||||
bCapt ? FColor::Green : FColor::Red,
|
||||
FString::Printf(TEXT("=== MIC: %s ==="),
|
||||
FString::Printf(TEXT("=== MIC [%s]: %s ==="), *OwnerName,
|
||||
bCapt ? TEXT("CAPTURING") : TEXT("STOPPED")));
|
||||
|
||||
GEngine->AddOnScreenDebugMessage(BaseKey + 1, DisplayTime, MainColor,
|
||||
|
||||
@ -127,6 +127,7 @@ void UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy::SendUserTurnStart()
|
||||
bWaitingForResponse = false;
|
||||
bFirstAudioResponseLogged = false;
|
||||
bAgentResponseStartedFired = false;
|
||||
LastInterruptEventId = 0; // New user turn — stale interrupt filter no longer valid.
|
||||
|
||||
// No log here — turn start is implicit from audio chunks following.
|
||||
}
|
||||
@ -267,6 +268,18 @@ void UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy::OnWsConnected()
|
||||
InitMsg->SetStringField(TEXT("type"), PS_AI_ConvAgent_MessageType_ElevenLabs::ConversationClientData);
|
||||
InitMsg->SetObjectField(TEXT("conversation_config_override"), ConversationConfigOverride);
|
||||
|
||||
// Send dynamic variables (global context + per-agent variables).
|
||||
// These resolve {{variable_name}} placeholders in the agent's system prompt.
|
||||
if (DynamicVariables.Num() > 0)
|
||||
{
|
||||
TSharedPtr<FJsonObject> DynVarsObj = MakeShareable(new FJsonObject());
|
||||
for (const auto& Pair : DynamicVariables)
|
||||
{
|
||||
DynVarsObj->SetStringField(Pair.Key, Pair.Value);
|
||||
}
|
||||
InitMsg->SetObjectField(TEXT("dynamic_variables"), DynVarsObj);
|
||||
}
|
||||
|
||||
// NOTE: We bypass SendJsonMessage() here intentionally.
|
||||
// SendJsonMessage() guards on WebSocket->IsConnected(), but OnWsConnected fires
|
||||
// during the handshake before IsConnected() returns true in some UE WS backends.
|
||||
@ -551,6 +564,13 @@ void UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy::HandleAgentResponse(const TShar
|
||||
// subsequent agent_chat_response_part is guaranteed to belong to a new turn.
|
||||
bAgentResponseStartedFired = false;
|
||||
|
||||
// Also reset the interrupt audio filter here. agent_response is the last message
|
||||
// of the current turn — any audio arriving after this belongs to a new generation
|
||||
// and must not be filtered by a stale interrupt event_id from this turn.
|
||||
// This covers the edge case where audio for the next turn arrives before
|
||||
// agent_chat_response_part (which also resets the filter).
|
||||
LastInterruptEventId = 0;
|
||||
|
||||
// { "type": "agent_response",
|
||||
// "agent_response_event": { "agent_response": "..." }
|
||||
// }
|
||||
|
||||
@ -59,6 +59,17 @@ public:
|
||||
UPROPERTY(Config, EditAnywhere, AdvancedDisplay, Category = "PS AI ConvAgent|ElevenLabs API")
|
||||
bool bVerboseLogging = false;
|
||||
|
||||
/**
|
||||
* Global context prompt shared by ALL agents.
|
||||
* Use this for world-building, era, common rules that every character should know.
|
||||
* Injected at conversation start via dynamic variables — changes take effect
|
||||
* on the next conversation without re-syncing agents.
|
||||
*/
|
||||
UPROPERTY(Config, EditAnywhere, Category = "PS AI ConvAgent|Global Prompt",
|
||||
meta = (MultiLine = "true",
|
||||
ToolTip = "Context prompt shared by all agents.\nDescribe the world, era, common rules.\nChanges take effect on next conversation start (no re-sync needed)."))
|
||||
FString GlobalContextPrompt;
|
||||
|
||||
/** Return the API base URL (https) for the selected region. */
|
||||
FString GetAPIBaseURL() const
|
||||
{
|
||||
|
||||
@ -0,0 +1,53 @@
|
||||
// Copyright ASTERION. All Rights Reserved.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "CoreMinimal.h"
|
||||
#include "Engine/DataAsset.h"
|
||||
#include "PS_AI_ConvAgent_Definitions.h"
|
||||
#include "PS_AI_ConvAgent_ActionSet_ElevenLabs.generated.h"
|
||||
|
||||
/**
|
||||
* Reusable set of physical actions that an agent can trigger during conversation.
|
||||
*
|
||||
* Create this as a Data Asset in the Content Browser
|
||||
* (right-click > Miscellaneous > Data Asset > PS AI ConvAgent Action Set),
|
||||
* define your actions once, then drag it into any Agent Config that should
|
||||
* use these actions.
|
||||
*
|
||||
* The same ActionSet can be shared by multiple agents — modify it once
|
||||
* and re-sync all agents to update them.
|
||||
*/
|
||||
UCLASS(BlueprintType, Blueprintable,
|
||||
DisplayName = "PS AI ConvAgent Action Set (ElevenLabs)")
|
||||
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_ActionSet_ElevenLabs : public UPrimaryDataAsset
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
public:
|
||||
/** List of actions available to agents using this set.
|
||||
* Each action has a Name (identifier sent by the LLM) and a Description
|
||||
* (guidance for when the LLM should use it). */
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Actions",
|
||||
meta = (TitleProperty = "Name",
|
||||
ToolTip = "Available actions.\nName = identifier sent by LLM (snake_case).\nDescription = guidance for the LLM."))
|
||||
TArray<FPS_AI_ConvAgent_AgentAction_ElevenLabs> Actions;
|
||||
|
||||
/** System prompt fragment appended to CharacterPrompt when this action set is active.
|
||||
* Describes when and how the agent should use the perform_action tool.
|
||||
* Customize this for your scenario. */
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Actions",
|
||||
meta = (MultiLine = "true",
|
||||
ToolTip = "Prompt instructions for the action tool.\nAppended to CharacterPrompt when creating/updating the agent."))
|
||||
FString ActionToolPromptFragment = TEXT(
|
||||
"## Physical Actions\n"
|
||||
"You have a perform_action tool to trigger physical actions.\n"
|
||||
"Use it when the conversation naturally leads to a physical reaction.\n"
|
||||
"Only call it when the action is clearly motivated by the situation.");
|
||||
|
||||
// UPrimaryDataAsset interface
|
||||
virtual FPrimaryAssetId GetPrimaryAssetId() const override
|
||||
{
|
||||
return FPrimaryAssetId(TEXT("ActionSet_ElevenLabs"), GetFName());
|
||||
}
|
||||
};
|
||||
@ -5,6 +5,7 @@
|
||||
#include "CoreMinimal.h"
|
||||
#include "Engine/DataAsset.h"
|
||||
#include "PS_AI_ConvAgent_Definitions.h"
|
||||
#include "PS_AI_ConvAgent_ActionSet_ElevenLabs.h"
|
||||
#include "PS_AI_ConvAgent_AgentConfig_ElevenLabs.generated.h"
|
||||
|
||||
/**
|
||||
@ -236,6 +237,24 @@ public:
|
||||
"- \"high\" for strong reactions (big laugh, deep sadness, shock)\n\n"
|
||||
"Always return to neutral when the emotional moment passes.");
|
||||
|
||||
// ── Action Tool ─────────────────────────────────────────────────────────
|
||||
|
||||
/** Include a configurable "perform_action" client tool in the agent configuration.
|
||||
* Allows the LLM to trigger physical actions defined in the referenced ActionSet.
|
||||
* Actions are handled via the OnAgentActionRequested event in Blueprint. */
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Action Tool",
|
||||
meta = (ToolTip = "Include the perform_action client tool.\nRequires an ActionSet Data Asset with at least one action."))
|
||||
bool bIncludeActionTool = false;
|
||||
|
||||
/** Reference to a reusable Action Set Data Asset.
|
||||
* Create one in Content Browser (Miscellaneous > Data Asset > PS AI ConvAgent Action Set),
|
||||
* define your actions there, then drag it here.
|
||||
* The same ActionSet can be shared by multiple agents. */
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Action Tool",
|
||||
meta = (EditCondition = "bIncludeActionTool",
|
||||
ToolTip = "Drag an ActionSet Data Asset here.\nDefines which actions the agent can trigger."))
|
||||
TObjectPtr<UPS_AI_ConvAgent_ActionSet_ElevenLabs> ActionSet;
|
||||
|
||||
// ── Dynamic Variables ────────────────────────────────────────────────────
|
||||
|
||||
/** Key-value pairs sent as dynamic_variables at conversation start.
|
||||
|
||||
@ -135,6 +135,10 @@ public:
|
||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|BodyExpression|Events")
|
||||
FOnBodyExpressionChanged OnBodyExpressionChanged;
|
||||
|
||||
/** Current activation blend alpha (0 = fully inactive, 1 = fully active).
|
||||
* Used by ForceDisableConversation to monitor blend-out completion. */
|
||||
float GetActiveAlpha() const { return CurrentActiveAlpha; }
|
||||
|
||||
// ── UActorComponent overrides ─────────────────────────────────────────────
|
||||
virtual void BeginPlay() override;
|
||||
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
|
||||
|
||||
@ -166,3 +166,21 @@ struct PS_AI_CONVAGENT_API FPS_AI_ConvAgent_ClientToolCall_ElevenLabs
|
||||
UPROPERTY(BlueprintReadOnly, Category = "PS AI ConvAgent|ElevenLabs")
|
||||
TMap<FString, FString> Parameters;
|
||||
};
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Agent action definition (used by ActionSet Data Asset)
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
/** Defines a single action that an agent can perform during conversation. */
|
||||
USTRUCT(BlueprintType)
|
||||
struct PS_AI_CONVAGENT_API FPS_AI_ConvAgent_AgentAction_ElevenLabs
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
/** Action identifier sent by the LLM (e.g. "flee", "draw_weapon"). Keep short, snake_case. */
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs")
|
||||
FString Name;
|
||||
|
||||
/** Human-readable description — helps the LLM understand when to use this action. */
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs")
|
||||
FString Description;
|
||||
};
|
||||
|
||||
@ -84,13 +84,31 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnAgentEmotionChanged,
|
||||
EPS_AI_ConvAgent_EmotionIntensity, Intensity);
|
||||
|
||||
/**
|
||||
* Fired for any client tool call that is NOT automatically handled (i.e. not "set_emotion").
|
||||
* Fired when the agent triggers a "perform_action" client tool call.
|
||||
* ActionName is the identifier from the ActionSet (e.g. "flee", "draw_weapon").
|
||||
* Handle this in Blueprint to play animations, trigger AI behavior, etc.
|
||||
* The tool call is auto-acknowledged — no need to call SendClientToolResult.
|
||||
*/
|
||||
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnAgentActionRequested,
|
||||
const FString&, ActionName);
|
||||
|
||||
/**
|
||||
* Fired for any client tool call that is NOT automatically handled (i.e. not "set_emotion", not "perform_action").
|
||||
* Use this to implement custom client tools in Blueprint.
|
||||
* You MUST call SendClientToolResult on the WebSocketProxy to acknowledge the call.
|
||||
*/
|
||||
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnAgentClientToolCall,
|
||||
const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs&, ToolCall);
|
||||
|
||||
/**
|
||||
* Fired when ForceDisableConversation() has finished blending all visual
|
||||
* components (gaze, lip sync, facial expression, body expression) back to
|
||||
* their neutral state. At this point the NPC is fully idle and the game
|
||||
* can safely start the physical action (flee, draw weapon, etc.).
|
||||
*/
|
||||
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnReadyForAction,
|
||||
const FString&, ActionName);
|
||||
|
||||
/**
|
||||
* Fired when the active speaker changes in a multi-player shared conversation.
|
||||
* Use this for UI indicators showing who is talking, or to drive camera focus.
|
||||
@ -98,6 +116,12 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnAgentClientToolCall,
|
||||
DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnActiveSpeakerChanged,
|
||||
APawn*, NewSpeaker, APawn*, PreviousSpeaker);
|
||||
|
||||
/** Fired when this agent starts passive gaze (proximity awareness — not selected). */
|
||||
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAgentPassiveGazeStarted);
|
||||
|
||||
/** Fired when this agent stops passive gaze (left range or became selected). */
|
||||
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAgentPassiveGazeStopped);
|
||||
|
||||
// Non-dynamic delegate for raw agent audio (high-frequency, C++ consumers only).
|
||||
// Delivers PCM chunks as int16, 16kHz mono, little-endian.
|
||||
DECLARE_MULTICAST_DELEGATE_OneParam(FOnAgentAudioData, const TArray<uint8>& /*PCMData*/);
|
||||
@ -315,9 +339,16 @@ public:
|
||||
meta = (ToolTip = "Fires when the agent sets an emotion (joy, sadness, surprise, fear, anger, disgust).\nDriven by the 'set_emotion' client tool. Arrives before the audio."))
|
||||
FOnAgentEmotionChanged OnAgentEmotionChanged;
|
||||
|
||||
/** Fired for client tool calls that are NOT automatically handled (i.e. not "set_emotion"). You must call GetWebSocketProxy()->SendClientToolResult() to respond. */
|
||||
/** Fired when the LLM triggers a perform_action tool call.
|
||||
* ActionName is the identifier from the ActionSet (e.g. "flee", "draw_weapon").
|
||||
* Handle this in Blueprint to play animations, trigger AI behavior, etc. */
|
||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
||||
meta = (ToolTip = "Fires for custom client tool calls (not set_emotion).\nYou must respond via GetWebSocketProxy()->SendClientToolResult()."))
|
||||
meta = (ToolTip = "Fires when the agent triggers a physical action.\nActionName matches the Name field from the ActionSet Data Asset."))
|
||||
FOnAgentActionRequested OnAgentActionRequested;
|
||||
|
||||
/** Fired for client tool calls that are NOT automatically handled (i.e. not "set_emotion", not "perform_action"). You must call GetWebSocketProxy()->SendClientToolResult() to respond. */
|
||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
||||
meta = (ToolTip = "Fires for custom client tool calls (not set_emotion, not perform_action).\nYou must respond via GetWebSocketProxy()->SendClientToolResult()."))
|
||||
FOnAgentClientToolCall OnAgentClientToolCall;
|
||||
|
||||
/** Fired when the active speaker changes in a multi-player shared conversation.
|
||||
@ -326,6 +357,24 @@ public:
|
||||
meta = (ToolTip = "Fires when the speaking player changes.\nNewSpeaker is null when no one is speaking."))
|
||||
FOnActiveSpeakerChanged OnActiveSpeakerChanged;
|
||||
|
||||
/** Fired when ForceDisableConversation() has finished blending all visual
|
||||
* components back to neutral. The NPC is fully idle and the game can start
|
||||
* the physical action (flee, draw weapon, etc.).
|
||||
* Call ForceEnableConversation() later to allow conversation again. */
|
||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
||||
meta = (ToolTip = "Fires when all animation components have returned to neutral after ForceDisableConversation().\nThe agent is ready for a game action."))
|
||||
FOnReadyForAction OnReadyForAction;
|
||||
|
||||
/** Fired when this agent starts passively tracking a nearby player (head/eyes gaze). */
|
||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
||||
meta = (ToolTip = "Fires when this agent starts passively tracking a nearby player.\nTriggered by proximity awareness — the agent is not selected for conversation."))
|
||||
FOnAgentPassiveGazeStarted OnPassiveGazeStarted;
|
||||
|
||||
/** Fired when this agent stops passively tracking a player (left range or selected). */
|
||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
||||
meta = (ToolTip = "Fires when this agent stops passively tracking a player.\nThe player left range or selected this agent for active conversation."))
|
||||
FOnAgentPassiveGazeStopped OnPassiveGazeStopped;
|
||||
|
||||
/** The current emotion of the agent, as set by the "set_emotion" client tool. Defaults to Neutral. */
|
||||
UPROPERTY(ReplicatedUsing = OnRep_Emotion, BlueprintReadOnly, Category = "PS AI ConvAgent|ElevenLabs")
|
||||
EPS_AI_ConvAgent_Emotion CurrentEmotion = EPS_AI_ConvAgent_Emotion::Neutral;
|
||||
@ -498,6 +547,35 @@ public:
|
||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
||||
void InterruptAgent();
|
||||
|
||||
/**
|
||||
* Cleanly disable conversation on this agent so the NPC can switch to game AI.
|
||||
* Ends the WebSocket connection, stops all audio, and begins blending all
|
||||
* visual components (gaze, lip sync, facial expression, body expression)
|
||||
* back to their neutral state. While disabled, the InteractionComponent
|
||||
* cannot auto-restart a conversation with this agent.
|
||||
*
|
||||
* When all components have blended to neutral, fires OnReadyForAction.
|
||||
* Call ForceEnableConversation() later to allow conversation again.
|
||||
*
|
||||
* Typical usage from Blueprint (OnAgentActionRequested):
|
||||
* 1. ForceDisableConversation("flee")
|
||||
* 2. Wait for OnReadyForAction — receives the same ActionName
|
||||
* 3. Play action montage based on ActionName
|
||||
* 4. When done, call ForceEnableConversation()
|
||||
*
|
||||
* @param ActionName The action to perform once neutral (passed through to OnReadyForAction).
|
||||
*/
|
||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
||||
void ForceDisableConversation(const FString& ActionName);
|
||||
|
||||
/**
|
||||
* Re-enable conversation after a ForceDisableConversation() call.
|
||||
* Clears the disabled flag so the InteractionComponent can auto-start
|
||||
* a new conversation on the next tick (if the player is still looking).
|
||||
*/
|
||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
||||
void ForceEnableConversation();
|
||||
|
||||
/**
|
||||
* Feed microphone audio from an external source (e.g. InteractionComponent on the pawn).
|
||||
* When an InteractionComponent exists on the player pawn, or in a network scenario,
|
||||
@ -523,9 +601,19 @@ public:
|
||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|ElevenLabs")
|
||||
bool IsAgentSpeaking() const { return bAgentSpeaking; }
|
||||
|
||||
/** True when the agent is speaking OR the server hasn't confirmed the full response yet.
|
||||
* Use this instead of IsAgentSpeaking() when you need to know if the agent MIGHT still
|
||||
* produce more audio (e.g. during TTS inter-batch gaps). */
|
||||
bool IsAgentSpeakingOrPending() const { return bAgentSpeaking || (bAgentGenerating && !bAgentResponseReceived); }
|
||||
|
||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|ElevenLabs")
|
||||
const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& GetConversationInfo() const;
|
||||
|
||||
/** True while conversation is force-disabled (ForceDisableConversation was called).
|
||||
* While disabled, StartConversation and ServerJoinConversation are blocked. */
|
||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|ElevenLabs")
|
||||
bool IsConversationDisabled() const { return bConversationDisabledByAction; }
|
||||
|
||||
/** True while audio is being pre-buffered (playback hasn't started yet).
|
||||
* Used by the LipSync component to pause viseme queue consumption. */
|
||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|ElevenLabs")
|
||||
@ -618,6 +706,20 @@ private:
|
||||
UPROPERTY()
|
||||
USoundWaveProcedural* ProceduralSoundWave = nullptr;
|
||||
|
||||
// ── ForceDisableConversation state ───────────────────────────────────────
|
||||
// Set by ForceDisableConversation(), cleared by ForceEnableConversation().
|
||||
// While true, StartConversation() and ServerJoinConversation() are blocked.
|
||||
bool bConversationDisabledByAction = false;
|
||||
// True while waiting for all visual components to blend back to neutral.
|
||||
// Monitored in TickComponent — fires OnReadyForAction when complete.
|
||||
bool bWaitingForBlendOut = false;
|
||||
// The action name passed to ForceDisableConversation(), forwarded to OnReadyForAction.
|
||||
FString PendingActionName;
|
||||
// Accumulated time waiting for blend-out. Safety timeout fires OnReadyForAction
|
||||
// if components haven't reached neutral within this limit.
|
||||
float BlendOutElapsedTime = 0.0f;
|
||||
static constexpr float BlendOutTimeoutSeconds = 5.0f;
|
||||
|
||||
// ── State ─────────────────────────────────────────────────────────────────
|
||||
// Atomic: read from WASAPI background thread (OnMicrophoneDataCaptured), written from game thread.
|
||||
std::atomic<bool> bIsListening{false};
|
||||
@ -698,25 +800,27 @@ private:
|
||||
// Debug: track when the AudioQueue runs dry during speech (one-shot log).
|
||||
bool bQueueWasDry = false;
|
||||
|
||||
// Silence detection: how many consecutive ticks with an empty audio queue.
|
||||
int32 SilentTickCount = 0;
|
||||
// Silence detection: accumulated seconds of empty audio queue.
|
||||
// Time-based (not tick-based) to behave consistently across frame rates.
|
||||
float SilentTime = 0.0f;
|
||||
|
||||
// Generating timeout: how many consecutive ticks bAgentGenerating has been true
|
||||
// without any audio arriving. If this reaches HardSilenceTimeoutTicks, bAgentGenerating
|
||||
// is force-cleared so StartListening() is no longer blocked. This covers the edge case
|
||||
// where the server sends agent_chat_response_part but the TTS pipeline produces no audio.
|
||||
int32 GeneratingTickCount = 0;
|
||||
// Generating timeout: accumulated seconds with bAgentGenerating=true but no audio.
|
||||
// If this exceeds GeneratingTimeoutSeconds, bAgentGenerating is force-cleared
|
||||
// so StartListening() is no longer blocked.
|
||||
float GeneratingTime = 0.0f;
|
||||
|
||||
// Primary threshold: fire OnAgentStoppedSpeaking after this many silent ticks
|
||||
// Primary threshold: fire OnAgentStoppedSpeaking after this many seconds of silence
|
||||
// once the server has confirmed the full response (bAgentResponseReceived=true).
|
||||
// 30 ticks ≈ 0.5s at 60fps — enough to bridge brief inter-chunk gaps in the TTS stream.
|
||||
static constexpr int32 SilenceThresholdTicks = 30;
|
||||
// 0.5s is enough to bridge brief inter-chunk gaps in the TTS stream.
|
||||
static constexpr float SilenceThresholdSeconds = 0.5f;
|
||||
|
||||
// Hard-timeout fallback: fire even without agent_response confirmation after 10s
|
||||
// of silence. This covers edge cases where agent_response is very late or missing,
|
||||
// while being long enough to bridge inter-batch TTS gaps (observed up to ~5s).
|
||||
// Previously 2s — raised after logs showed premature firing during multi-batch responses.
|
||||
static constexpr int32 HardSilenceTimeoutTicks = 600; // 10s at 60fps
|
||||
static constexpr float HardSilenceTimeoutSeconds = 10.0f;
|
||||
|
||||
// Generating timeout: same as hard silence timeout.
|
||||
static constexpr float GeneratingTimeoutSeconds = 10.0f;
|
||||
|
||||
// True once the server sends agent_response for the current turn.
|
||||
// The server sends the full text when generation is complete — this is the
|
||||
|
||||
@ -114,6 +114,10 @@ public:
|
||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|FacialExpression|Events")
|
||||
FOnExpressionChanged OnExpressionChanged;
|
||||
|
||||
/** Current activation blend alpha (0 = fully inactive, 1 = fully active).
|
||||
* Used by ForceDisableConversation to monitor blend-out completion. */
|
||||
float GetActiveAlpha() const { return CurrentActiveAlpha; }
|
||||
|
||||
// ── UActorComponent overrides ─────────────────────────────────────────────
|
||||
virtual void BeginPlay() override;
|
||||
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
|
||||
|
||||
@ -315,6 +315,10 @@ public:
|
||||
* fresh instead of chasing a stale yaw from the previous interaction. */
|
||||
void ResetBodyTarget();
|
||||
|
||||
/** Current activation blend alpha (0 = fully inactive, 1 = fully active).
|
||||
* Used by ForceDisableConversation to monitor blend-out completion. */
|
||||
float GetActiveAlpha() const { return CurrentActiveAlpha; }
|
||||
|
||||
// ── UActorComponent overrides ────────────────────────────────────────────
|
||||
virtual void BeginPlay() override;
|
||||
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
|
||||
|
||||
@ -115,6 +115,28 @@ public:
|
||||
ToolTip = "Seconds to wait before the agent stops looking at the pawn.\n0 = immediate."))
|
||||
float GazeDetachDelay = 0.0f;
|
||||
|
||||
/** Agents within range but NOT selected will passively track the player
|
||||
* with head+eyes (and optionally body). This creates natural "awareness"
|
||||
* without requiring an active conversation.
|
||||
* Also controls the passive gaze retained after switching conversations. */
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction|Gaze",
|
||||
meta = (EditCondition = "bAutoManageGaze",
|
||||
ToolTip = "Agents in range but not selected will passively track the player.\nAlso applies to retained gaze after switching conversations."))
|
||||
bool bAutoPassiveGaze = true;
|
||||
|
||||
/** In passive gaze mode, the agent's head and eyes follow the player. */
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction|Gaze",
|
||||
meta = (EditCondition = "bAutoManageGaze && bAutoPassiveGaze",
|
||||
ToolTip = "Enable head+eyes tracking in passive gaze mode."))
|
||||
bool bPassiveGazeHeadEyes = true;
|
||||
|
||||
/** In passive gaze mode, the agent's body also turns toward the player.
|
||||
* When false, only head and eyes track — the body stays in its default animation. */
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction|Gaze",
|
||||
meta = (EditCondition = "bAutoManageGaze && bAutoPassiveGaze",
|
||||
ToolTip = "Enable body rotation in passive gaze mode.\nWhen false, only head and eyes track."))
|
||||
bool bPassiveGazeBody = false;
|
||||
|
||||
// ── Conversation management ──────────────────────────────────────────────
|
||||
|
||||
/** How long (seconds) the player must look at a different agent before switching
|
||||
@ -310,4 +332,32 @@ private:
|
||||
// while in an active conversation. Switch only happens after the delay.
|
||||
TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> PendingSwitchAgent;
|
||||
double PendingSwitchStartTime = 0.0;
|
||||
|
||||
// ── Deferred leave & gaze retention ──────────────────────────────────
|
||||
// When switching agents mid-conversation:
|
||||
// 1. If the old agent is still speaking, the Leave is deferred until
|
||||
// audio finishes → PendingLeaveAgent.
|
||||
// 2. Once the Leave executes (or if the old agent was already silent),
|
||||
// the gaze stays on the player until they walk out of range →
|
||||
// GazeRetainedAgent.
|
||||
|
||||
/** Agent whose Leave is deferred because it is still speaking. */
|
||||
TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> PendingLeaveAgent;
|
||||
|
||||
/** Agent that already left the conversation but retains gaze
|
||||
* on the player until MaxInteractionDistance is exceeded. */
|
||||
TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> GazeRetainedAgent;
|
||||
|
||||
/** Execute the actual Leave + stop listening on an agent. Does NOT touch gaze. */
|
||||
void ExecuteLeave(UPS_AI_ConvAgent_ElevenLabsComponent* Agent);
|
||||
|
||||
/** Detach gaze from a retained agent and clean up. */
|
||||
void CleanupRetainedGaze(UPS_AI_ConvAgent_ElevenLabsComponent* Agent);
|
||||
|
||||
// ── Passive gaze (proximity awareness) ──────────────────────────────
|
||||
/** Agents currently tracked passively (in range, not selected). */
|
||||
TSet<TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent>> PassiveGazeAgents;
|
||||
|
||||
/** Update passive gaze on all nearby agents. Called each tick. */
|
||||
void UpdatePassiveGaze();
|
||||
};
|
||||
|
||||
@ -188,6 +188,10 @@ public:
|
||||
* Smooth transition between silence and speech states. */
|
||||
float GetSpeechBlendAlpha() const { return SpeechBlendAlpha; }
|
||||
|
||||
/** Current activation blend alpha (0 = fully inactive, 1 = fully active).
|
||||
* Used by ForceDisableConversation to monitor blend-out completion. */
|
||||
float GetActiveAlpha() const { return CurrentActiveAlpha; }
|
||||
|
||||
// ── UActorComponent overrides ─────────────────────────────────────────────
|
||||
virtual void BeginPlay() override;
|
||||
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
|
||||
|
||||
@ -288,4 +288,9 @@ public:
|
||||
|
||||
// Start generating before confirming end-of-speech (reduces latency, may cause false starts).
|
||||
bool bSpeculativeTurn = false;
|
||||
|
||||
// Dynamic variables sent in conversation_initiation_client_data.
|
||||
// Populated by UPS_AI_ConvAgent_ElevenLabsComponent before calling Connect().
|
||||
// Includes global_context (from Settings) + per-agent DefaultDynamicVariables.
|
||||
TMap<FString, FString> DynamicVariables;
|
||||
};
|
||||
|
||||
@ -33,6 +33,8 @@ public class PS_AI_ConvAgentEditor : ModuleRules
|
||||
"HTTP",
|
||||
"Json",
|
||||
"JsonUtilities",
|
||||
// Asset Registry for scanning AgentConfig assets (batch update)
|
||||
"AssetRegistry",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@ -4,6 +4,8 @@
|
||||
#include "PropertyEditorModule.h"
|
||||
#include "PS_AI_ConvAgent_AgentConfig_ElevenLabs.h"
|
||||
#include "PS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs.h"
|
||||
#include "PS_AI_ConvAgent_ActionSet_ElevenLabs.h"
|
||||
#include "PS_AI_ConvAgent_ActionSetCustomization_ElevenLabs.h"
|
||||
|
||||
/**
|
||||
* Editor module for PS_AI_ConvAgent plugin.
|
||||
@ -21,6 +23,11 @@ public:
|
||||
UPS_AI_ConvAgent_AgentConfig_ElevenLabs::StaticClass()->GetFName(),
|
||||
FOnGetDetailCustomizationInstance::CreateStatic(
|
||||
&FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::MakeInstance));
|
||||
|
||||
PropertyModule.RegisterCustomClassLayout(
|
||||
UPS_AI_ConvAgent_ActionSet_ElevenLabs::StaticClass()->GetFName(),
|
||||
FOnGetDetailCustomizationInstance::CreateStatic(
|
||||
&FPS_AI_ConvAgent_ActionSetCustomization_ElevenLabs::MakeInstance));
|
||||
}
|
||||
|
||||
virtual void ShutdownModule() override
|
||||
@ -32,6 +39,9 @@ public:
|
||||
|
||||
PropertyModule.UnregisterCustomClassLayout(
|
||||
UPS_AI_ConvAgent_AgentConfig_ElevenLabs::StaticClass()->GetFName());
|
||||
|
||||
PropertyModule.UnregisterCustomClassLayout(
|
||||
UPS_AI_ConvAgent_ActionSet_ElevenLabs::StaticClass()->GetFName());
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@ -0,0 +1,293 @@
|
||||
// Copyright ASTERION. All Rights Reserved.
|
||||
|
||||
#include "PS_AI_ConvAgent_ActionSetCustomization_ElevenLabs.h"
|
||||
#include "PS_AI_ConvAgent_ActionSet_ElevenLabs.h"
|
||||
#include "PS_AI_ConvAgent_AgentConfig_ElevenLabs.h"
|
||||
#include "PS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs.h"
|
||||
#include "PS_AI_ConvAgent.h"
|
||||
|
||||
#include "DetailLayoutBuilder.h"
|
||||
#include "DetailCategoryBuilder.h"
|
||||
#include "DetailWidgetRow.h"
|
||||
#include "Widgets/Input/SButton.h"
|
||||
#include "Widgets/Text/STextBlock.h"
|
||||
|
||||
#include "AssetRegistry/AssetRegistryModule.h"
|
||||
#include "HttpModule.h"
|
||||
#include "Interfaces/IHttpRequest.h"
|
||||
#include "Interfaces/IHttpResponse.h"
|
||||
#include "Dom/JsonObject.h"
|
||||
#include "Serialization/JsonWriter.h"
|
||||
#include "Serialization/JsonSerializer.h"
|
||||
|
||||
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ActionSetEditor, Log, All);
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Factory
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
TSharedRef<IDetailCustomization> FPS_AI_ConvAgent_ActionSetCustomization_ElevenLabs::MakeInstance()
|
||||
{
|
||||
return MakeShareable(new FPS_AI_ConvAgent_ActionSetCustomization_ElevenLabs());
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// CustomizeDetails
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
void FPS_AI_ConvAgent_ActionSetCustomization_ElevenLabs::CustomizeDetails(
|
||||
IDetailLayoutBuilder& DetailBuilder)
|
||||
{
|
||||
DetailBuilder.GetObjectsBeingCustomized(SelectedObjects);
|
||||
|
||||
// ── Agent Sync category ─────────────────────────────────────────────────
|
||||
IDetailCategoryBuilder& SyncCat = DetailBuilder.EditCategory(
|
||||
TEXT("Agent Sync"),
|
||||
FText::FromString(TEXT("Agent Sync")),
|
||||
ECategoryPriority::Important);
|
||||
|
||||
SyncCat.AddCustomRow(FText::FromString(TEXT("Update All Agents")))
|
||||
.WholeRowContent()
|
||||
[
|
||||
SNew(SVerticalBox)
|
||||
+ SVerticalBox::Slot()
|
||||
.AutoHeight()
|
||||
.Padding(0, 4)
|
||||
[
|
||||
SNew(SButton)
|
||||
.Text(FText::FromString(TEXT("Update All Agents")))
|
||||
.ToolTipText(FText::FromString(
|
||||
TEXT("PATCH all AgentConfig assets that reference this ActionSet.")))
|
||||
.OnClicked_Lambda([this]()
|
||||
{
|
||||
OnUpdateAllAgentsClicked();
|
||||
return FReply::Handled();
|
||||
})
|
||||
]
|
||||
+ SVerticalBox::Slot()
|
||||
.AutoHeight()
|
||||
.Padding(0, 2)
|
||||
[
|
||||
SAssignNew(StatusTextBlock, STextBlock)
|
||||
.Text(FText::GetEmpty())
|
||||
.Font(IDetailLayoutBuilder::GetDetailFont())
|
||||
.ColorAndOpacity(FSlateColor(FLinearColor(0.3f, 0.7f, 1.0f)))
|
||||
]
|
||||
];
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Update All Agents
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
void FPS_AI_ConvAgent_ActionSetCustomization_ElevenLabs::OnUpdateAllAgentsClicked()
|
||||
{
|
||||
const UPS_AI_ConvAgent_ActionSet_ElevenLabs* ActionSetAsset = GetEditedAsset();
|
||||
if (!ActionSetAsset)
|
||||
{
|
||||
SetStatusError(TEXT("No ActionSet asset selected."));
|
||||
return;
|
||||
}
|
||||
|
||||
const FString APIKey = GetAPIKey();
|
||||
if (APIKey.IsEmpty())
|
||||
{
|
||||
SetStatusError(TEXT("API Key not set in Project Settings > PS AI ConvAgent - ElevenLabs."));
|
||||
return;
|
||||
}
|
||||
|
||||
// ── Scan all AgentConfig assets via Asset Registry ───────────────────────
|
||||
FAssetRegistryModule& ARModule = FModuleManager::LoadModuleChecked<FAssetRegistryModule>("AssetRegistry");
|
||||
IAssetRegistry& AssetRegistry = ARModule.Get();
|
||||
|
||||
TArray<FAssetData> AllAgentConfigs;
|
||||
AssetRegistry.GetAssetsByClass(
|
||||
UPS_AI_ConvAgent_AgentConfig_ElevenLabs::StaticClass()->GetClassPathName(),
|
||||
AllAgentConfigs, true);
|
||||
|
||||
// ── Filter: bIncludeActionTool && ActionSet == this asset && AgentID not empty ─
|
||||
TArray<UPS_AI_ConvAgent_AgentConfig_ElevenLabs*> MatchingConfigs;
|
||||
for (const FAssetData& AD : AllAgentConfigs)
|
||||
{
|
||||
UPS_AI_ConvAgent_AgentConfig_ElevenLabs* Config =
|
||||
Cast<UPS_AI_ConvAgent_AgentConfig_ElevenLabs>(AD.GetAsset());
|
||||
if (!Config) continue;
|
||||
if (!Config->bIncludeActionTool) continue;
|
||||
if (Config->ActionSet != ActionSetAsset) continue;
|
||||
if (Config->AgentID.IsEmpty()) continue;
|
||||
|
||||
MatchingConfigs.Add(Config);
|
||||
}
|
||||
|
||||
if (MatchingConfigs.Num() == 0)
|
||||
{
|
||||
SetStatusError(TEXT("No AgentConfig assets reference this ActionSet (with AgentID set)."));
|
||||
return;
|
||||
}
|
||||
|
||||
SetStatusText(FString::Printf(TEXT("Updating %d agent(s)..."), MatchingConfigs.Num()));
|
||||
|
||||
// ── Shared counter for async completion tracking ─────────────────────────
|
||||
struct FBatchState
|
||||
{
|
||||
int32 Total = 0;
|
||||
FThreadSafeCounter Succeeded;
|
||||
FThreadSafeCounter Failed;
|
||||
TArray<FString> Errors;
|
||||
FCriticalSection ErrorLock;
|
||||
};
|
||||
TSharedPtr<FBatchState> State = MakeShareable(new FBatchState());
|
||||
State->Total = MatchingConfigs.Num();
|
||||
|
||||
TWeakPtr<FPS_AI_ConvAgent_ActionSetCustomization_ElevenLabs> WeakSelf =
|
||||
StaticCastSharedRef<FPS_AI_ConvAgent_ActionSetCustomization_ElevenLabs>(this->AsShared());
|
||||
|
||||
for (UPS_AI_ConvAgent_AgentConfig_ElevenLabs* Config : MatchingConfigs)
|
||||
{
|
||||
TSharedPtr<FJsonObject> Payload =
|
||||
FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::BuildAgentPayloadForAsset(Config);
|
||||
|
||||
FString PayloadStr;
|
||||
TSharedRef<TJsonWriter<>> Writer = TJsonWriterFactory<>::Create(&PayloadStr);
|
||||
FJsonSerializer::Serialize(Payload.ToSharedRef(), Writer);
|
||||
|
||||
const FString URL = FString::Printf(
|
||||
TEXT("https://api.elevenlabs.io/v1/convai/agents/%s"), *Config->AgentID);
|
||||
|
||||
TSharedRef<IHttpRequest, ESPMode::ThreadSafe> Request = FHttpModule::Get().CreateRequest();
|
||||
Request->SetURL(URL);
|
||||
Request->SetVerb(TEXT("PATCH"));
|
||||
Request->SetHeader(TEXT("xi-api-key"), APIKey);
|
||||
Request->SetHeader(TEXT("Content-Type"), TEXT("application/json"));
|
||||
Request->SetContentAsString(PayloadStr);
|
||||
|
||||
// Capture Config as weak pointer for safety.
|
||||
TWeakObjectPtr<UPS_AI_ConvAgent_AgentConfig_ElevenLabs> WeakConfig(Config);
|
||||
FString AgentName = Config->AgentName.IsEmpty() ? Config->AgentID : Config->AgentName;
|
||||
|
||||
Request->OnProcessRequestComplete().BindLambda(
|
||||
[WeakSelf, State, WeakConfig, AgentName]
|
||||
(FHttpRequestPtr Req, FHttpResponsePtr Resp, bool bConnected)
|
||||
{
|
||||
bool bSuccess = false;
|
||||
FString ErrorMsg;
|
||||
|
||||
if (!bConnected || !Resp.IsValid())
|
||||
{
|
||||
ErrorMsg = FString::Printf(TEXT("%s: connection failed"), *AgentName);
|
||||
}
|
||||
else if (Resp->GetResponseCode() != 200)
|
||||
{
|
||||
ErrorMsg = FString::Printf(TEXT("%s: HTTP %d"),
|
||||
*AgentName, Resp->GetResponseCode());
|
||||
}
|
||||
else
|
||||
{
|
||||
bSuccess = true;
|
||||
// Update LastSyncTimestamp on the asset.
|
||||
if (UPS_AI_ConvAgent_AgentConfig_ElevenLabs* Cfg = WeakConfig.Get())
|
||||
{
|
||||
Cfg->Modify();
|
||||
Cfg->LastSyncTimestamp = FDateTime::UtcNow().ToIso8601();
|
||||
}
|
||||
}
|
||||
|
||||
if (bSuccess)
|
||||
{
|
||||
State->Succeeded.Increment();
|
||||
}
|
||||
else
|
||||
{
|
||||
State->Failed.Increment();
|
||||
FScopeLock Lock(&State->ErrorLock);
|
||||
State->Errors.Add(ErrorMsg);
|
||||
}
|
||||
|
||||
// Check if all requests are done.
|
||||
const int32 Done = State->Succeeded.GetValue() + State->Failed.GetValue();
|
||||
if (Done >= State->Total)
|
||||
{
|
||||
auto Pinned = WeakSelf.Pin();
|
||||
if (!Pinned.IsValid()) return;
|
||||
|
||||
if (State->Failed.GetValue() == 0)
|
||||
{
|
||||
Pinned->SetStatusSuccess(FString::Printf(
|
||||
TEXT("Updated %d/%d agents successfully."),
|
||||
State->Succeeded.GetValue(), State->Total));
|
||||
}
|
||||
else
|
||||
{
|
||||
FString AllErrors;
|
||||
{
|
||||
FScopeLock Lock(&State->ErrorLock);
|
||||
AllErrors = FString::Join(State->Errors, TEXT(", "));
|
||||
}
|
||||
Pinned->SetStatusError(FString::Printf(
|
||||
TEXT("Updated %d/%d agents. Failures: %s"),
|
||||
State->Succeeded.GetValue(), State->Total, *AllErrors));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Request->ProcessRequest();
|
||||
|
||||
UE_LOG(LogPS_AI_ActionSetEditor, Log,
|
||||
TEXT(" → PATCH agent '%s' (ID: %s)"), *AgentName, *Config->AgentID);
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Helpers
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
FString FPS_AI_ConvAgent_ActionSetCustomization_ElevenLabs::GetAPIKey() const
|
||||
{
|
||||
if (FPS_AI_ConvAgentModule::IsAvailable())
|
||||
{
|
||||
if (const UPS_AI_ConvAgent_Settings_ElevenLabs* Settings = FPS_AI_ConvAgentModule::Get().GetSettings())
|
||||
{
|
||||
return Settings->API_Key;
|
||||
}
|
||||
}
|
||||
return FString();
|
||||
}
|
||||
|
||||
UPS_AI_ConvAgent_ActionSet_ElevenLabs* FPS_AI_ConvAgent_ActionSetCustomization_ElevenLabs::GetEditedAsset() const
|
||||
{
|
||||
for (const TWeakObjectPtr<UObject>& Obj : SelectedObjects)
|
||||
{
|
||||
if (UPS_AI_ConvAgent_ActionSet_ElevenLabs* Asset =
|
||||
Cast<UPS_AI_ConvAgent_ActionSet_ElevenLabs>(Obj.Get()))
|
||||
{
|
||||
return Asset;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void FPS_AI_ConvAgent_ActionSetCustomization_ElevenLabs::SetStatusText(const FString& Text)
|
||||
{
|
||||
UE_LOG(LogPS_AI_ActionSetEditor, Log, TEXT("%s"), *Text);
|
||||
if (StatusTextBlock.IsValid())
|
||||
{
|
||||
StatusTextBlock->SetText(FText::FromString(Text));
|
||||
StatusTextBlock->SetColorAndOpacity(FSlateColor(FLinearColor(0.3f, 0.7f, 1.0f))); // cyan/info
|
||||
}
|
||||
}
|
||||
|
||||
void FPS_AI_ConvAgent_ActionSetCustomization_ElevenLabs::SetStatusError(const FString& Text)
|
||||
{
|
||||
UE_LOG(LogPS_AI_ActionSetEditor, Error, TEXT("%s"), *Text);
|
||||
if (StatusTextBlock.IsValid())
|
||||
{
|
||||
StatusTextBlock->SetText(FText::FromString(Text));
|
||||
StatusTextBlock->SetColorAndOpacity(FSlateColor(FLinearColor(1.0f, 0.25f, 0.25f))); // red
|
||||
}
|
||||
}
|
||||
|
||||
void FPS_AI_ConvAgent_ActionSetCustomization_ElevenLabs::SetStatusSuccess(const FString& Text)
|
||||
{
|
||||
UE_LOG(LogPS_AI_ActionSetEditor, Log, TEXT("%s"), *Text);
|
||||
if (StatusTextBlock.IsValid())
|
||||
{
|
||||
StatusTextBlock->SetText(FText::FromString(Text));
|
||||
StatusTextBlock->SetColorAndOpacity(FSlateColor(FLinearColor(0.2f, 0.9f, 0.3f))); // green
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,43 @@
|
||||
// Copyright ASTERION. All Rights Reserved.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "CoreMinimal.h"
|
||||
#include "IDetailCustomization.h"
|
||||
|
||||
class IDetailLayoutBuilder;
|
||||
|
||||
/**
|
||||
* Detail Customization for UPS_AI_ConvAgent_ActionSet_ElevenLabs data assets.
|
||||
*
|
||||
* Provides:
|
||||
* - Agent Sync category: "Update All Agents" button + STextBlock status
|
||||
* Scans all UPS_AI_ConvAgent_AgentConfig_ElevenLabs assets that reference
|
||||
* this ActionSet, then sends HTTP PATCH to ElevenLabs for each one.
|
||||
*/
|
||||
class FPS_AI_ConvAgent_ActionSetCustomization_ElevenLabs : public IDetailCustomization
|
||||
{
|
||||
public:
|
||||
static TSharedRef<IDetailCustomization> MakeInstance();
|
||||
|
||||
virtual void CustomizeDetails(IDetailLayoutBuilder& DetailBuilder) override;
|
||||
|
||||
private:
|
||||
// ── Update All Agents ────────────────────────────────────────────────────
|
||||
void OnUpdateAllAgentsClicked();
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────
|
||||
FString GetAPIKey() const;
|
||||
|
||||
/** Retrieve the ActionSet data asset being edited (first selected object). */
|
||||
class UPS_AI_ConvAgent_ActionSet_ElevenLabs* GetEditedAsset() const;
|
||||
|
||||
/** Display a status message. Color: red for errors, green for success, cyan for info. */
|
||||
void SetStatusText(const FString& Text);
|
||||
void SetStatusError(const FString& Text);
|
||||
void SetStatusSuccess(const FString& Text);
|
||||
|
||||
// ── Cached state ─────────────────────────────────────────────────────────
|
||||
TArray<TWeakObjectPtr<UObject>> SelectedObjects;
|
||||
TSharedPtr<class STextBlock> StatusTextBlock;
|
||||
};
|
||||
@ -0,0 +1,29 @@
|
||||
// Copyright ASTERION. All Rights Reserved.
|
||||
|
||||
#include "PS_AI_ConvAgent_ActionSetFactory_ElevenLabs.h"
|
||||
#include "PS_AI_ConvAgent_ActionSet_ElevenLabs.h"
|
||||
#include "AssetTypeCategories.h"
|
||||
|
||||
UPS_AI_ConvAgent_ActionSetFactory_ElevenLabs::UPS_AI_ConvAgent_ActionSetFactory_ElevenLabs()
|
||||
{
|
||||
SupportedClass = UPS_AI_ConvAgent_ActionSet_ElevenLabs::StaticClass();
|
||||
bCreateNew = true;
|
||||
bEditAfterNew = true;
|
||||
}
|
||||
|
||||
UObject* UPS_AI_ConvAgent_ActionSetFactory_ElevenLabs::FactoryCreateNew(
|
||||
UClass* Class, UObject* InParent, FName Name, EObjectFlags Flags,
|
||||
UObject* Context, FFeedbackContext* Warn)
|
||||
{
|
||||
return NewObject<UPS_AI_ConvAgent_ActionSet_ElevenLabs>(InParent, Class, Name, Flags);
|
||||
}
|
||||
|
||||
FText UPS_AI_ConvAgent_ActionSetFactory_ElevenLabs::GetDisplayName() const
|
||||
{
|
||||
return FText::FromString(TEXT("PS AI ConvAgent Action Set (ElevenLabs)"));
|
||||
}
|
||||
|
||||
uint32 UPS_AI_ConvAgent_ActionSetFactory_ElevenLabs::GetMenuCategories() const
|
||||
{
|
||||
return EAssetTypeCategories::Misc;
|
||||
}
|
||||
@ -0,0 +1,27 @@
|
||||
// Copyright ASTERION. All Rights Reserved.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "CoreMinimal.h"
|
||||
#include "Factories/Factory.h"
|
||||
#include "PS_AI_ConvAgent_ActionSetFactory_ElevenLabs.generated.h"
|
||||
|
||||
/**
|
||||
* Factory that lets users create PS_AI_ConvAgent_ActionSet_ElevenLabs assets
|
||||
* directly from the Content Browser (right-click > Miscellaneous).
|
||||
*/
|
||||
UCLASS()
|
||||
class UPS_AI_ConvAgent_ActionSetFactory_ElevenLabs : public UFactory
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
public:
|
||||
UPS_AI_ConvAgent_ActionSetFactory_ElevenLabs();
|
||||
|
||||
virtual UObject* FactoryCreateNew(UClass* Class, UObject* InParent,
|
||||
FName Name, EObjectFlags Flags, UObject* Context,
|
||||
FFeedbackContext* Warn) override;
|
||||
|
||||
virtual FText GetDisplayName() const override;
|
||||
virtual uint32 GetMenuCategories() const override;
|
||||
};
|
||||
@ -1182,6 +1182,25 @@ void FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::OnFetchAgentClicked()
|
||||
// to avoid doubling them on next Update.
|
||||
// Order matters: strip from earliest marker to preserve CharacterPrompt.
|
||||
|
||||
// 0. Strip global context placeholder prepended by BuildAgentPayload.
|
||||
// Must be removed so it never appears in the CharacterPrompt UI field.
|
||||
{
|
||||
const FString GlobalContextPrefix = TEXT("{{global_context}}\n\n");
|
||||
if (Prompt.StartsWith(GlobalContextPrefix))
|
||||
{
|
||||
Prompt.RightChopInline(GlobalContextPrefix.Len());
|
||||
}
|
||||
else
|
||||
{
|
||||
// Fallback: ElevenLabs may collapse double-newline to single
|
||||
const FString GlobalContextPrefixAlt = TEXT("{{global_context}}\n");
|
||||
if (Prompt.StartsWith(GlobalContextPrefixAlt))
|
||||
{
|
||||
Prompt.RightChopInline(GlobalContextPrefixAlt.Len());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1. Language instruction — try exact fragment first, then marker fallback.
|
||||
// Mirrors the emotion-tool approach: ElevenLabs may normalise
|
||||
// double-newlines, so the "\n\n## …" marker alone can fail.
|
||||
@ -1277,6 +1296,28 @@ void FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::OnFetchAgentClicked()
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Action tool fragment (from ActionSet)
|
||||
if (Asset->ActionSet && !Asset->ActionSet->ActionToolPromptFragment.IsEmpty())
|
||||
{
|
||||
int32 Idx = Prompt.Find(Asset->ActionSet->ActionToolPromptFragment,
|
||||
ESearchCase::CaseSensitive);
|
||||
if (Idx != INDEX_NONE)
|
||||
{
|
||||
Prompt.LeftInline(Idx);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Fallback: strip by marker
|
||||
const FString ActionMarker = TEXT("\n\n## Physical Actions");
|
||||
int32 MarkerIdx = Prompt.Find(ActionMarker,
|
||||
ESearchCase::CaseSensitive);
|
||||
if (MarkerIdx != INDEX_NONE)
|
||||
{
|
||||
Prompt.LeftInline(MarkerIdx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Asset->CharacterPrompt = Prompt;
|
||||
}
|
||||
|
||||
@ -1499,12 +1540,26 @@ FString FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::ParseAPIError(
|
||||
|
||||
TSharedPtr<FJsonObject> FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::BuildAgentPayload() const
|
||||
{
|
||||
const UPS_AI_ConvAgent_AgentConfig_ElevenLabs* Asset = GetEditedAsset();
|
||||
return BuildAgentPayloadForAsset(GetEditedAsset());
|
||||
}
|
||||
|
||||
TSharedPtr<FJsonObject> FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::BuildAgentPayloadForAsset(
|
||||
const UPS_AI_ConvAgent_AgentConfig_ElevenLabs* Asset)
|
||||
{
|
||||
if (!Asset) return MakeShareable(new FJsonObject());
|
||||
|
||||
// Build the full system prompt by appending automated fragments.
|
||||
// Order: CharacterPrompt + Language/Multilingual instruction + Emotion tool
|
||||
FString FullPrompt = Asset->CharacterPrompt;
|
||||
// Order: [GlobalContext placeholder] + CharacterPrompt + Language/Multilingual instruction + Emotion tool
|
||||
|
||||
// Prepend global context placeholder — resolved at runtime via dynamic_variables.
|
||||
// Only added when the user has set a GlobalContextPrompt in project settings.
|
||||
FString FullPrompt;
|
||||
const UPS_AI_ConvAgent_Settings_ElevenLabs* Settings = FPS_AI_ConvAgentModule::Get().GetSettings();
|
||||
if (Settings && !Settings->GlobalContextPrompt.IsEmpty())
|
||||
{
|
||||
FullPrompt = TEXT("{{global_context}}\n\n");
|
||||
}
|
||||
FullPrompt += Asset->CharacterPrompt;
|
||||
|
||||
UE_LOG(LogPS_AI_AgentConfigEditor, Log,
|
||||
TEXT("BuildAgentPayload: CharacterPrompt=%d chars, bMultilingual=%d, bAutoLangInstr=%d, Language='%s', "
|
||||
@ -1553,6 +1608,16 @@ TSharedPtr<FJsonObject> FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::Bu
|
||||
UE_LOG(LogPS_AI_AgentConfigEditor, Log, TEXT(" → Appended EmotionToolPromptFragment"));
|
||||
}
|
||||
|
||||
// Append action tool instructions from ActionSet.
|
||||
if (Asset->bIncludeActionTool && Asset->ActionSet
|
||||
&& Asset->ActionSet->Actions.Num() > 0
|
||||
&& !Asset->ActionSet->ActionToolPromptFragment.IsEmpty())
|
||||
{
|
||||
FullPrompt += TEXT("\n\n");
|
||||
FullPrompt += Asset->ActionSet->ActionToolPromptFragment;
|
||||
UE_LOG(LogPS_AI_AgentConfigEditor, Log, TEXT(" → Appended ActionToolPromptFragment from ActionSet"));
|
||||
}
|
||||
|
||||
UE_LOG(LogPS_AI_AgentConfigEditor, Log, TEXT("BuildAgentPayload: FullPrompt = %d chars"), FullPrompt.Len());
|
||||
|
||||
// prompt object (includes LLM selection + tools)
|
||||
@ -1563,12 +1628,24 @@ TSharedPtr<FJsonObject> FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::Bu
|
||||
PromptObj->SetStringField(TEXT("llm"), Asset->LLMModel);
|
||||
}
|
||||
|
||||
// If emotion tool is enabled, add to prompt.tools[] (API path: conversation_config.agent.prompt.tools)
|
||||
// Build tools array: emotion tool + action tool (API path: conversation_config.agent.prompt.tools)
|
||||
TArray<TSharedPtr<FJsonValue>> Tools;
|
||||
if (Asset->bIncludeEmotionTool)
|
||||
{
|
||||
TSharedPtr<FJsonObject> EmotionTool = BuildEmotionToolDefinition();
|
||||
TArray<TSharedPtr<FJsonValue>> Tools;
|
||||
Tools.Add(MakeShareable(new FJsonValueObject(EmotionTool)));
|
||||
}
|
||||
if (Asset->bIncludeActionTool && Asset->ActionSet
|
||||
&& Asset->ActionSet->Actions.Num() > 0)
|
||||
{
|
||||
TSharedPtr<FJsonObject> ActionTool = BuildActionToolDefinition(Asset);
|
||||
if (ActionTool)
|
||||
{
|
||||
Tools.Add(MakeShareable(new FJsonValueObject(ActionTool)));
|
||||
}
|
||||
}
|
||||
if (Tools.Num() > 0)
|
||||
{
|
||||
PromptObj->SetArrayField(TEXT("tools"), Tools);
|
||||
}
|
||||
|
||||
@ -1670,7 +1747,7 @@ TSharedPtr<FJsonObject> FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::Bu
|
||||
return Root;
|
||||
}
|
||||
|
||||
TSharedPtr<FJsonObject> FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::BuildEmotionToolDefinition() const
|
||||
TSharedPtr<FJsonObject> FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::BuildEmotionToolDefinition()
|
||||
{
|
||||
// Build the set_emotion client tool definition.
|
||||
// Parameters: emotion (enum), intensity (enum).
|
||||
@ -1724,3 +1801,54 @@ TSharedPtr<FJsonObject> FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::Bu
|
||||
|
||||
return Tool;
|
||||
}
|
||||
|
||||
TSharedPtr<FJsonObject> FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::BuildActionToolDefinition(
|
||||
const UPS_AI_ConvAgent_AgentConfig_ElevenLabs* Asset)
|
||||
{
|
||||
if (!Asset || !Asset->ActionSet || Asset->ActionSet->Actions.Num() == 0)
|
||||
return nullptr;
|
||||
|
||||
// Build description with action list from the referenced ActionSet.
|
||||
FString ParamDesc = TEXT("The action to perform. Available actions:");
|
||||
TArray<TSharedPtr<FJsonValue>> ActionEnum;
|
||||
for (const auto& Action : Asset->ActionSet->Actions)
|
||||
{
|
||||
if (!Action.Name.IsEmpty())
|
||||
{
|
||||
ActionEnum.Add(MakeShareable(new FJsonValueString(Action.Name)));
|
||||
if (!Action.Description.IsEmpty())
|
||||
{
|
||||
ParamDesc += FString::Printf(TEXT("\n- %s: %s"),
|
||||
*Action.Name, *Action.Description);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (ActionEnum.Num() == 0) return nullptr;
|
||||
|
||||
// action parameter
|
||||
TSharedPtr<FJsonObject> ActionParam = MakeShareable(new FJsonObject());
|
||||
ActionParam->SetStringField(TEXT("type"), TEXT("string"));
|
||||
ActionParam->SetStringField(TEXT("description"), ParamDesc);
|
||||
ActionParam->SetArrayField(TEXT("enum"), ActionEnum);
|
||||
|
||||
// properties + required
|
||||
TSharedPtr<FJsonObject> Properties = MakeShareable(new FJsonObject());
|
||||
Properties->SetObjectField(TEXT("action"), ActionParam);
|
||||
TArray<TSharedPtr<FJsonValue>> Required;
|
||||
Required.Add(MakeShareable(new FJsonValueString(TEXT("action"))));
|
||||
|
||||
TSharedPtr<FJsonObject> Parameters = MakeShareable(new FJsonObject());
|
||||
Parameters->SetStringField(TEXT("type"), TEXT("object"));
|
||||
Parameters->SetObjectField(TEXT("properties"), Properties);
|
||||
Parameters->SetArrayField(TEXT("required"), Required);
|
||||
|
||||
// Tool definition
|
||||
TSharedPtr<FJsonObject> Tool = MakeShareable(new FJsonObject());
|
||||
Tool->SetStringField(TEXT("type"), TEXT("client"));
|
||||
Tool->SetStringField(TEXT("name"), TEXT("perform_action"));
|
||||
Tool->SetStringField(TEXT("description"),
|
||||
TEXT("Trigger a physical action or reaction for the character."));
|
||||
Tool->SetObjectField(TEXT("parameters"), Parameters);
|
||||
|
||||
return Tool;
|
||||
}
|
||||
|
||||
@ -50,7 +50,17 @@ private:
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────
|
||||
FString GetAPIKey() const;
|
||||
TSharedPtr<FJsonObject> BuildAgentPayload() const;
|
||||
TSharedPtr<FJsonObject> BuildEmotionToolDefinition() const;
|
||||
|
||||
public:
|
||||
/** Build the full ElevenLabs API payload for any AgentConfig asset.
|
||||
* Static so it can be reused from other customizations (e.g. ActionSet batch update). */
|
||||
static TSharedPtr<FJsonObject> BuildAgentPayloadForAsset(
|
||||
const class UPS_AI_ConvAgent_AgentConfig_ElevenLabs* Asset);
|
||||
static TSharedPtr<FJsonObject> BuildEmotionToolDefinition();
|
||||
static TSharedPtr<FJsonObject> BuildActionToolDefinition(
|
||||
const class UPS_AI_ConvAgent_AgentConfig_ElevenLabs* Asset);
|
||||
|
||||
private:
|
||||
|
||||
/** Display a status message in the Identity category.
|
||||
* Color: red for errors, green for success, blue/cyan for info. */
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user