Add Agent Config data asset system with ElevenLabs editor integration
Introduces UPS_AI_ConvAgent_AgentConfig_ElevenLabs data asset to encapsulate full agent configuration (voice, LLM, prompt, language, emotions) with a custom Detail Customization providing: - Voice/TTS Model/LLM/Language pickers with Fetch buttons (ElevenLabs API) - LLM latency hints in dropdown (~250ms, ~700ms, etc.) - Create/Update/Fetch Agent buttons for REST API CRUD - Auto-fetch on editor open, auto-select first voice for new assets - Prompt fragment management (language, multilingual, emotion tool) - Smart defaults: gemini-2.5-flash LLM, eleven_turbo_v2_5 TTS, English - Speed range expanded to 0.7-1.95 (was 0.7-1.2) - bAutoStartConversation + StartConversationWithSelectedAgent() on InteractionComponent Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,3 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#include "PS_AI_ConvAgent_AgentConfig_ElevenLabs.h"
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
// Copyright ASTERION. All Rights Reserved.
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
#include "PS_AI_ConvAgent_ElevenLabsComponent.h"
|
#include "PS_AI_ConvAgent_ElevenLabsComponent.h"
|
||||||
|
#include "PS_AI_ConvAgent_AgentConfig_ElevenLabs.h"
|
||||||
#include "PS_AI_ConvAgent_MicrophoneCaptureComponent.h"
|
#include "PS_AI_ConvAgent_MicrophoneCaptureComponent.h"
|
||||||
#include "PS_AI_ConvAgent_PostureComponent.h"
|
#include "PS_AI_ConvAgent_PostureComponent.h"
|
||||||
#include "PS_AI_ConvAgent_InteractionSubsystem.h"
|
#include "PS_AI_ConvAgent_InteractionSubsystem.h"
|
||||||
@@ -266,7 +267,13 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StartConversation_Internal()
|
|||||||
// Pass configuration to the proxy before connecting.
|
// Pass configuration to the proxy before connecting.
|
||||||
WebSocketProxy->TurnMode = TurnMode;
|
WebSocketProxy->TurnMode = TurnMode;
|
||||||
|
|
||||||
WebSocketProxy->Connect(AgentID);
|
// Resolve AgentID by priority: AgentConfig > component string > project default.
|
||||||
|
FString ResolvedAgentID = AgentID;
|
||||||
|
if (AgentConfig && !AgentConfig->AgentID.IsEmpty())
|
||||||
|
{
|
||||||
|
ResolvedAgentID = AgentConfig->AgentID;
|
||||||
|
}
|
||||||
|
WebSocketProxy->Connect(ResolvedAgentID);
|
||||||
}
|
}
|
||||||
|
|
||||||
void UPS_AI_ConvAgent_ElevenLabsComponent::EndConversation()
|
void UPS_AI_ConvAgent_ElevenLabsComponent::EndConversation()
|
||||||
|
|||||||
@@ -252,16 +252,18 @@ void UPS_AI_ConvAgent_InteractionComponent::SetSelectedAgent(UPS_AI_ConvAgent_El
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Network: auto-start conversation if the agent isn't connected yet.
|
// Network: auto-start conversation if the agent isn't connected yet.
|
||||||
if (!NewAgent->IsConnected() && !NewAgent->bNetIsConversing)
|
// Only when bAutoStartConversation is true — otherwise the user must
|
||||||
|
// call StartConversationWithSelectedAgent() explicitly (e.g. on key press).
|
||||||
|
if (bAutoStartConversation && !NewAgent->IsConnected() && !NewAgent->bNetIsConversing)
|
||||||
{
|
{
|
||||||
NewAgent->StartConversation();
|
NewAgent->StartConversation();
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure mic is capturing so we can route audio to the new agent.
|
// Ensure mic is capturing so we can route audio to the new agent.
|
||||||
if (MicComponent && !MicComponent->IsCapturing())
|
if (MicComponent && !MicComponent->IsCapturing())
|
||||||
{
|
{
|
||||||
MicComponent->StartCapture();
|
MicComponent->StartCapture();
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ── Posture: attach (eyes+head only — body tracking is enabled later
|
// ── Posture: attach (eyes+head only — body tracking is enabled later
|
||||||
// by ElevenLabsComponent when the agent starts responding) ──
|
// by ElevenLabsComponent when the agent starts responding) ──
|
||||||
@@ -351,6 +353,48 @@ void UPS_AI_ConvAgent_InteractionComponent::ClearSelection()
|
|||||||
SetSelectedAgent(nullptr);
|
SetSelectedAgent(nullptr);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_InteractionComponent::StartConversationWithSelectedAgent()
|
||||||
|
{
|
||||||
|
UPS_AI_ConvAgent_ElevenLabsComponent* Agent = SelectedAgent.Get();
|
||||||
|
if (!Agent)
|
||||||
|
{
|
||||||
|
if (bDebug)
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Select, Warning, TEXT("StartConversationWithSelectedAgent: no agent selected."));
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Agent->IsConnected() || Agent->bNetIsConversing)
|
||||||
|
{
|
||||||
|
if (bDebug)
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Select, Log, TEXT("StartConversationWithSelectedAgent: agent already connected/conversing."));
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (bDebug)
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Select, Log, TEXT("StartConversationWithSelectedAgent: starting conversation with %s"),
|
||||||
|
Agent->GetOwner() ? *Agent->GetOwner()->GetName() : TEXT("(null)"));
|
||||||
|
}
|
||||||
|
|
||||||
|
Agent->StartConversation();
|
||||||
|
|
||||||
|
// Ensure mic is capturing so we can route audio to the agent.
|
||||||
|
if (MicComponent && !MicComponent->IsCapturing())
|
||||||
|
{
|
||||||
|
MicComponent->StartCapture();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start listening if auto-managed.
|
||||||
|
if (bAutoManageListening)
|
||||||
|
{
|
||||||
|
Agent->StartListening();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
// Posture helpers
|
// Posture helpers
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|||||||
@@ -724,9 +724,9 @@ FString UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy::BuildWebSocketURL(const FStr
|
|||||||
return Settings->CustomWebSocketURL;
|
return Settings->CustomWebSocketURL;
|
||||||
}
|
}
|
||||||
|
|
||||||
const FString ResolvedAgentID = AgentIDOverride.IsEmpty() ? Settings->AgentID : AgentIDOverride;
|
if (AgentIDOverride.IsEmpty())
|
||||||
if (ResolvedAgentID.IsEmpty())
|
|
||||||
{
|
{
|
||||||
|
UE_LOG(LogTemp, Error, TEXT("[PS_AI_ConvAgent] No AgentID provided. Set one via AgentConfig data asset or the AgentID property on the component."));
|
||||||
return FString();
|
return FString();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -734,5 +734,5 @@ FString UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy::BuildWebSocketURL(const FStr
|
|||||||
// wss://api.elevenlabs.io/v1/convai/conversation?agent_id=<ID>
|
// wss://api.elevenlabs.io/v1/convai/conversation?agent_id=<ID>
|
||||||
return FString::Printf(
|
return FString::Printf(
|
||||||
TEXT("wss://api.elevenlabs.io/v1/convai/conversation?agent_id=%s"),
|
TEXT("wss://api.elevenlabs.io/v1/convai/conversation?agent_id=%s"),
|
||||||
*ResolvedAgentID);
|
*AgentIDOverride);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,13 +23,6 @@ public:
|
|||||||
UPROPERTY(Config, EditAnywhere, Category = "PS AI ConvAgent|ElevenLabs API")
|
UPROPERTY(Config, EditAnywhere, Category = "PS AI ConvAgent|ElevenLabs API")
|
||||||
FString API_Key;
|
FString API_Key;
|
||||||
|
|
||||||
/**
|
|
||||||
* The default ElevenLabs Agent ID to use when none is specified
|
|
||||||
* on the component. Create agents at https://elevenlabs.io/app/conversational-ai
|
|
||||||
*/
|
|
||||||
UPROPERTY(Config, EditAnywhere, Category = "PS AI ConvAgent|ElevenLabs API")
|
|
||||||
FString AgentID;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Override the ElevenLabs WebSocket base URL. Leave empty to use the default:
|
* Override the ElevenLabs WebSocket base URL. Leave empty to use the default:
|
||||||
* wss://api.elevenlabs.io/v1/convai/conversation
|
* wss://api.elevenlabs.io/v1/convai/conversation
|
||||||
|
|||||||
@@ -0,0 +1,234 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "CoreMinimal.h"
|
||||||
|
#include "Engine/DataAsset.h"
|
||||||
|
#include "PS_AI_ConvAgent_AgentConfig_ElevenLabs.generated.h"
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reusable data asset that encapsulates a full ElevenLabs agent configuration:
|
||||||
|
* voice, LLM prompt, language, emotion tool, and API identity.
|
||||||
|
*
|
||||||
|
* Create ONE instance per agent in the Content Browser
|
||||||
|
* (right-click > Miscellaneous > PS AI ConvAgent Agent Config),
|
||||||
|
* then assign it on the PS AI ConvAgent ElevenLabs component.
|
||||||
|
*
|
||||||
|
* The editor Detail Customization provides:
|
||||||
|
* - Voice picker (fetches available voices from the ElevenLabs API)
|
||||||
|
* - Model picker (fetches TTS models from the ElevenLabs API)
|
||||||
|
* - LLM picker (dropdown with supported LLMs)
|
||||||
|
* - Language picker (dropdown with supported languages)
|
||||||
|
* - Create / Update / Fetch Agent buttons (REST API)
|
||||||
|
* - Pre-configured emotion tool prompt fragment
|
||||||
|
*
|
||||||
|
* At runtime, the ElevenLabsComponent reads AgentID from this asset
|
||||||
|
* to establish the WebSocket conversation.
|
||||||
|
*/
|
||||||
|
UCLASS(BlueprintType, Blueprintable,
|
||||||
|
DisplayName = "PS AI ConvAgent Agent Config (ElevenLabs)")
|
||||||
|
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_AgentConfig_ElevenLabs : public UPrimaryDataAsset
|
||||||
|
{
|
||||||
|
GENERATED_BODY()
|
||||||
|
|
||||||
|
public:
|
||||||
|
// ── Identity ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Agent ID assigned by ElevenLabs after Create/Sync.
|
||||||
|
* Populated automatically by the "Create Agent" editor action.
|
||||||
|
* This is the ID used to connect the WebSocket conversation.
|
||||||
|
* You can also paste an existing ID here, then use "Fetch Agent" to pull its config. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadOnly, Category = "Identity",
|
||||||
|
meta = (ToolTip = "ElevenLabs Agent ID.\nPopulated when created/synced via API.\nPaste an existing ID + Fetch Agent to import."))
|
||||||
|
FString AgentID;
|
||||||
|
|
||||||
|
/** Human-readable name for this agent.
|
||||||
|
* Used as the agent name when creating on ElevenLabs. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Identity",
|
||||||
|
meta = (ToolTip = "Agent display name (visible on ElevenLabs dashboard)."))
|
||||||
|
FString AgentName;
|
||||||
|
|
||||||
|
// ── Voice ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Voice ID from ElevenLabs.
|
||||||
|
* Managed by the Voice picker dropdown — do not edit manually. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice",
|
||||||
|
meta = (ToolTip = "ElevenLabs Voice ID.\nManaged by the Voice picker dropdown."))
|
||||||
|
FString VoiceID;
|
||||||
|
|
||||||
|
/** Display name of the selected voice (informational, not sent to API). */
|
||||||
|
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Voice",
|
||||||
|
meta = (ToolTip = "Name of the selected voice (display only)."))
|
||||||
|
FString VoiceName;
|
||||||
|
|
||||||
|
/** TTS model ID (e.g. "eleven_turbo_v2_5", "eleven_multilingual_v2").
|
||||||
|
* Managed by the Model picker dropdown. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice",
|
||||||
|
meta = (ToolTip = "TTS model ID.\nManaged by the Model picker dropdown."))
|
||||||
|
FString TTSModelID = TEXT("eleven_turbo_v2_5");
|
||||||
|
|
||||||
|
/** TTS stability (0.0 - 1.0). Controls voice consistency. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice",
|
||||||
|
meta = (ClampMin = "0.0", ClampMax = "1.0",
|
||||||
|
ToolTip = "Voice stability.\n0 = variable/expressive, 1 = consistent.\nDefault: 0.5"))
|
||||||
|
float Stability = 0.5f;
|
||||||
|
|
||||||
|
/** TTS similarity boost (0.0 - 1.0). Higher = closer to original voice. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice",
|
||||||
|
meta = (ClampMin = "0.0", ClampMax = "1.0",
|
||||||
|
ToolTip = "Similarity boost.\n0 = less similar, 1 = more similar.\nDefault: 0.75"))
|
||||||
|
float SimilarityBoost = 0.75f;
|
||||||
|
|
||||||
|
/** TTS speed multiplier. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice",
|
||||||
|
meta = (ClampMin = "0.7", ClampMax = "1.95",
|
||||||
|
ToolTip = "Speech speed multiplier.\nRange: 0.7-1.95.\nDefault: 1.0"))
|
||||||
|
float Speed = 1.0f;
|
||||||
|
|
||||||
|
/** LLM model used by the agent (e.g. "gpt-4o-mini", "claude-3-5-sonnet").
|
||||||
|
* Managed by the LLM picker dropdown.
|
||||||
|
* Leave empty for ElevenLabs default. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice",
|
||||||
|
meta = (ToolTip = "LLM model.\nManaged by the LLM picker dropdown."))
|
||||||
|
FString LLMModel = TEXT("gemini-2.5-flash");
|
||||||
|
|
||||||
|
/** Agent language code (e.g. "en", "fr", "ja").
|
||||||
|
* Managed by the Language picker dropdown.
|
||||||
|
* Controls STT and TTS language selection on ElevenLabs. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice",
|
||||||
|
meta = (ToolTip = "Language code.\nManaged by the Language picker dropdown."))
|
||||||
|
FString Language = TEXT("en");
|
||||||
|
|
||||||
|
/** Enable multilingual mode: the agent dynamically adapts to whatever
|
||||||
|
* language the user speaks in, switching seamlessly mid-conversation.
|
||||||
|
* Requires a multilingual TTS model (e.g. eleven_multilingual_v2 or eleven_turbo_v2_5).
|
||||||
|
* When enabled, the fixed language instruction (bAutoLanguageInstruction) is replaced
|
||||||
|
* by a multilingual prompt that tells the LLM to mirror the user's language.
|
||||||
|
* The Language field still serves as the default/fallback for STT. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice",
|
||||||
|
meta = (ToolTip = "Allow the agent to switch languages dynamically.\nThe agent responds in whatever language the user speaks.\nRequires a multilingual TTS model (turbo_v2_5, multilingual_v2, flash_v2_5)."))
|
||||||
|
bool bMultilingual = false;
|
||||||
|
|
||||||
|
/** Prompt fragment appended when bMultilingual is true.
|
||||||
|
* Instructs the LLM to mirror the user's language. Editable for customization. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice",
|
||||||
|
meta = (MultiLine = "true", EditCondition = "bMultilingual",
|
||||||
|
ToolTip = "Prompt instructions for multilingual behavior.\nAppended when bMultilingual is true."))
|
||||||
|
FString MultilingualPromptFragment = TEXT(
|
||||||
|
"## Language\n"
|
||||||
|
"You are multilingual. ALWAYS respond in the same language the user is speaking. "
|
||||||
|
"If the user switches language mid-conversation, switch with them immediately. "
|
||||||
|
"Match the user's language exactly — do not default to English. "
|
||||||
|
"If the user has not spoken yet, use the language of your first message.");
|
||||||
|
|
||||||
|
/** Append a language instruction to the system prompt when not English.
|
||||||
|
* Ensures the LLM generates text in the correct language.
|
||||||
|
* The Language field controls STT/TTS, but NOT the LLM output language.
|
||||||
|
* Without this, the LLM may default to English for follow-up messages.
|
||||||
|
* Ignored when bMultilingual is true (multilingual prompt takes priority). */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice",
|
||||||
|
meta = (EditCondition = "!bMultilingual",
|
||||||
|
ToolTip = "Append a language instruction for non-English agents.\nThe Language field only controls STT/TTS, not the LLM output.\nIgnored when Multilingual is enabled."))
|
||||||
|
bool bAutoLanguageInstruction = true;
|
||||||
|
|
||||||
|
/** Prompt fragment appended when bAutoLanguageInstruction is true and language is not English.
|
||||||
|
* Use {Language} as a placeholder — it will be replaced by the language name (e.g. "French").
|
||||||
|
* Pre-filled with a standard instruction. Editable for customization. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice",
|
||||||
|
meta = (MultiLine = "true",
|
||||||
|
EditCondition = "bAutoLanguageInstruction && !bMultilingual",
|
||||||
|
ToolTip = "Prompt instruction for fixed-language mode.\n{Language} is replaced by the selected language name.\nAppended when language is not English."))
|
||||||
|
FString LanguagePromptFragment = TEXT(
|
||||||
|
"## Language\n"
|
||||||
|
"You MUST always respond in {Language}. "
|
||||||
|
"Never switch to any other language, "
|
||||||
|
"even for follow-up messages or when the user is silent.");
|
||||||
|
|
||||||
|
// ── Behavior ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Character-specific prompt describing THIS agent's personality and context.
|
||||||
|
* This is YOUR prompt — write what makes this character unique.
|
||||||
|
* The emotion tool instructions are appended automatically if enabled. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Behavior",
|
||||||
|
meta = (MultiLine = "true",
|
||||||
|
ToolTip = "Character-specific prompt.\nDescribe the character's personality, backstory, and behavior.\nEmotion tool instructions are appended automatically if enabled."))
|
||||||
|
FString CharacterPrompt;
|
||||||
|
|
||||||
|
/** First message the agent says when the conversation starts.
|
||||||
|
* Leave empty to let the agent wait for the user to speak first. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Behavior",
|
||||||
|
meta = (ToolTip = "Agent's opening message.\nLeave empty for no greeting."))
|
||||||
|
FString FirstMessage;
|
||||||
|
|
||||||
|
/** Disable the idle follow-up behavior where the agent automatically speaks again
|
||||||
|
* if the user remains silent after the greeting / last response.
|
||||||
|
* When enabled: sets turn_timeout to -1 (infinite wait) so the agent
|
||||||
|
* waits indefinitely for the user to speak first. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice",
|
||||||
|
meta = (ToolTip = "Prevent the agent from speaking again unprompted.\nSets turn_timeout to -1 (infinite wait).\nUseful when you want the player to initiate the conversation."))
|
||||||
|
bool bDisableIdleFollowUp = false;
|
||||||
|
|
||||||
|
/** Time (seconds) the agent waits for the user to speak before re-engaging.
|
||||||
|
* ElevenLabs API: conversation_config.turn.turn_timeout.
|
||||||
|
* Range: 1–30 seconds. Default: 7. -1 = wait indefinitely.
|
||||||
|
* Higher values make the agent more patient.
|
||||||
|
* When bDisableIdleFollowUp is true, this is overridden to -1 (infinite). */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice",
|
||||||
|
meta = (DisplayName = "Follow-up Timeout",
|
||||||
|
ClampMin = "-1.0", ClampMax = "30.0",
|
||||||
|
EditCondition = "!bDisableIdleFollowUp",
|
||||||
|
ToolTip = "Seconds before the agent speaks again if the user is silent.\nRange: 1-30. Default: 7. -1 = wait indefinitely.\nWhen 'Disable Idle Follow-up' is on, forced to -1 (infinite)."))
|
||||||
|
float TurnTimeout = 7.0f;
|
||||||
|
|
||||||
|
/** Maximum number of turns in a conversation. 0 = unlimited. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Voice",
|
||||||
|
meta = (ClampMin = "0",
|
||||||
|
ToolTip = "Max conversation turns.\n0 = unlimited."))
|
||||||
|
int32 MaxTurns = 0;
|
||||||
|
|
||||||
|
// ── Emotion Tool ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Include the built-in "set_emotion" client tool in the agent configuration.
|
||||||
|
* Allows the LLM to set facial expressions (Joy, Sadness, Anger, etc.)
|
||||||
|
* that drive the FacialExpression component in real-time. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Emotion Tool",
|
||||||
|
meta = (ToolTip = "Include the set_emotion client tool.\nAllows the LLM to drive facial expressions."))
|
||||||
|
bool bIncludeEmotionTool = true;
|
||||||
|
|
||||||
|
/** System prompt fragment appended to CharacterPrompt when bIncludeEmotionTool is true.
|
||||||
|
* Pre-filled with the standard emotion instruction. Editable for customization. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Emotion Tool",
|
||||||
|
meta = (MultiLine = "true", EditCondition = "bIncludeEmotionTool",
|
||||||
|
ToolTip = "Prompt instructions for the emotion tool.\nAppended to CharacterPrompt when creating/updating the agent."))
|
||||||
|
FString EmotionToolPromptFragment = TEXT(
|
||||||
|
"## Facial Expressions\n"
|
||||||
|
"You have a set_emotion tool to control your facial expression. "
|
||||||
|
"Use it whenever the emotional context changes:\n"
|
||||||
|
"- Call set_emotion with emotion=\"joy\" when happy, laughing, or excited\n"
|
||||||
|
"- Call set_emotion with emotion=\"sadness\" when empathetic or discussing sad topics\n"
|
||||||
|
"- Call set_emotion with emotion=\"anger\" when frustrated or discussing injustice\n"
|
||||||
|
"- Call set_emotion with emotion=\"surprise\" when reacting to unexpected information\n"
|
||||||
|
"- Call set_emotion with emotion=\"fear\" when discussing scary or worrying topics\n"
|
||||||
|
"- Call set_emotion with emotion=\"disgust\" when reacting to unpleasant things\n"
|
||||||
|
"- Call set_emotion with emotion=\"neutral\" to return to a calm expression\n\n"
|
||||||
|
"Use intensity to match the strength of the emotion:\n"
|
||||||
|
"- \"low\" for subtle hints (slight smile, mild concern)\n"
|
||||||
|
"- \"medium\" for normal expression (default)\n"
|
||||||
|
"- \"high\" for strong reactions (big laugh, deep sadness, shock)\n\n"
|
||||||
|
"Always return to neutral when the emotional moment passes.");
|
||||||
|
|
||||||
|
// ── Dynamic Variables ────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Key-value pairs sent as dynamic_variables at conversation start.
|
||||||
|
* Referenced in the system prompt as {{variable_name}}. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Dynamic Variables",
|
||||||
|
meta = (ToolTip = "Dynamic variables available in the system prompt as {{key}}.\nSent at conversation start."))
|
||||||
|
TMap<FString, FString> DefaultDynamicVariables;
|
||||||
|
|
||||||
|
// ── Metadata (read-only, populated by API) ───────────────────────────────
|
||||||
|
|
||||||
|
/** Timestamp of last API sync (ISO 8601). */
|
||||||
|
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Metadata",
|
||||||
|
meta = (ToolTip = "When this asset was last synced with ElevenLabs."))
|
||||||
|
FString LastSyncTimestamp;
|
||||||
|
};
|
||||||
@@ -14,6 +14,7 @@
|
|||||||
class UAudioComponent;
|
class UAudioComponent;
|
||||||
class USoundAttenuation;
|
class USoundAttenuation;
|
||||||
class UPS_AI_ConvAgent_MicrophoneCaptureComponent;
|
class UPS_AI_ConvAgent_MicrophoneCaptureComponent;
|
||||||
|
class UPS_AI_ConvAgent_AgentConfig_ElevenLabs;
|
||||||
class APlayerController;
|
class APlayerController;
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -110,9 +111,17 @@ public:
|
|||||||
|
|
||||||
// ── Configuration ─────────────────────────────────────────────────────────
|
// ── Configuration ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** ElevenLabs Agent ID used for this conversation. Leave empty to use the default from Project Settings > PS AI ConvAgent - ElevenLabs. */
|
/** Agent configuration data asset.
|
||||||
|
* When set, the AgentID is resolved from this asset at conversation start.
|
||||||
|
* Create one via Content Browser → right-click → Miscellaneous → PS AI ConvAgent Agent Config. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs",
|
||||||
meta = (ToolTip = "ElevenLabs Agent ID. Leave empty to use the project default from Project Settings."))
|
meta = (ToolTip = "Agent configuration data asset.\nOverrides the AgentID string below when set."))
|
||||||
|
TObjectPtr<UPS_AI_ConvAgent_AgentConfig_ElevenLabs> AgentConfig;
|
||||||
|
|
||||||
|
/** ElevenLabs Agent ID used for this conversation. Leave empty to use the default from Project Settings > PS AI ConvAgent - ElevenLabs.
|
||||||
|
* Overridden by AgentConfig if set. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs",
|
||||||
|
meta = (ToolTip = "ElevenLabs Agent ID. Leave empty to use the project default from Project Settings.\nOverridden by AgentConfig if set."))
|
||||||
FString AgentID;
|
FString AgentID;
|
||||||
|
|
||||||
/** How turn-taking is managed between the user and the agent.\n- Server VAD (recommended): ElevenLabs automatically detects when the user stops speaking.\n- Client Controlled: You manually call StartListening/StopListening (push-to-talk with a key). */
|
/** How turn-taking is managed between the user and the agent.\n- Server VAD (recommended): ElevenLabs automatically detects when the user stops speaking.\n- Client Controlled: You manually call StartListening/StopListening (push-to-talk with a key). */
|
||||||
|
|||||||
@@ -114,6 +114,18 @@ public:
|
|||||||
ToolTip = "Seconds to wait before the agent stops looking at the pawn.\n0 = immediate."))
|
ToolTip = "Seconds to wait before the agent stops looking at the pawn.\n0 = immediate."))
|
||||||
float PostureDetachDelay = 0.0f;
|
float PostureDetachDelay = 0.0f;
|
||||||
|
|
||||||
|
// ── Conversation management ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Automatically start the WebSocket conversation when an agent is selected
|
||||||
|
* (enters range + view cone). When false, selecting an agent only manages
|
||||||
|
* posture and visual awareness — the conversation must be started explicitly
|
||||||
|
* via StartConversationWithSelectedAgent() (e.g. on a key press).
|
||||||
|
* Set to false when you have multiple agents in a scene to prevent them
|
||||||
|
* all from greeting the player simultaneously. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction",
|
||||||
|
meta = (ToolTip = "Auto-start the WebSocket when an agent is selected by proximity.\nSet to false to require explicit interaction (call StartConversationWithSelectedAgent).\nUseful with multiple agents to prevent simultaneous greetings."))
|
||||||
|
bool bAutoStartConversation = true;
|
||||||
|
|
||||||
// ── Listening management ─────────────────────────────────────────────────
|
// ── Listening management ─────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Automatically call StartListening/StopListening on the agent's
|
/** Automatically call StartListening/StopListening on the agent's
|
||||||
@@ -163,6 +175,14 @@ public:
|
|||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
|
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
|
||||||
void ForceSelectAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Agent);
|
void ForceSelectAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Agent);
|
||||||
|
|
||||||
|
/** Start the WebSocket conversation with the currently selected agent.
|
||||||
|
* Use this when bAutoStartConversation is false and the player explicitly
|
||||||
|
* interacts (e.g. presses a key, enters a trigger zone).
|
||||||
|
* Does nothing if no agent is selected or the agent is already connected.
|
||||||
|
* Also starts mic capture and listening automatically. */
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
|
||||||
|
void StartConversationWithSelectedAgent();
|
||||||
|
|
||||||
/** Clear the current selection. Automatic selection resumes next tick. */
|
/** Clear the current selection. Automatic selection resumes next tick. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
|
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
|
||||||
void ClearSelection();
|
void ClearSelection();
|
||||||
|
|||||||
@@ -22,5 +22,17 @@ public class PS_AI_ConvAgentEditor : ModuleRules
|
|||||||
// Runtime module containing FAnimNode_PS_AI_ConvAgent_LipSync
|
// Runtime module containing FAnimNode_PS_AI_ConvAgent_LipSync
|
||||||
"PS_AI_ConvAgent",
|
"PS_AI_ConvAgent",
|
||||||
});
|
});
|
||||||
|
|
||||||
|
PrivateDependencyModuleNames.AddRange(new string[]
|
||||||
|
{
|
||||||
|
// Slate UI for Detail Customization
|
||||||
|
"Slate",
|
||||||
|
"SlateCore",
|
||||||
|
"PropertyEditor",
|
||||||
|
// HTTP requests for ElevenLabs API (voice list, agent CRUD)
|
||||||
|
"HTTP",
|
||||||
|
"Json",
|
||||||
|
"JsonUtilities",
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,16 +1,39 @@
|
|||||||
// Copyright ASTERION. All Rights Reserved.
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
#include "Modules/ModuleManager.h"
|
#include "Modules/ModuleManager.h"
|
||||||
|
#include "PropertyEditorModule.h"
|
||||||
|
#include "PS_AI_ConvAgent_AgentConfig_ElevenLabs.h"
|
||||||
|
#include "PS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs.h"
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Editor module for PS_AI_ConvAgent plugin.
|
* Editor module for PS_AI_ConvAgent plugin.
|
||||||
* Provides AnimGraph node(s) for the PS AI ConvAgent Lip Sync system.
|
* Provides AnimGraph nodes, asset factories, and Detail Customizations.
|
||||||
*/
|
*/
|
||||||
class FPS_AI_ConvAgentEditorModule : public IModuleInterface
|
class FPS_AI_ConvAgentEditorModule : public IModuleInterface
|
||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
virtual void StartupModule() override {}
|
virtual void StartupModule() override
|
||||||
virtual void ShutdownModule() override {}
|
{
|
||||||
|
FPropertyEditorModule& PropertyModule =
|
||||||
|
FModuleManager::LoadModuleChecked<FPropertyEditorModule>("PropertyEditor");
|
||||||
|
|
||||||
|
PropertyModule.RegisterCustomClassLayout(
|
||||||
|
UPS_AI_ConvAgent_AgentConfig_ElevenLabs::StaticClass()->GetFName(),
|
||||||
|
FOnGetDetailCustomizationInstance::CreateStatic(
|
||||||
|
&FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::MakeInstance));
|
||||||
|
}
|
||||||
|
|
||||||
|
virtual void ShutdownModule() override
|
||||||
|
{
|
||||||
|
if (FModuleManager::Get().IsModuleLoaded("PropertyEditor"))
|
||||||
|
{
|
||||||
|
FPropertyEditorModule& PropertyModule =
|
||||||
|
FModuleManager::GetModuleChecked<FPropertyEditorModule>("PropertyEditor");
|
||||||
|
|
||||||
|
PropertyModule.UnregisterCustomClassLayout(
|
||||||
|
UPS_AI_ConvAgent_AgentConfig_ElevenLabs::StaticClass()->GetFName());
|
||||||
|
}
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
IMPLEMENT_MODULE(FPS_AI_ConvAgentEditorModule, PS_AI_ConvAgentEditor)
|
IMPLEMENT_MODULE(FPS_AI_ConvAgentEditorModule, PS_AI_ConvAgentEditor)
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,95 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "CoreMinimal.h"
|
||||||
|
#include "IDetailCustomization.h"
|
||||||
|
|
||||||
|
class IDetailLayoutBuilder;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Detail Customization for UPS_AI_ConvAgent_AgentConfig_ElevenLabs data assets.
|
||||||
|
*
|
||||||
|
* Provides:
|
||||||
|
* - Voice category: "Fetch Voices" button + STextComboBox dropdown
|
||||||
|
* - Voice category: "Fetch Models" button + STextComboBox dropdown
|
||||||
|
* - Behavior category: LLM picker with "Fetch" button + STextComboBox dropdown
|
||||||
|
* - Behavior category: Language picker dropdown (static list)
|
||||||
|
* - Identity category: "Create Agent" / "Update Agent" / "Fetch Agent" buttons + status text
|
||||||
|
* - Hidden properties: VoiceID, VoiceName, TTSModelID, LLMModel, Language (managed by dropdowns)
|
||||||
|
*/
|
||||||
|
class FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs : public IDetailCustomization
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
static TSharedRef<IDetailCustomization> MakeInstance();
|
||||||
|
|
||||||
|
virtual void CustomizeDetails(IDetailLayoutBuilder& DetailBuilder) override;
|
||||||
|
|
||||||
|
private:
|
||||||
|
// ── Voice picker ─────────────────────────────────────────────────────────
|
||||||
|
void OnFetchVoicesClicked();
|
||||||
|
void OnVoiceSelected(TSharedPtr<FString> NewSelection, ESelectInfo::Type SelectInfo);
|
||||||
|
|
||||||
|
// ── Model picker ─────────────────────────────────────────────────────────
|
||||||
|
void OnFetchModelsClicked();
|
||||||
|
void OnModelSelected(TSharedPtr<FString> NewSelection, ESelectInfo::Type SelectInfo);
|
||||||
|
|
||||||
|
// ── LLM picker ──────────────────────────────────────────────────────────
|
||||||
|
void OnFetchLLMsClicked();
|
||||||
|
void OnLLMSelected(TSharedPtr<FString> NewSelection, ESelectInfo::Type SelectInfo);
|
||||||
|
|
||||||
|
// ── Language picker ──────────────────────────────────────────────────────
|
||||||
|
void OnFetchLanguagesClicked();
|
||||||
|
void OnLanguageSelected(TSharedPtr<FString> NewSelection, ESelectInfo::Type SelectInfo);
|
||||||
|
|
||||||
|
// ── Agent API ────────────────────────────────────────────────────────────
|
||||||
|
void OnCreateAgentClicked();
|
||||||
|
void OnUpdateAgentClicked();
|
||||||
|
void OnFetchAgentClicked();
|
||||||
|
|
||||||
|
// ── Helpers ──────────────────────────────────────────────────────────────
|
||||||
|
FString GetAPIKey() const;
|
||||||
|
TSharedPtr<FJsonObject> BuildAgentPayload() const;
|
||||||
|
TSharedPtr<FJsonObject> BuildEmotionToolDefinition() const;
|
||||||
|
|
||||||
|
/** Display a status message in the Identity category.
|
||||||
|
* Color: red for errors, green for success, blue/cyan for info. */
|
||||||
|
void SetStatusText(const FString& Text);
|
||||||
|
void SetStatusError(const FString& Text);
|
||||||
|
void SetStatusSuccess(const FString& Text);
|
||||||
|
|
||||||
|
/** Parse ElevenLabs API error JSON and return a human-readable message. */
|
||||||
|
static FString ParseAPIError(int32 HttpCode, const FString& ResponseBody);
|
||||||
|
|
||||||
|
/** Retrieve the data asset being edited (first selected object). */
|
||||||
|
class UPS_AI_ConvAgent_AgentConfig_ElevenLabs* GetEditedAsset() const;
|
||||||
|
|
||||||
|
// ── Cached state ─────────────────────────────────────────────────────────
|
||||||
|
TArray<TWeakObjectPtr<UObject>> SelectedObjects;
|
||||||
|
|
||||||
|
// Voice combo data
|
||||||
|
TArray<TSharedPtr<FString>> VoiceDisplayNames;
|
||||||
|
TArray<FString> VoiceIDs;
|
||||||
|
TSharedPtr<class STextComboBox> VoiceComboBox;
|
||||||
|
|
||||||
|
// Model combo data
|
||||||
|
TArray<TSharedPtr<FString>> ModelDisplayNames;
|
||||||
|
TArray<FString> ModelIDs;
|
||||||
|
TSharedPtr<class STextComboBox> ModelComboBox;
|
||||||
|
|
||||||
|
// LLM combo data
|
||||||
|
TArray<TSharedPtr<FString>> LLMDisplayNames;
|
||||||
|
TArray<FString> LLMModelIDs;
|
||||||
|
TSharedPtr<class STextComboBox> LLMComboBox;
|
||||||
|
|
||||||
|
// Language combo data
|
||||||
|
TArray<TSharedPtr<FString>> LanguageDisplayNames;
|
||||||
|
TArray<FString> LanguageCodes;
|
||||||
|
TSharedPtr<class STextComboBox> LanguageComboBox;
|
||||||
|
|
||||||
|
// Status feedback
|
||||||
|
TSharedPtr<class STextBlock> StatusTextBlock;
|
||||||
|
|
||||||
|
// Guard: prevents infinite auto-fetch loop when PostEditChange re-triggers CustomizeDetails.
|
||||||
|
bool bAutoFetchDone = false;
|
||||||
|
};
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#include "PS_AI_ConvAgent_AgentConfigFactory_ElevenLabs.h"
|
||||||
|
#include "PS_AI_ConvAgent_AgentConfig_ElevenLabs.h"
|
||||||
|
#include "AssetTypeCategories.h"
|
||||||
|
|
||||||
|
UPS_AI_ConvAgent_AgentConfigFactory_ElevenLabs::UPS_AI_ConvAgent_AgentConfigFactory_ElevenLabs()
|
||||||
|
{
|
||||||
|
SupportedClass = UPS_AI_ConvAgent_AgentConfig_ElevenLabs::StaticClass();
|
||||||
|
bCreateNew = true;
|
||||||
|
bEditAfterNew = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
UObject* UPS_AI_ConvAgent_AgentConfigFactory_ElevenLabs::FactoryCreateNew(
|
||||||
|
UClass* Class, UObject* InParent, FName Name, EObjectFlags Flags,
|
||||||
|
UObject* Context, FFeedbackContext* Warn)
|
||||||
|
{
|
||||||
|
return NewObject<UPS_AI_ConvAgent_AgentConfig_ElevenLabs>(InParent, Class, Name, Flags);
|
||||||
|
}
|
||||||
|
|
||||||
|
FText UPS_AI_ConvAgent_AgentConfigFactory_ElevenLabs::GetDisplayName() const
|
||||||
|
{
|
||||||
|
return FText::FromString(TEXT("PS AI ConvAgent Agent Config (ElevenLabs)"));
|
||||||
|
}
|
||||||
|
|
||||||
|
uint32 UPS_AI_ConvAgent_AgentConfigFactory_ElevenLabs::GetMenuCategories() const
|
||||||
|
{
|
||||||
|
return EAssetTypeCategories::Misc;
|
||||||
|
}
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "CoreMinimal.h"
|
||||||
|
#include "Factories/Factory.h"
|
||||||
|
#include "PS_AI_ConvAgent_AgentConfigFactory_ElevenLabs.generated.h"
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Factory that lets users create PS_AI_ConvAgent_AgentConfig_ElevenLabs assets
|
||||||
|
* directly from the Content Browser (right-click → Miscellaneous).
|
||||||
|
*/
|
||||||
|
UCLASS()
|
||||||
|
class UPS_AI_ConvAgent_AgentConfigFactory_ElevenLabs : public UFactory
|
||||||
|
{
|
||||||
|
GENERATED_BODY()
|
||||||
|
|
||||||
|
public:
|
||||||
|
UPS_AI_ConvAgent_AgentConfigFactory_ElevenLabs();
|
||||||
|
|
||||||
|
virtual UObject* FactoryCreateNew(UClass* Class, UObject* InParent,
|
||||||
|
FName Name, EObjectFlags Flags, UObject* Context,
|
||||||
|
FFeedbackContext* Warn) override;
|
||||||
|
|
||||||
|
virtual FText GetDisplayName() const override;
|
||||||
|
virtual uint32 GetMenuCategories() const override;
|
||||||
|
};
|
||||||
Reference in New Issue
Block a user