Compare commits
34 Commits
fda2e4e32f
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 2f35c134aa | |||
| 086311da38 | |||
| c783613117 | |||
| d92e402722 | |||
| 231ca4145d | |||
| 087c83018c | |||
| f49a57f892 | |||
| 6d92ec3c26 | |||
| bcb6c44f57 | |||
| cf83a2302d | |||
| 5b3a11a733 | |||
| 2ca5643f32 | |||
| 4854a7fa9a | |||
| 41900991ef | |||
| efd9921fde | |||
| fee7d0c679 | |||
| 777ffa09a8 | |||
| ab2482a209 | |||
| c2ca2e3dbb | |||
| 8ae73bbacb | |||
| 6a8c22c77e | |||
| 7d99a89ac7 | |||
| a5b0d5db84 | |||
| cfade2a265 | |||
| 27d0906d21 | |||
| 89d555b734 | |||
| 7430a18e66 | |||
| ceaf89f836 | |||
| a1e34e6dfa | |||
| f338c0586e | |||
| 48d1543329 | |||
| 15871be9e9 | |||
| 6fcbdb2005 | |||
| e322a400be |
@@ -1,5 +1,20 @@
|
|||||||
# PS_AI_Agent — Notes pour Claude
|
# PS_AI_Agent — Notes pour Claude
|
||||||
|
|
||||||
|
## Workflow règles
|
||||||
|
|
||||||
|
**TOUJOURS demander avant de lancer un changement de code.** Avant d'appeler Edit/Write/sed sur un fichier source :
|
||||||
|
1. Expliquer ce qu'on s'apprête à faire (quel fichier, quelle logique, quel impact)
|
||||||
|
2. Attendre la confirmation explicite de l'utilisateur
|
||||||
|
3. Seulement après → effectuer le changement
|
||||||
|
|
||||||
|
Cela s'applique à : modifications de code (.cpp/.h), refactors, renames bulk, ajouts de nouvelles fonctions.
|
||||||
|
Cela ne s'applique PAS à : lecture de fichiers, recherches, git status/log/diff, builds, questions diagnostiques.
|
||||||
|
|
||||||
|
Exceptions où on peut agir directement :
|
||||||
|
- L'utilisateur dit explicitement "fais-le" / "go" / "lance" / "commit"
|
||||||
|
- L'utilisateur demande explicitement un changement précis ("renomme X en Y", "ajoute cette fonction")
|
||||||
|
- Annulation / revert demandé par l'utilisateur
|
||||||
|
|
||||||
## Plugins dans ce repo
|
## Plugins dans ce repo
|
||||||
|
|
||||||
### PS_AI_ConvAgent
|
### PS_AI_ConvAgent
|
||||||
|
|||||||
3
.gitignore
vendored
3
.gitignore
vendored
@@ -5,7 +5,8 @@ Unreal/PS_AI_Agent/Intermediate/
|
|||||||
Unreal/PS_AI_Agent/Saved/
|
Unreal/PS_AI_Agent/Saved/
|
||||||
ConvAI/Convai/Binaries/
|
ConvAI/Convai/Binaries/
|
||||||
|
|
||||||
# Plugin Intermediate folders (build artifacts, regenerated)
|
# Plugin build artifacts (regenerated)
|
||||||
|
Unreal/PS_AI_Agent/Plugins/*/Binaries/
|
||||||
Unreal/PS_AI_Agent/Plugins/*/Intermediate/
|
Unreal/PS_AI_Agent/Plugins/*/Intermediate/
|
||||||
|
|
||||||
# Live Coding patch files (temporary, regenerated each session)
|
# Live Coding patch files (temporary, regenerated each session)
|
||||||
|
|||||||
BIN
Docs/PS_AI_Behavior_Documentation.pdf
Normal file
BIN
Docs/PS_AI_Behavior_Documentation.pdf
Normal file
Binary file not shown.
BIN
Docs/PS_AI_Behavior_Documentation.pptx
Normal file
BIN
Docs/PS_AI_Behavior_Documentation.pptx
Normal file
Binary file not shown.
BIN
Docs/PS_AI_ConvAgent_Documentation.pdf
Normal file
BIN
Docs/PS_AI_ConvAgent_Documentation.pdf
Normal file
Binary file not shown.
BIN
Docs/PS_AI_ConvAgent_Documentation.pptx
Normal file
BIN
Docs/PS_AI_ConvAgent_Documentation.pptx
Normal file
Binary file not shown.
691
Docs/gen_behavior_doc.js
Normal file
691
Docs/gen_behavior_doc.js
Normal file
@@ -0,0 +1,691 @@
|
|||||||
|
// PS_AI_Behavior User Documentation Generator
|
||||||
|
// Palette: Warm Terracotta (terracotta #B85042, sand #E7E8D1, sage #A7BEAE)
|
||||||
|
|
||||||
|
const pptxgen = require("pptxgenjs");
|
||||||
|
const pres = new pptxgen();
|
||||||
|
|
||||||
|
pres.layout = "LAYOUT_WIDE"; // 13.333 x 7.5 inch
|
||||||
|
|
||||||
|
// ─── Color Palette ─────────────────────────────────
|
||||||
|
const TERRA = "B85042";
|
||||||
|
const TERRA_D = "7A3229";
|
||||||
|
const SAND = "E7E8D1";
|
||||||
|
const SAND_LT = "F5F5EA";
|
||||||
|
const SAGE = "A7BEAE";
|
||||||
|
const SAGE_D = "5F7566";
|
||||||
|
const WHITE = "FFFFFF";
|
||||||
|
const CHARCOAL = "2E2A26";
|
||||||
|
const GOLD = "D4A84B";
|
||||||
|
|
||||||
|
// ─── Font ──────────────────────────────────────────
|
||||||
|
const FONT_H = "Calibri";
|
||||||
|
const FONT_B = "Calibri";
|
||||||
|
|
||||||
|
// ─── Master slide backgrounds ──────────────────────
|
||||||
|
pres.defineSlideMaster({
|
||||||
|
title: "CONTENT",
|
||||||
|
background: { color: SAND_LT },
|
||||||
|
objects: [
|
||||||
|
{ rect: { x: 0, y: 0, w: 13.333, h: 0.35, fill: { color: TERRA } } },
|
||||||
|
{ rect: { x: 0, y: 7.15, w: 13.333, h: 0.35, fill: { color: TERRA } } },
|
||||||
|
{ text: {
|
||||||
|
text: "PS_AI_Behavior — User Documentation",
|
||||||
|
options: { x: 0.3, y: 7.18, w: 8, h: 0.3, fontSize: 10, color: SAND, fontFace: FONT_B }
|
||||||
|
} },
|
||||||
|
{ text: {
|
||||||
|
text: "ASTERION",
|
||||||
|
options: { x: 11.5, y: 7.18, w: 1.5, h: 0.3, fontSize: 10, color: SAND, fontFace: FONT_B, align: "right", bold: true }
|
||||||
|
} },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 1 — TITLE
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
let s = pres.addSlide();
|
||||||
|
s.background = { color: TERRA };
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0, y: 3.3, w: 13.333, h: 0.05, fill: { color: GOLD } });
|
||||||
|
s.addText("PS_AI_Behavior", { x: 0.6, y: 2.2, w: 12, h: 1.2, fontSize: 60, bold: true, color: WHITE, fontFace: FONT_H });
|
||||||
|
s.addText("Personality-Driven NPC AI for Unreal Engine", { x: 0.6, y: 3.5, w: 12, h: 0.6, fontSize: 22, color: SAND, fontFace: FONT_B, italic: true });
|
||||||
|
s.addText("User Documentation", { x: 0.6, y: 4.4, w: 12, h: 0.5, fontSize: 18, color: SAND, fontFace: FONT_B });
|
||||||
|
s.addText("Civilians • Enemies • Protectors • Cover System • Splines", { x: 0.6, y: 5.2, w: 12, h: 0.4, fontSize: 14, color: SAND, fontFace: FONT_B });
|
||||||
|
s.addText("ASTERION • 2026", { x: 0.6, y: 6.6, w: 12, h: 0.4, fontSize: 12, color: SAND, fontFace: FONT_B });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 2 — OVERVIEW
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Plugin Overview", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: GOLD } });
|
||||||
|
|
||||||
|
s.addText("What it does", { x: 0.5, y: 1.7, w: 6, h: 0.4, fontSize: 18, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "Personality-driven AI for NPCs", options: { bullet: true, fontSize: 14 } },
|
||||||
|
{ text: "Three archetypes: Civilian, Enemy, Protector", options: { bullet: true, fontSize: 14 } },
|
||||||
|
{ text: "Trait-based reactions: Courage, Aggressivity, Caution", options: { bullet: true, fontSize: 14 } },
|
||||||
|
{ text: "Tactical combat + cover system + EQS", options: { bullet: true, fontSize: 14 } },
|
||||||
|
{ text: "Spline-based patrol networks", options: { bullet: true, fontSize: 14 } },
|
||||||
|
{ text: "Interface-driven — no Pawn class dependency", options: { bullet: true, fontSize: 14 } },
|
||||||
|
], { x: 0.5, y: 2.1, w: 6, h: 3, color: CHARCOAL, fontFace: FONT_B, paraSpaceAfter: 8 });
|
||||||
|
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 7, y: 1.7, w: 5.8, h: 4.8, fill: { color: TERRA }, rectRadius: 0.1 });
|
||||||
|
s.addText("Primary Use Case", { x: 7.2, y: 1.9, w: 5.4, h: 0.4, fontSize: 16, bold: true, color: GOLD, fontFace: FONT_H });
|
||||||
|
s.addText("Populate your world with intelligent NPCs that perceive threats, evaluate risk based on personality, and respond appropriately — civilians flee from gunfire, enemies engage in tactical combat, protectors defend civilians.", {
|
||||||
|
x: 7.2, y: 2.4, w: 5.4, h: 1.7, fontSize: 13, color: SAND, fontFace: FONT_B, italic: true
|
||||||
|
});
|
||||||
|
s.addText("Key Differentiators", { x: 7.2, y: 4.2, w: 5.4, h: 0.4, fontSize: 16, bold: true, color: GOLD, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "Data-driven personality profiles", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "Built-in cover cycle state machine", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "Spline network with junction detection", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "Gaze bridge to ConvAgent (reflection)", options: { bullet: true, fontSize: 12 } },
|
||||||
|
], { x: 7.2, y: 4.7, w: 5.4, h: 1.7, color: SAND, fontFace: FONT_B, paraSpaceAfter: 4 });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 3 — CORE ARCHITECTURE
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Core Architecture", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: GOLD } });
|
||||||
|
|
||||||
|
s.addText("The plugin is built around four concepts:", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: CHARCOAL, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
const pillars = [
|
||||||
|
{ title: "Interface", subtitle: "IPS_AI_Behavior_Interface", desc: "Implemented on host project Pawns. Zero compile dependency — identity, team, combat, movement." },
|
||||||
|
{ title: "NPC Type", subtitle: "Civilian / Enemy / Protector", desc: "Determines team, spline access, default tree. Enemies can be disguised (hostile=false)." },
|
||||||
|
{ title: "State", subtitle: "Behavior State enum", desc: "Idle, Patrol, Alerted, Combat, Fleeing, TakingCover, Dead, Scripted. Written to Blackboard." },
|
||||||
|
{ title: "Profile", subtitle: "PersonalityProfile data asset", desc: "Traits, thresholds, target priority, speed per state, default BT." },
|
||||||
|
];
|
||||||
|
pillars.forEach((p, i) => {
|
||||||
|
const col = i % 2;
|
||||||
|
const row = Math.floor(i / 2);
|
||||||
|
const x = 0.5 + col * 6.3;
|
||||||
|
const y = 2.2 + row * 2.3;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: x, y: y, w: 6.0, h: 2.1, fill: { color: TERRA }, rectRadius: 0.1 });
|
||||||
|
s.addText(p.title, { x: x + 0.2, y: y + 0.15, w: 5.6, h: 0.45, fontSize: 20, bold: true, color: GOLD, fontFace: FONT_H });
|
||||||
|
s.addText(p.subtitle, { x: x + 0.2, y: y + 0.6, w: 5.6, h: 0.35, fontSize: 12, color: SAND, italic: true, fontFace: "Consolas" });
|
||||||
|
s.addText(p.desc, { x: x + 0.2, y: y + 1.0, w: 5.6, h: 1.0, fontSize: 12, color: SAND, fontFace: FONT_B });
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 4 — NPC TYPES & TEAM ID
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("NPC Types & TeamId", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: GOLD } });
|
||||||
|
s.addText("Identity drives perception, targeting, and access control.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: CHARCOAL, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
// Three NPC cards
|
||||||
|
const npcs = [
|
||||||
|
{ name: "CIVILIAN", color: SAGE, id: "0x00", desc: "Non-hostile. Flees. Allied with Protector." },
|
||||||
|
{ name: "ENEMY", color: TERRA, id: "0x10", desc: "Hostile. Tactical combat. Can be disguised." },
|
||||||
|
{ name: "PROTECTOR", color: SAGE_D, id: "0x20", desc: "Police / guard. Defends Civilians." },
|
||||||
|
];
|
||||||
|
npcs.forEach((n, i) => {
|
||||||
|
const x = 0.5 + i * 4.3;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: x, y: 2.15, w: 4.0, h: 2.2, fill: { color: n.color }, rectRadius: 0.1 });
|
||||||
|
s.addText(n.name, { x: x + 0.2, y: 2.3, w: 3.6, h: 0.5, fontSize: 22, bold: true, color: WHITE, align: "center", fontFace: FONT_H });
|
||||||
|
s.addText(`TeamId = ${n.id}`, { x: x + 0.2, y: 2.85, w: 3.6, h: 0.35, fontSize: 14, color: WHITE, align: "center", italic: true, fontFace: "Consolas" });
|
||||||
|
s.addText(n.desc, { x: x + 0.2, y: 3.3, w: 3.6, h: 0.95, fontSize: 12, color: WHITE, align: "center", fontFace: FONT_B });
|
||||||
|
});
|
||||||
|
|
||||||
|
// TeamId encoding
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: 4.6, w: 12.3, h: 2.3, fill: { color: TERRA }, rectRadius: 0.1 });
|
||||||
|
s.addText("TeamId Encoding", { x: 0.7, y: 4.7, w: 12, h: 0.4, fontSize: 16, bold: true, color: GOLD, fontFace: FONT_H });
|
||||||
|
s.addText("uint8 = (high nibble: NPCType) | (low nibble: Faction 0-15)", { x: 0.7, y: 5.1, w: 12, h: 0.3, fontSize: 13, color: SAND, italic: true, fontFace: "Consolas" });
|
||||||
|
|
||||||
|
s.addText([
|
||||||
|
{ text: "Same type + same faction → Friendly (allies)", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "Same type + different faction → Hostile (rival gangs)", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "Civilian ↔ Protector → always Friendly", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "Everything else → Hostile", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "Disguised Enemy (hostile=false) → TeamId 0x01 (civilian disguise)", options: { bullet: true, fontSize: 12 } },
|
||||||
|
], { x: 0.7, y: 5.45, w: 12, h: 1.4, color: SAND, fontFace: FONT_B, paraSpaceAfter: 3 });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 5 — BEHAVIOR STATES
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Behavior States", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: GOLD } });
|
||||||
|
s.addText("Enum written to Blackboard, drives BT branching.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: CHARCOAL, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
const states = [
|
||||||
|
{ name: "Idle", desc: "Resting / waiting. Proximity gaze active." },
|
||||||
|
{ name: "Patrol", desc: "Wandering on waypoints or splines." },
|
||||||
|
{ name: "Alerted", desc: "Suspicious, heightened awareness." },
|
||||||
|
{ name: "Combat", desc: "Engaged with a threat, attacking." },
|
||||||
|
{ name: "Fleeing", desc: "Running away from danger." },
|
||||||
|
{ name: "TakingCover", desc: "Behind tactical cover during combat." },
|
||||||
|
{ name: "Dead", desc: "NPC deceased, all systems shut down." },
|
||||||
|
{ name: "Scripted", desc: "Manual override (conversations, cinematics)." },
|
||||||
|
];
|
||||||
|
states.forEach((state, i) => {
|
||||||
|
const col = i % 4;
|
||||||
|
const row = Math.floor(i / 4);
|
||||||
|
const x = 0.5 + col * 3.15;
|
||||||
|
const y = 2.2 + row * 2.3;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: x, y: y, w: 2.95, h: 2.1, fill: { color: SAND }, line: { color: TERRA, width: 1 }, rectRadius: 0.08 });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: x, y: y, w: 2.95, h: 0.55, fill: { color: TERRA } });
|
||||||
|
s.addText(state.name, { x: x, y: y + 0.05, w: 2.95, h: 0.5, fontSize: 16, bold: true, color: WHITE, align: "center", valign: "middle", fontFace: FONT_H });
|
||||||
|
s.addText(state.desc, { x: x + 0.15, y: y + 0.7, w: 2.65, h: 1.3, fontSize: 11, color: CHARCOAL, valign: "top", fontFace: FONT_B });
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 6 — AI CONTROLLER
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("AI Controller", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: GOLD } });
|
||||||
|
s.addText("APS_AI_Behavior_AIController — the central orchestrator.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: CHARCOAL, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
s.addText("Auto-setup on Possession", { x: 0.5, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "Discovers PersonalityComponent on Pawn", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "Computes TeamId from NPCType + Faction", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "Initializes Blackboard with required keys", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "Binds gaze bridge to ConvAgent (if present)", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "Starts BT (unless bAutoStartBehavior = false)", options: { bullet: true, fontSize: 12 } },
|
||||||
|
], { x: 0.5, y: 2.55, w: 6, h: 2.5, color: CHARCOAL, fontFace: FONT_B, paraSpaceAfter: 4 });
|
||||||
|
|
||||||
|
s.addText("Key Methods", { x: 7, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "StartBehavior() / StopBehavior()", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "SetBehaviorState() / GetBehaviorState()", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "SetTeamId() — runtime team change", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "GetBehaviorPerception()", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "GetPersonalityComponent()", options: { bullet: true, fontSize: 12 } },
|
||||||
|
], { x: 7, y: 2.55, w: 6, h: 2.5, color: CHARCOAL, fontFace: FONT_B, paraSpaceAfter: 4 });
|
||||||
|
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: 5.2, w: 12.3, h: 1.8, fill: { color: TERRA }, rectRadius: 0.1 });
|
||||||
|
s.addText("Gaze Bridge (Proximity)", { x: 0.7, y: 5.3, w: 12, h: 0.35, fontSize: 14, bold: true, color: GOLD, fontFace: FONT_H });
|
||||||
|
s.addText("Reflection-based integration with PS_AI_ConvAgent. During Idle/Patrol, NPC glances at nearby actors within GazeProximityRadius. Configurable duration and cooldown. Auto-pauses during active conversations.", {
|
||||||
|
x: 0.7, y: 5.7, w: 12, h: 1.25, fontSize: 12, color: SAND, italic: true, fontFace: FONT_B
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 7 — PERSONALITY COMPONENT
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Personality Component", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: GOLD } });
|
||||||
|
s.addText("Runtime traits & state reactions for the NPC.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: CHARCOAL, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
// Traits triangle
|
||||||
|
s.addText("The Three Traits", { x: 0.5, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
|
||||||
|
const traits = [
|
||||||
|
{ name: "Courage", desc: "0 = coward, 1 = fearless", effect: "Gates flee threshold & cover advancement", color: "C9463A" },
|
||||||
|
{ name: "Aggressivity", desc: "0 = peaceful, 1 = violent", effect: "Drives attack threshold, combat/cover cycle", color: "D4803A" },
|
||||||
|
{ name: "Caution", desc: "0 = reckless, 1 = prudent", effect: "Flee threshold, cover duration, spline threat", color: "7C9D7B" },
|
||||||
|
];
|
||||||
|
traits.forEach((t, i) => {
|
||||||
|
const y = 2.6 + i * 0.95;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: y, w: 1.8, h: 0.8, fill: { color: t.color }, rectRadius: 0.08 });
|
||||||
|
s.addText(t.name, { x: 0.5, y: y + 0.1, w: 1.8, h: 0.3, fontSize: 14, bold: true, color: WHITE, align: "center", fontFace: FONT_H });
|
||||||
|
s.addText(t.desc, { x: 0.5, y: y + 0.45, w: 1.8, h: 0.3, fontSize: 10, color: WHITE, align: "center", italic: true, fontFace: FONT_B });
|
||||||
|
s.addText(t.effect, { x: 2.4, y: y + 0.15, w: 4.2, h: 0.6, fontSize: 12, color: CHARCOAL, valign: "middle", fontFace: FONT_B });
|
||||||
|
});
|
||||||
|
|
||||||
|
s.addText("Properties & Events", { x: 7, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "Profile — PersonalityProfile DataAsset", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "bAutoStartBehavior — per-NPC override", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "RuntimeTraits — editable at runtime", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "PerceivedThreatLevel — replicated", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "CurrentState — replicated with OnRep", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "EvaluateReaction() / ApplyReaction()", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "ModifyTrait() / GetTrait()", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "ForceState() — scripting override", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "OnBehaviorStateChanged delegate", options: { bullet: true, fontSize: 12 } },
|
||||||
|
], { x: 7, y: 2.55, w: 6, h: 4.5, color: CHARCOAL, fontFace: FONT_B, paraSpaceAfter: 3 });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 8 — PERSONALITY PROFILE
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("PersonalityProfile DataAsset", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 32, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: GOLD } });
|
||||||
|
s.addText("The behavioral recipe — one profile per NPC archetype.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: CHARCOAL, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
const profileCats = [
|
||||||
|
{ title: "Identity", items: ["ProfileName", "NPCType", "Faction (0-15)"] },
|
||||||
|
{ title: "Traits", items: ["Courage (0-1)", "Aggressivity (0-1)", "Caution (0-1)"] },
|
||||||
|
{ title: "Thresholds", items: ["FleeThreshold", "AttackThreshold", "AlertThreshold", "ThreatDecayRate"] },
|
||||||
|
{ title: "Combat", items: ["TargetPriority[]", "MinAttackRange / MaxAttackRange", "CombatCoverCycleDuration"] },
|
||||||
|
{ title: "Movement", items: ["SpeedPerState map", "DefaultWalkSpeed"] },
|
||||||
|
{ title: "Behavior", items: ["DefaultBehaviorTree (soft ref)"] },
|
||||||
|
];
|
||||||
|
profileCats.forEach((c, i) => {
|
||||||
|
const col = i % 3;
|
||||||
|
const row = Math.floor(i / 3);
|
||||||
|
const x = 0.5 + col * 4.15;
|
||||||
|
const y = 2.2 + row * 2.3;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: x, y: y, w: 3.95, h: 2.1, fill: { color: SAND }, line: { color: TERRA, width: 1 }, rectRadius: 0.08 });
|
||||||
|
s.addText(c.title, { x: x + 0.15, y: y + 0.1, w: 3.65, h: 0.35, fontSize: 14, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: x + 0.15, y: y + 0.45, w: 0.35, h: 0.04, fill: { color: GOLD } });
|
||||||
|
s.addText(c.items.map(t => ({ text: t, options: { bullet: true, fontSize: 11 } })),
|
||||||
|
{ x: x + 0.15, y: y + 0.55, w: 3.65, h: 1.4, color: CHARCOAL, fontFace: FONT_B, paraSpaceAfter: 3 });
|
||||||
|
});
|
||||||
|
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: 6.85, w: 12.3, h: 0.35, fill: { color: TERRA }, rectRadius: 0.04 });
|
||||||
|
s.addText("Example presets: Cowardly Villager • Aggressive Guard • Rival Gang Member", { x: 0.7, y: 6.88, w: 12, h: 0.3, fontSize: 11, color: SAND, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 9 — COMPONENTS OVERVIEW
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Components", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: GOLD } });
|
||||||
|
s.addText("Attach to Pawns as needed — most are optional.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: CHARCOAL, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
const components = [
|
||||||
|
{ name: "Personality", desc: "Traits, profile, state reactions", required: true },
|
||||||
|
{ name: "Perception", desc: "AI Perception wrapper, threat computation", required: "auto" },
|
||||||
|
{ name: "Team", desc: "Identity for non-AI pawns (player)", required: false },
|
||||||
|
{ name: "Combat", desc: "Attack events, fire/kill delegates", required: false },
|
||||||
|
{ name: "Spline Follower", desc: "Spline walking, junction detection", required: false },
|
||||||
|
];
|
||||||
|
components.forEach((c, i) => {
|
||||||
|
const y = 2.2 + i * 0.9;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: y, w: 3.5, h: 0.8, fill: { color: TERRA }, rectRadius: 0.08 });
|
||||||
|
s.addText(c.name, { x: 0.5, y: y, w: 3.5, h: 0.8, fontSize: 16, bold: true, color: WHITE, align: "center", valign: "middle", fontFace: FONT_H });
|
||||||
|
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 4.2, y: y, w: 7.0, h: 0.8, fill: { color: SAND }, line: { color: TERRA, width: 1 }, rectRadius: 0.06 });
|
||||||
|
s.addText(c.desc, { x: 4.4, y: y + 0.1, w: 6.8, h: 0.6, fontSize: 13, color: CHARCOAL, valign: "middle", fontFace: FONT_B });
|
||||||
|
|
||||||
|
let tagText = c.required === true ? "Required" : (c.required === "auto" ? "Auto" : "Optional");
|
||||||
|
let tagColor = c.required === true ? TERRA : (c.required === "auto" ? SAGE_D : GOLD);
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 11.4, y: y + 0.2, w: 1.4, h: 0.4, fill: { color: tagColor }, rectRadius: 0.08 });
|
||||||
|
s.addText(tagText, { x: 11.4, y: y + 0.2, w: 1.4, h: 0.4, fontSize: 11, bold: true, color: WHITE, align: "center", valign: "middle", fontFace: FONT_H });
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 10 — PERCEPTION COMPONENT
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Perception Component", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: GOLD } });
|
||||||
|
s.addText("Sight, hearing, damage senses — auto-configured.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: CHARCOAL, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
s.addText("Senses", { x: 0.5, y: 2.15, w: 4, h: 0.4, fontSize: 16, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "Sight — vision cone + line of sight", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Hearing — noise events with tags", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Damage — damage received tracking", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 0.5, y: 2.55, w: 4, h: 2, color: CHARCOAL, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
s.addText("Threat Scoring", { x: 4.8, y: 2.15, w: 4, h: 0.4, fontSize: 16, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "Target priority (from Profile)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Recent damage weight", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Proximity weight", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Target persistence (80% score hold)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 4.8, y: 2.55, w: 4.2, h: 2.2, color: CHARCOAL, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
s.addText("Key Methods", { x: 9.3, y: 2.15, w: 3.5, h: 0.4, fontSize: 16, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "GetHighestThreatActor()", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "CalculateThreatLevel()", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "GetThreatLocation()", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "ClassifyActor()", options: { bullet: true, fontSize: 12 } },
|
||||||
|
], { x: 9.3, y: 2.55, w: 3.5, h: 2, color: CHARCOAL, fontFace: "Consolas", paraSpaceAfter: 4 });
|
||||||
|
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: 5.15, w: 12.3, h: 1.85, fill: { color: TERRA }, rectRadius: 0.1 });
|
||||||
|
s.addText("Perception / Threat Resolution", { x: 0.7, y: 5.25, w: 12, h: 0.35, fontSize: 14, bold: true, color: GOLD, fontFace: FONT_H });
|
||||||
|
s.addText("Detects ALL affiliations → filters by Hostile attitude. Supports separate ThreatTarget (e.g. AimTargetActor sphere) vs. owning Pawn. Line-of-sight tracking with last-known-position memory.", {
|
||||||
|
x: 0.7, y: 5.65, w: 12, h: 1.3, fontSize: 12, color: SAND, italic: true, fontFace: FONT_B
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 11 — COMBAT COMPONENT
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Combat Component", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: GOLD } });
|
||||||
|
s.addText("Attack execution, cooldown, damage events.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: CHARCOAL, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
s.addText("Properties", { x: 0.5, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "AttackRange — max distance (cm)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "AttackCooldown — minimum between attacks", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "AttackDamage — base damage value", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "CurrentTarget — replicated", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 0.5, y: 2.55, w: 6, h: 2.3, color: CHARCOAL, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
s.addText("Methods & Events", { x: 7, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "CanAttack() — cooldown check", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "IsInAttackRange() — distance test", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "ExecuteAttack() — apply damage + multicast", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "OnAttackExecuted (multicast)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "OnDamageReceived (server-only)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 7, y: 2.55, w: 6, h: 2.5, color: CHARCOAL, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: 5.3, w: 12.3, h: 1.7, fill: { color: TERRA }, rectRadius: 0.1 });
|
||||||
|
s.addText("Combat Type", { x: 0.7, y: 5.4, w: 12, h: 0.35, fontSize: 14, bold: true, color: GOLD, fontFace: FONT_H });
|
||||||
|
s.addText("NPC's combat type (Melee / Ranged) comes from IPS_AI_Behavior::GetBehaviorCombatType(). Melee = rush the target (no cooldown between re-positions). Ranged = maintain distance, use cover cycle, peek to fire.", {
|
||||||
|
x: 0.7, y: 5.75, w: 12, h: 1.2, fontSize: 12, color: SAND, italic: true, fontFace: FONT_B
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 12 — WORLD ACTORS
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("World Actors", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: GOLD } });
|
||||||
|
s.addText("Placed by level designers — drive navigation and tactics.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: CHARCOAL, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
const actors = [
|
||||||
|
{ name: "SplinePath", desc: "Placeable spline defining a navigation corridor. Filtered by NPCType. Bidirectional, with priority for junction selection." },
|
||||||
|
{ name: "SplineNetwork", desc: "WorldSubsystem — auto-detects intersections between SplinePath actors. Provides navigation queries, threat-aware switching." },
|
||||||
|
{ name: "CoverPoint", desc: "Manually placed tactical position. Cover or HidingSpot type, quality score, crouch flag, max occupants, type filter." },
|
||||||
|
];
|
||||||
|
actors.forEach((a, i) => {
|
||||||
|
const y = 2.2 + i * 1.5;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: y, w: 12.3, h: 1.35, fill: { color: SAND }, line: { color: TERRA, width: 1 }, rectRadius: 0.1 });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: y, w: 3.5, h: 1.35, fill: { color: TERRA } });
|
||||||
|
s.addText(a.name, { x: 0.5, y: y, w: 3.5, h: 1.35, fontSize: 20, bold: true, color: WHITE, align: "center", valign: "middle", fontFace: FONT_H });
|
||||||
|
s.addText(a.desc, { x: 4.2, y: y + 0.15, w: 8.4, h: 1.05, fontSize: 13, color: CHARCOAL, valign: "middle", fontFace: FONT_B });
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 13 — BEHAVIOR TREE SERVICES & TASKS
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Behavior Tree — Services & Tasks", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 30, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: GOLD } });
|
||||||
|
s.addText("Reusable BT nodes — build custom trees per NPC type.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: CHARCOAL, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
s.addText("Services (root-level)", { x: 0.5, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
const services = [
|
||||||
|
{ name: "UpdateThreat", desc: "Queries perception, writes threat BB keys" },
|
||||||
|
{ name: "EvaluateReaction", desc: "Computes state from traits + threat" },
|
||||||
|
{ name: "UpdateGaze", desc: "Manages eye contact, ConvAgent bridge" },
|
||||||
|
];
|
||||||
|
services.forEach((svc, i) => {
|
||||||
|
const y = 2.55 + i * 0.7;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: y, w: 2.3, h: 0.55, fill: { color: SAGE_D }, rectRadius: 0.06 });
|
||||||
|
s.addText(svc.name, { x: 0.5, y: y, w: 2.3, h: 0.55, fontSize: 12, bold: true, color: WHITE, align: "center", valign: "middle", fontFace: FONT_H });
|
||||||
|
s.addText(svc.desc, { x: 2.9, y: y, w: 3.7, h: 0.55, fontSize: 11, color: CHARCOAL, valign: "middle", fontFace: FONT_B });
|
||||||
|
});
|
||||||
|
|
||||||
|
s.addText("Tasks (leaves)", { x: 7, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
const tasks = [
|
||||||
|
{ name: "FollowSpline", desc: "Drives SplineFollower" },
|
||||||
|
{ name: "FindAndFollowSpline", desc: "Auto-locate + follow" },
|
||||||
|
{ name: "Patrol", desc: "Cycle PatrolPoints" },
|
||||||
|
{ name: "FindCover", desc: "Manual + procedural cover" },
|
||||||
|
{ name: "CoverShootCycle", desc: "Cover → peek → fire cycle" },
|
||||||
|
{ name: "Attack", desc: "Chase + attack (Melee/Ranged)" },
|
||||||
|
{ name: "FleeFrom", desc: "Move away from threat" },
|
||||||
|
];
|
||||||
|
tasks.forEach((task, i) => {
|
||||||
|
const y = 2.55 + i * 0.58;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 7, y: y, w: 2.3, h: 0.45, fill: { color: TERRA }, rectRadius: 0.06 });
|
||||||
|
s.addText(task.name, { x: 7, y: y, w: 2.3, h: 0.45, fontSize: 11, bold: true, color: WHITE, align: "center", valign: "middle", fontFace: FONT_H });
|
||||||
|
s.addText(task.desc, { x: 9.4, y: y, w: 3.5, h: 0.45, fontSize: 10, color: CHARCOAL, valign: "middle", fontFace: FONT_B });
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 14 — COVER SYSTEM
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Cover System", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: GOLD } });
|
||||||
|
s.addText("Manual CoverPoints + procedural raycast fallback + EQS refinement.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: CHARCOAL, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
const coverCols = [
|
||||||
|
{
|
||||||
|
title: "Manual CoverPoints",
|
||||||
|
items: ["Placed by level designer", "Type: Cover or HidingSpot", "Quality score (0–1)", "MaxOccupants, Crouch flag", "Type filter (Any / Civ / Enemy / Protector)"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: "Procedural",
|
||||||
|
items: ["Fallback if no manual points", "Raycast sampling around threat", "LOS blockage test", "Advancement bias toward threat"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: "EQS Refinement",
|
||||||
|
items: ["OnCircle generator around cover", "LineOfSight filter", "Distance + quality tests", "Optional firing position refine"]
|
||||||
|
},
|
||||||
|
];
|
||||||
|
coverCols.forEach((c, i) => {
|
||||||
|
const x = 0.5 + i * 4.28;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: x, y: 2.2, w: 4.08, h: 3.1, fill: { color: SAND }, line: { color: TERRA, width: 1 }, rectRadius: 0.1 });
|
||||||
|
s.addText(c.title, { x: x + 0.2, y: 2.3, w: 3.78, h: 0.4, fontSize: 16, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: x + 0.2, y: 2.72, w: 0.4, h: 0.04, fill: { color: GOLD } });
|
||||||
|
s.addText(c.items.map(t => ({ text: t, options: { bullet: true, fontSize: 11 } })),
|
||||||
|
{ x: x + 0.2, y: 2.85, w: 3.78, h: 2.3, color: CHARCOAL, fontFace: FONT_B, paraSpaceAfter: 3 });
|
||||||
|
});
|
||||||
|
|
||||||
|
// Cover cycle state machine
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: 5.55, w: 12.3, h: 1.45, fill: { color: TERRA }, rectRadius: 0.1 });
|
||||||
|
s.addText("Cover Cycle State Machine (CombatSubState)", { x: 0.7, y: 5.65, w: 12, h: 0.35, fontSize: 14, bold: true, color: GOLD, fontFace: FONT_H });
|
||||||
|
|
||||||
|
const subStates = ["Engaging", "AtCover", "MovingToFire", "Peeking", "ReturningToCover", "Advancing"];
|
||||||
|
subStates.forEach((sub, i) => {
|
||||||
|
const x = 0.7 + i * 2.0;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: x, y: 6.15, w: 1.9, h: 0.7, fill: { color: TERRA_D }, rectRadius: 0.05 });
|
||||||
|
s.addText(sub, { x: x, y: 6.15, w: 1.9, h: 0.7, fontSize: 11, bold: true, color: WHITE, align: "center", valign: "middle", fontFace: FONT_H });
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 15 — INTERFACE (IMPLEMENTERS GUIDE)
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("IPS_AI_Behavior_Interface", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 34, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: GOLD } });
|
||||||
|
s.addText("What your Pawn must implement (C++ or Blueprint).", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: CHARCOAL, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
const ifCats = [
|
||||||
|
{
|
||||||
|
title: "Identity",
|
||||||
|
items: ["GetBehaviorNPCType()", "SetBehaviorNPCType()", "IsBehaviorHostile()", "SetBehaviorHostile()"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: "Movement",
|
||||||
|
items: ["SetBehaviorMovementSpeed()", "GetBehaviorMovementSpeed()", "SetBehaviorCrouch()"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: "State Feedback",
|
||||||
|
items: ["OnBehaviorStateChanged()", "IsTargetActorValid()", "GetBehaviorThreatActor()"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: "Combat",
|
||||||
|
items: ["BehaviorStartAttack()", "BehaviorStopAttack()", "CanBehaviorAttack()", "GetBehaviorCombatType()"]
|
||||||
|
},
|
||||||
|
];
|
||||||
|
ifCats.forEach((c, i) => {
|
||||||
|
const col = i % 2;
|
||||||
|
const row = Math.floor(i / 2);
|
||||||
|
const x = 0.5 + col * 6.3;
|
||||||
|
const y = 2.2 + row * 2.4;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: x, y: y, w: 6.0, h: 2.2, fill: { color: TERRA }, rectRadius: 0.1 });
|
||||||
|
s.addText(c.title, { x: x + 0.2, y: y + 0.15, w: 5.6, h: 0.4, fontSize: 16, bold: true, color: GOLD, fontFace: FONT_H });
|
||||||
|
s.addText(c.items.map(t => ({ text: t, options: { bullet: true, fontSize: 12 } })),
|
||||||
|
{ x: x + 0.2, y: y + 0.6, w: 5.6, h: 1.55, color: SAND, fontFace: "Consolas", paraSpaceAfter: 3 });
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 16 — SETUP WORKFLOW
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Setup Workflow", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: GOLD } });
|
||||||
|
s.addText("From zero to a behaving NPC in 6 steps.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: CHARCOAL, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
const bsteps = [
|
||||||
|
{ num: "1", title: "Personality Profile", desc: "Create DataAsset — traits, thresholds, DefaultBehaviorTree." },
|
||||||
|
{ num: "2", title: "Implement Interface", desc: "Pawn: implement IPS_AI_Behavior_Interface (identity, combat, movement)." },
|
||||||
|
{ num: "3", title: "Add Components", desc: "Pawn: add PersonalityComponent. Optional: Combat, SplineFollower." },
|
||||||
|
{ num: "4", title: "Behavior Tree", desc: "Branch on BehaviorState. Add services UpdateThreat + EvaluateReaction at root." },
|
||||||
|
{ num: "5", title: "Set AIController", desc: "Pawn's AIController class = APS_AI_Behavior_AIController." },
|
||||||
|
{ num: "6", title: "Place World Actors", desc: "Optional: SplinePaths, CoverPoints, SplineNetwork (auto)." },
|
||||||
|
];
|
||||||
|
bsteps.forEach((step, i) => {
|
||||||
|
const y = 2.2 + i * 0.78;
|
||||||
|
s.addShape(pres.ShapeType.ellipse, { x: 0.5, y: y, w: 0.7, h: 0.7, fill: { color: GOLD } });
|
||||||
|
s.addText(step.num, { x: 0.5, y: y, w: 0.7, h: 0.7, fontSize: 24, bold: true, color: TERRA, align: "center", valign: "middle", fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 1.4, y: y, w: 11.4, h: 0.7, fill: { color: SAND }, line: { color: TERRA, width: 1 }, rectRadius: 0.05 });
|
||||||
|
s.addText(step.title, { x: 1.6, y: y + 0.05, w: 3.5, h: 0.3, fontSize: 13, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addText(step.desc, { x: 1.6, y: y + 0.35, w: 11.2, h: 0.35, fontSize: 11, color: CHARCOAL, italic: true, fontFace: FONT_B });
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 17 — BLACKBOARD KEYS
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Blackboard Keys", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: GOLD } });
|
||||||
|
s.addText("Auto-created by the AIController — use in your BT decorators and tasks.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: CHARCOAL, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
const bbKeys = [
|
||||||
|
{ name: "BehaviorState", type: "Enum", desc: "Current state (Idle, Combat, etc.)" },
|
||||||
|
{ name: "ThreatActor", type: "Object", desc: "Current highest-scored threat" },
|
||||||
|
{ name: "ThreatLocation", type: "Vector", desc: "Last known threat position" },
|
||||||
|
{ name: "ThreatLevel", type: "Float", desc: "Perceived threat (0–1+)" },
|
||||||
|
{ name: "CoverLocation", type: "Vector", desc: "Target cover position" },
|
||||||
|
{ name: "CoverPoint", type: "Object", desc: "Current CoverPoint actor" },
|
||||||
|
{ name: "PatrolIndex", type: "Int", desc: "Current patrol waypoint index" },
|
||||||
|
{ name: "HomeLocation", type: "Vector", desc: "Spawn / home point" },
|
||||||
|
{ name: "CurrentSpline", type: "Object", desc: "Active SplinePath" },
|
||||||
|
{ name: "SplineProgress", type: "Float", desc: "Distance along current spline" },
|
||||||
|
{ name: "CombatSubState", type: "Enum", desc: "Cover cycle sub-state (Peeking, etc.)" },
|
||||||
|
{ name: "LastKnownTargetPosition", type: "Vector", desc: "LOS-lost target memory" },
|
||||||
|
{ name: "PreferCover", type: "Bool", desc: "Combat/cover cycle toggle" },
|
||||||
|
{ name: "ConversationPaused", type: "Bool", desc: "Active dialogue pause flag" },
|
||||||
|
];
|
||||||
|
bbKeys.forEach((k, i) => {
|
||||||
|
const col = i % 2;
|
||||||
|
const row = Math.floor(i / 2);
|
||||||
|
const x = 0.5 + col * 6.3;
|
||||||
|
const y = 2.2 + row * 0.62;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: x, y: y, w: 1.8, h: 0.5, fill: { color: TERRA }, rectRadius: 0.05 });
|
||||||
|
s.addText(k.name, { x: x, y: y, w: 1.8, h: 0.5, fontSize: 10, bold: true, color: WHITE, align: "center", valign: "middle", fontFace: "Consolas" });
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: x + 1.85, y: y + 0.1, w: 0.6, h: 0.3, fill: { color: GOLD }, rectRadius: 0.05 });
|
||||||
|
s.addText(k.type, { x: x + 1.85, y: y + 0.1, w: 0.6, h: 0.3, fontSize: 9, bold: true, color: WHITE, align: "center", valign: "middle", fontFace: FONT_H });
|
||||||
|
s.addText(k.desc, { x: x + 2.55, y: y, w: 3.7, h: 0.5, fontSize: 10, color: CHARCOAL, valign: "middle", italic: true, fontFace: FONT_B });
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 18 — NETWORKING
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Networking", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: GOLD } });
|
||||||
|
s.addText("Server-authoritative AI, minimal replication for visuals.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: CHARCOAL, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
s.addText("Server-only", { x: 0.5, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "All AI planning & decision-making", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Threat evaluation, state transitions", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Perception stimuli processing", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Path selection, cover choice", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Combat targeting", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 0.5, y: 2.55, w: 6, h: 2.7, color: CHARCOAL, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
s.addText("Replicated to clients", { x: 7, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "CurrentState (OnRep fires delegate)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "PerceivedThreatLevel (HUD/debug)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "CurrentTarget (anims/visuals)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "CurrentSpline, bIsFollowing", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "OnAttackExecuted (multicast)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 7, y: 2.55, w: 6, h: 2.7, color: CHARCOAL, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: 5.4, w: 12.3, h: 1.6, fill: { color: TERRA }, rectRadius: 0.1 });
|
||||||
|
s.addText("Character movement syncs via CharacterMovementComponent. Spline follower updates server-side, position replicates naturally. Clients receive enough state for animation and HUD without any AI logic running locally.", {
|
||||||
|
x: 0.7, y: 5.6, w: 12, h: 1.2, fontSize: 13, color: SAND, italic: true, fontFace: FONT_B, valign: "middle"
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 19 — INTEGRATION WITH CONVAGENT
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Integration with PS_AI_ConvAgent", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 28, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: GOLD } });
|
||||||
|
s.addText("Optional — behavior and conversation plugins talk via reflection.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: CHARCOAL, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
const integrations = [
|
||||||
|
{ title: "Gaze Bridge", desc: "AIController discovers GazeComponent on the Pawn, sets target during Idle/Patrol (proximity gaze) and forwards current threat during Combat." },
|
||||||
|
{ title: "Conversation Pause", desc: "When ConvAgent is conversing, AI planning pauses — NPC stops moving, waits. Local VAD detects user speech." },
|
||||||
|
{ title: "Disguise Reveal", desc: "Enemy NPC with IsBehaviorHostile=false starts as Civilian TeamId. Flips to hostile via interface when triggered by dialogue." },
|
||||||
|
{ title: "Forced State", desc: "ConvAgent LLM can invoke tools that call ForceState(Scripted) or ForceState(Fleeing) for narrative control." },
|
||||||
|
];
|
||||||
|
integrations.forEach((item, i) => {
|
||||||
|
const col = i % 2;
|
||||||
|
const row = Math.floor(i / 2);
|
||||||
|
const x = 0.5 + col * 6.3;
|
||||||
|
const y = 2.2 + row * 2.3;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: x, y: y, w: 6.0, h: 2.1, fill: { color: SAND }, line: { color: TERRA, width: 1 }, rectRadius: 0.1 });
|
||||||
|
s.addText(item.title, { x: x + 0.2, y: y + 0.15, w: 5.6, h: 0.4, fontSize: 16, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: x + 0.2, y: y + 0.58, w: 0.4, h: 0.04, fill: { color: GOLD } });
|
||||||
|
s.addText(item.desc, { x: x + 0.2, y: y + 0.7, w: 5.6, h: 1.3, fontSize: 12, color: CHARCOAL, fontFace: FONT_B });
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 20 — DEBUG TOOLS
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Debug Tools", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: GOLD } });
|
||||||
|
s.addText("Visualize state, perception, and targeting at runtime.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: CHARCOAL, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
s.addText("Per-NPC Debug", { x: 0.5, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "PersonalityComponent.bDebug — floating text", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Name, NPCType, TeamId, State", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "ThreatLevel, ThreatActor", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Hostile, active spline", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "SplineFollower debug sphere (green)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 0.5, y: 2.55, w: 6, h: 2.8, color: CHARCOAL, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
s.addText("Global Logging", { x: 7, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: TERRA, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "CVar: ps.ai.Behavior.Debug 1", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Toggles verbose logs globally", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "LogPS_AI_Behavior category", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "BTView / AIDebug (Unreal built-in)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "EQS Visualizer for cover queries", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 7, y: 2.55, w: 6, h: 2.8, color: CHARCOAL, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: 5.5, w: 12.3, h: 1.5, fill: { color: TERRA }, rectRadius: 0.1 });
|
||||||
|
s.addText("Pro tip", { x: 0.7, y: 5.6, w: 12, h: 0.35, fontSize: 14, bold: true, color: GOLD, fontFace: FONT_H });
|
||||||
|
s.addText("Enable bDebug only on specific NPCs (or via a debug HUD key). Global debug logs can be overwhelming in crowded scenes. The floating text is cheap but accumulates.",
|
||||||
|
{ x: 0.7, y: 5.95, w: 12, h: 1.0, fontSize: 12, color: SAND, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 21 — SUMMARY
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide();
|
||||||
|
s.background = { color: TERRA };
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0, y: 2.5, w: 13.333, h: 0.05, fill: { color: GOLD } });
|
||||||
|
s.addText("Summary", { x: 0.6, y: 0.8, w: 12, h: 1.2, fontSize: 54, bold: true, color: WHITE, fontFace: FONT_H });
|
||||||
|
s.addText("Complete NPC behavior stack for tactical game AI.", { x: 0.6, y: 2.0, w: 12, h: 0.5, fontSize: 18, color: SAND, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
const bsummary = [
|
||||||
|
{ h: "Personality-driven", t: "Three traits modulate every decision" },
|
||||||
|
{ h: "Cover-aware", t: "Manual CoverPoints + procedural fallback + EQS" },
|
||||||
|
{ h: "Spline navigation", t: "Network with junction detection, type filtering" },
|
||||||
|
{ h: "Perception built-in", t: "Sight + hearing + damage, priority-based threat" },
|
||||||
|
{ h: "Server-authoritative", t: "Minimal replication, client just animates" },
|
||||||
|
{ h: "Plugin-friendly", t: "Gaze bridge to ConvAgent, interface-only dependency" },
|
||||||
|
];
|
||||||
|
bsummary.forEach((item, i) => {
|
||||||
|
const col = i % 2;
|
||||||
|
const row = Math.floor(i / 2);
|
||||||
|
const x = 0.6 + col * 6.2;
|
||||||
|
const y = 3.0 + row * 1.4;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: x, y: y, w: 5.9, h: 1.2, fill: { color: TERRA_D }, line: { color: GOLD, width: 1 }, rectRadius: 0.1 });
|
||||||
|
s.addText(item.h, { x: x + 0.2, y: y + 0.1, w: 5.5, h: 0.4, fontSize: 18, bold: true, color: GOLD, fontFace: FONT_H });
|
||||||
|
s.addText(item.t, { x: x + 0.2, y: y + 0.55, w: 5.5, h: 0.6, fontSize: 12, color: SAND, italic: true, fontFace: FONT_B });
|
||||||
|
});
|
||||||
|
|
||||||
|
s.addText("ASTERION • PS_AI_Behavior • 2026", { x: 0.6, y: 6.9, w: 12, h: 0.4, fontSize: 12, color: SAND, fontFace: FONT_B });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SAVE
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
pres.writeFile({ fileName: "PS_AI_Behavior_Documentation.pptx" })
|
||||||
|
.then(name => console.log(`Generated: ${name}`))
|
||||||
|
.catch(err => { console.error(err); process.exit(1); });
|
||||||
711
Docs/gen_convagent_doc.js
Normal file
711
Docs/gen_convagent_doc.js
Normal file
@@ -0,0 +1,711 @@
|
|||||||
|
// PS_AI_ConvAgent User Documentation Generator
|
||||||
|
// Palette: Midnight Executive (navy #1E2761, ice blue #CADCFC, white)
|
||||||
|
|
||||||
|
const pptxgen = require("pptxgenjs");
|
||||||
|
const pres = new pptxgen();
|
||||||
|
|
||||||
|
pres.layout = "LAYOUT_WIDE"; // 13.333 x 7.5 inch
|
||||||
|
|
||||||
|
// ─── Color Palette ─────────────────────────────────
|
||||||
|
const NAVY = "1E2761";
|
||||||
|
const NAVY_DARK = "141B45";
|
||||||
|
const ICE = "CADCFC";
|
||||||
|
const ICE_DEEP = "7892D6";
|
||||||
|
const WHITE = "FFFFFF";
|
||||||
|
const GREY = "3A3F5C";
|
||||||
|
const LIGHT_GREY= "F5F7FC";
|
||||||
|
const ACCENT = "F9A825"; // warm gold accent
|
||||||
|
|
||||||
|
// ─── Font ──────────────────────────────────────────
|
||||||
|
const FONT_H = "Calibri";
|
||||||
|
const FONT_B = "Calibri";
|
||||||
|
|
||||||
|
// ─── Master slide backgrounds ──────────────────────
|
||||||
|
pres.defineSlideMaster({
|
||||||
|
title: "CONTENT",
|
||||||
|
background: { color: LIGHT_GREY },
|
||||||
|
objects: [
|
||||||
|
{ rect: { x: 0, y: 0, w: 13.333, h: 0.35, fill: { color: NAVY } } },
|
||||||
|
{ rect: { x: 0, y: 7.15, w: 13.333, h: 0.35, fill: { color: NAVY } } },
|
||||||
|
{ text: {
|
||||||
|
text: "PS_AI_ConvAgent — User Documentation",
|
||||||
|
options: { x: 0.3, y: 7.18, w: 8, h: 0.3, fontSize: 10, color: ICE, fontFace: FONT_B }
|
||||||
|
} },
|
||||||
|
{ text: {
|
||||||
|
text: "ASTERION",
|
||||||
|
options: { x: 11.5, y: 7.18, w: 1.5, h: 0.3, fontSize: 10, color: ICE, fontFace: FONT_B, align: "right", bold: true }
|
||||||
|
} },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 1 — TITLE
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
let s = pres.addSlide();
|
||||||
|
s.background = { color: NAVY };
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0, y: 3.3, w: 13.333, h: 0.05, fill: { color: ACCENT } });
|
||||||
|
s.addText("PS_AI_ConvAgent", { x: 0.6, y: 2.2, w: 12, h: 1.2, fontSize: 60, bold: true, color: WHITE, fontFace: FONT_H });
|
||||||
|
s.addText("Conversational AI for MetaHumans in Unreal Engine", { x: 0.6, y: 3.5, w: 12, h: 0.6, fontSize: 22, color: ICE, fontFace: FONT_B, italic: true });
|
||||||
|
s.addText("User Documentation", { x: 0.6, y: 4.4, w: 12, h: 0.5, fontSize: 18, color: ICE_DEEP, fontFace: FONT_B });
|
||||||
|
s.addText("ElevenLabs • MetaHuman • Listen Server", { x: 0.6, y: 5.2, w: 12, h: 0.4, fontSize: 14, color: ICE_DEEP, fontFace: FONT_B });
|
||||||
|
s.addText("ASTERION • 2026", { x: 0.6, y: 6.6, w: 12, h: 0.4, fontSize: 12, color: ICE_DEEP, fontFace: FONT_B });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 2 — OVERVIEW
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Plugin Overview", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: ACCENT } });
|
||||||
|
|
||||||
|
s.addText("What it does", { x: 0.5, y: 1.7, w: 6, h: 0.4, fontSize: 18, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "Full conversational AI NPC integration for Unreal Engine", options: { bullet: true, fontSize: 14 } },
|
||||||
|
{ text: "Real-time voice capture, turn-taking, TTS playback", options: { bullet: true, fontSize: 14 } },
|
||||||
|
{ text: "Emotion-driven MetaHuman animations (face, body, gaze)", options: { bullet: true, fontSize: 14 } },
|
||||||
|
{ text: "Multiplayer support with speaker arbitration", options: { bullet: true, fontSize: 14 } },
|
||||||
|
{ text: "Tool system for LLM-triggered game actions", options: { bullet: true, fontSize: 14 } },
|
||||||
|
], { x: 0.5, y: 2.1, w: 6, h: 2.6, color: GREY, fontFace: FONT_B, paraSpaceAfter: 8 });
|
||||||
|
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 7, y: 1.7, w: 5.8, h: 4.8, fill: { color: NAVY }, rectRadius: 0.1 });
|
||||||
|
s.addText("Primary Use Case", { x: 7.2, y: 1.9, w: 5.4, h: 0.4, fontSize: 16, bold: true, color: ACCENT, fontFace: FONT_H });
|
||||||
|
s.addText("Interactive, emotionally expressive voice NPCs that listen to players, respond with contextual answers, express emotion through facial animation and body language, and trigger physical actions based on conversation flow.", {
|
||||||
|
x: 7.2, y: 2.4, w: 5.4, h: 1.7, fontSize: 13, color: ICE, fontFace: FONT_B, italic: true
|
||||||
|
});
|
||||||
|
s.addText("Key Differentiators", { x: 7.2, y: 4.2, w: 5.4, h: 0.4, fontSize: 16, bold: true, color: ACCENT, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "Full-duplex Server VAD or push-to-talk", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "Adaptive pre-buffering & speculative TTS", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "Spectral lip sync + artist-driven poses", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "Opus codec + distance LOD for multiplayer", options: { bullet: true, fontSize: 12 } },
|
||||||
|
], { x: 7.2, y: 4.7, w: 5.4, h: 1.7, color: ICE, fontFace: FONT_B, paraSpaceAfter: 4 });
|
||||||
|
|
||||||
|
s.addText("\"Drop components on a MetaHuman, configure a few data assets, and talk.\"", {
|
||||||
|
x: 0.5, y: 5.3, w: 6, h: 0.6, fontSize: 13, color: NAVY, italic: true, fontFace: FONT_B, bold: true
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 3 — ARCHITECTURE DIAGRAM
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Architecture", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: ACCENT } });
|
||||||
|
|
||||||
|
s.addText("Component chain on a MetaHuman NPC", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
// Input row
|
||||||
|
const boxY = 2.4;
|
||||||
|
const boxH = 1.0;
|
||||||
|
const boxes = [
|
||||||
|
{ x: 0.5, label: "Microphone\nCapture", sub: "VAD • 16kHz" },
|
||||||
|
{ x: 2.6, label: "Interaction\nComponent", sub: "Agent select\n(player side)" },
|
||||||
|
{ x: 4.7, label: "ElevenLabs\nComponent", sub: "WebSocket\nTurn-taking" },
|
||||||
|
{ x: 6.8, label: "Lip Sync\nComponent", sub: "Spectral\nFFT → visemes" },
|
||||||
|
{ x: 8.9, label: "Facial Expr.\nComponent", sub: "Emotion\ncurves" },
|
||||||
|
{ x: 11.0, label: "Body Expr.\nComponent", sub: "Gesture\nanim blend" },
|
||||||
|
];
|
||||||
|
boxes.forEach((b, i) => {
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: b.x, y: boxY, w: 2.0, h: boxH, fill: { color: NAVY }, rectRadius: 0.1 });
|
||||||
|
s.addText(b.label, { x: b.x, y: boxY + 0.1, w: 2.0, h: 0.5, fontSize: 13, bold: true, color: WHITE, align: "center", fontFace: FONT_H });
|
||||||
|
s.addText(b.sub, { x: b.x, y: boxY + 0.58, w: 2.0, h: 0.4, fontSize: 9, color: ICE, align: "center", fontFace: FONT_B });
|
||||||
|
});
|
||||||
|
|
||||||
|
// Arrows between boxes
|
||||||
|
for (let i = 0; i < boxes.length - 1; i++) {
|
||||||
|
const ax = boxes[i].x + 2.0;
|
||||||
|
s.addShape(pres.ShapeType.line, { x: ax, y: boxY + 0.5, w: 0.1, h: 0, line: { color: ACCENT, width: 2 } });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Gaze row
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: 4.0, w: 2.0, h: boxH, fill: { color: ICE_DEEP }, rectRadius: 0.1 });
|
||||||
|
s.addText("Gaze\nComponent", { x: 0.5, y: 4.1, w: 2.0, h: 0.5, fontSize: 13, bold: true, color: WHITE, align: "center", fontFace: FONT_H });
|
||||||
|
s.addText("Eyes • Head\n• Body", { x: 0.5, y: 4.58, w: 2.0, h: 0.4, fontSize: 9, color: WHITE, align: "center", fontFace: FONT_B });
|
||||||
|
|
||||||
|
// AnimGraph pipeline
|
||||||
|
s.addText("→ AnimGraph (Face + Body)", { x: 2.8, y: 4.2, w: 4.5, h: 0.5, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText("FacialExpression → LipSync | Gaze → BodyExpression", { x: 2.8, y: 4.65, w: 8, h: 0.4, fontSize: 12, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
// Project settings layer
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: 5.6, w: 12.3, h: 0.9, fill: { color: LIGHT_GREY }, line: { color: NAVY, width: 1 }, rectRadius: 0.08 });
|
||||||
|
s.addText("PROJECT SETTINGS — API Key • Server Region • Global Context Prompt", { x: 0.7, y: 5.75, w: 12, h: 0.3, fontSize: 13, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText("Data Assets: AgentConfig • EmotionPoseMap • LipSyncPoseMap • BodyPoseMap • ActionSet • Tool", { x: 0.7, y: 6.1, w: 12, h: 0.3, fontSize: 11, color: GREY, fontFace: FONT_B });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 4 — ELEVENLABS COMPONENT
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("ElevenLabs Component", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: ACCENT } });
|
||||||
|
s.addText("The core conversation engine — WebSocket, turn-taking, audio pipeline.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
s.addText("Key Properties", { x: 0.5, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "AgentConfig — identity, voice, tools", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "TurnMode — ServerVAD / Client (push-to-talk)", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "MicChunkDurationMs (50–250 ms)", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "AudioPreBufferMs / bAdaptivePreBuffer", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "bAllowInterruption, bPersistentSession", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "SoundAttenuation (optional 3D audio)", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "AudioLODCullDistance / LipSyncLODDistance", options: { bullet: true, fontSize: 12 } },
|
||||||
|
], { x: 0.5, y: 2.55, w: 6, h: 3.6, color: GREY, fontFace: FONT_B, paraSpaceAfter: 4 });
|
||||||
|
|
||||||
|
s.addText("Key Methods", { x: 7, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "StartConversation() / EndConversation()", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "StartListening() / StopListening()", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "SendTextMessage() — text without voice", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "InterruptAgent() — force stop mid-speech", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "ForceDisableConversation(Action) — blend to neutral", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "ForceEnableConversation() — re-enable", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "FeedExternalAudio() — external mic source", options: { bullet: true, fontSize: 12 } },
|
||||||
|
], { x: 7, y: 2.55, w: 6, h: 3.6, color: GREY, fontFace: FONT_B, paraSpaceAfter: 4 });
|
||||||
|
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: 6.2, w: 12.3, h: 0.75, fill: { color: NAVY }, rectRadius: 0.08 });
|
||||||
|
s.addText("Major events: OnAgentConnected • OnAgentTranscript • OnAgentTextResponse • OnAgentStartedSpeaking • OnAgentEmotionChanged • OnAgentActionRequested • OnActiveSpeakerChanged • OnReadyForAction", { x: 0.7, y: 6.3, w: 12, h: 0.55, fontSize: 11, color: ICE, fontFace: FONT_B, italic: true });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 5 — MICROPHONE CAPTURE
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Microphone Capture Component", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 34, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: ACCENT } });
|
||||||
|
s.addText("Captures, resamples, and performs local voice activity detection.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
s.addText("Audio Pipeline", { x: 0.5, y: 2.2, w: 6, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "WASAPI 48kHz stereo float32 input", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Resample to 16kHz mono float", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Convert to int16 LE byte stream", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Delivered via multicast delegate", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 0.5, y: 2.6, w: 6, h: 2.4, color: GREY, fontFace: FONT_B, paraSpaceAfter: 6 });
|
||||||
|
|
||||||
|
s.addText("Voice Activity Detection", { x: 7, y: 2.2, w: 6, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "VolumeMultiplier — mic gain", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "VoiceActivityThreshold — RMS threshold", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "VoiceOnsetTime — debounce on speech start", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "VoiceSilenceTime — debounce on end", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "OnUserVoiceActivityChanged event", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 7, y: 2.6, w: 6, h: 2.4, color: GREY, fontFace: FONT_B, paraSpaceAfter: 6 });
|
||||||
|
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: 5.4, w: 12.3, h: 1.4, fill: { color: LIGHT_GREY }, line: { color: NAVY, width: 1 }, rectRadius: 0.08 });
|
||||||
|
s.addText("Public API", { x: 0.7, y: 5.5, w: 12, h: 0.3, fontSize: 13, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText("StartCapture() • StopCapture() • IsCapturing()", { x: 0.7, y: 5.85, w: 12, h: 0.3, fontSize: 13, color: GREY, fontFace: "Consolas" });
|
||||||
|
s.addText("The component auto-opens the default input device. Output is consumed by ElevenLabsComponent (or InteractionComponent, which relays via Server RPC in networked games).", { x: 0.7, y: 6.2, w: 12, h: 0.55, fontSize: 11, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 6 — LIP SYNC
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Lip Sync Component", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: ACCENT } });
|
||||||
|
s.addText("Spectral analysis of agent audio → 15 OVR visemes → ARKit blendshapes.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
s.addText("How it works", { x: 0.5, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "FFT on agent audio stream", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Estimates 15 OVR viseme weights", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Maps to ARKit blendshapes (jawOpen, mouthFunnel…)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Optional: custom PoseMap with artist-crafted poses", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Thread-safe curve output for AnimGraph", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 0.5, y: 2.55, w: 6, h: 3.5, color: GREY, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
s.addText("Key Properties", { x: 7, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "LipSyncStrength — overall intensity", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "SmoothingSpeed (35–65) — viseme interp", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "SpeechBlendDuration — fade in/out", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "EmotionExpressionBlend (0–1) — emotion bleed through mouth", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "EnvelopeAttackMs / EnvelopeReleaseMs", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "PoseMap — optional artist override", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 7, y: 2.55, w: 6, h: 3.5, color: GREY, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: 6.2, w: 12.3, h: 0.75, fill: { color: NAVY }, rectRadius: 0.08 });
|
||||||
|
s.addText("AnimGraph placement (Face AnimBP): Live Link → FacialExpression → LipSync → mh_arkit_mapping_pose", { x: 0.7, y: 6.3, w: 12, h: 0.55, fontSize: 12, color: ICE, fontFace: FONT_B, italic: true, bold: true });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 7 — FACIAL EXPRESSION
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Facial Expression Component", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 34, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: ACCENT } });
|
||||||
|
s.addText("Emotion-driven face animation via real-time AnimSequence sampling.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
s.addText("Design", { x: 0.5, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "Plays AnimSequences in real-time (looping)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Extracts emotion curves (eyes, brows, cheeks, mouth mood)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Injects into AnimGraph via FacialExpression AnimNode", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Lip sync overrides mouth area during speech", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "3 intensity levels: Normal / Medium / Extreme", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Crossfades between emotions", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 0.5, y: 2.55, w: 6, h: 3.5, color: GREY, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
s.addText("Properties", { x: 7, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "EmotionPoseMap — data asset of AnimSequences", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "ActivationBlendDuration (default 0.5s)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "EmotionBlendDuration — crossfade time", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "bActive — auto-managed by conversation", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 7, y: 2.55, w: 6, h: 2.4, color: GREY, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 7, y: 5.0, w: 5.8, h: 1.5, fill: { color: LIGHT_GREY }, line: { color: NAVY, width: 1 }, rectRadius: 0.08 });
|
||||||
|
s.addText("Emotions covered", { x: 7.15, y: 5.1, w: 5.6, h: 0.3, fontSize: 13, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText("Joy • Sadness • Anger • Fear • Surprise • Disgust • Neutral", { x: 7.15, y: 5.45, w: 5.6, h: 0.9, fontSize: 12, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 8 — BODY EXPRESSION
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Body Expression Component", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 34, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: ACCENT } });
|
||||||
|
s.addText("Upper-body gestures and posture driven by emotion + speaking state.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
s.addText("Animation lists per emotion", { x: 0.5, y: 2.15, w: 6.3, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
|
||||||
|
const animLists = [
|
||||||
|
{ name: "Idle", desc: "Played while listening (not speaking)" },
|
||||||
|
{ name: "Normal", desc: "Low-intensity speaking anims" },
|
||||||
|
{ name: "Medium", desc: "Medium-intensity speaking" },
|
||||||
|
{ name: "Extreme", desc: "High-intensity speaking" },
|
||||||
|
];
|
||||||
|
animLists.forEach((a, i) => {
|
||||||
|
const y = 2.7 + i * 0.75;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: y, w: 1.5, h: 0.55, fill: { color: NAVY }, rectRadius: 0.05 });
|
||||||
|
s.addText(a.name, { x: 0.5, y: y, w: 1.5, h: 0.55, fontSize: 13, bold: true, color: WHITE, align: "center", valign: "middle", fontFace: FONT_H });
|
||||||
|
s.addText(a.desc, { x: 2.15, y: y + 0.05, w: 4.6, h: 0.45, fontSize: 12, color: GREY, valign: "middle", fontFace: FONT_B });
|
||||||
|
});
|
||||||
|
|
||||||
|
s.addText("Key Properties", { x: 7.2, y: 2.15, w: 5.8, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "BodyPoseMap — emotion → anim lists", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "ActivationBlendDuration (0.5s)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "EmotionBlendDuration (1.0s)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "BlendWeight (0–1)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 7.2, y: 2.55, w: 5.8, h: 2, color: GREY, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
s.addText("AnimNode behaviour", { x: 7.2, y: 4.7, w: 5.8, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "Override mode, per-bone mask", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "Default BlendRootBone = spine_02", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "Excludes neck_01 (Gaze handles it)", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "Lower body passes through upstream", options: { bullet: true, fontSize: 12 } },
|
||||||
|
], { x: 7.2, y: 5.1, w: 5.8, h: 2, color: GREY, fontFace: FONT_B, paraSpaceAfter: 3 });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 9 — GAZE
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Gaze Component", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: ACCENT } });
|
||||||
|
s.addText("Multi-layer look-at: body • head • eyes.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
const layers = [
|
||||||
|
{ name: "BODY", ratio: "full 360°", desc: "Actor yaw — follows target around the character" },
|
||||||
|
{ name: "HEAD", ratio: "limited", desc: "Head bone yaw + pitch, clamped" },
|
||||||
|
{ name: "EYES", ratio: "limited", desc: "ARKit eye curves, fine targeting" },
|
||||||
|
];
|
||||||
|
layers.forEach((l, i) => {
|
||||||
|
const y = 2.3 + i * 1.1;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: y, w: 2.2, h: 0.9, fill: { color: NAVY }, rectRadius: 0.08 });
|
||||||
|
s.addText(l.name, { x: 0.5, y: y + 0.05, w: 2.2, h: 0.4, fontSize: 18, bold: true, color: WHITE, align: "center", fontFace: FONT_H });
|
||||||
|
s.addText(l.ratio, { x: 0.5, y: y + 0.48, w: 2.2, h: 0.3, fontSize: 11, color: ACCENT, align: "center", italic: true, fontFace: FONT_B });
|
||||||
|
s.addText(l.desc, { x: 3.0, y: y + 0.15, w: 5.0, h: 0.7, fontSize: 13, color: GREY, valign: "middle", fontFace: FONT_B });
|
||||||
|
});
|
||||||
|
|
||||||
|
s.addText("Key Properties", { x: 8.5, y: 2.15, w: 4.5, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "TargetActor — who to look at", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "MaxHeadYaw / MaxHeadPitch", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "MaxEyeHorizontal / MaxEyeVertical", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "BodyInterpSpeed / HeadInterpSpeed / EyeInterpSpeed", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "HeadAnimationCompensation (0.9)", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "EyeAnimationCompensation (0.6)", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "BodyDriftCompensation (0.8)", options: { bullet: true, fontSize: 12 } },
|
||||||
|
{ text: "NeckBoneChain — multi-bone distribution", options: { bullet: true, fontSize: 12 } },
|
||||||
|
], { x: 8.5, y: 2.55, w: 4.5, h: 4, color: GREY, fontFace: FONT_B, paraSpaceAfter: 3 });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 10 — INTERACTION COMPONENT
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Interaction Component (Player)", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 32, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: ACCENT } });
|
||||||
|
s.addText("Placed on the player pawn — selects NPCs, routes mic, manages gaze.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
s.addText("Selection Logic", { x: 0.5, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "Proximity + view cone detection", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "MaxInteractionDistance (300 cm default)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "ViewConeHalfAngle (45° default)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "SelectionStickyAngle — hysteresis", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "ConversationSwitchDelay — anti-flicker", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 0.5, y: 2.55, w: 6, h: 3, color: GREY, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
s.addText("Auto-management", { x: 7, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "bAutoStartConversation — open WS on select", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "bAutoManageListening — mic open/close", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "bAutoManageGaze — set agent gaze to player", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "bAutoPassiveGaze — out-of-range passive tracking", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 7, y: 2.55, w: 6, h: 2.5, color: GREY, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: 5.8, w: 12.3, h: 1.2, fill: { color: NAVY }, rectRadius: 0.08 });
|
||||||
|
s.addText("Events: OnAgentSelected • OnAgentDeselected • OnNoAgentInRange", { x: 0.7, y: 5.95, w: 12, h: 0.3, fontSize: 13, bold: true, color: ACCENT, fontFace: FONT_H });
|
||||||
|
s.addText("Methods: GetSelectedAgent() • ForceSelectAgent() • StartConversationWithSelectedAgent() • SendTextToSelectedAgent() • ClearSelection()", { x: 0.7, y: 6.3, w: 12, h: 0.55, fontSize: 11, color: ICE, fontFace: FONT_B });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 11 — DATA ASSETS OVERVIEW
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Data Assets", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: ACCENT } });
|
||||||
|
s.addText("Reusable configuration assets for agent identity, animations, and tools.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
const dataAssets = [
|
||||||
|
{ name: "AgentConfig", desc: "Agent identity, voice, LLM model, tools", color: NAVY },
|
||||||
|
{ name: "EmotionPoseMap", desc: "Emotion → facial AnimSequences (3 levels)", color: NAVY },
|
||||||
|
{ name: "LipSyncPoseMap", desc: "15 OVR visemes → mouth poses", color: NAVY },
|
||||||
|
{ name: "BodyPoseMap", desc: "Emotion → body gesture anim lists", color: NAVY },
|
||||||
|
{ name: "ActionSet", desc: "Actions the agent can trigger", color: ICE_DEEP },
|
||||||
|
{ name: "Tool", desc: "Custom LLM tools with parameters", color: ICE_DEEP },
|
||||||
|
];
|
||||||
|
dataAssets.forEach((d, i) => {
|
||||||
|
const col = i % 2;
|
||||||
|
const row = Math.floor(i / 2);
|
||||||
|
const x = 0.5 + col * 6.3;
|
||||||
|
const y = 2.2 + row * 1.5;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: x, y: y, w: 6.0, h: 1.3, fill: { color: d.color }, rectRadius: 0.1 });
|
||||||
|
s.addText(d.name, { x: x + 0.2, y: y + 0.2, w: 5.6, h: 0.4, fontSize: 18, bold: true, color: ACCENT, fontFace: FONT_H });
|
||||||
|
s.addText(d.desc, { x: x + 0.2, y: y + 0.65, w: 5.6, h: 0.6, fontSize: 12, color: ICE, italic: true, fontFace: FONT_B });
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 12 — AGENT CONFIG DETAILS
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("AgentConfig (ElevenLabs)", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 34, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: ACCENT } });
|
||||||
|
s.addText("Complete agent definition — editable in Content Browser with API buttons.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
const configCats = [
|
||||||
|
{ title: "Identity", items: ["AgentID", "AgentName"] },
|
||||||
|
{ title: "Voice", items: ["VoiceID / VoiceName", "TTSModelID", "Stability • Similarity • Speed"] },
|
||||||
|
{ title: "LLM", items: ["LLMModel (gemini-2.5-flash default)", "Language (en default)"] },
|
||||||
|
{ title: "Behavior", items: ["CharacterPrompt (personality)", "FirstMessage (greeting)", "TurnTimeout, MaxTurns"] },
|
||||||
|
{ title: "Latency", items: ["TurnEagerness (Eager/Normal/Patient)", "bSpeculativeTurn"] },
|
||||||
|
{ title: "Tools", items: ["Tools[] array of Tool assets", "ActionSet integration"] },
|
||||||
|
];
|
||||||
|
|
||||||
|
configCats.forEach((c, i) => {
|
||||||
|
const col = i % 3;
|
||||||
|
const row = Math.floor(i / 3);
|
||||||
|
const x = 0.5 + col * 4.15;
|
||||||
|
const y = 2.2 + row * 2.15;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: x, y: y, w: 3.95, h: 1.95, fill: { color: LIGHT_GREY }, line: { color: NAVY, width: 1 }, rectRadius: 0.08 });
|
||||||
|
s.addText(c.title, { x: x + 0.15, y: y + 0.1, w: 3.65, h: 0.35, fontSize: 14, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: x + 0.15, y: y + 0.45, w: 0.35, h: 0.04, fill: { color: ACCENT } });
|
||||||
|
s.addText(c.items.map(t => ({ text: t, options: { bullet: true, fontSize: 11 } })),
|
||||||
|
{ x: x + 0.15, y: y + 0.55, w: 3.65, h: 1.3, color: GREY, fontFace: FONT_B, paraSpaceAfter: 2 });
|
||||||
|
});
|
||||||
|
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: 6.55, w: 12.3, h: 0.45, fill: { color: NAVY }, rectRadius: 0.05 });
|
||||||
|
s.addText("Editor: Voice picker • Model picker • LLM picker • Create / Update / Fetch Agent buttons", { x: 0.7, y: 6.6, w: 12, h: 0.35, fontSize: 11, color: ICE, fontFace: FONT_B, italic: true, bold: true });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 13 — POSE MAPS
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Pose Maps", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: ACCENT } });
|
||||||
|
s.addText("Three pose-map data assets map abstract states → AnimSequences.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
// Emotion
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: 2.15, w: 4.0, h: 4.5, fill: { color: NAVY }, rectRadius: 0.1 });
|
||||||
|
s.addText("EmotionPoseMap", { x: 0.6, y: 2.3, w: 3.8, h: 0.4, fontSize: 17, bold: true, color: ACCENT, fontFace: FONT_H });
|
||||||
|
s.addText("Emotion → facial AnimSequences\n(3 intensity levels per emotion)", { x: 0.6, y: 2.7, w: 3.8, h: 0.8, fontSize: 11, color: ICE, italic: true, fontFace: FONT_B });
|
||||||
|
s.addText([
|
||||||
|
{ text: "Joy (N / M / E)", options: { bullet: true, fontSize: 11 } },
|
||||||
|
{ text: "Sadness (N / M / E)", options: { bullet: true, fontSize: 11 } },
|
||||||
|
{ text: "Anger (N / M / E)", options: { bullet: true, fontSize: 11 } },
|
||||||
|
{ text: "Fear (N / M / E)", options: { bullet: true, fontSize: 11 } },
|
||||||
|
{ text: "Surprise (N / M / E)", options: { bullet: true, fontSize: 11 } },
|
||||||
|
{ text: "Disgust (N / M / E)", options: { bullet: true, fontSize: 11 } },
|
||||||
|
{ text: "Neutral (N / M / E)", options: { bullet: true, fontSize: 11 } },
|
||||||
|
], { x: 0.7, y: 3.7, w: 3.7, h: 2.8, color: ICE, fontFace: FONT_B, paraSpaceAfter: 3 });
|
||||||
|
|
||||||
|
// LipSync
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 4.7, y: 2.15, w: 4.0, h: 4.5, fill: { color: NAVY }, rectRadius: 0.1 });
|
||||||
|
s.addText("LipSyncPoseMap", { x: 4.8, y: 2.3, w: 3.8, h: 0.4, fontSize: 17, bold: true, color: ACCENT, fontFace: FONT_H });
|
||||||
|
s.addText("15 OVR visemes →\nphoneme AnimSequences", { x: 4.8, y: 2.7, w: 3.8, h: 0.7, fontSize: 11, color: ICE, italic: true, fontFace: FONT_B });
|
||||||
|
s.addText([
|
||||||
|
{ text: "sil • PP • FF • TH • DD", options: { bullet: true, fontSize: 11 } },
|
||||||
|
{ text: "kk • CH • SS • nn • RR", options: { bullet: true, fontSize: 11 } },
|
||||||
|
{ text: "aa • E • ih • oh • ou", options: { bullet: true, fontSize: 11 } },
|
||||||
|
{ text: "MetaHuman MHF_* recommended", options: { bullet: true, fontSize: 11 } },
|
||||||
|
{ text: "Shareable across characters", options: { bullet: true, fontSize: 11 } },
|
||||||
|
], { x: 4.9, y: 3.5, w: 3.7, h: 2.9, color: ICE, fontFace: FONT_B, paraSpaceAfter: 4 });
|
||||||
|
|
||||||
|
// Body
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 8.9, y: 2.15, w: 4.0, h: 4.5, fill: { color: NAVY }, rectRadius: 0.1 });
|
||||||
|
s.addText("BodyPoseMap", { x: 9.0, y: 2.3, w: 3.8, h: 0.4, fontSize: 17, bold: true, color: ACCENT, fontFace: FONT_H });
|
||||||
|
s.addText("Emotion → body anim lists\n(Idle / N / M / E per emotion)", { x: 9.0, y: 2.7, w: 3.8, h: 0.8, fontSize: 11, color: ICE, italic: true, fontFace: FONT_B });
|
||||||
|
s.addText([
|
||||||
|
{ text: "Idle: listening animations", options: { bullet: true, fontSize: 11 } },
|
||||||
|
{ text: "Normal: light speaking", options: { bullet: true, fontSize: 11 } },
|
||||||
|
{ text: "Medium: expressive speaking", options: { bullet: true, fontSize: 11 } },
|
||||||
|
{ text: "Extreme: emphatic speaking", options: { bullet: true, fontSize: 11 } },
|
||||||
|
{ text: "Random selection per loop", options: { bullet: true, fontSize: 11 } },
|
||||||
|
{ text: "Continuous variety", options: { bullet: true, fontSize: 11 } },
|
||||||
|
], { x: 9.0, y: 3.7, w: 3.8, h: 2.8, color: ICE, fontFace: FONT_B, paraSpaceAfter: 3 });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 14 — TOOLS & ACTIONSETS
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Tools & ActionSets", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: ACCENT } });
|
||||||
|
s.addText("LLM-triggered actions that bridge conversation → gameplay.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
s.addText("Tool asset", { x: 0.5, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "ToolName — snake_case identifier", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "ToolDescription — when LLM should use it", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "PromptFragment — appended system prompt", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Parameters[] — typed LLM arguments", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "ToolID auto-populated on Create", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 0.5, y: 2.55, w: 6, h: 2.8, color: GREY, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
s.addText("Built-in Tools", { x: 7, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "set_emotion(emotion, intensity)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "perform_action(action) — from ActionSet", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Custom tools — any parameters", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 7, y: 2.55, w: 6, h: 2, color: GREY, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: 5.4, w: 12.3, h: 1.6, fill: { color: LIGHT_GREY }, line: { color: NAVY, width: 1 }, rectRadius: 0.08 });
|
||||||
|
s.addText("Action flow", { x: 0.7, y: 5.5, w: 12, h: 0.3, fontSize: 14, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText("1. LLM decides to call a tool (e.g. perform_action(\"flee\"))\n2. Plugin fires OnAgentActionRequested(\"flee\")\n3. Game calls ForceDisableConversation(\"flee\") → animations blend to neutral\n4. OnReadyForAction(\"flee\") fires → play gameplay montage\n5. Game calls ForceEnableConversation() when done", { x: 0.7, y: 5.85, w: 12, h: 1.1, fontSize: 11, color: GREY, fontFace: "Consolas" });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 15 — ANIMGRAPH INTEGRATION
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("AnimGraph Integration", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 34, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: ACCENT } });
|
||||||
|
s.addText("Four AnimNodes auto-discover components — no manual wiring.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
// Face AnimBP chain
|
||||||
|
s.addText("Face AnimBP", { x: 0.5, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
const faceChain = [
|
||||||
|
{ label: "Live Link", color: ICE_DEEP },
|
||||||
|
{ label: "Facial\nExpression", color: "E88B3E" },
|
||||||
|
{ label: "Gaze\n(eyes only)", color: "5E7AC7" },
|
||||||
|
{ label: "Lip Sync", color: "3EB086" },
|
||||||
|
{ label: "mh_arkit\n_mapping", color: GREY },
|
||||||
|
];
|
||||||
|
faceChain.forEach((n, i) => {
|
||||||
|
const x = 0.5 + i * 2.55;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: x, y: 2.65, w: 2.3, h: 1.0, fill: { color: n.color }, rectRadius: 0.1 });
|
||||||
|
s.addText(n.label, { x: x, y: 2.75, w: 2.3, h: 0.8, fontSize: 12, bold: true, color: WHITE, align: "center", valign: "middle", fontFace: FONT_H });
|
||||||
|
});
|
||||||
|
|
||||||
|
// Body AnimBP chain
|
||||||
|
s.addText("Body AnimBP", { x: 0.5, y: 4.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
const bodyChain = [
|
||||||
|
{ label: "Locomotion\n/ Idle", color: ICE_DEEP },
|
||||||
|
{ label: "Gaze\n(head rot)", color: "5E7AC7" },
|
||||||
|
{ label: "Body\nExpression", color: "B85042" },
|
||||||
|
{ label: "Output Pose", color: GREY },
|
||||||
|
];
|
||||||
|
bodyChain.forEach((n, i) => {
|
||||||
|
const x = 0.5 + i * 3.2;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: x, y: 4.65, w: 2.9, h: 1.0, fill: { color: n.color }, rectRadius: 0.1 });
|
||||||
|
s.addText(n.label, { x: x, y: 4.75, w: 2.9, h: 0.8, fontSize: 12, bold: true, color: WHITE, align: "center", valign: "middle", fontFace: FONT_H });
|
||||||
|
});
|
||||||
|
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: 6.15, w: 12.3, h: 0.85, fill: { color: NAVY }, rectRadius: 0.08 });
|
||||||
|
s.addText("Why this order?", { x: 0.7, y: 6.2, w: 12, h: 0.3, fontSize: 13, bold: true, color: ACCENT, fontFace: FONT_H });
|
||||||
|
s.addText("Emotion curves set mood (brows, eyes, cheeks). Lip sync overrides mouth during speech. Gaze head rotation runs before body expression so it doesn't fight arms.", { x: 0.7, y: 6.55, w: 12, h: 0.4, fontSize: 11, color: ICE, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 16 — PROJECT SETTINGS
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Project Settings", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: ACCENT } });
|
||||||
|
s.addText("Project Settings → Plugins → PS AI ConvAgent - ElevenLabs", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
const settings = [
|
||||||
|
{ name: "API_Key", desc: "ElevenLabs API key — required for authentication" },
|
||||||
|
{ name: "ServerRegion", desc: "Global / US / EU / India" },
|
||||||
|
{ name: "CustomWebSocketURL", desc: "Override WebSocket URL (advanced)" },
|
||||||
|
{ name: "bVerboseLogging", desc: "Verbose WebSocket logging (development)" },
|
||||||
|
{ name: "GlobalContextPrompt", desc: "Shared context injected as dynamic variables for all agents" },
|
||||||
|
];
|
||||||
|
|
||||||
|
settings.forEach((setting, i) => {
|
||||||
|
const y = 2.3 + i * 0.85;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: y, w: 3.5, h: 0.65, fill: { color: NAVY }, rectRadius: 0.05 });
|
||||||
|
s.addText(setting.name, { x: 0.5, y: y, w: 3.5, h: 0.65, fontSize: 13, bold: true, color: ACCENT, align: "center", valign: "middle", fontFace: "Consolas" });
|
||||||
|
s.addText(setting.desc, { x: 4.2, y: y + 0.05, w: 8.6, h: 0.55, fontSize: 12, color: GREY, valign: "middle", fontFace: FONT_B });
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 17 — SETUP WORKFLOW
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Setup Workflow", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: ACCENT } });
|
||||||
|
s.addText("From zero to a talking MetaHuman in 5 steps.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
const steps = [
|
||||||
|
{ num: "1", title: "Project Settings", desc: "Enter API_Key, pick ServerRegion." },
|
||||||
|
{ num: "2", title: "Data Assets", desc: "Create AgentConfig, EmotionPoseMap, LipSyncPoseMap, BodyPoseMap." },
|
||||||
|
{ num: "3", title: "NPC Components", desc: "Add ElevenLabs + Mic + LipSync + FacialExpr + BodyExpr + Gaze on the MetaHuman." },
|
||||||
|
{ num: "4", title: "AnimGraphs", desc: "Wire AnimNodes in Face + Body AnimBPs. Components auto-discover." },
|
||||||
|
{ num: "5", title: "Player Pawn", desc: "Add InteractionComponent. Bind selection events to UI. Play." },
|
||||||
|
];
|
||||||
|
steps.forEach((step, i) => {
|
||||||
|
const y = 2.25 + i * 0.9;
|
||||||
|
s.addShape(pres.ShapeType.ellipse, { x: 0.5, y: y, w: 0.8, h: 0.8, fill: { color: ACCENT } });
|
||||||
|
s.addText(step.num, { x: 0.5, y: y, w: 0.8, h: 0.8, fontSize: 28, bold: true, color: NAVY, align: "center", valign: "middle", fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 1.6, y: y, w: 11.2, h: 0.8, fill: { color: LIGHT_GREY }, line: { color: NAVY, width: 1 }, rectRadius: 0.05 });
|
||||||
|
s.addText(step.title, { x: 1.8, y: y + 0.1, w: 3.5, h: 0.3, fontSize: 14, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText(step.desc, { x: 1.8, y: y + 0.4, w: 11, h: 0.35, fontSize: 11, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 18 — NETWORKING
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Networking & Multiplayer", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 34, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: ACCENT } });
|
||||||
|
s.addText("Listen Server model — server owns WebSocket, clients relay via RPCs.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
s.addText("Audio Pipeline", { x: 0.5, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "Client mic → Server RPC (Opus-encoded)", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Server forwards to ElevenLabs WebSocket", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Agent audio → Multicast RPC to all clients", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Clients decode Opus → procedural sound wave", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 0.5, y: 2.55, w: 6, h: 2.4, color: GREY, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
s.addText("Speaker Arbitration", { x: 7, y: 2.15, w: 6, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "SpeakerSwitchHysteresis — min silence before switching", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "SpeakerIdleTimeout — clear active speaker", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "NetActiveSpeakerPawn replicated for gaze", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Only active speaker's audio forwarded to LLM", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 7, y: 2.55, w: 6, h: 2.4, color: GREY, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
s.addText("Level of Detail (LOD)", { x: 0.5, y: 5.1, w: 6, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "AudioLODCullDistance — skip audio at distance", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "LipSyncLODDistance — skip lip sync at distance", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "Active speaker always full quality", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 0.5, y: 5.5, w: 6, h: 1.5, color: GREY, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
s.addText("Replicated State", { x: 7, y: 5.1, w: 6, h: 0.4, fontSize: 16, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addText([
|
||||||
|
{ text: "bNetIsConversing", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "NetConnectedPawns[]", options: { bullet: true, fontSize: 13 } },
|
||||||
|
{ text: "CurrentEmotion + CurrentEmotionIntensity", options: { bullet: true, fontSize: 13 } },
|
||||||
|
], { x: 7, y: 5.5, w: 6, h: 1.5, color: GREY, fontFace: FONT_B, paraSpaceAfter: 5 });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 19 — FORCE DISABLE FLOW
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("ForceDisableConversation Flow", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 30, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: ACCENT } });
|
||||||
|
s.addText("Clean handoff between conversation and gameplay actions.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
const flow = [
|
||||||
|
{ step: "LLM Tool Call", desc: "perform_action(\"flee\")" },
|
||||||
|
{ step: "OnAgentActionRequested", desc: "Event fires on ElevenLabs component" },
|
||||||
|
{ step: "ForceDisableConversation", desc: "Blend out face/body/gaze animations" },
|
||||||
|
{ step: "OnReadyForAction", desc: "All components have reached neutral" },
|
||||||
|
{ step: "Play Gameplay", desc: "Montage / behavior switch" },
|
||||||
|
{ step: "ForceEnableConversation", desc: "Resume conversation when done" },
|
||||||
|
];
|
||||||
|
flow.forEach((f, i) => {
|
||||||
|
const y = 2.2 + i * 0.75;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: 0.5, y: y, w: 4, h: 0.6, fill: { color: NAVY }, rectRadius: 0.08 });
|
||||||
|
s.addText(f.step, { x: 0.5, y: y, w: 4, h: 0.6, fontSize: 13, bold: true, color: WHITE, align: "center", valign: "middle", fontFace: "Consolas" });
|
||||||
|
s.addText("→", { x: 4.6, y: y + 0.05, w: 0.5, h: 0.5, fontSize: 22, color: ACCENT, align: "center", valign: "middle", bold: true, fontFace: FONT_H });
|
||||||
|
s.addText(f.desc, { x: 5.2, y: y + 0.05, w: 7.6, h: 0.5, fontSize: 13, color: GREY, italic: true, valign: "middle", fontFace: FONT_B });
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 20 — EVENTS CHEAT SHEET
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide({ masterName: "CONTENT" });
|
||||||
|
s.addText("Events Cheat Sheet", { x: 0.5, y: 0.6, w: 12, h: 0.8, fontSize: 36, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0.5, y: 1.45, w: 0.6, h: 0.06, fill: { color: ACCENT } });
|
||||||
|
s.addText("Blueprint-assignable delegates on ElevenLabs component.", { x: 0.5, y: 1.65, w: 12, h: 0.4, fontSize: 14, color: GREY, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
const eventCols = [
|
||||||
|
{
|
||||||
|
title: "Connection",
|
||||||
|
items: ["OnAgentConnected", "OnAgentDisconnected", "OnAgentError"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: "Dialogue",
|
||||||
|
items: ["OnAgentTranscript", "OnAgentTextResponse", "OnAgentPartialResponse", "OnAgentStartedGenerating"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: "Audio",
|
||||||
|
items: ["OnAgentStartedSpeaking", "OnAudioPlaybackStarted", "OnAgentStoppedSpeaking", "OnAgentInterrupted"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: "State / Actions",
|
||||||
|
items: ["OnAgentEmotionChanged", "OnAgentActionRequested", "OnAgentClientToolCall", "OnActiveSpeakerChanged", "OnReadyForAction", "OnAgentResponseTimeout"]
|
||||||
|
},
|
||||||
|
];
|
||||||
|
eventCols.forEach((c, i) => {
|
||||||
|
const col = i % 2;
|
||||||
|
const row = Math.floor(i / 2);
|
||||||
|
const x = 0.5 + col * 6.3;
|
||||||
|
const y = 2.15 + row * 2.45;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: x, y: y, w: 6.0, h: 2.25, fill: { color: LIGHT_GREY }, line: { color: NAVY, width: 1 }, rectRadius: 0.08 });
|
||||||
|
s.addText(c.title, { x: x + 0.2, y: y + 0.1, w: 5.6, h: 0.3, fontSize: 14, bold: true, color: NAVY, fontFace: FONT_H });
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: x + 0.2, y: y + 0.42, w: 0.35, h: 0.04, fill: { color: ACCENT } });
|
||||||
|
s.addText(c.items.map(t => ({ text: t, options: { bullet: true, fontSize: 11 } })),
|
||||||
|
{ x: x + 0.2, y: y + 0.52, w: 5.6, h: 1.7, color: GREY, fontFace: "Consolas", paraSpaceAfter: 2 });
|
||||||
|
});
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SLIDE 21 — CLOSING / SUMMARY
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
s = pres.addSlide();
|
||||||
|
s.background = { color: NAVY };
|
||||||
|
s.addShape(pres.ShapeType.rect, { x: 0, y: 2.5, w: 13.333, h: 0.05, fill: { color: ACCENT } });
|
||||||
|
s.addText("Summary", { x: 0.6, y: 0.8, w: 12, h: 1.2, fontSize: 54, bold: true, color: WHITE, fontFace: FONT_H });
|
||||||
|
s.addText("A complete conversational AI stack for Unreal Engine.", { x: 0.6, y: 2.0, w: 12, h: 0.5, fontSize: 18, color: ICE, italic: true, fontFace: FONT_B });
|
||||||
|
|
||||||
|
const summary = [
|
||||||
|
{ h: "Real-time", t: "Spectral lip sync, speculative TTS, adaptive pre-buffering" },
|
||||||
|
{ h: "Expressive", t: "Emotion-driven face/body animation via AnimSequences" },
|
||||||
|
{ h: "Multiplayer", t: "Listen Server + Opus codec + distance LOD + speaker arbitration" },
|
||||||
|
{ h: "Data-driven", t: "Reusable PoseMaps, AgentConfigs, Tools, ActionSets" },
|
||||||
|
{ h: "Bridgeable", t: "Behavior plugin gaze bridge, ForceDisable handoff for gameplay" },
|
||||||
|
{ h: "BP-friendly", t: "All components blueprintable, rich event system" },
|
||||||
|
];
|
||||||
|
summary.forEach((item, i) => {
|
||||||
|
const col = i % 2;
|
||||||
|
const row = Math.floor(i / 2);
|
||||||
|
const x = 0.6 + col * 6.2;
|
||||||
|
const y = 3.0 + row * 1.4;
|
||||||
|
s.addShape(pres.ShapeType.roundRect, { x: x, y: y, w: 5.9, h: 1.2, fill: { color: NAVY_DARK }, line: { color: ACCENT, width: 1 }, rectRadius: 0.1 });
|
||||||
|
s.addText(item.h, { x: x + 0.2, y: y + 0.1, w: 5.5, h: 0.4, fontSize: 18, bold: true, color: ACCENT, fontFace: FONT_H });
|
||||||
|
s.addText(item.t, { x: x + 0.2, y: y + 0.55, w: 5.5, h: 0.6, fontSize: 12, color: ICE, italic: true, fontFace: FONT_B });
|
||||||
|
});
|
||||||
|
|
||||||
|
s.addText("ASTERION • PS_AI_ConvAgent • 2026", { x: 0.6, y: 6.9, w: 12, h: 0.4, fontSize: 12, color: ICE_DEEP, fontFace: FONT_B });
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
// SAVE
|
||||||
|
// ═══════════════════════════════════════════════════════════════════
|
||||||
|
pres.writeFile({ fileName: "PS_AI_ConvAgent_Documentation.pptx" })
|
||||||
|
.then(name => console.log(`Generated: ${name}`))
|
||||||
|
.catch(err => { console.error(err); process.exit(1); });
|
||||||
@@ -166,6 +166,14 @@ DefaultPlatformService=Null
|
|||||||
[OnlineSubsystemNull]
|
[OnlineSubsystemNull]
|
||||||
bEnabled=true
|
bEnabled=true
|
||||||
|
|
||||||
|
[Voice]
|
||||||
|
; Enable the Voice module so FVoiceModule::CreateVoiceEncoder() returns a valid
|
||||||
|
; Opus encoder/decoder. The engine default is bEnabled=false, which leaves the
|
||||||
|
; encoder NULL and makes networked agent audio fall back to raw PCM in large
|
||||||
|
; reliable multicast RPCs — these overflow the reliable buffer and drop VR
|
||||||
|
; client connections the moment the avatar speaks. With Opus, chunks are ~1-4KB.
|
||||||
|
bEnabled=true
|
||||||
|
|
||||||
[/Script/AndroidFileServerEditor.AndroidFileServerRuntimeSettings]
|
[/Script/AndroidFileServerEditor.AndroidFileServerRuntimeSettings]
|
||||||
bEnablePlugin=True
|
bEnablePlugin=True
|
||||||
bAllowNetworkConnection=True
|
bAllowNetworkConnection=True
|
||||||
@@ -181,6 +189,9 @@ bUseManualIPAddress=False
|
|||||||
ManualIPAddress=
|
ManualIPAddress=
|
||||||
|
|
||||||
[/Script/PS_AI_ConvAgent.PS_AI_ConvAgent_Settings_ElevenLabs]
|
[/Script/PS_AI_ConvAgent.PS_AI_ConvAgent_Settings_ElevenLabs]
|
||||||
API_Key=7b73c4244ccbec394cc010aaab01b0ec59ce0b11fc636ce4828354f675ca14a5
|
; NOTE: there is intentionally no API_Key here. The ElevenLabs key is provided
|
||||||
|
; per-install by the client at runtime (BlueprintLibrary "Set ElevenLabs API Key")
|
||||||
|
; and stored in a SaveGame slot (Saved/SaveGames/PS_AI_ConvAgent_ElevenLabs.sav),
|
||||||
|
; so it is never committed or baked into a shipped build.
|
||||||
ServerRegion=Global
|
ServerRegion=Global
|
||||||
|
|
||||||
|
|||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,8 +0,0 @@
|
|||||||
{
|
|
||||||
"BuildId": "37670630",
|
|
||||||
"Modules":
|
|
||||||
{
|
|
||||||
"PS_AI_Behavior": "UnrealEditor-PS_AI_Behavior.dll",
|
|
||||||
"PS_AI_BehaviorEditor": "UnrealEditor-PS_AI_BehaviorEditor.dll"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -16,7 +16,8 @@
|
|||||||
"PlatformAllowList": [
|
"PlatformAllowList": [
|
||||||
"Win64",
|
"Win64",
|
||||||
"Mac",
|
"Mac",
|
||||||
"Linux"
|
"Linux",
|
||||||
|
"Android"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -62,6 +62,24 @@ void UPS_AI_Behavior_BTService_EvaluateReaction::TickNode(
|
|||||||
UE_LOG(LogPS_AI_Behavior, Log, TEXT("[%s] Hostility changed: TeamId 0x%02X -> 0x%02X (hostile=%d)"),
|
UE_LOG(LogPS_AI_Behavior, Log, TEXT("[%s] Hostility changed: TeamId 0x%02X -> 0x%02X (hostile=%d)"),
|
||||||
*AIC->GetName(), AIC->GetGenericTeamId().GetId(), ExpectedTeamId, (int32)bHostile);
|
*AIC->GetName(), AIC->GetGenericTeamId().GetId(), ExpectedTeamId, (int32)bHostile);
|
||||||
AIC->SetTeamId(ExpectedTeamId);
|
AIC->SetTeamId(ExpectedTeamId);
|
||||||
|
|
||||||
|
// Enemy becoming non-hostile (disguise) — soft de-escalation:
|
||||||
|
// clear threat so the BT re-evaluates to Idle naturally on the next
|
||||||
|
// ApplyReaction below. Do NOT stop movement or force speed abruptly —
|
||||||
|
// the natural state transition will handle that via HandleStateChanged.
|
||||||
|
if (NPCType == EPS_AI_Behavior_NPCType::Enemy && !bHostile)
|
||||||
|
{
|
||||||
|
BB->ClearValue(PS_AI_Behavior_BB::ThreatActor);
|
||||||
|
BB->ClearValue(PS_AI_Behavior_BB::ThreatLocation);
|
||||||
|
BB->SetValueAsFloat(PS_AI_Behavior_BB::ThreatLevel, 0.0f);
|
||||||
|
if (Personality)
|
||||||
|
{
|
||||||
|
Personality->PerceivedThreatLevel = 0.0f;
|
||||||
|
}
|
||||||
|
UE_LOG(LogPS_AI_Behavior, Log,
|
||||||
|
TEXT("[%s] Hostile→false: threat cleared, BT will transition naturally."),
|
||||||
|
*AIC->GetName());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -32,6 +32,14 @@ EBTNodeResult::Type UPS_AI_Behavior_BTTask_FindAndFollowSpline::ExecuteTask(
|
|||||||
return EBTNodeResult::Failed;
|
return EBTNodeResult::Failed;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Respect a manual StopFollowing(): do not search for or resume a spline until
|
||||||
|
// the caller explicitly StartFollowing() again. Without this, the task would
|
||||||
|
// resume any CurrentSpline and override the stop on the very next tick.
|
||||||
|
if (Follower->bManuallyStopped)
|
||||||
|
{
|
||||||
|
return EBTNodeResult::Failed;
|
||||||
|
}
|
||||||
|
|
||||||
// Debug: log state on entry
|
// Debug: log state on entry
|
||||||
if (UPS_AI_Behavior_Statics::IsDebugEnabled(AIC->GetPawn()))
|
if (UPS_AI_Behavior_Statics::IsDebugEnabled(AIC->GetPawn()))
|
||||||
{
|
{
|
||||||
@@ -268,6 +276,18 @@ void UPS_AI_Behavior_BTTask_FindAndFollowSpline::TickTask(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Manual StopFollowing() requested mid-approach: cancel the move-to-spline and bail.
|
||||||
|
if (UPS_AI_Behavior_SplineFollowerComponent* StopCheck =
|
||||||
|
AIC->GetPawn()->FindComponentByClass<UPS_AI_Behavior_SplineFollowerComponent>())
|
||||||
|
{
|
||||||
|
if (StopCheck->bManuallyStopped)
|
||||||
|
{
|
||||||
|
AIC->StopMovement();
|
||||||
|
FinishLatentTask(OwnerComp, EBTNodeResult::Failed);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Check if we've reached the spline
|
// Check if we've reached the spline
|
||||||
if (AIC->GetMoveStatus() == EPathFollowingStatus::Idle)
|
if (AIC->GetMoveStatus() == EPathFollowingStatus::Idle)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -24,6 +24,12 @@ EBTNodeResult::Type UPS_AI_Behavior_BTTask_FollowSpline::ExecuteTask(
|
|||||||
AIC->GetPawn()->FindComponentByClass<UPS_AI_Behavior_SplineFollowerComponent>();
|
AIC->GetPawn()->FindComponentByClass<UPS_AI_Behavior_SplineFollowerComponent>();
|
||||||
if (!Follower) return EBTNodeResult::Failed;
|
if (!Follower) return EBTNodeResult::Failed;
|
||||||
|
|
||||||
|
// Respect a manual StopFollowing(): refuse to resume until an explicit restart.
|
||||||
|
if (Follower->bManuallyStopped)
|
||||||
|
{
|
||||||
|
return EBTNodeResult::Failed;
|
||||||
|
}
|
||||||
|
|
||||||
if (!Follower->CurrentSpline)
|
if (!Follower->CurrentSpline)
|
||||||
{
|
{
|
||||||
UE_LOG(LogPS_AI_Behavior, Warning, TEXT("[%s] FollowSpline: no current spline set."),
|
UE_LOG(LogPS_AI_Behavior, Warning, TEXT("[%s] FollowSpline: no current spline set."),
|
||||||
|
|||||||
@@ -81,7 +81,12 @@ void APS_AI_Behavior_AIController::OnPossess(APawn* InPawn)
|
|||||||
|
|
||||||
SetupBlackboard();
|
SetupBlackboard();
|
||||||
|
|
||||||
if (bAutoStartBehavior)
|
// Read auto-start from PersonalityComponent (defaults to true if no component)
|
||||||
|
const bool bAutoStart = PersonalityComp
|
||||||
|
? PersonalityComp->bAutoStartBehavior
|
||||||
|
: true;
|
||||||
|
|
||||||
|
if (bAutoStart)
|
||||||
{
|
{
|
||||||
StartBehavior();
|
StartBehavior();
|
||||||
}
|
}
|
||||||
@@ -89,7 +94,7 @@ void APS_AI_Behavior_AIController::OnPossess(APawn* InPawn)
|
|||||||
{
|
{
|
||||||
// Stay in Scripted state — no BT, no services, controlled externally
|
// Stay in Scripted state — no BT, no services, controlled externally
|
||||||
SetBehaviorState(EPS_AI_Behavior_State::Scripted);
|
SetBehaviorState(EPS_AI_Behavior_State::Scripted);
|
||||||
UE_LOG(LogPS_AI_Behavior, Log, TEXT("[%s] bAutoStartBehavior=false — entering Scripted state."),
|
UE_LOG(LogPS_AI_Behavior, Log, TEXT("[%s] bAutoStartBehavior=false (from Profile) — entering Scripted state."),
|
||||||
*GetName());
|
*GetName());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -97,7 +102,7 @@ void APS_AI_Behavior_AIController::OnPossess(APawn* InPawn)
|
|||||||
TryBindGazeComponent();
|
TryBindGazeComponent();
|
||||||
|
|
||||||
UE_LOG(LogPS_AI_Behavior, Log, TEXT("[%s] Possessed Pawn '%s' — TeamId=%d, AutoStart=%d."),
|
UE_LOG(LogPS_AI_Behavior, Log, TEXT("[%s] Possessed Pawn '%s' — TeamId=%d, AutoStart=%d."),
|
||||||
*GetName(), *InPawn->GetName(), TeamId, (int32)bAutoStartBehavior);
|
*GetName(), *InPawn->GetName(), TeamId, (int32)bAutoStart);
|
||||||
}
|
}
|
||||||
|
|
||||||
void APS_AI_Behavior_AIController::OnUnPossess()
|
void APS_AI_Behavior_AIController::OnUnPossess()
|
||||||
@@ -319,6 +324,111 @@ void APS_AI_Behavior_AIController::StopBehavior()
|
|||||||
UE_LOG(LogPS_AI_Behavior, Log, TEXT("[%s] StopBehavior — BT stopped, entering Scripted state."), *GetName());
|
UE_LOG(LogPS_AI_Behavior, Log, TEXT("[%s] StopBehavior — BT stopped, entering Scripted state."), *GetName());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void APS_AI_Behavior_AIController::ResetBehavior()
|
||||||
|
{
|
||||||
|
APawn* MyPawn = GetPawn();
|
||||||
|
if (!MyPawn)
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_Behavior, Warning, TEXT("[%s] ResetBehavior: no Pawn."), *GetName());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── 1. Stop any active movement ────────────────────────────────────
|
||||||
|
StopMovement();
|
||||||
|
|
||||||
|
if (auto* Spline = MyPawn->FindComponentByClass<UPS_AI_Behavior_SplineFollowerComponent>())
|
||||||
|
{
|
||||||
|
Spline->PauseFollowing();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── 2. Reset PersonalityComponent runtime state ────────────────────
|
||||||
|
if (PersonalityComp)
|
||||||
|
{
|
||||||
|
PersonalityComp->ResetRuntimeState();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── 3. Recompute TeamId from current NPCType + Faction + Hostile ──
|
||||||
|
{
|
||||||
|
EPS_AI_Behavior_NPCType NPCType = EPS_AI_Behavior_NPCType::Any;
|
||||||
|
if (MyPawn->Implements<UPS_AI_Behavior_Interface>())
|
||||||
|
{
|
||||||
|
NPCType = IPS_AI_Behavior_Interface::Execute_GetBehaviorNPCType(MyPawn);
|
||||||
|
}
|
||||||
|
else if (PersonalityComp)
|
||||||
|
{
|
||||||
|
NPCType = PersonalityComp->GetNPCType();
|
||||||
|
}
|
||||||
|
|
||||||
|
const uint8 Faction = (PersonalityComp && PersonalityComp->Profile)
|
||||||
|
? PersonalityComp->Profile->Faction : 0;
|
||||||
|
|
||||||
|
uint8 NewTeamId;
|
||||||
|
if (NPCType == EPS_AI_Behavior_NPCType::Enemy &&
|
||||||
|
MyPawn->Implements<UPS_AI_Behavior_Interface>() &&
|
||||||
|
!IPS_AI_Behavior_Interface::Execute_IsBehaviorHostile(MyPawn))
|
||||||
|
{
|
||||||
|
NewTeamId = PS_AI_Behavior_Team::DisguisedTeamId;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
NewTeamId = PS_AI_Behavior_Team::MakeTeamId(NPCType, Faction);
|
||||||
|
}
|
||||||
|
|
||||||
|
SetTeamId(NewTeamId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── 4. Reset Blackboard keys to initial-tick defaults ──────────────
|
||||||
|
if (Blackboard)
|
||||||
|
{
|
||||||
|
// Threat
|
||||||
|
Blackboard->ClearValue(PS_AI_Behavior_BB::ThreatActor);
|
||||||
|
Blackboard->ClearValue(PS_AI_Behavior_BB::ThreatLocation);
|
||||||
|
Blackboard->SetValueAsFloat(PS_AI_Behavior_BB::ThreatLevel, 0.0f);
|
||||||
|
Blackboard->ClearValue(PS_AI_Behavior_BB::LastKnownTargetPosition);
|
||||||
|
Blackboard->ClearValue(PS_AI_Behavior_BB::ThreatPawnName);
|
||||||
|
|
||||||
|
// Cover
|
||||||
|
Blackboard->ClearValue(PS_AI_Behavior_BB::CoverLocation);
|
||||||
|
Blackboard->ClearValue(PS_AI_Behavior_BB::CoverPoint);
|
||||||
|
Blackboard->SetValueAsBool(PS_AI_Behavior_BB::PreferCover, false);
|
||||||
|
Blackboard->SetValueAsEnum(PS_AI_Behavior_BB::CombatSubState,
|
||||||
|
static_cast<uint8>(EPS_AI_Behavior_CombatSubState::Engaging));
|
||||||
|
|
||||||
|
// Patrol / Spline
|
||||||
|
Blackboard->SetValueAsInt(PS_AI_Behavior_BB::PatrolIndex, 0);
|
||||||
|
Blackboard->ClearValue(PS_AI_Behavior_BB::CurrentSpline);
|
||||||
|
Blackboard->SetValueAsFloat(PS_AI_Behavior_BB::SplineProgress, 0.0f);
|
||||||
|
|
||||||
|
// Re-capture HomeLocation to current actor location (initial-tick semantics)
|
||||||
|
Blackboard->SetValueAsVector(PS_AI_Behavior_BB::HomeLocation,
|
||||||
|
MyPawn->GetActorLocation());
|
||||||
|
|
||||||
|
// Conversation pause
|
||||||
|
Blackboard->SetValueAsBool(PS_AI_Behavior_BB::ConversationPaused, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── 5. Clear local gaze / conversation caches ──────────────────────
|
||||||
|
bConversationPaused = false;
|
||||||
|
bUserHasSpokenInConversation = false;
|
||||||
|
ProximityGazeTarget = nullptr;
|
||||||
|
ProximityGazeDuration = 0.0f;
|
||||||
|
ProximityGazeCooldownTimer = 0.0f;
|
||||||
|
ProximityGazeCooldownActor = nullptr;
|
||||||
|
ClearGazeTarget();
|
||||||
|
|
||||||
|
// ─── 6. Force state back to Idle ────────────────────────────────────
|
||||||
|
// ForceState on PersonalityComp updates CurrentState + triggers HandleStateChanged
|
||||||
|
// which calls SetBehaviorMovementSpeed on the Pawn (so walk speed is reset).
|
||||||
|
// SetBehaviorState syncs the Blackboard for the BT.
|
||||||
|
if (PersonalityComp)
|
||||||
|
{
|
||||||
|
PersonalityComp->ForceState(EPS_AI_Behavior_State::Idle);
|
||||||
|
}
|
||||||
|
SetBehaviorState(EPS_AI_Behavior_State::Idle);
|
||||||
|
|
||||||
|
UE_LOG(LogPS_AI_Behavior, Log, TEXT("[%s] ResetBehavior — back to initial-tick state (Idle)."), *GetName());
|
||||||
|
}
|
||||||
|
|
||||||
void APS_AI_Behavior_AIController::SetBehaviorState(EPS_AI_Behavior_State NewState)
|
void APS_AI_Behavior_AIController::SetBehaviorState(EPS_AI_Behavior_State NewState)
|
||||||
{
|
{
|
||||||
if (Blackboard)
|
if (Blackboard)
|
||||||
@@ -620,6 +730,11 @@ void APS_AI_Behavior_AIController::TryBindGazeComponent()
|
|||||||
CachedConvAgentComponent = ConvComp;
|
CachedConvAgentComponent = ConvComp;
|
||||||
ConvProp_bNetIsConversing = CastField<FBoolProperty>(
|
ConvProp_bNetIsConversing = CastField<FBoolProperty>(
|
||||||
ConvClass->FindPropertyByName(TEXT("bNetIsConversing")));
|
ConvClass->FindPropertyByName(TEXT("bNetIsConversing")));
|
||||||
|
// Agent-to-agent dialogue: the orchestrator pins this agent's gaze on
|
||||||
|
// the other agent via GazeOverrideTarget. Cache it so we can treat that
|
||||||
|
// as an active conversation (see IsConversationActiveOnPawn).
|
||||||
|
ConvProp_GazeOverrideTarget = CastField<FObjectProperty>(
|
||||||
|
ConvClass->FindPropertyByName(TEXT("GazeOverrideTarget")));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -687,9 +802,73 @@ void APS_AI_Behavior_AIController::ClearGazeTarget()
|
|||||||
bool APS_AI_Behavior_AIController::IsConversationActiveOnPawn() const
|
bool APS_AI_Behavior_AIController::IsConversationActiveOnPawn() const
|
||||||
{
|
{
|
||||||
UActorComponent* ConvComp = CachedConvAgentComponent.Get();
|
UActorComponent* ConvComp = CachedConvAgentComponent.Get();
|
||||||
if (!ConvComp || !ConvProp_bNetIsConversing) return false;
|
if (!ConvComp) return false;
|
||||||
|
|
||||||
return ConvProp_bNetIsConversing->GetPropertyValue_InContainer(ConvComp);
|
// Player-driven conversation (a human joined this agent).
|
||||||
|
if (ConvProp_bNetIsConversing &&
|
||||||
|
ConvProp_bNetIsConversing->GetPropertyValue_InContainer(ConvComp))
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Agent-to-agent dialogue: the orchestrator sets GazeOverrideTarget on this
|
||||||
|
// agent. Treat it as an active conversation so proximity gaze / wandering back
|
||||||
|
// off and the ConvAgent owns the gaze (agents look at each other, not a passing
|
||||||
|
// player). Clearing the override (StopDialogue) restores normal behavior.
|
||||||
|
if (ConvProp_GazeOverrideTarget &&
|
||||||
|
ConvProp_GazeOverrideTarget->GetObjectPropertyValue_InContainer(ConvComp))
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_Behavior_AIController::ApplyMovementPaused(bool bPaused)
|
||||||
|
{
|
||||||
|
if (bPaused == bConversationPaused)
|
||||||
|
{
|
||||||
|
return; // no change — idempotent
|
||||||
|
}
|
||||||
|
bConversationPaused = bPaused;
|
||||||
|
|
||||||
|
// Blackboard flag read by the movement-branch decorators (Patrol / Spline).
|
||||||
|
if (Blackboard)
|
||||||
|
{
|
||||||
|
Blackboard->SetValueAsBool(PS_AI_Behavior_BB::ConversationPaused, bPaused);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (bPaused)
|
||||||
|
{
|
||||||
|
StopMovement(); // cancel any in-flight MoveTo immediately
|
||||||
|
}
|
||||||
|
|
||||||
|
if (APawn* MyPawn = GetPawn())
|
||||||
|
{
|
||||||
|
if (auto* Spline = MyPawn->FindComponentByClass<UPS_AI_Behavior_SplineFollowerComponent>())
|
||||||
|
{
|
||||||
|
if (bPaused) { Spline->PauseFollowing(); }
|
||||||
|
else { Spline->ResumeFollowing(); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
UE_LOG(LogPS_AI_Behavior, Log, TEXT("[%s] Movement %s."),
|
||||||
|
*GetName(), bPaused ? TEXT("paused") : TEXT("resumed"));
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_Behavior_AIController::DisableBehaviorMovement()
|
||||||
|
{
|
||||||
|
bMovementManuallyDisabled = true;
|
||||||
|
ApplyMovementPaused(true);
|
||||||
|
UE_LOG(LogPS_AI_Behavior, Log, TEXT("[%s] DisableBehaviorMovement — manual freeze (BT keeps running)."), *GetName());
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_Behavior_AIController::EnableBehaviorMovement()
|
||||||
|
{
|
||||||
|
bMovementManuallyDisabled = false;
|
||||||
|
// Stay paused if a conversation is still active; otherwise resume.
|
||||||
|
ApplyMovementPaused(IsConversationActiveOnPawn());
|
||||||
|
UE_LOG(LogPS_AI_Behavior, Log, TEXT("[%s] EnableBehaviorMovement — manual freeze released."), *GetName());
|
||||||
}
|
}
|
||||||
|
|
||||||
void APS_AI_Behavior_AIController::UpdateGazeTarget(float DeltaSeconds)
|
void APS_AI_Behavior_AIController::UpdateGazeTarget(float DeltaSeconds)
|
||||||
@@ -732,117 +911,26 @@ void APS_AI_Behavior_AIController::UpdateGazeTarget(float DeltaSeconds)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Priority 2: Conversation active → manage movement pause via local VAD
|
// ── Priority 2: Conversation active OR manual freeze → pause movement ──
|
||||||
|
// The BT keeps running; a "ConversationPaused" Blackboard decorator blocks the
|
||||||
|
// movement branches (Patrol / Spline). Leave the Flee branch un-decorated so
|
||||||
|
// fleeing still works. Movement pauses as soon as the conversation is genuinely
|
||||||
|
// active (agent greeting included) — it no longer waits for the user to speak —
|
||||||
|
// or whenever DisableBehaviorMovement() was called.
|
||||||
{
|
{
|
||||||
const bool bConversing = IsConversationActiveOnPawn();
|
const bool bConversing = IsConversationActiveOnPawn();
|
||||||
|
|
||||||
|
ApplyMovementPaused(bMovementManuallyDisabled || bConversing);
|
||||||
// Check local VAD: is the user actually speaking?
|
|
||||||
// The mic is on the PLAYER's Pawn — find it dynamically
|
|
||||||
bool bUserSpeaking = false;
|
|
||||||
if (MicProp_bIsUserSpeaking)
|
|
||||||
{
|
|
||||||
// Try cached mic first
|
|
||||||
UActorComponent* MicComp = CachedMicComponent.Get();
|
|
||||||
if (!MicComp)
|
|
||||||
{
|
|
||||||
// Find mic on any player pawn
|
|
||||||
static UClass* MicClass = LoadClass<UActorComponent>(nullptr,
|
|
||||||
TEXT("/Script/PS_AI_ConvAgent.PS_AI_ConvAgent_MicrophoneCaptureComponent"));
|
|
||||||
if (MicClass)
|
|
||||||
{
|
|
||||||
for (FConstPlayerControllerIterator It = GetWorld()->GetPlayerControllerIterator(); It; ++It)
|
|
||||||
{
|
|
||||||
if (APlayerController* PC = It->Get())
|
|
||||||
{
|
|
||||||
if (APawn* PlayerPawn = PC->GetPawn())
|
|
||||||
{
|
|
||||||
MicComp = PlayerPawn->FindComponentByClass(MicClass);
|
|
||||||
if (MicComp)
|
|
||||||
{
|
|
||||||
CachedMicComponent = MicComp;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (MicComp)
|
|
||||||
{
|
|
||||||
bUserSpeaking = MicProp_bIsUserSpeaking->GetPropertyValue_InContainer(MicComp);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Once user has spoken during this conversation, stay paused until conversation ends
|
|
||||||
if (bConversing && bUserSpeaking)
|
|
||||||
{
|
|
||||||
bUserHasSpokenInConversation = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (bConversing && bUserHasSpokenInConversation)
|
|
||||||
{
|
|
||||||
// Set BB flag to block movement branches via decorator
|
|
||||||
if (!bConversationPaused)
|
|
||||||
{
|
|
||||||
bConversationPaused = true;
|
|
||||||
|
|
||||||
if (Blackboard)
|
|
||||||
{
|
|
||||||
Blackboard->SetValueAsBool(PS_AI_Behavior_BB::ConversationPaused, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
StopMovement();
|
|
||||||
|
|
||||||
APawn* ConvPawn = GetPawn();
|
|
||||||
if (ConvPawn)
|
|
||||||
{
|
|
||||||
auto* Spline = ConvPawn->FindComponentByClass<UPS_AI_Behavior_SplineFollowerComponent>();
|
|
||||||
if (Spline)
|
|
||||||
{
|
|
||||||
Spline->PauseFollowing();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
UE_LOG(LogPS_AI_Behavior, Log, TEXT("[%s] User speaking — conversation pause."), *GetName());
|
|
||||||
}
|
|
||||||
|
|
||||||
ProximityGazeTarget = nullptr;
|
|
||||||
ProximityGazeDuration = 0.0f;
|
|
||||||
return; // Gaze managed by ConvAgent
|
|
||||||
}
|
|
||||||
|
|
||||||
if (bConversing)
|
if (bConversing)
|
||||||
{
|
{
|
||||||
// Connected but user hasn't spoken yet — don't touch gaze, don't pause
|
// Gaze is driven by the ConvAgent during conversation.
|
||||||
ProximityGazeTarget = nullptr;
|
ProximityGazeTarget = nullptr;
|
||||||
ProximityGazeDuration = 0.0f;
|
ProximityGazeDuration = 0.0f;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
// Manual-only freeze (no active conversation): movement stays paused above,
|
||||||
// Conversation ended — clear BB flag, resume movement
|
// but proximity gaze below still runs so the NPC can look around.
|
||||||
if (bConversationPaused)
|
|
||||||
{
|
|
||||||
bConversationPaused = false;
|
|
||||||
bUserHasSpokenInConversation = false;
|
|
||||||
|
|
||||||
if (Blackboard)
|
|
||||||
{
|
|
||||||
Blackboard->SetValueAsBool(PS_AI_Behavior_BB::ConversationPaused, false);
|
|
||||||
}
|
|
||||||
|
|
||||||
APawn* ConvPawn = GetPawn();
|
|
||||||
if (ConvPawn)
|
|
||||||
{
|
|
||||||
if (auto* Spline = ConvPawn->FindComponentByClass<UPS_AI_Behavior_SplineFollowerComponent>())
|
|
||||||
{
|
|
||||||
Spline->ResumeFollowing();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
UE_LOG(LogPS_AI_Behavior, Log, TEXT("[%s] Conversation ended — resumed."), *GetName());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Priority 3/4: Proximity gaze ────────────────────────────────
|
// ── Priority 3/4: Proximity gaze ────────────────────────────────
|
||||||
|
|||||||
@@ -263,7 +263,7 @@ AActor* UPS_AI_Behavior_PerceptionComponent::GetHighestThreatActor()
|
|||||||
{
|
{
|
||||||
// Owner is the AIController, get the Pawn
|
// Owner is the AIController, get the Pawn
|
||||||
const AAIController* AIC = Cast<AAIController>(Owner);
|
const AAIController* AIC = Cast<AAIController>(Owner);
|
||||||
const APawn* MyPawn = AIC ? AIC->GetPawn() : Cast<APawn>(const_cast<AActor*>(Owner));
|
const APawn* MyPawn = AIC ? ToRawPtr(AIC->GetPawn()) : Cast<APawn>(const_cast<AActor*>(Owner));
|
||||||
|
|
||||||
if (MyPawn)
|
if (MyPawn)
|
||||||
{
|
{
|
||||||
@@ -369,7 +369,7 @@ AActor* UPS_AI_Behavior_PerceptionComponent::GetHighestThreatActor(
|
|||||||
|
|
||||||
// Get our Pawn for IsTargetActorValid checks
|
// Get our Pawn for IsTargetActorValid checks
|
||||||
const AAIController* AIC = Cast<AAIController>(Owner);
|
const AAIController* AIC = Cast<AAIController>(Owner);
|
||||||
APawn* MyPawn = AIC ? AIC->GetPawn() : Cast<APawn>(const_cast<AActor*>(Owner));
|
APawn* MyPawn = AIC ? ToRawPtr(AIC->GetPawn()) : Cast<APawn>(const_cast<AActor*>(Owner));
|
||||||
|
|
||||||
// ─── Score accumulation per resolved actor ───────────────────────
|
// ─── Score accumulation per resolved actor ───────────────────────
|
||||||
// Multiple RawActors can resolve to the same Pawn (e.g. Pawn seen + weapon heard).
|
// Multiple RawActors can resolve to the same Pawn (e.g. Pawn seen + weapon heard).
|
||||||
@@ -587,8 +587,10 @@ AActor* UPS_AI_Behavior_PerceptionComponent::GetHighestThreatActor(
|
|||||||
}
|
}
|
||||||
else if (PerceivedActors.Num() > 0)
|
else if (PerceivedActors.Num() > 0)
|
||||||
{
|
{
|
||||||
UE_LOG(LogPS_AI_Behavior, Warning,
|
// Normal case: civilian surrounded by non-hostile actors, or disguised enemy
|
||||||
TEXT("[%s] GetHighestThreatActor: perceived %d actors but ALL were filtered out (myTeam=0x%02X). Check TeamIds and attitude."),
|
// seeing allies. Only log at Verbose — not an issue.
|
||||||
|
UE_LOG(LogPS_AI_Behavior, Verbose,
|
||||||
|
TEXT("[%s] GetHighestThreatActor: perceived %d actors, none hostile (myTeam=0x%02X)."),
|
||||||
*Owner->GetName(), PerceivedActors.Num(),
|
*Owner->GetName(), PerceivedActors.Num(),
|
||||||
AIC ? AIC->GetGenericTeamId().GetId() : 255);
|
AIC ? AIC->GetGenericTeamId().GetId() : 255);
|
||||||
}
|
}
|
||||||
@@ -609,7 +611,7 @@ float UPS_AI_Behavior_PerceptionComponent::CalculateThreatLevel()
|
|||||||
|
|
||||||
// Get our AIController and Pawn for checks
|
// Get our AIController and Pawn for checks
|
||||||
const AAIController* AIC = Cast<AAIController>(Owner);
|
const AAIController* AIC = Cast<AAIController>(Owner);
|
||||||
APawn* MyPawn = AIC ? AIC->GetPawn() : Cast<APawn>(const_cast<AActor*>(Owner));
|
APawn* MyPawn = AIC ? ToRawPtr(AIC->GetPawn()) : Cast<APawn>(const_cast<AActor*>(Owner));
|
||||||
|
|
||||||
for (AActor* RawActor : PerceivedActors)
|
for (AActor* RawActor : PerceivedActors)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -77,6 +77,21 @@ EPS_AI_Behavior_State UPS_AI_Behavior_PersonalityComponent::EvaluateReaction() c
|
|||||||
return EPS_AI_Behavior_State::Scripted;
|
return EPS_AI_Behavior_State::Scripted;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Disguised Enemy (NPCType=Enemy but IsBehaviorHostile=false) — cannot enter
|
||||||
|
// active combat states. Behave like a civilian (Alerted max, or Fleeing if terrified).
|
||||||
|
bool bDisguisedEnemy = false;
|
||||||
|
if (APawn* OwnerPawn = Cast<APawn>(GetOwner()))
|
||||||
|
{
|
||||||
|
if (OwnerPawn->Implements<UPS_AI_Behavior_Interface>())
|
||||||
|
{
|
||||||
|
const EPS_AI_Behavior_NPCType NPCT =
|
||||||
|
IPS_AI_Behavior_Interface::Execute_GetBehaviorNPCType(OwnerPawn);
|
||||||
|
const bool bHostile =
|
||||||
|
IPS_AI_Behavior_Interface::Execute_IsBehaviorHostile(OwnerPawn);
|
||||||
|
bDisguisedEnemy = (NPCT == EPS_AI_Behavior_NPCType::Enemy && !bHostile);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const float Courage = GetTrait(EPS_AI_Behavior_TraitAxis::Courage);
|
const float Courage = GetTrait(EPS_AI_Behavior_TraitAxis::Courage);
|
||||||
const float Aggressivity = GetTrait(EPS_AI_Behavior_TraitAxis::Aggressivity);
|
const float Aggressivity = GetTrait(EPS_AI_Behavior_TraitAxis::Aggressivity);
|
||||||
const float Caution = GetTrait(EPS_AI_Behavior_TraitAxis::Caution);
|
const float Caution = GetTrait(EPS_AI_Behavior_TraitAxis::Caution);
|
||||||
@@ -119,18 +134,22 @@ EPS_AI_Behavior_State UPS_AI_Behavior_PersonalityComponent::EvaluateReaction() c
|
|||||||
return EPS_AI_Behavior_State::Fleeing;
|
return EPS_AI_Behavior_State::Fleeing;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (CurrentState == EPS_AI_Behavior_State::Combat)
|
// Disguised enemies never enter active combat — skip the Combat branch entirely.
|
||||||
|
if (!bDisguisedEnemy)
|
||||||
{
|
{
|
||||||
// Stay in combat until threat drops well below the threshold
|
if (CurrentState == EPS_AI_Behavior_State::Combat)
|
||||||
if (PerceivedThreatLevel >= (EffectiveAttackThresh - Hysteresis))
|
{
|
||||||
|
// Stay in combat until threat drops well below the threshold
|
||||||
|
if (PerceivedThreatLevel >= (EffectiveAttackThresh - Hysteresis))
|
||||||
|
{
|
||||||
|
return EPS_AI_Behavior_State::Combat;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (PerceivedThreatLevel >= EffectiveAttackThresh && Aggressivity > 0.0f)
|
||||||
{
|
{
|
||||||
return EPS_AI_Behavior_State::Combat;
|
return EPS_AI_Behavior_State::Combat;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else if (PerceivedThreatLevel >= EffectiveAttackThresh && Aggressivity > 0.0f)
|
|
||||||
{
|
|
||||||
return EPS_AI_Behavior_State::Combat;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (PerceivedThreatLevel >= AlertThresh)
|
if (PerceivedThreatLevel >= AlertThresh)
|
||||||
{
|
{
|
||||||
@@ -280,6 +299,22 @@ float UPS_AI_Behavior_PersonalityComponent::CalculatePhaseDuration(EPS_AI_Behavi
|
|||||||
return Duration;
|
return Duration;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void UPS_AI_Behavior_PersonalityComponent::ResetRuntimeState()
|
||||||
|
{
|
||||||
|
if (!GetOwner() || !GetOwner()->HasAuthority())
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
PerceivedThreatLevel = 0.0f;
|
||||||
|
CombatCoverTimer = 0.0f;
|
||||||
|
bCombatCoverCycleActive = false;
|
||||||
|
bPreferCover = false;
|
||||||
|
|
||||||
|
UE_LOG(LogPS_AI_Behavior, Log, TEXT("[%s] PersonalityComponent runtime state reset."),
|
||||||
|
*GetOwner()->GetName());
|
||||||
|
}
|
||||||
|
|
||||||
void UPS_AI_Behavior_PersonalityComponent::ForceState(EPS_AI_Behavior_State NewState)
|
void UPS_AI_Behavior_PersonalityComponent::ForceState(EPS_AI_Behavior_State NewState)
|
||||||
{
|
{
|
||||||
// Only server can change replicated state
|
// Only server can change replicated state
|
||||||
|
|||||||
@@ -58,6 +58,7 @@ bool UPS_AI_Behavior_SplineFollowerComponent::StartFollowingAtDistance(
|
|||||||
CurrentDistance = FMath::Clamp(StartDistance, 0.0f, Spline->GetSplineLength());
|
CurrentDistance = FMath::Clamp(StartDistance, 0.0f, Spline->GetSplineLength());
|
||||||
bMovingForward = bForward;
|
bMovingForward = bForward;
|
||||||
bIsFollowing = true;
|
bIsFollowing = true;
|
||||||
|
bManuallyStopped = false; // an explicit Start lifts any previous manual stop
|
||||||
LastHandledJunctionIndex = -1;
|
LastHandledJunctionIndex = -1;
|
||||||
|
|
||||||
UE_LOG(LogPS_AI_Behavior, Warning,
|
UE_LOG(LogPS_AI_Behavior, Warning,
|
||||||
@@ -82,18 +83,23 @@ bool UPS_AI_Behavior_SplineFollowerComponent::StartFollowingAtDistance(
|
|||||||
void UPS_AI_Behavior_SplineFollowerComponent::StopFollowing()
|
void UPS_AI_Behavior_SplineFollowerComponent::StopFollowing()
|
||||||
{
|
{
|
||||||
bIsFollowing = false;
|
bIsFollowing = false;
|
||||||
UE_LOG(LogPS_AI_Behavior, Verbose, TEXT("[%s] Stopped following spline."),
|
// Firm, manual stop: the Behavior Tree spline tasks will refuse to re-start
|
||||||
|
// following until StartFollowing()/StartFollowingAtDistance() is called again.
|
||||||
|
bManuallyStopped = true;
|
||||||
|
UE_LOG(LogPS_AI_Behavior, Log, TEXT("[%s] StopFollowing: manual stop (BT will not auto-resume)."),
|
||||||
GetOwner() ? *GetOwner()->GetName() : TEXT("?"));
|
GetOwner() ? *GetOwner()->GetName() : TEXT("?"));
|
||||||
}
|
}
|
||||||
|
|
||||||
void UPS_AI_Behavior_SplineFollowerComponent::PauseFollowing()
|
void UPS_AI_Behavior_SplineFollowerComponent::PauseFollowing()
|
||||||
{
|
{
|
||||||
|
// Pause is NOT a manual stop — the BT is allowed to resume it.
|
||||||
bIsFollowing = false;
|
bIsFollowing = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
void UPS_AI_Behavior_SplineFollowerComponent::ResumeFollowing()
|
void UPS_AI_Behavior_SplineFollowerComponent::ResumeFollowing()
|
||||||
{
|
{
|
||||||
if (CurrentSpline)
|
// Do not resume over a manual stop — only an explicit Start lifts that.
|
||||||
|
if (CurrentSpline && !bManuallyStopped)
|
||||||
{
|
{
|
||||||
bIsFollowing = true;
|
bIsFollowing = true;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -48,7 +48,7 @@ public:
|
|||||||
uint8 TeamId = FGenericTeamId::NoTeam;
|
uint8 TeamId = FGenericTeamId::NoTeam;
|
||||||
|
|
||||||
/** Set the team ID at runtime. */
|
/** Set the team ID at runtime. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Team")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Team")
|
||||||
void SetTeamId(uint8 NewTeamId);
|
void SetTeamId(uint8 NewTeamId);
|
||||||
|
|
||||||
// ─── IGenericTeamAgentInterface (inherited from AAIController) ────
|
// ─── IGenericTeamAgentInterface (inherited from AAIController) ────
|
||||||
@@ -59,10 +59,6 @@ public:
|
|||||||
|
|
||||||
// ─── Configuration ──────────────────────────────────────────────────
|
// ─── Configuration ──────────────────────────────────────────────────
|
||||||
|
|
||||||
/** If false, the Behavior Tree does NOT start on possess. Call StartBehavior() manually. */
|
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Behavior")
|
|
||||||
bool bAutoStartBehavior = true;
|
|
||||||
|
|
||||||
/** Behavior Tree to run. If null, uses the Profile's DefaultBehaviorTree. */
|
/** Behavior Tree to run. If null, uses the Profile's DefaultBehaviorTree. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Behavior")
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Behavior")
|
||||||
TObjectPtr<UBehaviorTree> BehaviorTreeAsset;
|
TObjectPtr<UBehaviorTree> BehaviorTreeAsset;
|
||||||
@@ -78,47 +74,82 @@ public:
|
|||||||
// ─── Gaze Configuration ────────────────────────────────────────────
|
// ─── Gaze Configuration ────────────────────────────────────────────
|
||||||
|
|
||||||
/** Detection radius for proximity gaze (cm). NPC glances at nearby actors within this range. */
|
/** Detection radius for proximity gaze (cm). NPC glances at nearby actors within this range. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI Behavior|Gaze", meta = (ClampMin = "100.0", ClampMax = "1500.0"))
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_Behavior|Gaze", meta = (ClampMin = "100.0", ClampMax = "1500.0"))
|
||||||
float GazeProximityRadius = 400.0f;
|
float GazeProximityRadius = 400.0f;
|
||||||
|
|
||||||
/** Maximum duration (seconds) the NPC looks at a proximity target before releasing. */
|
/** Maximum duration (seconds) the NPC looks at a proximity target before releasing. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI Behavior|Gaze", meta = (ClampMin = "0.5", ClampMax = "30.0"))
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_Behavior|Gaze", meta = (ClampMin = "0.5", ClampMax = "30.0"))
|
||||||
float GazeMaxDuration = 3.0f;
|
float GazeMaxDuration = 3.0f;
|
||||||
|
|
||||||
/** Cooldown (seconds) before re-targeting the same actor after releasing. */
|
/** Cooldown (seconds) before re-targeting the same actor after releasing. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI Behavior|Gaze", meta = (ClampMin = "0.0", ClampMax = "30.0"))
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_Behavior|Gaze", meta = (ClampMin = "0.0", ClampMax = "30.0"))
|
||||||
float GazeCooldown = 5.0f;
|
float GazeCooldown = 5.0f;
|
||||||
|
|
||||||
/** Enable/disable proximity gaze (NPC glances at nearby actors in Idle/Patrol). */
|
/** Enable/disable proximity gaze (NPC glances at nearby actors in Idle/Patrol). */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI Behavior|Gaze")
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_Behavior|Gaze")
|
||||||
bool bEnableProximityGaze = true;
|
bool bEnableProximityGaze = true;
|
||||||
|
|
||||||
// ─── Component Access ───────────────────────────────────────────────
|
// ─── Component Access ───────────────────────────────────────────────
|
||||||
|
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
UPS_AI_Behavior_PerceptionComponent* GetBehaviorPerception() const { return BehaviorPerception; }
|
UPS_AI_Behavior_PerceptionComponent* GetBehaviorPerception() const { return BehaviorPerception; }
|
||||||
|
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
UPS_AI_Behavior_PersonalityComponent* GetPersonalityComponent() const { return PersonalityComp; }
|
UPS_AI_Behavior_PersonalityComponent* GetPersonalityComponent() const { return PersonalityComp; }
|
||||||
|
|
||||||
// ─── Blackboard Helpers ─────────────────────────────────────────────
|
// ─── Blackboard Helpers ─────────────────────────────────────────────
|
||||||
|
|
||||||
/** Write the current behavior state to the Blackboard. */
|
/** Write the current behavior state to the Blackboard. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Blackboard")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Blackboard")
|
||||||
void SetBehaviorState(EPS_AI_Behavior_State NewState);
|
void SetBehaviorState(EPS_AI_Behavior_State NewState);
|
||||||
|
|
||||||
/** Read the current behavior state from the Blackboard. */
|
/** Read the current behavior state from the Blackboard. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Blackboard")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Blackboard")
|
||||||
EPS_AI_Behavior_State GetBehaviorState() const;
|
EPS_AI_Behavior_State GetBehaviorState() const;
|
||||||
|
|
||||||
/** Start (or restart) the Behavior Tree. If state is Scripted, transitions to Idle. */
|
/** Start (or restart) the Behavior Tree. If state is Scripted, transitions to Idle. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
void StartBehavior();
|
void StartBehavior();
|
||||||
|
|
||||||
/** Stop the Behavior Tree and enter Scripted state. NPC stays alive and perceptible. */
|
/** Stop the Behavior Tree and enter Scripted state. NPC stays alive and perceptible. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
void StopBehavior();
|
void StopBehavior();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reset the NPC to its initial post-possession state (as if OnPossess had just run).
|
||||||
|
* Works for any NPC type (civilian, enemy, protector).
|
||||||
|
*
|
||||||
|
* Resets:
|
||||||
|
* - Blackboard: all threat, cover, patrol, spline keys back to defaults
|
||||||
|
* - Blackboard: HomeLocation re-captured to current actor location
|
||||||
|
* - PersonalityComponent: PerceivedThreatLevel = 0, combat/cover cycle cleared
|
||||||
|
* - TeamId: recomputed from current NPCType + Faction + Hostile flag
|
||||||
|
* - BehaviorState: forced to Idle (BT transitions to Patrol if on a spline)
|
||||||
|
* - Movement: any pending MoveTo cancelled, spline follower paused/reset
|
||||||
|
*
|
||||||
|
* Does NOT reset: runtime traits (ModifyTrait changes are kept), BT (stays running).
|
||||||
|
*
|
||||||
|
* Use cases: civilian calmed down by dialogue, enemy surrendering, mission reset.
|
||||||
|
*/
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
|
void ResetBehavior();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Freeze the NPC's autonomous movement WITHOUT stopping the Behavior Tree.
|
||||||
|
* The BT keeps running (perception, threat, gaze, reactions) but movement
|
||||||
|
* branches are blocked via the ConversationPaused Blackboard flag, and any
|
||||||
|
* in-flight move is cancelled. Fleeing is NOT blocked (leave the Flee branch
|
||||||
|
* un-decorated). Use this to keep an NPC in place during a scripted moment
|
||||||
|
* (e.g. dialogue) regardless of the automatic conversation pause.
|
||||||
|
*/
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
|
void DisableBehaviorMovement();
|
||||||
|
|
||||||
|
/** Release the manual movement freeze set by DisableBehaviorMovement().
|
||||||
|
* Movement stays paused if a conversation is still active. */
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
|
void EnableBehaviorMovement();
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
virtual void OnPossess(APawn* InPawn) override;
|
virtual void OnPossess(APawn* InPawn) override;
|
||||||
virtual void OnUnPossess() override;
|
virtual void OnUnPossess() override;
|
||||||
@@ -161,6 +192,11 @@ private:
|
|||||||
/** Check if a conversation is active on the Pawn (via reflection on ElevenLabsComponent). */
|
/** Check if a conversation is active on the Pawn (via reflection on ElevenLabsComponent). */
|
||||||
bool IsConversationActiveOnPawn() const;
|
bool IsConversationActiveOnPawn() const;
|
||||||
|
|
||||||
|
/** Apply (or release) the movement pause: syncs the ConversationPaused Blackboard
|
||||||
|
* flag, stops in-flight movement, and pauses/resumes the spline follower.
|
||||||
|
* No-op if the state is unchanged. */
|
||||||
|
void ApplyMovementPaused(bool bPaused);
|
||||||
|
|
||||||
// Cached reflection pointers (null if PS_AI_ConvAgent not loaded)
|
// Cached reflection pointers (null if PS_AI_ConvAgent not loaded)
|
||||||
TWeakObjectPtr<UActorComponent> CachedGazeComponent;
|
TWeakObjectPtr<UActorComponent> CachedGazeComponent;
|
||||||
FObjectProperty* GazeProp_TargetActor = nullptr;
|
FObjectProperty* GazeProp_TargetActor = nullptr;
|
||||||
@@ -170,6 +206,8 @@ private:
|
|||||||
// Conversation detection cache
|
// Conversation detection cache
|
||||||
TWeakObjectPtr<UActorComponent> CachedConvAgentComponent;
|
TWeakObjectPtr<UActorComponent> CachedConvAgentComponent;
|
||||||
FBoolProperty* ConvProp_bNetIsConversing = nullptr;
|
FBoolProperty* ConvProp_bNetIsConversing = nullptr;
|
||||||
|
// Set by the ConvAgent agent-to-agent orchestrator while two agents converse.
|
||||||
|
FObjectProperty* ConvProp_GazeOverrideTarget = nullptr;
|
||||||
|
|
||||||
// Mic VAD cache (local voice activity detection)
|
// Mic VAD cache (local voice activity detection)
|
||||||
TWeakObjectPtr<UActorComponent> CachedMicComponent;
|
TWeakObjectPtr<UActorComponent> CachedMicComponent;
|
||||||
@@ -179,6 +217,11 @@ private:
|
|||||||
bool bConversationPaused = false;
|
bool bConversationPaused = false;
|
||||||
bool bUserHasSpokenInConversation = false;
|
bool bUserHasSpokenInConversation = false;
|
||||||
|
|
||||||
|
// True while movement is frozen by an explicit DisableBehaviorMovement() call.
|
||||||
|
// Combined with the automatic conversation pause: movement is paused when
|
||||||
|
// either this is set OR a conversation is active.
|
||||||
|
bool bMovementManuallyDisabled = false;
|
||||||
|
|
||||||
// Proximity gaze state
|
// Proximity gaze state
|
||||||
TWeakObjectPtr<AActor> ProximityGazeTarget;
|
TWeakObjectPtr<AActor> ProximityGazeTarget;
|
||||||
float ProximityGazeDuration = 0.0f;
|
float ProximityGazeDuration = 0.0f;
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnDamageReceived, float, Damage, A
|
|||||||
*
|
*
|
||||||
* Attach to the NPC Pawn alongside PersonalityComponent.
|
* Attach to the NPC Pawn alongside PersonalityComponent.
|
||||||
*/
|
*/
|
||||||
UCLASS(ClassGroup = "PS AI Behavior", meta = (BlueprintSpawnableComponent, DisplayName = "PS AI Behavior - Combat"))
|
UCLASS(ClassGroup = "ASTERION|PS_AI_Behavior", meta = (BlueprintSpawnableComponent, DisplayName = "PS AI Behavior - Combat"))
|
||||||
class PS_AI_BEHAVIOR_API UPS_AI_Behavior_CombatComponent : public UActorComponent
|
class PS_AI_BEHAVIOR_API UPS_AI_Behavior_CombatComponent : public UActorComponent
|
||||||
{
|
{
|
||||||
GENERATED_BODY()
|
GENERATED_BODY()
|
||||||
@@ -52,21 +52,21 @@ public:
|
|||||||
// ─── Delegates ──────────────────────────────────────────────────────
|
// ─── Delegates ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Fired on ALL machines (server + clients) when an attack is executed. */
|
/** Fired on ALL machines (server + clients) when an attack is executed. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI Behavior|Combat")
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_Behavior|Combat")
|
||||||
FOnAttackExecuted OnAttackExecuted;
|
FOnAttackExecuted OnAttackExecuted;
|
||||||
|
|
||||||
/** Fired on server when damage is received. */
|
/** Fired on server when damage is received. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI Behavior|Combat")
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_Behavior|Combat")
|
||||||
FOnDamageReceived OnDamageReceived;
|
FOnDamageReceived OnDamageReceived;
|
||||||
|
|
||||||
// ─── API ────────────────────────────────────────────────────────────
|
// ─── API ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Whether the NPC can currently attack (cooldown elapsed). Server-only. */
|
/** Whether the NPC can currently attack (cooldown elapsed). Server-only. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Combat")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Combat")
|
||||||
bool CanAttack() const;
|
bool CanAttack() const;
|
||||||
|
|
||||||
/** Whether the target is within attack range. */
|
/** Whether the target is within attack range. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Combat")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Combat")
|
||||||
bool IsInAttackRange(AActor* Target) const;
|
bool IsInAttackRange(AActor* Target) const;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -74,14 +74,14 @@ public:
|
|||||||
* cooldown, and multicasts cosmetic event to all clients.
|
* cooldown, and multicasts cosmetic event to all clients.
|
||||||
* @return True if the attack was executed.
|
* @return True if the attack was executed.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Combat")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Combat")
|
||||||
bool ExecuteAttack(AActor* Target);
|
bool ExecuteAttack(AActor* Target);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Called when this NPC takes damage. Updates threat perception.
|
* Called when this NPC takes damage. Updates threat perception.
|
||||||
* Hook this into the owning Actor's OnTakeAnyDamage.
|
* Hook this into the owning Actor's OnTakeAnyDamage.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Combat")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Combat")
|
||||||
void NotifyDamageReceived(float Damage, AActor* DamageInstigator);
|
void NotifyDamageReceived(float Damage, AActor* DamageInstigator);
|
||||||
|
|
||||||
// ─── Replication ────────────────────────────────────────────────────
|
// ─── Replication ────────────────────────────────────────────────────
|
||||||
|
|||||||
@@ -88,23 +88,23 @@ public:
|
|||||||
// ─── API ────────────────────────────────────────────────────────────
|
// ─── API ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Can this point be used by the given NPC type? */
|
/** Can this point be used by the given NPC type? */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Cover")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Cover")
|
||||||
bool IsAccessibleTo(EPS_AI_Behavior_NPCType NPCType) const;
|
bool IsAccessibleTo(EPS_AI_Behavior_NPCType NPCType) const;
|
||||||
|
|
||||||
/** Is there room for one more occupant? */
|
/** Is there room for one more occupant? */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Cover")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Cover")
|
||||||
bool HasRoom() const;
|
bool HasRoom() const;
|
||||||
|
|
||||||
/** Try to claim this point for an NPC. Returns true if successful. */
|
/** Try to claim this point for an NPC. Returns true if successful. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Cover")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Cover")
|
||||||
bool Claim(AActor* Occupant);
|
bool Claim(AActor* Occupant);
|
||||||
|
|
||||||
/** Release this point (NPC leaves cover). */
|
/** Release this point (NPC leaves cover). */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Cover")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Cover")
|
||||||
void Release(AActor* Occupant);
|
void Release(AActor* Occupant);
|
||||||
|
|
||||||
/** Get the facing direction (forward vector of the actor). */
|
/** Get the facing direction (forward vector of the actor). */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Cover")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Cover")
|
||||||
FVector GetCoverDirection() const;
|
FVector GetCoverDirection() const;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -113,11 +113,11 @@ public:
|
|||||||
* @param ThreatLocation Where the threat is.
|
* @param ThreatLocation Where the threat is.
|
||||||
* @return Combined score 0.0 to 1.0.
|
* @return Combined score 0.0 to 1.0.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Cover")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Cover")
|
||||||
float EvaluateAgainstThreat(const FVector& ThreatLocation) const;
|
float EvaluateAgainstThreat(const FVector& ThreatLocation) const;
|
||||||
|
|
||||||
/** Enable/disable at runtime (e.g. barricade destroyed). */
|
/** Enable/disable at runtime (e.g. barricade destroyed). */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Cover")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Cover")
|
||||||
void SetEnabled(bool bNewEnabled);
|
void SetEnabled(bool bNewEnabled);
|
||||||
|
|
||||||
#if WITH_EDITOR
|
#if WITH_EDITOR
|
||||||
|
|||||||
@@ -44,11 +44,11 @@ public:
|
|||||||
// ─── NPC Type ───────────────────────────────────────────────────────
|
// ─── NPC Type ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Get this NPC's type (Civilian, Enemy, Protector). */
|
/** Get this NPC's type (Civilian, Enemy, Protector). */
|
||||||
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "PS AI Behavior")
|
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
EPS_AI_Behavior_NPCType GetBehaviorNPCType() const;
|
EPS_AI_Behavior_NPCType GetBehaviorNPCType() const;
|
||||||
|
|
||||||
/** Set this NPC's type. Called by gameplay logic or plugin actions. */
|
/** Set this NPC's type. Called by gameplay logic or plugin actions. */
|
||||||
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "PS AI Behavior")
|
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
void SetBehaviorNPCType(EPS_AI_Behavior_NPCType NewType);
|
void SetBehaviorNPCType(EPS_AI_Behavior_NPCType NewType);
|
||||||
|
|
||||||
// ─── Hostility ──────────────────────────────────────────────────────
|
// ─── Hostility ──────────────────────────────────────────────────────
|
||||||
@@ -58,14 +58,14 @@ public:
|
|||||||
* An infiltrated Enemy with IsHostile=false appears as Civilian to the perception system.
|
* An infiltrated Enemy with IsHostile=false appears as Civilian to the perception system.
|
||||||
* When SetHostile(true) is called, the NPC reveals itself and TeamId changes.
|
* When SetHostile(true) is called, the NPC reveals itself and TeamId changes.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "PS AI Behavior")
|
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
bool IsBehaviorHostile() const;
|
bool IsBehaviorHostile() const;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set hostility state. Typically called by gameplay scripts or ConvAgent actions.
|
* Set hostility state. Typically called by gameplay scripts or ConvAgent actions.
|
||||||
* Implementors should update their TeamId accordingly.
|
* Implementors should update their TeamId accordingly.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "PS AI Behavior")
|
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
void SetBehaviorHostile(bool bNewHostile);
|
void SetBehaviorHostile(bool bNewHostile);
|
||||||
|
|
||||||
// ─── Movement ───────────────────────────────────────────────────────
|
// ─── Movement ───────────────────────────────────────────────────────
|
||||||
@@ -80,13 +80,13 @@ public:
|
|||||||
*
|
*
|
||||||
* @param NewSpeed Desired walk speed in cm/s.
|
* @param NewSpeed Desired walk speed in cm/s.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "PS AI Behavior")
|
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
void SetBehaviorMovementSpeed(float NewSpeed);
|
void SetBehaviorMovementSpeed(float NewSpeed);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the Pawn's current movement speed (cm/s).
|
* Get the Pawn's current movement speed (cm/s).
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "PS AI Behavior")
|
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
float GetBehaviorMovementSpeed() const;
|
float GetBehaviorMovementSpeed() const;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -98,7 +98,7 @@ public:
|
|||||||
* @param NewState The new behavioral state.
|
* @param NewState The new behavioral state.
|
||||||
* @param OldState The previous behavioral state.
|
* @param OldState The previous behavioral state.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "PS AI Behavior")
|
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
void OnBehaviorStateChanged(EPS_AI_Behavior_State NewState, EPS_AI_Behavior_State OldState);
|
void OnBehaviorStateChanged(EPS_AI_Behavior_State NewState, EPS_AI_Behavior_State OldState);
|
||||||
|
|
||||||
// ─── Target Validation ──────────────────────────────────────────────
|
// ─── Target Validation ──────────────────────────────────────────────
|
||||||
@@ -113,7 +113,7 @@ public:
|
|||||||
* @param TargetActor The actor to validate.
|
* @param TargetActor The actor to validate.
|
||||||
* @return True if the actor can still be targeted.
|
* @return True if the actor can still be targeted.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "PS AI Behavior")
|
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
bool IsTargetActorValid(AActor* TargetActor) const;
|
bool IsTargetActorValid(AActor* TargetActor) const;
|
||||||
|
|
||||||
// ─── Combat ─────────────────────────────────────────────────────────
|
// ─── Combat ─────────────────────────────────────────────────────────
|
||||||
@@ -125,21 +125,21 @@ public:
|
|||||||
*
|
*
|
||||||
* @param Target The actor to attack.
|
* @param Target The actor to attack.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "PS AI Behavior")
|
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
void BehaviorStartAttack(AActor* Target);
|
void BehaviorStartAttack(AActor* Target);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Order the Pawn to stop attacking.
|
* Order the Pawn to stop attacking.
|
||||||
* Called when the BT exits the Attack task (state changed, target lost, etc.).
|
* Called when the BT exits the Attack task (state changed, target lost, etc.).
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "PS AI Behavior")
|
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
void BehaviorStopAttack();
|
void BehaviorStopAttack();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Query whether the Pawn can currently attack the target (has ammo, weapon ready, etc.).
|
* Query whether the Pawn can currently attack the target (has ammo, weapon ready, etc.).
|
||||||
* If false, the BT will keep the NPC in combat stance but won't call BehaviorStartAttack.
|
* If false, the BT will keep the NPC in combat stance but won't call BehaviorStartAttack.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "PS AI Behavior")
|
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
bool CanBehaviorAttack(AActor* Target) const;
|
bool CanBehaviorAttack(AActor* Target) const;
|
||||||
|
|
||||||
// ─── Stance ─────────────────────────────────────────────────────────
|
// ─── Stance ─────────────────────────────────────────────────────────
|
||||||
@@ -151,7 +151,7 @@ public:
|
|||||||
*
|
*
|
||||||
* @param bCrouch True = crouch, False = stand up.
|
* @param bCrouch True = crouch, False = stand up.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "PS AI Behavior")
|
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
void SetBehaviorCrouch(bool bCrouch);
|
void SetBehaviorCrouch(bool bCrouch);
|
||||||
|
|
||||||
// ─── Combat Style ───────────────────────────────────────────────────
|
// ─── Combat Style ───────────────────────────────────────────────────
|
||||||
@@ -162,7 +162,7 @@ public:
|
|||||||
* Melee NPCs rush the target; Ranged NPCs use cover and maintain distance.
|
* Melee NPCs rush the target; Ranged NPCs use cover and maintain distance.
|
||||||
* Default: Melee.
|
* Default: Melee.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "PS AI Behavior")
|
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
EPS_AI_Behavior_CombatType GetBehaviorCombatType() const;
|
EPS_AI_Behavior_CombatType GetBehaviorCombatType() const;
|
||||||
|
|
||||||
// ─── Actor Resolution ──────────────────────────────────────────────
|
// ─── Actor Resolution ──────────────────────────────────────────────
|
||||||
@@ -177,6 +177,6 @@ public:
|
|||||||
*
|
*
|
||||||
* @return The actor to use as threat source / target.
|
* @return The actor to use as threat source / target.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "PS AI Behavior")
|
UFUNCTION(BlueprintNativeEvent, BlueprintCallable, Category = "ASTERION|PS_AI_Behavior")
|
||||||
AActor* GetBehaviorThreatActor() const;
|
AActor* GetBehaviorThreatActor() const;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -14,7 +14,7 @@
|
|||||||
*
|
*
|
||||||
* Automatically added by PS_AI_Behavior_AIController — you don't need to add it manually.
|
* Automatically added by PS_AI_Behavior_AIController — you don't need to add it manually.
|
||||||
*/
|
*/
|
||||||
UCLASS(ClassGroup = "PS AI Behavior", meta = (DisplayName = "PS AI Behavior - Perception"))
|
UCLASS(ClassGroup = "ASTERION|PS_AI_Behavior", meta = (DisplayName = "PS AI Behavior - Perception"))
|
||||||
class PS_AI_BEHAVIOR_API UPS_AI_Behavior_PerceptionComponent : public UAIPerceptionComponent
|
class PS_AI_BEHAVIOR_API UPS_AI_Behavior_PerceptionComponent : public UAIPerceptionComponent
|
||||||
{
|
{
|
||||||
GENERATED_BODY()
|
GENERATED_BODY()
|
||||||
@@ -32,7 +32,7 @@ public:
|
|||||||
* If empty, uses default [Protector, Player, Civilian].
|
* If empty, uses default [Protector, Player, Civilian].
|
||||||
* @return The most threatening actor, or nullptr if none perceived.
|
* @return The most threatening actor, or nullptr if none perceived.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Perception")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Perception")
|
||||||
AActor* GetHighestThreatActor(const TArray<EPS_AI_Behavior_TargetType>& TargetPriority);
|
AActor* GetHighestThreatActor(const TArray<EPS_AI_Behavior_TargetType>& TargetPriority);
|
||||||
|
|
||||||
/** Convenience overload — reads priority from the Pawn's PersonalityProfile. */
|
/** Convenience overload — reads priority from the Pawn's PersonalityProfile. */
|
||||||
@@ -42,7 +42,7 @@ public:
|
|||||||
* Compute an aggregate threat level from all currently perceived hostile stimuli.
|
* Compute an aggregate threat level from all currently perceived hostile stimuli.
|
||||||
* Returns 0.0 (no threat) to 1.0+ (extreme danger).
|
* Returns 0.0 (no threat) to 1.0+ (extreme danger).
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Perception")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Perception")
|
||||||
float CalculateThreatLevel();
|
float CalculateThreatLevel();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -50,7 +50,7 @@ public:
|
|||||||
* @param OutLocation Filled with the threat location if any threat exists.
|
* @param OutLocation Filled with the threat location if any threat exists.
|
||||||
* @return True if a threat was found.
|
* @return True if a threat was found.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Perception")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Perception")
|
||||||
bool GetThreatLocation(FVector& OutLocation);
|
bool GetThreatLocation(FVector& OutLocation);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -82,7 +82,7 @@ protected:
|
|||||||
* Uses IsPlayerControlled() for Player, IPS_AI_Behavior interface or
|
* Uses IsPlayerControlled() for Player, IPS_AI_Behavior interface or
|
||||||
* PersonalityComponent for NPC type.
|
* PersonalityComponent for NPC type.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Perception")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Perception")
|
||||||
static EPS_AI_Behavior_TargetType ClassifyActor(const AActor* Actor);
|
static EPS_AI_Behavior_TargetType ClassifyActor(const AActor* Actor);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnBehaviorStateChanged, EPS_AI_Beh
|
|||||||
*
|
*
|
||||||
* Attach to the NPC Pawn/Character.
|
* Attach to the NPC Pawn/Character.
|
||||||
*/
|
*/
|
||||||
UCLASS(ClassGroup = "PS AI Behavior", meta = (BlueprintSpawnableComponent, DisplayName = "PS AI Behavior - Personality"))
|
UCLASS(ClassGroup = "ASTERION|PS_AI_Behavior", meta = (BlueprintSpawnableComponent, DisplayName = "PS AI Behavior - Personality"))
|
||||||
class PS_AI_BEHAVIOR_API UPS_AI_Behavior_PersonalityComponent : public UActorComponent
|
class PS_AI_BEHAVIOR_API UPS_AI_Behavior_PersonalityComponent : public UActorComponent
|
||||||
{
|
{
|
||||||
GENERATED_BODY()
|
GENERATED_BODY()
|
||||||
@@ -36,6 +36,14 @@ public:
|
|||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Personality")
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Personality")
|
||||||
TObjectPtr<UPS_AI_Behavior_PersonalityProfile> Profile;
|
TObjectPtr<UPS_AI_Behavior_PersonalityProfile> Profile;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* If false, the Behavior Tree does NOT start on possess.
|
||||||
|
* Use this for NPCs controlled externally (conversations, cinematics).
|
||||||
|
* Call StartBehavior() on the AIController to activate manually.
|
||||||
|
*/
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Personality")
|
||||||
|
bool bAutoStartBehavior = true;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Master debug toggle for this NPC.
|
* Master debug toggle for this NPC.
|
||||||
* When enabled, draws floating text above the NPC's head with:
|
* When enabled, draws floating text above the NPC's head with:
|
||||||
@@ -73,7 +81,7 @@ public:
|
|||||||
// ─── Delegates ──────────────────────────────────────────────────────
|
// ─── Delegates ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Fired when the behavioral state changes (on server AND clients via OnRep). */
|
/** Fired when the behavioral state changes (on server AND clients via OnRep). */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI Behavior|Personality")
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_Behavior|Personality")
|
||||||
FOnBehaviorStateChanged OnBehaviorStateChanged;
|
FOnBehaviorStateChanged OnBehaviorStateChanged;
|
||||||
|
|
||||||
// ─── API ────────────────────────────────────────────────────────────
|
// ─── API ────────────────────────────────────────────────────────────
|
||||||
@@ -82,7 +90,7 @@ public:
|
|||||||
* Evaluate the NPC's reaction based on current traits and perceived threat.
|
* Evaluate the NPC's reaction based on current traits and perceived threat.
|
||||||
* Returns the recommended behavioral state. Server-only.
|
* Returns the recommended behavioral state. Server-only.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Personality")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Personality")
|
||||||
EPS_AI_Behavior_State EvaluateReaction() const;
|
EPS_AI_Behavior_State EvaluateReaction() const;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -90,23 +98,31 @@ public:
|
|||||||
* Server-only: state is replicated to clients via OnRep.
|
* Server-only: state is replicated to clients via OnRep.
|
||||||
* @return The new behavioral state.
|
* @return The new behavioral state.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Personality")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Personality")
|
||||||
EPS_AI_Behavior_State ApplyReaction();
|
EPS_AI_Behavior_State ApplyReaction();
|
||||||
|
|
||||||
/** Get a runtime trait value (returns 0.5 if undefined). */
|
/** Get a runtime trait value (returns 0.5 if undefined). */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Personality")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Personality")
|
||||||
float GetTrait(EPS_AI_Behavior_TraitAxis Axis) const;
|
float GetTrait(EPS_AI_Behavior_TraitAxis Axis) const;
|
||||||
|
|
||||||
/** Modify a runtime trait by delta, clamped to [0, 1]. Server-only. */
|
/** Modify a runtime trait by delta, clamped to [0, 1]. Server-only. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Personality")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Personality")
|
||||||
void ModifyTrait(EPS_AI_Behavior_TraitAxis Axis, float Delta);
|
void ModifyTrait(EPS_AI_Behavior_TraitAxis Axis, float Delta);
|
||||||
|
|
||||||
/** Force a specific state (e.g. from conversation agent action). Server-only. */
|
/** Force a specific state (e.g. from conversation agent action). Server-only. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Personality")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Personality")
|
||||||
void ForceState(EPS_AI_Behavior_State NewState);
|
void ForceState(EPS_AI_Behavior_State NewState);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reset runtime state — perceived threat level, combat/cover cycle.
|
||||||
|
* Does NOT reset traits (ModifyTrait changes are preserved).
|
||||||
|
* Called by AIController::ResetBehavior().
|
||||||
|
*/
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Personality")
|
||||||
|
void ResetRuntimeState();
|
||||||
|
|
||||||
/** Get the NPC type from the interface or profile. */
|
/** Get the NPC type from the interface or profile. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Personality")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Personality")
|
||||||
EPS_AI_Behavior_NPCType GetNPCType() const;
|
EPS_AI_Behavior_NPCType GetNPCType() const;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -114,7 +130,7 @@ public:
|
|||||||
* Driven by the Combat/Cover cycle timer based on Aggressivity vs Caution.
|
* Driven by the Combat/Cover cycle timer based on Aggressivity vs Caution.
|
||||||
* Used by the IsCoverNeeded BT decorator.
|
* Used by the IsCoverNeeded BT decorator.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Personality")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Personality")
|
||||||
bool ShouldPreferCover() const { return bPreferCover; }
|
bool ShouldPreferCover() const { return bPreferCover; }
|
||||||
|
|
||||||
// ─── Replication ────────────────────────────────────────────────────
|
// ─── Replication ────────────────────────────────────────────────────
|
||||||
|
|||||||
@@ -160,11 +160,11 @@ public:
|
|||||||
/**
|
/**
|
||||||
* Get the score for a given trait axis. Returns 0.5 if the axis is not defined.
|
* Get the score for a given trait axis. Returns 0.5 if the axis is not defined.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Personality")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Personality")
|
||||||
float GetTrait(EPS_AI_Behavior_TraitAxis Axis) const;
|
float GetTrait(EPS_AI_Behavior_TraitAxis Axis) const;
|
||||||
|
|
||||||
/** Get the speed for a given state. Returns DefaultWalkSpeed if state not in SpeedPerState. */
|
/** Get the speed for a given state. Returns DefaultWalkSpeed if state not in SpeedPerState. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Personality")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Personality")
|
||||||
float GetSpeedForState(EPS_AI_Behavior_State State) const;
|
float GetSpeedForState(EPS_AI_Behavior_State State) const;
|
||||||
|
|
||||||
/** UPrimaryDataAsset interface */
|
/** UPrimaryDataAsset interface */
|
||||||
|
|||||||
@@ -10,7 +10,7 @@
|
|||||||
* Project-wide settings for the PS AI Behavior plugin.
|
* Project-wide settings for the PS AI Behavior plugin.
|
||||||
* Accessible via Project Settings -> Plugins -> PS AI Behavior.
|
* Accessible via Project Settings -> Plugins -> PS AI Behavior.
|
||||||
*/
|
*/
|
||||||
UCLASS(config = Game, defaultconfig, meta = (DisplayName = "PS AI Behavior"))
|
UCLASS(config = Game, defaultconfig, meta = (DisplayName = "ASTERION|PS_AI_Behavior"))
|
||||||
class PS_AI_BEHAVIOR_API UPS_AI_Behavior_Settings : public UDeveloperSettings
|
class PS_AI_BEHAVIOR_API UPS_AI_Behavior_Settings : public UDeveloperSettings
|
||||||
{
|
{
|
||||||
GENERATED_BODY()
|
GENERATED_BODY()
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnSplineEndReached,
|
|||||||
*
|
*
|
||||||
* Attach to the NPC Pawn. Works with or without the AI Controller.
|
* Attach to the NPC Pawn. Works with or without the AI Controller.
|
||||||
*/
|
*/
|
||||||
UCLASS(ClassGroup = "PS AI Behavior", meta = (BlueprintSpawnableComponent, DisplayName = "PS AI Behavior - Spline Follower"))
|
UCLASS(ClassGroup = "ASTERION|PS_AI_Behavior", meta = (BlueprintSpawnableComponent, DisplayName = "PS AI Behavior - Spline Follower"))
|
||||||
class PS_AI_BEHAVIOR_API UPS_AI_Behavior_SplineFollowerComponent : public UActorComponent
|
class PS_AI_BEHAVIOR_API UPS_AI_Behavior_SplineFollowerComponent : public UActorComponent
|
||||||
{
|
{
|
||||||
GENERATED_BODY()
|
GENERATED_BODY()
|
||||||
@@ -106,18 +106,25 @@ public:
|
|||||||
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Replicated, Category = "Spline Follower|Runtime")
|
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Replicated, Category = "Spline Follower|Runtime")
|
||||||
bool bIsFollowing = false;
|
bool bIsFollowing = false;
|
||||||
|
|
||||||
|
/** Set by StopFollowing(): a firm, manual stop. While true, the Behavior Tree
|
||||||
|
* spline tasks refuse to (re)start following — only an explicit StartFollowing()
|
||||||
|
* / StartFollowingAtDistance() clears it. This is what distinguishes a deliberate
|
||||||
|
* stop from PauseFollowing() (which the BT is allowed to auto-resume). */
|
||||||
|
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "Spline Follower|Runtime")
|
||||||
|
bool bManuallyStopped = false;
|
||||||
|
|
||||||
// ─── Delegates ──────────────────────────────────────────────────────
|
// ─── Delegates ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Fired when approaching a junction. Use to make custom spline selection. */
|
/** Fired when approaching a junction. Use to make custom spline selection. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI Behavior|Spline Follower")
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_Behavior|Spline Follower")
|
||||||
FOnApproachingJunction OnApproachingJunction;
|
FOnApproachingJunction OnApproachingJunction;
|
||||||
|
|
||||||
/** Fired when the NPC switches to a different spline. */
|
/** Fired when the NPC switches to a different spline. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI Behavior|Spline Follower")
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_Behavior|Spline Follower")
|
||||||
FOnSplineChanged OnSplineChanged;
|
FOnSplineChanged OnSplineChanged;
|
||||||
|
|
||||||
/** Fired when the NPC reaches the end of a spline (if bReverseAtEnd is false). */
|
/** Fired when the NPC reaches the end of a spline (if bReverseAtEnd is false). */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI Behavior|Spline Follower")
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_Behavior|Spline Follower")
|
||||||
FOnSplineEndReached OnSplineEndReached;
|
FOnSplineEndReached OnSplineEndReached;
|
||||||
|
|
||||||
// ─── API ────────────────────────────────────────────────────────────
|
// ─── API ────────────────────────────────────────────────────────────
|
||||||
@@ -128,24 +135,24 @@ public:
|
|||||||
* @param bForward Direction of travel.
|
* @param bForward Direction of travel.
|
||||||
* @return True if successfully started.
|
* @return True if successfully started.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Spline Follower")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Spline Follower")
|
||||||
bool StartFollowing(APS_AI_Behavior_SplinePath* Spline, bool bForward = true);
|
bool StartFollowing(APS_AI_Behavior_SplinePath* Spline, bool bForward = true);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Start following the given spline from a specific distance.
|
* Start following the given spline from a specific distance.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Spline Follower")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Spline Follower")
|
||||||
bool StartFollowingAtDistance(APS_AI_Behavior_SplinePath* Spline, float StartDistance, bool bForward = true);
|
bool StartFollowingAtDistance(APS_AI_Behavior_SplinePath* Spline, float StartDistance, bool bForward = true);
|
||||||
|
|
||||||
/** Stop following the current spline. */
|
/** Stop following the current spline. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Spline Follower")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Spline Follower")
|
||||||
void StopFollowing();
|
void StopFollowing();
|
||||||
|
|
||||||
/** Pause/resume without losing state. */
|
/** Pause/resume without losing state. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Spline Follower")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Spline Follower")
|
||||||
void PauseFollowing();
|
void PauseFollowing();
|
||||||
|
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Spline Follower")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Spline Follower")
|
||||||
void ResumeFollowing();
|
void ResumeFollowing();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -154,15 +161,15 @@ public:
|
|||||||
* @param DistanceOnNew Distance along the new spline to start from.
|
* @param DistanceOnNew Distance along the new spline to start from.
|
||||||
* @param bNewForward Direction on the new spline.
|
* @param bNewForward Direction on the new spline.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Spline Follower")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Spline Follower")
|
||||||
void SwitchToSpline(APS_AI_Behavior_SplinePath* NewSpline, float DistanceOnNew, bool bNewForward = true);
|
void SwitchToSpline(APS_AI_Behavior_SplinePath* NewSpline, float DistanceOnNew, bool bNewForward = true);
|
||||||
|
|
||||||
/** Get the effective walk speed (considering spline override). */
|
/** Get the effective walk speed (considering spline override). */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Spline Follower")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Spline Follower")
|
||||||
float GetEffectiveSpeed() const;
|
float GetEffectiveSpeed() const;
|
||||||
|
|
||||||
/** Get progress as a 0-1 ratio. */
|
/** Get progress as a 0-1 ratio. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Spline Follower")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Spline Follower")
|
||||||
float GetProgress() const;
|
float GetProgress() const;
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ public:
|
|||||||
* Called automatically after world initialization. Can be called again
|
* Called automatically after world initialization. Can be called again
|
||||||
* if splines are added/removed at runtime.
|
* if splines are added/removed at runtime.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|SplineNetwork")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|SplineNetwork")
|
||||||
void RebuildNetwork();
|
void RebuildNetwork();
|
||||||
|
|
||||||
// ─── Queries ────────────────────────────────────────────────────────
|
// ─── Queries ────────────────────────────────────────────────────────
|
||||||
@@ -49,7 +49,7 @@ public:
|
|||||||
* @param OutDistance Distance along the spline to the closest point.
|
* @param OutDistance Distance along the spline to the closest point.
|
||||||
* @return True if a spline was found within MaxDistance.
|
* @return True if a spline was found within MaxDistance.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|SplineNetwork")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|SplineNetwork")
|
||||||
bool FindClosestSpline(const FVector& WorldLocation,
|
bool FindClosestSpline(const FVector& WorldLocation,
|
||||||
EPS_AI_Behavior_NPCType NPCType, float MaxDistance,
|
EPS_AI_Behavior_NPCType NPCType, float MaxDistance,
|
||||||
APS_AI_Behavior_SplinePath*& OutSpline, float& OutDistance) const;
|
APS_AI_Behavior_SplinePath*& OutSpline, float& OutDistance) const;
|
||||||
@@ -57,7 +57,7 @@ public:
|
|||||||
/**
|
/**
|
||||||
* Get all splines of a given category.
|
* Get all splines of a given category.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|SplineNetwork")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|SplineNetwork")
|
||||||
TArray<APS_AI_Behavior_SplinePath*> GetSplinesForCategory(
|
TArray<APS_AI_Behavior_SplinePath*> GetSplinesForCategory(
|
||||||
EPS_AI_Behavior_NPCType Category) const;
|
EPS_AI_Behavior_NPCType Category) const;
|
||||||
|
|
||||||
@@ -73,7 +73,7 @@ public:
|
|||||||
* @param CautionScore Optional: NPC's caution trait (0-1). Higher = prefer quieter paths.
|
* @param CautionScore Optional: NPC's caution trait (0-1). Higher = prefer quieter paths.
|
||||||
* @return The chosen spline (could be the same if staying is best).
|
* @return The chosen spline (could be the same if staying is best).
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|SplineNetwork")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|SplineNetwork")
|
||||||
APS_AI_Behavior_SplinePath* ChooseSplineAtJunction(
|
APS_AI_Behavior_SplinePath* ChooseSplineAtJunction(
|
||||||
APS_AI_Behavior_SplinePath* CurrentSpline, int32 JunctionIndex,
|
APS_AI_Behavior_SplinePath* CurrentSpline, int32 JunctionIndex,
|
||||||
EPS_AI_Behavior_NPCType NPCType,
|
EPS_AI_Behavior_NPCType NPCType,
|
||||||
@@ -81,11 +81,11 @@ public:
|
|||||||
float CautionScore = 0.5f) const;
|
float CautionScore = 0.5f) const;
|
||||||
|
|
||||||
/** Total number of splines in the network. */
|
/** Total number of splines in the network. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|SplineNetwork")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|SplineNetwork")
|
||||||
int32 GetSplineCount() const { return AllSplines.Num(); }
|
int32 GetSplineCount() const { return AllSplines.Num(); }
|
||||||
|
|
||||||
/** Total number of junctions detected. */
|
/** Total number of junctions detected. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|SplineNetwork")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|SplineNetwork")
|
||||||
int32 GetJunctionCount() const { return TotalJunctions; }
|
int32 GetJunctionCount() const { return TotalJunctions; }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|||||||
@@ -94,7 +94,7 @@ public:
|
|||||||
// ─── API ────────────────────────────────────────────────────────────
|
// ─── API ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Can the given NPC type use this spline? */
|
/** Can the given NPC type use this spline? */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Spline")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Spline")
|
||||||
bool IsAccessibleTo(EPS_AI_Behavior_NPCType NPCType) const;
|
bool IsAccessibleTo(EPS_AI_Behavior_NPCType NPCType) const;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -104,7 +104,7 @@ public:
|
|||||||
* @param OutWorldPoint World location of the closest point on the spline.
|
* @param OutWorldPoint World location of the closest point on the spline.
|
||||||
* @return Distance from WorldLocation to the closest spline point.
|
* @return Distance from WorldLocation to the closest spline point.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Spline")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Spline")
|
||||||
float GetClosestPointOnSpline(const FVector& WorldLocation,
|
float GetClosestPointOnSpline(const FVector& WorldLocation,
|
||||||
float& OutDistance, FVector& OutWorldPoint) const;
|
float& OutDistance, FVector& OutWorldPoint) const;
|
||||||
|
|
||||||
@@ -115,24 +115,24 @@ public:
|
|||||||
* @param bForward Travel direction (true = increasing distance).
|
* @param bForward Travel direction (true = increasing distance).
|
||||||
* @return Array of upcoming junctions.
|
* @return Array of upcoming junctions.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Spline")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Spline")
|
||||||
TArray<FPS_AI_Behavior_SplineJunction> GetUpcomingJunctions(
|
TArray<FPS_AI_Behavior_SplineJunction> GetUpcomingJunctions(
|
||||||
float CurrentDistance, float LookAheadDist, bool bForward) const;
|
float CurrentDistance, float LookAheadDist, bool bForward) const;
|
||||||
|
|
||||||
/** Total spline length. */
|
/** Total spline length. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Spline")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Spline")
|
||||||
float GetSplineLength() const;
|
float GetSplineLength() const;
|
||||||
|
|
||||||
/** Get world location at a distance along the spline. */
|
/** Get world location at a distance along the spline. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Spline")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Spline")
|
||||||
FVector GetWorldLocationAtDistance(float Distance) const;
|
FVector GetWorldLocationAtDistance(float Distance) const;
|
||||||
|
|
||||||
/** Get world rotation at a distance along the spline. */
|
/** Get world rotation at a distance along the spline. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Spline")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Spline")
|
||||||
FRotator GetWorldRotationAtDistance(float Distance) const;
|
FRotator GetWorldRotationAtDistance(float Distance) const;
|
||||||
|
|
||||||
/** Get world-space tangent direction at a distance along the spline. */
|
/** Get world-space tangent direction at a distance along the spline. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Spline")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Spline")
|
||||||
FVector GetWorldDirectionAtDistance(float Distance) const;
|
FVector GetWorldDirectionAtDistance(float Distance) const;
|
||||||
|
|
||||||
#if WITH_EDITOR
|
#if WITH_EDITOR
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ public:
|
|||||||
* @param Loudness Noise loudness (default 1.0). Scales with HearingRange.
|
* @param Loudness Noise loudness (default 1.0). Scales with HearingRange.
|
||||||
* @param MaxRange Override max hearing range (0 = use sense default).
|
* @param MaxRange Override max hearing range (0 = use sense default).
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Perception",
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Perception",
|
||||||
meta = (WorldContext = "WorldContext", DefaultToSelf = "Shooter"))
|
meta = (WorldContext = "WorldContext", DefaultToSelf = "Shooter"))
|
||||||
static void ReportGunfire(UObject* WorldContext, FVector Location,
|
static void ReportGunfire(UObject* WorldContext, FVector Location,
|
||||||
AActor* Shooter, bool bIsEnemyFire = false,
|
AActor* Shooter, bool bIsEnemyFire = false,
|
||||||
|
|||||||
@@ -21,7 +21,7 @@
|
|||||||
* Enemy (Team 2) — Protectors and Civilians react
|
* Enemy (Team 2) — Protectors and Civilians react
|
||||||
* Protector (Team 3) — Allied with Civilians, hostile to Enemies
|
* Protector (Team 3) — Allied with Civilians, hostile to Enemies
|
||||||
*/
|
*/
|
||||||
UCLASS(ClassGroup = "PS AI Behavior", meta = (BlueprintSpawnableComponent,
|
UCLASS(ClassGroup = "ASTERION|PS_AI_Behavior", meta = (BlueprintSpawnableComponent,
|
||||||
DisplayName = "PS AI Behavior Pawn Identity"))
|
DisplayName = "PS AI Behavior Pawn Identity"))
|
||||||
class PS_AI_BEHAVIOR_API UPS_AI_Behavior_TeamComponent : public UActorComponent
|
class PS_AI_BEHAVIOR_API UPS_AI_Behavior_TeamComponent : public UActorComponent
|
||||||
{
|
{
|
||||||
@@ -31,16 +31,16 @@ public:
|
|||||||
UPS_AI_Behavior_TeamComponent();
|
UPS_AI_Behavior_TeamComponent();
|
||||||
|
|
||||||
/** Role of this actor in the behavior system. Determines team affiliation. */
|
/** Role of this actor in the behavior system. Determines team affiliation. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI Behavior|Team")
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_Behavior|Team")
|
||||||
EPS_AI_Behavior_NPCType Role = EPS_AI_Behavior_NPCType::Civilian;
|
EPS_AI_Behavior_NPCType Role = EPS_AI_Behavior_NPCType::Civilian;
|
||||||
|
|
||||||
/** Faction within the same role. Same role + different faction = rivals. */
|
/** Faction within the same role. Same role + different faction = rivals. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI Behavior|Team",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_Behavior|Team",
|
||||||
meta = (ClampMin = "0", ClampMax = "15"))
|
meta = (ClampMin = "0", ClampMax = "15"))
|
||||||
uint8 Faction = 0;
|
uint8 Faction = 0;
|
||||||
|
|
||||||
/** Change role at runtime. */
|
/** Change role at runtime. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI Behavior|Team")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_Behavior|Team")
|
||||||
void SetRole(EPS_AI_Behavior_NPCType NewRole) { Role = NewRole; }
|
void SetRole(EPS_AI_Behavior_NPCType NewRole) { Role = NewRole; }
|
||||||
|
|
||||||
/** Returns the TeamId derived from the current Role. */
|
/** Returns the TeamId derived from the current Role. */
|
||||||
|
|||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,8 +0,0 @@
|
|||||||
{
|
|
||||||
"BuildId": "37670630",
|
|
||||||
"Modules":
|
|
||||||
{
|
|
||||||
"PS_AI_ConvAgent": "UnrealEditor-PS_AI_ConvAgent.dll",
|
|
||||||
"PS_AI_ConvAgentEditor": "UnrealEditor-PS_AI_ConvAgentEditor.dll"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
Binary file not shown.
@@ -19,6 +19,10 @@ void FPS_AI_ConvAgentModule::StartupModule()
|
|||||||
Settings = NewObject<UPS_AI_ConvAgent_Settings_ElevenLabs>(GetTransientPackage(), "PS_AI_ConvAgent_Settings_ElevenLabs", RF_Standalone);
|
Settings = NewObject<UPS_AI_ConvAgent_Settings_ElevenLabs>(GetTransientPackage(), "PS_AI_ConvAgent_Settings_ElevenLabs", RF_Standalone);
|
||||||
Settings->AddToRoot();
|
Settings->AddToRoot();
|
||||||
|
|
||||||
|
// NOTE: the client-provided API key (if any) is loaded lazily on the first
|
||||||
|
// GetSettings() call rather than here — see GetSettings(). Loading at module
|
||||||
|
// startup would be too early: the SaveGame system may not be ready yet.
|
||||||
|
|
||||||
if (ISettingsModule* SettingsModule = FModuleManager::GetModulePtr<ISettingsModule>("Settings"))
|
if (ISettingsModule* SettingsModule = FModuleManager::GetModulePtr<ISettingsModule>("Settings"))
|
||||||
{
|
{
|
||||||
SettingsModule->RegisterSettings(
|
SettingsModule->RegisterSettings(
|
||||||
|
|||||||
@@ -0,0 +1,561 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#include "PS_AI_ConvAgent_AgentDialogue.h"
|
||||||
|
#include "PS_AI_ConvAgent_ElevenLabsComponent.h"
|
||||||
|
#include "GameFramework/Actor.h"
|
||||||
|
#include "GameFramework/Pawn.h"
|
||||||
|
#include "GameFramework/PlayerController.h"
|
||||||
|
#include "TimerManager.h"
|
||||||
|
#include "Engine/World.h"
|
||||||
|
|
||||||
|
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_Dialogue, Log, All);
|
||||||
|
|
||||||
|
static const TCHAR* FloorName(uint8 F)
|
||||||
|
{
|
||||||
|
switch (F) { case 1: return TEXT("A"); case 2: return TEXT("B"); default: return TEXT("None"); }
|
||||||
|
}
|
||||||
|
|
||||||
|
APS_AI_ConvAgent_AgentDialogue::APS_AI_ConvAgent_AgentDialogue()
|
||||||
|
{
|
||||||
|
// Ticks only while "addressing player" (auto-release watcher); disabled otherwise.
|
||||||
|
PrimaryActorTick.bCanEverTick = true;
|
||||||
|
PrimaryActorTick.bStartWithTickEnabled = false;
|
||||||
|
bReplicates = false; // server-side orchestration; agents replicate their own audio
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::BeginPlay()
|
||||||
|
{
|
||||||
|
Super::BeginPlay();
|
||||||
|
if (bAutoStartOnBeginPlay && HasAuthority())
|
||||||
|
{
|
||||||
|
StartDialogue();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::EndPlay(const EEndPlayReason::Type Reason)
|
||||||
|
{
|
||||||
|
StopDialogue(false);
|
||||||
|
UnbindAll();
|
||||||
|
Super::EndPlay(Reason);
|
||||||
|
}
|
||||||
|
|
||||||
|
UPS_AI_ConvAgent_ElevenLabsComponent* APS_AI_ConvAgent_AgentDialogue::ResolveAgentComp(AActor* Actor) const
|
||||||
|
{
|
||||||
|
return Actor ? Actor->FindComponentByClass<UPS_AI_ConvAgent_ElevenLabsComponent>() : nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::StartDialogue()
|
||||||
|
{
|
||||||
|
if (!HasAuthority())
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Dialogue, Warning,
|
||||||
|
TEXT("StartDialogue ignored — must run on the server (authority)."));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (bRunning)
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Dialogue, Log, TEXT("StartDialogue: already running."));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
CompA = ResolveAgentComp(AgentA);
|
||||||
|
CompB = ResolveAgentComp(AgentB);
|
||||||
|
if (!CompA || !CompB)
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Dialogue, Warning,
|
||||||
|
TEXT("StartDialogue: no ElevenLabs component on %s and/or %s."),
|
||||||
|
*GetNameSafe(AgentA), *GetNameSafe(AgentB));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (CompA == CompB)
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Dialogue, Warning, TEXT("StartDialogue: AgentA and AgentB are the same agent."));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
UnbindAll();
|
||||||
|
BindAgent(CompA, /*bIsA=*/true);
|
||||||
|
BindAgent(CompB, /*bIsA=*/false);
|
||||||
|
|
||||||
|
// CRITICAL: this is a TEXT-only loop. The agents must NEVER open their mics,
|
||||||
|
// otherwise each one HEARS the other's TTS and self-replies (via STT) — that
|
||||||
|
// produces the overlap / "already has the answer before the other spoke" chaos.
|
||||||
|
CompA->bAutoStartListening = false;
|
||||||
|
CompB->bAutoStartListening = false;
|
||||||
|
|
||||||
|
// The dialogue is interrupted when the PLAYER speaks — detected via a local mic
|
||||||
|
// voice-onset (primary) and OnAgentTranscript (backup). Enable transcript, and
|
||||||
|
// force Server turn mode + interruption so per-agent editor config can't silently
|
||||||
|
// block the mic path / the player barging in.
|
||||||
|
CompA->bEnableUserTranscript = true;
|
||||||
|
CompB->bEnableUserTranscript = true;
|
||||||
|
CompA->TurnMode = EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server;
|
||||||
|
CompB->TurnMode = EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server;
|
||||||
|
CompA->bAllowInterruption = true;
|
||||||
|
CompB->bAllowInterruption = true;
|
||||||
|
|
||||||
|
// Make the two agents look at EACH OTHER (not at a bystanding player).
|
||||||
|
// Replicated, so the audience sees it correctly on every machine.
|
||||||
|
CompA->SetGazeOverrideTarget(AgentB);
|
||||||
|
CompB->SetGazeOverrideTarget(AgentA);
|
||||||
|
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Dialogue, Warning,
|
||||||
|
TEXT("[Dialogue] Reminder: both agents must have an EMPTY FirstMessage in their AgentConfig "
|
||||||
|
"(a configured first message auto-greets on connect and collides with the opening)."));
|
||||||
|
|
||||||
|
// Reset state.
|
||||||
|
Floor = EFloor::None;
|
||||||
|
TurnId = 0;
|
||||||
|
bTextReady = false;
|
||||||
|
bSpeechDone = false;
|
||||||
|
bForwardedThisTurn = false;
|
||||||
|
PendingLine.Reset();
|
||||||
|
bOpeningSent = false;
|
||||||
|
bRunning = true;
|
||||||
|
|
||||||
|
CompA->StartConversation();
|
||||||
|
CompB->StartConversation();
|
||||||
|
|
||||||
|
TryKickoff();
|
||||||
|
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Dialogue, Log,
|
||||||
|
TEXT("[Dialogue] started: %s <-> %s (starter=%s, maxTurns=%d, waitForSpeech=%d)."),
|
||||||
|
*GetNameSafe(AgentA), *GetNameSafe(AgentB),
|
||||||
|
bAgentAStarts ? *GetNameSafe(AgentA) : *GetNameSafe(AgentB),
|
||||||
|
MaxTurns, bWaitForSpeechToFinish ? 1 : 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::StopDialogue(bool bEndConversations)
|
||||||
|
{
|
||||||
|
if (!bRunning && !bAddressingPlayer) { return; }
|
||||||
|
bRunning = false;
|
||||||
|
bAddressingPlayer = false;
|
||||||
|
AddressedByPlayer = nullptr;
|
||||||
|
EngagedFloor = EFloor::None;
|
||||||
|
SetActorTickEnabled(false);
|
||||||
|
|
||||||
|
if (UWorld* W = GetWorld())
|
||||||
|
{
|
||||||
|
W->GetTimerManager().ClearTimer(TurnTimer);
|
||||||
|
}
|
||||||
|
// Restore normal (player-tracking) gaze.
|
||||||
|
if (CompA) { CompA->SetGazeOverrideTarget(nullptr); }
|
||||||
|
if (CompB) { CompB->SetGazeOverrideTarget(nullptr); }
|
||||||
|
|
||||||
|
if (bEndConversations)
|
||||||
|
{
|
||||||
|
if (CompA) { CompA->EndConversation(); }
|
||||||
|
if (CompB) { CompB->EndConversation(); }
|
||||||
|
}
|
||||||
|
UnbindAll();
|
||||||
|
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Dialogue, Log, TEXT("[Dialogue] stopped after %d turns."), TurnId);
|
||||||
|
OnDialogueFinished.Broadcast(TurnId);
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::BindAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Comp, bool bIsA)
|
||||||
|
{
|
||||||
|
if (!Comp) { return; }
|
||||||
|
if (bIsA)
|
||||||
|
{
|
||||||
|
Comp->OnAgentTextResponse.AddDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleTextA);
|
||||||
|
Comp->OnAgentStoppedSpeaking.AddDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleStoppedA);
|
||||||
|
Comp->OnAgentConnected.AddDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleConnectedA);
|
||||||
|
// Per-agent interrupt/relay triggers (voice-onset = primary, transcript = words).
|
||||||
|
Comp->OnUserVoiceDetected.AddDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleVoiceOnsetA);
|
||||||
|
Comp->OnAgentTranscript.AddDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleTranscriptA);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
Comp->OnAgentTextResponse.AddDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleTextB);
|
||||||
|
Comp->OnAgentStoppedSpeaking.AddDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleStoppedB);
|
||||||
|
Comp->OnAgentConnected.AddDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleConnectedB);
|
||||||
|
Comp->OnUserVoiceDetected.AddDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleVoiceOnsetB);
|
||||||
|
Comp->OnAgentTranscript.AddDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleTranscriptB);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::UnbindAll()
|
||||||
|
{
|
||||||
|
if (CompA)
|
||||||
|
{
|
||||||
|
CompA->OnAgentTextResponse.RemoveDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleTextA);
|
||||||
|
CompA->OnAgentStoppedSpeaking.RemoveDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleStoppedA);
|
||||||
|
CompA->OnAgentConnected.RemoveDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleConnectedA);
|
||||||
|
CompA->OnAgentTranscript.RemoveDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleTranscriptA);
|
||||||
|
CompA->OnUserVoiceDetected.RemoveDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleVoiceOnsetA);
|
||||||
|
}
|
||||||
|
if (CompB)
|
||||||
|
{
|
||||||
|
CompB->OnAgentTextResponse.RemoveDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleTextB);
|
||||||
|
CompB->OnAgentStoppedSpeaking.RemoveDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleStoppedB);
|
||||||
|
CompB->OnAgentConnected.RemoveDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleConnectedB);
|
||||||
|
CompB->OnAgentTranscript.RemoveDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleTranscriptB);
|
||||||
|
CompB->OnUserVoiceDetected.RemoveDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleVoiceOnsetB);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::HandleVoiceOnsetA() { OnPlayerSpoke(EFloor::A, FString()); }
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::HandleVoiceOnsetB() { OnPlayerSpoke(EFloor::B, FString()); }
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::HandleTranscriptA(const FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs& Segment)
|
||||||
|
{
|
||||||
|
if (Segment.Speaker == TEXT("user")) { OnPlayerSpoke(EFloor::A, Segment.Text); }
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::HandleTranscriptB(const FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs& Segment)
|
||||||
|
{
|
||||||
|
if (Segment.Speaker == TEXT("user")) { OnPlayerSpoke(EFloor::B, Segment.Text); }
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::OnPlayerSpoke(EFloor Which, const FString& PlayerWords)
|
||||||
|
{
|
||||||
|
if (bRunning)
|
||||||
|
{
|
||||||
|
// The player barged into the agent-to-agent dialogue: enter 3-way floor mode.
|
||||||
|
InterruptForPlayer(Which);
|
||||||
|
}
|
||||||
|
else if (bAddressingPlayer)
|
||||||
|
{
|
||||||
|
// Already talking to the player — did they turn to the OTHER agent?
|
||||||
|
if (Which != EngagedFloor) { SetEngaged(Which); }
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
return; // orchestrator idle
|
||||||
|
}
|
||||||
|
|
||||||
|
// Relay the player's actual words (transcript path) to the WITNESS agent, so it
|
||||||
|
// "overhears" the exchange and can reference it later. Only while in floor mode.
|
||||||
|
if (bAddressingPlayer && !PlayerWords.IsEmpty() && !OverheardPrefix.IsEmpty())
|
||||||
|
{
|
||||||
|
RelayToBystander(Which, OverheardPrefix + TEXT("(la personne) ") + PlayerWords);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::InterruptForPlayer(EFloor Engaged)
|
||||||
|
{
|
||||||
|
APawn* Player = ResolveSpeakingPlayer();
|
||||||
|
if (!Player)
|
||||||
|
{
|
||||||
|
// Couldn't identify the speaker — fall back to a plain stop.
|
||||||
|
StopDialogue(/*bEndConversations=*/false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enter 3-way "floor" mode: stop the agent↔agent forwarding, turn BOTH agents to the
|
||||||
|
// player, and keep relaying the exchange to the witness agent. Agents stay connected;
|
||||||
|
// the orchestrator ticks to auto-release once the player leaves both (see Tick).
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Dialogue, Log,
|
||||||
|
TEXT("[Dialogue] player '%s' spoke to %s — entering 3-way floor mode."),
|
||||||
|
*GetNameSafe(Player), FloorName((uint8)Engaged));
|
||||||
|
|
||||||
|
bRunning = false;
|
||||||
|
if (UWorld* W = GetWorld())
|
||||||
|
{
|
||||||
|
W->GetTimerManager().ClearTimer(TurnTimer);
|
||||||
|
}
|
||||||
|
|
||||||
|
// The player barged in — cut BOTH agents' current speech now. The engaged agent
|
||||||
|
// is already interrupted by the player's mic (ElevenLabs), but the WITNESS agent
|
||||||
|
// never hears the player, so without this it keeps finishing its dealer<->client
|
||||||
|
// line ("...I'll give you the cash...") while the player is talking. Interrupting
|
||||||
|
// leaves the conversations open; the witness still receives the contextual_updates.
|
||||||
|
if (CompA) { CompA->InterruptAgent(); }
|
||||||
|
if (CompB) { CompB->InterruptAgent(); }
|
||||||
|
// Keep the agent bindings: in floor mode the text/transcript handlers RELAY to the
|
||||||
|
// witness instead of forwarding (their dialogue logic no-ops while !bRunning).
|
||||||
|
|
||||||
|
if (CompA) { CompA->SetGazeOverrideTarget(Player); }
|
||||||
|
if (CompB) { CompB->SetGazeOverrideTarget(Player); }
|
||||||
|
|
||||||
|
bAddressingPlayer = true;
|
||||||
|
AddressedByPlayer = Player;
|
||||||
|
EngagedFloor = EFloor::None; // force SetEngaged to send the reframe messages
|
||||||
|
SetEngaged(Engaged);
|
||||||
|
SetActorTickEnabled(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::SetEngaged(EFloor Which)
|
||||||
|
{
|
||||||
|
if (Which == EFloor::None || Which == EngagedFloor) { return; }
|
||||||
|
EngagedFloor = Which;
|
||||||
|
|
||||||
|
UPS_AI_ConvAgent_ElevenLabsComponent* Engaged = (Which == EFloor::A) ? CompA.Get() : CompB.Get();
|
||||||
|
UPS_AI_ConvAgent_ElevenLabsComponent* Bystander = (Which == EFloor::A) ? CompB.Get() : CompA.Get();
|
||||||
|
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Dialogue, Log,
|
||||||
|
TEXT("[Dialogue] player now addressing %s (the other agent becomes witness)."),
|
||||||
|
FloorName((uint8)Which));
|
||||||
|
|
||||||
|
// Silent reframe (contextual_update = no reply triggered).
|
||||||
|
if (Engaged && !InterruptContextMessage.IsEmpty())
|
||||||
|
{
|
||||||
|
Engaged->SendContextualUpdate(InterruptContextMessage);
|
||||||
|
}
|
||||||
|
if (Bystander && !BystanderContextMessage.IsEmpty())
|
||||||
|
{
|
||||||
|
Bystander->SendContextualUpdate(BystanderContextMessage);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::RelayToBystander(EFloor EngagedSide, const FString& FormattedLine)
|
||||||
|
{
|
||||||
|
UPS_AI_ConvAgent_ElevenLabsComponent* Bystander = (EngagedSide == EFloor::A) ? CompB.Get() : CompA.Get();
|
||||||
|
if (Bystander)
|
||||||
|
{
|
||||||
|
Bystander->SendContextualUpdate(FormattedLine);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::Tick(float DeltaSeconds)
|
||||||
|
{
|
||||||
|
Super::Tick(DeltaSeconds);
|
||||||
|
if (!bAddressingPlayer) { return; }
|
||||||
|
|
||||||
|
APawn* Player = AddressedByPlayer.Get();
|
||||||
|
if (!Player)
|
||||||
|
{
|
||||||
|
EndPlayerAddress();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Release once the player has walked farther than the threshold from BOTH agents.
|
||||||
|
const FVector P = Player->GetActorLocation();
|
||||||
|
const float Threshold = PlayerAddressReleaseDistance;
|
||||||
|
const bool bFarFromA = !AgentA || FVector::Dist(P, AgentA->GetActorLocation()) > Threshold;
|
||||||
|
const bool bFarFromB = !AgentB || FVector::Dist(P, AgentB->GetActorLocation()) > Threshold;
|
||||||
|
if (bFarFromA && bFarFromB)
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Dialogue, Log,
|
||||||
|
TEXT("[Dialogue] player left both agents — releasing."));
|
||||||
|
EndPlayerAddress();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::EndPlayerAddress()
|
||||||
|
{
|
||||||
|
bAddressingPlayer = false;
|
||||||
|
AddressedByPlayer = nullptr;
|
||||||
|
EngagedFloor = EFloor::None;
|
||||||
|
SetActorTickEnabled(false);
|
||||||
|
|
||||||
|
// Hand the agents back to normal gaze and end their (now idle) conversations.
|
||||||
|
if (CompA) { CompA->SetGazeOverrideTarget(nullptr); CompA->EndConversation(); }
|
||||||
|
if (CompB) { CompB->SetGazeOverrideTarget(nullptr); CompB->EndConversation(); }
|
||||||
|
UnbindAll(); // floor mode kept the bindings active — release them now.
|
||||||
|
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Dialogue, Log,
|
||||||
|
TEXT("[Dialogue] player address ended — agents released (after %d turns)."), TurnId);
|
||||||
|
OnDialogueFinished.Broadcast(TurnId);
|
||||||
|
}
|
||||||
|
|
||||||
|
APawn* APS_AI_ConvAgent_AgentDialogue::ResolveSpeakingPlayer() const
|
||||||
|
{
|
||||||
|
// The player who spoke is the active speaker on one of the agents (a pawn that is
|
||||||
|
// neither AgentA nor AgentB), else a connected non-agent pawn, else the local player.
|
||||||
|
AActor* A = AgentA.Get();
|
||||||
|
AActor* B = AgentB.Get();
|
||||||
|
auto IsPlayer = [A, B](APawn* P) -> bool { return P && P != A && P != B; };
|
||||||
|
|
||||||
|
if (CompA && IsPlayer(CompA->NetActiveSpeakerPawn)) { return CompA->NetActiveSpeakerPawn; }
|
||||||
|
if (CompB && IsPlayer(CompB->NetActiveSpeakerPawn)) { return CompB->NetActiveSpeakerPawn; }
|
||||||
|
|
||||||
|
if (CompA) { for (APawn* P : CompA->NetConnectedPawns) { if (IsPlayer(P)) { return P; } } }
|
||||||
|
if (CompB) { for (APawn* P : CompB->NetConnectedPawns) { if (IsPlayer(P)) { return P; } } }
|
||||||
|
|
||||||
|
// Voice-onset path: NetActiveSpeakerPawn is usually unset (the player never formally
|
||||||
|
// joined the agent). Pick the player pawn NEAREST to either agent.
|
||||||
|
UWorld* W = GetWorld();
|
||||||
|
if (!W) { return nullptr; }
|
||||||
|
|
||||||
|
APawn* Nearest = nullptr;
|
||||||
|
double BestDistSq = TNumericLimits<double>::Max();
|
||||||
|
for (FConstPlayerControllerIterator It = W->GetPlayerControllerIterator(); It; ++It)
|
||||||
|
{
|
||||||
|
APlayerController* PC = It->Get();
|
||||||
|
APawn* P = PC ? PC->GetPawn() : nullptr;
|
||||||
|
if (!IsPlayer(P)) { continue; }
|
||||||
|
const FVector L = P->GetActorLocation();
|
||||||
|
double D = TNumericLimits<double>::Max();
|
||||||
|
if (A) { D = FMath::Min(D, FVector::DistSquared(L, A->GetActorLocation())); }
|
||||||
|
if (B) { D = FMath::Min(D, FVector::DistSquared(L, B->GetActorLocation())); }
|
||||||
|
if (D < BestDistSq) { BestDistSq = D; Nearest = P; }
|
||||||
|
}
|
||||||
|
if (Nearest) { return Nearest; }
|
||||||
|
|
||||||
|
if (APlayerController* PC = W->GetFirstPlayerController())
|
||||||
|
{
|
||||||
|
return PC->GetPawn();
|
||||||
|
}
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::BeginTurn(EFloor NewFloor)
|
||||||
|
{
|
||||||
|
Floor = NewFloor;
|
||||||
|
bTextReady = false;
|
||||||
|
bSpeechDone = false;
|
||||||
|
bForwardedThisTurn = false;
|
||||||
|
PendingLine.Reset();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool APS_AI_ConvAgent_AgentDialogue::IsFloor(bool bFromA) const
|
||||||
|
{
|
||||||
|
return (bFromA && Floor == EFloor::A) || (!bFromA && Floor == EFloor::B);
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::TryKickoff()
|
||||||
|
{
|
||||||
|
if (!bRunning || bOpeningSent) { return; }
|
||||||
|
|
||||||
|
const bool bBothReady = CompA && CompB && CompA->IsConnected() && CompB->IsConnected();
|
||||||
|
if (!bBothReady)
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Dialogue, Log, TEXT("[Dialogue] Kickoff waiting: A.connected=%d B.connected=%d"),
|
||||||
|
(CompA && CompA->IsConnected()) ? 1 : 0, (CompB && CompB->IsConnected()) ? 1 : 0);
|
||||||
|
return; // wait for the other OnAgentConnected
|
||||||
|
}
|
||||||
|
|
||||||
|
bOpeningSent = true;
|
||||||
|
BeginTurn(bAgentAStarts ? EFloor::A : EFloor::B);
|
||||||
|
UPS_AI_ConvAgent_ElevenLabsComponent* Starter = (Floor == EFloor::A) ? CompA : CompB;
|
||||||
|
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Dialogue, Log, TEXT("[Dialogue] FLOOR=%s SEND-OPENING '%s'"),
|
||||||
|
FloorName((uint8)Floor), *OpeningMessage);
|
||||||
|
Starter->SendTextMessage(OpeningMessage);
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::TryForward()
|
||||||
|
{
|
||||||
|
if (!bRunning || bForwardedThisTurn) { return; }
|
||||||
|
|
||||||
|
// Need the floor-holder's full text; and, unless we're on a headless server,
|
||||||
|
// also require it to have finished speaking so the voices never overlap.
|
||||||
|
if (!bTextReady) { return; }
|
||||||
|
if (bWaitForSpeechToFinish && !bSpeechDone) { return; }
|
||||||
|
|
||||||
|
UPS_AI_ConvAgent_ElevenLabsComponent* Listener = (Floor == EFloor::A) ? CompB : CompA;
|
||||||
|
AActor* SpeakerActor = (Floor == EFloor::A) ? AgentA : AgentB;
|
||||||
|
if (!Listener) { return; }
|
||||||
|
|
||||||
|
if (MaxTurns > 0 && TurnId >= MaxTurns)
|
||||||
|
{
|
||||||
|
StopDialogue(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
bForwardedThisTurn = true;
|
||||||
|
++TurnId;
|
||||||
|
const FString Line = PendingLine;
|
||||||
|
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Dialogue, Log, TEXT("[Dialogue] turn=%d FORWARD %s->%s : '%.60s'"),
|
||||||
|
TurnId, FloorName((uint8)Floor), FloorName((uint8)(Floor == EFloor::A ? EFloor::B : EFloor::A)), *Line);
|
||||||
|
OnDialogueTurn.Broadcast(TurnId, SpeakerActor, Line);
|
||||||
|
|
||||||
|
// Hand the floor to the listener BEFORE sending, so the listener's upcoming
|
||||||
|
// text/speech events are attributed to it (and the old speaker's late/duplicate
|
||||||
|
// events are now non-floor and get ignored).
|
||||||
|
BeginTurn(Floor == EFloor::A ? EFloor::B : EFloor::A);
|
||||||
|
|
||||||
|
auto DoSend = [this, Line, Listener]()
|
||||||
|
{
|
||||||
|
if (bRunning && Listener && Listener->IsConnected())
|
||||||
|
{
|
||||||
|
Listener->SendTextMessage(Line);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if (TurnDelaySeconds > 0.0f)
|
||||||
|
{
|
||||||
|
if (UWorld* W = GetWorld())
|
||||||
|
{
|
||||||
|
W->GetTimerManager().SetTimer(TurnTimer, FTimerDelegate::CreateLambda(DoSend), TurnDelaySeconds, false);
|
||||||
|
}
|
||||||
|
else { DoSend(); }
|
||||||
|
}
|
||||||
|
else { DoSend(); }
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Delegate handlers ─────────────────────────────────────────────────────────
|
||||||
|
// Every handler ignores events from the agent that does NOT hold the floor.
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::HandleTextA(const FString& Text)
|
||||||
|
{
|
||||||
|
if (bAddressingPlayer)
|
||||||
|
{
|
||||||
|
// Floor mode: the ENGAGED agent's spoken line is overheard by the witness.
|
||||||
|
if (EngagedFloor == EFloor::A && !Text.IsEmpty() && !OverheardPrefix.IsEmpty())
|
||||||
|
{
|
||||||
|
RelayToBystander(EFloor::A, OverheardPrefix + TEXT("(l'autre personnage) ") + Text);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!bRunning) { return; }
|
||||||
|
if (!IsFloor(/*bFromA=*/true))
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Dialogue, Verbose, TEXT("[Dialogue] IGNORED text from NON-FLOOR A (floor=%s)"), FloorName((uint8)Floor));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (Text.IsEmpty()) { return; }
|
||||||
|
PendingLine = Text;
|
||||||
|
bTextReady = true;
|
||||||
|
TryForward();
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::HandleTextB(const FString& Text)
|
||||||
|
{
|
||||||
|
if (bAddressingPlayer)
|
||||||
|
{
|
||||||
|
// Floor mode: the ENGAGED agent's spoken line is overheard by the witness.
|
||||||
|
if (EngagedFloor == EFloor::B && !Text.IsEmpty() && !OverheardPrefix.IsEmpty())
|
||||||
|
{
|
||||||
|
RelayToBystander(EFloor::B, OverheardPrefix + TEXT("(l'autre personnage) ") + Text);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!bRunning) { return; }
|
||||||
|
if (!IsFloor(/*bFromA=*/false))
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Dialogue, Verbose, TEXT("[Dialogue] IGNORED text from NON-FLOOR B (floor=%s)"), FloorName((uint8)Floor));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (Text.IsEmpty()) { return; }
|
||||||
|
PendingLine = Text;
|
||||||
|
bTextReady = true;
|
||||||
|
TryForward();
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::HandleStoppedA()
|
||||||
|
{
|
||||||
|
if (!bRunning) { return; }
|
||||||
|
if (!IsFloor(/*bFromA=*/true))
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Dialogue, Verbose, TEXT("[Dialogue] IGNORED stopped from NON-FLOOR A (floor=%s)"), FloorName((uint8)Floor));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
bSpeechDone = true;
|
||||||
|
TryForward();
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::HandleStoppedB()
|
||||||
|
{
|
||||||
|
if (!bRunning) { return; }
|
||||||
|
if (!IsFloor(/*bFromA=*/false))
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_Dialogue, Verbose, TEXT("[Dialogue] IGNORED stopped from NON-FLOOR B (floor=%s)"), FloorName((uint8)Floor));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
bSpeechDone = true;
|
||||||
|
TryForward();
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::HandleConnectedA(const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& Info)
|
||||||
|
{
|
||||||
|
TryKickoff();
|
||||||
|
}
|
||||||
|
|
||||||
|
void APS_AI_ConvAgent_AgentDialogue::HandleConnectedB(const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& Info)
|
||||||
|
{
|
||||||
|
TryKickoff();
|
||||||
|
}
|
||||||
@@ -1,7 +1,98 @@
|
|||||||
// Copyright ASTERION. All Rights Reserved.
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
#include "PS_AI_ConvAgent_BlueprintLibrary.h"
|
#include "PS_AI_ConvAgent_BlueprintLibrary.h"
|
||||||
|
#include "PS_AI_ConvAgent.h"
|
||||||
|
#include "PS_AI_ConvAgent_SaveGame_ElevenLabs.h"
|
||||||
#include "Components/SkeletalMeshComponent.h"
|
#include "Components/SkeletalMeshComponent.h"
|
||||||
|
#include "Kismet/GameplayStatics.h"
|
||||||
|
#include "Misc/AES.h"
|
||||||
|
|
||||||
|
namespace
|
||||||
|
{
|
||||||
|
// In-memory cache of the client API key (PLAINTEXT). This BlueprintLibrary is
|
||||||
|
// the single owner of the key — every consumer (WebSocket auth, region probe,
|
||||||
|
// editor sync tools) reads it via GetElevenLabsAPIKey(). The SaveGame on disk
|
||||||
|
// is the persistent backing store (AES-encrypted); this cache avoids a disk
|
||||||
|
// read + decrypt on every access. All access is on the game thread, so no
|
||||||
|
// synchronization is needed.
|
||||||
|
bool GElevenLabsKeyLoaded = false;
|
||||||
|
FString GElevenLabsKeyCache;
|
||||||
|
|
||||||
|
// 32-byte AES key used to obfuscate the API key at rest in the .sav file.
|
||||||
|
// IMPORTANT: this is embedded in the binary, so it is NOT real security — a
|
||||||
|
// determined attacker can extract it. It only raises the bar against casual
|
||||||
|
// inspection of the save file (agreed "trusted distribution" threat model).
|
||||||
|
void GetObfuscationAESKey(FAES::FAESKey& OutKey)
|
||||||
|
{
|
||||||
|
static const uint8 KeyBytes[FAES::FAESKey::KeySize] = {
|
||||||
|
0x9A, 0x3C, 0x71, 0x05, 0xE2, 0x4F, 0xB8, 0x16,
|
||||||
|
0x2D, 0xC4, 0x88, 0x5B, 0x37, 0xF1, 0x60, 0xAA,
|
||||||
|
0x14, 0x7E, 0xD9, 0x42, 0x6B, 0x90, 0x0C, 0xF5,
|
||||||
|
0x53, 0x21, 0xBE, 0x8D, 0x47, 0xA0, 0x39, 0xCF
|
||||||
|
};
|
||||||
|
FMemory::Memcpy(OutKey.Key, KeyBytes, FAES::FAESKey::KeySize);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Encrypt a plaintext key to AES block-padded bytes. Sets OutPlainLen to the
|
||||||
|
* original UTF-8 byte length so padding can be stripped on decrypt. */
|
||||||
|
TArray<uint8> EncryptElevenLabsKey(const FString& Plain, int32& OutPlainLen)
|
||||||
|
{
|
||||||
|
FTCHARToUTF8 Utf8(*Plain);
|
||||||
|
OutPlainLen = Utf8.Length();
|
||||||
|
|
||||||
|
TArray<uint8> Buffer;
|
||||||
|
Buffer.Append(reinterpret_cast<const uint8*>(Utf8.Get()), Utf8.Length());
|
||||||
|
if (Buffer.Num() == 0)
|
||||||
|
{
|
||||||
|
return Buffer; // empty key → empty blob
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pad up to a multiple of the AES block size (required by FAES).
|
||||||
|
const int32 Remainder = Buffer.Num() % FAES::AESBlockSize;
|
||||||
|
if (Remainder != 0)
|
||||||
|
{
|
||||||
|
Buffer.AddZeroed(FAES::AESBlockSize - Remainder);
|
||||||
|
}
|
||||||
|
|
||||||
|
FAES::FAESKey Key;
|
||||||
|
GetObfuscationAESKey(Key);
|
||||||
|
FAES::EncryptData(Buffer.GetData(), Buffer.Num(), Key);
|
||||||
|
return Buffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Decrypt AES block-padded bytes back to the original plaintext key. */
|
||||||
|
FString DecryptElevenLabsKey(const TArray<uint8>& Encrypted, int32 PlainLen)
|
||||||
|
{
|
||||||
|
if (Encrypted.Num() == 0 || PlainLen <= 0 || (Encrypted.Num() % FAES::AESBlockSize) != 0)
|
||||||
|
{
|
||||||
|
return FString();
|
||||||
|
}
|
||||||
|
|
||||||
|
TArray<uint8> Buffer = Encrypted;
|
||||||
|
FAES::FAESKey Key;
|
||||||
|
GetObfuscationAESKey(Key);
|
||||||
|
FAES::DecryptData(Buffer.GetData(), Buffer.Num(), Key);
|
||||||
|
|
||||||
|
// Strip padding back to the original UTF-8 length, then null-terminate.
|
||||||
|
const int32 Len = FMath::Clamp(PlainLen, 0, Buffer.Num());
|
||||||
|
Buffer.SetNum(Len);
|
||||||
|
Buffer.Add(0);
|
||||||
|
return FString(FUTF8ToTCHAR(reinterpret_cast<const ANSICHAR*>(Buffer.GetData())).Get());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Load the API-key SaveGame from its slot, or null if none exists yet. */
|
||||||
|
UPS_AI_ConvAgent_SaveGame_ElevenLabs* LoadElevenLabsKeySaveGame()
|
||||||
|
{
|
||||||
|
const TCHAR* Slot = UPS_AI_ConvAgent_SaveGame_ElevenLabs::GetSlotName();
|
||||||
|
const int32 UserIndex = UPS_AI_ConvAgent_SaveGame_ElevenLabs::GetUserIndex();
|
||||||
|
if (UGameplayStatics::DoesSaveGameExist(Slot, UserIndex))
|
||||||
|
{
|
||||||
|
return Cast<UPS_AI_ConvAgent_SaveGame_ElevenLabs>(
|
||||||
|
UGameplayStatics::LoadGameFromSlot(Slot, UserIndex));
|
||||||
|
}
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
void UPS_AI_ConvAgent_BlueprintLibrary::SetPostProcessAnimBlueprint(
|
void UPS_AI_ConvAgent_BlueprintLibrary::SetPostProcessAnimBlueprint(
|
||||||
USkeletalMeshComponent* SkelMeshComp,
|
USkeletalMeshComponent* SkelMeshComp,
|
||||||
@@ -15,3 +106,73 @@ void UPS_AI_ConvAgent_BlueprintLibrary::SetPostProcessAnimBlueprint(
|
|||||||
|
|
||||||
SkelMeshComp->SetOverridePostProcessAnimBP(AnimBPClass);
|
SkelMeshComp->SetOverridePostProcessAnimBP(AnimBPClass);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── ElevenLabs API key (per-user, client-provided) ──────────────────────────
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_BlueprintLibrary::SetElevenLabsAPIKey(const FString& Key)
|
||||||
|
{
|
||||||
|
// 1) Persist to the SaveGame slot (writes to disk immediately).
|
||||||
|
UPS_AI_ConvAgent_SaveGame_ElevenLabs* Save = LoadElevenLabsKeySaveGame();
|
||||||
|
if (!Save)
|
||||||
|
{
|
||||||
|
Save = Cast<UPS_AI_ConvAgent_SaveGame_ElevenLabs>(
|
||||||
|
UGameplayStatics::CreateSaveGameObject(UPS_AI_ConvAgent_SaveGame_ElevenLabs::StaticClass()));
|
||||||
|
}
|
||||||
|
if (!Save)
|
||||||
|
{
|
||||||
|
UE_LOG(LogTemp, Warning, TEXT("[PS_AI_ConvAgent] SetElevenLabsAPIKey: failed to create SaveGame object."));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
int32 PlainLen = 0;
|
||||||
|
Save->EncryptedAPIKey = EncryptElevenLabsKey(Key, PlainLen);
|
||||||
|
Save->PlainKeyLength = PlainLen;
|
||||||
|
const bool bSaved = UGameplayStatics::SaveGameToSlot(
|
||||||
|
Save,
|
||||||
|
UPS_AI_ConvAgent_SaveGame_ElevenLabs::GetSlotName(),
|
||||||
|
UPS_AI_ConvAgent_SaveGame_ElevenLabs::GetUserIndex());
|
||||||
|
|
||||||
|
if (!bSaved)
|
||||||
|
{
|
||||||
|
UE_LOG(LogTemp, Warning, TEXT("[PS_AI_ConvAgent] SetElevenLabsAPIKey: SaveGameToSlot failed "
|
||||||
|
"(check write permissions at the install location)."));
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2) Update the in-memory cache so the new key takes effect immediately
|
||||||
|
// (next conversation / region probe / editor sync) without a restart.
|
||||||
|
GElevenLabsKeyCache = Key;
|
||||||
|
GElevenLabsKeyLoaded = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
FString UPS_AI_ConvAgent_BlueprintLibrary::GetElevenLabsAPIKey()
|
||||||
|
{
|
||||||
|
// Lazy-load once from the SaveGame (decrypting), then serve from the cache.
|
||||||
|
if (!GElevenLabsKeyLoaded)
|
||||||
|
{
|
||||||
|
GElevenLabsKeyLoaded = true;
|
||||||
|
if (const UPS_AI_ConvAgent_SaveGame_ElevenLabs* Save = LoadElevenLabsKeySaveGame())
|
||||||
|
{
|
||||||
|
GElevenLabsKeyCache = DecryptElevenLabsKey(Save->EncryptedAPIKey, Save->PlainKeyLength);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return GElevenLabsKeyCache;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool UPS_AI_ConvAgent_BlueprintLibrary::HasElevenLabsAPIKey()
|
||||||
|
{
|
||||||
|
return !GetElevenLabsAPIKey().IsEmpty();
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_BlueprintLibrary::ClearElevenLabsAPIKey()
|
||||||
|
{
|
||||||
|
const TCHAR* Slot = UPS_AI_ConvAgent_SaveGame_ElevenLabs::GetSlotName();
|
||||||
|
const int32 UserIndex = UPS_AI_ConvAgent_SaveGame_ElevenLabs::GetUserIndex();
|
||||||
|
if (UGameplayStatics::DoesSaveGameExist(Slot, UserIndex))
|
||||||
|
{
|
||||||
|
UGameplayStatics::DeleteGameInSlot(Slot, UserIndex);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clear the cache too so the running session stops using the old key.
|
||||||
|
GElevenLabsKeyCache.Empty();
|
||||||
|
GElevenLabsKeyLoaded = true;
|
||||||
|
}
|
||||||
|
|||||||
@@ -10,6 +10,7 @@
|
|||||||
#include "PS_AI_ConvAgent_InteractionSubsystem.h"
|
#include "PS_AI_ConvAgent_InteractionSubsystem.h"
|
||||||
#include "PS_AI_ConvAgent_InteractionComponent.h"
|
#include "PS_AI_ConvAgent_InteractionComponent.h"
|
||||||
#include "PS_AI_ConvAgent.h"
|
#include "PS_AI_ConvAgent.h"
|
||||||
|
#include "PS_AI_ConvAgent_BlueprintLibrary.h"
|
||||||
|
|
||||||
#include "Components/AudioComponent.h"
|
#include "Components/AudioComponent.h"
|
||||||
#include "Sound/SoundAttenuation.h"
|
#include "Sound/SoundAttenuation.h"
|
||||||
@@ -101,10 +102,9 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::TickComponent(float DeltaTime, ELevel
|
|||||||
{
|
{
|
||||||
Super::TickComponent(DeltaTime, TickType, ThisTickFunction);
|
Super::TickComponent(DeltaTime, TickType, ThisTickFunction);
|
||||||
|
|
||||||
// ── ForceDisableConversation blend-out monitoring ─────────────────────
|
// ── Conversation disable: blend-out monitoring ─────────────────────────
|
||||||
// After ForceDisableConversation(), sub-components are blending their
|
// After ForceDisableConversation(), sub-components blend their CurrentActiveAlpha
|
||||||
// CurrentActiveAlpha to 0. Once all are at (near) zero, fire OnReadyForAction
|
// to 0. Once all near zero (or timeout), clear the wait flag.
|
||||||
// so the game can start the physical action.
|
|
||||||
if (bWaitingForBlendOut)
|
if (bWaitingForBlendOut)
|
||||||
{
|
{
|
||||||
BlendOutElapsedTime += DeltaTime;
|
BlendOutElapsedTime += DeltaTime;
|
||||||
@@ -139,23 +139,33 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::TickComponent(float DeltaTime, ELevel
|
|||||||
bWaitingForBlendOut = false;
|
bWaitingForBlendOut = false;
|
||||||
BlendOutElapsedTime = 0.0f;
|
BlendOutElapsedTime = 0.0f;
|
||||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
||||||
TEXT("ForceDisableConversation: all components blended to neutral — firing OnReadyForAction for action '%s'."),
|
TEXT("ForceDisableConversation: all components blended to neutral."));
|
||||||
*PendingActionName);
|
|
||||||
OnReadyForAction.Broadcast(PendingActionName);
|
|
||||||
}
|
}
|
||||||
else if (bTimedOut)
|
else if (bTimedOut)
|
||||||
{
|
{
|
||||||
// Safety timeout — some component didn't reach alpha 0 in time.
|
// Safety timeout — some component didn't reach alpha 0 in time.
|
||||||
// Fire OnReadyForAction anyway to avoid blocking the game action forever.
|
|
||||||
bWaitingForBlendOut = false;
|
bWaitingForBlendOut = false;
|
||||||
BlendOutElapsedTime = 0.0f;
|
BlendOutElapsedTime = 0.0f;
|
||||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
|
||||||
TEXT("ForceDisableConversation: blend-out timed out after %.1fs — not all components at neutral. Firing OnReadyForAction for action '%s' anyway."),
|
TEXT("ForceDisableConversation: blend-out timed out after %.1fs — not all components at neutral."),
|
||||||
BlendOutTimeoutSeconds, *PendingActionName);
|
BlendOutTimeoutSeconds);
|
||||||
OnReadyForAction.Broadcast(PendingActionName);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Action flow: fire OnReadyForAction when ready ──────────────────────
|
||||||
|
// An action was requested via OnAgentActionRequested. Fire OnReadyForAction
|
||||||
|
// as soon as the agent is ready:
|
||||||
|
// - If ForceDisableConversation was called, wait for blend-out to complete.
|
||||||
|
// - Otherwise, fire on the next tick (quasi-immediate).
|
||||||
|
if (!PendingActionName.IsEmpty() && !bWaitingForBlendOut)
|
||||||
|
{
|
||||||
|
const FString ActionToFire = PendingActionName;
|
||||||
|
PendingActionName.Empty();
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
||||||
|
TEXT("OnReadyForAction: firing for action '%s'."), *ActionToFire);
|
||||||
|
OnReadyForAction.Broadcast(ActionToFire);
|
||||||
|
}
|
||||||
|
|
||||||
// Response timeout: if the server hasn't started generating within ResponseTimeoutSeconds
|
// Response timeout: if the server hasn't started generating within ResponseTimeoutSeconds
|
||||||
// after the user stopped speaking, notify Blueprint so it can react (e.g. show "try again").
|
// after the user stopped speaking, notify Blueprint so it can react (e.g. show "try again").
|
||||||
if (bWaitingForAgentResponse && ResponseTimeoutSeconds > 0.0f && TurnEndTime > 0.0)
|
if (bWaitingForAgentResponse && ResponseTimeoutSeconds > 0.0f && TurnEndTime > 0.0)
|
||||||
@@ -400,7 +410,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::TickComponent(float DeltaTime, ELevel
|
|||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
void UPS_AI_ConvAgent_ElevenLabsComponent::StartConversation()
|
void UPS_AI_ConvAgent_ElevenLabsComponent::StartConversation()
|
||||||
{
|
{
|
||||||
if (bConversationDisabledByAction)
|
if (bConversationForceDisabled)
|
||||||
{
|
{
|
||||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
|
||||||
TEXT("StartConversation: blocked — conversation disabled by ForceDisableConversation(). Call ForceEnableConversation() first."));
|
TEXT("StartConversation: blocked — conversation disabled by ForceDisableConversation(). Call ForceEnableConversation() first."));
|
||||||
@@ -492,6 +502,9 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StartConversation_Internal()
|
|||||||
|
|
||||||
void UPS_AI_ConvAgent_ElevenLabsComponent::EndConversation()
|
void UPS_AI_ConvAgent_ElevenLabsComponent::EndConversation()
|
||||||
{
|
{
|
||||||
|
// Clear any pending listen request so it doesn't leak across sessions.
|
||||||
|
bPendingStartListening = false;
|
||||||
|
|
||||||
if (GetOwnerRole() == ROLE_Authority)
|
if (GetOwnerRole() == ROLE_Authority)
|
||||||
{
|
{
|
||||||
// Standalone / listen-server: leave via the local player controller.
|
// Standalone / listen-server: leave via the local player controller.
|
||||||
@@ -525,7 +538,12 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StartListening()
|
|||||||
const bool bEffectivelyConnected = IsConnected() || (GetOwnerRole() != ROLE_Authority && bNetIsConversing);
|
const bool bEffectivelyConnected = IsConnected() || (GetOwnerRole() != ROLE_Authority && bNetIsConversing);
|
||||||
if (!bEffectivelyConnected)
|
if (!bEffectivelyConnected)
|
||||||
{
|
{
|
||||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning, TEXT("StartListening: not connected (IsConnected=%s bNetIsConversing=%s Role=%d)."),
|
// Not ready yet — defer until HandleConnected fires.
|
||||||
|
// Prevents silent audio loss when StartListening is called synchronously
|
||||||
|
// after StartConversation (async WebSocket handshake).
|
||||||
|
bPendingStartListening = true;
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
||||||
|
TEXT("StartListening deferred: WebSocket not connected yet (IsConnected=%s bNetIsConversing=%s Role=%d). Will open mic on connect."),
|
||||||
IsConnected() ? TEXT("true") : TEXT("false"),
|
IsConnected() ? TEXT("true") : TEXT("false"),
|
||||||
bNetIsConversing ? TEXT("true") : TEXT("false"),
|
bNetIsConversing ? TEXT("true") : TEXT("false"),
|
||||||
static_cast<int32>(GetOwnerRole()));
|
static_cast<int32>(GetOwnerRole()));
|
||||||
@@ -621,6 +639,10 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StartListening()
|
|||||||
|
|
||||||
void UPS_AI_ConvAgent_ElevenLabsComponent::StopListening()
|
void UPS_AI_ConvAgent_ElevenLabsComponent::StopListening()
|
||||||
{
|
{
|
||||||
|
// Also cancel any pending-start request (e.g. StartListening was deferred
|
||||||
|
// before WS connect and the user now wants to cancel).
|
||||||
|
bPendingStartListening = false;
|
||||||
|
|
||||||
if (!bIsListening) return;
|
if (!bIsListening) return;
|
||||||
bIsListening = false;
|
bIsListening = false;
|
||||||
|
|
||||||
@@ -717,6 +739,23 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StopListening()
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_ElevenLabsComponent::SendContextualUpdate(const FString& Text)
|
||||||
|
{
|
||||||
|
// Server-authoritative: the WebSocket lives on the server. The agent-to-agent
|
||||||
|
// orchestrator (server) uses this to reframe the agents when the player takes over.
|
||||||
|
if (GetOwnerRole() == ROLE_Authority)
|
||||||
|
{
|
||||||
|
if (IsConnected() && WebSocketProxy)
|
||||||
|
{
|
||||||
|
WebSocketProxy->SendContextualUpdate(Text);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning, TEXT("SendContextualUpdate: not connected."));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
void UPS_AI_ConvAgent_ElevenLabsComponent::SendTextMessage(const FString& Text)
|
void UPS_AI_ConvAgent_ElevenLabsComponent::SendTextMessage(const FString& Text)
|
||||||
{
|
{
|
||||||
if (GetOwnerRole() == ROLE_Authority)
|
if (GetOwnerRole() == ROLE_Authority)
|
||||||
@@ -781,21 +820,19 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::InterruptAgent()
|
|||||||
// ForceDisableConversation / ForceEnableConversation
|
// ForceDisableConversation / ForceEnableConversation
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
void UPS_AI_ConvAgent_ElevenLabsComponent::ForceDisableConversation(const FString& ActionName)
|
void UPS_AI_ConvAgent_ElevenLabsComponent::ForceDisableConversation()
|
||||||
{
|
{
|
||||||
if (bConversationDisabledByAction)
|
if (bConversationForceDisabled)
|
||||||
{
|
{
|
||||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
|
||||||
TEXT("ForceDisableConversation: already disabled (pending action: %s)."), *PendingActionName);
|
TEXT("ForceDisableConversation: already disabled."));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
||||||
TEXT("ForceDisableConversation: shutting down conversation for action '%s', blending to neutral."), *ActionName);
|
TEXT("ForceDisableConversation: shutting down conversation, blending to neutral."));
|
||||||
|
|
||||||
PendingActionName = ActionName;
|
bConversationForceDisabled = true;
|
||||||
|
|
||||||
bConversationDisabledByAction = true;
|
|
||||||
bWaitingForBlendOut = true;
|
bWaitingForBlendOut = true;
|
||||||
BlendOutElapsedTime = 0.0f;
|
BlendOutElapsedTime = 0.0f;
|
||||||
|
|
||||||
@@ -856,7 +893,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ForceDisableConversation(const FStrin
|
|||||||
|
|
||||||
void UPS_AI_ConvAgent_ElevenLabsComponent::ForceEnableConversation()
|
void UPS_AI_ConvAgent_ElevenLabsComponent::ForceEnableConversation()
|
||||||
{
|
{
|
||||||
if (!bConversationDisabledByAction)
|
if (!bConversationForceDisabled)
|
||||||
{
|
{
|
||||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
|
||||||
TEXT("ForceEnableConversation: not currently disabled."));
|
TEXT("ForceEnableConversation: not currently disabled."));
|
||||||
@@ -866,7 +903,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ForceEnableConversation()
|
|||||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
||||||
TEXT("ForceEnableConversation: conversation re-enabled."));
|
TEXT("ForceEnableConversation: conversation re-enabled."));
|
||||||
|
|
||||||
bConversationDisabledByAction = false;
|
bConversationForceDisabled = false;
|
||||||
bWaitingForBlendOut = false;
|
bWaitingForBlendOut = false;
|
||||||
BlendOutElapsedTime = 0.0f;
|
BlendOutElapsedTime = 0.0f;
|
||||||
|
|
||||||
@@ -880,6 +917,27 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::FeedExternalAudio(const TArray<float>
|
|||||||
// Same logic as OnMicrophoneDataCaptured but called from an external source
|
// Same logic as OnMicrophoneDataCaptured but called from an external source
|
||||||
// (e.g. InteractionComponent on the pawn) instead of a local mic component.
|
// (e.g. InteractionComponent on the pawn) instead of a local mic component.
|
||||||
// On server: check local WebSocket. On client: check replicated conversation state.
|
// On server: check local WebSocket. On client: check replicated conversation state.
|
||||||
|
|
||||||
|
// Local voice-onset detection: broadcast on the FIRST voiced frame BEFORE the
|
||||||
|
// listening/echo gates below, so an agent-to-agent dialogue can be interrupted the
|
||||||
|
// instant the player speaks — no "listening" state and no ElevenLabs STT round-trip
|
||||||
|
// needed. Debounced to fire once per utterance. (OnAudioCaptured is marshaled to the
|
||||||
|
// game thread by the mic component, so this runs on the game thread — safe to broadcast.)
|
||||||
|
{
|
||||||
|
float OnsetSumSq = 0.0f;
|
||||||
|
for (const float S : FloatPCM) { OnsetSumSq += S * S; }
|
||||||
|
const float OnsetRms = FMath::Sqrt(OnsetSumSq / FMath::Max(1, FloatPCM.Num()));
|
||||||
|
if (OnsetRms >= 0.02f)
|
||||||
|
{
|
||||||
|
const double NowSec = FPlatformTime::Seconds();
|
||||||
|
if (NowSec - LastUserVoiceOnset > 0.5)
|
||||||
|
{
|
||||||
|
LastUserVoiceOnset = NowSec;
|
||||||
|
OnUserVoiceDetected.Broadcast();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const bool bCanSend = (GetOwnerRole() == ROLE_Authority) ? IsConnected() : bNetIsConversing;
|
const bool bCanSend = (GetOwnerRole() == ROLE_Authority) ? IsConnected() : bNetIsConversing;
|
||||||
if (!bCanSend || !bIsListening) return;
|
if (!bCanSend || !bIsListening) return;
|
||||||
|
|
||||||
@@ -889,13 +947,15 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::FeedExternalAudio(const TArray<float>
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Network silence gate ────────────────────────────────────────────────
|
// ── Network silence gate (clients only) ─────────────────────────────────
|
||||||
// On clients the mic audio is sent via unreliable RPCs (~3200 bytes every
|
// On clients the mic audio is sent via unreliable RPCs (~256 Kbits/s).
|
||||||
// 100ms = ~256 Kbits/s). Sending silence saturates the connection and
|
// Streaming silence the whole session saturates the connection and starves
|
||||||
// starves movement replication, causing chaotic teleporting.
|
// movement replication (teleporting). But ElevenLabs Server VAD needs the
|
||||||
// Skip silent chunks on clients only — the server path uses a local
|
// silence AFTER speech to detect end-of-turn — dropping it entirely means the
|
||||||
// WebSocket and doesn't touch the network, so it keeps the full stream
|
// agent hears the words but never the pause, so it never replies.
|
||||||
// for proper ElevenLabs VAD (voice-activity detection).
|
// Compromise: keep streaming a short silence "tail" after the last voiced
|
||||||
|
// chunk (MicSilenceTailSeconds), then drop silence during long idle gaps.
|
||||||
|
// The authority path is untouched — it always streams the full mic.
|
||||||
if (GetOwnerRole() != ROLE_Authority)
|
if (GetOwnerRole() != ROLE_Authority)
|
||||||
{
|
{
|
||||||
float SumSq = 0.0f;
|
float SumSq = 0.0f;
|
||||||
@@ -904,9 +964,22 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::FeedExternalAudio(const TArray<float>
|
|||||||
SumSq += Sample * Sample;
|
SumSq += Sample * Sample;
|
||||||
}
|
}
|
||||||
const float Rms = FMath::Sqrt(SumSq / FMath::Max(1, FloatPCM.Num()));
|
const float Rms = FMath::Sqrt(SumSq / FMath::Max(1, FloatPCM.Num()));
|
||||||
if (Rms < 0.005f) // ~-46 dBFS — well below any speech level
|
|
||||||
|
const double NowSec = FPlatformTime::Seconds();
|
||||||
|
if (Rms >= 0.005f) // ~-46 dBFS — above this is voiced speech.
|
||||||
{
|
{
|
||||||
return;
|
LastMicVoicedTime = NowSec; // Voiced: always send, remember when.
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// Silence: send only within the tail window after the last voiced
|
||||||
|
// chunk (so VAD sees end-of-turn); drop once the tail has elapsed.
|
||||||
|
const bool bWithinTail = (LastMicVoicedTime > 0.0)
|
||||||
|
&& (NowSec - LastMicVoicedTime) <= MicSilenceTailSeconds;
|
||||||
|
if (!bWithinTail)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1069,7 +1142,14 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleConnected(const FPS_AI_ConvAgen
|
|||||||
{
|
{
|
||||||
StartListening();
|
StartListening();
|
||||||
}
|
}
|
||||||
|
// Consume any pending listen request deferred before WebSocket was ready.
|
||||||
|
else if (bPendingStartListening)
|
||||||
|
{
|
||||||
|
bPendingStartListening = false;
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
||||||
|
TEXT("HandleConnected: consuming pending StartListening request."));
|
||||||
|
StartListening();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void UPS_AI_ConvAgent_ElevenLabsComponent::HandleDisconnected(int32 StatusCode, const FString& Reason)
|
void UPS_AI_ConvAgent_ElevenLabsComponent::HandleDisconnected(int32 StatusCode, const FString& Reason)
|
||||||
@@ -1182,41 +1262,123 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleAudioReceived(const TArray<uint
|
|||||||
// Network: broadcast audio to all clients.
|
// Network: broadcast audio to all clients.
|
||||||
if (GetOwnerRole() == ROLE_Authority)
|
if (GetOwnerRole() == ROLE_Authority)
|
||||||
{
|
{
|
||||||
|
// Net-audio size debug: toggle at runtime with
|
||||||
|
// ps.ai.ConvAgent.Debug.ElevenLabs 1 (or set bDebug on the component).
|
||||||
|
const int32 NetDbgCVar = CVarDebugElevenLabs.GetValueOnGameThread();
|
||||||
|
const bool bNetAudioDebug = (NetDbgCVar >= 0) ? (NetDbgCVar > 0) : bDebug;
|
||||||
|
|
||||||
if (OpusEncoder.IsValid())
|
if (OpusEncoder.IsValid())
|
||||||
{
|
{
|
||||||
// Opus path: compress then send.
|
// Opus path: compress then send, in sub-blocks.
|
||||||
uint32 CompressedSize = static_cast<uint32>(OpusWorkBuffer.Num());
|
//
|
||||||
OpusEncoder->Encode(PCMData.GetData(), PCMData.Num(),
|
// A whole ElevenLabs message can be several seconds. Two hard limits
|
||||||
OpusWorkBuffer.GetData(), CompressedSize);
|
// force us to split it before each Encode()/RPC:
|
||||||
|
// 1. UE's Opus encoder caps ONE packet at 76 frames (~1.52s) — frames
|
||||||
|
// beyond that are silently dropped at encode (VoiceCodecOpus.cpp).
|
||||||
|
// 2. The client decode buffer only holds ~44 frames (~0.88s) before
|
||||||
|
// the engine decoder aborts with "Decompression buffer too small"
|
||||||
|
// and drops the packet tail (MulticastReceiveAgentAudio_Implementation).
|
||||||
|
// 40 frames (25600 bytes, 0.8s) stays safely under BOTH, so nothing is
|
||||||
|
// truncated on server or client. It also keeps each Unreliable RPC small.
|
||||||
|
static constexpr int32 BytesPerFrame =
|
||||||
|
(PS_AI_ConvAgent_Audio_ElevenLabs::SampleRate / 50)
|
||||||
|
* PS_AI_ConvAgent_Audio_ElevenLabs::Channels
|
||||||
|
* static_cast<int32>(sizeof(int16)); // 640 at 16kHz mono
|
||||||
|
static constexpr int32 MaxOpusChunkBytes = 40 * BytesPerFrame; // 25600 = 0.8s
|
||||||
|
|
||||||
if (CompressedSize > 0)
|
// Carry the sub-frame remainder from the PREVIOUS message so no samples
|
||||||
|
// are dropped at message boundaries. Opus encodes whole 640-byte frames
|
||||||
|
// only; dropping each message's <1-frame tail left a ~10-16ms hole and a
|
||||||
|
// waveform discontinuity at every message splice → audible pops on clients
|
||||||
|
// (the host plays raw PCM directly, so standalone/host never had it).
|
||||||
|
// Prepend the leftover, encode only whole frames, stash the new tail.
|
||||||
|
if (!bAgentSpeaking)
|
||||||
{
|
{
|
||||||
TArray<uint8> CompressedData;
|
// First message of a new turn — don't glue on the previous turn's tail.
|
||||||
CompressedData.Append(OpusWorkBuffer.GetData(), CompressedSize);
|
AgentAudioEncodeLeftover.Reset();
|
||||||
MulticastReceiveAgentAudio(CompressedData);
|
}
|
||||||
|
TArray<uint8> ContinuousPCM = MoveTemp(AgentAudioEncodeLeftover);
|
||||||
|
ContinuousPCM.Append(PCMData);
|
||||||
|
|
||||||
|
const int32 UsableBytes = (ContinuousPCM.Num() / BytesPerFrame) * BytesPerFrame;
|
||||||
|
if (ContinuousPCM.Num() > UsableBytes)
|
||||||
|
{
|
||||||
|
AgentAudioEncodeLeftover.Append(
|
||||||
|
ContinuousPCM.GetData() + UsableBytes, ContinuousPCM.Num() - UsableBytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
int32 Offset = 0;
|
||||||
|
while (Offset < UsableBytes)
|
||||||
|
{
|
||||||
|
const int32 BlockBytes = FMath::Min(MaxOpusChunkBytes, UsableBytes - Offset);
|
||||||
|
const bool bLast = (Offset + BlockBytes >= UsableBytes);
|
||||||
|
|
||||||
|
uint32 CompressedSize = static_cast<uint32>(OpusWorkBuffer.Num());
|
||||||
|
OpusEncoder->Encode(ContinuousPCM.GetData() + Offset, BlockBytes,
|
||||||
|
OpusWorkBuffer.GetData(), CompressedSize);
|
||||||
|
|
||||||
|
if (CompressedSize > 0)
|
||||||
|
{
|
||||||
|
TArray<uint8> CompressedData;
|
||||||
|
CompressedData.Append(OpusWorkBuffer.GetData(), CompressedSize);
|
||||||
|
|
||||||
|
if (bNetAudioDebug)
|
||||||
|
{
|
||||||
|
const float Ratio = static_cast<float>(BlockBytes) / static_cast<float>(CompressedSize);
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
||||||
|
TEXT("[NET-SRV] Opus audio chunk: %u bytes sent (raw %d → %.1fx compression, last=%d)."),
|
||||||
|
CompressedSize, BlockBytes, Ratio, bLast ? 1 : 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
MulticastReceiveAgentAudio(CompressedData, bLast);
|
||||||
|
}
|
||||||
|
|
||||||
|
Offset += BlockBytes;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
// Fallback: send raw PCM (no compression). ~32 KB/s at 16kHz 16-bit mono.
|
// Fallback: Opus init failed → send raw PCM (no compression, ~32 KB/s).
|
||||||
// Fine for LAN; revisit with proper Opus if internet play is needed.
|
// This should not happen in the working config — it means [Voice]
|
||||||
// UE5 limits replicated TArrays to 65535 elements, so we must chunk.
|
// bEnabled=true is missing in DefaultEngine.ini. Warn on every message
|
||||||
static bool bWarnedOnce = false;
|
// so it is loud, not hidden behind a one-shot flag.
|
||||||
if (!bWarnedOnce)
|
// The RPC is now Reliable, so we must protect the reliable buffer (512
|
||||||
|
// bunches): each ~4 KB chunk ≈ ~4 partial bunches. Hard-cap one message
|
||||||
|
// at MaxFallbackBytes and drop the tail so a single message can never
|
||||||
|
// approach the buffer and trigger a reliable-buffer overflow (the exact
|
||||||
|
// cause of the old VR disconnect).
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
|
||||||
|
TEXT("[NET-SRV] Opus encoder unavailable — sending raw PCM (no compression). "
|
||||||
|
"Set [Voice] bEnabled=true in DefaultEngine.ini to restore Opus."));
|
||||||
|
|
||||||
|
static constexpr int32 MaxChunkBytes = 4000;
|
||||||
|
static constexpr int32 MaxFallbackBytes = 320000; // ~80 reliable chunks/message, well under the 512-bunch buffer
|
||||||
|
|
||||||
|
const int32 SendableBytes = FMath::Min(PCMData.Num(), MaxFallbackBytes);
|
||||||
|
if (PCMData.Num() > MaxFallbackBytes)
|
||||||
{
|
{
|
||||||
bWarnedOnce = true;
|
|
||||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
|
||||||
TEXT("[NET-SRV] Opus encoder unavailable — sending raw PCM in chunks (no compression)."));
|
TEXT("[NET-SRV] Raw-PCM fallback capped at %d bytes — dropping %d-byte tail "
|
||||||
|
"to protect the reliable buffer."),
|
||||||
|
MaxFallbackBytes, PCMData.Num() - MaxFallbackBytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
static constexpr int32 MaxChunkBytes = 32000; // 1s of 16kHz 16-bit mono, well under 65535 limit
|
|
||||||
int32 Offset = 0;
|
int32 Offset = 0;
|
||||||
while (Offset < PCMData.Num())
|
while (Offset < SendableBytes)
|
||||||
{
|
{
|
||||||
const int32 ChunkSize = FMath::Min(MaxChunkBytes, PCMData.Num() - Offset);
|
const int32 ChunkSize = FMath::Min(MaxChunkBytes, SendableBytes - Offset);
|
||||||
|
const bool bLast = (Offset + ChunkSize >= SendableBytes);
|
||||||
TArray<uint8> Chunk;
|
TArray<uint8> Chunk;
|
||||||
Chunk.Append(PCMData.GetData() + Offset, ChunkSize);
|
Chunk.Append(PCMData.GetData() + Offset, ChunkSize);
|
||||||
MulticastReceiveAgentAudio(Chunk);
|
|
||||||
|
if (bNetAudioDebug)
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
|
||||||
|
TEXT("[NET-SRV] RAW PCM chunk: %d bytes sent (NO compression — Opus disabled, last=%d)."),
|
||||||
|
ChunkSize, bLast ? 1 : 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
MulticastReceiveAgentAudio(Chunk, bLast);
|
||||||
Offset += ChunkSize;
|
Offset += ChunkSize;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1451,6 +1613,10 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleClientToolCall(const FPS_AI_Con
|
|||||||
TEXT("[T+%.2fs] Agent action requested: %s"), T, *ActionName);
|
TEXT("[T+%.2fs] Agent action requested: %s"), T, *ActionName);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Track the pending action so OnReadyForAction can fire with the same name.
|
||||||
|
// If the user synchronously calls ForceDisableConversation() in their handler,
|
||||||
|
// OnReadyForAction will wait for blend-out. Otherwise it fires on next tick.
|
||||||
|
PendingActionName = ActionName;
|
||||||
OnAgentActionRequested.Broadcast(ActionName);
|
OnAgentActionRequested.Broadcast(ActionName);
|
||||||
|
|
||||||
// Auto-respond so the agent can continue speaking.
|
// Auto-respond so the agent can continue speaking.
|
||||||
@@ -1604,6 +1770,42 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::OnProceduralUnderflow(
|
|||||||
|
|
||||||
void UPS_AI_ConvAgent_ElevenLabsComponent::EnqueueAgentAudio(const TArray<uint8>& PCMData)
|
void UPS_AI_ConvAgent_ElevenLabsComponent::EnqueueAgentAudio(const TArray<uint8>& PCMData)
|
||||||
{
|
{
|
||||||
|
// Debug: flag a waveform discontinuity at the splice between the previous
|
||||||
|
// enqueued block and this one — the signature of the audio pops. The host's
|
||||||
|
// raw-PCM stream is continuous so its count should stay ~0; if a client's
|
||||||
|
// count climbs at message cadence, a splice problem remains.
|
||||||
|
{
|
||||||
|
const int32 DbgVal = CVarDebugElevenLabs.GetValueOnGameThread();
|
||||||
|
const bool bAudioDbg = (DbgVal >= 0) ? (DbgVal > 0) : bDebug;
|
||||||
|
if (bAudioDbg && PCMData.Num() >= static_cast<int32>(sizeof(int16)))
|
||||||
|
{
|
||||||
|
const int16* Samples = reinterpret_cast<const int16*>(PCMData.GetData());
|
||||||
|
const int32 NumSamples = PCMData.Num() / static_cast<int32>(sizeof(int16));
|
||||||
|
if (bHasLastEnqueuedSample)
|
||||||
|
{
|
||||||
|
const int32 Delta = FMath::Abs(static_cast<int32>(Samples[0]) - static_cast<int32>(LastEnqueuedSample));
|
||||||
|
static constexpr int32 DiscontinuityThreshold = 3000; // |delta| on the int16 scale
|
||||||
|
if (Delta > DiscontinuityThreshold)
|
||||||
|
{
|
||||||
|
++AudioSpliceDiscontinuityCount;
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
|
||||||
|
TEXT("[AUDIO-DBG] Role=%d splice discontinuity: |delta|=%d (prev=%d first=%d) count=%d — pop likely here."),
|
||||||
|
static_cast<int32>(GetOwnerRole()), Delta,
|
||||||
|
static_cast<int32>(LastEnqueuedSample), static_cast<int32>(Samples[0]),
|
||||||
|
AudioSpliceDiscontinuityCount);
|
||||||
|
}
|
||||||
|
else if (DbgVal >= 2)
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
||||||
|
TEXT("[AUDIO-DBG] Role=%d splice ok: |delta|=%d"),
|
||||||
|
static_cast<int32>(GetOwnerRole()), Delta);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
LastEnqueuedSample = Samples[NumSamples - 1];
|
||||||
|
bHasLastEnqueuedSample = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
FScopeLock Lock(&AudioQueueLock);
|
FScopeLock Lock(&AudioQueueLock);
|
||||||
AudioQueue.Append(PCMData);
|
AudioQueue.Append(PCMData);
|
||||||
@@ -1778,6 +1980,9 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StopAgentAudio()
|
|||||||
bool bWasSpeaking = false;
|
bool bWasSpeaking = false;
|
||||||
double Now = 0.0;
|
double Now = 0.0;
|
||||||
bPreBuffering = false; // Clear pre-buffer state on stop.
|
bPreBuffering = false; // Clear pre-buffer state on stop.
|
||||||
|
ClientReassemblyPCM.Reset(); // Client: drop any half-received message so it can't leak into the next turn.
|
||||||
|
AgentAudioEncodeLeftover.Reset(); // Server: drop the carried sub-frame tail so it can't glue onto the next turn.
|
||||||
|
bHasLastEnqueuedSample = false; // Debug: don't count the inter-turn splice as a discontinuity.
|
||||||
{
|
{
|
||||||
FScopeLock Lock(&AudioQueueLock);
|
FScopeLock Lock(&AudioQueueLock);
|
||||||
AudioQueue.Empty();
|
AudioQueue.Empty();
|
||||||
@@ -1987,6 +2192,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::GetLifetimeReplicatedProps(
|
|||||||
DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, bNetIsConversing);
|
DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, bNetIsConversing);
|
||||||
DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, NetConnectedPawns);
|
DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, NetConnectedPawns);
|
||||||
DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, NetActiveSpeakerPawn);
|
DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, NetActiveSpeakerPawn);
|
||||||
|
DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, GazeOverrideTarget);
|
||||||
DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, CurrentEmotion);
|
DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, CurrentEmotion);
|
||||||
DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, CurrentEmotionIntensity);
|
DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, CurrentEmotionIntensity);
|
||||||
}
|
}
|
||||||
@@ -2010,16 +2216,33 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::OnRep_ConversationState()
|
|||||||
// on the local pawn, but remote clients never run that code path.
|
// on the local pawn, but remote clients never run that code path.
|
||||||
if (UPS_AI_ConvAgent_GazeComponent* Gaze = Owner->FindComponentByClass<UPS_AI_ConvAgent_GazeComponent>())
|
if (UPS_AI_ConvAgent_GazeComponent* Gaze = Owner->FindComponentByClass<UPS_AI_ConvAgent_GazeComponent>())
|
||||||
{
|
{
|
||||||
if (bNetIsConversing && NetConnectedPawns.Num() > 0)
|
if (GazeOverrideTarget)
|
||||||
|
{
|
||||||
|
// Override (agent-to-agent dialogue): look at the fixed actor.
|
||||||
|
Gaze->bActive = true;
|
||||||
|
if (Gaze->TargetActor != GazeOverrideTarget)
|
||||||
|
{
|
||||||
|
Gaze->TargetActor = GazeOverrideTarget;
|
||||||
|
Gaze->ResetBodyTarget();
|
||||||
|
}
|
||||||
|
Gaze->bEnableBodyTracking = true;
|
||||||
|
}
|
||||||
|
else if (bNetIsConversing && NetConnectedPawns.Num() > 0)
|
||||||
{
|
{
|
||||||
Gaze->bActive = true;
|
Gaze->bActive = true;
|
||||||
// Use active speaker if set, otherwise first connected pawn.
|
// Use active speaker if set, otherwise first connected pawn.
|
||||||
|
// GazeTarget can be null on a late joiner if the replicated pawn
|
||||||
|
// reference hasn't resolved yet (actor channel not open). Skip until
|
||||||
|
// it resolves — OnRep fires again once the reference is patched in.
|
||||||
APawn* GazeTarget = NetActiveSpeakerPawn ? NetActiveSpeakerPawn : NetConnectedPawns.Last();
|
APawn* GazeTarget = NetActiveSpeakerPawn ? NetActiveSpeakerPawn : NetConnectedPawns.Last();
|
||||||
Gaze->TargetActor = GazeTarget;
|
if (GazeTarget)
|
||||||
Gaze->ResetBodyTarget();
|
{
|
||||||
Gaze->bEnableBodyTracking = true;
|
Gaze->TargetActor = GazeTarget;
|
||||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
Gaze->ResetBodyTarget();
|
||||||
TEXT("[NET-REP] Gaze ACTIVATED, TargetActor set to %s"), *GazeTarget->GetName());
|
Gaze->bEnableBodyTracking = true;
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
||||||
|
TEXT("[NET-REP] Gaze ACTIVATED, TargetActor set to %s"), *GazeTarget->GetName());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@@ -2053,6 +2276,25 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::OnRep_ConversationState()
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Client: (re)start listening once the conversation is confirmed live ──
|
||||||
|
// StartListening() called early on the client defers (bPendingStartListening)
|
||||||
|
// because bNetIsConversing wasn't replicated yet. The ClientConversationStarted
|
||||||
|
// relay RPC may also race ahead of this replication. OnRep is the reliable
|
||||||
|
// "conversation is live" signal, so consume the deferred listen here — this
|
||||||
|
// closes the race regardless of RPC/replication ordering. Gated on
|
||||||
|
// IsLocalPlayerConversating() so only the joining client opens its mic path.
|
||||||
|
if (GetOwnerRole() != ROLE_Authority && bNetIsConversing && IsLocalPlayerConversating())
|
||||||
|
{
|
||||||
|
if (bPendingStartListening
|
||||||
|
|| (bAutoStartListening && TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server))
|
||||||
|
{
|
||||||
|
bPendingStartListening = false;
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
||||||
|
TEXT("[NET-REP] Conversation live on client — starting listening (mic uplink)."));
|
||||||
|
StartListening();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (!bNetIsConversing)
|
if (!bNetIsConversing)
|
||||||
{
|
{
|
||||||
// Conversation ended on server — clean up local playback.
|
// Conversation ended on server — clean up local playback.
|
||||||
@@ -2069,16 +2311,22 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::OnRep_ActiveSpeaker()
|
|||||||
AActor* Owner = GetOwner();
|
AActor* Owner = GetOwner();
|
||||||
if (!Owner) return;
|
if (!Owner) return;
|
||||||
|
|
||||||
if (auto* Gaze = Owner->FindComponentByClass<UPS_AI_ConvAgent_GazeComponent>())
|
// Guard: during an agent-to-agent dialogue the gaze is pinned to the other
|
||||||
|
// agent (GazeOverrideTarget) — a replicated active-speaker change (e.g. a
|
||||||
|
// player joining/speaking) must NOT yank the eyes back onto the player.
|
||||||
|
if (!GazeOverrideTarget)
|
||||||
{
|
{
|
||||||
if (NetActiveSpeakerPawn)
|
if (auto* Gaze = Owner->FindComponentByClass<UPS_AI_ConvAgent_GazeComponent>())
|
||||||
{
|
{
|
||||||
Gaze->TargetActor = NetActiveSpeakerPawn;
|
if (NetActiveSpeakerPawn)
|
||||||
Gaze->ResetBodyTarget();
|
{
|
||||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
Gaze->TargetActor = NetActiveSpeakerPawn;
|
||||||
TEXT("[NET-REP] ActiveSpeaker changed to %s"), *NetActiveSpeakerPawn->GetName());
|
Gaze->ResetBodyTarget();
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
||||||
|
TEXT("[NET-REP] ActiveSpeaker changed to %s"), *NetActiveSpeakerPawn->GetName());
|
||||||
|
}
|
||||||
|
// Don't clear gaze when null — keep looking at last speaker.
|
||||||
}
|
}
|
||||||
// Don't clear gaze when null — keep looking at last speaker.
|
|
||||||
}
|
}
|
||||||
|
|
||||||
OnActiveSpeakerChanged.Broadcast(NetActiveSpeakerPawn, nullptr);
|
OnActiveSpeakerChanged.Broadcast(NetActiveSpeakerPawn, nullptr);
|
||||||
@@ -2100,7 +2348,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ServerJoinConversation_Implementation
|
|||||||
if (!Pawn) return;
|
if (!Pawn) return;
|
||||||
|
|
||||||
// Block join while force-disabled (ForceDisableConversation active).
|
// Block join while force-disabled (ForceDisableConversation active).
|
||||||
if (bConversationDisabledByAction)
|
if (bConversationForceDisabled)
|
||||||
{
|
{
|
||||||
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
||||||
TEXT("[NET] ServerJoinConversation: blocked — conversation disabled by ForceDisableConversation()."));
|
TEXT("[NET] ServerJoinConversation: blocked — conversation disabled by ForceDisableConversation()."));
|
||||||
@@ -2192,8 +2440,11 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ServerLeaveConversation_Implementatio
|
|||||||
SetActiveSpeaker(NetConnectedPawns.Num() > 0 ? NetConnectedPawns[0] : nullptr);
|
SetActiveSpeaker(NetConnectedPawns.Num() > 0 ? NetConnectedPawns[0] : nullptr);
|
||||||
}
|
}
|
||||||
|
|
||||||
// If no players left, fully end the conversation.
|
// If no players left, fully end the conversation — UNLESS the agent-to-agent
|
||||||
if (NetConnectedPawns.Num() == 0)
|
// orchestrator owns it (GazeOverrideTarget set). A player merely WATCHING the
|
||||||
|
// dialogue joins/leaves just by looking around; that must NOT close the agent's
|
||||||
|
// WebSocket, stop its audio, or deactivate its lip sync. Keep the dialogue alive.
|
||||||
|
if (NetConnectedPawns.Num() == 0 && !GazeOverrideTarget)
|
||||||
{
|
{
|
||||||
// Cancel any pending reconnection (ephemeral mode only).
|
// Cancel any pending reconnection (ephemeral mode only).
|
||||||
if (!bPersistentSession)
|
if (!bPersistentSession)
|
||||||
@@ -2229,7 +2480,8 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ServerLeaveConversation_Implementatio
|
|||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
// Other players still connected — update gaze.
|
// Players still connected, OR an orchestrated agent-to-agent dialogue owns
|
||||||
|
// this conversation (0 players but GazeOverrideTarget set) — keep it running.
|
||||||
ApplyConversationGaze();
|
ApplyConversationGaze();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2273,6 +2525,38 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ServerSendMicAudioFromPlayer(
|
|||||||
{
|
{
|
||||||
if (!WebSocketProxy || !WebSocketProxy->IsConnected()) return;
|
if (!WebSocketProxy || !WebSocketProxy->IsConnected()) return;
|
||||||
|
|
||||||
|
// Server-side voice-onset. This runs on the SERVER (only the server has a live
|
||||||
|
// WebSocket — clients returned above), and the player's mic — the host's OR a
|
||||||
|
// remote client's, relayed here — always passes through this function. Broadcasting
|
||||||
|
// OnUserVoiceDetected here is what makes the agent-to-agent dialogue interrupt work
|
||||||
|
// on a DEDICATED server (no host player). Debounced (shares LastUserVoiceOnset with
|
||||||
|
// the FeedExternalAudio path, so the host doesn't double-fire).
|
||||||
|
{
|
||||||
|
const int32 OnsetSamples = PCMBytes.Num() / (int32)sizeof(int16);
|
||||||
|
if (OnsetSamples > 0)
|
||||||
|
{
|
||||||
|
const int16* OS = reinterpret_cast<const int16*>(PCMBytes.GetData());
|
||||||
|
double OnsetSumSq = 0.0;
|
||||||
|
for (int32 i = 0; i < OnsetSamples; ++i) { const double v = OS[i] / 32768.0; OnsetSumSq += v * v; }
|
||||||
|
const double OnsetRms = FMath::Sqrt(OnsetSumSq / OnsetSamples);
|
||||||
|
if (OnsetRms >= 0.02)
|
||||||
|
{
|
||||||
|
const double NowSec = FPlatformTime::Seconds();
|
||||||
|
if (NowSec - LastUserVoiceOnset > 0.5)
|
||||||
|
{
|
||||||
|
LastUserVoiceOnset = NowSec;
|
||||||
|
OnUserVoiceDetected.Broadcast();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Replay capture: expose every relayed player mic chunk (with its pawn) so an
|
||||||
|
// external recorder (PS_AI_ConvReplay) can save player speech. Server-only — clients
|
||||||
|
// have no live WebSocket and returned above. Fired before arbitration so audio from
|
||||||
|
// every player is captured, even a non-active speaker.
|
||||||
|
OnPlayerAudioCaptured.Broadcast(SpeakerPawn, PCMBytes);
|
||||||
|
|
||||||
// Standalone / single-player: bypass speaker arbitration entirely.
|
// Standalone / single-player: bypass speaker arbitration entirely.
|
||||||
// There's only one player — no need for Contains check or speaker switching.
|
// There's only one player — no need for Contains check or speaker switching.
|
||||||
if (NetConnectedPawns.Num() <= 1)
|
if (NetConnectedPawns.Num() <= 1)
|
||||||
@@ -2304,16 +2588,44 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ServerSendMicAudioFromPlayer(
|
|||||||
}
|
}
|
||||||
|
|
||||||
const double Now = FPlatformTime::Seconds();
|
const double Now = FPlatformTime::Seconds();
|
||||||
LastSpeakTime.FindOrAdd(SpeakerPawn) = Now;
|
|
||||||
|
|
||||||
// If this player IS the active speaker, forward immediately.
|
// Real voice energy of this chunk (int16 mono PCM). Only ACTUAL speech may
|
||||||
|
// hold or take the floor — not the silence-tail or ambient that an open mic
|
||||||
|
// streams continuously. Otherwise the first speaker's continuous stream keeps
|
||||||
|
// refreshing LastSpeakTime, so no one else can ever switch in and the agent
|
||||||
|
// stays locked on them (multi-player gaze/turn-taking broken).
|
||||||
|
const int32 NumSamples = PCMBytes.Num() / (int32)sizeof(int16);
|
||||||
|
double SumSq = 0.0;
|
||||||
|
if (NumSamples > 0)
|
||||||
|
{
|
||||||
|
const int16* S = reinterpret_cast<const int16*>(PCMBytes.GetData());
|
||||||
|
for (int32 i = 0; i < NumSamples; ++i) { const double v = S[i] / 32768.0; SumSq += v * v; }
|
||||||
|
}
|
||||||
|
const double Rms = (NumSamples > 0) ? FMath::Sqrt(SumSq / NumSamples) : 0.0;
|
||||||
|
const bool bVoiced = (Rms >= 0.02); // speech vs ambient/breath; tune if needed.
|
||||||
|
|
||||||
|
// Only real voice asserts holding/taking the floor.
|
||||||
|
if (bVoiced)
|
||||||
|
{
|
||||||
|
LastSpeakTime.FindOrAdd(SpeakerPawn) = Now;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The active speaker is always forwarded (keeps the silence tail flowing to
|
||||||
|
// ElevenLabs VAD for end-of-turn detection).
|
||||||
if (NetActiveSpeakerPawn == SpeakerPawn)
|
if (NetActiveSpeakerPawn == SpeakerPawn)
|
||||||
{
|
{
|
||||||
WebSocketProxy->SendAudioChunk(PCMBytes);
|
WebSocketProxy->SendAudioChunk(PCMBytes);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Speaker switch: only switch if current speaker has been silent
|
// A different player can only TAKE the floor with actual voice — their
|
||||||
|
// silence/tail must not steal it from the current speaker.
|
||||||
|
if (!bVoiced)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Speaker switch: only switch if current speaker has been voice-silent
|
||||||
// for at least SpeakerSwitchHysteresis seconds.
|
// for at least SpeakerSwitchHysteresis seconds.
|
||||||
bool bCanSwitch = true;
|
bool bCanSwitch = true;
|
||||||
if (NetActiveSpeakerPawn)
|
if (NetActiveSpeakerPawn)
|
||||||
@@ -2367,6 +2679,14 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ClientConversationStarted_Implementat
|
|||||||
{
|
{
|
||||||
StartListening();
|
StartListening();
|
||||||
}
|
}
|
||||||
|
// Consume any pending listen request deferred before conversation was ready.
|
||||||
|
else if (bPendingStartListening)
|
||||||
|
{
|
||||||
|
bPendingStartListening = false;
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
|
||||||
|
TEXT("[Client] Consuming pending StartListening request."));
|
||||||
|
StartListening();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void UPS_AI_ConvAgent_ElevenLabsComponent::ClientConversationFailed_Implementation(
|
void UPS_AI_ConvAgent_ElevenLabsComponent::ClientConversationFailed_Implementation(
|
||||||
@@ -2381,7 +2701,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ClientConversationFailed_Implementati
|
|||||||
// Network: Multicast RPCs
|
// Network: Multicast RPCs
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastReceiveAgentAudio_Implementation(
|
void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastReceiveAgentAudio_Implementation(
|
||||||
const TArray<uint8>& AudioData)
|
const TArray<uint8>& AudioData, bool bLastSubChunk)
|
||||||
{
|
{
|
||||||
// Server already handled playback in HandleAudioReceived.
|
// Server already handled playback in HandleAudioReceived.
|
||||||
if (GetOwnerRole() == ROLE_Authority) return;
|
if (GetOwnerRole() == ROLE_Authority) return;
|
||||||
@@ -2399,13 +2719,17 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastReceiveAgentAudio_Implementa
|
|||||||
if (OpusDecoder.IsValid())
|
if (OpusDecoder.IsValid())
|
||||||
{
|
{
|
||||||
// Opus path: decode compressed audio.
|
// Opus path: decode compressed audio.
|
||||||
const uint32 MaxDecompressedSize = 16000 * 2;
|
// Buffer must hold a full packet's decoded PCM. The engine decoder aborts
|
||||||
|
// with "Decompression buffer too small to decode voice" (dropping the
|
||||||
|
// packet tail) if this is smaller than the packet needs. The server caps
|
||||||
|
// packets at 40 frames (25600 bytes); 64KB gives ample headroom even up to
|
||||||
|
// the engine's 76-frame (~48640 bytes) maximum, so no packet is truncated.
|
||||||
|
const uint32 MaxDecompressedSize = 64 * 1024;
|
||||||
PCMBuffer.SetNumUninitialized(MaxDecompressedSize);
|
PCMBuffer.SetNumUninitialized(MaxDecompressedSize);
|
||||||
uint32 DecompressedSize = MaxDecompressedSize;
|
uint32 DecompressedSize = MaxDecompressedSize;
|
||||||
OpusDecoder->Decode(AudioData.GetData(), AudioData.Num(),
|
OpusDecoder->Decode(AudioData.GetData(), AudioData.Num(),
|
||||||
PCMBuffer.GetData(), DecompressedSize);
|
PCMBuffer.GetData(), DecompressedSize);
|
||||||
if (DecompressedSize == 0) return;
|
PCMBuffer.SetNum(DecompressedSize); // may be 0 — handled by reassembly below
|
||||||
PCMBuffer.SetNum(DecompressedSize);
|
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@@ -2413,14 +2737,73 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastReceiveAgentAudio_Implementa
|
|||||||
PCMBuffer = AudioData;
|
PCMBuffer = AudioData;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Local playback.
|
// Reassemble one ElevenLabs message from its N sub-chunks, then enqueue ONCE.
|
||||||
EnqueueAgentAudio(PCMBuffer);
|
// The server splits a message into sub-chunks (to respect the Opus 76-frame /
|
||||||
|
// client decode-buffer limits), sent back-to-back in the same tick. Enqueuing
|
||||||
// Feed lip-sync (within LOD or speaker).
|
// each sub-chunk separately re-ran the playback state machine → a pop at each
|
||||||
if (bIsSpeaker || LipSyncLODDistance <= 0.f || Dist <= LipSyncLODDistance)
|
// ~0.8s seam. Accumulating and enqueuing once per message plays it contiguously.
|
||||||
|
if (PCMBuffer.Num() > 0)
|
||||||
{
|
{
|
||||||
OnAgentAudioData.Broadcast(PCMBuffer);
|
ClientReassemblyPCM.Append(PCMBuffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!bLastSubChunk)
|
||||||
|
{
|
||||||
|
return; // wait for the remaining sub-chunks of this message
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ClientReassemblyPCM.Num() > 0)
|
||||||
|
{
|
||||||
|
// Local playback (one contiguous block per message).
|
||||||
|
EnqueueAgentAudio(ClientReassemblyPCM);
|
||||||
|
|
||||||
|
// Feed lip-sync (within LOD or speaker).
|
||||||
|
if (bIsSpeaker || LipSyncLODDistance <= 0.f || Dist <= LipSyncLODDistance)
|
||||||
|
{
|
||||||
|
OnAgentAudioData.Broadcast(ClientReassemblyPCM);
|
||||||
|
}
|
||||||
|
|
||||||
|
ClientReassemblyPCM.Reset();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Replay support (PS_AI_ConvReplay bridge)
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
void UPS_AI_ConvAgent_ElevenLabsComponent::PlayExternalAudioPCM(const TArray<uint8>& PCM16)
|
||||||
|
{
|
||||||
|
if (PCM16.Num() == 0) return;
|
||||||
|
|
||||||
|
// Same tail as MulticastReceiveAgentAudio_Implementation, minus the LOD culls:
|
||||||
|
// enqueue for playback (drives body expression + start/stop via bAgentSpeaking) and
|
||||||
|
// broadcast the raw PCM so the LipSync component animates visemes.
|
||||||
|
EnqueueAgentAudio(PCM16);
|
||||||
|
|
||||||
|
// During replay there is no live agent_response message, so the normal
|
||||||
|
// silence-detection stop path (TickComponent) would otherwise wait for the long
|
||||||
|
// hard-timeout. Mark the response as received so OnAgentStoppedSpeaking fires
|
||||||
|
// shortly after the buffer drains and body expression returns to idle.
|
||||||
|
// (Set AFTER EnqueueAgentAudio, which resets this flag on the first chunk.)
|
||||||
|
bAgentResponseReceived = true;
|
||||||
|
|
||||||
|
OnAgentAudioData.Broadcast(PCM16);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_ElevenLabsComponent::StopExternalAudio()
|
||||||
|
{
|
||||||
|
StopAgentAudio();
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_ElevenLabsComponent::TriggerReplayedAction(const FString& ActionName)
|
||||||
|
{
|
||||||
|
PendingActionName = ActionName;
|
||||||
|
OnAgentActionRequested.Broadcast(ActionName);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_ElevenLabsComponent::TriggerReplayedClientToolCall(
|
||||||
|
const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall)
|
||||||
|
{
|
||||||
|
OnAgentClientToolCall.Broadcast(ToolCall);
|
||||||
}
|
}
|
||||||
|
|
||||||
void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastAgentStartedSpeaking_Implementation()
|
void UPS_AI_ConvAgent_ElevenLabsComponent::MulticastAgentStartedSpeaking_Implementation()
|
||||||
@@ -2599,14 +2982,27 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ApplyConversationGaze()
|
|||||||
auto* Gaze = Owner->FindComponentByClass<UPS_AI_ConvAgent_GazeComponent>();
|
auto* Gaze = Owner->FindComponentByClass<UPS_AI_ConvAgent_GazeComponent>();
|
||||||
if (!Gaze) return;
|
if (!Gaze) return;
|
||||||
|
|
||||||
if (bNetIsConversing && NetConnectedPawns.Num() > 0)
|
// Override (agent-to-agent dialogue): look at a fixed actor, ignore players.
|
||||||
|
if (GazeOverrideTarget)
|
||||||
|
{
|
||||||
|
Gaze->bActive = true;
|
||||||
|
if (Gaze->TargetActor != GazeOverrideTarget)
|
||||||
|
{
|
||||||
|
Gaze->TargetActor = GazeOverrideTarget;
|
||||||
|
Gaze->ResetBodyTarget();
|
||||||
|
}
|
||||||
|
Gaze->bEnableBodyTracking = true;
|
||||||
|
}
|
||||||
|
else if (bNetIsConversing && NetConnectedPawns.Num() > 0)
|
||||||
{
|
{
|
||||||
Gaze->bActive = true;
|
Gaze->bActive = true;
|
||||||
// Look at active speaker if set, otherwise last connected pawn.
|
// Look at active speaker if set, otherwise last connected pawn.
|
||||||
|
// GazeTarget can be null if a replicated pawn ref hasn't resolved yet
|
||||||
|
// (late joiner). Don't overwrite the target with null in that case.
|
||||||
AActor* GazeTarget = NetActiveSpeakerPawn
|
AActor* GazeTarget = NetActiveSpeakerPawn
|
||||||
? static_cast<AActor*>(NetActiveSpeakerPawn)
|
? static_cast<AActor*>(NetActiveSpeakerPawn)
|
||||||
: static_cast<AActor*>(NetConnectedPawns.Last());
|
: static_cast<AActor*>(NetConnectedPawns.Last());
|
||||||
if (Gaze->TargetActor != GazeTarget)
|
if (GazeTarget && Gaze->TargetActor != GazeTarget)
|
||||||
{
|
{
|
||||||
Gaze->TargetActor = GazeTarget;
|
Gaze->TargetActor = GazeTarget;
|
||||||
Gaze->ResetBodyTarget();
|
Gaze->ResetBodyTarget();
|
||||||
@@ -2637,6 +3033,20 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ApplyConversationGaze()
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_ElevenLabsComponent::SetGazeOverrideTarget(AActor* Target)
|
||||||
|
{
|
||||||
|
if (GazeOverrideTarget == Target) return;
|
||||||
|
GazeOverrideTarget = Target; // replicates → OnRep_GazeOverrideTarget on clients
|
||||||
|
ApplyConversationGaze(); // apply immediately on the server/authority
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_ElevenLabsComponent::OnRep_GazeOverrideTarget()
|
||||||
|
{
|
||||||
|
// Re-evaluate the gaze on clients (honours the override, or restores normal
|
||||||
|
// player-tracking when it was cleared to null).
|
||||||
|
ApplyConversationGaze();
|
||||||
|
}
|
||||||
|
|
||||||
void UPS_AI_ConvAgent_ElevenLabsComponent::SetActiveSpeaker(APawn* NewSpeaker)
|
void UPS_AI_ConvAgent_ElevenLabsComponent::SetActiveSpeaker(APawn* NewSpeaker)
|
||||||
{
|
{
|
||||||
if (NetActiveSpeakerPawn == NewSpeaker) return;
|
if (NetActiveSpeakerPawn == NewSpeaker) return;
|
||||||
@@ -2675,12 +3085,18 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::DrawDebugHUD() const
|
|||||||
const FColor GoodColor = FColor::Green;
|
const FColor GoodColor = FColor::Green;
|
||||||
const FString OwnerName = GetOwner()->GetName();
|
const FString OwnerName = GetOwner()->GetName();
|
||||||
|
|
||||||
const bool bConnected = IsConnected();
|
// On a client there is no local WebSocket (it lives on the server), so
|
||||||
|
// IsConnected() is always false. Report the effective networked state via the
|
||||||
|
// replicated bNetIsConversing flag instead, to avoid a misleading "DISCONNECTED".
|
||||||
|
const bool bIsAuthority = (GetOwnerRole() == ROLE_Authority);
|
||||||
|
const bool bConnected = bIsAuthority ? IsConnected() : bNetIsConversing;
|
||||||
|
const TCHAR* ConnStr = bConnected
|
||||||
|
? (bIsAuthority ? TEXT("CONNECTED") : TEXT("CONNECTED (via server)"))
|
||||||
|
: TEXT("DISCONNECTED");
|
||||||
|
|
||||||
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime,
|
GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime,
|
||||||
bConnected ? GoodColor : FColor::Red,
|
bConnected ? GoodColor : FColor::Red,
|
||||||
FString::Printf(TEXT("=== ELEVENLABS [%s]: %s ==="), *OwnerName,
|
FString::Printf(TEXT("=== ELEVENLABS [%s]: %s ==="), *OwnerName, ConnStr));
|
||||||
bConnected ? TEXT("CONNECTED") : TEXT("DISCONNECTED")));
|
|
||||||
|
|
||||||
// Session info
|
// Session info
|
||||||
FString ModeStr = (TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server)
|
FString ModeStr = (TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server)
|
||||||
@@ -2745,12 +3161,13 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::DrawDebugHUD() const
|
|||||||
bWantsReconnect ? TEXT(" (ACTIVE)") : TEXT("")));
|
bWantsReconnect ? TEXT(" (ACTIVE)") : TEXT("")));
|
||||||
|
|
||||||
// ForceDisable state
|
// ForceDisable state
|
||||||
if (bConversationDisabledByAction || bWaitingForBlendOut)
|
if (bConversationForceDisabled || bWaitingForBlendOut)
|
||||||
{
|
{
|
||||||
const FColor DisableColor = FColor::Orange;
|
const FColor DisableColor = FColor::Orange;
|
||||||
|
const FString PendingStr = PendingActionName.IsEmpty() ? TEXT("<none>") : PendingActionName;
|
||||||
GEngine->AddOnScreenDebugMessage(BaseKey + 9, DisplayTime, DisableColor,
|
GEngine->AddOnScreenDebugMessage(BaseKey + 9, DisplayTime, DisableColor,
|
||||||
FString::Printf(TEXT(" FORCE DISABLED — action: '%s' blendOut: %s (%.1fs)"),
|
FString::Printf(TEXT(" FORCE DISABLED — pending action: '%s' blendOut: %s (%.1fs)"),
|
||||||
*PendingActionName,
|
*PendingStr,
|
||||||
bWaitingForBlendOut ? TEXT("WAITING") : TEXT("DONE"),
|
bWaitingForBlendOut ? TEXT("WAITING") : TEXT("DONE"),
|
||||||
BlendOutElapsedTime));
|
BlendOutElapsedTime));
|
||||||
|
|
||||||
@@ -2783,12 +3200,13 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::DrawDebugHUD() const
|
|||||||
void UPS_AI_ConvAgent_ElevenLabsComponent::FetchServerRegion()
|
void UPS_AI_ConvAgent_ElevenLabsComponent::FetchServerRegion()
|
||||||
{
|
{
|
||||||
const UPS_AI_ConvAgent_Settings_ElevenLabs* Settings = FPS_AI_ConvAgentModule::Get().GetSettings();
|
const UPS_AI_ConvAgent_Settings_ElevenLabs* Settings = FPS_AI_ConvAgentModule::Get().GetSettings();
|
||||||
if (!Settings || Settings->API_Key.IsEmpty()) return;
|
const FString APIKey = UPS_AI_ConvAgent_BlueprintLibrary::GetElevenLabsAPIKey();
|
||||||
|
if (!Settings || APIKey.IsEmpty()) return;
|
||||||
|
|
||||||
auto Request = FHttpModule::Get().CreateRequest();
|
auto Request = FHttpModule::Get().CreateRequest();
|
||||||
Request->SetURL(Settings->GetAPIBaseURL() + TEXT("/v1/models"));
|
Request->SetURL(Settings->GetAPIBaseURL() + TEXT("/v1/models"));
|
||||||
Request->SetVerb(TEXT("GET"));
|
Request->SetVerb(TEXT("GET"));
|
||||||
Request->SetHeader(TEXT("xi-api-key"), Settings->API_Key);
|
Request->SetHeader(TEXT("xi-api-key"), APIKey);
|
||||||
|
|
||||||
TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> WeakThis(this);
|
TWeakObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> WeakThis(this);
|
||||||
Request->OnProcessRequestComplete().BindLambda(
|
Request->OnProcessRequestComplete().BindLambda(
|
||||||
|
|||||||
@@ -405,11 +405,7 @@ void UPS_AI_ConvAgent_GazeComponent::TickComponent(
|
|||||||
|
|
||||||
const FVector ToTarget = TargetPos - EyeOrigin;
|
const FVector ToTarget = TargetPos - EyeOrigin;
|
||||||
|
|
||||||
// ── 2. Body: smooth interp toward sticky target ────────────────────
|
// ── 1b. Horizontal direction / world yaw to the target ─────────────
|
||||||
//
|
|
||||||
// TargetBodyWorldYaw is persistent — only updated when head+eyes
|
|
||||||
// can't reach the target. Same sticky pattern as TargetHeadYaw.
|
|
||||||
|
|
||||||
const FVector HorizontalDir = FVector(ToTarget.X, ToTarget.Y, 0.0f);
|
const FVector HorizontalDir = FVector(ToTarget.X, ToTarget.Y, 0.0f);
|
||||||
float TargetWorldYaw = 0.0f;
|
float TargetWorldYaw = 0.0f;
|
||||||
if (!HorizontalDir.IsNearlyZero(1.0f))
|
if (!HorizontalDir.IsNearlyZero(1.0f))
|
||||||
@@ -417,11 +413,21 @@ void UPS_AI_ConvAgent_GazeComponent::TickComponent(
|
|||||||
TargetWorldYaw = HorizontalDir.Rotation().Yaw;
|
TargetWorldYaw = HorizontalDir.Rotation().Yaw;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Body smoothly interpolates toward its persistent target.
|
// ── 2. BODY (authority only): lazy turn ────────────────────────────
|
||||||
// Server/standalone: directly rotates the actor.
|
// Rotate the actor to face the target only once head+eyes can no longer
|
||||||
// Client: does NOT rotate — accepts replicated rotation from server.
|
// cover it. Clients NEVER rotate the body — they receive it replicated.
|
||||||
if (bEnableBodyTracking && Owner->HasAuthority())
|
// Body turn state (TargetBodyWorldYaw) lives on the authority only.
|
||||||
|
if (bEnableBodyTracking && Owner->HasAuthority() && !HorizontalDir.IsNearlyZero(1.0f))
|
||||||
{
|
{
|
||||||
|
const float BodyFacingYaw = Owner->GetActorRotation().Yaw + MeshForwardYawOffset;
|
||||||
|
const float DeltaToTarget = FMath::FindDeltaAngleDegrees(BodyFacingYaw, TargetWorldYaw);
|
||||||
|
|
||||||
|
// Commit to facing the target once it leaves the head+eyes range.
|
||||||
|
if (FMath::Abs(DeltaToTarget) > MaxHeadYaw + MaxEyeHorizontal)
|
||||||
|
{
|
||||||
|
TargetBodyWorldYaw = TargetWorldYaw - MeshForwardYawOffset;
|
||||||
|
}
|
||||||
|
|
||||||
const float BodyDelta = FMath::FindDeltaAngleDegrees(
|
const float BodyDelta = FMath::FindDeltaAngleDegrees(
|
||||||
Owner->GetActorRotation().Yaw, TargetBodyWorldYaw);
|
Owner->GetActorRotation().Yaw, TargetBodyWorldYaw);
|
||||||
if (FMath::Abs(BodyDelta) > 0.1f)
|
if (FMath::Abs(BodyDelta) > 0.1f)
|
||||||
@@ -431,122 +437,49 @@ void UPS_AI_ConvAgent_GazeComponent::TickComponent(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Smoothed body yaw for cascade computations ──────────────────
|
// Smoothed mirror of the (authoritative/replicated) body yaw — DEBUG
|
||||||
// Server: direct copy (no lag). Client: interpolate toward replicated
|
// only. NEVER written back to the actor: the old client override
|
||||||
// rotation to avoid step artifacts from discrete ~30Hz network updates.
|
// (SetActorRotation) corrupted the head input and made the client head
|
||||||
// The client also overrides the actor's rotation with the smoothed value
|
// diverge by up to MaxHeadYaw. The head reads the real actor yaw below.
|
||||||
// so the body mesh doesn't visually jump at network tick rate.
|
|
||||||
if (Owner->HasAuthority())
|
|
||||||
{
|
{
|
||||||
SmoothedBodyYaw = Owner->GetActorRotation().Yaw;
|
const float ActorYaw = Owner->GetActorRotation().Yaw;
|
||||||
}
|
const float YawDelta = FMath::FindDeltaAngleDegrees(SmoothedBodyYaw, ActorYaw);
|
||||||
else
|
SmoothedBodyYaw += FMath::FInterpTo(0.0f, YawDelta, SafeDeltaTime, BodyInterpSpeed * 3.0f);
|
||||||
{
|
|
||||||
const float ReplicatedYaw = Owner->GetActorRotation().Yaw;
|
|
||||||
const float ClientDelta = FMath::FindDeltaAngleDegrees(SmoothedBodyYaw, ReplicatedYaw);
|
|
||||||
SmoothedBodyYaw += FMath::FInterpTo(0.0f, ClientDelta, SafeDeltaTime, BodyInterpSpeed * 3.0f);
|
|
||||||
|
|
||||||
// Override actor rotation with smoothed value so the body mesh
|
|
||||||
// shows the interpolated rotation instead of the raw replicated steps.
|
|
||||||
// This is local-only — replication will overwrite on next network update,
|
|
||||||
// and SmoothedBodyYaw will absorb the correction smoothly.
|
|
||||||
FRotator SmoothedRot = Owner->GetActorRotation();
|
|
||||||
SmoothedRot.Yaw = SmoothedBodyYaw;
|
|
||||||
Owner->SetActorRotation(SmoothedRot);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── 3. Compute DeltaYaw after body interp ──────────────────────────
|
// When body tracking is off, keep the body target synced (debug only).
|
||||||
// When body tracking is disabled (e.g. proximity gaze during spline patrol),
|
|
||||||
// sync SmoothedBodyYaw and TargetBodyWorldYaw to current actor rotation.
|
|
||||||
// This ensures the head offset is always relative to the actual body facing,
|
|
||||||
// not a stale cached value from a previous frame.
|
|
||||||
if (!bEnableBodyTracking)
|
if (!bEnableBodyTracking)
|
||||||
{
|
{
|
||||||
SmoothedBodyYaw = Owner->GetActorRotation().Yaw;
|
TargetBodyWorldYaw = Owner->GetActorRotation().Yaw;
|
||||||
TargetBodyWorldYaw = SmoothedBodyYaw;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── 3. HEAD + EYES — STATELESS cascade ─────────────────────────────
|
||||||
|
// Pure function of the ACTUAL body facing (authoritative on the server,
|
||||||
|
// replicated on the client) and the target direction. No sticky /
|
||||||
|
// path-dependent state, so server and client converge to the same pose
|
||||||
|
// regardless of how the body rotation arrived (smooth vs ~30Hz steps).
|
||||||
|
// FInterpTo only eases the visual; it never latches.
|
||||||
|
const float BodyFacingYaw = Owner->GetActorRotation().Yaw + MeshForwardYawOffset;
|
||||||
|
|
||||||
float DeltaYaw = 0.0f;
|
float DeltaYaw = 0.0f;
|
||||||
if (!HorizontalDir.IsNearlyZero(1.0f))
|
if (!HorizontalDir.IsNearlyZero(1.0f))
|
||||||
{
|
{
|
||||||
const float CurrentFacingYaw = SmoothedBodyYaw + MeshForwardYawOffset;
|
DeltaYaw = FMath::FindDeltaAngleDegrees(BodyFacingYaw, TargetWorldYaw);
|
||||||
DeltaYaw = FMath::FindDeltaAngleDegrees(CurrentFacingYaw, TargetWorldYaw);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── 4. Pitch from 3D direction ─────────────────────────────────────
|
|
||||||
|
|
||||||
const float HorizontalDist = HorizontalDir.Size();
|
const float HorizontalDist = HorizontalDir.Size();
|
||||||
const float TargetPitch = (HorizontalDist > 1.0f)
|
const float TargetPitch = (HorizontalDist > 1.0f)
|
||||||
? FMath::RadiansToDegrees(FMath::Atan2(ToTarget.Z, HorizontalDist))
|
? FMath::RadiansToDegrees(FMath::Atan2(ToTarget.Z, HorizontalDist))
|
||||||
: 0.0f;
|
: 0.0f;
|
||||||
|
|
||||||
// ── 5. BODY overflow: check against persistent TargetBodyWorldYaw ──
|
// Head takes as much of the yaw gap as its range allows; eyes the rest.
|
||||||
//
|
// (TargetHeadYaw/Pitch are kept only as debug mirrors — no longer sticky.)
|
||||||
// Same pattern as head: check from body's TARGET position (not current).
|
TargetHeadYaw = FMath::Clamp(DeltaYaw, -MaxHeadYaw, MaxHeadYaw);
|
||||||
// Body triggers when head+eyes combined range is exceeded.
|
|
||||||
|
|
||||||
const float BodyTargetFacing = TargetBodyWorldYaw + MeshForwardYawOffset;
|
|
||||||
const float DeltaFromBodyTarget = FMath::FindDeltaAngleDegrees(
|
|
||||||
BodyTargetFacing, TargetWorldYaw);
|
|
||||||
|
|
||||||
bool bBodyOverflowed = false;
|
|
||||||
if (bEnableBodyTracking && FMath::Abs(DeltaFromBodyTarget) > MaxHeadYaw + MaxEyeHorizontal)
|
|
||||||
{
|
|
||||||
// Body realigns to face target
|
|
||||||
TargetBodyWorldYaw = TargetWorldYaw - MeshForwardYawOffset;
|
|
||||||
// Head returns to ~0° since body will face target directly
|
|
||||||
TargetHeadYaw = 0.0f;
|
|
||||||
bBodyOverflowed = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── 6. HEAD: realign when eyes overflow (check against body TARGET) ──
|
|
||||||
//
|
|
||||||
// Head overflow is checked relative to where the BODY IS GOING
|
|
||||||
// (TargetBodyWorldYaw), not where it currently is. This prevents
|
|
||||||
// the head from overcompensating during body interpolation —
|
|
||||||
// otherwise the head turns to track while body catches up, then
|
|
||||||
// snaps back when body arrives (two-step animation artifact).
|
|
||||||
//
|
|
||||||
// GUARD: Skip eye overflow check when body just overflowed in this
|
|
||||||
// same frame. Body overflow already set TargetHeadYaw=0 (head will
|
|
||||||
// recenter as body turns). Allowing eye overflow to also fire would
|
|
||||||
// snap TargetHeadYaw to a second value in the same frame, causing
|
|
||||||
// a visible 1-2 frame jerk as FInterpTo chases two targets.
|
|
||||||
|
|
||||||
if (!bBodyOverflowed)
|
|
||||||
{
|
|
||||||
const float HeadDeltaYaw = FMath::FindDeltaAngleDegrees(
|
|
||||||
TargetBodyWorldYaw + MeshForwardYawOffset, TargetWorldYaw);
|
|
||||||
|
|
||||||
const float EyeDeltaYaw = HeadDeltaYaw - TargetHeadYaw;
|
|
||||||
|
|
||||||
if (FMath::Abs(EyeDeltaYaw) > MaxEyeHorizontal)
|
|
||||||
{
|
|
||||||
TargetHeadYaw = FMath::Clamp(HeadDeltaYaw, -MaxHeadYaw, MaxHeadYaw);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Head smoothly interpolates toward its persistent target
|
|
||||||
CurrentHeadYaw = FMath::FInterpTo(CurrentHeadYaw, TargetHeadYaw, SafeDeltaTime, HeadInterpSpeed);
|
CurrentHeadYaw = FMath::FInterpTo(CurrentHeadYaw, TargetHeadYaw, SafeDeltaTime, HeadInterpSpeed);
|
||||||
|
|
||||||
// Eyes = remaining gap (during transients, eyes may sit at MaxEye while
|
|
||||||
// the head catches up — ARKit normalization keeps visual deflection small)
|
|
||||||
CurrentEyeYaw = FMath::Clamp(DeltaYaw - CurrentHeadYaw, -MaxEyeHorizontal, MaxEyeHorizontal);
|
CurrentEyeYaw = FMath::Clamp(DeltaYaw - CurrentHeadYaw, -MaxEyeHorizontal, MaxEyeHorizontal);
|
||||||
|
|
||||||
// ── 5. PITCH: relative cascade (Eyes → Head, no body pitch) ────────
|
TargetHeadPitch = FMath::Clamp(TargetPitch, -MaxHeadPitch, MaxHeadPitch);
|
||||||
|
|
||||||
// Same pattern: check against persistent TargetHeadPitch
|
|
||||||
const float EyeDeltaPitch = TargetPitch - TargetHeadPitch;
|
|
||||||
|
|
||||||
if (FMath::Abs(EyeDeltaPitch) > MaxEyeVertical)
|
|
||||||
{
|
|
||||||
// Eyes overflow → head realigns toward target pitch
|
|
||||||
TargetHeadPitch = FMath::Clamp(TargetPitch, -MaxHeadPitch, MaxHeadPitch);
|
|
||||||
}
|
|
||||||
|
|
||||||
CurrentHeadPitch = FMath::FInterpTo(CurrentHeadPitch, TargetHeadPitch, SafeDeltaTime, HeadInterpSpeed);
|
CurrentHeadPitch = FMath::FInterpTo(CurrentHeadPitch, TargetHeadPitch, SafeDeltaTime, HeadInterpSpeed);
|
||||||
|
|
||||||
// Eyes = remaining pitch gap
|
|
||||||
CurrentEyePitch = FMath::Clamp(TargetPitch - CurrentHeadPitch, -MaxEyeVertical, MaxEyeVertical);
|
CurrentEyePitch = FMath::Clamp(TargetPitch - CurrentHeadPitch, -MaxEyeVertical, MaxEyeVertical);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
|||||||
@@ -21,6 +21,17 @@ static TAutoConsoleVariable<int32> CVarDebugInteraction(
|
|||||||
TEXT("Debug HUD for Interaction. -1=use property, 0=off, 1-3=verbosity."),
|
TEXT("Debug HUD for Interaction. -1=use property, 0=off, 1-3=verbosity."),
|
||||||
ECVF_Default);
|
ECVF_Default);
|
||||||
|
|
||||||
|
// Convert normalized float mic samples to int16 little-endian bytes (16kHz mono).
|
||||||
|
static void ConvReplay_FloatToInt16Bytes(const TArray<float>& In, TArray<uint8>& Out)
|
||||||
|
{
|
||||||
|
Out.SetNumUninitialized(In.Num() * static_cast<int32>(sizeof(int16)));
|
||||||
|
int16* D = reinterpret_cast<int16*>(Out.GetData());
|
||||||
|
for (int32 i = 0; i < In.Num(); ++i)
|
||||||
|
{
|
||||||
|
D[i] = static_cast<int16>(FMath::Clamp<int32>(FMath::RoundToInt(In[i] * 32767.0f), -32768, 32767));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
// Constructor
|
// Constructor
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -132,6 +143,9 @@ void UPS_AI_ConvAgent_InteractionComponent::TickComponent(float DeltaTime, ELeve
|
|||||||
|
|
||||||
bInitialized = true;
|
bInitialized = true;
|
||||||
|
|
||||||
|
// If a replay recording started before the mic existed, begin capturing now.
|
||||||
|
UpdateReplayCapture();
|
||||||
|
|
||||||
if (bDebug)
|
if (bDebug)
|
||||||
{
|
{
|
||||||
UE_LOG(LogPS_AI_ConvAgent_Select, Log,
|
UE_LOG(LogPS_AI_ConvAgent_Select, Log,
|
||||||
@@ -191,7 +205,8 @@ void UPS_AI_ConvAgent_InteractionComponent::TickComponent(float DeltaTime, ELeve
|
|||||||
|
|
||||||
// Re-activate passive gaze using configured checkboxes.
|
// Re-activate passive gaze using configured checkboxes.
|
||||||
// ExecuteLeave killed the gaze — re-enable passively.
|
// ExecuteLeave killed the gaze — re-enable passively.
|
||||||
if (bAutoManageGaze)
|
// (Skip if the agent is in an agent-to-agent dialogue.)
|
||||||
|
if (bAutoManageGaze && !Pending->GazeOverrideTarget)
|
||||||
{
|
{
|
||||||
if (UPS_AI_ConvAgent_GazeComponent* Gaze = FindGazeOnAgent(Pending))
|
if (UPS_AI_ConvAgent_GazeComponent* Gaze = FindGazeOnAgent(Pending))
|
||||||
{
|
{
|
||||||
@@ -301,6 +316,9 @@ void UPS_AI_ConvAgent_InteractionComponent::UpdatePassiveGaze()
|
|||||||
if (Agent == GazeRetainedAgent.Get()) continue;
|
if (Agent == GazeRetainedAgent.Get()) continue;
|
||||||
if (Agent == PendingLeaveAgent.Get()) continue;
|
if (Agent == PendingLeaveAgent.Get()) continue;
|
||||||
if (Agent->IsConversationDisabled()) continue;
|
if (Agent->IsConversationDisabled()) continue;
|
||||||
|
// Skip agents driven by an agent-to-agent dialogue (they look at each
|
||||||
|
// other via GazeOverrideTarget — don't pull their gaze toward the player).
|
||||||
|
if (Agent->GazeOverrideTarget) continue;
|
||||||
|
|
||||||
AActor* AgentActor = Agent->GetOwner();
|
AActor* AgentActor = Agent->GetOwner();
|
||||||
if (!AgentActor) continue;
|
if (!AgentActor) continue;
|
||||||
@@ -600,7 +618,8 @@ void UPS_AI_ConvAgent_InteractionComponent::SetSelectedAgent(UPS_AI_ConvAgent_El
|
|||||||
// ExecuteLeave → ServerLeaveConversation → ApplyConversationGaze()
|
// ExecuteLeave → ServerLeaveConversation → ApplyConversationGaze()
|
||||||
// already killed bActive and cleared TargetActor. Override that:
|
// already killed bActive and cleared TargetActor. Override that:
|
||||||
// the agent should keep looking at the player passively.
|
// the agent should keep looking at the player passively.
|
||||||
if (bAutoManageGaze)
|
// (Skip while an agent-to-agent dialogue owns the gaze.)
|
||||||
|
if (bAutoManageGaze && !OldAgent->GazeOverrideTarget)
|
||||||
{
|
{
|
||||||
if (UPS_AI_ConvAgent_GazeComponent* Gaze = FindGazeOnAgent(OldAgent))
|
if (UPS_AI_ConvAgent_GazeComponent* Gaze = FindGazeOnAgent(OldAgent))
|
||||||
{
|
{
|
||||||
@@ -919,6 +938,11 @@ void UPS_AI_ConvAgent_InteractionComponent::AttachGazeTarget(
|
|||||||
// from ever firing.
|
// from ever firing.
|
||||||
if (AgentPtr->IsConversationDisabled()) return;
|
if (AgentPtr->IsConversationDisabled()) return;
|
||||||
|
|
||||||
|
// Agent-to-agent dialogue: this agent's gaze is pinned on the OTHER agent via
|
||||||
|
// GazeOverrideTarget. Don't attach it to the player just because the player is
|
||||||
|
// looking at / selecting it — let the override hold (the dialogue owns the gaze).
|
||||||
|
if (AgentPtr->GazeOverrideTarget) return;
|
||||||
|
|
||||||
if (UPS_AI_ConvAgent_GazeComponent* Gaze = FindGazeOnAgent(AgentPtr))
|
if (UPS_AI_ConvAgent_GazeComponent* Gaze = FindGazeOnAgent(AgentPtr))
|
||||||
{
|
{
|
||||||
Gaze->TargetActor = GetOwner();
|
Gaze->TargetActor = GetOwner();
|
||||||
@@ -1052,10 +1076,34 @@ void UPS_AI_ConvAgent_InteractionComponent::CleanupRetainedGaze(UPS_AI_ConvAgent
|
|||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
void UPS_AI_ConvAgent_InteractionComponent::OnMicAudioCaptured(const TArray<float>& FloatPCM)
|
void UPS_AI_ConvAgent_InteractionComponent::OnMicAudioCaptured(const TArray<float>& FloatPCM)
|
||||||
{
|
{
|
||||||
UPS_AI_ConvAgent_ElevenLabsComponent* Agent = SelectedAgent.Get();
|
if (UPS_AI_ConvAgent_ElevenLabsComponent* Agent = SelectedAgent.Get())
|
||||||
if (!Agent) return;
|
{
|
||||||
|
Agent->FeedExternalAudio(FloatPCM);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
Agent->FeedExternalAudio(FloatPCM);
|
// No agent selected: if a replay is being recorded, still capture this pawn's
|
||||||
|
// voice for the recorder. Convert to int16, accumulate, and relay in chunks —
|
||||||
|
// the host broadcasts locally; a remote client sends a Server RPC.
|
||||||
|
if (!bReplayRecordActive) return;
|
||||||
|
|
||||||
|
TArray<uint8> Bytes;
|
||||||
|
ConvReplay_FloatToInt16Bytes(FloatPCM, Bytes);
|
||||||
|
RecordMicBuffer.Append(Bytes);
|
||||||
|
|
||||||
|
static constexpr int32 RecordChunkMinBytes = 3200; // ~100ms @ 16kHz mono int16
|
||||||
|
if (RecordMicBuffer.Num() >= RecordChunkMinBytes)
|
||||||
|
{
|
||||||
|
if (GetOwnerRole() == ROLE_Authority)
|
||||||
|
{
|
||||||
|
OnPlayerVoiceForRecording.Broadcast(Cast<APawn>(GetOwner()), RecordMicBuffer);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
ServerRecordVoice(RecordMicBuffer);
|
||||||
|
}
|
||||||
|
RecordMicBuffer.Reset();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -1144,7 +1192,7 @@ void UPS_AI_ConvAgent_InteractionComponent::GetLifetimeReplicatedProps(
|
|||||||
TArray<FLifetimeProperty>& OutLifetimeProps) const
|
TArray<FLifetimeProperty>& OutLifetimeProps) const
|
||||||
{
|
{
|
||||||
Super::GetLifetimeReplicatedProps(OutLifetimeProps);
|
Super::GetLifetimeReplicatedProps(OutLifetimeProps);
|
||||||
// No replicated properties — but the component must be replicated for RPCs.
|
DOREPLIFETIME(UPS_AI_ConvAgent_InteractionComponent, bReplayRecordActive);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -1221,6 +1269,50 @@ void UPS_AI_ConvAgent_InteractionComponent::ServerRelayMicAudio_Implementation(
|
|||||||
Agent->ServerSendMicAudioFromPlayer(SenderPawn, PCMToSend);
|
Agent->ServerSendMicAudioFromPlayer(SenderPawn, PCMToSend);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Agent-less replay recording (driven by PS_AI_ConvReplay)
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
void UPS_AI_ConvAgent_InteractionComponent::SetReplayRecordActive(bool bActive)
|
||||||
|
{
|
||||||
|
if (GetOwnerRole() != ROLE_Authority) return;
|
||||||
|
if (bReplayRecordActive == bActive) return;
|
||||||
|
bReplayRecordActive = bActive;
|
||||||
|
// Authority (host player) receives no OnRep — apply locally now.
|
||||||
|
UpdateReplayCapture();
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_InteractionComponent::OnRep_ReplayRecordActive()
|
||||||
|
{
|
||||||
|
UpdateReplayCapture();
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_InteractionComponent::UpdateReplayCapture()
|
||||||
|
{
|
||||||
|
if (!bReplayRecordActive)
|
||||||
|
{
|
||||||
|
RecordMicBuffer.Reset();
|
||||||
|
// Only stop the mic if no agent conversation still needs it.
|
||||||
|
if (MicComponent && MicComponent->IsCapturing() && !SelectedAgent.IsValid())
|
||||||
|
{
|
||||||
|
MicComponent->StopCapture();
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recording: make sure the mic is capturing. On a remote pawn's server-side
|
||||||
|
// replica MicComponent is null (tick disabled) and this is a harmless no-op —
|
||||||
|
// that client captures on its own machine and relays via ServerRecordVoice.
|
||||||
|
if (MicComponent && !MicComponent->IsCapturing())
|
||||||
|
{
|
||||||
|
MicComponent->StartCapture();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_InteractionComponent::ServerRecordVoice_Implementation(const TArray<uint8>& PCMBytes)
|
||||||
|
{
|
||||||
|
OnPlayerVoiceForRecording.Broadcast(Cast<APawn>(GetOwner()), PCMBytes);
|
||||||
|
}
|
||||||
|
|
||||||
void UPS_AI_ConvAgent_InteractionComponent::ServerRelaySendText_Implementation(
|
void UPS_AI_ConvAgent_InteractionComponent::ServerRelaySendText_Implementation(
|
||||||
AActor* AgentActor, const FString& Text)
|
AActor* AgentActor, const FString& Text)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -0,0 +1,78 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#include "PS_AI_ConvAgent_TestAPIKey_AsyncAction.h"
|
||||||
|
#include "PS_AI_ConvAgent.h"
|
||||||
|
#include "HttpModule.h"
|
||||||
|
#include "Interfaces/IHttpRequest.h"
|
||||||
|
#include "Interfaces/IHttpResponse.h"
|
||||||
|
|
||||||
|
UPS_AI_ConvAgent_TestAPIKey_AsyncAction* UPS_AI_ConvAgent_TestAPIKey_AsyncAction::TestElevenLabsAPIKey(const FString& Key)
|
||||||
|
{
|
||||||
|
UPS_AI_ConvAgent_TestAPIKey_AsyncAction* Action = NewObject<UPS_AI_ConvAgent_TestAPIKey_AsyncAction>();
|
||||||
|
Action->KeyToTest = Key;
|
||||||
|
return Action;
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_TestAPIKey_AsyncAction::Activate()
|
||||||
|
{
|
||||||
|
// Keep this action alive across the async HTTP round-trip. Removed right
|
||||||
|
// before SetReadyToDestroy() once a result has been broadcast.
|
||||||
|
AddToRoot();
|
||||||
|
|
||||||
|
if (KeyToTest.IsEmpty())
|
||||||
|
{
|
||||||
|
OnInvalid.Broadcast(TEXT("API key is empty."));
|
||||||
|
RemoveFromRoot();
|
||||||
|
SetReadyToDestroy();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Respect the configured server region for the probe URL; fall back to global.
|
||||||
|
FString BaseURL = TEXT("https://api.elevenlabs.io");
|
||||||
|
if (FPS_AI_ConvAgentModule::IsAvailable())
|
||||||
|
{
|
||||||
|
if (const UPS_AI_ConvAgent_Settings_ElevenLabs* Settings = FPS_AI_ConvAgentModule::Get().GetSettings())
|
||||||
|
{
|
||||||
|
BaseURL = Settings->GetAPIBaseURL();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const TSharedRef<IHttpRequest, ESPMode::ThreadSafe> Request = FHttpModule::Get().CreateRequest();
|
||||||
|
Request->SetURL(BaseURL + TEXT("/v1/models"));
|
||||||
|
Request->SetVerb(TEXT("GET"));
|
||||||
|
Request->SetHeader(TEXT("xi-api-key"), KeyToTest);
|
||||||
|
|
||||||
|
TWeakObjectPtr<UPS_AI_ConvAgent_TestAPIKey_AsyncAction> WeakThis(this);
|
||||||
|
Request->OnProcessRequestComplete().BindLambda(
|
||||||
|
[WeakThis](FHttpRequestPtr /*Req*/, FHttpResponsePtr Resp, bool bSuccess)
|
||||||
|
{
|
||||||
|
if (!WeakThis.IsValid())
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
UPS_AI_ConvAgent_TestAPIKey_AsyncAction* Self = WeakThis.Get();
|
||||||
|
|
||||||
|
const int32 Code = Resp.IsValid() ? Resp->GetResponseCode() : 0;
|
||||||
|
if (bSuccess && Code == 200)
|
||||||
|
{
|
||||||
|
Self->OnValid.Broadcast(TEXT("API key is valid."));
|
||||||
|
}
|
||||||
|
else if (Code == 401 || Code == 403)
|
||||||
|
{
|
||||||
|
Self->OnInvalid.Broadcast(FString::Printf(TEXT("Invalid API key (HTTP %d)."), Code));
|
||||||
|
}
|
||||||
|
else if (Code == 0)
|
||||||
|
{
|
||||||
|
Self->OnInvalid.Broadcast(TEXT("Could not reach ElevenLabs (network error)."));
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
Self->OnInvalid.Broadcast(FString::Printf(TEXT("Key test failed (HTTP %d)."), Code));
|
||||||
|
}
|
||||||
|
|
||||||
|
Self->RemoveFromRoot();
|
||||||
|
Self->SetReadyToDestroy();
|
||||||
|
});
|
||||||
|
|
||||||
|
Request->ProcessRequest();
|
||||||
|
}
|
||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
#include "PS_AI_ConvAgent_WebSocket_ElevenLabsProxy.h"
|
#include "PS_AI_ConvAgent_WebSocket_ElevenLabsProxy.h"
|
||||||
#include "PS_AI_ConvAgent.h"
|
#include "PS_AI_ConvAgent.h"
|
||||||
|
#include "PS_AI_ConvAgent_BlueprintLibrary.h"
|
||||||
|
|
||||||
#include "WebSocketsModule.h"
|
#include "WebSocketsModule.h"
|
||||||
#include "IWebSocket.h"
|
#include "IWebSocket.h"
|
||||||
@@ -9,6 +10,10 @@
|
|||||||
#include "Json.h"
|
#include "Json.h"
|
||||||
#include "Misc/Base64.h"
|
#include "Misc/Base64.h"
|
||||||
|
|
||||||
|
#include "HttpModule.h"
|
||||||
|
#include "Interfaces/IHttpRequest.h"
|
||||||
|
#include "Interfaces/IHttpResponse.h"
|
||||||
|
|
||||||
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_WS_ElevenLabs, Log, All);
|
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_WS_ElevenLabs, Log, All);
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -28,7 +33,36 @@ void UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy::Connect(const FString& AgentIDO
|
|||||||
FModuleManager::LoadModuleChecked<FWebSocketsModule>("WebSockets");
|
FModuleManager::LoadModuleChecked<FWebSocketsModule>("WebSockets");
|
||||||
}
|
}
|
||||||
|
|
||||||
const FString URL = BuildWebSocketURL(AgentIDOverride, APIKeyOverride);
|
const UPS_AI_ConvAgent_Settings_ElevenLabs* Settings = FPS_AI_ConvAgentModule::Get().GetSettings();
|
||||||
|
const FString ResolvedKey = APIKeyOverride.IsEmpty()
|
||||||
|
? UPS_AI_ConvAgent_BlueprintLibrary::GetElevenLabsAPIKey()
|
||||||
|
: APIKeyOverride;
|
||||||
|
|
||||||
|
// Private agents (authentication enabled) require a Signed URL: a direct
|
||||||
|
// ?agent_id=... connection is rejected by ElevenLabs even with the xi-api-key
|
||||||
|
// header. So whenever we have a key (and no manual URL override), fetch a
|
||||||
|
// short-lived signed URL with that key and connect to it. This also works for
|
||||||
|
// public agents, and enforces "valid key required" — no valid key, no signed
|
||||||
|
// URL, no conversation.
|
||||||
|
if (!ResolvedKey.IsEmpty() && Settings->CustomWebSocketURL.IsEmpty())
|
||||||
|
{
|
||||||
|
if (AgentIDOverride.IsEmpty())
|
||||||
|
{
|
||||||
|
const FString Msg = TEXT("Cannot connect: no Agent ID configured. Set it in Project Settings or pass it to Connect().");
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_WS_ElevenLabs, Error, TEXT("%s"), *Msg);
|
||||||
|
ConnectionState = EPS_AI_ConvAgent_ConnectionState_ElevenLabs::Error;
|
||||||
|
OnError.Broadcast(Msg);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
ConnectionState = EPS_AI_ConvAgent_ConnectionState_ElevenLabs::Connecting;
|
||||||
|
RequestSignedURLAndConnect(AgentIDOverride, ResolvedKey);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback: direct connection. Used for the advanced custom-URL override, or
|
||||||
|
// for a public agent when no key is available (no authentication).
|
||||||
|
const FString URL = BuildWebSocketURL(AgentIDOverride, ResolvedKey);
|
||||||
if (URL.IsEmpty())
|
if (URL.IsEmpty())
|
||||||
{
|
{
|
||||||
const FString Msg = TEXT("Cannot connect: no Agent ID configured. Set it in Project Settings or pass it to Connect().");
|
const FString Msg = TEXT("Cannot connect: no Agent ID configured. Set it in Project Settings or pass it to Connect().");
|
||||||
@@ -38,18 +72,105 @@ void UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy::Connect(const FString& AgentIDO
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
UE_LOG(LogPS_AI_ConvAgent_WS_ElevenLabs, Log, TEXT("Connecting to ElevenLabs: %s"), *URL);
|
|
||||||
ConnectionState = EPS_AI_ConvAgent_ConnectionState_ElevenLabs::Connecting;
|
ConnectionState = EPS_AI_ConvAgent_ConnectionState_ElevenLabs::Connecting;
|
||||||
|
|
||||||
// Headers: the ElevenLabs Conversational AI WS endpoint accepts the
|
|
||||||
// xi-api-key header on the initial HTTP upgrade request.
|
|
||||||
TMap<FString, FString> UpgradeHeaders;
|
TMap<FString, FString> UpgradeHeaders;
|
||||||
const UPS_AI_ConvAgent_Settings_ElevenLabs* Settings = FPS_AI_ConvAgentModule::Get().GetSettings();
|
|
||||||
const FString ResolvedKey = APIKeyOverride.IsEmpty() ? Settings->API_Key : APIKeyOverride;
|
|
||||||
if (!ResolvedKey.IsEmpty())
|
if (!ResolvedKey.IsEmpty())
|
||||||
{
|
{
|
||||||
UpgradeHeaders.Add(TEXT("xi-api-key"), ResolvedKey);
|
UpgradeHeaders.Add(TEXT("xi-api-key"), ResolvedKey);
|
||||||
}
|
}
|
||||||
|
OpenWebSocket(URL, UpgradeHeaders);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy::RequestSignedURLAndConnect(const FString& AgentID, const FString& APIKey)
|
||||||
|
{
|
||||||
|
const UPS_AI_ConvAgent_Settings_ElevenLabs* Settings = FPS_AI_ConvAgentModule::Get().GetSettings();
|
||||||
|
const FString RequestURL = FString::Printf(
|
||||||
|
TEXT("%s/v1/convai/conversation/get-signed-url?agent_id=%s"),
|
||||||
|
*Settings->GetAPIBaseURL(), *AgentID);
|
||||||
|
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_WS_ElevenLabs, Log, TEXT("Requesting signed URL for agent %s"), *AgentID);
|
||||||
|
|
||||||
|
TSharedRef<IHttpRequest, ESPMode::ThreadSafe> Request = FHttpModule::Get().CreateRequest();
|
||||||
|
Request->SetURL(RequestURL);
|
||||||
|
Request->SetVerb(TEXT("GET"));
|
||||||
|
Request->SetHeader(TEXT("xi-api-key"), APIKey);
|
||||||
|
|
||||||
|
TWeakObjectPtr<UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy> WeakThis(this);
|
||||||
|
Request->OnProcessRequestComplete().BindLambda(
|
||||||
|
[WeakThis](FHttpRequestPtr /*Req*/, FHttpResponsePtr Resp, bool bSuccess)
|
||||||
|
{
|
||||||
|
if (!WeakThis.IsValid())
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy* Self = WeakThis.Get();
|
||||||
|
|
||||||
|
const int32 Code = Resp.IsValid() ? Resp->GetResponseCode() : 0;
|
||||||
|
if (bSuccess && Code == 200 && Resp.IsValid())
|
||||||
|
{
|
||||||
|
FString SignedURL;
|
||||||
|
TSharedPtr<FJsonObject> Root;
|
||||||
|
const TSharedRef<TJsonReader<>> Reader = TJsonReaderFactory<>::Create(Resp->GetContentAsString());
|
||||||
|
if (FJsonSerializer::Deserialize(Reader, Root) && Root.IsValid())
|
||||||
|
{
|
||||||
|
Root->TryGetStringField(TEXT("signed_url"), SignedURL);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!SignedURL.IsEmpty())
|
||||||
|
{
|
||||||
|
// Token is embedded in the signed URL — no auth header needed.
|
||||||
|
Self->OpenWebSocket(SignedURL, TMap<FString, FString>());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const FString Msg = TEXT("Signed URL request succeeded but the response had no 'signed_url'.");
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_WS_ElevenLabs, Error, TEXT("%s"), *Msg);
|
||||||
|
Self->ConnectionState = EPS_AI_ConvAgent_ConnectionState_ElevenLabs::Error;
|
||||||
|
Self->OnError.Broadcast(Msg);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
FString Msg;
|
||||||
|
if (Code == 401 || Code == 403)
|
||||||
|
{
|
||||||
|
Msg = FString::Printf(TEXT("Authentication failed (HTTP %d): invalid or unauthorized ElevenLabs API key."), Code);
|
||||||
|
}
|
||||||
|
else if (Code == 0)
|
||||||
|
{
|
||||||
|
Msg = TEXT("Could not reach ElevenLabs to obtain a signed URL (network error).");
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
Msg = FString::Printf(TEXT("Failed to obtain a signed URL (HTTP %d)."), Code);
|
||||||
|
}
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_WS_ElevenLabs, Error, TEXT("%s"), *Msg);
|
||||||
|
Self->ConnectionState = EPS_AI_ConvAgent_ConnectionState_ElevenLabs::Error;
|
||||||
|
Self->OnError.Broadcast(Msg);
|
||||||
|
});
|
||||||
|
|
||||||
|
Request->ProcessRequest();
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy::OpenWebSocket(const FString& URL, const TMap<FString, FString>& UpgradeHeaders)
|
||||||
|
{
|
||||||
|
if (URL.IsEmpty())
|
||||||
|
{
|
||||||
|
const FString Msg = TEXT("Cannot open WebSocket: empty URL.");
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_WS_ElevenLabs, Error, TEXT("%s"), *Msg);
|
||||||
|
ConnectionState = EPS_AI_ConvAgent_ConnectionState_ElevenLabs::Error;
|
||||||
|
OnError.Broadcast(Msg);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Redact the query string (signed URLs carry an auth token there) before logging.
|
||||||
|
FString LogURL = URL;
|
||||||
|
int32 QueryIdx = INDEX_NONE;
|
||||||
|
if (URL.FindChar(TEXT('?'), QueryIdx))
|
||||||
|
{
|
||||||
|
LogURL = URL.Left(QueryIdx) + TEXT("?<params hidden>");
|
||||||
|
}
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_WS_ElevenLabs, Log, TEXT("Connecting to ElevenLabs: %s"), *LogURL);
|
||||||
|
|
||||||
WebSocket = FWebSocketsModule::Get().CreateWebSocket(URL, TEXT(""), UpgradeHeaders);
|
WebSocket = FWebSocketsModule::Get().CreateWebSocket(URL, TEXT(""), UpgradeHeaders);
|
||||||
|
|
||||||
@@ -164,6 +285,23 @@ void UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy::SendTextMessage(const FString&
|
|||||||
SendJsonMessage(Msg);
|
SendJsonMessage(Msg);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy::SendContextualUpdate(const FString& Text)
|
||||||
|
{
|
||||||
|
if (!IsConnected())
|
||||||
|
{
|
||||||
|
UE_LOG(LogPS_AI_ConvAgent_WS_ElevenLabs, Warning, TEXT("SendContextualUpdate: not connected."));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (Text.IsEmpty()) return;
|
||||||
|
|
||||||
|
// API: { "type": "contextual_update", "text": "..." } — injects context WITHOUT
|
||||||
|
// triggering an agent response (unlike user_message).
|
||||||
|
TSharedPtr<FJsonObject> Msg = MakeShareable(new FJsonObject());
|
||||||
|
Msg->SetStringField(TEXT("type"), TEXT("contextual_update"));
|
||||||
|
Msg->SetStringField(TEXT("text"), Text);
|
||||||
|
SendJsonMessage(Msg);
|
||||||
|
}
|
||||||
|
|
||||||
void UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy::SendInterrupt()
|
void UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy::SendInterrupt()
|
||||||
{
|
{
|
||||||
if (!IsConnected()) return;
|
if (!IsConnected()) return;
|
||||||
@@ -685,15 +823,18 @@ void UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy::HandleClientToolCall(const TSha
|
|||||||
{
|
{
|
||||||
for (const auto& Pair : (*ParamsObj)->Values)
|
for (const auto& Pair : (*ParamsObj)->Values)
|
||||||
{
|
{
|
||||||
|
// UE 5.8 changed FJsonObject::Values from TMap<FString,...> to TMap<FSharedString,...>.
|
||||||
|
// FSharedString has no implicit FString conversion; deref (*) yields const TCHAR*.
|
||||||
|
FString Key(*Pair.Key);
|
||||||
FString Value;
|
FString Value;
|
||||||
if (Pair.Value->TryGetString(Value))
|
if (Pair.Value->TryGetString(Value))
|
||||||
{
|
{
|
||||||
ToolCall.Parameters.Add(Pair.Key, Value);
|
ToolCall.Parameters.Add(Key, Value);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
// For non-string values, serialize to string
|
// For non-string values, serialize to string
|
||||||
ToolCall.Parameters.Add(Pair.Key, Pair.Value->AsString());
|
ToolCall.Parameters.Add(Key, Pair.Value->AsString());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -31,13 +31,12 @@ class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_Settings_ElevenLabs : public UObject
|
|||||||
GENERATED_BODY()
|
GENERATED_BODY()
|
||||||
|
|
||||||
public:
|
public:
|
||||||
/**
|
// NOTE: The ElevenLabs API key is NOT stored here on purpose.
|
||||||
* ElevenLabs API key.
|
// It is provided per-install by the client at runtime and persisted in a
|
||||||
* Obtain from https://elevenlabs.io – used to authenticate WebSocket connections.
|
// SaveGame slot (see UPS_AI_ConvAgent_SaveGame_ElevenLabs). Read/write it via
|
||||||
* Keep this secret; do not ship with the key hard-coded in a shipping build.
|
// UPS_AI_ConvAgent_BlueprintLibrary::Get/SetElevenLabsAPIKey(). This keeps the
|
||||||
*/
|
// key out of Project Settings / DefaultEngine.ini so it can never be committed
|
||||||
UPROPERTY(Config, EditAnywhere, Category = "PS AI ConvAgent|ElevenLabs API")
|
// or baked into a shipped build.
|
||||||
FString API_Key;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Server region for ElevenLabs API.
|
* Server region for ElevenLabs API.
|
||||||
@@ -45,18 +44,18 @@ public:
|
|||||||
* - US: force US servers (api.us.elevenlabs.io).
|
* - US: force US servers (api.us.elevenlabs.io).
|
||||||
* - EU / India: Enterprise-only data residency endpoints.
|
* - EU / India: Enterprise-only data residency endpoints.
|
||||||
*/
|
*/
|
||||||
UPROPERTY(Config, EditAnywhere, Category = "PS AI ConvAgent|ElevenLabs API")
|
UPROPERTY(Config, EditAnywhere, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs API")
|
||||||
EPS_AI_ConvAgent_ElevenLabsRegion ServerRegion = EPS_AI_ConvAgent_ElevenLabsRegion::Global;
|
EPS_AI_ConvAgent_ElevenLabsRegion ServerRegion = EPS_AI_ConvAgent_ElevenLabsRegion::Global;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Override the ElevenLabs WebSocket URL entirely. Leave empty to use ServerRegion setting.
|
* Override the ElevenLabs WebSocket URL entirely. Leave empty to use ServerRegion setting.
|
||||||
* Example: wss://custom-proxy.example.com/v1/convai/conversation?agent_id=YOUR_ID
|
* Example: wss://custom-proxy.example.com/v1/convai/conversation?agent_id=YOUR_ID
|
||||||
*/
|
*/
|
||||||
UPROPERTY(Config, EditAnywhere, AdvancedDisplay, Category = "PS AI ConvAgent|ElevenLabs API")
|
UPROPERTY(Config, EditAnywhere, AdvancedDisplay, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs API")
|
||||||
FString CustomWebSocketURL;
|
FString CustomWebSocketURL;
|
||||||
|
|
||||||
/** Log verbose WebSocket messages to the Output Log (useful during development). */
|
/** Log verbose WebSocket messages to the Output Log (useful during development). */
|
||||||
UPROPERTY(Config, EditAnywhere, AdvancedDisplay, Category = "PS AI ConvAgent|ElevenLabs API")
|
UPROPERTY(Config, EditAnywhere, AdvancedDisplay, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs API")
|
||||||
bool bVerboseLogging = false;
|
bool bVerboseLogging = false;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -65,7 +64,7 @@ public:
|
|||||||
* Injected at conversation start via dynamic variables — changes take effect
|
* Injected at conversation start via dynamic variables — changes take effect
|
||||||
* on the next conversation without re-syncing agents.
|
* on the next conversation without re-syncing agents.
|
||||||
*/
|
*/
|
||||||
UPROPERTY(Config, EditAnywhere, Category = "PS AI ConvAgent|Global Prompt",
|
UPROPERTY(Config, EditAnywhere, Category = "ASTERION|PS_AI_ConvAgent|Global Prompt",
|
||||||
meta = (MultiLine = "true",
|
meta = (MultiLine = "true",
|
||||||
ToolTip = "Context prompt shared by all agents.\nDescribe the world, era, common rules.\nChanges take effect on next conversation start (no re-sync needed)."))
|
ToolTip = "Context prompt shared by all agents.\nDescribe the world, era, common rules.\nChanges take effect on next conversation start (no re-sync needed)."))
|
||||||
FString GlobalContextPrompt;
|
FString GlobalContextPrompt;
|
||||||
@@ -119,7 +118,8 @@ public:
|
|||||||
return FModuleManager::Get().IsModuleLoaded("PS_AI_ConvAgent");
|
return FModuleManager::Get().IsModuleLoaded("PS_AI_ConvAgent");
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Access the settings object at runtime */
|
/** Access the settings object at runtime (region, custom URL, logging, global
|
||||||
|
* prompt). The API key is NOT here — see UPS_AI_ConvAgent_BlueprintLibrary. */
|
||||||
UPS_AI_ConvAgent_Settings_ElevenLabs* GetSettings() const;
|
UPS_AI_ConvAgent_Settings_ElevenLabs* GetSettings() const;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|||||||
@@ -0,0 +1,181 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "CoreMinimal.h"
|
||||||
|
#include "GameFramework/Actor.h"
|
||||||
|
#include "PS_AI_ConvAgent_ElevenLabsComponent.h" // component type + delegates + ConversationInfo struct
|
||||||
|
#include "PS_AI_ConvAgent_AgentDialogue.generated.h"
|
||||||
|
|
||||||
|
class UPS_AI_ConvAgent_ElevenLabsComponent;
|
||||||
|
|
||||||
|
/** Fired each time a speaker's line is forwarded to the other agent (subtitles / logging). */
|
||||||
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FPS_AI_ConvAgent_OnDialogueTurn,
|
||||||
|
int32, TurnIndex, AActor*, Speaker, const FString&, Text);
|
||||||
|
|
||||||
|
/** Fired when the dialogue stops (MaxTurns reached or StopDialogue). */
|
||||||
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPS_AI_ConvAgent_OnDialogueFinished,
|
||||||
|
int32, TotalTurns);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Orchestrator actor that makes TWO ElevenLabs agents converse with each other
|
||||||
|
* over TEXT (no STT): each agent's completed text response is fed as a user
|
||||||
|
* message to the OTHER agent, which then replies. Both agents still speak aloud
|
||||||
|
* (TTS), so humans in the level hear the whole conversation.
|
||||||
|
*
|
||||||
|
* Setup: place this actor in the level, assign AgentA / AgentB to two actors that
|
||||||
|
* each already carry a PS_AI_ConvAgent_ElevenLabsComponent, set an OpeningMessage,
|
||||||
|
* then call StartDialogue() (or enable bAutoStartOnBeginPlay).
|
||||||
|
*
|
||||||
|
* Runs on the SERVER (authority) — the agents' WebSockets are server-owned.
|
||||||
|
* By default forwarding happens when a speaker FINISHES speaking so the two
|
||||||
|
* voices never overlap (needs audio playback → listen-server/host). On a
|
||||||
|
* dedicated server with no audio device, set bWaitForSpeechToFinish=false to
|
||||||
|
* drive turns from the text event instead.
|
||||||
|
*/
|
||||||
|
UCLASS(Blueprintable)
|
||||||
|
class PS_AI_CONVAGENT_API APS_AI_ConvAgent_AgentDialogue : public AActor
|
||||||
|
{
|
||||||
|
GENERATED_BODY()
|
||||||
|
|
||||||
|
public:
|
||||||
|
APS_AI_ConvAgent_AgentDialogue();
|
||||||
|
|
||||||
|
/** Actor carrying the FIRST agent's ElevenLabs component. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue")
|
||||||
|
TObjectPtr<AActor> AgentA;
|
||||||
|
|
||||||
|
/** Actor carrying the SECOND agent's ElevenLabs component. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue")
|
||||||
|
TObjectPtr<AActor> AgentB;
|
||||||
|
|
||||||
|
/** Opening line sent to the first speaker to kick off the dialogue. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue")
|
||||||
|
FString OpeningMessage = TEXT("Bonjour.");
|
||||||
|
|
||||||
|
/** If true the opening message goes to AgentA (A replies first); else to AgentB. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue")
|
||||||
|
bool bAgentAStarts = true;
|
||||||
|
|
||||||
|
/** Max forwarded turns before auto-stopping. 0 = unlimited. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue", meta = (ClampMin = "0"))
|
||||||
|
int32 MaxTurns = 20;
|
||||||
|
|
||||||
|
/** Pause (s) after a speaker's turn before the other is prompted. Adds breathing room. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue", meta = (ClampMin = "0.0"))
|
||||||
|
float TurnDelaySeconds = 0.0f;
|
||||||
|
|
||||||
|
/** True (default): forward a line when the speaker FINISHES speaking (clean, non-overlapping voices; needs audio playback).
|
||||||
|
* False: forward as soon as the text is ready (robust on dedicated servers, but voices may overlap). */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue")
|
||||||
|
bool bWaitForSpeechToFinish = true;
|
||||||
|
|
||||||
|
/** Start the dialogue automatically on BeginPlay (authority only). */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue")
|
||||||
|
bool bAutoStartOnBeginPlay = false;
|
||||||
|
|
||||||
|
/** After the player interrupts (by speaking), ALL dialogue agents look at the
|
||||||
|
* player until the player moves farther than this (cm) from BOTH agents, then
|
||||||
|
* the agents are released back to normal behavior and their conversations end. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue", meta = (ClampMin = "0.0"))
|
||||||
|
float PlayerAddressReleaseDistance = 500.0f;
|
||||||
|
|
||||||
|
/** Sent to BOTH agents as a SILENT context update (ElevenLabs contextual_update — no
|
||||||
|
* reply triggered) the instant the player interrupts, so the agents know a new/real
|
||||||
|
* person is now addressing them while KEEPING the prior conversation context.
|
||||||
|
* Leave empty to inject nothing. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue", meta = (MultiLine = "true"))
|
||||||
|
FString InterruptContextMessage = TEXT("Une nouvelle personne, bien reelle, s'adresse maintenant directement a toi, ce n'est plus ton interlocuteur precedent. Garde en memoire la conversation qui vient d'avoir lieu, mais reponds desormais a cette personne.");
|
||||||
|
|
||||||
|
/** Sent (contextual_update, no reply) to the agent the player is NOT talking to: it
|
||||||
|
* is a witness. Empty = don't inform the bystander. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue", meta = (MultiLine = "true"))
|
||||||
|
FString BystanderContextMessage = TEXT("Une nouvelle personne, bien reelle, vient d'entrer et s'adresse a ton interlocuteur, pas a toi pour l'instant. Ecoute ce qui se dit, tu pourras y faire reference plus tard quand elle se tournera vers toi.");
|
||||||
|
|
||||||
|
/** Prefix for the "overheard" lines relayed to the witness agent during a 3-way
|
||||||
|
* exchange (contextual_update, no reply). Empty = disable passive overhearing. */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue")
|
||||||
|
FString OverheardPrefix = TEXT("[Tu entends la conversation a cote] ");
|
||||||
|
|
||||||
|
/** Fired each time a line is forwarded (TurnIndex, Speaker actor, the text). */
|
||||||
|
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|Dialogue")
|
||||||
|
FPS_AI_ConvAgent_OnDialogueTurn OnDialogueTurn;
|
||||||
|
|
||||||
|
/** Fired when the dialogue ends. */
|
||||||
|
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|Dialogue")
|
||||||
|
FPS_AI_ConvAgent_OnDialogueFinished OnDialogueFinished;
|
||||||
|
|
||||||
|
/** Open both conversations and kick off the exchange. Authority only. */
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Dialogue")
|
||||||
|
void StartDialogue();
|
||||||
|
|
||||||
|
/** Stop forwarding turns. Agents stay connected unless bEndConversations. */
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Dialogue")
|
||||||
|
void StopDialogue(bool bEndConversations = false);
|
||||||
|
|
||||||
|
/** True while the loop is active. */
|
||||||
|
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|Dialogue")
|
||||||
|
bool IsRunning() const { return bRunning; }
|
||||||
|
|
||||||
|
protected:
|
||||||
|
virtual void BeginPlay() override;
|
||||||
|
virtual void EndPlay(const EEndPlayReason::Type Reason) override;
|
||||||
|
virtual void Tick(float DeltaSeconds) override; // only active in "addressing player" mode
|
||||||
|
|
||||||
|
private:
|
||||||
|
UPROPERTY(Transient) TObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> CompA;
|
||||||
|
UPROPERTY(Transient) TObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> CompB;
|
||||||
|
|
||||||
|
// Single-floor turn state: exactly ONE agent holds the floor at a time; every
|
||||||
|
// event from the OTHER agent is ignored. A turn forwards exactly once, when the
|
||||||
|
// floor-holder's text AND its speech are both complete.
|
||||||
|
enum class EFloor : uint8 { None, A, B };
|
||||||
|
EFloor Floor = EFloor::None;
|
||||||
|
int32 TurnId = 0; // number of forwarded turns
|
||||||
|
bool bTextReady = false; // floor-holder's full text arrived this turn
|
||||||
|
bool bSpeechDone = false; // floor-holder finished speaking this turn
|
||||||
|
bool bForwardedThisTurn = false; // exactly-one-forward guard
|
||||||
|
FString PendingLine; // floor-holder's line for this turn
|
||||||
|
bool bRunning = false;
|
||||||
|
bool bOpeningSent = false;
|
||||||
|
FTimerHandle TurnTimer;
|
||||||
|
|
||||||
|
// "Addressing player" mode: after the player interrupts by speaking, ALL dialogue
|
||||||
|
// agents are pinned to look at the player (fast, via the gaze override) and the
|
||||||
|
// orchestrator ticks to auto-release them once the player leaves both agents.
|
||||||
|
bool bAddressingPlayer = false;
|
||||||
|
UPROPERTY(Transient) TObjectPtr<APawn> AddressedByPlayer;
|
||||||
|
EFloor EngagedFloor = EFloor::None; // which agent the player is currently addressing
|
||||||
|
|
||||||
|
APawn* ResolveSpeakingPlayer() const; // the player whose voice was just transcribed
|
||||||
|
void EndPlayerAddress(); // release agents → normal + end conversations
|
||||||
|
|
||||||
|
UPS_AI_ConvAgent_ElevenLabsComponent* ResolveAgentComp(AActor* Actor) const;
|
||||||
|
void BindAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Comp, bool bIsA);
|
||||||
|
void UnbindAll();
|
||||||
|
void TryKickoff();
|
||||||
|
void BeginTurn(EFloor NewFloor); // set floor + reset per-turn flags
|
||||||
|
bool IsFloor(bool bFromA) const; // is the given agent the current floor-holder?
|
||||||
|
void TryForward(); // forward once, when text AND speech done
|
||||||
|
|
||||||
|
UFUNCTION() void HandleTextA(const FString& Text);
|
||||||
|
UFUNCTION() void HandleTextB(const FString& Text);
|
||||||
|
UFUNCTION() void HandleStoppedA();
|
||||||
|
UFUNCTION() void HandleStoppedB();
|
||||||
|
UFUNCTION() void HandleConnectedA(const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& Info);
|
||||||
|
UFUNCTION() void HandleConnectedB(const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& Info);
|
||||||
|
|
||||||
|
// Per-agent interrupt/relay triggers. Knowing WHICH agent the player addressed lets
|
||||||
|
// us track the engaged agent and relay the exchange to the other ("witness") agent.
|
||||||
|
// Voice-onset = primary immediate trigger; transcript = backup + carries the words.
|
||||||
|
// Merely LOOKING at an agent never triggers these — only actually speaking does.
|
||||||
|
UFUNCTION() void HandleVoiceOnsetA();
|
||||||
|
UFUNCTION() void HandleVoiceOnsetB();
|
||||||
|
UFUNCTION() void HandleTranscriptA(const FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs& Segment);
|
||||||
|
UFUNCTION() void HandleTranscriptB(const FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs& Segment);
|
||||||
|
|
||||||
|
void OnPlayerSpoke(EFloor Which, const FString& PlayerWords); // player addressed agent Which
|
||||||
|
void InterruptForPlayer(EFloor Engaged); // enter 3-way floor mode (Engaged = addressee)
|
||||||
|
void SetEngaged(EFloor Which); // reframe both agents when the addressee changes
|
||||||
|
void RelayToBystander(EFloor EngagedSide, const FString& FormattedLine); // "overheard" context
|
||||||
|
};
|
||||||
@@ -24,8 +24,34 @@ public:
|
|||||||
* This allows per-instance override without modifying the Skeletal Mesh asset.
|
* This allows per-instance override without modifying the Skeletal Mesh asset.
|
||||||
* @param SkelMeshComp The target Skeletal Mesh Component.
|
* @param SkelMeshComp The target Skeletal Mesh Component.
|
||||||
* @param AnimBPClass The AnimBlueprint class to use as post-process (nullptr to clear). */
|
* @param AnimBPClass The AnimBlueprint class to use as post-process (nullptr to clear). */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Utilities",
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Utilities",
|
||||||
meta = (DisplayName = "Set Post Process Anim Blueprint"))
|
meta = (DisplayName = "Set Post Process Anim Blueprint"))
|
||||||
static void SetPostProcessAnimBlueprint(USkeletalMeshComponent* SkelMeshComp,
|
static void SetPostProcessAnimBlueprint(USkeletalMeshComponent* SkelMeshComp,
|
||||||
TSubclassOf<UAnimInstance> AnimBPClass);
|
TSubclassOf<UAnimInstance> AnimBPClass);
|
||||||
|
|
||||||
|
// ── ElevenLabs API key (per-user, client-provided) ────────────────────────
|
||||||
|
|
||||||
|
/** Store the client's ElevenLabs API key. Persists it per-user in
|
||||||
|
* Saved/Config/.../Game.ini AND applies it immediately to the running plugin
|
||||||
|
* (no restart needed). Call this from your settings UI once the user has
|
||||||
|
* entered — and ideally validated via "Test ElevenLabs API Key" — their key. */
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs API Key",
|
||||||
|
meta = (DisplayName = "Set ElevenLabs API Key"))
|
||||||
|
static void SetElevenLabsAPIKey(const FString& Key);
|
||||||
|
|
||||||
|
/** Return the currently stored client ElevenLabs API key (empty if none). */
|
||||||
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs API Key",
|
||||||
|
meta = (DisplayName = "Get ElevenLabs API Key"))
|
||||||
|
static FString GetElevenLabsAPIKey();
|
||||||
|
|
||||||
|
/** True if a non-empty client ElevenLabs API key has been stored.
|
||||||
|
* Use this on startup to decide whether to force the user to the key entry screen. */
|
||||||
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs API Key",
|
||||||
|
meta = (DisplayName = "Has ElevenLabs API Key"))
|
||||||
|
static bool HasElevenLabsAPIKey();
|
||||||
|
|
||||||
|
/** Clear the stored client ElevenLabs API key (both on disk and in memory). */
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs API Key",
|
||||||
|
meta = (DisplayName = "Clear ElevenLabs API Key"))
|
||||||
|
static void ClearElevenLabsAPIKey();
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnBodyExpressionChanged,
|
|||||||
// 4. The component listens to conversation lifecycle + emotion events.
|
// 4. The component listens to conversation lifecycle + emotion events.
|
||||||
// 5. The AnimNode reads GetSnapshot() and blends the upper body pose.
|
// 5. The AnimNode reads GetSnapshot() and blends the upper body pose.
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
UCLASS(ClassGroup = "PS AI ConvAgent", meta = (BlueprintSpawnableComponent),
|
UCLASS(ClassGroup = "ASTERION|PS_AI_ConvAgent", meta = (BlueprintSpawnableComponent),
|
||||||
DisplayName = "PS AI ConvAgent Body Expression")
|
DisplayName = "PS AI ConvAgent Body Expression")
|
||||||
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_BodyExpressionComponent : public UActorComponent
|
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_BodyExpressionComponent : public UActorComponent
|
||||||
{
|
{
|
||||||
@@ -68,28 +68,28 @@ public:
|
|||||||
|
|
||||||
/** Body pose map asset containing body animation lists per emotion.
|
/** Body pose map asset containing body animation lists per emotion.
|
||||||
* Create a dedicated PS_AI_ConvAgent_BodyPoseMap asset in the Content Browser. */
|
* Create a dedicated PS_AI_ConvAgent_BodyPoseMap asset in the Content Browser. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|BodyExpression",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|BodyExpression",
|
||||||
meta = (ToolTip = "Body Pose Map asset with animation lists per emotion.\nRight-click Content Browser > Miscellaneous > PS AI ConvAgent Body Pose Map."))
|
meta = (ToolTip = "Body Pose Map asset with animation lists per emotion.\nRight-click Content Browser > Miscellaneous > PS AI ConvAgent Body Pose Map."))
|
||||||
TObjectPtr<UPS_AI_ConvAgent_BodyPoseMap> BodyPoseMap;
|
TObjectPtr<UPS_AI_ConvAgent_BodyPoseMap> BodyPoseMap;
|
||||||
|
|
||||||
/** Whether body expressions are currently active (read-only).
|
/** Whether body expressions are currently active (read-only).
|
||||||
* Managed automatically: activates on conversation connect, deactivates on disconnect. */
|
* Managed automatically: activates on conversation connect, deactivates on disconnect. */
|
||||||
UPROPERTY(BlueprintReadOnly, Category = "PS AI ConvAgent|BodyExpression")
|
UPROPERTY(BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|BodyExpression")
|
||||||
bool bActive = false;
|
bool bActive = false;
|
||||||
|
|
||||||
/** How long (seconds) to blend in/out when conversation starts/ends. */
|
/** How long (seconds) to blend in/out when conversation starts/ends. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|BodyExpression",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|BodyExpression",
|
||||||
meta = (ClampMin = "0.05", ClampMax = "3.0"))
|
meta = (ClampMin = "0.05", ClampMax = "3.0"))
|
||||||
float ActivationBlendDuration = 0.5f;
|
float ActivationBlendDuration = 0.5f;
|
||||||
|
|
||||||
/** Crossfade duration (seconds) when switching between animations. */
|
/** Crossfade duration (seconds) when switching between animations. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|BodyExpression",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|BodyExpression",
|
||||||
meta = (ClampMin = "0.1", ClampMax = "3.0",
|
meta = (ClampMin = "0.1", ClampMax = "3.0",
|
||||||
ToolTip = "How long (seconds) to crossfade between animations.\n0.5 = snappy, 1.5 = smooth."))
|
ToolTip = "How long (seconds) to crossfade between animations.\n0.5 = snappy, 1.5 = smooth."))
|
||||||
float EmotionBlendDuration = 1.0f;
|
float EmotionBlendDuration = 1.0f;
|
||||||
|
|
||||||
/** Overall blend weight for body expressions. 1.0 = full, 0.5 = subtle. */
|
/** Overall blend weight for body expressions. 1.0 = full, 0.5 = subtle. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|BodyExpression",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|BodyExpression",
|
||||||
meta = (ClampMin = "0.0", ClampMax = "1.0",
|
meta = (ClampMin = "0.0", ClampMax = "1.0",
|
||||||
ToolTip = "Overall blend weight for body expressions.\n1.0 = full, 0.5 = subtle."))
|
ToolTip = "Overall blend weight for body expressions.\n1.0 = full, 0.5 = subtle."))
|
||||||
float BlendWeight = 1.0f;
|
float BlendWeight = 1.0f;
|
||||||
@@ -97,12 +97,12 @@ public:
|
|||||||
// ── Debug ────────────────────────────────────────────────────────────────
|
// ── Debug ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Enable debug logging for this component. */
|
/** Enable debug logging for this component. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Debug")
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Debug")
|
||||||
bool bDebug = false;
|
bool bDebug = false;
|
||||||
|
|
||||||
/** Verbosity level when bDebug is true.
|
/** Verbosity level when bDebug is true.
|
||||||
* 0 = minimal extras, 1 = normal debug, 2 = detailed, 3 = per-frame data. */
|
* 0 = minimal extras, 1 = normal debug, 2 = detailed, 3 = per-frame data. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Debug",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Debug",
|
||||||
meta = (ClampMin = "0", ClampMax = "3", EditCondition = "bDebug"))
|
meta = (ClampMin = "0", ClampMax = "3", EditCondition = "bDebug"))
|
||||||
int32 DebugVerbosity = 1;
|
int32 DebugVerbosity = 1;
|
||||||
|
|
||||||
@@ -117,22 +117,22 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** Get the active emotion. */
|
/** Get the active emotion. */
|
||||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|BodyExpression")
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|BodyExpression")
|
||||||
EPS_AI_ConvAgent_Emotion GetActiveEmotion() const { return ActiveEmotion; }
|
EPS_AI_ConvAgent_Emotion GetActiveEmotion() const { return ActiveEmotion; }
|
||||||
|
|
||||||
/** Get the active emotion intensity. */
|
/** Get the active emotion intensity. */
|
||||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|BodyExpression")
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|BodyExpression")
|
||||||
EPS_AI_ConvAgent_EmotionIntensity GetActiveIntensity() const { return ActiveEmotionIntensity; }
|
EPS_AI_ConvAgent_EmotionIntensity GetActiveIntensity() const { return ActiveEmotionIntensity; }
|
||||||
|
|
||||||
/** True when the agent is currently speaking (body uses N/M/E lists). */
|
/** True when the agent is currently speaking (body uses N/M/E lists). */
|
||||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|BodyExpression")
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|BodyExpression")
|
||||||
bool IsSpeaking() const { return bIsSpeaking; }
|
bool IsSpeaking() const { return bIsSpeaking; }
|
||||||
|
|
||||||
// ── Events ───────────────────────────────────────────────────────────────
|
// ── Events ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Fired when the body expression changes (emotion + intensity).
|
/** Fired when the body expression changes (emotion + intensity).
|
||||||
* Only fires when the emotion actually differs from the previous one. */
|
* Only fires when the emotion actually differs from the previous one. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|BodyExpression|Events")
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|BodyExpression|Events")
|
||||||
FOnBodyExpressionChanged OnBodyExpressionChanged;
|
FOnBodyExpressionChanged OnBodyExpressionChanged;
|
||||||
|
|
||||||
/** Current activation blend alpha (0 = fully inactive, 1 = fully active).
|
/** Current activation blend alpha (0 = fully inactive, 1 = fully active).
|
||||||
|
|||||||
@@ -91,11 +91,11 @@ struct PS_AI_CONVAGENT_API FPS_AI_ConvAgent_ConversationInfo_ElevenLabs
|
|||||||
GENERATED_BODY()
|
GENERATED_BODY()
|
||||||
|
|
||||||
/** Unique ID of this conversation session assigned by ElevenLabs. */
|
/** Unique ID of this conversation session assigned by ElevenLabs. */
|
||||||
UPROPERTY(BlueprintReadOnly, Category = "PS AI ConvAgent|ElevenLabs")
|
UPROPERTY(BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
FString ConversationID;
|
FString ConversationID;
|
||||||
|
|
||||||
/** Agent ID that is responding. */
|
/** Agent ID that is responding. */
|
||||||
UPROPERTY(BlueprintReadOnly, Category = "PS AI ConvAgent|ElevenLabs")
|
UPROPERTY(BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
FString AgentID;
|
FString AgentID;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -108,15 +108,15 @@ struct PS_AI_CONVAGENT_API FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs
|
|||||||
GENERATED_BODY()
|
GENERATED_BODY()
|
||||||
|
|
||||||
/** Transcribed text. */
|
/** Transcribed text. */
|
||||||
UPROPERTY(BlueprintReadOnly, Category = "PS AI ConvAgent|ElevenLabs")
|
UPROPERTY(BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
FString Text;
|
FString Text;
|
||||||
|
|
||||||
/** "user" or "agent". */
|
/** "user" or "agent". */
|
||||||
UPROPERTY(BlueprintReadOnly, Category = "PS AI ConvAgent|ElevenLabs")
|
UPROPERTY(BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
FString Speaker;
|
FString Speaker;
|
||||||
|
|
||||||
/** Whether this is a final transcript or a tentative (in-progress) one. */
|
/** Whether this is a final transcript or a tentative (in-progress) one. */
|
||||||
UPROPERTY(BlueprintReadOnly, Category = "PS AI ConvAgent|ElevenLabs")
|
UPROPERTY(BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
bool bIsFinal = false;
|
bool bIsFinal = false;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -155,15 +155,15 @@ struct PS_AI_CONVAGENT_API FPS_AI_ConvAgent_ClientToolCall_ElevenLabs
|
|||||||
GENERATED_BODY()
|
GENERATED_BODY()
|
||||||
|
|
||||||
/** Name of the tool the agent wants to invoke (e.g. "set_emotion"). */
|
/** Name of the tool the agent wants to invoke (e.g. "set_emotion"). */
|
||||||
UPROPERTY(BlueprintReadOnly, Category = "PS AI ConvAgent|ElevenLabs")
|
UPROPERTY(BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
FString ToolName;
|
FString ToolName;
|
||||||
|
|
||||||
/** Unique ID for this tool invocation — must be echoed back in client_tool_result. */
|
/** Unique ID for this tool invocation — must be echoed back in client_tool_result. */
|
||||||
UPROPERTY(BlueprintReadOnly, Category = "PS AI ConvAgent|ElevenLabs")
|
UPROPERTY(BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
FString ToolCallId;
|
FString ToolCallId;
|
||||||
|
|
||||||
/** Raw JSON parameters as key-value string pairs. */
|
/** Raw JSON parameters as key-value string pairs. */
|
||||||
UPROPERTY(BlueprintReadOnly, Category = "PS AI ConvAgent|ElevenLabs")
|
UPROPERTY(BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
TMap<FString, FString> Parameters;
|
TMap<FString, FString> Parameters;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -177,11 +177,11 @@ struct PS_AI_CONVAGENT_API FPS_AI_ConvAgent_AgentAction_ElevenLabs
|
|||||||
GENERATED_BODY()
|
GENERATED_BODY()
|
||||||
|
|
||||||
/** Action identifier sent by the LLM (e.g. "flee", "draw_weapon"). Keep short, snake_case. */
|
/** Action identifier sent by the LLM (e.g. "flee", "draw_weapon"). Keep short, snake_case. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs")
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
FString Name;
|
FString Name;
|
||||||
|
|
||||||
/** Human-readable description — helps the LLM understand when to use this action. */
|
/** Human-readable description — helps the LLM understand when to use this action. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs")
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
FString Description;
|
FString Description;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -210,23 +210,23 @@ struct PS_AI_CONVAGENT_API FPS_AI_ConvAgent_ToolParameter_ElevenLabs
|
|||||||
|
|
||||||
/** Parameter identifier (snake_case). Must match the name expected by the client code.
|
/** Parameter identifier (snake_case). Must match the name expected by the client code.
|
||||||
* Examples: "emotion", "intensity", "action", "target_name" */
|
* Examples: "emotion", "intensity", "action", "target_name" */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs",
|
||||||
meta = (ToolTip = "Parameter name (snake_case).\nThe key sent in the tool call JSON."))
|
meta = (ToolTip = "Parameter name (snake_case).\nThe key sent in the tool call JSON."))
|
||||||
FString Name;
|
FString Name;
|
||||||
|
|
||||||
/** Data type for this parameter. Maps to JSON Schema "type". */
|
/** Data type for this parameter. Maps to JSON Schema "type". */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs",
|
||||||
meta = (ToolTip = "Data type: String, Integer, Number, or Boolean."))
|
meta = (ToolTip = "Data type: String, Integer, Number, or Boolean."))
|
||||||
EPS_AI_ConvAgent_ToolParamType Type = EPS_AI_ConvAgent_ToolParamType::String;
|
EPS_AI_ConvAgent_ToolParamType Type = EPS_AI_ConvAgent_ToolParamType::String;
|
||||||
|
|
||||||
/** Whether this parameter is required in the tool call. */
|
/** Whether this parameter is required in the tool call. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs",
|
||||||
meta = (ToolTip = "If true, the LLM must provide this parameter."))
|
meta = (ToolTip = "If true, the LLM must provide this parameter."))
|
||||||
bool bRequired = true;
|
bool bRequired = true;
|
||||||
|
|
||||||
/** Description passed to the LLM explaining what this parameter is
|
/** Description passed to the LLM explaining what this parameter is
|
||||||
* and how to extract or determine its value from the conversation. */
|
* and how to extract or determine its value from the conversation. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs",
|
||||||
meta = (MultiLine = "true",
|
meta = (MultiLine = "true",
|
||||||
ToolTip = "LLM prompt describing this parameter.\nGuides the LLM on what value to provide."))
|
ToolTip = "LLM prompt describing this parameter.\nGuides the LLM on what value to provide."))
|
||||||
FString Description;
|
FString Description;
|
||||||
@@ -234,7 +234,7 @@ struct PS_AI_CONVAGENT_API FPS_AI_ConvAgent_ToolParameter_ElevenLabs
|
|||||||
/** Optional predefined values the LLM can choose from.
|
/** Optional predefined values the LLM can choose from.
|
||||||
* If empty, the LLM can use any value of the specified type.
|
* If empty, the LLM can use any value of the specified type.
|
||||||
* Examples: ["joy","sadness","anger"] for emotion, ["low","medium","high"] for intensity. */
|
* Examples: ["joy","sadness","anger"] for emotion, ["low","medium","high"] for intensity. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs",
|
||||||
meta = (ToolTip = "Predefined enum values (optional).\nIf set, the LLM must choose from these."))
|
meta = (ToolTip = "Predefined enum values (optional).\nIf set, the LLM must choose from these."))
|
||||||
TArray<FString> EnumValues;
|
TArray<FString> EnumValues;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -122,10 +122,21 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAgentPassiveGazeStarted);
|
|||||||
/** Fired when this agent stops passive gaze (left range or became selected). */
|
/** Fired when this agent stops passive gaze (left range or became selected). */
|
||||||
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAgentPassiveGazeStopped);
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAgentPassiveGazeStopped);
|
||||||
|
|
||||||
|
/** Fired on the FIRST voiced frame of the local user's mic (via FeedExternalAudio),
|
||||||
|
* independent of the ElevenLabs STT round-trip — used to interrupt an agent-to-agent
|
||||||
|
* dialogue on voice onset. Debounced (~once per utterance). */
|
||||||
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnUserVoiceDetected);
|
||||||
|
|
||||||
// Non-dynamic delegate for raw agent audio (high-frequency, C++ consumers only).
|
// Non-dynamic delegate for raw agent audio (high-frequency, C++ consumers only).
|
||||||
// Delivers PCM chunks as int16, 16kHz mono, little-endian.
|
// Delivers PCM chunks as int16, 16kHz mono, little-endian.
|
||||||
DECLARE_MULTICAST_DELEGATE_OneParam(FOnAgentAudioData, const TArray<uint8>& /*PCMData*/);
|
DECLARE_MULTICAST_DELEGATE_OneParam(FOnAgentAudioData, const TArray<uint8>& /*PCMData*/);
|
||||||
|
|
||||||
|
/** Fired on the SERVER for each player mic chunk relayed via ServerSendMicAudioFromPlayer
|
||||||
|
* (int16, 16kHz mono). Carries the speaking pawn so an external recorder (e.g. the
|
||||||
|
* PS_AI_ConvReplay bridge) can capture player speech for replay. */
|
||||||
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnPlayerAudioCaptured,
|
||||||
|
APawn*, SpeakerPawn, const TArray<uint8>&, PCMData);
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
// UPS_AI_ConvAgent_ElevenLabsComponent
|
// UPS_AI_ConvAgent_ElevenLabsComponent
|
||||||
//
|
//
|
||||||
@@ -139,7 +150,7 @@ DECLARE_MULTICAST_DELEGATE_OneParam(FOnAgentAudioData, const TArray<uint8>& /*PC
|
|||||||
// 4. React to events (OnAgentTranscript, OnAgentTextResponse, etc.) in Blueprint.
|
// 4. React to events (OnAgentTranscript, OnAgentTextResponse, etc.) in Blueprint.
|
||||||
// 5. Call EndConversation() when done.
|
// 5. Call EndConversation() when done.
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
UCLASS(ClassGroup = "PS AI ConvAgent", meta = (BlueprintSpawnableComponent),
|
UCLASS(ClassGroup = "ASTERION|PS_AI_ConvAgent", meta = (BlueprintSpawnableComponent),
|
||||||
DisplayName = "PS AI ConvAgent - ElevenLabs Conv")
|
DisplayName = "PS AI ConvAgent - ElevenLabs Conv")
|
||||||
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_ElevenLabsComponent : public UActorComponent
|
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_ElevenLabsComponent : public UActorComponent
|
||||||
{
|
{
|
||||||
@@ -153,18 +164,18 @@ public:
|
|||||||
/** Agent configuration data asset.
|
/** Agent configuration data asset.
|
||||||
* When set, the AgentID is resolved from this asset at conversation start.
|
* When set, the AgentID is resolved from this asset at conversation start.
|
||||||
* Create one via Content Browser → right-click → Miscellaneous → PS AI ConvAgent Agent Config. */
|
* Create one via Content Browser → right-click → Miscellaneous → PS AI ConvAgent Agent Config. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs",
|
||||||
meta = (ToolTip = "Agent configuration data asset.\nOverrides the AgentID string below when set."))
|
meta = (ToolTip = "Agent configuration data asset.\nOverrides the AgentID string below when set."))
|
||||||
TObjectPtr<UPS_AI_ConvAgent_AgentConfig_ElevenLabs> AgentConfig;
|
TObjectPtr<UPS_AI_ConvAgent_AgentConfig_ElevenLabs> AgentConfig;
|
||||||
|
|
||||||
/** ElevenLabs Agent ID used for this conversation. Leave empty to use the default from Project Settings > PS AI ConvAgent - ElevenLabs.
|
/** ElevenLabs Agent ID used for this conversation. Leave empty to use the default from Project Settings > PS AI ConvAgent - ElevenLabs.
|
||||||
* Overridden by AgentConfig if set. */
|
* Overridden by AgentConfig if set. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs",
|
||||||
meta = (ToolTip = "ElevenLabs Agent ID. Leave empty to use the project default from Project Settings.\nOverridden by AgentConfig if set."))
|
meta = (ToolTip = "ElevenLabs Agent ID. Leave empty to use the project default from Project Settings.\nOverridden by AgentConfig if set."))
|
||||||
FString AgentID;
|
FString AgentID;
|
||||||
|
|
||||||
/** How turn-taking is managed between the user and the agent.\n- Server VAD (recommended): ElevenLabs automatically detects when the user stops speaking.\n- Client Controlled: You manually call StartListening/StopListening (push-to-talk with a key). */
|
/** How turn-taking is managed between the user and the agent.\n- Server VAD (recommended): ElevenLabs automatically detects when the user stops speaking.\n- Client Controlled: You manually call StartListening/StopListening (push-to-talk with a key). */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs",
|
||||||
meta = (ToolTip = "Turn-taking mode.\n- Server VAD: ElevenLabs detects end-of-speech automatically (hands-free).\n- Client Controlled: You call StartListening/StopListening manually (push-to-talk)."))
|
meta = (ToolTip = "Turn-taking mode.\n- Server VAD: ElevenLabs detects end-of-speech automatically (hands-free).\n- Client Controlled: You call StartListening/StopListening manually (push-to-talk)."))
|
||||||
EPS_AI_ConvAgent_TurnMode_ElevenLabs TurnMode = EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server;
|
EPS_AI_ConvAgent_TurnMode_ElevenLabs TurnMode = EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server;
|
||||||
|
|
||||||
@@ -174,38 +185,38 @@ public:
|
|||||||
* The agent remembers the full conversation context between interactions.
|
* The agent remembers the full conversation context between interactions.
|
||||||
* When false (ephemeral), each StartConversation opens a fresh WebSocket session
|
* When false (ephemeral), each StartConversation opens a fresh WebSocket session
|
||||||
* and EndConversation closes it. */
|
* and EndConversation closes it. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs",
|
||||||
meta = (ToolTip = "Keep the WebSocket open across conversation cycles.\n- True: agent remembers the full conversation between interactions.\n- False: each StartConversation opens a new session."))
|
meta = (ToolTip = "Keep the WebSocket open across conversation cycles.\n- True: agent remembers the full conversation between interactions.\n- False: each StartConversation opens a new session."))
|
||||||
bool bPersistentSession = true;
|
bool bPersistentSession = true;
|
||||||
|
|
||||||
/** Automatically open the microphone as soon as the WebSocket connection is established. Only applies in Server VAD mode. In Client (push-to-talk) mode, you must call StartListening manually. */
|
/** Automatically open the microphone as soon as the WebSocket connection is established. Only applies in Server VAD mode. In Client (push-to-talk) mode, you must call StartListening manually. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs",
|
||||||
meta = (ToolTip = "Auto-open the microphone when the conversation starts.\nOnly applies in Server VAD mode. In push-to-talk mode, call StartListening() manually."))
|
meta = (ToolTip = "Auto-open the microphone when the conversation starts.\nOnly applies in Server VAD mode. In push-to-talk mode, call StartListening() manually."))
|
||||||
bool bAutoStartListening = true;
|
bool bAutoStartListening = true;
|
||||||
|
|
||||||
/** How many milliseconds of microphone audio to accumulate before sending a chunk to ElevenLabs. Lower values reduce latency but may degrade voice detection accuracy. Higher values are more reliable but add delay. */
|
/** How many milliseconds of microphone audio to accumulate before sending a chunk to ElevenLabs. Lower values reduce latency but may degrade voice detection accuracy. Higher values are more reliable but add delay. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs|Latency",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Latency",
|
||||||
meta = (ClampMin = "20", ClampMax = "500",
|
meta = (ClampMin = "20", ClampMax = "500",
|
||||||
ToolTip = "Mic audio chunk duration sent to ElevenLabs.\n- 50-80ms: lower latency, less reliable voice detection.\n- 100ms (default): good balance.\n- 150-250ms: more reliable, higher latency."))
|
ToolTip = "Mic audio chunk duration sent to ElevenLabs.\n- 50-80ms: lower latency, less reliable voice detection.\n- 100ms (default): good balance.\n- 150-250ms: more reliable, higher latency."))
|
||||||
int32 MicChunkDurationMs = 100;
|
int32 MicChunkDurationMs = 100;
|
||||||
|
|
||||||
/** Allow the user to interrupt the agent while it is speaking.\n- In Server VAD mode: the microphone stays open during agent speech and the server detects interruptions automatically.\n- In Client (push-to-talk) mode: pressing the talk key while the agent speaks sends an interrupt signal.\n- When disabled: the user must wait for the agent to finish speaking before talking. */
|
/** Allow the user to interrupt the agent while it is speaking.\n- In Server VAD mode: the microphone stays open during agent speech and the server detects interruptions automatically.\n- In Client (push-to-talk) mode: pressing the talk key while the agent speaks sends an interrupt signal.\n- When disabled: the user must wait for the agent to finish speaking before talking. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs",
|
||||||
meta = (ToolTip = "Allow the user to interrupt the agent while it speaks.\n- Server VAD: mic stays open, server detects user voice automatically.\n- Push-to-talk: pressing the talk key interrupts the agent.\n- Disabled: user must wait for the agent to finish."))
|
meta = (ToolTip = "Allow the user to interrupt the agent while it speaks.\n- Server VAD: mic stays open, server detects user voice automatically.\n- Push-to-talk: pressing the talk key interrupts the agent.\n- Disabled: user must wait for the agent to finish."))
|
||||||
bool bAllowInterruption = true;
|
bool bAllowInterruption = true;
|
||||||
|
|
||||||
/** Enable the OnAgentTranscript event, which provides real-time speech-to-text of what the user is saying. Disable if you don't need to display user speech to reduce processing overhead. */
|
/** Enable the OnAgentTranscript event, which provides real-time speech-to-text of what the user is saying. Disable if you don't need to display user speech to reduce processing overhead. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Fire OnAgentTranscript with real-time speech-to-text of user speech.\nDisable if you don't need to display what the user said."))
|
meta = (ToolTip = "Fire OnAgentTranscript with real-time speech-to-text of user speech.\nDisable if you don't need to display what the user said."))
|
||||||
bool bEnableUserTranscript = true;
|
bool bEnableUserTranscript = true;
|
||||||
|
|
||||||
/** Enable the OnAgentTextResponse event, which provides the agent's complete text response once fully generated. Disable if you only need the audio output. */
|
/** Enable the OnAgentTextResponse event, which provides the agent's complete text response once fully generated. Disable if you only need the audio output. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Fire OnAgentTextResponse with the agent's complete text once fully generated.\nDisable if you only need the audio output."))
|
meta = (ToolTip = "Fire OnAgentTextResponse with the agent's complete text once fully generated.\nDisable if you only need the audio output."))
|
||||||
bool bEnableAgentTextResponse = true;
|
bool bEnableAgentTextResponse = true;
|
||||||
|
|
||||||
/** Enable the OnAgentPartialResponse event, which streams the agent's text word-by-word as the LLM generates it. Use this for real-time subtitles that appear while the agent speaks, rather than waiting for the full response. */
|
/** Enable the OnAgentPartialResponse event, which streams the agent's text word-by-word as the LLM generates it. Use this for real-time subtitles that appear while the agent speaks, rather than waiting for the full response. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Fire OnAgentPartialResponse with streaming text fragments as the LLM generates them.\nIdeal for real-time subtitles. Each event gives one text chunk, not the accumulated text."))
|
meta = (ToolTip = "Fire OnAgentPartialResponse with streaming text fragments as the LLM generates them.\nIdeal for real-time subtitles. Each event gives one text chunk, not the accumulated text."))
|
||||||
bool bEnableAgentPartialResponse = false;
|
bool bEnableAgentPartialResponse = false;
|
||||||
|
|
||||||
@@ -214,7 +225,7 @@ public:
|
|||||||
* When adaptive mode is on, the system starts here and can only decrease
|
* When adaptive mode is on, the system starts here and can only decrease
|
||||||
* (never increase) as it measures that chunks arrive fast enough.
|
* (never increase) as it measures that chunks arrive fast enough.
|
||||||
* Set to 0 to disable pre-buffering entirely. */
|
* Set to 0 to disable pre-buffering entirely. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs|Latency",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Latency",
|
||||||
meta = (ClampMin = "0", ClampMax = "4000",
|
meta = (ClampMin = "0", ClampMax = "4000",
|
||||||
ToolTip = "Pre-buffer delay (ms) — your safe 'worst case' value.\nAdaptive mode can only decrease from here, never increase.\nSet 0 to disable pre-buffering entirely."))
|
ToolTip = "Pre-buffer delay (ms) — your safe 'worst case' value.\nAdaptive mode can only decrease from here, never increase.\nSet 0 to disable pre-buffering entirely."))
|
||||||
int32 AudioPreBufferMs = 300;
|
int32 AudioPreBufferMs = 300;
|
||||||
@@ -223,13 +234,13 @@ public:
|
|||||||
* lowers the pre-buffer when TTS chunks arrive fast enough.
|
* lowers the pre-buffer when TTS chunks arrive fast enough.
|
||||||
* The system can only decrease from AudioPreBufferMs — never increase.
|
* The system can only decrease from AudioPreBufferMs — never increase.
|
||||||
* Resets to AudioPreBufferMs at the start of each conversation. */
|
* Resets to AudioPreBufferMs at the start of each conversation. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs|Latency",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Latency",
|
||||||
meta = (EditCondition = "AudioPreBufferMs > 0",
|
meta = (EditCondition = "AudioPreBufferMs > 0",
|
||||||
ToolTip = "Automatically lower pre-buffer when connection is good.\nCan only decrease, never increase beyond AudioPreBufferMs.\nResets each conversation."))
|
ToolTip = "Automatically lower pre-buffer when connection is good.\nCan only decrease, never increase beyond AudioPreBufferMs.\nResets each conversation."))
|
||||||
bool bAdaptivePreBuffer = true;
|
bool bAdaptivePreBuffer = true;
|
||||||
|
|
||||||
/** Safety timeout: if the server does not start generating a response within this many seconds after the user stops speaking, fire OnAgentResponseTimeout. Set to 0 to disable. A normal response starts within 0.1-0.8s. */
|
/** Safety timeout: if the server does not start generating a response within this many seconds after the user stops speaking, fire OnAgentResponseTimeout. Set to 0 to disable. A normal response starts within 0.1-0.8s. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs",
|
||||||
meta = (ClampMin = "0.0",
|
meta = (ClampMin = "0.0",
|
||||||
ToolTip = "Seconds to wait for a server response after the user stops speaking.\nFires OnAgentResponseTimeout if exceeded. Normal latency is 0.1-0.8s.\nSet to 0 to disable. Default: 10s."))
|
ToolTip = "Seconds to wait for a server response after the user stops speaking.\nFires OnAgentResponseTimeout if exceeded. Normal latency is 0.1-0.8s.\nSet to 0 to disable. Default: 10s."))
|
||||||
float ResponseTimeoutSeconds = 10.0f;
|
float ResponseTimeoutSeconds = 10.0f;
|
||||||
@@ -237,7 +248,7 @@ public:
|
|||||||
/** Maximum auto-reconnection attempts on unexpected WebSocket disconnect.
|
/** Maximum auto-reconnection attempts on unexpected WebSocket disconnect.
|
||||||
* Uses exponential backoff (1s, 2s, 4s, ... up to 30s).
|
* Uses exponential backoff (1s, 2s, 4s, ... up to 30s).
|
||||||
* Set to 0 to disable auto-reconnection. */
|
* Set to 0 to disable auto-reconnection. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs",
|
||||||
meta = (ClampMin = "0",
|
meta = (ClampMin = "0",
|
||||||
ToolTip = "Auto-reconnect attempts on unexpected disconnect.\n0 = disabled. Uses exponential backoff."))
|
ToolTip = "Auto-reconnect attempts on unexpected disconnect.\n0 = disabled. Uses exponential backoff."))
|
||||||
int32 MaxReconnectAttempts = 5;
|
int32 MaxReconnectAttempts = 5;
|
||||||
@@ -248,7 +259,7 @@ public:
|
|||||||
* When set, the agent's audio playback will be spatialized in 3D — volume
|
* When set, the agent's audio playback will be spatialized in 3D — volume
|
||||||
* and panning change based on distance and direction from the listener.
|
* and panning change based on distance and direction from the listener.
|
||||||
* Leave null for non-spatialized (2D) audio (default, same as before). */
|
* Leave null for non-spatialized (2D) audio (default, same as before). */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs|Audio",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Audio",
|
||||||
meta = (ToolTip = "Sound attenuation for 3D spatialized audio.\nLeave null for non-spatialized (2D) playback."))
|
meta = (ToolTip = "Sound attenuation for 3D spatialized audio.\nLeave null for non-spatialized (2D) playback."))
|
||||||
USoundAttenuation* SoundAttenuation = nullptr;
|
USoundAttenuation* SoundAttenuation = nullptr;
|
||||||
|
|
||||||
@@ -256,104 +267,110 @@ public:
|
|||||||
|
|
||||||
/** Enable debug logging for this component.
|
/** Enable debug logging for this component.
|
||||||
* When false, only key events (init, errors, state changes) are logged. */
|
* When false, only key events (init, errors, state changes) are logged. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Debug")
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Debug")
|
||||||
bool bDebug = false;
|
bool bDebug = false;
|
||||||
|
|
||||||
/** Verbosity level when bDebug is true.
|
/** Verbosity level when bDebug is true.
|
||||||
* 0 = minimal extras, 1 = normal debug, 2 = detailed, 3 = per-frame data. */
|
* 0 = minimal extras, 1 = normal debug, 2 = detailed, 3 = per-frame data. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Debug",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Debug",
|
||||||
meta = (ClampMin = "0", ClampMax = "3", EditCondition = "bDebug"))
|
meta = (ClampMin = "0", ClampMax = "3", EditCondition = "bDebug"))
|
||||||
int32 DebugVerbosity = 1;
|
int32 DebugVerbosity = 1;
|
||||||
|
|
||||||
/** Show a separate latency debug HUD with detailed per-turn timing breakdown.
|
/** Show a separate latency debug HUD with detailed per-turn timing breakdown.
|
||||||
* Independent from bDebug — can be enabled alone via CVar ps.ai.ConvAgent.Debug.Latency. */
|
* Independent from bDebug — can be enabled alone via CVar ps.ai.ConvAgent.Debug.Latency. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Debug")
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Debug")
|
||||||
bool bDebugLatency = false;
|
bool bDebugLatency = false;
|
||||||
|
|
||||||
// ── Events ────────────────────────────────────────────────────────────────
|
// ── Events ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Fired when the WebSocket connection is established and the conversation session is ready. Provides the ConversationID and AgentID. */
|
/** Fired when the WebSocket connection is established and the conversation session is ready. Provides the ConversationID and AgentID. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Fires when the connection to ElevenLabs is established and the conversation is ready to begin."))
|
meta = (ToolTip = "Fires when the connection to ElevenLabs is established and the conversation is ready to begin."))
|
||||||
FOnAgentConnected OnAgentConnected;
|
FOnAgentConnected OnAgentConnected;
|
||||||
|
|
||||||
/** Fired when the WebSocket connection is closed (gracefully or due to an error). Provides the status code and reason. */
|
/** Fired when the WebSocket connection is closed (gracefully or due to an error). Provides the status code and reason. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Fires when the connection to ElevenLabs is closed. Check StatusCode and Reason for details."))
|
meta = (ToolTip = "Fires when the connection to ElevenLabs is closed. Check StatusCode and Reason for details."))
|
||||||
FOnAgentDisconnected OnAgentDisconnected;
|
FOnAgentDisconnected OnAgentDisconnected;
|
||||||
|
|
||||||
/** Fired on any connection or protocol error. The error message describes what went wrong. */
|
/** Fired on any connection or protocol error. The error message describes what went wrong. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Fires on connection or protocol errors. The ErrorMessage describes the issue."))
|
meta = (ToolTip = "Fires on connection or protocol errors. The ErrorMessage describes the issue."))
|
||||||
FOnAgentError OnAgentError;
|
FOnAgentError OnAgentError;
|
||||||
|
|
||||||
/** Fired with real-time speech-to-text of the user's voice. Includes both tentative (in-progress) and final transcripts. Requires bEnableUserTranscript to be true. */
|
/** Fired with real-time speech-to-text of the user's voice. Includes both tentative (in-progress) and final transcripts. Requires bEnableUserTranscript to be true. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Real-time speech-to-text of the user's voice.\nIncludes tentative and final transcripts. Enable with bEnableUserTranscript."))
|
meta = (ToolTip = "Real-time speech-to-text of the user's voice.\nIncludes tentative and final transcripts. Enable with bEnableUserTranscript."))
|
||||||
FOnAgentTranscript OnAgentTranscript;
|
FOnAgentTranscript OnAgentTranscript;
|
||||||
|
|
||||||
/** Fired once when the agent's complete text response is available. This is the full text that corresponds to the audio the agent speaks. Requires bEnableAgentTextResponse to be true. */
|
/** Fired once when the agent's complete text response is available. This is the full text that corresponds to the audio the agent speaks. Requires bEnableAgentTextResponse to be true. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "The agent's complete text response (matches the spoken audio).\nFires once when the full text is ready. Enable with bEnableAgentTextResponse."))
|
meta = (ToolTip = "The agent's complete text response (matches the spoken audio).\nFires once when the full text is ready. Enable with bEnableAgentTextResponse."))
|
||||||
FOnAgentTextResponse OnAgentTextResponse;
|
FOnAgentTextResponse OnAgentTextResponse;
|
||||||
|
|
||||||
/** Fired repeatedly as the LLM generates text, providing one word/fragment at a time. Use for real-time subtitles. Each call gives a new fragment, NOT the accumulated text. Requires bEnableAgentPartialResponse to be true. */
|
/** Fired repeatedly as the LLM generates text, providing one word/fragment at a time. Use for real-time subtitles. Each call gives a new fragment, NOT the accumulated text. Requires bEnableAgentPartialResponse to be true. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Streaming text fragments as the LLM generates them (word by word).\nIdeal for real-time subtitles. Enable with bEnableAgentPartialResponse."))
|
meta = (ToolTip = "Streaming text fragments as the LLM generates them (word by word).\nIdeal for real-time subtitles. Enable with bEnableAgentPartialResponse."))
|
||||||
FOnAgentPartialResponse OnAgentPartialResponse;
|
FOnAgentPartialResponse OnAgentPartialResponse;
|
||||||
|
|
||||||
/** Fired when the agent begins playing audio (first audio chunk received). Use this to trigger speech animations or UI indicators. */
|
/** Fired when the agent begins playing audio (first audio chunk received). Use this to trigger speech animations or UI indicators. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Fires when the agent starts speaking (first audio chunk). Use for lip-sync or UI feedback."))
|
meta = (ToolTip = "Fires when the agent starts speaking (first audio chunk). Use for lip-sync or UI feedback."))
|
||||||
FOnAgentStartedSpeaking OnAgentStartedSpeaking;
|
FOnAgentStartedSpeaking OnAgentStartedSpeaking;
|
||||||
|
|
||||||
/** Fired when audio playback actually starts — AFTER any pre-buffering delay.
|
/** Fired when audio playback actually starts — AFTER any pre-buffering delay.
|
||||||
* Unlike OnAgentStartedSpeaking (first chunk arrival), this fires when audio is audible.
|
* Unlike OnAgentStartedSpeaking (first chunk arrival), this fires when audio is audible.
|
||||||
* Use this for body/gesture animations that should be synced with audible speech. */
|
* Use this for body/gesture animations that should be synced with audible speech. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Fires when audio playback actually starts (after pre-buffering).\nSynced with audible speech. Use for body animations."))
|
meta = (ToolTip = "Fires when audio playback actually starts (after pre-buffering).\nSynced with audible speech. Use for body animations."))
|
||||||
FOnAudioPlaybackStarted OnAudioPlaybackStarted;
|
FOnAudioPlaybackStarted OnAudioPlaybackStarted;
|
||||||
|
|
||||||
/** Fired when the agent finishes playing all audio. Use this to re-open the microphone (in Server VAD mode without interruption) or update UI. */
|
/** Fired when the agent finishes playing all audio. Use this to re-open the microphone (in Server VAD mode without interruption) or update UI. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Fires when the agent finishes speaking. Use to re-open the mic or update UI."))
|
meta = (ToolTip = "Fires when the agent finishes speaking. Use to re-open the mic or update UI."))
|
||||||
FOnAgentStoppedSpeaking OnAgentStoppedSpeaking;
|
FOnAgentStoppedSpeaking OnAgentStoppedSpeaking;
|
||||||
|
|
||||||
/** Fired when the agent's speech is interrupted (either by the user speaking over it, or by a manual InterruptAgent call). The audio playback is automatically stopped. */
|
/** Fired when the agent's speech is interrupted (either by the user speaking over it, or by a manual InterruptAgent call). The audio playback is automatically stopped. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Fires when the agent is interrupted mid-speech. Audio is automatically stopped."))
|
meta = (ToolTip = "Fires when the agent is interrupted mid-speech. Audio is automatically stopped."))
|
||||||
FOnAgentInterrupted OnAgentInterrupted;
|
FOnAgentInterrupted OnAgentInterrupted;
|
||||||
|
|
||||||
|
/** Fires on local user voice onset (first voiced mic frame via FeedExternalAudio),
|
||||||
|
* without waiting for the ElevenLabs STT round-trip. Used to interrupt an
|
||||||
|
* agent-to-agent dialogue the instant the player speaks. */
|
||||||
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events")
|
||||||
|
FOnUserVoiceDetected OnUserVoiceDetected;
|
||||||
|
|
||||||
/** Fired when the server starts generating a response (before any audio arrives). Use this for "thinking..." UI feedback. In push-to-talk mode, the microphone is automatically closed when this fires. */
|
/** Fired when the server starts generating a response (before any audio arrives). Use this for "thinking..." UI feedback. In push-to-talk mode, the microphone is automatically closed when this fires. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Fires when the server starts generating (before audio arrives).\nUse for 'thinking...' UI. Mic is auto-closed in push-to-talk mode."))
|
meta = (ToolTip = "Fires when the server starts generating (before audio arrives).\nUse for 'thinking...' UI. Mic is auto-closed in push-to-talk mode."))
|
||||||
FOnAgentStartedGenerating OnAgentStartedGenerating;
|
FOnAgentStartedGenerating OnAgentStartedGenerating;
|
||||||
|
|
||||||
/** Fired if the server does not start generating a response within ResponseTimeoutSeconds after the user stops speaking. Use this to show a "try again" message or automatically re-open the microphone. */
|
/** Fired if the server does not start generating a response within ResponseTimeoutSeconds after the user stops speaking. Use this to show a "try again" message or automatically re-open the microphone. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Fires if the server doesn't respond within ResponseTimeoutSeconds.\nUse to show 'try again' or re-open the mic automatically."))
|
meta = (ToolTip = "Fires if the server doesn't respond within ResponseTimeoutSeconds.\nUse to show 'try again' or re-open the mic automatically."))
|
||||||
FOnAgentResponseTimeout OnAgentResponseTimeout;
|
FOnAgentResponseTimeout OnAgentResponseTimeout;
|
||||||
|
|
||||||
/** Fired when the agent changes emotion via the "set_emotion" client tool. The emotion is set BEFORE the corresponding audio arrives, giving you time to smoothly blend facial expressions. */
|
/** Fired when the agent changes emotion via the "set_emotion" client tool. The emotion is set BEFORE the corresponding audio arrives, giving you time to smoothly blend facial expressions. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Fires when the agent sets an emotion (joy, sadness, surprise, fear, anger, disgust).\nDriven by the 'set_emotion' client tool. Arrives before the audio."))
|
meta = (ToolTip = "Fires when the agent sets an emotion (joy, sadness, surprise, fear, anger, disgust).\nDriven by the 'set_emotion' client tool. Arrives before the audio."))
|
||||||
FOnAgentEmotionChanged OnAgentEmotionChanged;
|
FOnAgentEmotionChanged OnAgentEmotionChanged;
|
||||||
|
|
||||||
/** Fired when the LLM triggers a perform_action tool call.
|
/** Fired when the LLM triggers a perform_action tool call.
|
||||||
* ActionName is the identifier from the ActionSet (e.g. "flee", "draw_weapon").
|
* ActionName is the identifier from the ActionSet (e.g. "flee", "draw_weapon").
|
||||||
* Handle this in Blueprint to play animations, trigger AI behavior, etc. */
|
* Handle this in Blueprint to play animations, trigger AI behavior, etc. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Fires when the agent triggers a physical action.\nActionName matches the Name field from the ActionSet Data Asset."))
|
meta = (ToolTip = "Fires when the agent triggers a physical action.\nActionName matches the Name field from the ActionSet Data Asset."))
|
||||||
FOnAgentActionRequested OnAgentActionRequested;
|
FOnAgentActionRequested OnAgentActionRequested;
|
||||||
|
|
||||||
/** Fired for client tool calls that are NOT automatically handled (i.e. not "set_emotion", not "perform_action"). You must call GetWebSocketProxy()->SendClientToolResult() to respond. */
|
/** Fired for client tool calls that are NOT automatically handled (i.e. not "set_emotion", not "perform_action"). You must call GetWebSocketProxy()->SendClientToolResult() to respond. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Fires for custom client tool calls (not set_emotion, not perform_action).\nYou must respond via GetWebSocketProxy()->SendClientToolResult()."))
|
meta = (ToolTip = "Fires for custom client tool calls (not set_emotion, not perform_action).\nYou must respond via GetWebSocketProxy()->SendClientToolResult()."))
|
||||||
FOnAgentClientToolCall OnAgentClientToolCall;
|
FOnAgentClientToolCall OnAgentClientToolCall;
|
||||||
|
|
||||||
/** Fired when the active speaker changes in a multi-player shared conversation.
|
/** Fired when the active speaker changes in a multi-player shared conversation.
|
||||||
* NewSpeaker is null when no one is speaking (idle timeout). */
|
* NewSpeaker is null when no one is speaking (idle timeout). */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Fires when the speaking player changes.\nNewSpeaker is null when no one is speaking."))
|
meta = (ToolTip = "Fires when the speaking player changes.\nNewSpeaker is null when no one is speaking."))
|
||||||
FOnActiveSpeakerChanged OnActiveSpeakerChanged;
|
FOnActiveSpeakerChanged OnActiveSpeakerChanged;
|
||||||
|
|
||||||
@@ -361,26 +378,26 @@ public:
|
|||||||
* components back to neutral. The NPC is fully idle and the game can start
|
* components back to neutral. The NPC is fully idle and the game can start
|
||||||
* the physical action (flee, draw weapon, etc.).
|
* the physical action (flee, draw weapon, etc.).
|
||||||
* Call ForceEnableConversation() later to allow conversation again. */
|
* Call ForceEnableConversation() later to allow conversation again. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Fires when all animation components have returned to neutral after ForceDisableConversation().\nThe agent is ready for a game action."))
|
meta = (ToolTip = "Fires when all animation components have returned to neutral after ForceDisableConversation().\nThe agent is ready for a game action."))
|
||||||
FOnReadyForAction OnReadyForAction;
|
FOnReadyForAction OnReadyForAction;
|
||||||
|
|
||||||
/** Fired when this agent starts passively tracking a nearby player (head/eyes gaze). */
|
/** Fired when this agent starts passively tracking a nearby player (head/eyes gaze). */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Fires when this agent starts passively tracking a nearby player.\nTriggered by proximity awareness — the agent is not selected for conversation."))
|
meta = (ToolTip = "Fires when this agent starts passively tracking a nearby player.\nTriggered by proximity awareness — the agent is not selected for conversation."))
|
||||||
FOnAgentPassiveGazeStarted OnPassiveGazeStarted;
|
FOnAgentPassiveGazeStarted OnPassiveGazeStarted;
|
||||||
|
|
||||||
/** Fired when this agent stops passively tracking a player (left range or selected). */
|
/** Fired when this agent stops passively tracking a player (left range or selected). */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events",
|
||||||
meta = (ToolTip = "Fires when this agent stops passively tracking a player.\nThe player left range or selected this agent for active conversation."))
|
meta = (ToolTip = "Fires when this agent stops passively tracking a player.\nThe player left range or selected this agent for active conversation."))
|
||||||
FOnAgentPassiveGazeStopped OnPassiveGazeStopped;
|
FOnAgentPassiveGazeStopped OnPassiveGazeStopped;
|
||||||
|
|
||||||
/** The current emotion of the agent, as set by the "set_emotion" client tool. Defaults to Neutral. */
|
/** The current emotion of the agent, as set by the "set_emotion" client tool. Defaults to Neutral. */
|
||||||
UPROPERTY(ReplicatedUsing = OnRep_Emotion, BlueprintReadOnly, Category = "PS AI ConvAgent|ElevenLabs")
|
UPROPERTY(ReplicatedUsing = OnRep_Emotion, BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
EPS_AI_ConvAgent_Emotion CurrentEmotion = EPS_AI_ConvAgent_Emotion::Neutral;
|
EPS_AI_ConvAgent_Emotion CurrentEmotion = EPS_AI_ConvAgent_Emotion::Neutral;
|
||||||
|
|
||||||
/** The current emotion intensity. Defaults to Medium. */
|
/** The current emotion intensity. Defaults to Medium. */
|
||||||
UPROPERTY(ReplicatedUsing = OnRep_Emotion, BlueprintReadOnly, Category = "PS AI ConvAgent|ElevenLabs")
|
UPROPERTY(ReplicatedUsing = OnRep_Emotion, BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
EPS_AI_ConvAgent_EmotionIntensity CurrentEmotionIntensity = EPS_AI_ConvAgent_EmotionIntensity::Medium;
|
EPS_AI_ConvAgent_EmotionIntensity CurrentEmotionIntensity = EPS_AI_ConvAgent_EmotionIntensity::Medium;
|
||||||
|
|
||||||
// ── Raw audio data (C++ only, used by LipSync component) ────────────────
|
// ── Raw audio data (C++ only, used by LipSync component) ────────────────
|
||||||
@@ -388,30 +405,50 @@ public:
|
|||||||
* Used internally by UPS_AI_ConvAgent_LipSyncComponent for spectral analysis. */
|
* Used internally by UPS_AI_ConvAgent_LipSyncComponent for spectral analysis. */
|
||||||
FOnAgentAudioData OnAgentAudioData;
|
FOnAgentAudioData OnAgentAudioData;
|
||||||
|
|
||||||
|
/** Fired on the SERVER for each player mic chunk relayed via ServerSendMicAudioFromPlayer
|
||||||
|
* (int16, 16kHz mono). Lets an external recorder capture player speech for replay.
|
||||||
|
* Not fired on clients — only the server holds the relayed mic stream. */
|
||||||
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events")
|
||||||
|
FOnPlayerAudioCaptured OnPlayerAudioCaptured;
|
||||||
|
|
||||||
// ── Network state (replicated) ───────────────────────────────────────────
|
// ── Network state (replicated) ───────────────────────────────────────────
|
||||||
|
|
||||||
/** True when one or more players are in conversation with this NPC.
|
/** True when one or more players are in conversation with this NPC.
|
||||||
* Replicated to all clients for UI feedback, gaze, LOD, etc. */
|
* Replicated to all clients for UI feedback, gaze, LOD, etc. */
|
||||||
UPROPERTY(ReplicatedUsing = OnRep_ConversationState, BlueprintReadOnly, Category = "PS AI ConvAgent|Network")
|
UPROPERTY(ReplicatedUsing = OnRep_ConversationState, BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|Network")
|
||||||
bool bNetIsConversing = false;
|
bool bNetIsConversing = false;
|
||||||
|
|
||||||
/** All player pawns currently in conversation with this NPC.
|
/** All player pawns currently in conversation with this NPC.
|
||||||
* Multiple players can share the same agent — audio is routed via speaker arbitration.
|
* Multiple players can share the same agent — audio is routed via speaker arbitration.
|
||||||
* Replicated to ALL clients for gaze target and LOD distance checks. */
|
* Replicated to ALL clients for gaze target and LOD distance checks. */
|
||||||
UPROPERTY(ReplicatedUsing = OnRep_ConversationState, BlueprintReadOnly, Category = "PS AI ConvAgent|Network")
|
UPROPERTY(ReplicatedUsing = OnRep_ConversationState, BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|Network")
|
||||||
TArray<TObjectPtr<APawn>> NetConnectedPawns;
|
TArray<TObjectPtr<APawn>> NetConnectedPawns;
|
||||||
|
|
||||||
/** The player currently speaking (active audio sender). Null if no one is speaking.
|
/** The player currently speaking (active audio sender). Null if no one is speaking.
|
||||||
* Used by GazeComponent for target switching — the NPC looks at whoever is talking.
|
* Used by GazeComponent for target switching — the NPC looks at whoever is talking.
|
||||||
* Replicated to ALL clients so gaze updates everywhere. */
|
* Replicated to ALL clients so gaze updates everywhere. */
|
||||||
UPROPERTY(ReplicatedUsing = OnRep_ActiveSpeaker, BlueprintReadOnly, Category = "PS AI ConvAgent|Network")
|
UPROPERTY(ReplicatedUsing = OnRep_ActiveSpeaker, BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|Network")
|
||||||
TObjectPtr<APawn> NetActiveSpeakerPawn = nullptr;
|
TObjectPtr<APawn> NetActiveSpeakerPawn = nullptr;
|
||||||
|
|
||||||
|
/** When set, the conversation gaze looks at THIS actor instead of any connected
|
||||||
|
* player pawn — used by the agent-to-agent dialogue so the two agents look at
|
||||||
|
* EACH OTHER rather than at a bystander. Replicated so clients honor it too. */
|
||||||
|
UPROPERTY(ReplicatedUsing = OnRep_GazeOverrideTarget, BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|Gaze")
|
||||||
|
TObjectPtr<AActor> GazeOverrideTarget = nullptr;
|
||||||
|
|
||||||
|
UFUNCTION()
|
||||||
|
void OnRep_GazeOverrideTarget();
|
||||||
|
|
||||||
|
/** Force the conversation gaze to look at Target (pass null to clear and restore
|
||||||
|
* the normal player-tracking behaviour). Call on the server; replicates to clients. */
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Gaze")
|
||||||
|
void SetGazeOverrideTarget(AActor* Target);
|
||||||
|
|
||||||
// ── Multi-player speaker arbitration (server only) ──────────────────────
|
// ── Multi-player speaker arbitration (server only) ──────────────────────
|
||||||
|
|
||||||
/** Minimum seconds of silence from the current speaker before allowing a speaker switch.
|
/** Minimum seconds of silence from the current speaker before allowing a speaker switch.
|
||||||
* Prevents rapid ping-pong switching when both players make brief sounds. */
|
* Prevents rapid ping-pong switching when both players make brief sounds. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Network|MultiPlayer",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Network|MultiPlayer",
|
||||||
meta = (ClampMin = "0.0", ClampMax = "5.0",
|
meta = (ClampMin = "0.0", ClampMax = "5.0",
|
||||||
ToolTip = "Minimum silence from current speaker before switching to another.\nPrevents rapid gaze flip-flop."))
|
ToolTip = "Minimum silence from current speaker before switching to another.\nPrevents rapid gaze flip-flop."))
|
||||||
float SpeakerSwitchHysteresis = 0.3f;
|
float SpeakerSwitchHysteresis = 0.3f;
|
||||||
@@ -419,7 +456,7 @@ public:
|
|||||||
/** Seconds after last speech before the active speaker is cleared.
|
/** Seconds after last speech before the active speaker is cleared.
|
||||||
* When cleared, gaze returns to the last speaker position (or closest connected player).
|
* When cleared, gaze returns to the last speaker position (or closest connected player).
|
||||||
* Set to 0 to never clear (last speaker stays active indefinitely). */
|
* Set to 0 to never clear (last speaker stays active indefinitely). */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Network|MultiPlayer",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Network|MultiPlayer",
|
||||||
meta = (ClampMin = "0.0",
|
meta = (ClampMin = "0.0",
|
||||||
ToolTip = "Seconds of silence before clearing the active speaker.\n0 = last speaker stays active indefinitely."))
|
ToolTip = "Seconds of silence before clearing the active speaker.\n0 = last speaker stays active indefinitely."))
|
||||||
float SpeakerIdleTimeout = 3.0f;
|
float SpeakerIdleTimeout = 3.0f;
|
||||||
@@ -428,17 +465,29 @@ public:
|
|||||||
|
|
||||||
/** Distance (cm) beyond which remote clients stop receiving agent audio entirely.
|
/** Distance (cm) beyond which remote clients stop receiving agent audio entirely.
|
||||||
* The speaking player always receives full audio regardless of distance. */
|
* The speaking player always receives full audio regardless of distance. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Network|LOD",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Network|LOD",
|
||||||
meta = (ClampMin = "0", ToolTip = "Distance beyond which audio is culled for non-speaking players. 0 = no cull."))
|
meta = (ClampMin = "0", ToolTip = "Distance beyond which audio is culled for non-speaking players. 0 = no cull."))
|
||||||
float AudioLODCullDistance = 3000.f;
|
float AudioLODCullDistance = 3000.f;
|
||||||
|
|
||||||
/** Distance (cm) beyond which remote clients skip lip-sync / emotion processing.
|
/** Distance (cm) beyond which remote clients skip lip-sync / emotion processing.
|
||||||
* Audio still plays (if within AudioLODCullDistance) but without facial animation.
|
* Audio still plays (if within AudioLODCullDistance) but without facial animation.
|
||||||
* The speaking player always gets full lip-sync regardless of distance. */
|
* The speaking player always gets full lip-sync regardless of distance. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Network|LOD",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Network|LOD",
|
||||||
meta = (ClampMin = "0", ToolTip = "Distance beyond which lip-sync is skipped for non-speaking players. 0 = no LOD."))
|
meta = (ClampMin = "0", ToolTip = "Distance beyond which lip-sync is skipped for non-speaking players. 0 = no LOD."))
|
||||||
float LipSyncLODDistance = 1500.f;
|
float LipSyncLODDistance = 1500.f;
|
||||||
|
|
||||||
|
// ── Network mic uplink (clients) ─────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Seconds of silence to keep streaming to the server AFTER the player's last
|
||||||
|
* voiced chunk, so ElevenLabs Server VAD can detect end-of-turn. Beyond this
|
||||||
|
* tail, silent chunks are dropped to save bandwidth (clients only — the
|
||||||
|
* authority path always streams the full mic). Set >= the agent's end-of-turn
|
||||||
|
* silence threshold (+ margin). 0 = drop all silence (breaks Server VAD). */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Network|MultiPlayer",
|
||||||
|
meta = (ClampMin = "0.0", ClampMax = "5.0",
|
||||||
|
ToolTip = "Silence streamed after last voice so ElevenLabs detects end-of-turn (Server VAD).\nMust be >= the agent's end-of-turn silence. 0 = drop all silence (breaks Server VAD on clients)."))
|
||||||
|
float MicSilenceTailSeconds = 2.0f;
|
||||||
|
|
||||||
// ── Network RPCs ─────────────────────────────────────────────────────────
|
// ── Network RPCs ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Join a shared conversation with this NPC. Multiple players can join.
|
/** Join a shared conversation with this NPC. Multiple players can join.
|
||||||
@@ -472,9 +521,22 @@ public:
|
|||||||
UFUNCTION(Server, Reliable)
|
UFUNCTION(Server, Reliable)
|
||||||
void ServerRequestInterrupt();
|
void ServerRequestInterrupt();
|
||||||
|
|
||||||
/** Broadcast agent audio to all clients (Opus-compressed or raw PCM fallback). */
|
/** Broadcast agent audio to all clients (Opus-compressed or raw PCM fallback).
|
||||||
|
* Reliable: agent TTS must arrive complete and IN ORDER. Opus is a stateful
|
||||||
|
* (inter-frame-predictive) codec, so a dropped or reordered packet on an
|
||||||
|
* Unreliable channel desyncs the decoder → missing words + crackle. Reliable
|
||||||
|
* guarantees ordered, lossless delivery, keeping the decoder in sync.
|
||||||
|
* This does NOT reintroduce the old reliable-buffer overflow (which dropped
|
||||||
|
* VR connections): that came from 32-152 KB raw-PCM RPCs. With Opus each
|
||||||
|
* sub-chunk is ~2 KB (2-3 partial bunches, below the 8-bunch reliable-
|
||||||
|
* promotion threshold) at ~1.25 RPC/s, so in-flight bunches stay in single
|
||||||
|
* digits vs the 512-bunch buffer. Sub-chunking in HandleAudioReceived is what
|
||||||
|
* keeps packets this small — it must stay.
|
||||||
|
* bLastSubChunk marks the final sub-chunk of one ElevenLabs message so the
|
||||||
|
* client can reassemble all sub-chunks and enqueue ONCE per message (enqueuing
|
||||||
|
* each sub-chunk separately re-ran the playback state machine → pops). */
|
||||||
UFUNCTION(NetMulticast, Reliable)
|
UFUNCTION(NetMulticast, Reliable)
|
||||||
void MulticastReceiveAgentAudio(const TArray<uint8>& AudioData);
|
void MulticastReceiveAgentAudio(const TArray<uint8>& AudioData, bool bLastSubChunk);
|
||||||
|
|
||||||
/** Notify all clients that the agent started speaking (first audio chunk). */
|
/** Notify all clients that the agent started speaking (first audio chunk). */
|
||||||
UFUNCTION(NetMulticast, Reliable)
|
UFUNCTION(NetMulticast, Reliable)
|
||||||
@@ -514,25 +576,25 @@ public:
|
|||||||
* Open the WebSocket connection and start the conversation.
|
* Open the WebSocket connection and start the conversation.
|
||||||
* If bAutoStartListening is true, microphone capture also starts once connected.
|
* If bAutoStartListening is true, microphone capture also starts once connected.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
void StartConversation();
|
void StartConversation();
|
||||||
|
|
||||||
/** Close the WebSocket and stop all audio. */
|
/** Close the WebSocket and stop all audio. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
void EndConversation();
|
void EndConversation();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Start capturing microphone audio and streaming it to ElevenLabs.
|
* Start capturing microphone audio and streaming it to ElevenLabs.
|
||||||
* In Client turn mode, also sends a UserTurnStart signal.
|
* In Client turn mode, also sends a UserTurnStart signal.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
void StartListening();
|
void StartListening();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Stop capturing microphone audio.
|
* Stop capturing microphone audio.
|
||||||
* In Client turn mode, also sends a UserTurnEnd signal.
|
* In Client turn mode, also sends a UserTurnEnd signal.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
void StopListening();
|
void StopListening();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -540,40 +602,66 @@ public:
|
|||||||
* The agent will respond with audio and text just as if it heard you speak.
|
* The agent will respond with audio and text just as if it heard you speak.
|
||||||
* Useful for testing in the Editor or for text-based interaction.
|
* Useful for testing in the Editor or for text-based interaction.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
void SendTextMessage(const FString& Text);
|
void SendTextMessage(const FString& Text);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Inject a "silent" context update: the agent incorporates it as context but does
|
||||||
|
* NOT reply to it (unlike SendTextMessage). Used to tell an agent that a new/real
|
||||||
|
* interlocutor is now speaking while keeping the prior conversation. Authority only.
|
||||||
|
*/
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
|
void SendContextualUpdate(const FString& Text);
|
||||||
|
|
||||||
/** Interrupt the agent's current utterance. */
|
/** Interrupt the agent's current utterance. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
void InterruptAgent();
|
void InterruptAgent();
|
||||||
|
|
||||||
|
// ── Replay support (PS_AI_ConvReplay bridge) ─────────────────────────────
|
||||||
/**
|
/**
|
||||||
* Cleanly disable conversation on this agent so the NPC can switch to game AI.
|
* Play an external PCM buffer through this agent's voice, exactly as if it were
|
||||||
|
* live TTS: drives audio playback, lip-sync, body expression and start/stop events.
|
||||||
|
* Role-agnostic (works on a replay/simulated proxy); no LOD culling.
|
||||||
|
* @param PCM16 int16 samples, 16000 Hz mono, little-endian.
|
||||||
|
*/
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Replay")
|
||||||
|
void PlayExternalAudioPCM(const TArray<uint8>& PCM16);
|
||||||
|
|
||||||
|
/** Stop/flush any audio currently playing (used when scrubbing a replay). */
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Replay")
|
||||||
|
void StopExternalAudio();
|
||||||
|
|
||||||
|
/** Re-broadcast a recorded perform_action reaction during replay (fires OnAgentActionRequested). */
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Replay")
|
||||||
|
void TriggerReplayedAction(const FString& ActionName);
|
||||||
|
|
||||||
|
/** Re-broadcast a recorded custom client tool call during replay (fires OnAgentClientToolCall). */
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Replay")
|
||||||
|
void TriggerReplayedClientToolCall(const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Cleanly disable conversation on this agent.
|
||||||
* Ends the WebSocket connection, stops all audio, and begins blending all
|
* Ends the WebSocket connection, stops all audio, and begins blending all
|
||||||
* visual components (gaze, lip sync, facial expression, body expression)
|
* visual components (gaze, lip sync, facial expression, body expression)
|
||||||
* back to their neutral state. While disabled, the InteractionComponent
|
* back to their neutral state. While disabled, the InteractionComponent
|
||||||
* cannot auto-restart a conversation with this agent.
|
* cannot auto-restart a conversation with this agent.
|
||||||
*
|
*
|
||||||
* When all components have blended to neutral, fires OnReadyForAction.
|
* This function ONLY handles the conversation master-switch. It has no
|
||||||
|
* knowledge of action/reaction flow — if an action was pending via
|
||||||
|
* OnAgentActionRequested, OnReadyForAction will fire automatically once
|
||||||
|
* blend-out completes. If no action is pending, nothing else happens.
|
||||||
|
*
|
||||||
* Call ForceEnableConversation() later to allow conversation again.
|
* Call ForceEnableConversation() later to allow conversation again.
|
||||||
*
|
|
||||||
* Typical usage from Blueprint (OnAgentActionRequested):
|
|
||||||
* 1. ForceDisableConversation("flee")
|
|
||||||
* 2. Wait for OnReadyForAction — receives the same ActionName
|
|
||||||
* 3. Play action montage based on ActionName
|
|
||||||
* 4. When done, call ForceEnableConversation()
|
|
||||||
*
|
|
||||||
* @param ActionName The action to perform once neutral (passed through to OnReadyForAction).
|
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
void ForceDisableConversation(const FString& ActionName);
|
void ForceDisableConversation();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Re-enable conversation after a ForceDisableConversation() call.
|
* Re-enable conversation after a ForceDisableConversation() call.
|
||||||
* Clears the disabled flag so the InteractionComponent can auto-start
|
* Clears the disabled flag so the InteractionComponent can auto-start
|
||||||
* a new conversation on the next tick (if the player is still looking).
|
* a new conversation on the next tick (if the player is still looking).
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
void ForceEnableConversation();
|
void ForceEnableConversation();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -583,7 +671,7 @@ public:
|
|||||||
* The component must be connected and listening (StartListening called) for audio to be sent.
|
* The component must be connected and listening (StartListening called) for audio to be sent.
|
||||||
* @param FloatPCM Float32 samples, 16000 Hz mono (same format as MicrophoneCaptureComponent output).
|
* @param FloatPCM Float32 samples, 16000 Hz mono (same format as MicrophoneCaptureComponent output).
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
void FeedExternalAudio(const TArray<float>& FloatPCM);
|
void FeedExternalAudio(const TArray<float>& FloatPCM);
|
||||||
|
|
||||||
/** Receive mic audio from a specific player with speaker arbitration.
|
/** Receive mic audio from a specific player with speaker arbitration.
|
||||||
@@ -592,13 +680,13 @@ public:
|
|||||||
|
|
||||||
// ── State queries ─────────────────────────────────────────────────────────
|
// ── State queries ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
bool IsConnected() const;
|
bool IsConnected() const;
|
||||||
|
|
||||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
bool IsListening() const { return bIsListening; }
|
bool IsListening() const { return bIsListening; }
|
||||||
|
|
||||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
bool IsAgentSpeaking() const { return bAgentSpeaking; }
|
bool IsAgentSpeaking() const { return bAgentSpeaking; }
|
||||||
|
|
||||||
/** True when the agent is speaking OR the server hasn't confirmed the full response yet.
|
/** True when the agent is speaking OR the server hasn't confirmed the full response yet.
|
||||||
@@ -606,21 +694,21 @@ public:
|
|||||||
* produce more audio (e.g. during TTS inter-batch gaps). */
|
* produce more audio (e.g. during TTS inter-batch gaps). */
|
||||||
bool IsAgentSpeakingOrPending() const { return bAgentSpeaking || (bAgentGenerating && !bAgentResponseReceived); }
|
bool IsAgentSpeakingOrPending() const { return bAgentSpeaking || (bAgentGenerating && !bAgentResponseReceived); }
|
||||||
|
|
||||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& GetConversationInfo() const;
|
const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& GetConversationInfo() const;
|
||||||
|
|
||||||
/** True while conversation is force-disabled (ForceDisableConversation was called).
|
/** True while conversation is force-disabled (ForceDisableConversation was called).
|
||||||
* While disabled, StartConversation and ServerJoinConversation are blocked. */
|
* While disabled, StartConversation and ServerJoinConversation are blocked. */
|
||||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
bool IsConversationDisabled() const { return bConversationDisabledByAction; }
|
bool IsConversationDisabled() const { return bConversationForceDisabled; }
|
||||||
|
|
||||||
/** True while audio is being pre-buffered (playback hasn't started yet).
|
/** True while audio is being pre-buffered (playback hasn't started yet).
|
||||||
* Used by the LipSync component to pause viseme queue consumption. */
|
* Used by the LipSync component to pause viseme queue consumption. */
|
||||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
bool IsPreBuffering() const { return bPreBuffering; }
|
bool IsPreBuffering() const { return bPreBuffering; }
|
||||||
|
|
||||||
/** Access the underlying WebSocket proxy (advanced use). */
|
/** Access the underlying WebSocket proxy (advanced use). */
|
||||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy* GetWebSocketProxy() const { return WebSocketProxy; }
|
UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy* GetWebSocketProxy() const { return WebSocketProxy; }
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────
|
||||||
@@ -643,6 +731,9 @@ public:
|
|||||||
int32 GetMicChunkMinBytesPublic() const;
|
int32 GetMicChunkMinBytesPublic() const;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
// Debounce timestamp (seconds) for the OnUserVoiceDetected voice-onset broadcast.
|
||||||
|
double LastUserVoiceOnset = 0.0;
|
||||||
|
|
||||||
// ── Network OnRep handlers ───────────────────────────────────────────────
|
// ── Network OnRep handlers ───────────────────────────────────────────────
|
||||||
UFUNCTION()
|
UFUNCTION()
|
||||||
void OnRep_ConversationState();
|
void OnRep_ConversationState();
|
||||||
@@ -706,24 +797,33 @@ private:
|
|||||||
UPROPERTY()
|
UPROPERTY()
|
||||||
USoundWaveProcedural* ProceduralSoundWave = nullptr;
|
USoundWaveProcedural* ProceduralSoundWave = nullptr;
|
||||||
|
|
||||||
// ── ForceDisableConversation state ───────────────────────────────────────
|
// ── Conversation disable state ───────────────────────────────────────────
|
||||||
// Set by ForceDisableConversation(), cleared by ForceEnableConversation().
|
// Set by ForceDisableConversation(), cleared by ForceEnableConversation().
|
||||||
// While true, StartConversation() and ServerJoinConversation() are blocked.
|
// While true, StartConversation() and ServerJoinConversation() are blocked.
|
||||||
bool bConversationDisabledByAction = false;
|
bool bConversationForceDisabled = false;
|
||||||
// True while waiting for all visual components to blend back to neutral.
|
// True while waiting for all visual components to blend back to neutral
|
||||||
// Monitored in TickComponent — fires OnReadyForAction when complete.
|
// after ForceDisableConversation(). Monitored in TickComponent.
|
||||||
bool bWaitingForBlendOut = false;
|
bool bWaitingForBlendOut = false;
|
||||||
// The action name passed to ForceDisableConversation(), forwarded to OnReadyForAction.
|
// Accumulated time waiting for blend-out. Safety timeout to unblock
|
||||||
FString PendingActionName;
|
|
||||||
// Accumulated time waiting for blend-out. Safety timeout fires OnReadyForAction
|
|
||||||
// if components haven't reached neutral within this limit.
|
// if components haven't reached neutral within this limit.
|
||||||
float BlendOutElapsedTime = 0.0f;
|
float BlendOutElapsedTime = 0.0f;
|
||||||
static constexpr float BlendOutTimeoutSeconds = 5.0f;
|
static constexpr float BlendOutTimeoutSeconds = 5.0f;
|
||||||
|
|
||||||
|
// ── Action flow state (independent from disable state) ───────────────────
|
||||||
|
// Set when OnAgentActionRequested is broadcasted (LLM tool call).
|
||||||
|
// Cleared when OnReadyForAction is broadcasted.
|
||||||
|
// - If conversation is being disabled, OnReadyForAction waits for blend-out.
|
||||||
|
// - If not, OnReadyForAction fires on the next tick (quasi-immediate).
|
||||||
|
FString PendingActionName;
|
||||||
|
|
||||||
// ── State ─────────────────────────────────────────────────────────────────
|
// ── State ─────────────────────────────────────────────────────────────────
|
||||||
// Atomic: read from WASAPI background thread (OnMicrophoneDataCaptured), written from game thread.
|
// Atomic: read from WASAPI background thread (OnMicrophoneDataCaptured), written from game thread.
|
||||||
std::atomic<bool> bIsListening{false};
|
std::atomic<bool> bIsListening{false};
|
||||||
std::atomic<bool> bAgentSpeaking{false};
|
std::atomic<bool> bAgentSpeaking{false};
|
||||||
|
// Set when StartListening() is called before the WebSocket is connected.
|
||||||
|
// Consumed in HandleConnected() to open the microphone automatically once ready.
|
||||||
|
// Cleared by StopListening() or EndConversation() to cancel the pending request.
|
||||||
|
bool bPendingStartListening = false;
|
||||||
// True from the first agent_chat_response_part until the first audio chunk arrives.
|
// True from the first agent_chat_response_part until the first audio chunk arrives.
|
||||||
// Used to block StartListening() while the server is processing the previous turn.
|
// Used to block StartListening() while the server is processing the previous turn.
|
||||||
// Atomic: defensive — currently game-thread only, but documents thread-safety contract.
|
// Atomic: defensive — currently game-thread only, but documents thread-safety contract.
|
||||||
@@ -777,6 +877,26 @@ private:
|
|||||||
int32 AudioQueueReadOffset = 0;
|
int32 AudioQueueReadOffset = 0;
|
||||||
FCriticalSection AudioQueueLock;
|
FCriticalSection AudioQueueLock;
|
||||||
|
|
||||||
|
// Client-only: accumulates decoded PCM across the N sub-chunks of one
|
||||||
|
// ElevenLabs message; flushed to EnqueueAgentAudio() once, on the last
|
||||||
|
// sub-chunk (bLastSubChunk). Game-thread only. Reset in StopAgentAudio().
|
||||||
|
TArray<uint8> ClientReassemblyPCM;
|
||||||
|
|
||||||
|
// Server-only: sub-frame (<640 byte) tail carried from the previous agent
|
||||||
|
// audio message. Opus encodes whole frames only; without carry-over each
|
||||||
|
// message dropped its <1-frame tail, leaving a ~10-16ms hole + waveform
|
||||||
|
// discontinuity at every message splice → audible pops on clients. Reset
|
||||||
|
// per turn (new-turn guard in HandleAudioReceived + StopAgentAudio()).
|
||||||
|
TArray<uint8> AgentAudioEncodeLeftover;
|
||||||
|
|
||||||
|
// Debug: detects waveform discontinuities at enqueue/message splices (the
|
||||||
|
// signature of the audio "pops"). Tracks the last enqueued sample; a large
|
||||||
|
// jump to the next block's first sample is flagged and counted. Toggle with
|
||||||
|
// ps.ai.ConvAgent.Debug.ElevenLabs. Reset in StopAgentAudio().
|
||||||
|
int16 LastEnqueuedSample = 0;
|
||||||
|
bool bHasLastEnqueuedSample = false;
|
||||||
|
int32 AudioSpliceDiscontinuityCount = 0;
|
||||||
|
|
||||||
// Pre-buffer state: delay playback start to absorb TTS inter-chunk gaps.
|
// Pre-buffer state: delay playback start to absorb TTS inter-chunk gaps.
|
||||||
bool bPreBuffering = false;
|
bool bPreBuffering = false;
|
||||||
double PreBufferStartTime = 0.0;
|
double PreBufferStartTime = 0.0;
|
||||||
@@ -838,6 +958,11 @@ private:
|
|||||||
TArray<uint8> MicAccumulationBuffer;
|
TArray<uint8> MicAccumulationBuffer;
|
||||||
FCriticalSection MicSendLock;
|
FCriticalSection MicSendLock;
|
||||||
|
|
||||||
|
// Client mic uplink: timestamp (FPlatformTime::Seconds) of the last voiced chunk.
|
||||||
|
// Used to keep streaming a short silence tail after speech so ElevenLabs Server
|
||||||
|
// VAD can detect end-of-turn (see MicSilenceTailSeconds). 0 = never voiced yet.
|
||||||
|
double LastMicVoicedTime = 0.0;
|
||||||
|
|
||||||
/** Compute the minimum bytes from the user-facing MicChunkDurationMs.
|
/** Compute the minimum bytes from the user-facing MicChunkDurationMs.
|
||||||
* Formula: bytes = SampleRate * (ms / 1000) * BytesPerSample = 16000 * ms / 1000 * 2 = 32 * ms */
|
* Formula: bytes = SampleRate * (ms / 1000) * BytesPerSample = 16000 * ms / 1000 * 2 = 32 * ms */
|
||||||
int32 GetMicChunkMinBytes() const { return MicChunkDurationMs * 32; }
|
int32 GetMicChunkMinBytes() const { return MicChunkDurationMs * 32; }
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnExpressionChanged,
|
|||||||
// 4. Emotion animations crossfade smoothly (configurable duration).
|
// 4. Emotion animations crossfade smoothly (configurable duration).
|
||||||
// 5. The AnimNode reads GetCurrentEmotionCurves() and injects them into the pose.
|
// 5. The AnimNode reads GetCurrentEmotionCurves() and injects them into the pose.
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
UCLASS(ClassGroup = "PS AI ConvAgent", meta = (BlueprintSpawnableComponent),
|
UCLASS(ClassGroup = "ASTERION|PS_AI_ConvAgent", meta = (BlueprintSpawnableComponent),
|
||||||
DisplayName = "PS AI ConvAgent Facial Expression")
|
DisplayName = "PS AI ConvAgent Facial Expression")
|
||||||
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_FacialExpressionComponent : public UActorComponent
|
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_FacialExpressionComponent : public UActorComponent
|
||||||
{
|
{
|
||||||
@@ -48,23 +48,23 @@ public:
|
|||||||
|
|
||||||
/** Emotion pose map asset containing emotion AnimSequences (Normal / Medium / Extreme per emotion).
|
/** Emotion pose map asset containing emotion AnimSequences (Normal / Medium / Extreme per emotion).
|
||||||
* Create a dedicated PS_AI_ConvAgent_EmotionPoseMap asset in the Content Browser. */
|
* Create a dedicated PS_AI_ConvAgent_EmotionPoseMap asset in the Content Browser. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|FacialExpression",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|FacialExpression",
|
||||||
meta = (ToolTip = "Dedicated Emotion Pose Map asset.\nRight-click Content Browser → Miscellaneous → PS AI ConvAgent Emotion Pose Map."))
|
meta = (ToolTip = "Dedicated Emotion Pose Map asset.\nRight-click Content Browser → Miscellaneous → PS AI ConvAgent Emotion Pose Map."))
|
||||||
TObjectPtr<UPS_AI_ConvAgent_EmotionPoseMap> EmotionPoseMap;
|
TObjectPtr<UPS_AI_ConvAgent_EmotionPoseMap> EmotionPoseMap;
|
||||||
|
|
||||||
/** Whether facial expressions are currently active (read-only).
|
/** Whether facial expressions are currently active (read-only).
|
||||||
* Managed automatically: activates on conversation connect, deactivates on disconnect.
|
* Managed automatically: activates on conversation connect, deactivates on disconnect.
|
||||||
* The underlying emotion playback keeps running so reactivation is seamless. */
|
* The underlying emotion playback keeps running so reactivation is seamless. */
|
||||||
UPROPERTY(BlueprintReadOnly, Category = "PS AI ConvAgent|FacialExpression")
|
UPROPERTY(BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|FacialExpression")
|
||||||
bool bActive = true;
|
bool bActive = true;
|
||||||
|
|
||||||
/** How long (seconds) to blend in/out when bActive changes. */
|
/** How long (seconds) to blend in/out when bActive changes. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|FacialExpression",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|FacialExpression",
|
||||||
meta = (ClampMin = "0.05", ClampMax = "3.0"))
|
meta = (ClampMin = "0.05", ClampMax = "3.0"))
|
||||||
float ActivationBlendDuration = 0.5f;
|
float ActivationBlendDuration = 0.5f;
|
||||||
|
|
||||||
/** Emotion crossfade duration in seconds. */
|
/** Emotion crossfade duration in seconds. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|FacialExpression",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|FacialExpression",
|
||||||
meta = (ClampMin = "0.1", ClampMax = "3.0",
|
meta = (ClampMin = "0.1", ClampMax = "3.0",
|
||||||
ToolTip = "How long (seconds) to crossfade between emotions.\n0.5 = snappy, 1.5 = smooth."))
|
ToolTip = "How long (seconds) to crossfade between emotions.\n0.5 = snappy, 1.5 = smooth."))
|
||||||
float EmotionBlendDuration = 0.5f;
|
float EmotionBlendDuration = 0.5f;
|
||||||
@@ -73,12 +73,12 @@ public:
|
|||||||
|
|
||||||
/** Enable debug logging for this component.
|
/** Enable debug logging for this component.
|
||||||
* When false, only key events (init, errors, state changes) are logged. */
|
* When false, only key events (init, errors, state changes) are logged. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Debug")
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Debug")
|
||||||
bool bDebug = false;
|
bool bDebug = false;
|
||||||
|
|
||||||
/** Verbosity level when bDebug is true.
|
/** Verbosity level when bDebug is true.
|
||||||
* 0 = minimal extras, 1 = normal debug, 2 = detailed, 3 = per-frame data. */
|
* 0 = minimal extras, 1 = normal debug, 2 = detailed, 3 = per-frame data. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Debug",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Debug",
|
||||||
meta = (ClampMin = "0", ClampMax = "3", EditCondition = "bDebug"))
|
meta = (ClampMin = "0", ClampMax = "3", EditCondition = "bDebug"))
|
||||||
int32 DebugVerbosity = 1;
|
int32 DebugVerbosity = 1;
|
||||||
|
|
||||||
@@ -86,7 +86,7 @@ public:
|
|||||||
|
|
||||||
/** Get the current emotion curves evaluated from the playing AnimSequence.
|
/** Get the current emotion curves evaluated from the playing AnimSequence.
|
||||||
* Returns a COPY — safe to call from any thread (AnimNode worker thread). */
|
* Returns a COPY — safe to call from any thread (AnimNode worker thread). */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|FacialExpression")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|FacialExpression")
|
||||||
TMap<FName, float> GetCurrentEmotionCurves() const
|
TMap<FName, float> GetCurrentEmotionCurves() const
|
||||||
{
|
{
|
||||||
FScopeLock Lock(&EmotionCurveLock);
|
FScopeLock Lock(&EmotionCurveLock);
|
||||||
@@ -94,15 +94,15 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** Get the active emotion. */
|
/** Get the active emotion. */
|
||||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|FacialExpression")
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|FacialExpression")
|
||||||
EPS_AI_ConvAgent_Emotion GetActiveEmotion() const { return ActiveEmotion; }
|
EPS_AI_ConvAgent_Emotion GetActiveEmotion() const { return ActiveEmotion; }
|
||||||
|
|
||||||
/** Get the active emotion intensity. */
|
/** Get the active emotion intensity. */
|
||||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|FacialExpression")
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|FacialExpression")
|
||||||
EPS_AI_ConvAgent_EmotionIntensity GetActiveIntensity() const { return ActiveEmotionIntensity; }
|
EPS_AI_ConvAgent_EmotionIntensity GetActiveIntensity() const { return ActiveEmotionIntensity; }
|
||||||
|
|
||||||
/** Check if a curve name belongs to the mouth area (overridden by lip sync). */
|
/** Check if a curve name belongs to the mouth area (overridden by lip sync). */
|
||||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|FacialExpression")
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|FacialExpression")
|
||||||
static bool IsMouthCurve(const FName& CurveName);
|
static bool IsMouthCurve(const FName& CurveName);
|
||||||
|
|
||||||
// ── Events ───────────────────────────────────────────────────────────────
|
// ── Events ───────────────────────────────────────────────────────────────
|
||||||
@@ -111,7 +111,7 @@ public:
|
|||||||
* Subscribe in Blueprints to react to expression changes (trigger animations,
|
* Subscribe in Blueprints to react to expression changes (trigger animations,
|
||||||
* particles, sounds, etc.). Only fires when the emotion actually differs
|
* particles, sounds, etc.). Only fires when the emotion actually differs
|
||||||
* from the previous one. */
|
* from the previous one. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|FacialExpression|Events")
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|FacialExpression|Events")
|
||||||
FOnExpressionChanged OnExpressionChanged;
|
FOnExpressionChanged OnExpressionChanged;
|
||||||
|
|
||||||
/** Current activation blend alpha (0 = fully inactive, 1 = fully active).
|
/** Current activation blend alpha (0 = fully inactive, 1 = fully active).
|
||||||
|
|||||||
@@ -58,7 +58,7 @@ struct PS_AI_CONVAGENT_API FPS_AI_ConvAgent_NeckBoneEntry
|
|||||||
// 5. Set TargetOffset for actors without a skeleton (e.g. (0,0,160) for
|
// 5. Set TargetOffset for actors without a skeleton (e.g. (0,0,160) for
|
||||||
// eye-level on a simple actor).
|
// eye-level on a simple actor).
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
UCLASS(ClassGroup = "PS AI ConvAgent", meta = (BlueprintSpawnableComponent),
|
UCLASS(ClassGroup = "ASTERION|PS_AI_ConvAgent", meta = (BlueprintSpawnableComponent),
|
||||||
DisplayName = "PS AI ConvAgent Gaze")
|
DisplayName = "PS AI ConvAgent Gaze")
|
||||||
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_GazeComponent : public UActorComponent
|
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_GazeComponent : public UActorComponent
|
||||||
{
|
{
|
||||||
@@ -71,24 +71,24 @@ public:
|
|||||||
|
|
||||||
/** The actor to look at. Can be any actor (player, prop, etc.).
|
/** The actor to look at. Can be any actor (player, prop, etc.).
|
||||||
* Set to null to smoothly return to neutral. */
|
* Set to null to smoothly return to neutral. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Gaze",
|
||||||
meta = (ToolTip = "Target actor to look at.\nSet to null to return to neutral."))
|
meta = (ToolTip = "Target actor to look at.\nSet to null to return to neutral."))
|
||||||
TObjectPtr<AActor> TargetActor;
|
TObjectPtr<AActor> TargetActor;
|
||||||
|
|
||||||
/** When false, all gaze outputs smoothly blend to neutral (passthrough).
|
/** When false, all gaze outputs smoothly blend to neutral (passthrough).
|
||||||
* The underlying tracking keeps running so reactivation is seamless.
|
* The underlying tracking keeps running so reactivation is seamless.
|
||||||
* Managed automatically: activates on conversation connect, deactivates on disconnect. */
|
* Managed automatically: activates on conversation connect, deactivates on disconnect. */
|
||||||
UPROPERTY(BlueprintReadOnly, Category = "PS AI ConvAgent|Gaze")
|
UPROPERTY(BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|Gaze")
|
||||||
bool bActive = true;
|
bool bActive = true;
|
||||||
|
|
||||||
/** How long (seconds) to blend in/out when bActive changes. */
|
/** How long (seconds) to blend in/out when bActive changes. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Gaze",
|
||||||
meta = (ClampMin = "0.05", ClampMax = "3.0"))
|
meta = (ClampMin = "0.05", ClampMax = "3.0"))
|
||||||
float ActivationBlendDuration = 0.5f;
|
float ActivationBlendDuration = 0.5f;
|
||||||
|
|
||||||
/** When false, body rotation is frozen — only head and eyes track the target.
|
/** When false, body rotation is frozen — only head and eyes track the target.
|
||||||
* Useful to have the agent notice the player (eyes+head) before fully engaging (body). */
|
* Useful to have the agent notice the player (eyes+head) before fully engaging (body). */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Gaze",
|
||||||
meta = (ToolTip = "Enable body rotation toward the target.\nWhen false, only head and eyes track."))
|
meta = (ToolTip = "Enable body rotation toward the target.\nWhen false, only head and eyes track."))
|
||||||
bool bEnableBodyTracking = true;
|
bool bEnableBodyTracking = true;
|
||||||
|
|
||||||
@@ -96,7 +96,7 @@ public:
|
|||||||
* When enabled, TargetOffset is ignored and the agent looks at the midpoint
|
* When enabled, TargetOffset is ignored and the agent looks at the midpoint
|
||||||
* between the target pawn's eye bones.
|
* between the target pawn's eye bones.
|
||||||
* Fallback chain: eye bones → head bone → PawnViewLocation → FallbackEyeHeight. */
|
* Fallback chain: eye bones → head bone → PawnViewLocation → FallbackEyeHeight. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Gaze",
|
||||||
meta = (ToolTip = "Auto-target the pawn's eye bones for eye contact.\nFallback: eye bones > head bone > PawnViewLocation > FallbackEyeHeight."))
|
meta = (ToolTip = "Auto-target the pawn's eye bones for eye contact.\nFallback: eye bones > head bone > PawnViewLocation > FallbackEyeHeight."))
|
||||||
bool bAutoTargetEyes = true;
|
bool bAutoTargetEyes = true;
|
||||||
|
|
||||||
@@ -104,7 +104,7 @@ public:
|
|||||||
* AND the target is not a Pawn. For Pawns without skeleton, GetPawnViewLocation()
|
* AND the target is not a Pawn. For Pawns without skeleton, GetPawnViewLocation()
|
||||||
* is used instead (accounts for BaseEyeHeight automatically).
|
* is used instead (accounts for BaseEyeHeight automatically).
|
||||||
* Only applies to non-Pawn actors (props, triggers, etc.). */
|
* Only applies to non-Pawn actors (props, triggers, etc.). */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Gaze",
|
||||||
meta = (EditCondition = "bAutoTargetEyes", ClampMin = "0",
|
meta = (EditCondition = "bAutoTargetEyes", ClampMin = "0",
|
||||||
ToolTip = "Height offset (cm) for non-Pawn targets without skeleton.\nPawns use BaseEyeHeight automatically.\nOnly used as last-resort fallback for non-Pawn actors."))
|
ToolTip = "Height offset (cm) for non-Pawn targets without skeleton.\nPawns use BaseEyeHeight automatically.\nOnly used as last-resort fallback for non-Pawn actors."))
|
||||||
float FallbackEyeHeight = 160.0f;
|
float FallbackEyeHeight = 160.0f;
|
||||||
@@ -112,7 +112,7 @@ public:
|
|||||||
/** Offset from the target actor's origin to aim at.
|
/** Offset from the target actor's origin to aim at.
|
||||||
* Only used when bAutoTargetEyes is false.
|
* Only used when bAutoTargetEyes is false.
|
||||||
* E.g. (0,0,160) for eye-level on a simple actor. */
|
* E.g. (0,0,160) for eye-level on a simple actor. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Gaze",
|
||||||
meta = (EditCondition = "!bAutoTargetEyes",
|
meta = (EditCondition = "!bAutoTargetEyes",
|
||||||
ToolTip = "Offset from target actor origin.\nE.g. (0,0,160) for eye-level.\nOnly used when Auto Target Eyes is disabled."))
|
ToolTip = "Offset from target actor origin.\nE.g. (0,0,160) for eye-level.\nOnly used when Auto Target Eyes is disabled."))
|
||||||
FVector TargetOffset = FVector(0.0f, 0.0f, 0.0f);
|
FVector TargetOffset = FVector(0.0f, 0.0f, 0.0f);
|
||||||
@@ -127,44 +127,44 @@ public:
|
|||||||
// so small movements around the target don't re-trigger higher layers.
|
// so small movements around the target don't re-trigger higher layers.
|
||||||
|
|
||||||
/** Maximum head yaw rotation in degrees. */
|
/** Maximum head yaw rotation in degrees. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Gaze",
|
||||||
meta = (ClampMin = "0", ClampMax = "90"))
|
meta = (ClampMin = "0", ClampMax = "90"))
|
||||||
float MaxHeadYaw = 40.0f;
|
float MaxHeadYaw = 40.0f;
|
||||||
|
|
||||||
/** Maximum head pitch rotation in degrees. */
|
/** Maximum head pitch rotation in degrees. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Gaze",
|
||||||
meta = (ClampMin = "0", ClampMax = "90"))
|
meta = (ClampMin = "0", ClampMax = "90"))
|
||||||
float MaxHeadPitch = 30.0f;
|
float MaxHeadPitch = 30.0f;
|
||||||
|
|
||||||
/** Maximum horizontal eye angle in degrees. */
|
/** Maximum horizontal eye angle in degrees. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Gaze",
|
||||||
meta = (ClampMin = "0", ClampMax = "90"))
|
meta = (ClampMin = "0", ClampMax = "90"))
|
||||||
float MaxEyeHorizontal = 15.0f;
|
float MaxEyeHorizontal = 15.0f;
|
||||||
|
|
||||||
/** Maximum vertical eye angle in degrees. */
|
/** Maximum vertical eye angle in degrees. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Gaze",
|
||||||
meta = (ClampMin = "0", ClampMax = "90"))
|
meta = (ClampMin = "0", ClampMax = "90"))
|
||||||
float MaxEyeVertical = 10.0f;
|
float MaxEyeVertical = 10.0f;
|
||||||
|
|
||||||
// ── Smoothing speeds ─────────────────────────────────────────────────────
|
// ── Smoothing speeds ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Body rotation interpolation speed (lower = slower, more natural). */
|
/** Body rotation interpolation speed (lower = slower, more natural). */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Gaze",
|
||||||
meta = (ClampMin = "0.1", ClampMax = "20"))
|
meta = (ClampMin = "0.1", ClampMax = "20"))
|
||||||
float BodyInterpSpeed = 4.0f;
|
float BodyInterpSpeed = 4.0f;
|
||||||
|
|
||||||
/** Head rotation interpolation speed. */
|
/** Head rotation interpolation speed. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Gaze",
|
||||||
meta = (ClampMin = "0.1", ClampMax = "20"))
|
meta = (ClampMin = "0.1", ClampMax = "20"))
|
||||||
float HeadInterpSpeed = 4.0f;
|
float HeadInterpSpeed = 4.0f;
|
||||||
|
|
||||||
/** Eye movement interpolation speed (higher = snappier). */
|
/** Eye movement interpolation speed (higher = snappier). */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Gaze",
|
||||||
meta = (ClampMin = "0.1", ClampMax = "20"))
|
meta = (ClampMin = "0.1", ClampMax = "20"))
|
||||||
float EyeInterpSpeed = 5.0f;
|
float EyeInterpSpeed = 5.0f;
|
||||||
|
|
||||||
/** Interpolation speed when returning to neutral (TargetActor is null). */
|
/** Interpolation speed when returning to neutral (TargetActor is null). */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Gaze",
|
||||||
meta = (ClampMin = "0.1", ClampMax = "20"))
|
meta = (ClampMin = "0.1", ClampMax = "20"))
|
||||||
float ReturnToNeutralSpeed = 3.0f;
|
float ReturnToNeutralSpeed = 3.0f;
|
||||||
|
|
||||||
@@ -178,7 +178,7 @@ public:
|
|||||||
* 1.0 = head always points at target regardless of animation.
|
* 1.0 = head always points at target regardless of animation.
|
||||||
* 0.0 = gaze is additive on top of animation (old behavior).
|
* 0.0 = gaze is additive on top of animation (old behavior).
|
||||||
* Default: 1.0 for conversational AI (always look at who you talk to). */
|
* Default: 1.0 for conversational AI (always look at who you talk to). */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Gaze",
|
||||||
meta = (ClampMin = "0", ClampMax = "1"))
|
meta = (ClampMin = "0", ClampMax = "1"))
|
||||||
float HeadAnimationCompensation = 0.9f;
|
float HeadAnimationCompensation = 0.9f;
|
||||||
|
|
||||||
@@ -186,7 +186,7 @@ public:
|
|||||||
* 1.0 = eyes frozen on gaze target, animation's eye movement removed.
|
* 1.0 = eyes frozen on gaze target, animation's eye movement removed.
|
||||||
* 0.0 = animation's eyes play through, gaze is additive.
|
* 0.0 = animation's eyes play through, gaze is additive.
|
||||||
* Intermediate (e.g. 0.5) = smooth 50/50 blend. */
|
* Intermediate (e.g. 0.5) = smooth 50/50 blend. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Gaze",
|
||||||
meta = (ClampMin = "0", ClampMax = "1"))
|
meta = (ClampMin = "0", ClampMax = "1"))
|
||||||
float EyeAnimationCompensation = 0.6f;
|
float EyeAnimationCompensation = 0.6f;
|
||||||
|
|
||||||
@@ -195,7 +195,7 @@ public:
|
|||||||
* counter-rotates the gaze to keep the head pointing at the target.
|
* counter-rotates the gaze to keep the head pointing at the target.
|
||||||
* 1.0 = full compensation — head stays locked on target even during bows.
|
* 1.0 = full compensation — head stays locked on target even during bows.
|
||||||
* 0.0 = no compensation — head follows body movement naturally. */
|
* 0.0 = no compensation — head follows body movement naturally. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Gaze",
|
||||||
meta = (ClampMin = "0", ClampMax = "1"))
|
meta = (ClampMin = "0", ClampMax = "1"))
|
||||||
float BodyDriftCompensation = 0.8f;
|
float BodyDriftCompensation = 0.8f;
|
||||||
|
|
||||||
@@ -204,12 +204,12 @@ public:
|
|||||||
/** Enable debug logging for this component.
|
/** Enable debug logging for this component.
|
||||||
* When false, only key events (init, errors, state changes) are logged.
|
* When false, only key events (init, errors, state changes) are logged.
|
||||||
* When true, additional diagnostic info is logged based on DebugVerbosity. */
|
* When true, additional diagnostic info is logged based on DebugVerbosity. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Debug")
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Debug")
|
||||||
bool bDebug = false;
|
bool bDebug = false;
|
||||||
|
|
||||||
/** Verbosity level when bDebug is true.
|
/** Verbosity level when bDebug is true.
|
||||||
* 0 = minimal extras, 1 = normal debug, 2 = detailed, 3 = per-frame data. */
|
* 0 = minimal extras, 1 = normal debug, 2 = detailed, 3 = per-frame data. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Debug",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Debug",
|
||||||
meta = (ClampMin = "0", ClampMax = "3", EditCondition = "bDebug"))
|
meta = (ClampMin = "0", ClampMax = "3", EditCondition = "bDebug"))
|
||||||
int32 DebugVerbosity = 1;
|
int32 DebugVerbosity = 1;
|
||||||
|
|
||||||
@@ -217,7 +217,7 @@ public:
|
|||||||
* Green = head bone → target (desired).
|
* Green = head bone → target (desired).
|
||||||
* Cyan = computed gaze direction (body yaw + head + eyes).
|
* Cyan = computed gaze direction (body yaw + head + eyes).
|
||||||
* Useful for verifying that eye contact is accurate. */
|
* Useful for verifying that eye contact is accurate. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Debug",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Debug",
|
||||||
meta = (EditCondition = "bDebug"))
|
meta = (EditCondition = "bDebug"))
|
||||||
bool bDrawDebugGaze = false;
|
bool bDrawDebugGaze = false;
|
||||||
|
|
||||||
@@ -228,14 +228,14 @@ public:
|
|||||||
* 0 = mesh faces +X (default UE convention)
|
* 0 = mesh faces +X (default UE convention)
|
||||||
* 90 = mesh faces +Y
|
* 90 = mesh faces +Y
|
||||||
* -90 = mesh faces -Y */
|
* -90 = mesh faces -Y */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Gaze",
|
||||||
meta = (ClampMin = "-180", ClampMax = "180"))
|
meta = (ClampMin = "-180", ClampMax = "180"))
|
||||||
float MeshForwardYawOffset = 90.0f;
|
float MeshForwardYawOffset = 90.0f;
|
||||||
|
|
||||||
// ── Head bone ────────────────────────────────────────────────────────────
|
// ── Head bone ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Name of the head bone on the skeletal mesh (used for eye origin calculation). */
|
/** Name of the head bone on the skeletal mesh (used for eye origin calculation). */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze")
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Gaze")
|
||||||
FName HeadBoneName = FName(TEXT("head"));
|
FName HeadBoneName = FName(TEXT("head"));
|
||||||
|
|
||||||
// ── Neck bone chain ─────────────────────────────────────────────────────
|
// ── Neck bone chain ─────────────────────────────────────────────────────
|
||||||
@@ -244,7 +244,7 @@ public:
|
|||||||
* Order: root-to-tip (e.g. neck_01 → neck_02 → head).
|
* Order: root-to-tip (e.g. neck_01 → neck_02 → head).
|
||||||
* Weights should sum to ~1.0.
|
* Weights should sum to ~1.0.
|
||||||
* If empty, falls back to single-bone behavior (HeadBoneName, weight 1.0). */
|
* If empty, falls back to single-bone behavior (HeadBoneName, weight 1.0). */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Gaze")
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Gaze")
|
||||||
TArray<FPS_AI_ConvAgent_NeckBoneEntry> NeckBoneChain;
|
TArray<FPS_AI_ConvAgent_NeckBoneEntry> NeckBoneChain;
|
||||||
|
|
||||||
// ── Getters (read by AnimNode) ───────────────────────────────────────────
|
// ── Getters (read by AnimNode) ───────────────────────────────────────────
|
||||||
@@ -252,7 +252,7 @@ public:
|
|||||||
/** Get current eye gaze curves (8 ARKit eye look curves).
|
/** Get current eye gaze curves (8 ARKit eye look curves).
|
||||||
* Returns a COPY scaled by activation alpha — safe to call from any thread.
|
* Returns a COPY scaled by activation alpha — safe to call from any thread.
|
||||||
* NOTE: Prefer FillCurrentEyeCurves() for hot paths to avoid per-frame allocation. */
|
* NOTE: Prefer FillCurrentEyeCurves() for hot paths to avoid per-frame allocation. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Gaze")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Gaze")
|
||||||
TMap<FName, float> GetCurrentEyeCurves() const
|
TMap<FName, float> GetCurrentEyeCurves() const
|
||||||
{
|
{
|
||||||
FScopeLock Lock(&GazeDataLock);
|
FScopeLock Lock(&GazeDataLock);
|
||||||
|
|||||||
@@ -26,6 +26,12 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnConvAgentDeselected,
|
|||||||
/** Fired when no agent is within interaction range/view. */
|
/** Fired when no agent is within interaction range/view. */
|
||||||
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnNoConvAgentInRange);
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnNoConvAgentInRange);
|
||||||
|
|
||||||
|
/** Fired on the SERVER for each player mic chunk captured for agent-less replay
|
||||||
|
* recording (int16, 16kHz mono). Lets an external recorder (PS_AI_ConvReplay)
|
||||||
|
* save player speech even when no agent is present in the scene. */
|
||||||
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnConvAgentPlayerVoiceForRecording,
|
||||||
|
APawn*, SpeakerPawn, const TArray<uint8>&, PCMData);
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
// UPS_AI_ConvAgent_InteractionComponent
|
// UPS_AI_ConvAgent_InteractionComponent
|
||||||
//
|
//
|
||||||
@@ -45,7 +51,7 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnNoConvAgentInRange);
|
|||||||
// 4. Mic management is auto-detected (no manual flag needed on agents).
|
// 4. Mic management is auto-detected (no manual flag needed on agents).
|
||||||
// 5. Agents manage their own WebSocket connections independently.
|
// 5. Agents manage their own WebSocket connections independently.
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
UCLASS(ClassGroup = "PS AI ConvAgent", meta = (BlueprintSpawnableComponent),
|
UCLASS(ClassGroup = "ASTERION|PS_AI_ConvAgent", meta = (BlueprintSpawnableComponent),
|
||||||
DisplayName = "PS AI ConvAgent Interaction")
|
DisplayName = "PS AI ConvAgent Interaction")
|
||||||
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_InteractionComponent : public UActorComponent
|
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_InteractionComponent : public UActorComponent
|
||||||
{
|
{
|
||||||
@@ -57,7 +63,7 @@ public:
|
|||||||
// ── Configuration ─────────────────────────────────────────────────────────
|
// ── Configuration ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Maximum distance (cm) at which an agent can be selected. Agents beyond this are ignored. */
|
/** Maximum distance (cm) at which an agent can be selected. Agents beyond this are ignored. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Interaction",
|
||||||
meta = (ClampMin = "0",
|
meta = (ClampMin = "0",
|
||||||
ToolTip = "Max distance in cm to interact with an agent.\nAgents beyond this distance are never selected."))
|
ToolTip = "Max distance in cm to interact with an agent.\nAgents beyond this distance are never selected."))
|
||||||
float MaxInteractionDistance = 300.0f;
|
float MaxInteractionDistance = 300.0f;
|
||||||
@@ -65,7 +71,7 @@ public:
|
|||||||
/** Half-angle (degrees) of the view cone used for agent selection.
|
/** Half-angle (degrees) of the view cone used for agent selection.
|
||||||
* The cone is centered on the pawn's view direction (camera or control rotation).
|
* The cone is centered on the pawn's view direction (camera or control rotation).
|
||||||
* Agents outside this cone are not considered unless they are the current sticky selection. */
|
* Agents outside this cone are not considered unless they are the current sticky selection. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Interaction",
|
||||||
meta = (ClampMin = "0", ClampMax = "180",
|
meta = (ClampMin = "0", ClampMax = "180",
|
||||||
ToolTip = "Half-angle of the view cone (degrees).\nAgents outside this cone are not selected.\n45 = 90-degree total FOV."))
|
ToolTip = "Half-angle of the view cone (degrees).\nAgents outside this cone are not selected.\n45 = 90-degree total FOV."))
|
||||||
float ViewConeHalfAngle = 45.0f;
|
float ViewConeHalfAngle = 45.0f;
|
||||||
@@ -73,7 +79,7 @@ public:
|
|||||||
/** Wider half-angle (degrees) used for the currently selected agent.
|
/** Wider half-angle (degrees) used for the currently selected agent.
|
||||||
* This provides hysteresis so the player can look slightly away from
|
* This provides hysteresis so the player can look slightly away from
|
||||||
* the current agent without losing selection (prevents flickering). */
|
* the current agent without losing selection (prevents flickering). */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Interaction",
|
||||||
meta = (ClampMin = "0", ClampMax = "180",
|
meta = (ClampMin = "0", ClampMax = "180",
|
||||||
ToolTip = "Sticky cone half-angle for the current agent (degrees).\nMust be >= ViewConeHalfAngle for hysteresis to work.\n60 = agent stays selected until 120-degree total cone is exceeded."))
|
ToolTip = "Sticky cone half-angle for the current agent (degrees).\nMust be >= ViewConeHalfAngle for hysteresis to work.\n60 = agent stays selected until 120-degree total cone is exceeded."))
|
||||||
float SelectionStickyAngle = 60.0f;
|
float SelectionStickyAngle = 60.0f;
|
||||||
@@ -81,14 +87,14 @@ public:
|
|||||||
/** Height offset (cm) added to the agent's origin for the view cone check.
|
/** Height offset (cm) added to the agent's origin for the view cone check.
|
||||||
* Agent origins are typically at foot level; this shifts the target point
|
* Agent origins are typically at foot level; this shifts the target point
|
||||||
* up to chest/head height so close-range look-at checks don't fail. */
|
* up to chest/head height so close-range look-at checks don't fail. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Interaction",
|
||||||
meta = (ClampMin = "0",
|
meta = (ClampMin = "0",
|
||||||
ToolTip = "Height offset (cm) above the agent origin for the view cone check.\nPrevents losing selection at close range because the camera looks above the agent's feet.\n150 = roughly chest height for a MetaHuman."))
|
ToolTip = "Height offset (cm) above the agent origin for the view cone check.\nPrevents losing selection at close range because the camera looks above the agent's feet.\n150 = roughly chest height for a MetaHuman."))
|
||||||
float AgentEyeLevelOffset = 150.0f;
|
float AgentEyeLevelOffset = 150.0f;
|
||||||
|
|
||||||
/** When false, only distance matters for selection (no view cone check).
|
/** When false, only distance matters for selection (no view cone check).
|
||||||
* The closest agent within MaxInteractionDistance is always selected. */
|
* The closest agent within MaxInteractionDistance is always selected. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Interaction",
|
||||||
meta = (ToolTip = "Require the player to look at an agent to select it.\nWhen false, the closest agent within range is always selected."))
|
meta = (ToolTip = "Require the player to look at an agent to select it.\nWhen false, the closest agent within range is always selected."))
|
||||||
bool bRequireLookAt = true;
|
bool bRequireLookAt = true;
|
||||||
|
|
||||||
@@ -97,20 +103,20 @@ public:
|
|||||||
/** Automatically set/clear the agent's GazeComponent TargetActor
|
/** Automatically set/clear the agent's GazeComponent TargetActor
|
||||||
* when the agent is selected/deselected. When false, gaze must
|
* when the agent is selected/deselected. When false, gaze must
|
||||||
* be managed from Blueprint (e.g. on conversation start). */
|
* be managed from Blueprint (e.g. on conversation start). */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Interaction|Gaze",
|
||||||
meta = (ToolTip = "Automatically point the agent's gaze at the pawn on selection.\nDisable for manual control (e.g. set target only when conversation starts)."))
|
meta = (ToolTip = "Automatically point the agent's gaze at the pawn on selection.\nDisable for manual control (e.g. set target only when conversation starts)."))
|
||||||
bool bAutoManageGaze = true;
|
bool bAutoManageGaze = true;
|
||||||
|
|
||||||
/** Delay (seconds) before setting the agent's gaze target after selection.
|
/** Delay (seconds) before setting the agent's gaze target after selection.
|
||||||
* 0 = immediate. Useful to let the agent "notice" the player with a beat. */
|
* 0 = immediate. Useful to let the agent "notice" the player with a beat. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Interaction|Gaze",
|
||||||
meta = (EditCondition = "bAutoManageGaze", ClampMin = "0",
|
meta = (EditCondition = "bAutoManageGaze", ClampMin = "0",
|
||||||
ToolTip = "Seconds to wait before the agent looks at the pawn.\n0 = immediate."))
|
ToolTip = "Seconds to wait before the agent looks at the pawn.\n0 = immediate."))
|
||||||
float GazeAttachDelay = 0.0f;
|
float GazeAttachDelay = 0.0f;
|
||||||
|
|
||||||
/** Delay (seconds) before clearing the agent's gaze target after deselection.
|
/** Delay (seconds) before clearing the agent's gaze target after deselection.
|
||||||
* 0 = immediate. Useful to have the agent keep looking briefly as the player leaves. */
|
* 0 = immediate. Useful to have the agent keep looking briefly as the player leaves. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Interaction|Gaze",
|
||||||
meta = (EditCondition = "bAutoManageGaze", ClampMin = "0",
|
meta = (EditCondition = "bAutoManageGaze", ClampMin = "0",
|
||||||
ToolTip = "Seconds to wait before the agent stops looking at the pawn.\n0 = immediate."))
|
ToolTip = "Seconds to wait before the agent stops looking at the pawn.\n0 = immediate."))
|
||||||
float GazeDetachDelay = 0.0f;
|
float GazeDetachDelay = 0.0f;
|
||||||
@@ -119,20 +125,20 @@ public:
|
|||||||
* with head+eyes (and optionally body). This creates natural "awareness"
|
* with head+eyes (and optionally body). This creates natural "awareness"
|
||||||
* without requiring an active conversation.
|
* without requiring an active conversation.
|
||||||
* Also controls the passive gaze retained after switching conversations. */
|
* Also controls the passive gaze retained after switching conversations. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Interaction|Gaze",
|
||||||
meta = (EditCondition = "bAutoManageGaze",
|
meta = (EditCondition = "bAutoManageGaze",
|
||||||
ToolTip = "Agents in range but not selected will passively track the player.\nAlso applies to retained gaze after switching conversations."))
|
ToolTip = "Agents in range but not selected will passively track the player.\nAlso applies to retained gaze after switching conversations."))
|
||||||
bool bAutoPassiveGaze = true;
|
bool bAutoPassiveGaze = true;
|
||||||
|
|
||||||
/** In passive gaze mode, the agent's head and eyes follow the player. */
|
/** In passive gaze mode, the agent's head and eyes follow the player. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Interaction|Gaze",
|
||||||
meta = (EditCondition = "bAutoManageGaze && bAutoPassiveGaze",
|
meta = (EditCondition = "bAutoManageGaze && bAutoPassiveGaze",
|
||||||
ToolTip = "Enable head+eyes tracking in passive gaze mode."))
|
ToolTip = "Enable head+eyes tracking in passive gaze mode."))
|
||||||
bool bPassiveGazeHeadEyes = true;
|
bool bPassiveGazeHeadEyes = true;
|
||||||
|
|
||||||
/** In passive gaze mode, the agent's body also turns toward the player.
|
/** In passive gaze mode, the agent's body also turns toward the player.
|
||||||
* When false, only head and eyes track — the body stays in its default animation. */
|
* When false, only head and eyes track — the body stays in its default animation. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction|Gaze",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Interaction|Gaze",
|
||||||
meta = (EditCondition = "bAutoManageGaze && bAutoPassiveGaze",
|
meta = (EditCondition = "bAutoManageGaze && bAutoPassiveGaze",
|
||||||
ToolTip = "Enable body rotation in passive gaze mode.\nWhen false, only head and eyes track."))
|
ToolTip = "Enable body rotation in passive gaze mode.\nWhen false, only head and eyes track."))
|
||||||
bool bPassiveGazeBody = false;
|
bool bPassiveGazeBody = false;
|
||||||
@@ -142,7 +148,7 @@ public:
|
|||||||
/** How long (seconds) the player must look at a different agent before switching
|
/** How long (seconds) the player must look at a different agent before switching
|
||||||
* during an active conversation. Prevents accidental switches when glancing around.
|
* during an active conversation. Prevents accidental switches when glancing around.
|
||||||
* Only applies when a conversation is active (bNetIsConversing). */
|
* Only applies when a conversation is active (bNetIsConversing). */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Interaction",
|
||||||
meta = (ClampMin = "0.0", ClampMax = "5.0",
|
meta = (ClampMin = "0.0", ClampMax = "5.0",
|
||||||
ToolTip = "Seconds the player must look at another agent before switching mid-conversation.\nPrevents accidental switches when glancing around.\n0 = instant switch."))
|
ToolTip = "Seconds the player must look at another agent before switching mid-conversation.\nPrevents accidental switches when glancing around.\n0 = instant switch."))
|
||||||
float ConversationSwitchDelay = 1.0f;
|
float ConversationSwitchDelay = 1.0f;
|
||||||
@@ -153,7 +159,7 @@ public:
|
|||||||
* via StartConversationWithSelectedAgent() (e.g. on a key press).
|
* via StartConversationWithSelectedAgent() (e.g. on a key press).
|
||||||
* Set to false when you have multiple agents in a scene to prevent them
|
* Set to false when you have multiple agents in a scene to prevent them
|
||||||
* all from greeting the player simultaneously. */
|
* all from greeting the player simultaneously. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Interaction",
|
||||||
meta = (ToolTip = "Auto-start the WebSocket when an agent is selected by proximity.\nSet to false to require explicit interaction (call StartConversationWithSelectedAgent).\nUseful with multiple agents to prevent simultaneous greetings."))
|
meta = (ToolTip = "Auto-start the WebSocket when an agent is selected by proximity.\nSet to false to require explicit interaction (call StartConversationWithSelectedAgent).\nUseful with multiple agents to prevent simultaneous greetings."))
|
||||||
bool bAutoStartConversation = true;
|
bool bAutoStartConversation = true;
|
||||||
|
|
||||||
@@ -162,48 +168,53 @@ public:
|
|||||||
/** Automatically call StartListening/StopListening on the agent's
|
/** Automatically call StartListening/StopListening on the agent's
|
||||||
* ElevenLabsComponent when selected/deselected. When false, listening
|
* ElevenLabsComponent when selected/deselected. When false, listening
|
||||||
* must be managed from Blueprint. */
|
* must be managed from Blueprint. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Interaction|Listening",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Interaction|Listening",
|
||||||
meta = (ToolTip = "Automatically open/close the agent's mic turn on selection.\nDisable for manual control (e.g. push-to-talk)."))
|
meta = (ToolTip = "Automatically open/close the agent's mic turn on selection.\nDisable for manual control (e.g. push-to-talk)."))
|
||||||
bool bAutoManageListening = true;
|
bool bAutoManageListening = true;
|
||||||
|
|
||||||
// ── Debug ────────────────────────────────────────────────────────────────
|
// ── Debug ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Enable debug logging for this component. */
|
/** Enable debug logging for this component. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Debug")
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Debug")
|
||||||
bool bDebug = false;
|
bool bDebug = false;
|
||||||
|
|
||||||
/** Verbosity level when bDebug is true.
|
/** Verbosity level when bDebug is true.
|
||||||
* 0 = selection changes only, 1 = candidate evaluation, 2 = per-frame data. */
|
* 0 = selection changes only, 1 = candidate evaluation, 2 = per-frame data. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Debug",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Debug",
|
||||||
meta = (ClampMin = "0", ClampMax = "3", EditCondition = "bDebug"))
|
meta = (ClampMin = "0", ClampMax = "3", EditCondition = "bDebug"))
|
||||||
int32 DebugVerbosity = 1;
|
int32 DebugVerbosity = 1;
|
||||||
|
|
||||||
// ── Events ────────────────────────────────────────────────────────────────
|
// ── Events ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Fired when a new agent enters selection. Use this to set gaze targets, show UI, etc. */
|
/** Fired when a new agent enters selection. Use this to set gaze targets, show UI, etc. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|Interaction|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|Interaction|Events",
|
||||||
meta = (ToolTip = "Fires when a new agent is selected.\nSet gaze targets, update UI, etc."))
|
meta = (ToolTip = "Fires when a new agent is selected.\nSet gaze targets, update UI, etc."))
|
||||||
FOnConvAgentSelected OnAgentSelected;
|
FOnConvAgentSelected OnAgentSelected;
|
||||||
|
|
||||||
/** Fired when the previously selected agent loses selection. */
|
/** Fired when the previously selected agent loses selection. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|Interaction|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|Interaction|Events",
|
||||||
meta = (ToolTip = "Fires when the current agent is deselected.\nClear gaze targets, hide UI, etc."))
|
meta = (ToolTip = "Fires when the current agent is deselected.\nClear gaze targets, hide UI, etc."))
|
||||||
FOnConvAgentDeselected OnAgentDeselected;
|
FOnConvAgentDeselected OnAgentDeselected;
|
||||||
|
|
||||||
/** Fired when no agent is within range or view cone. */
|
/** Fired when no agent is within range or view cone. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|Interaction|Events",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|Interaction|Events",
|
||||||
meta = (ToolTip = "Fires when no agent meets the distance/view criteria."))
|
meta = (ToolTip = "Fires when no agent meets the distance/view criteria."))
|
||||||
FOnNoConvAgentInRange OnNoAgentInRange;
|
FOnNoConvAgentInRange OnNoAgentInRange;
|
||||||
|
|
||||||
|
/** Fired on the SERVER for each player mic chunk captured for agent-less replay
|
||||||
|
* recording. An external recorder (PS_AI_ConvReplay) subscribes to save player voice. */
|
||||||
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|Interaction|Events")
|
||||||
|
FOnConvAgentPlayerVoiceForRecording OnPlayerVoiceForRecording;
|
||||||
|
|
||||||
// ── Blueprint API ─────────────────────────────────────────────────────────
|
// ── Blueprint API ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Get the currently selected agent (null if none). */
|
/** Get the currently selected agent (null if none). */
|
||||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|Interaction")
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|Interaction")
|
||||||
UPS_AI_ConvAgent_ElevenLabsComponent* GetSelectedAgent() const;
|
UPS_AI_ConvAgent_ElevenLabsComponent* GetSelectedAgent() const;
|
||||||
|
|
||||||
/** Manually select a specific agent, bypassing distance/view checks.
|
/** Manually select a specific agent, bypassing distance/view checks.
|
||||||
* Pass null to clear the selection. Automatic selection resumes next tick. */
|
* Pass null to clear the selection. Automatic selection resumes next tick. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Interaction")
|
||||||
void ForceSelectAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Agent);
|
void ForceSelectAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Agent);
|
||||||
|
|
||||||
/** Start the WebSocket conversation with the currently selected agent.
|
/** Start the WebSocket conversation with the currently selected agent.
|
||||||
@@ -211,20 +222,26 @@ public:
|
|||||||
* interacts (e.g. presses a key, enters a trigger zone).
|
* interacts (e.g. presses a key, enters a trigger zone).
|
||||||
* Does nothing if no agent is selected or the agent is already connected.
|
* Does nothing if no agent is selected or the agent is already connected.
|
||||||
* Also starts mic capture and listening automatically. */
|
* Also starts mic capture and listening automatically. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Interaction")
|
||||||
void StartConversationWithSelectedAgent();
|
void StartConversationWithSelectedAgent();
|
||||||
|
|
||||||
/** Send a text message to the currently selected agent.
|
/** Send a text message to the currently selected agent.
|
||||||
* The agent responds with audio and text, just as if it heard the player speak.
|
* The agent responds with audio and text, just as if it heard the player speak.
|
||||||
* Handles network relay automatically (works on both server and client).
|
* Handles network relay automatically (works on both server and client).
|
||||||
* Does nothing if no agent is selected or not connected. */
|
* Does nothing if no agent is selected or not connected. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Interaction")
|
||||||
void SendTextToSelectedAgent(const FString& Text);
|
void SendTextToSelectedAgent(const FString& Text);
|
||||||
|
|
||||||
/** Clear the current selection. Automatic selection resumes next tick. */
|
/** Clear the current selection. Automatic selection resumes next tick. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Interaction")
|
||||||
void ClearSelection();
|
void ClearSelection();
|
||||||
|
|
||||||
|
/** Enable/disable capturing this pawn's mic for replay recording even when no
|
||||||
|
* agent is selected. Server authority only; the flag replicates so the owning
|
||||||
|
* client starts capturing + relaying its mic. Driven by PS_AI_ConvReplay. */
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Interaction")
|
||||||
|
void SetReplayRecordActive(bool bActive);
|
||||||
|
|
||||||
// ── Network relay RPCs ───────────────────────────────────────────────────
|
// ── Network relay RPCs ───────────────────────────────────────────────────
|
||||||
// UE5 clients can only call Server RPCs on actors they own. NPC actors are
|
// UE5 clients can only call Server RPCs on actors they own. NPC actors are
|
||||||
// server-owned, so clients can't call RPCs on them directly.
|
// server-owned, so clients can't call RPCs on them directly.
|
||||||
@@ -253,6 +270,11 @@ public:
|
|||||||
UFUNCTION(Server, Unreliable)
|
UFUNCTION(Server, Unreliable)
|
||||||
void ServerRelayMicAudio(AActor* AgentActor, const TArray<uint8>& PCMBytes);
|
void ServerRelayMicAudio(AActor* AgentActor, const TArray<uint8>& PCMBytes);
|
||||||
|
|
||||||
|
/** Relay: stream this pawn's mic to the server purely for replay recording
|
||||||
|
* (no agent). The server broadcasts OnPlayerVoiceForRecording for the recorder. */
|
||||||
|
UFUNCTION(Server, Unreliable)
|
||||||
|
void ServerRecordVoice(const TArray<uint8>& PCMBytes);
|
||||||
|
|
||||||
/** Relay: send text message to an NPC agent. */
|
/** Relay: send text message to an NPC agent. */
|
||||||
UFUNCTION(Server, Reliable)
|
UFUNCTION(Server, Reliable)
|
||||||
void ServerRelaySendText(AActor* AgentActor, const FString& Text);
|
void ServerRelaySendText(AActor* AgentActor, const FString& Text);
|
||||||
@@ -317,6 +339,22 @@ private:
|
|||||||
UPROPERTY()
|
UPROPERTY()
|
||||||
UPS_AI_ConvAgent_MicrophoneCaptureComponent* MicComponent = nullptr;
|
UPS_AI_ConvAgent_MicrophoneCaptureComponent* MicComponent = nullptr;
|
||||||
|
|
||||||
|
// ── Agent-less replay recording ──────────────────────────────────────────
|
||||||
|
/** Replicated: the server sets this while a replay is recording so the owning
|
||||||
|
* client captures + relays its mic even without a selected agent. */
|
||||||
|
UPROPERTY(ReplicatedUsing = OnRep_ReplayRecordActive)
|
||||||
|
bool bReplayRecordActive = false;
|
||||||
|
|
||||||
|
UFUNCTION()
|
||||||
|
void OnRep_ReplayRecordActive();
|
||||||
|
|
||||||
|
/** Start/stop mic capture to honour bReplayRecordActive (never stops the mic
|
||||||
|
* while an agent conversation still needs it). */
|
||||||
|
void UpdateReplayCapture();
|
||||||
|
|
||||||
|
/** Accumulates int16 mic bytes for the recording relay, chunked to limit RPC count. */
|
||||||
|
TArray<uint8> RecordMicBuffer;
|
||||||
|
|
||||||
/** True once the one-time lazy init in TickComponent has completed.
|
/** True once the one-time lazy init in TickComponent has completed.
|
||||||
* Deferred from BeginPlay because IsLocallyControlled() may return false
|
* Deferred from BeginPlay because IsLocallyControlled() may return false
|
||||||
* before the PlayerController has been replicated/possessed. */
|
* before the PlayerController has been replicated/possessed. */
|
||||||
|
|||||||
@@ -29,11 +29,11 @@ public:
|
|||||||
void UnregisterAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Agent);
|
void UnregisterAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Agent);
|
||||||
|
|
||||||
/** Get all currently registered agent components (weak pointers, may contain stale entries that are cleaned on access). */
|
/** Get all currently registered agent components (weak pointers, may contain stale entries that are cleaned on access). */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Interaction")
|
||||||
TArray<UPS_AI_ConvAgent_ElevenLabsComponent*> GetRegisteredAgents() const;
|
TArray<UPS_AI_ConvAgent_ElevenLabsComponent*> GetRegisteredAgents() const;
|
||||||
|
|
||||||
/** Get the number of registered agents. */
|
/** Get the number of registered agents. */
|
||||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|Interaction")
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|Interaction")
|
||||||
int32 GetNumRegisteredAgents() const;
|
int32 GetNumRegisteredAgents() const;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnPS_AI_ConvAgent_VisemesReady);
|
|||||||
* 3. Conversation starts → lip sync works automatically.
|
* 3. Conversation starts → lip sync works automatically.
|
||||||
* 4. (Optional) Bind OnVisemesReady for custom Blueprint handling.
|
* 4. (Optional) Bind OnVisemesReady for custom Blueprint handling.
|
||||||
*/
|
*/
|
||||||
UCLASS(ClassGroup = "PS AI ConvAgent", meta = (BlueprintSpawnableComponent),
|
UCLASS(ClassGroup = "ASTERION|PS_AI_ConvAgent", meta = (BlueprintSpawnableComponent),
|
||||||
DisplayName = "PS AI ConvAgent Lip Sync")
|
DisplayName = "PS AI ConvAgent Lip Sync")
|
||||||
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_LipSyncComponent : public UActorComponent
|
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_LipSyncComponent : public UActorComponent
|
||||||
{
|
{
|
||||||
@@ -61,16 +61,16 @@ public:
|
|||||||
/** Whether lip sync is currently active (read-only).
|
/** Whether lip sync is currently active (read-only).
|
||||||
* Managed automatically: activates on conversation connect, deactivates on disconnect.
|
* Managed automatically: activates on conversation connect, deactivates on disconnect.
|
||||||
* The underlying audio analysis keeps running so reactivation is seamless. */
|
* The underlying audio analysis keeps running so reactivation is seamless. */
|
||||||
UPROPERTY(BlueprintReadOnly, Category = "PS AI ConvAgent|LipSync")
|
UPROPERTY(BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|LipSync")
|
||||||
bool bActive = true;
|
bool bActive = true;
|
||||||
|
|
||||||
/** How long (seconds) to blend in/out when bActive changes. */
|
/** How long (seconds) to blend in/out when bActive changes. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|LipSync",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|LipSync",
|
||||||
meta = (ClampMin = "0.05", ClampMax = "3.0"))
|
meta = (ClampMin = "0.05", ClampMax = "3.0"))
|
||||||
float ActivationBlendDuration = 0.5f;
|
float ActivationBlendDuration = 0.5f;
|
||||||
|
|
||||||
/** Overall mouth movement intensity multiplier. */
|
/** Overall mouth movement intensity multiplier. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|LipSync",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|LipSync",
|
||||||
meta = (ClampMin = "0.0", ClampMax = "3.0",
|
meta = (ClampMin = "0.0", ClampMax = "3.0",
|
||||||
ToolTip = "Lip sync intensity.\n1.0 = normal, higher = more expressive, lower = subtler."))
|
ToolTip = "Lip sync intensity.\n1.0 = normal, higher = more expressive, lower = subtler."))
|
||||||
float LipSyncStrength = 1.0f;
|
float LipSyncStrength = 1.0f;
|
||||||
@@ -78,13 +78,13 @@ public:
|
|||||||
/** Scales the audio amplitude driving mouth movement.
|
/** Scales the audio amplitude driving mouth movement.
|
||||||
* Lower values produce subtler animation, higher values are more pronounced.
|
* Lower values produce subtler animation, higher values are more pronounced.
|
||||||
* Use this to tone down overly strong lip movement without changing the shape. */
|
* Use this to tone down overly strong lip movement without changing the shape. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|LipSync",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|LipSync",
|
||||||
meta = (ClampMin = "0.5", ClampMax = "1.0",
|
meta = (ClampMin = "0.5", ClampMax = "1.0",
|
||||||
ToolTip = "Audio amplitude scale.\n0.5 = subtle, 0.75 = balanced, 1.0 = full.\nReduces overall mouth movement without affecting viseme shape."))
|
ToolTip = "Audio amplitude scale.\n0.5 = subtle, 0.75 = balanced, 1.0 = full.\nReduces overall mouth movement without affecting viseme shape."))
|
||||||
float AmplitudeScale = 1.0f;
|
float AmplitudeScale = 1.0f;
|
||||||
|
|
||||||
/** How quickly viseme weights interpolate towards new values each frame. */
|
/** How quickly viseme weights interpolate towards new values each frame. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|LipSync",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|LipSync",
|
||||||
meta = (ClampMin = "35.0", ClampMax = "65.0",
|
meta = (ClampMin = "35.0", ClampMax = "65.0",
|
||||||
ToolTip = "Smoothing speed for viseme transitions.\n35 = smooth and soft, 50 = balanced, 65 = sharp and responsive."))
|
ToolTip = "Smoothing speed for viseme transitions.\n35 = smooth and soft, 50 = balanced, 65 = sharp and responsive."))
|
||||||
float SmoothingSpeed = 50.0f;
|
float SmoothingSpeed = 50.0f;
|
||||||
@@ -95,7 +95,7 @@ public:
|
|||||||
* When the agent is NOT speaking, lip sync releases all mouth curves so the
|
* When the agent is NOT speaking, lip sync releases all mouth curves so the
|
||||||
* FacialExpressionComponent's emotion curves (including mouth) can show through.
|
* FacialExpressionComponent's emotion curves (including mouth) can show through.
|
||||||
* When the agent starts speaking, lip sync smoothly takes over mouth curves. */
|
* When the agent starts speaking, lip sync smoothly takes over mouth curves. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|LipSync",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|LipSync",
|
||||||
meta = (ClampMin = "0.05", ClampMax = "1.0",
|
meta = (ClampMin = "0.05", ClampMax = "1.0",
|
||||||
ToolTip = "Blend duration when speech starts/stops.\nDuring silence, emotion facial curves control the mouth.\nDuring speech, lip sync takes over."))
|
ToolTip = "Blend duration when speech starts/stops.\nDuring silence, emotion facial curves control the mouth.\nDuring speech, lip sync takes over."))
|
||||||
float SpeechBlendDuration = 0.15f;
|
float SpeechBlendDuration = 0.15f;
|
||||||
@@ -108,7 +108,7 @@ public:
|
|||||||
* 0.0 = lip sync completely overrides expression curves (old behavior).
|
* 0.0 = lip sync completely overrides expression curves (old behavior).
|
||||||
* 1.0 = emotion fully additive on expression curves during speech.
|
* 1.0 = emotion fully additive on expression curves during speech.
|
||||||
* 0.5 = balanced blend (recommended). */
|
* 0.5 = balanced blend (recommended). */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|LipSync",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|LipSync",
|
||||||
meta = (ClampMin = "0.0", ClampMax = "1.0",
|
meta = (ClampMin = "0.0", ClampMax = "1.0",
|
||||||
ToolTip = "Emotion bleed-through during speech.\n0 = pure lip sync, 0.5 = balanced, 1 = full emotion on expression curves.\nOnly affects expression curves (smile, frown, etc.), not lip shape."))
|
ToolTip = "Emotion bleed-through during speech.\n0 = pure lip sync, 0.5 = balanced, 1 = full emotion on expression curves.\nOnly affects expression curves (smile, frown, etc.), not lip shape."))
|
||||||
float EmotionExpressionBlend = 0.5f;
|
float EmotionExpressionBlend = 0.5f;
|
||||||
@@ -118,7 +118,7 @@ public:
|
|||||||
/** Envelope attack time in milliseconds.
|
/** Envelope attack time in milliseconds.
|
||||||
* Controls how fast the mouth opens when speech starts or gets louder.
|
* Controls how fast the mouth opens when speech starts or gets louder.
|
||||||
* Lower = snappier onset, higher = gentler opening. */
|
* Lower = snappier onset, higher = gentler opening. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|LipSync",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|LipSync",
|
||||||
meta = (ClampMin = "1.0", ClampMax = "50.0",
|
meta = (ClampMin = "1.0", ClampMax = "50.0",
|
||||||
ToolTip = "Envelope attack (ms).\n5 = snappy, 10 = balanced, 25 = gentle.\nHow fast the mouth opens on speech onset."))
|
ToolTip = "Envelope attack (ms).\n5 = snappy, 10 = balanced, 25 = gentle.\nHow fast the mouth opens on speech onset."))
|
||||||
float EnvelopeAttackMs = 10.0f;
|
float EnvelopeAttackMs = 10.0f;
|
||||||
@@ -126,7 +126,7 @@ public:
|
|||||||
/** Envelope release time in milliseconds.
|
/** Envelope release time in milliseconds.
|
||||||
* Controls how slowly the mouth closes when speech gets quieter.
|
* Controls how slowly the mouth closes when speech gets quieter.
|
||||||
* Higher = smoother, more natural decay. */
|
* Higher = smoother, more natural decay. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|LipSync",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|LipSync",
|
||||||
meta = (ClampMin = "10.0", ClampMax = "200.0",
|
meta = (ClampMin = "10.0", ClampMax = "200.0",
|
||||||
ToolTip = "Envelope release (ms).\n25 = responsive, 50 = balanced, 120 = smooth/cinematic.\nHow slowly the mouth closes between syllables."))
|
ToolTip = "Envelope release (ms).\n25 = responsive, 50 = balanced, 120 = smooth/cinematic.\nHow slowly the mouth closes between syllables."))
|
||||||
float EnvelopeReleaseMs = 50.0f;
|
float EnvelopeReleaseMs = 50.0f;
|
||||||
@@ -135,12 +135,12 @@ public:
|
|||||||
|
|
||||||
/** Enable debug logging for this component.
|
/** Enable debug logging for this component.
|
||||||
* When false, only key events (init, errors, state changes) are logged. */
|
* When false, only key events (init, errors, state changes) are logged. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Debug")
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Debug")
|
||||||
bool bDebug = false;
|
bool bDebug = false;
|
||||||
|
|
||||||
/** Verbosity level when bDebug is true.
|
/** Verbosity level when bDebug is true.
|
||||||
* 0 = minimal extras, 1 = normal debug, 2 = detailed, 3 = per-frame data. */
|
* 0 = minimal extras, 1 = normal debug, 2 = detailed, 3 = per-frame data. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Debug",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Debug",
|
||||||
meta = (ClampMin = "0", ClampMax = "3", EditCondition = "bDebug"))
|
meta = (ClampMin = "0", ClampMax = "3", EditCondition = "bDebug"))
|
||||||
int32 DebugVerbosity = 1;
|
int32 DebugVerbosity = 1;
|
||||||
|
|
||||||
@@ -152,7 +152,7 @@ public:
|
|||||||
* When assigned, curve data is extracted at BeginPlay and replaces the
|
* When assigned, curve data is extracted at BeginPlay and replaces the
|
||||||
* hardcoded ARKit blendshape mapping with artist-crafted poses.
|
* hardcoded ARKit blendshape mapping with artist-crafted poses.
|
||||||
* Leave empty to use the built-in ARKit mapping (backward compatible). */
|
* Leave empty to use the built-in ARKit mapping (backward compatible). */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|LipSync",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|LipSync",
|
||||||
meta = (ToolTip = "Reusable pose map asset.\nAssign your MHF_* AnimSequences once, reuse on all MetaHumans.\nLeave empty for built-in ARKit mapping."))
|
meta = (ToolTip = "Reusable pose map asset.\nAssign your MHF_* AnimSequences once, reuse on all MetaHumans.\nLeave empty for built-in ARKit mapping."))
|
||||||
TObjectPtr<UPS_AI_ConvAgent_LipSyncPoseMap> PoseMap;
|
TObjectPtr<UPS_AI_ConvAgent_LipSyncPoseMap> PoseMap;
|
||||||
|
|
||||||
@@ -160,19 +160,19 @@ public:
|
|||||||
|
|
||||||
/** Fires every tick when viseme data has been updated.
|
/** Fires every tick when viseme data has been updated.
|
||||||
* Use GetCurrentVisemes() or GetCurrentBlendshapes() to read values. */
|
* Use GetCurrentVisemes() or GetCurrentBlendshapes() to read values. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|LipSync",
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|LipSync",
|
||||||
meta = (ToolTip = "Fires each frame with updated viseme data.\nCall GetCurrentVisemes() or GetCurrentBlendshapes() to read values."))
|
meta = (ToolTip = "Fires each frame with updated viseme data.\nCall GetCurrentVisemes() or GetCurrentBlendshapes() to read values."))
|
||||||
FOnPS_AI_ConvAgent_VisemesReady OnVisemesReady;
|
FOnPS_AI_ConvAgent_VisemesReady OnVisemesReady;
|
||||||
|
|
||||||
// ── Getters ───────────────────────────────────────────────────────────────
|
// ── Getters ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Get current OVR viseme weights (15 values: sil, PP, FF, TH, DD, kk, CH, SS, nn, RR, aa, E, ih, oh, ou). */
|
/** Get current OVR viseme weights (15 values: sil, PP, FF, TH, DD, kk, CH, SS, nn, RR, aa, E, ih, oh, ou). */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|LipSync")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|LipSync")
|
||||||
TMap<FName, float> GetCurrentVisemes() const { return SmoothedVisemes; }
|
TMap<FName, float> GetCurrentVisemes() const { return SmoothedVisemes; }
|
||||||
|
|
||||||
/** Get current ARKit blendshape weights (MetaHuman compatible: jawOpen, mouthFunnel, mouthClose, etc.).
|
/** Get current ARKit blendshape weights (MetaHuman compatible: jawOpen, mouthFunnel, mouthClose, etc.).
|
||||||
* Thread-safe: returns a snapshot updated each tick. Safe to call from anim worker threads. */
|
* Thread-safe: returns a snapshot updated each tick. Safe to call from anim worker threads. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|LipSync")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|LipSync")
|
||||||
TMap<FName, float> GetCurrentBlendshapes() const
|
TMap<FName, float> GetCurrentBlendshapes() const
|
||||||
{
|
{
|
||||||
FScopeLock Lock(&BlendshapeLock);
|
FScopeLock Lock(&BlendshapeLock);
|
||||||
@@ -181,7 +181,7 @@ public:
|
|||||||
|
|
||||||
/** True when the agent is currently producing speech audio.
|
/** True when the agent is currently producing speech audio.
|
||||||
* When false, lip sync releases mouth curves to let emotion curves through. */
|
* When false, lip sync releases mouth curves to let emotion curves through. */
|
||||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|LipSync")
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|LipSync")
|
||||||
bool IsSpeaking() const { return bIsSpeaking; }
|
bool IsSpeaking() const { return bIsSpeaking; }
|
||||||
|
|
||||||
/** Current speech blend alpha (0 = silent/emotion mouth, 1 = lip sync mouth).
|
/** Current speech blend alpha (0 = silent/emotion mouth, 1 = lip sync mouth).
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnPS_AI_ConvAgent_VoiceActivityChan
|
|||||||
* Modelled after Convai's ConvaiAudioCaptureComponent but stripped to the
|
* Modelled after Convai's ConvaiAudioCaptureComponent but stripped to the
|
||||||
* minimal functionality needed for conversational AI agents.
|
* minimal functionality needed for conversational AI agents.
|
||||||
*/
|
*/
|
||||||
UCLASS(ClassGroup = "PS AI ConvAgent", meta = (BlueprintSpawnableComponent),
|
UCLASS(ClassGroup = "ASTERION|PS_AI_ConvAgent", meta = (BlueprintSpawnableComponent),
|
||||||
DisplayName = "PS AI ConvAgent Microphone Capture")
|
DisplayName = "PS AI ConvAgent Microphone Capture")
|
||||||
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_MicrophoneCaptureComponent : public UActorComponent
|
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_MicrophoneCaptureComponent : public UActorComponent
|
||||||
{
|
{
|
||||||
@@ -32,7 +32,7 @@ public:
|
|||||||
UPS_AI_ConvAgent_MicrophoneCaptureComponent();
|
UPS_AI_ConvAgent_MicrophoneCaptureComponent();
|
||||||
|
|
||||||
/** Multiplier applied to the microphone input volume before sending to the agent. Increase if the agent has trouble hearing you, decrease if your audio is clipping. Default: 1.0 (no change). */
|
/** Multiplier applied to the microphone input volume before sending to the agent. Increase if the agent has trouble hearing you, decrease if your audio is clipping. Default: 1.0 (no change). */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|MicCapture",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|MicCapture",
|
||||||
meta = (ClampMin = "0.0", ClampMax = "4.0",
|
meta = (ClampMin = "0.0", ClampMax = "4.0",
|
||||||
ToolTip = "Microphone volume multiplier.\n1.0 = no change. Increase if the agent can't hear you, decrease if audio clips."))
|
ToolTip = "Microphone volume multiplier.\n1.0 = no change. Increase if the agent can't hear you, decrease if audio clips."))
|
||||||
float VolumeMultiplier = 1.0f;
|
float VolumeMultiplier = 1.0f;
|
||||||
@@ -52,50 +52,50 @@ public:
|
|||||||
// ── Local Voice Activity Detection ──────────────────────────────────
|
// ── Local Voice Activity Detection ──────────────────────────────────
|
||||||
|
|
||||||
/** RMS threshold above which audio is considered voice. Adjust based on mic sensitivity and environment noise. */
|
/** RMS threshold above which audio is considered voice. Adjust based on mic sensitivity and environment noise. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Voice Activity",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Voice Activity",
|
||||||
meta = (ClampMin = "0.001", ClampMax = "0.2"))
|
meta = (ClampMin = "0.001", ClampMax = "0.2"))
|
||||||
float VoiceActivityThreshold = 0.015f;
|
float VoiceActivityThreshold = 0.015f;
|
||||||
|
|
||||||
/** Time (seconds) RMS must stay above threshold before declaring speech onset. Prevents false triggers from clicks/bumps. */
|
/** Time (seconds) RMS must stay above threshold before declaring speech onset. Prevents false triggers from clicks/bumps. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Voice Activity",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Voice Activity",
|
||||||
meta = (ClampMin = "0.01", ClampMax = "1.0"))
|
meta = (ClampMin = "0.01", ClampMax = "1.0"))
|
||||||
float VoiceOnsetTime = 0.1f;
|
float VoiceOnsetTime = 0.1f;
|
||||||
|
|
||||||
/** Time (seconds) RMS must stay below threshold before declaring speech ended. Prevents cutting off between words. */
|
/** Time (seconds) RMS must stay below threshold before declaring speech ended. Prevents cutting off between words. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Voice Activity",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Voice Activity",
|
||||||
meta = (ClampMin = "0.1", ClampMax = "3.0"))
|
meta = (ClampMin = "0.1", ClampMax = "3.0"))
|
||||||
float VoiceSilenceTime = 0.5f;
|
float VoiceSilenceTime = 0.5f;
|
||||||
|
|
||||||
/** Whether the user is currently speaking (local VAD, independent of ElevenLabs server). */
|
/** Whether the user is currently speaking (local VAD, independent of ElevenLabs server). */
|
||||||
UPROPERTY(BlueprintReadOnly, Category = "PS AI ConvAgent|Voice Activity")
|
UPROPERTY(BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|Voice Activity")
|
||||||
bool bIsUserSpeaking = false;
|
bool bIsUserSpeaking = false;
|
||||||
|
|
||||||
/** Fired when voice activity changes. Use to pause NPC movement, trigger animations, etc. */
|
/** Fired when voice activity changes. Use to pause NPC movement, trigger animations, etc. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|Voice Activity")
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|Voice Activity")
|
||||||
FOnPS_AI_ConvAgent_VoiceActivityChanged OnUserVoiceActivityChanged;
|
FOnPS_AI_ConvAgent_VoiceActivityChanged OnUserVoiceActivityChanged;
|
||||||
|
|
||||||
// ── Debug ────────────────────────────────────────────────────────────────
|
// ── Debug ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Enable debug logging for this component.
|
/** Enable debug logging for this component.
|
||||||
* When false, only key events (init, errors, state changes) are logged. */
|
* When false, only key events (init, errors, state changes) are logged. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Debug")
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Debug")
|
||||||
bool bDebug = false;
|
bool bDebug = false;
|
||||||
|
|
||||||
/** Verbosity level when bDebug is true.
|
/** Verbosity level when bDebug is true.
|
||||||
* 0 = minimal extras, 1 = normal debug, 2 = detailed, 3 = per-frame data. */
|
* 0 = minimal extras, 1 = normal debug, 2 = detailed, 3 = per-frame data. */
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Debug",
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Debug",
|
||||||
meta = (ClampMin = "0", ClampMax = "3", EditCondition = "bDebug"))
|
meta = (ClampMin = "0", ClampMax = "3", EditCondition = "bDebug"))
|
||||||
int32 DebugVerbosity = 1;
|
int32 DebugVerbosity = 1;
|
||||||
|
|
||||||
/** Open the default capture device and begin streaming audio. */
|
/** Open the default capture device and begin streaming audio. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|MicCapture")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|MicCapture")
|
||||||
void StartCapture();
|
void StartCapture();
|
||||||
|
|
||||||
/** Stop streaming and close the capture device. */
|
/** Stop streaming and close the capture device. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|MicCapture")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|MicCapture")
|
||||||
void StopCapture();
|
void StopCapture();
|
||||||
|
|
||||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|MicCapture")
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|MicCapture")
|
||||||
bool IsCapturing() const { return bCapturing; }
|
bool IsCapturing() const { return bCapturing; }
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────
|
||||||
|
|||||||
@@ -0,0 +1,42 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "CoreMinimal.h"
|
||||||
|
#include "GameFramework/SaveGame.h"
|
||||||
|
#include "PS_AI_ConvAgent_SaveGame_ElevenLabs.generated.h"
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// UPS_AI_ConvAgent_SaveGame_ElevenLabs
|
||||||
|
//
|
||||||
|
// Per-install persistence for the CLIENT's own ElevenLabs API key. Written via
|
||||||
|
// UGameplayStatics::SaveGameToSlot to:
|
||||||
|
// - Editor/PIE : <Project>/Saved/SaveGames/PS_AI_ConvAgent_ElevenLabs.sav
|
||||||
|
// - Packaged : <Build>/<Project>/Saved/SaveGames/PS_AI_ConvAgent_ElevenLabs.sav
|
||||||
|
//
|
||||||
|
// Binary file → the key is never stored in plaintext, never committed, never
|
||||||
|
// baked into the shipped config. The key is AES-encrypted at rest (see the codec
|
||||||
|
// in UPS_AI_ConvAgent_BlueprintLibrary) and entered once at runtime via the
|
||||||
|
// settings UI (BlueprintLibrary "Set ElevenLabs API Key").
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
UCLASS()
|
||||||
|
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_SaveGame_ElevenLabs : public USaveGame
|
||||||
|
{
|
||||||
|
GENERATED_BODY()
|
||||||
|
|
||||||
|
public:
|
||||||
|
/** AES-encrypted ElevenLabs API key (UTF-8, block-padded). Empty until set.
|
||||||
|
* Encrypted/decrypted by UPS_AI_ConvAgent_BlueprintLibrary. */
|
||||||
|
UPROPERTY()
|
||||||
|
TArray<uint8> EncryptedAPIKey;
|
||||||
|
|
||||||
|
/** Plaintext (UTF-8) byte length, used to strip AES block padding on decrypt. */
|
||||||
|
UPROPERTY()
|
||||||
|
int32 PlainKeyLength = 0;
|
||||||
|
|
||||||
|
/** Fixed save slot name used for the key. */
|
||||||
|
static const TCHAR* GetSlotName() { return TEXT("PS_AI_ConvAgent_ElevenLabs"); }
|
||||||
|
|
||||||
|
/** Fixed user index used for the key slot. */
|
||||||
|
static constexpr int32 GetUserIndex() { return 0; }
|
||||||
|
};
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "CoreMinimal.h"
|
||||||
|
#include "Kismet/BlueprintAsyncActionBase.h"
|
||||||
|
#include "PS_AI_ConvAgent_TestAPIKey_AsyncAction.generated.h"
|
||||||
|
|
||||||
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPS_AI_ConvAgent_TestAPIKeyResult, const FString&, Message);
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// UPS_AI_ConvAgent_TestAPIKey_AsyncAction
|
||||||
|
//
|
||||||
|
// Latent Blueprint node that validates an ElevenLabs API key by issuing a
|
||||||
|
// lightweight GET /v1/models request with the key as the xi-api-key header.
|
||||||
|
// - HTTP 200 → OnValid (key works)
|
||||||
|
// - HTTP 401 / other → OnInvalid (bad key, or network/server error)
|
||||||
|
//
|
||||||
|
// Designed for a settings UI: let the user type their key, run this node, and
|
||||||
|
// only call "Set ElevenLabs API Key" on the OnValid branch.
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
UCLASS()
|
||||||
|
class PS_AI_CONVAGENT_API UPS_AI_ConvAgent_TestAPIKey_AsyncAction : public UBlueprintAsyncActionBase
|
||||||
|
{
|
||||||
|
GENERATED_BODY()
|
||||||
|
|
||||||
|
public:
|
||||||
|
/** Test an ElevenLabs API key. Pass the key the user just entered in your settings UI. */
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs API Key",
|
||||||
|
meta = (BlueprintInternalUseOnly = "true", DisplayName = "Test ElevenLabs API Key"))
|
||||||
|
static UPS_AI_ConvAgent_TestAPIKey_AsyncAction* TestElevenLabsAPIKey(const FString& Key);
|
||||||
|
|
||||||
|
/** Fired when the key is valid (HTTP 200). The Message contains a human-readable status. */
|
||||||
|
UPROPERTY(BlueprintAssignable)
|
||||||
|
FPS_AI_ConvAgent_TestAPIKeyResult OnValid;
|
||||||
|
|
||||||
|
/** Fired when the key is invalid or the request failed. The Message explains why. */
|
||||||
|
UPROPERTY(BlueprintAssignable)
|
||||||
|
FPS_AI_ConvAgent_TestAPIKeyResult OnInvalid;
|
||||||
|
|
||||||
|
//~ Begin UBlueprintAsyncActionBase interface
|
||||||
|
virtual void Activate() override;
|
||||||
|
//~ End UBlueprintAsyncActionBase interface
|
||||||
|
|
||||||
|
private:
|
||||||
|
FString KeyToTest;
|
||||||
|
};
|
||||||
@@ -69,31 +69,31 @@ public:
|
|||||||
// ── Events ────────────────────────────────────────────────────────────────
|
// ── Events ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Called once the WebSocket handshake succeeds and the agent sends its initiation metadata. */
|
/** Called once the WebSocket handshake succeeds and the agent sends its initiation metadata. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events")
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events")
|
||||||
FOnPS_AI_ConvAgent_ElevenLabs_Connected OnConnected;
|
FOnPS_AI_ConvAgent_ElevenLabs_Connected OnConnected;
|
||||||
|
|
||||||
/** Called when the WebSocket closes (graceful or remote). */
|
/** Called when the WebSocket closes (graceful or remote). */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events")
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events")
|
||||||
FOnPS_AI_ConvAgent_ElevenLabs_Disconnected OnDisconnected;
|
FOnPS_AI_ConvAgent_ElevenLabs_Disconnected OnDisconnected;
|
||||||
|
|
||||||
/** Called on any connection or protocol error. */
|
/** Called on any connection or protocol error. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events")
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events")
|
||||||
FOnPS_AI_ConvAgent_ElevenLabs_Error OnError;
|
FOnPS_AI_ConvAgent_ElevenLabs_Error OnError;
|
||||||
|
|
||||||
/** Raw PCM audio coming from the agent — feed this into your audio component. */
|
/** Raw PCM audio coming from the agent — feed this into your audio component. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events")
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events")
|
||||||
FOnPS_AI_ConvAgent_ElevenLabs_AudioReceived OnAudioReceived;
|
FOnPS_AI_ConvAgent_ElevenLabs_AudioReceived OnAudioReceived;
|
||||||
|
|
||||||
/** User or agent transcript (may be tentative while the conversation is ongoing). */
|
/** User or agent transcript (may be tentative while the conversation is ongoing). */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events")
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events")
|
||||||
FOnPS_AI_ConvAgent_ElevenLabs_Transcript OnTranscript;
|
FOnPS_AI_ConvAgent_ElevenLabs_Transcript OnTranscript;
|
||||||
|
|
||||||
/** Final text response from the agent (complements audio). */
|
/** Final text response from the agent (complements audio). */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events")
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events")
|
||||||
FOnPS_AI_ConvAgent_ElevenLabs_AgentResponse OnAgentResponse;
|
FOnPS_AI_ConvAgent_ElevenLabs_AgentResponse OnAgentResponse;
|
||||||
|
|
||||||
/** The agent was interrupted by new user speech. */
|
/** The agent was interrupted by new user speech. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events")
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events")
|
||||||
FOnPS_AI_ConvAgent_ElevenLabs_Interrupted OnInterrupted;
|
FOnPS_AI_ConvAgent_ElevenLabs_Interrupted OnInterrupted;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -101,15 +101,15 @@ public:
|
|||||||
* starts generating. Fires well before audio. The component uses this to stop the
|
* starts generating. Fires well before audio. The component uses this to stop the
|
||||||
* microphone if it was restarted before the server finished processing the previous turn.
|
* microphone if it was restarted before the server finished processing the previous turn.
|
||||||
*/
|
*/
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events")
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events")
|
||||||
FOnPS_AI_ConvAgent_ElevenLabs_ResponseStarted OnAgentResponseStarted;
|
FOnPS_AI_ConvAgent_ElevenLabs_ResponseStarted OnAgentResponseStarted;
|
||||||
|
|
||||||
/** Fired for every agent_chat_response_part with the streaming text fragment. */
|
/** Fired for every agent_chat_response_part with the streaming text fragment. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events")
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events")
|
||||||
FOnPS_AI_ConvAgent_ElevenLabs_ResponsePart OnAgentResponsePart;
|
FOnPS_AI_ConvAgent_ElevenLabs_ResponsePart OnAgentResponsePart;
|
||||||
|
|
||||||
/** Fired when the agent invokes a client tool. Handle the call and reply with SendClientToolResult. */
|
/** Fired when the agent invokes a client tool. Handle the call and reply with SendClientToolResult. */
|
||||||
UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events")
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events")
|
||||||
FOnPS_AI_ConvAgent_ElevenLabs_ClientToolCall OnClientToolCall;
|
FOnPS_AI_ConvAgent_ElevenLabs_ClientToolCall OnClientToolCall;
|
||||||
|
|
||||||
// ── Lifecycle ─────────────────────────────────────────────────────────────
|
// ── Lifecycle ─────────────────────────────────────────────────────────────
|
||||||
@@ -121,21 +121,21 @@ public:
|
|||||||
* @param AgentID ElevenLabs agent ID. Overrides the project-level default when non-empty.
|
* @param AgentID ElevenLabs agent ID. Overrides the project-level default when non-empty.
|
||||||
* @param APIKey API key. Overrides the project-level default when non-empty.
|
* @param APIKey API key. Overrides the project-level default when non-empty.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
void Connect(const FString& AgentID = TEXT(""), const FString& APIKey = TEXT(""));
|
void Connect(const FString& AgentID = TEXT(""), const FString& APIKey = TEXT(""));
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Gracefully close the WebSocket connection.
|
* Gracefully close the WebSocket connection.
|
||||||
* OnDisconnected will fire after the server acknowledges.
|
* OnDisconnected will fire after the server acknowledges.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
void Disconnect();
|
void Disconnect();
|
||||||
|
|
||||||
/** Current connection state. */
|
/** Current connection state. */
|
||||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
EPS_AI_ConvAgent_ConnectionState_ElevenLabs GetConnectionState() const { return ConnectionState; }
|
EPS_AI_ConvAgent_ConnectionState_ElevenLabs GetConnectionState() const { return ConnectionState; }
|
||||||
|
|
||||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
bool IsConnected() const { return ConnectionState == EPS_AI_ConvAgent_ConnectionState_ElevenLabs::Connected; }
|
bool IsConnected() const { return ConnectionState == EPS_AI_ConvAgent_ConnectionState_ElevenLabs::Connected; }
|
||||||
|
|
||||||
// ── Audio sending ─────────────────────────────────────────────────────────
|
// ── Audio sending ─────────────────────────────────────────────────────────
|
||||||
@@ -148,7 +148,7 @@ public:
|
|||||||
*
|
*
|
||||||
* @param PCMData Raw PCM bytes (16-bit LE, 16kHz, mono).
|
* @param PCMData Raw PCM bytes (16-bit LE, 16kHz, mono).
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
void SendAudioChunk(const TArray<uint8>& PCMData);
|
void SendAudioChunk(const TArray<uint8>& PCMData);
|
||||||
|
|
||||||
// ── Turn control (only relevant in Client turn mode) ──────────────────────
|
// ── Turn control (only relevant in Client turn mode) ──────────────────────
|
||||||
@@ -158,7 +158,7 @@ public:
|
|||||||
* Sends a { "type": "user_activity" } message to the server.
|
* Sends a { "type": "user_activity" } message to the server.
|
||||||
* Call this periodically while the user is speaking (e.g. every audio chunk).
|
* Call this periodically while the user is speaking (e.g. every audio chunk).
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
void SendUserTurnStart();
|
void SendUserTurnStart();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -166,7 +166,7 @@ public:
|
|||||||
* No explicit API message — simply stop sending user_activity.
|
* No explicit API message — simply stop sending user_activity.
|
||||||
* The server detects silence and hands the turn to the agent.
|
* The server detects silence and hands the turn to the agent.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
void SendUserTurnEnd();
|
void SendUserTurnEnd();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -174,11 +174,17 @@ public:
|
|||||||
* Useful for testing or text-only interaction.
|
* Useful for testing or text-only interaction.
|
||||||
* Sends: { "type": "user_message", "text": "..." }
|
* Sends: { "type": "user_message", "text": "..." }
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
void SendTextMessage(const FString& Text);
|
void SendTextMessage(const FString& Text);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Inject a context update WITHOUT triggering an agent response.
|
||||||
|
* Sends: { "type": "contextual_update", "text": "..." }
|
||||||
|
*/
|
||||||
|
void SendContextualUpdate(const FString& Text);
|
||||||
|
|
||||||
/** Ask the agent to stop the current utterance. */
|
/** Ask the agent to stop the current utterance. */
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
void SendInterrupt();
|
void SendInterrupt();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -189,12 +195,12 @@ public:
|
|||||||
* @param Result A string result to return to the agent.
|
* @param Result A string result to return to the agent.
|
||||||
* @param bIsError True if the tool execution failed.
|
* @param bIsError True if the tool execution failed.
|
||||||
*/
|
*/
|
||||||
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
void SendClientToolResult(const FString& ToolCallId, const FString& Result, bool bIsError = false);
|
void SendClientToolResult(const FString& ToolCallId, const FString& Result, bool bIsError = false);
|
||||||
|
|
||||||
// ── Info ──────────────────────────────────────────────────────────────────
|
// ── Info ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|ElevenLabs")
|
UFUNCTION(BlueprintPure, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs")
|
||||||
const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& GetConversationInfo() const { return ConversationInfo; }
|
const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& GetConversationInfo() const { return ConversationInfo; }
|
||||||
|
|
||||||
/** Latest WebSocket round-trip latency reported by the server (ms).
|
/** Latest WebSocket round-trip latency reported by the server (ms).
|
||||||
@@ -234,6 +240,14 @@ private:
|
|||||||
/** Resolve the WebSocket URL from settings / parameters. */
|
/** Resolve the WebSocket URL from settings / parameters. */
|
||||||
FString BuildWebSocketURL(const FString& AgentID, const FString& APIKey) const;
|
FString BuildWebSocketURL(const FString& AgentID, const FString& APIKey) const;
|
||||||
|
|
||||||
|
/** Fetch a short-lived signed URL for a (private) agent using the API key,
|
||||||
|
* then open the WebSocket on it. Required for agents with authentication
|
||||||
|
* enabled — a direct ?agent_id=... connection is rejected by ElevenLabs. */
|
||||||
|
void RequestSignedURLAndConnect(const FString& AgentID, const FString& APIKey);
|
||||||
|
|
||||||
|
/** Create the WebSocket on the given URL, bind callbacks, and connect. */
|
||||||
|
void OpenWebSocket(const FString& URL, const TMap<FString, FString>& UpgradeHeaders);
|
||||||
|
|
||||||
TSharedPtr<IWebSocket> WebSocket;
|
TSharedPtr<IWebSocket> WebSocket;
|
||||||
EPS_AI_ConvAgent_ConnectionState_ElevenLabs ConnectionState = EPS_AI_ConvAgent_ConnectionState_ElevenLabs::Disconnected;
|
EPS_AI_ConvAgent_ConnectionState_ElevenLabs ConnectionState = EPS_AI_ConvAgent_ConnectionState_ElevenLabs::Disconnected;
|
||||||
FPS_AI_ConvAgent_ConversationInfo_ElevenLabs ConversationInfo;
|
FPS_AI_ConvAgent_ConversationInfo_ElevenLabs ConversationInfo;
|
||||||
|
|||||||
@@ -35,6 +35,8 @@ public class PS_AI_ConvAgentEditor : ModuleRules
|
|||||||
"JsonUtilities",
|
"JsonUtilities",
|
||||||
// Asset Registry for scanning AgentConfig assets (batch update)
|
// Asset Registry for scanning AgentConfig assets (batch update)
|
||||||
"AssetRegistry",
|
"AssetRegistry",
|
||||||
|
// UDeveloperSettings base class for editor-only API key settings
|
||||||
|
"DeveloperSettings",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ FText UAnimGraphNode_PS_AI_ConvAgent_BodyExpression::GetTooltipText() const
|
|||||||
|
|
||||||
FString UAnimGraphNode_PS_AI_ConvAgent_BodyExpression::GetNodeCategory() const
|
FString UAnimGraphNode_PS_AI_ConvAgent_BodyExpression::GetNodeCategory() const
|
||||||
{
|
{
|
||||||
return TEXT("PS AI ConvAgent");
|
return TEXT("ASTERION|PS_AI_ConvAgent");
|
||||||
}
|
}
|
||||||
|
|
||||||
FLinearColor UAnimGraphNode_PS_AI_ConvAgent_BodyExpression::GetNodeTitleColor() const
|
FLinearColor UAnimGraphNode_PS_AI_ConvAgent_BodyExpression::GetNodeTitleColor() const
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ FText UAnimGraphNode_PS_AI_ConvAgent_FacialExpression::GetTooltipText() const
|
|||||||
|
|
||||||
FString UAnimGraphNode_PS_AI_ConvAgent_FacialExpression::GetNodeCategory() const
|
FString UAnimGraphNode_PS_AI_ConvAgent_FacialExpression::GetNodeCategory() const
|
||||||
{
|
{
|
||||||
return TEXT("PS AI ConvAgent");
|
return TEXT("ASTERION|PS_AI_ConvAgent");
|
||||||
}
|
}
|
||||||
|
|
||||||
FLinearColor UAnimGraphNode_PS_AI_ConvAgent_FacialExpression::GetNodeTitleColor() const
|
FLinearColor UAnimGraphNode_PS_AI_ConvAgent_FacialExpression::GetNodeTitleColor() const
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ FText UAnimGraphNode_PS_AI_ConvAgent_Gaze::GetTooltipText() const
|
|||||||
|
|
||||||
FString UAnimGraphNode_PS_AI_ConvAgent_Gaze::GetNodeCategory() const
|
FString UAnimGraphNode_PS_AI_ConvAgent_Gaze::GetNodeCategory() const
|
||||||
{
|
{
|
||||||
return TEXT("PS AI ConvAgent");
|
return TEXT("ASTERION|PS_AI_ConvAgent");
|
||||||
}
|
}
|
||||||
|
|
||||||
FLinearColor UAnimGraphNode_PS_AI_ConvAgent_Gaze::GetNodeTitleColor() const
|
FLinearColor UAnimGraphNode_PS_AI_ConvAgent_Gaze::GetNodeTitleColor() const
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ FText UAnimGraphNode_PS_AI_ConvAgent_LipSync::GetTooltipText() const
|
|||||||
|
|
||||||
FString UAnimGraphNode_PS_AI_ConvAgent_LipSync::GetNodeCategory() const
|
FString UAnimGraphNode_PS_AI_ConvAgent_LipSync::GetNodeCategory() const
|
||||||
{
|
{
|
||||||
return TEXT("PS AI ConvAgent");
|
return TEXT("ASTERION|PS_AI_ConvAgent");
|
||||||
}
|
}
|
||||||
|
|
||||||
FLinearColor UAnimGraphNode_PS_AI_ConvAgent_LipSync::GetNodeTitleColor() const
|
FLinearColor UAnimGraphNode_PS_AI_ConvAgent_LipSync::GetNodeTitleColor() const
|
||||||
|
|||||||
@@ -3,6 +3,8 @@
|
|||||||
#include "PS_AI_ConvAgent_ActionSetCustomization_ElevenLabs.h"
|
#include "PS_AI_ConvAgent_ActionSetCustomization_ElevenLabs.h"
|
||||||
#include "PS_AI_ConvAgent_ActionSet_ElevenLabs.h"
|
#include "PS_AI_ConvAgent_ActionSet_ElevenLabs.h"
|
||||||
#include "PS_AI_ConvAgent.h"
|
#include "PS_AI_ConvAgent.h"
|
||||||
|
#include "PS_AI_ConvAgent_BlueprintLibrary.h"
|
||||||
|
#include "PS_AI_ConvAgent_EditorSettings_ElevenLabs.h"
|
||||||
|
|
||||||
#include "DetailLayoutBuilder.h"
|
#include "DetailLayoutBuilder.h"
|
||||||
#include "DetailCategoryBuilder.h"
|
#include "DetailCategoryBuilder.h"
|
||||||
@@ -65,14 +67,9 @@ void FPS_AI_ConvAgent_ActionSetCustomization_ElevenLabs::OnUpdateAllAgentsClicke
|
|||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
FString FPS_AI_ConvAgent_ActionSetCustomization_ElevenLabs::GetAPIKey() const
|
FString FPS_AI_ConvAgent_ActionSetCustomization_ElevenLabs::GetAPIKey() const
|
||||||
{
|
{
|
||||||
if (FPS_AI_ConvAgentModule::IsAvailable())
|
// Editor-only authoring key (Project Settings → Editor, or ELEVENLABS_API_KEY env).
|
||||||
{
|
// Never committed by a build, never packaged — see UPS_AI_ConvAgent_EditorSettings_ElevenLabs.
|
||||||
if (const UPS_AI_ConvAgent_Settings_ElevenLabs* Settings = FPS_AI_ConvAgentModule::Get().GetSettings())
|
return UPS_AI_ConvAgent_EditorSettings_ElevenLabs::ResolveEditorApiKey();
|
||||||
{
|
|
||||||
return Settings->API_Key;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return FString();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
UPS_AI_ConvAgent_ActionSet_ElevenLabs* FPS_AI_ConvAgent_ActionSetCustomization_ElevenLabs::GetEditedAsset() const
|
UPS_AI_ConvAgent_ActionSet_ElevenLabs* FPS_AI_ConvAgent_ActionSetCustomization_ElevenLabs::GetEditedAsset() const
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
#include "PS_AI_ConvAgent_AgentConfig_ElevenLabs.h"
|
#include "PS_AI_ConvAgent_AgentConfig_ElevenLabs.h"
|
||||||
#include "PS_AI_ConvAgent_Tool_ElevenLabs.h"
|
#include "PS_AI_ConvAgent_Tool_ElevenLabs.h"
|
||||||
#include "PS_AI_ConvAgent.h"
|
#include "PS_AI_ConvAgent.h"
|
||||||
|
#include "PS_AI_ConvAgent_BlueprintLibrary.h"
|
||||||
|
#include "PS_AI_ConvAgent_EditorSettings_ElevenLabs.h"
|
||||||
|
|
||||||
#include "AssetRegistry/AssetRegistryModule.h"
|
#include "AssetRegistry/AssetRegistryModule.h"
|
||||||
|
|
||||||
@@ -425,7 +427,7 @@ void FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::OnFetchVoicesClicked(
|
|||||||
const FString APIKey = GetAPIKey();
|
const FString APIKey = GetAPIKey();
|
||||||
if (APIKey.IsEmpty())
|
if (APIKey.IsEmpty())
|
||||||
{
|
{
|
||||||
SetStatusError(TEXT("API Key not set in Project Settings > PS AI ConvAgent - ElevenLabs."));
|
SetStatusError(TEXT("ElevenLabs API Key not set. Set it in Project Settings > Editor > 'PS AI ConvAgent - ElevenLabs (Editor)' (or the ELEVENLABS_API_KEY env var)."));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -631,7 +633,7 @@ void FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::OnFetchLLMsClicked()
|
|||||||
const FString APIKey = GetAPIKey();
|
const FString APIKey = GetAPIKey();
|
||||||
if (APIKey.IsEmpty())
|
if (APIKey.IsEmpty())
|
||||||
{
|
{
|
||||||
SetStatusError(TEXT("API Key not set in Project Settings > PS AI ConvAgent - ElevenLabs."));
|
SetStatusError(TEXT("ElevenLabs API Key not set. Set it in Project Settings > Editor > 'PS AI ConvAgent - ElevenLabs (Editor)' (or the ELEVENLABS_API_KEY env var)."));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -917,7 +919,7 @@ void FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::OnCreateAgentClicked(
|
|||||||
const FString APIKey = GetAPIKey();
|
const FString APIKey = GetAPIKey();
|
||||||
if (APIKey.IsEmpty())
|
if (APIKey.IsEmpty())
|
||||||
{
|
{
|
||||||
SetStatusError(TEXT("API Key not set in Project Settings."));
|
SetStatusError(TEXT("ElevenLabs API Key not set. Set it in Project Settings > Editor > 'PS AI ConvAgent - ElevenLabs (Editor)' (or the ELEVENLABS_API_KEY env var)."));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1019,7 +1021,7 @@ void FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::OnUpdateAgentClicked(
|
|||||||
const FString APIKey = GetAPIKey();
|
const FString APIKey = GetAPIKey();
|
||||||
if (APIKey.IsEmpty())
|
if (APIKey.IsEmpty())
|
||||||
{
|
{
|
||||||
SetStatusError(TEXT("API Key not set in Project Settings."));
|
SetStatusError(TEXT("ElevenLabs API Key not set. Set it in Project Settings > Editor > 'PS AI ConvAgent - ElevenLabs (Editor)' (or the ELEVENLABS_API_KEY env var)."));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1107,7 +1109,7 @@ void FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::OnFetchAgentClicked()
|
|||||||
const FString APIKey = GetAPIKey();
|
const FString APIKey = GetAPIKey();
|
||||||
if (APIKey.IsEmpty())
|
if (APIKey.IsEmpty())
|
||||||
{
|
{
|
||||||
SetStatusError(TEXT("API Key not set in Project Settings."));
|
SetStatusError(TEXT("ElevenLabs API Key not set. Set it in Project Settings > Editor > 'PS AI ConvAgent - ElevenLabs (Editor)' (or the ELEVENLABS_API_KEY env var)."));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1508,14 +1510,9 @@ void FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::OnFetchAgentClicked()
|
|||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
FString FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::GetAPIKey() const
|
FString FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::GetAPIKey() const
|
||||||
{
|
{
|
||||||
if (FPS_AI_ConvAgentModule::IsAvailable())
|
// Editor-only authoring key (Project Settings → Editor, or ELEVENLABS_API_KEY env).
|
||||||
{
|
// Never committed by a build, never packaged — see UPS_AI_ConvAgent_EditorSettings_ElevenLabs.
|
||||||
if (const UPS_AI_ConvAgent_Settings_ElevenLabs* Settings = FPS_AI_ConvAgentModule::Get().GetSettings())
|
return UPS_AI_ConvAgent_EditorSettings_ElevenLabs::ResolveEditorApiKey();
|
||||||
{
|
|
||||||
return Settings->API_Key;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return FString();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
UPS_AI_ConvAgent_AgentConfig_ElevenLabs* FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::GetEditedAsset() const
|
UPS_AI_ConvAgent_AgentConfig_ElevenLabs* FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::GetEditedAsset() const
|
||||||
@@ -1827,6 +1824,20 @@ TSharedPtr<FJsonObject> FPS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs::Bu
|
|||||||
}
|
}
|
||||||
Root->SetObjectField(TEXT("conversation_config"), ConvConfig);
|
Root->SetObjectField(TEXT("conversation_config"), ConvConfig);
|
||||||
|
|
||||||
|
// platform_settings.auth.enable_auth = true → force every created/updated
|
||||||
|
// agent to be PRIVATE (authentication required). Hardcoded on purpose (no
|
||||||
|
// toggle): a public agent can be used by anyone who knows the agent_id,
|
||||||
|
// running up the owner's ElevenLabs bill without a valid API key.
|
||||||
|
{
|
||||||
|
TSharedPtr<FJsonObject> AuthObj = MakeShareable(new FJsonObject());
|
||||||
|
AuthObj->SetBoolField(TEXT("enable_auth"), true);
|
||||||
|
|
||||||
|
TSharedPtr<FJsonObject> PlatformSettings = MakeShareable(new FJsonObject());
|
||||||
|
PlatformSettings->SetObjectField(TEXT("auth"), AuthObj);
|
||||||
|
|
||||||
|
Root->SetObjectField(TEXT("platform_settings"), PlatformSettings);
|
||||||
|
}
|
||||||
|
|
||||||
return Root;
|
return Root;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#include "PS_AI_ConvAgent_EditorSettings_ElevenLabs.h"
|
||||||
|
#include "PS_AI_ConvAgent_BlueprintLibrary.h"
|
||||||
|
#include "HAL/PlatformMisc.h"
|
||||||
|
|
||||||
|
FString UPS_AI_ConvAgent_EditorSettings_ElevenLabs::ResolveEditorApiKey()
|
||||||
|
{
|
||||||
|
// 1) Explicit editor setting (Project Settings → Editor).
|
||||||
|
if (const UPS_AI_ConvAgent_EditorSettings_ElevenLabs* Settings = GetDefault<UPS_AI_ConvAgent_EditorSettings_ElevenLabs>())
|
||||||
|
{
|
||||||
|
if (!Settings->EditorApiKey.IsEmpty())
|
||||||
|
{
|
||||||
|
return Settings->EditorApiKey;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2) Environment variable (CI / shared machines).
|
||||||
|
const FString EnvKey = FPlatformMisc::GetEnvironmentVariable(TEXT("ELEVENLABS_API_KEY"));
|
||||||
|
if (!EnvKey.IsEmpty())
|
||||||
|
{
|
||||||
|
return EnvKey;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3) Runtime client key from the SaveGame, if one was set on this machine.
|
||||||
|
return UPS_AI_ConvAgent_BlueprintLibrary::GetElevenLabsAPIKey();
|
||||||
|
}
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "CoreMinimal.h"
|
||||||
|
#include "Engine/DeveloperSettings.h"
|
||||||
|
#include "PS_AI_ConvAgent_EditorSettings_ElevenLabs.generated.h"
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Editor-only settings for ElevenLabs agent authoring.
|
||||||
|
*
|
||||||
|
* The API key entered here is used EXCLUSIVELY by the editor (voice/model/LLM
|
||||||
|
* fetch, agent create/sync, tool & action-set sync). It is:
|
||||||
|
* - declared in the EDITOR module → never compiled into a packaged game,
|
||||||
|
* - stored with config = EditorPerProjectUserSettings → written to
|
||||||
|
* Saved/Config/.../EditorPerProjectUserSettings.ini, which is never cooked
|
||||||
|
* into a build.
|
||||||
|
* → The key therefore can never end up in a shipped package.
|
||||||
|
*
|
||||||
|
* The shipped game does NOT use this key. The final client provides their own
|
||||||
|
* key at runtime, persisted in a SaveGame slot (see UPS_AI_ConvAgent_BlueprintLibrary).
|
||||||
|
*
|
||||||
|
* Exposed in Project Settings → Editor → "PS AI ConvAgent - ElevenLabs (Editor)".
|
||||||
|
*/
|
||||||
|
UCLASS(config = EditorPerProjectUserSettings, defaultconfig,
|
||||||
|
meta = (DisplayName = "PS AI ConvAgent - ElevenLabs (Editor)"))
|
||||||
|
class UPS_AI_ConvAgent_EditorSettings_ElevenLabs : public UDeveloperSettings
|
||||||
|
{
|
||||||
|
GENERATED_BODY()
|
||||||
|
|
||||||
|
public:
|
||||||
|
/**
|
||||||
|
* ElevenLabs API key used by the editor to author and sync agents.
|
||||||
|
* Editor-only — never committed by a build, never packaged.
|
||||||
|
* Leave empty and set the ELEVENLABS_API_KEY environment variable instead
|
||||||
|
* if you prefer (useful for CI / shared machines).
|
||||||
|
*/
|
||||||
|
UPROPERTY(config, EditAnywhere, Category = "ElevenLabs API",
|
||||||
|
meta = (DisplayName = "Editor API Key",
|
||||||
|
ToolTip = "ElevenLabs API key used ONLY by the editor for authoring/syncing agents.\nStored per-user under Saved/Config - never committed by a build, never packaged.\nThe shipped game uses the client-provided runtime key instead.\nAlternatively, leave empty and set the ELEVENLABS_API_KEY environment variable."))
|
||||||
|
FString EditorApiKey;
|
||||||
|
|
||||||
|
/** Show under the "Editor" section of Project Settings (not "Game"/"Plugins"). */
|
||||||
|
virtual FName GetCategoryName() const override { return FName(TEXT("Editor")); }
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve the API key used by editor authoring, in priority order:
|
||||||
|
* 1) the EditorApiKey above,
|
||||||
|
* 2) the ELEVENLABS_API_KEY environment variable (CI / shared machines),
|
||||||
|
* 3) the runtime client key from the SaveGame, if one happens to exist locally.
|
||||||
|
* Returns an empty string if none is set.
|
||||||
|
*/
|
||||||
|
static FString ResolveEditorApiKey();
|
||||||
|
};
|
||||||
@@ -5,6 +5,8 @@
|
|||||||
#include "PS_AI_ConvAgent_AgentConfig_ElevenLabs.h"
|
#include "PS_AI_ConvAgent_AgentConfig_ElevenLabs.h"
|
||||||
#include "PS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs.h"
|
#include "PS_AI_ConvAgent_AgentConfigCustomization_ElevenLabs.h"
|
||||||
#include "PS_AI_ConvAgent.h"
|
#include "PS_AI_ConvAgent.h"
|
||||||
|
#include "PS_AI_ConvAgent_BlueprintLibrary.h"
|
||||||
|
#include "PS_AI_ConvAgent_EditorSettings_ElevenLabs.h"
|
||||||
|
|
||||||
#include "DetailLayoutBuilder.h"
|
#include "DetailLayoutBuilder.h"
|
||||||
#include "DetailCategoryBuilder.h"
|
#include "DetailCategoryBuilder.h"
|
||||||
@@ -242,7 +244,7 @@ void FPS_AI_ConvAgent_ToolCustomization_ElevenLabs::OnCreateToolClicked()
|
|||||||
const FString APIKey = GetAPIKey();
|
const FString APIKey = GetAPIKey();
|
||||||
if (APIKey.IsEmpty())
|
if (APIKey.IsEmpty())
|
||||||
{
|
{
|
||||||
SetStatusError(TEXT("API Key not set in Project Settings."));
|
SetStatusError(TEXT("ElevenLabs API Key not set. Set it in Project Settings > Editor > 'PS AI ConvAgent - ElevenLabs (Editor)' (or the ELEVENLABS_API_KEY env var)."));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -353,7 +355,7 @@ void FPS_AI_ConvAgent_ToolCustomization_ElevenLabs::OnUpdateToolClicked()
|
|||||||
const FString APIKey = GetAPIKey();
|
const FString APIKey = GetAPIKey();
|
||||||
if (APIKey.IsEmpty())
|
if (APIKey.IsEmpty())
|
||||||
{
|
{
|
||||||
SetStatusError(TEXT("API Key not set in Project Settings."));
|
SetStatusError(TEXT("ElevenLabs API Key not set. Set it in Project Settings > Editor > 'PS AI ConvAgent - ElevenLabs (Editor)' (or the ELEVENLABS_API_KEY env var)."));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -443,7 +445,7 @@ void FPS_AI_ConvAgent_ToolCustomization_ElevenLabs::OnFetchToolClicked()
|
|||||||
const FString APIKey = GetAPIKey();
|
const FString APIKey = GetAPIKey();
|
||||||
if (APIKey.IsEmpty())
|
if (APIKey.IsEmpty())
|
||||||
{
|
{
|
||||||
SetStatusError(TEXT("API Key not set in Project Settings."));
|
SetStatusError(TEXT("ElevenLabs API Key not set. Set it in Project Settings > Editor > 'PS AI ConvAgent - ElevenLabs (Editor)' (or the ELEVENLABS_API_KEY env var)."));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -560,8 +562,10 @@ void FPS_AI_ConvAgent_ToolCustomization_ElevenLabs::OnFetchToolClicked()
|
|||||||
if (!Pair.Value->TryGetObject(PropObj)) continue;
|
if (!Pair.Value->TryGetObject(PropObj)) continue;
|
||||||
|
|
||||||
FPS_AI_ConvAgent_ToolParameter_ElevenLabs Param;
|
FPS_AI_ConvAgent_ToolParameter_ElevenLabs Param;
|
||||||
Param.Name = Pair.Key;
|
// UE 5.8: FJsonObject::Values key is now FSharedString, convert explicitly.
|
||||||
Param.bRequired = RequiredSet.Contains(Pair.Key);
|
FString KeyStr(*Pair.Key);
|
||||||
|
Param.Name = KeyStr;
|
||||||
|
Param.bRequired = RequiredSet.Contains(KeyStr);
|
||||||
|
|
||||||
// Try ElevenLabs format: type-as-key
|
// Try ElevenLabs format: type-as-key
|
||||||
static const TArray<TPair<FString, EPS_AI_ConvAgent_ToolParamType>> TypeKeys = {
|
static const TArray<TPair<FString, EPS_AI_ConvAgent_ToolParamType>> TypeKeys = {
|
||||||
@@ -658,7 +662,7 @@ void FPS_AI_ConvAgent_ToolCustomization_ElevenLabs::OnUpdateAllAgentsClicked()
|
|||||||
const FString APIKey = GetAPIKey();
|
const FString APIKey = GetAPIKey();
|
||||||
if (APIKey.IsEmpty())
|
if (APIKey.IsEmpty())
|
||||||
{
|
{
|
||||||
SetStatusError(TEXT("API Key not set in Project Settings > PS AI ConvAgent - ElevenLabs."));
|
SetStatusError(TEXT("ElevenLabs API Key not set. Set it in Project Settings > Editor > 'PS AI ConvAgent - ElevenLabs (Editor)' (or the ELEVENLABS_API_KEY env var)."));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -818,15 +822,9 @@ void FPS_AI_ConvAgent_ToolCustomization_ElevenLabs::OnUpdateAllAgentsClicked()
|
|||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
FString FPS_AI_ConvAgent_ToolCustomization_ElevenLabs::GetAPIKey() const
|
FString FPS_AI_ConvAgent_ToolCustomization_ElevenLabs::GetAPIKey() const
|
||||||
{
|
{
|
||||||
if (FPS_AI_ConvAgentModule::IsAvailable())
|
// Editor-only authoring key (Project Settings → Editor, or ELEVENLABS_API_KEY env).
|
||||||
{
|
// Never committed by a build, never packaged — see UPS_AI_ConvAgent_EditorSettings_ElevenLabs.
|
||||||
if (const UPS_AI_ConvAgent_Settings_ElevenLabs* Settings =
|
return UPS_AI_ConvAgent_EditorSettings_ElevenLabs::ResolveEditorApiKey();
|
||||||
FPS_AI_ConvAgentModule::Get().GetSettings())
|
|
||||||
{
|
|
||||||
return Settings->API_Key;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return FString();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
UPS_AI_ConvAgent_Tool_ElevenLabs* FPS_AI_ConvAgent_ToolCustomization_ElevenLabs::GetEditedAsset() const
|
UPS_AI_ConvAgent_Tool_ElevenLabs* FPS_AI_ConvAgent_ToolCustomization_ElevenLabs::GetEditedAsset() const
|
||||||
|
|||||||
@@ -10,7 +10,7 @@
|
|||||||
/**
|
/**
|
||||||
* AnimGraph editor node for the PS AI ConvAgent Body Expression AnimNode.
|
* AnimGraph editor node for the PS AI ConvAgent Body Expression AnimNode.
|
||||||
*
|
*
|
||||||
* This node appears in the AnimBP graph editor under the "PS AI ConvAgent" category.
|
* This node appears in the AnimBP graph editor under the "ASTERION|PS_AI_ConvAgent" category.
|
||||||
* Place it in the character's Body AnimBP after upstream body animations
|
* Place it in the character's Body AnimBP after upstream body animations
|
||||||
* (idle, locomotion). It blends emotion-driven body poses onto the upper body
|
* (idle, locomotion). It blends emotion-driven body poses onto the upper body
|
||||||
* while passing the lower body through unchanged.
|
* while passing the lower body through unchanged.
|
||||||
|
|||||||
@@ -10,7 +10,7 @@
|
|||||||
/**
|
/**
|
||||||
* AnimGraph editor node for the PS AI ConvAgent Facial Expression AnimNode.
|
* AnimGraph editor node for the PS AI ConvAgent Facial Expression AnimNode.
|
||||||
*
|
*
|
||||||
* This node appears in the AnimBP graph editor under the "PS AI ConvAgent" category.
|
* This node appears in the AnimBP graph editor under the "ASTERION|PS_AI_ConvAgent" category.
|
||||||
* Place it BEFORE the PS AI ConvAgent Lip Sync node in the MetaHuman Face AnimBP.
|
* Place it BEFORE the PS AI ConvAgent Lip Sync node in the MetaHuman Face AnimBP.
|
||||||
* It auto-discovers the PS_AI_ConvAgent_FacialExpressionComponent on the owning Actor
|
* It auto-discovers the PS_AI_ConvAgent_FacialExpressionComponent on the owning Actor
|
||||||
* and injects CTRL_expressions_* curves for emotion-driven facial expressions.
|
* and injects CTRL_expressions_* curves for emotion-driven facial expressions.
|
||||||
|
|||||||
@@ -10,7 +10,7 @@
|
|||||||
/**
|
/**
|
||||||
* AnimGraph editor node for the PS AI ConvAgent Gaze AnimNode.
|
* AnimGraph editor node for the PS AI ConvAgent Gaze AnimNode.
|
||||||
*
|
*
|
||||||
* This node appears in the AnimBP graph editor under the "PS AI ConvAgent" category.
|
* This node appears in the AnimBP graph editor under the "ASTERION|PS_AI_ConvAgent" category.
|
||||||
* Place it AFTER the PS AI ConvAgent Facial Expression node and BEFORE the
|
* Place it AFTER the PS AI ConvAgent Facial Expression node and BEFORE the
|
||||||
* PS AI ConvAgent Lip Sync node in the MetaHuman Face AnimBP.
|
* PS AI ConvAgent Lip Sync node in the MetaHuman Face AnimBP.
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -10,7 +10,7 @@
|
|||||||
/**
|
/**
|
||||||
* AnimGraph editor node for the PS AI ConvAgent Lip Sync AnimNode.
|
* AnimGraph editor node for the PS AI ConvAgent Lip Sync AnimNode.
|
||||||
*
|
*
|
||||||
* This node appears in the AnimBP graph editor under the "PS AI ConvAgent" category.
|
* This node appears in the AnimBP graph editor under the "ASTERION|PS_AI_ConvAgent" category.
|
||||||
* Drop it into your MetaHuman Face AnimBP between the pose source and the
|
* Drop it into your MetaHuman Face AnimBP between the pose source and the
|
||||||
* Control Rig node. It auto-discovers the PS_AI_ConvAgent_LipSyncComponent on the
|
* Control Rig node. It auto-discovers the PS_AI_ConvAgent_LipSyncComponent on the
|
||||||
* owning Actor and injects CTRL_expressions_* curves for lip sync.
|
* owning Actor and injects CTRL_expressions_* curves for lip sync.
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
{
|
||||||
|
"FileVersion": 3,
|
||||||
|
"Version": 1,
|
||||||
|
"VersionName": "1.0",
|
||||||
|
"FriendlyName": "PS AI ConvReplay",
|
||||||
|
"Description": "Bridge that records ElevenLabs agent + player conversation audio (and AI reactions) into Unreal replays, and plays it back during replay. Keeps PS_AI_ConvAgent and PS_ReplaySystem independent.",
|
||||||
|
"Category": "ASTERION",
|
||||||
|
"CreatedBy": "ASTERION",
|
||||||
|
"CanContainContent": false,
|
||||||
|
"IsBetaVersion": false,
|
||||||
|
"Installed": false,
|
||||||
|
"EnabledByDefault": false,
|
||||||
|
"Modules": [
|
||||||
|
{
|
||||||
|
"Name": "PS_AI_ConvReplay",
|
||||||
|
"Type": "Runtime",
|
||||||
|
"LoadingPhase": "Default"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"Plugins": [
|
||||||
|
{
|
||||||
|
"Name": "PS_AI_ConvAgent",
|
||||||
|
"Enabled": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Name": "ReplaySystem",
|
||||||
|
"Enabled": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
using UnrealBuildTool;
|
||||||
|
|
||||||
|
public class PS_AI_ConvReplay : ModuleRules
|
||||||
|
{
|
||||||
|
public PS_AI_ConvReplay(ReadOnlyTargetRules Target) : base(Target)
|
||||||
|
{
|
||||||
|
DefaultBuildSettings = BuildSettingsVersion.Latest;
|
||||||
|
PCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs;
|
||||||
|
|
||||||
|
PublicDependencyModuleNames.AddRange(new string[]
|
||||||
|
{
|
||||||
|
"Core",
|
||||||
|
"CoreUObject",
|
||||||
|
"Engine",
|
||||||
|
// Opus voice codec (own encoder/decoder instances — not shared with ConvAgent).
|
||||||
|
"Voice",
|
||||||
|
// The conversational-agent plugin whose delegates we tap and whose audio we replay.
|
||||||
|
"PS_AI_ConvAgent",
|
||||||
|
// Native replay wrapper (record/playback state, demo time, save path).
|
||||||
|
"ReplaySystem",
|
||||||
|
});
|
||||||
|
|
||||||
|
PrivateDependencyModuleNames.AddRange(new string[]
|
||||||
|
{
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#include "PS_AI_ConvReplay_AgentTag.h"
|
||||||
|
#include "PS_AI_ConvReplay_Subsystem.h"
|
||||||
|
#include "GameFramework/Actor.h"
|
||||||
|
|
||||||
|
UPS_AI_ConvReplay_AgentTag::UPS_AI_ConvReplay_AgentTag()
|
||||||
|
{
|
||||||
|
PrimaryComponentTick.bCanEverTick = false;
|
||||||
|
// Exists on server (recording) and on replay clients (identity lookup). It is not
|
||||||
|
// itself replicated — the owner agent already replicates; the tag only needs to be
|
||||||
|
// present on both ends with the same resolved id.
|
||||||
|
SetIsReplicatedByDefault(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
FString UPS_AI_ConvReplay_AgentTag::GetResolvedId() const
|
||||||
|
{
|
||||||
|
if (!AgentReplayId.IsNone())
|
||||||
|
{
|
||||||
|
return AgentReplayId.ToString();
|
||||||
|
}
|
||||||
|
return GetOwner() ? GetOwner()->GetName() : FString();
|
||||||
|
}
|
||||||
|
|
||||||
|
UPS_AI_ConvAgent_ElevenLabsComponent* UPS_AI_ConvReplay_AgentTag::GetAgentComponent() const
|
||||||
|
{
|
||||||
|
if (AActor* Owner = GetOwner())
|
||||||
|
{
|
||||||
|
return Owner->FindComponentByClass<UPS_AI_ConvAgent_ElevenLabsComponent>();
|
||||||
|
}
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_AgentTag::BeginRecording(UPS_AI_ConvReplay_Subsystem* InSubsystem)
|
||||||
|
{
|
||||||
|
Subsystem = InSubsystem;
|
||||||
|
Agent = GetAgentComponent();
|
||||||
|
if (!Agent || bBound)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Agent->OnAgentStartedSpeaking.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleStarted);
|
||||||
|
Agent->OnAgentStoppedSpeaking.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleStopped);
|
||||||
|
Agent->OnAgentActionRequested.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleActionRequested);
|
||||||
|
Agent->OnAgentClientToolCall.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleClientToolCall);
|
||||||
|
Agent->OnAgentTextResponse.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleTextResponse);
|
||||||
|
Agent->OnAgentTranscript.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleTranscript);
|
||||||
|
Agent->OnPlayerAudioCaptured.AddDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandlePlayerAudio);
|
||||||
|
|
||||||
|
// Raw PCM is a non-dynamic C++ multicast delegate.
|
||||||
|
AudioDataHandle = Agent->OnAgentAudioData.AddUObject(this, &UPS_AI_ConvReplay_AgentTag::HandleAgentAudioData);
|
||||||
|
|
||||||
|
bBound = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_AgentTag::EndRecording()
|
||||||
|
{
|
||||||
|
if (Agent && bBound)
|
||||||
|
{
|
||||||
|
Agent->OnAgentStartedSpeaking.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleStarted);
|
||||||
|
Agent->OnAgentStoppedSpeaking.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleStopped);
|
||||||
|
Agent->OnAgentActionRequested.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleActionRequested);
|
||||||
|
Agent->OnAgentClientToolCall.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleClientToolCall);
|
||||||
|
Agent->OnAgentTextResponse.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleTextResponse);
|
||||||
|
Agent->OnAgentTranscript.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandleTranscript);
|
||||||
|
Agent->OnPlayerAudioCaptured.RemoveDynamic(this, &UPS_AI_ConvReplay_AgentTag::HandlePlayerAudio);
|
||||||
|
Agent->OnAgentAudioData.Remove(AudioDataHandle);
|
||||||
|
}
|
||||||
|
AudioDataHandle.Reset();
|
||||||
|
bBound = false;
|
||||||
|
Subsystem = nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_AgentTag::HandleStarted()
|
||||||
|
{
|
||||||
|
if (Subsystem) Subsystem->RecordAgentStarted(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_AgentTag::HandleStopped()
|
||||||
|
{
|
||||||
|
if (Subsystem) Subsystem->RecordAgentStopped(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_AgentTag::HandleActionRequested(const FString& ActionName)
|
||||||
|
{
|
||||||
|
if (Subsystem) Subsystem->RecordAction(this, ActionName);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_AgentTag::HandleClientToolCall(const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall)
|
||||||
|
{
|
||||||
|
if (Subsystem) Subsystem->RecordToolCall(this, ToolCall);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_AgentTag::HandleTextResponse(const FString& Text)
|
||||||
|
{
|
||||||
|
if (Subsystem) Subsystem->RecordAgentText(this, Text);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_AgentTag::HandleTranscript(const FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs& Segment)
|
||||||
|
{
|
||||||
|
// Only final user transcripts become player-text markers.
|
||||||
|
if (Subsystem && Segment.Speaker == TEXT("user") && Segment.bIsFinal)
|
||||||
|
{
|
||||||
|
Subsystem->RecordUserTranscript(this, Segment.Text);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_AgentTag::HandlePlayerAudio(APawn* SpeakerPawn, const TArray<uint8>& PCM)
|
||||||
|
{
|
||||||
|
if (Subsystem) Subsystem->RecordPlayerAudio(SpeakerPawn, PCM);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UPS_AI_ConvReplay_AgentTag::HandleAgentAudioData(const TArray<uint8>& PCM)
|
||||||
|
{
|
||||||
|
if (Subsystem) Subsystem->RecordAgentAudio(this, PCM);
|
||||||
|
}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#include "Modules/ModuleManager.h"
|
||||||
|
|
||||||
|
IMPLEMENT_MODULE(FDefaultModuleImpl, PS_AI_ConvReplay);
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,71 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "CoreMinimal.h"
|
||||||
|
#include "Components/ActorComponent.h"
|
||||||
|
// Pulls in the ElevenLabs component + FPS_AI_ConvAgent_ClientToolCall_ElevenLabs /
|
||||||
|
// FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs (needed for the UFUNCTION handler signatures).
|
||||||
|
#include "PS_AI_ConvAgent_ElevenLabsComponent.h"
|
||||||
|
#include "PS_AI_ConvReplay_AgentTag.generated.h"
|
||||||
|
|
||||||
|
class UPS_AI_ConvReplay_Subsystem;
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// Per-agent bridge component.
|
||||||
|
//
|
||||||
|
// Add one to each conversational agent to give it a stable, replay-safe identity
|
||||||
|
// (AgentReplayId). During recording it binds the sibling ElevenLabs component's
|
||||||
|
// delegates and forwards every conversation event to the subsystem, tagged with
|
||||||
|
// this agent's identity. During replay the subsystem looks the agent up by id to
|
||||||
|
// route recorded audio / reactions back to it.
|
||||||
|
//
|
||||||
|
// If you don't add one, the subsystem auto-provisions a tag on every agent at
|
||||||
|
// record/replay start, using the owner actor name as the id (fine for
|
||||||
|
// placed-in-level agents; add an explicit tag for runtime-spawned ones).
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
UCLASS(ClassGroup = "ASTERION|PS_AI_ConvReplay", meta = (BlueprintSpawnableComponent),
|
||||||
|
DisplayName = "PS AI ConvReplay - Agent Tag")
|
||||||
|
class PS_AI_CONVREPLAY_API UPS_AI_ConvReplay_AgentTag : public UActorComponent
|
||||||
|
{
|
||||||
|
GENERATED_BODY()
|
||||||
|
|
||||||
|
public:
|
||||||
|
UPS_AI_ConvReplay_AgentTag();
|
||||||
|
|
||||||
|
/** Stable id used to match recorded audio/reactions back to this agent on replay.
|
||||||
|
* Leave as None to fall back to the owner actor name (stable for placed agents). */
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvReplay")
|
||||||
|
FName AgentReplayId;
|
||||||
|
|
||||||
|
/** Resolved identity string (AgentReplayId if set, else the owner actor name). */
|
||||||
|
FString GetResolvedId() const;
|
||||||
|
|
||||||
|
/** The sibling ElevenLabs component on the owner actor (may be null). */
|
||||||
|
UPS_AI_ConvAgent_ElevenLabsComponent* GetAgentComponent() const;
|
||||||
|
|
||||||
|
/** Bind the sibling component's delegates and start forwarding events. Called by the subsystem. */
|
||||||
|
void BeginRecording(UPS_AI_ConvReplay_Subsystem* InSubsystem);
|
||||||
|
|
||||||
|
/** Unbind delegates and stop forwarding. Called by the subsystem. */
|
||||||
|
void EndRecording();
|
||||||
|
|
||||||
|
private:
|
||||||
|
UPROPERTY(Transient) TObjectPtr<UPS_AI_ConvReplay_Subsystem> Subsystem;
|
||||||
|
UPROPERTY(Transient) TObjectPtr<UPS_AI_ConvAgent_ElevenLabsComponent> Agent;
|
||||||
|
|
||||||
|
FDelegateHandle AudioDataHandle;
|
||||||
|
bool bBound = false;
|
||||||
|
|
||||||
|
// Dynamic-delegate handlers (must be UFUNCTION and match the delegate signatures).
|
||||||
|
UFUNCTION() void HandleStarted();
|
||||||
|
UFUNCTION() void HandleStopped();
|
||||||
|
UFUNCTION() void HandleActionRequested(const FString& ActionName);
|
||||||
|
UFUNCTION() void HandleClientToolCall(const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall);
|
||||||
|
UFUNCTION() void HandleTextResponse(const FString& Text);
|
||||||
|
UFUNCTION() void HandleTranscript(const FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs& Segment);
|
||||||
|
UFUNCTION() void HandlePlayerAudio(APawn* SpeakerPawn, const TArray<uint8>& PCM);
|
||||||
|
|
||||||
|
// Non-dynamic (raw C++ multicast) handler for agent PCM.
|
||||||
|
void HandleAgentAudioData(const TArray<uint8>& PCM);
|
||||||
|
};
|
||||||
@@ -0,0 +1,173 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "CoreMinimal.h"
|
||||||
|
#include "Subsystems/WorldSubsystem.h"
|
||||||
|
#include "PS_AI_ConvReplay_Types.h"
|
||||||
|
#include "PS_AI_ConvReplay_Subsystem.generated.h"
|
||||||
|
|
||||||
|
class UPS_AI_ConvReplay_AgentTag;
|
||||||
|
class UPS_AI_ConvAgent_ElevenLabsComponent;
|
||||||
|
class UPS_AI_ConvAgent_InteractionComponent;
|
||||||
|
class UAudioComponent;
|
||||||
|
class APawn;
|
||||||
|
class IVoiceEncoder;
|
||||||
|
class IVoiceDecoder;
|
||||||
|
struct FPS_AI_ConvAgent_ClientToolCall_ElevenLabs;
|
||||||
|
|
||||||
|
/** Fired on the replay client when a recorded line's timestamp is reached.
|
||||||
|
* Use it to drive subtitles / a conversation log. Audio playback is automatic. */
|
||||||
|
DECLARE_DYNAMIC_MULTICAST_DELEGATE_FourParams(FPS_AI_ConvReplay_OnReplayLinePlayed,
|
||||||
|
FName, AgentId, const FString&, Speaker, const FString&, Text, float, TimeSec);
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// World subsystem that records the conversation audio track while a replay is
|
||||||
|
// being recorded, and plays it back while a replay is being played.
|
||||||
|
//
|
||||||
|
// • Record: taps every agent's ElevenLabs delegates (via AgentTag), accumulates
|
||||||
|
// PCM per speaker, Opus-encodes each utterance and appends it to an external
|
||||||
|
// sidecar file (<ReplayName>.convreplay) next to the .replay — nothing heavy
|
||||||
|
// goes into the demo stream. Also records AI reactions (perform_action / custom
|
||||||
|
// tools) and player transcripts.
|
||||||
|
// • Replay: loads the sidecar, and each tick fires the records whose timestamp is
|
||||||
|
// reached — agent audio through the agent's own voice (drives lip-sync + body
|
||||||
|
// expression), player audio spatialized at the recorded location, reactions
|
||||||
|
// re-broadcast. Gaze / emotion / facial expression replay for free (replicated).
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
UCLASS()
|
||||||
|
class PS_AI_CONVREPLAY_API UPS_AI_ConvReplay_Subsystem : public UTickableWorldSubsystem
|
||||||
|
{
|
||||||
|
GENERATED_BODY()
|
||||||
|
|
||||||
|
public:
|
||||||
|
// USubsystem / UWorldSubsystem
|
||||||
|
virtual void Initialize(FSubsystemCollectionBase& Collection) override;
|
||||||
|
virtual void Deinitialize() override;
|
||||||
|
virtual bool DoesSupportWorldType(const EWorldType::Type WorldType) const override;
|
||||||
|
|
||||||
|
// FTickableGameObject
|
||||||
|
virtual void Tick(float DeltaTime) override;
|
||||||
|
virtual TStatId GetStatId() const override;
|
||||||
|
|
||||||
|
/** Fired on the replay client for each recorded line as its timestamp is reached. */
|
||||||
|
UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvReplay")
|
||||||
|
FPS_AI_ConvReplay_OnReplayLinePlayed OnReplayLinePlayed;
|
||||||
|
|
||||||
|
// ── Record callbacks (invoked by AgentTag on the recording machine) ──────
|
||||||
|
void RecordAgentStarted(UPS_AI_ConvReplay_AgentTag* Tag);
|
||||||
|
void RecordAgentAudio(UPS_AI_ConvReplay_AgentTag* Tag, const TArray<uint8>& PCM);
|
||||||
|
void RecordAgentText(UPS_AI_ConvReplay_AgentTag* Tag, const FString& Text);
|
||||||
|
void RecordAgentStopped(UPS_AI_ConvReplay_AgentTag* Tag);
|
||||||
|
void RecordAction(UPS_AI_ConvReplay_AgentTag* Tag, const FString& ActionName);
|
||||||
|
void RecordToolCall(UPS_AI_ConvReplay_AgentTag* Tag, const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs& ToolCall);
|
||||||
|
void RecordUserTranscript(UPS_AI_ConvReplay_AgentTag* Tag, const FString& Text);
|
||||||
|
void RecordPlayerAudio(APawn* SpeakerPawn, const TArray<uint8>& PCM);
|
||||||
|
|
||||||
|
private:
|
||||||
|
// Session transitions.
|
||||||
|
void BeginRecordingSession();
|
||||||
|
void EndRecordingSession();
|
||||||
|
void BeginReplaySession();
|
||||||
|
void EndReplaySession();
|
||||||
|
|
||||||
|
void TickRecording();
|
||||||
|
void TickReplay();
|
||||||
|
|
||||||
|
// Agent-less player-voice capture (InteractionComponents on player pawns).
|
||||||
|
void RefreshRecordingMics();
|
||||||
|
UFUNCTION()
|
||||||
|
void HandlePlayerVoiceForRecording(APawn* SpeakerPawn, const TArray<uint8>& PCM);
|
||||||
|
|
||||||
|
// Agent discovery / lookup.
|
||||||
|
void RebuildTagList();
|
||||||
|
UPS_AI_ConvReplay_AgentTag* FindTagById(const FString& Id);
|
||||||
|
// Robust match for a recorded agent record: exact id first (placed agents /
|
||||||
|
// explicit AgentReplayId), else the nearest agent to the recorded position
|
||||||
|
// (handles runtime-spawned agents whose actor names differ on replay).
|
||||||
|
UPS_AI_ConvReplay_AgentTag* FindAgentForRecord(const FString& Id, const FVector& Pos);
|
||||||
|
|
||||||
|
// Sidecar IO.
|
||||||
|
FString GetSidecarPath() const;
|
||||||
|
void WriteRecord(FPS_AI_ConvReplay_Record& Rec);
|
||||||
|
void LoadRecords();
|
||||||
|
|
||||||
|
// Opus (own instances — not shared with ConvAgent's live encoder).
|
||||||
|
void EnsureCodec();
|
||||||
|
bool EncodeUtterance(const TArray<uint8>& PCM, TArray<uint8>& OutPackets);
|
||||||
|
bool DecodeUtterance(const TArray<uint8>& Packets, TArray<uint8>& OutPCM);
|
||||||
|
|
||||||
|
// Replay dispatch.
|
||||||
|
void DispatchDueRecords(double NowSec, bool bAudioAllowed);
|
||||||
|
void ResetPlaybackFrom(double NowSec);
|
||||||
|
void PlayPlayerAudioAtLocation(const FVector& Loc, const TArray<uint8>& PCM);
|
||||||
|
|
||||||
|
// Helpers.
|
||||||
|
double NowDemoSeconds() const;
|
||||||
|
void FlushAgentSegment(UPS_AI_ConvReplay_AgentTag* Tag); // write + reset an agent utterance
|
||||||
|
void FlushPlayerSegment(APawn* Pawn);
|
||||||
|
static bool IsVoiced(const TArray<uint8>& PCM16);
|
||||||
|
static FString ResolvePlayerId(APawn* Pawn);
|
||||||
|
// Boost quiet mic capture: normalize a replayed player utterance toward a target
|
||||||
|
// RMS (peak-safe), then apply the ps.convreplay.PlayerGain CVar. In-place on int16 PCM.
|
||||||
|
static void NormalizePlayerPcm(TArray<uint8>& Pcm16);
|
||||||
|
|
||||||
|
// ── State ────────────────────────────────────────────────────────────────
|
||||||
|
bool bRecording = false;
|
||||||
|
bool bPlaying = false;
|
||||||
|
double LastReplayTime = -1.0;
|
||||||
|
|
||||||
|
IFileHandle* WriteHandle = nullptr;
|
||||||
|
|
||||||
|
TSharedPtr<IVoiceEncoder> Encoder;
|
||||||
|
TSharedPtr<IVoiceDecoder> Decoder;
|
||||||
|
|
||||||
|
UPROPERTY(Transient)
|
||||||
|
TArray<TObjectPtr<UPS_AI_ConvReplay_AgentTag>> Tags;
|
||||||
|
|
||||||
|
// Player InteractionComponents currently armed for agent-less voice recording.
|
||||||
|
UPROPERTY(Transient)
|
||||||
|
TArray<TObjectPtr<UPS_AI_ConvAgent_InteractionComponent>> RecordingMicComps;
|
||||||
|
|
||||||
|
// Per-agent recording accumulator.
|
||||||
|
struct FAgentAccum
|
||||||
|
{
|
||||||
|
double StartTime = 0.0;
|
||||||
|
double LastAudioTime = 0.0; // demo time of the last appended chunk (gap detection)
|
||||||
|
FVector Pos = FVector::ZeroVector;
|
||||||
|
FString Text;
|
||||||
|
TArray<uint8> PCM;
|
||||||
|
bool bOpen = false;
|
||||||
|
};
|
||||||
|
TMap<TWeakObjectPtr<UPS_AI_ConvReplay_AgentTag>, FAgentAccum> AgentAccum;
|
||||||
|
|
||||||
|
// Per-player recording accumulator (segmented by silence).
|
||||||
|
struct FPlayerAccum
|
||||||
|
{
|
||||||
|
double StartTime = 0.0;
|
||||||
|
double LastVoiceTime = 0.0;
|
||||||
|
FVector Pos = FVector::ZeroVector;
|
||||||
|
FString Id;
|
||||||
|
TArray<uint8> PCM;
|
||||||
|
bool bActive = false;
|
||||||
|
};
|
||||||
|
TMap<TWeakObjectPtr<APawn>, FPlayerAccum> PlayerAccum;
|
||||||
|
|
||||||
|
// Loaded conversation track (replay).
|
||||||
|
TArray<FPS_AI_ConvReplay_Record> Records;
|
||||||
|
|
||||||
|
// Transient one-shot components used to play back player voices during replay.
|
||||||
|
UPROPERTY(Transient)
|
||||||
|
TArray<TObjectPtr<UAudioComponent>> PlayerAudioPool;
|
||||||
|
|
||||||
|
// Tunables.
|
||||||
|
static constexpr float AgentGapSplitSeconds = 1.5f; // split an agent utterance if audio resumes after this gap (greeting vs reply)
|
||||||
|
static constexpr float PlayerSilenceFlushSec = 0.6f;
|
||||||
|
static constexpr float VoicedRmsThreshold = 0.02f;
|
||||||
|
static constexpr float ScrubBackThreshold = 0.05f; // seconds jumped back => scrub
|
||||||
|
static constexpr float ScrubFwdThreshold = 1.5f; // seconds jumped forward => scrub / fast-seek
|
||||||
|
static constexpr float NormalSpeedTolerance = 0.35f; // only play audio within this of 1x
|
||||||
|
// Target loudness (RMS, −1..1) is the live CVar ps.convreplay.PlayerTargetRms (default 0.14).
|
||||||
|
static constexpr float PlayerMinGain = 0.1f; // allow attenuating LOUD utterances (bidirectional AGC)
|
||||||
|
static constexpr float PlayerMaxGain = 16.0f; // cap so near-silence isn't blown up
|
||||||
|
};
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
// Copyright ASTERION. All Rights Reserved.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "CoreMinimal.h"
|
||||||
|
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
// One entry in the conversation sidecar track (<ReplayName>.convreplay).
|
||||||
|
//
|
||||||
|
// The sidecar is an append-only sequence of these records, each serialized
|
||||||
|
// back-to-back with FMemoryWriter (self-delimiting via FString/TArray length
|
||||||
|
// prefixes) so it can be streamed while recording and read in one pass at replay.
|
||||||
|
// Audio is stored as a sequence of length-prefixed Opus packets (see the
|
||||||
|
// subsystem's EncodeUtterance/DecodeUtterance), NOT inside the demo metadata.
|
||||||
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
enum class EPS_AI_ConvReplay_Kind : uint8
|
||||||
|
{
|
||||||
|
AgentSpeech = 0, // an agent utterance: Audio + Text, played through the agent's voice
|
||||||
|
PlayerSpeech = 1, // a player utterance: Audio, played spatialized at Position
|
||||||
|
Action = 2, // a perform_action reaction: Text = action name
|
||||||
|
ToolCall = 3, // a custom client tool call: Text = tool name, Params = parameters
|
||||||
|
PlayerText = 4, // a player transcript marker (subtitle only, no audio)
|
||||||
|
};
|
||||||
|
|
||||||
|
struct FPS_AI_ConvReplay_Record
|
||||||
|
{
|
||||||
|
/** EPS_AI_ConvReplay_Kind. */
|
||||||
|
uint8 Kind = 0;
|
||||||
|
|
||||||
|
/** Start time in demo seconds (captured via GetCurrentReplayTime at record time). */
|
||||||
|
double TimeSec = 0.0;
|
||||||
|
|
||||||
|
/** Audio duration in seconds (0 for non-audio records). */
|
||||||
|
double DurationSec = 0.0;
|
||||||
|
|
||||||
|
/** Agent id (resolved from AgentTag / actor name) or player id. */
|
||||||
|
FString Id;
|
||||||
|
|
||||||
|
/** 0 = agent, 1 = user. */
|
||||||
|
uint8 Speaker = 0;
|
||||||
|
|
||||||
|
/** Transcript, or action name, or tool name depending on Kind. */
|
||||||
|
FString Text;
|
||||||
|
|
||||||
|
/** World location of the speaker at utterance start (used to spatialize player audio). */
|
||||||
|
FVector Position = FVector::ZeroVector;
|
||||||
|
|
||||||
|
/** Tool-call parameters (empty otherwise). */
|
||||||
|
TMap<FString, FString> Params;
|
||||||
|
|
||||||
|
/** Sequence of length-prefixed Opus packets (empty for non-audio records). */
|
||||||
|
TArray<uint8> Audio;
|
||||||
|
|
||||||
|
/** Runtime-only: set once this record has been dispatched during playback. NOT serialized. */
|
||||||
|
bool bPlayed = false;
|
||||||
|
|
||||||
|
friend FArchive& operator<<(FArchive& Ar, FPS_AI_ConvReplay_Record& R)
|
||||||
|
{
|
||||||
|
Ar << R.Kind;
|
||||||
|
Ar << R.TimeSec;
|
||||||
|
Ar << R.DurationSec;
|
||||||
|
Ar << R.Id;
|
||||||
|
Ar << R.Speaker;
|
||||||
|
Ar << R.Text;
|
||||||
|
Ar << R.Position;
|
||||||
|
Ar << R.Params;
|
||||||
|
Ar << R.Audio;
|
||||||
|
return Ar;
|
||||||
|
}
|
||||||
|
};
|
||||||
Reference in New Issue
Block a user