From d035f5410a27f86373e5ac96564588830ddef34e Mon Sep 17 00:00:00 2001 From: "j.foucher" Date: Tue, 3 Mar 2026 12:50:56 +0100 Subject: [PATCH] Add bPersistentSession: keep WebSocket alive across conversation cycles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When true (default), the first StartConversation() opens the WebSocket and EndConversation() only stops the mic/posture — the WebSocket stays open until EndPlay. The agent remembers the full conversation context. When false, each Start/EndConversation opens/closes the WebSocket (previous behavior). Co-Authored-By: Claude Opus 4.6 --- .../PS_AI_ConvAgent_ElevenLabsComponent.cpp | 97 ++++++++++++++++--- .../PS_AI_ConvAgent_ElevenLabsComponent.h | 10 ++ 2 files changed, 91 insertions(+), 16 deletions(-) diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp index 49c844f..78609ee 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp @@ -51,6 +51,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::BeginPlay() Sub->RegisterAgent(this); } } + } void UPS_AI_ConvAgent_ElevenLabsComponent::EndPlay(const EEndPlayReason::Type EndPlayReason) @@ -65,6 +66,17 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::EndPlay(const EEndPlayReason::Type En } EndConversation(); + + // In persistent mode EndConversation() deliberately keeps the WebSocket open. + // EndPlay is the final teardown — force-close it now. + if (bPersistentSession && WebSocketProxy) + { + bWantsReconnect = false; + bIntentionalDisconnect = true; + WebSocketProxy->Disconnect(); + WebSocketProxy = nullptr; + } + Super::EndPlay(EndPlayReason); } @@ -257,7 +269,25 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StartConversation() NetConversatingPlayer = PC; NetConversatingPawn = PC ? PC->GetPawn() : nullptr; } - StartConversation_Internal(); + + // In persistent mode the WebSocket was already opened by the first StartConversation. + // Reuse the existing connection — only set up conversation state. + if (bPersistentSession && IsConnected()) + { + // WebSocket already alive — just set up conversation state (posture, etc.). + ApplyConversationPosture(); + OnAgentConnected.Broadcast(WebSocketProxy->GetConversationInfo()); + + // Auto-start listening if configured (same as HandleConnected). + if (bAutoStartListening && TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server) + { + StartListening(); + } + } + else + { + StartConversation_Internal(); + } } else { @@ -319,9 +349,12 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::EndConversation() { if (GetOwnerRole() == ROLE_Authority) { - // Cancel any pending reconnection. - bWantsReconnect = false; - ReconnectAttemptCount = 0; + // Cancel any pending reconnection (ephemeral mode only — persistent keeps reconnecting). + if (!bPersistentSession) + { + bWantsReconnect = false; + ReconnectAttemptCount = 0; + } StopListening(); // ISSUE-4: StopListening() may set bWaitingForAgentResponse=true (normal turn end path). @@ -330,11 +363,15 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::EndConversation() bWaitingForAgentResponse = false; StopAgentAudio(); - if (WebSocketProxy) + // In persistent mode, keep the WebSocket open — only manage conversation state. + if (!bPersistentSession) { - bIntentionalDisconnect = true; - WebSocketProxy->Disconnect(); - WebSocketProxy = nullptr; + if (WebSocketProxy) + { + bIntentionalDisconnect = true; + WebSocketProxy->Disconnect(); + WebSocketProxy = nullptr; + } } // Reset replicated state so other players can talk to this NPC. @@ -1633,24 +1670,52 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ServerRequestConversation_Implementat // Update NPC posture on the server (OnRep never fires on Authority). ApplyConversationPosture(); - StartConversation_Internal(); + // In persistent mode the WebSocket is already open — skip reconnection. + if (bPersistentSession && IsConnected()) + { + // Notify the requesting client that conversation started (normally done in HandleConnected). + if (NetConversatingPawn) + { + if (auto* Relay = NetConversatingPawn->FindComponentByClass()) + { + Relay->ClientRelayConversationStarted(GetOwner(), WebSocketProxy->GetConversationInfo()); + } + } + + // Auto-start listening if configured. + if (bAutoStartListening && TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server) + { + StartListening(); + } + } + else + { + StartConversation_Internal(); + } } void UPS_AI_ConvAgent_ElevenLabsComponent::ServerReleaseConversation_Implementation() { - // Cancel any pending reconnection. - bWantsReconnect = false; - ReconnectAttemptCount = 0; + // Cancel any pending reconnection (ephemeral mode only). + if (!bPersistentSession) + { + bWantsReconnect = false; + ReconnectAttemptCount = 0; + } StopListening(); bWaitingForAgentResponse = false; StopAgentAudio(); - if (WebSocketProxy) + // In persistent mode, keep the WebSocket open. + if (!bPersistentSession) { - bIntentionalDisconnect = true; - WebSocketProxy->Disconnect(); - WebSocketProxy = nullptr; + if (WebSocketProxy) + { + bIntentionalDisconnect = true; + WebSocketProxy->Disconnect(); + WebSocketProxy = nullptr; + } } // Clear posture before nullifying the pawn pointer (ApplyConversationPosture diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h index a945e8d..d2817b3 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h @@ -129,6 +129,16 @@ public: meta = (ToolTip = "Turn-taking mode.\n- Server VAD: ElevenLabs detects end-of-speech automatically (hands-free).\n- Client Controlled: You call StartListening/StopListening manually (push-to-talk).")) EPS_AI_ConvAgent_TurnMode_ElevenLabs TurnMode = EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server; + /** Keep the WebSocket open across multiple StartConversation / EndConversation cycles. + * When true, the first StartConversation opens the WebSocket and EndConversation only + * stops the microphone and resets posture — the WebSocket stays alive until EndPlay. + * The agent remembers the full conversation context between interactions. + * When false (ephemeral), each StartConversation opens a fresh WebSocket session + * and EndConversation closes it. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs", + meta = (ToolTip = "Keep the WebSocket open across conversation cycles.\n- True: agent remembers the full conversation between interactions.\n- False: each StartConversation opens a new session.")) + bool bPersistentSession = true; + /** Automatically open the microphone as soon as the WebSocket connection is established. Only applies in Server VAD mode. In Client (push-to-talk) mode, you must call StartListening manually. */ UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|ElevenLabs", meta = (ToolTip = "Auto-open the microphone when the conversation starts.\nOnly applies in Server VAD mode. In push-to-talk mode, call StartListening() manually."))