Add relay RPC pattern for client→NPC communication over network

UE5 clients cannot call Server RPCs on actors they don't own. NPC actors
are server-owned, causing "No owning connection" errors when remote clients
try to start conversations, send mic audio, or interrupt agents.

Solution: relay all client→NPC RPCs through the InteractionComponent on
the player's pawn (which IS owned by the client). The relay forwards
commands to the NPC's ElevenLabsComponent on the server side.

Changes:
- InteractionComponent: add Server relay RPCs (Start/End conversation,
  mic audio, text message, interrupt) and Client relay RPCs
  (ConversationStarted/Failed) with GetLifetimeReplicatedProps
- ElevenLabsComponent: implement FindLocalRelayComponent(), route all
  client-side calls through relay (StartConversation, EndConversation,
  SendTextMessage, InterruptAgent, FeedExternalAudio, mic capture)
- Fix HandleConnected/ServerRequestConversation to route Client RPCs
  through the player pawn's relay instead of the NPC (no owning connection)
- Fix StartListening/FeedExternalAudio/StopListening to accept
  bNetIsConversing on clients (WebSocket only exists on server)
- Fix EvaluateBestAgent to use NetConversatingPawn instead of
  NetConversatingPlayer (NULL on remote clients due to bOnlyRelevantToOwner)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
j.foucher 2026-03-02 16:40:27 +01:00
parent ce84a5dc58
commit 8a3304fea9
4 changed files with 306 additions and 27 deletions

View File

@ -7,6 +7,7 @@
#include "PS_AI_ConvAgent_FacialExpressionComponent.h"
#include "PS_AI_ConvAgent_LipSyncComponent.h"
#include "PS_AI_ConvAgent_InteractionSubsystem.h"
#include "PS_AI_ConvAgent_InteractionComponent.h"
#include "PS_AI_ConvAgent.h"
#include "Components/AudioComponent.h"
@ -231,11 +232,17 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StartConversation()
}
else
{
// Client: request conversation via Server RPC.
APlayerController* PC = GetWorld() ? GetWorld()->GetFirstPlayerController() : nullptr;
if (PC)
// Client: route through InteractionComponent relay (clients can't call
// Server RPCs on NPC actors — no owning connection).
if (auto* Relay = FindLocalRelayComponent())
{
ServerRequestConversation(PC);
Relay->ServerRelayStartConversation(GetOwner());
}
else
{
// Fallback: try direct RPC (will fail with "No owning connection" warning).
APlayerController* PC = GetWorld() ? GetWorld()->GetFirstPlayerController() : nullptr;
if (PC) ServerRequestConversation(PC);
}
}
}
@ -303,16 +310,32 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::EndConversation()
}
else
{
// Client: request release via Server RPC.
ServerReleaseConversation();
// Client: route through InteractionComponent relay (clients can't call
// Server RPCs on NPC actors — no owning connection).
if (auto* Relay = FindLocalRelayComponent())
{
Relay->ServerRelayEndConversation(GetOwner());
}
else
{
// Fallback: try direct RPC (will fail with "No owning connection" warning).
ServerReleaseConversation();
}
}
}
void UPS_AI_ConvAgent_ElevenLabsComponent::StartListening()
{
if (!IsConnected())
// On the server, the local WebSocket must be connected.
// On clients, there is no local WebSocket — use the replicated conversation state
// (the actual WebSocket lives on the server and audio is routed through relay RPCs).
const bool bEffectivelyConnected = IsConnected() || (GetOwnerRole() != ROLE_Authority && bNetIsConversing);
if (!bEffectivelyConnected)
{
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning, TEXT("StartListening: not connected."));
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning, TEXT("StartListening: not connected (IsConnected=%s bNetIsConversing=%s Role=%d)."),
IsConnected() ? TEXT("true") : TEXT("false"),
bNetIsConversing ? TEXT("true") : TEXT("false"),
static_cast<int32>(GetOwnerRole()));
return;
}
@ -352,7 +375,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StartListening()
bIsListening = true;
TurnStartTime = FPlatformTime::Seconds();
if (TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Client)
if (TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Client && WebSocketProxy)
{
WebSocketProxy->SendUserTurnStart();
}
@ -438,9 +461,23 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StopListening()
MicAccumulationBuffer.Num());
}
}
else if (MicAccumulationBuffer.Num() > 0 && WebSocketProxy && IsConnected())
else if (MicAccumulationBuffer.Num() > 0)
{
WebSocketProxy->SendAudioChunk(MicAccumulationBuffer);
if (GetOwnerRole() == ROLE_Authority)
{
if (WebSocketProxy && IsConnected())
{
WebSocketProxy->SendAudioChunk(MicAccumulationBuffer);
}
}
else
{
// Client: flush remaining mic audio through relay.
if (auto* Relay = FindLocalRelayComponent())
{
Relay->ServerRelayMicAudio(GetOwner(), MicAccumulationBuffer);
}
}
}
MicAccumulationBuffer.Reset();
}
@ -483,12 +520,27 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StopListening()
void UPS_AI_ConvAgent_ElevenLabsComponent::SendTextMessage(const FString& Text)
{
if (!IsConnected())
if (GetOwnerRole() == ROLE_Authority)
{
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning, TEXT("SendTextMessage: not connected. Call StartConversation() first."));
return;
if (!IsConnected())
{
UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning, TEXT("SendTextMessage: not connected. Call StartConversation() first."));
return;
}
WebSocketProxy->SendTextMessage(Text);
}
else
{
// Client: route through InteractionComponent relay (WebSocket is on server).
if (auto* Relay = FindLocalRelayComponent())
{
Relay->ServerRelaySendText(GetOwner(), Text);
}
else
{
ServerSendTextMessage(Text);
}
}
WebSocketProxy->SendTextMessage(Text);
// Enable body tracking on the sibling PostureComponent (if present).
// Text input counts as conversation engagement, same as voice.
@ -505,7 +557,24 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::SendTextMessage(const FString& Text)
void UPS_AI_ConvAgent_ElevenLabsComponent::InterruptAgent()
{
bWaitingForAgentResponse = false; // Interrupting — no response expected from previous turn.
if (WebSocketProxy) WebSocketProxy->SendInterrupt();
if (GetOwnerRole() == ROLE_Authority)
{
if (WebSocketProxy) WebSocketProxy->SendInterrupt();
}
else
{
// Client: route through InteractionComponent relay (WebSocket is on server).
if (auto* Relay = FindLocalRelayComponent())
{
Relay->ServerRelayInterrupt(GetOwner());
}
else
{
ServerRequestInterrupt();
}
}
StopAgentAudio();
}
@ -513,7 +582,9 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::FeedExternalAudio(const TArray<float>
{
// Same logic as OnMicrophoneDataCaptured but called from an external source
// (e.g. InteractionComponent on the pawn) instead of a local mic component.
if (!IsConnected() || !bIsListening) return;
// On server: check local WebSocket. On client: check replicated conversation state.
const bool bCanSend = (GetOwnerRole() == ROLE_Authority) ? IsConnected() : bNetIsConversing;
if (!bCanSend || !bIsListening) return;
// Echo suppression: skip sending mic audio while the agent is speaking.
if (bAgentSpeaking && !(TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server && bAllowInterruption))
@ -534,7 +605,15 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::FeedExternalAudio(const TArray<float>
}
else
{
ServerSendMicAudio(MicAccumulationBuffer);
// Route through relay (clients can't call Server RPCs on NPC actors).
if (auto* Relay = FindLocalRelayComponent())
{
Relay->ServerRelayMicAudio(GetOwner(), MicAccumulationBuffer);
}
else
{
ServerSendMicAudio(MicAccumulationBuffer);
}
}
MicAccumulationBuffer.Reset();
}
@ -566,9 +645,14 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleConnected(const FPS_AI_ConvAgen
OnAgentConnected.Broadcast(Info);
// Network: notify the requesting remote client that conversation started.
if (GetOwnerRole() == ROLE_Authority && NetConversatingPlayer)
// Client RPCs on NPC actors have no owning connection — route through the
// player pawn's InteractionComponent which IS owned by the client.
if (GetOwnerRole() == ROLE_Authority && NetConversatingPawn)
{
ClientConversationStarted(Info);
if (auto* Relay = NetConversatingPawn->FindComponentByClass<UPS_AI_ConvAgent_InteractionComponent>())
{
Relay->ClientRelayConversationStarted(GetOwner(), Info);
}
}
// In Client turn mode (push-to-talk), the user controls listening manually via
@ -1139,7 +1223,9 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StopAgentAudio()
// ─────────────────────────────────────────────────────────────────────────────
void UPS_AI_ConvAgent_ElevenLabsComponent::OnMicrophoneDataCaptured(const TArray<float>& FloatPCM)
{
if (!IsConnected() || !bIsListening) return;
// On server: check local WebSocket. On client: check replicated conversation state.
const bool bCanSend = (GetOwnerRole() == ROLE_Authority) ? IsConnected() : bNetIsConversing;
if (!bCanSend || !bIsListening) return;
// Echo suppression: skip sending mic audio while the agent is speaking.
// This prevents the agent from hearing its own voice through the speakers,
@ -1170,7 +1256,15 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::OnMicrophoneDataCaptured(const TArray
}
else
{
ServerSendMicAudio(MicAccumulationBuffer);
// Route through relay (clients can't call Server RPCs on NPC actors).
if (auto* Relay = FindLocalRelayComponent())
{
Relay->ServerRelayMicAudio(GetOwner(), MicAccumulationBuffer);
}
else
{
ServerSendMicAudio(MicAccumulationBuffer);
}
}
MicAccumulationBuffer.Reset();
}
@ -1296,7 +1390,18 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ServerRequestConversation_Implementat
{
if (bNetIsConversing)
{
ClientConversationFailed(TEXT("NPC is already in conversation with another player."));
// Route failure notification through the player's InteractionComponent relay.
// Client RPCs on NPC actors have no owning connection.
if (RequestingPlayer)
{
if (APawn* Pawn = RequestingPlayer->GetPawn())
{
if (auto* Relay = Pawn->FindComponentByClass<UPS_AI_ConvAgent_InteractionComponent>())
{
Relay->ClientRelayConversationFailed(TEXT("NPC is already in conversation with another player."));
}
}
}
return;
}
@ -1538,3 +1643,18 @@ bool UPS_AI_ConvAgent_ElevenLabsComponent::IsLocalPlayerConversating() const
}
return false;
}
UPS_AI_ConvAgent_InteractionComponent* UPS_AI_ConvAgent_ElevenLabsComponent::FindLocalRelayComponent() const
{
if (UWorld* World = GetWorld())
{
if (APlayerController* PC = World->GetFirstPlayerController())
{
if (APawn* Pawn = PC->GetPawn())
{
return Pawn->FindComponentByClass<UPS_AI_ConvAgent_InteractionComponent>();
}
}
}
return nullptr;
}

View File

@ -10,6 +10,7 @@
#include "GameFramework/PlayerController.h"
#include "Camera/PlayerCameraManager.h"
#include "TimerManager.h"
#include "Net/UnrealNetwork.h"
DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_Select, Log, All);
@ -20,6 +21,7 @@ UPS_AI_ConvAgent_InteractionComponent::UPS_AI_ConvAgent_InteractionComponent()
{
PrimaryComponentTick.bCanEverTick = true;
PrimaryComponentTick.TickInterval = 1.0f / 30.0f; // 30 Hz is enough for selection evaluation.
SetIsReplicatedByDefault(true); // Required for Server/Client RPCs on this component.
}
// ─────────────────────────────────────────────────────────────────────────────
@ -127,8 +129,11 @@ UPS_AI_ConvAgent_ElevenLabsComponent* UPS_AI_ConvAgent_InteractionComponent::Eva
UPS_AI_ConvAgent_ElevenLabsComponent* CurrentAgent = SelectedAgent.Get();
// Get local player controller for occupied-NPC check.
// Get local player's pawn for occupied-NPC check.
// Use pawn (replicated to ALL clients) instead of PlayerController
// (only replicated to owning client due to bOnlyRelevantToOwner=true).
APlayerController* LocalPC = World->GetFirstPlayerController();
APawn* LocalPawn = LocalPC ? LocalPC->GetPawn() : nullptr;
for (UPS_AI_ConvAgent_ElevenLabsComponent* Agent : Agents)
{
@ -136,7 +141,9 @@ UPS_AI_ConvAgent_ElevenLabsComponent* UPS_AI_ConvAgent_InteractionComponent::Eva
if (!AgentActor) continue;
// Network: skip agents that are in conversation with a different player.
if (Agent->bNetIsConversing && Agent->NetConversatingPlayer != LocalPC)
// Use NetConversatingPawn (replicated to all) instead of NetConversatingPlayer
// (NULL on remote clients because APlayerController has bOnlyRelevantToOwner=true).
if (Agent->bNetIsConversing && Agent->NetConversatingPawn != LocalPawn)
{
continue;
}
@ -256,7 +263,17 @@ void UPS_AI_ConvAgent_InteractionComponent::SetSelectedAgent(UPS_AI_ConvAgent_El
// call StartConversationWithSelectedAgent() explicitly (e.g. on key press).
if (bAutoStartConversation && !NewAgent->IsConnected() && !NewAgent->bNetIsConversing)
{
NewAgent->StartConversation();
// On the server, call StartConversation() directly.
// On clients, route through our relay RPC (clients can't call
// Server RPCs on NPC actors — no owning connection).
if (GetOwnerRole() == ROLE_Authority || (GetWorld() && GetWorld()->GetNetMode() == NM_Standalone))
{
NewAgent->StartConversation();
}
else
{
ServerRelayStartConversation(NewAgent->GetOwner());
}
// Ensure mic is capturing so we can route audio to the new agent.
if (MicComponent && !MicComponent->IsCapturing())
@ -380,7 +397,15 @@ void UPS_AI_ConvAgent_InteractionComponent::StartConversationWithSelectedAgent()
Agent->GetOwner() ? *Agent->GetOwner()->GetName() : TEXT("(null)"));
}
Agent->StartConversation();
// Route through relay on clients (can't call Server RPCs on NPC actors).
if (GetOwnerRole() == ROLE_Authority || (GetWorld() && GetWorld()->GetNetMode() == NM_Standalone))
{
Agent->StartConversation();
}
else
{
ServerRelayStartConversation(Agent->GetOwner());
}
// Ensure mic is capturing so we can route audio to the agent.
if (MicComponent && !MicComponent->IsCapturing())
@ -467,3 +492,98 @@ void UPS_AI_ConvAgent_InteractionComponent::OnMicAudioCaptured(const TArray<floa
Agent->FeedExternalAudio(FloatPCM);
}
// ─────────────────────────────────────────────────────────────────────────────
// Replication
// ─────────────────────────────────────────────────────────────────────────────
void UPS_AI_ConvAgent_InteractionComponent::GetLifetimeReplicatedProps(
TArray<FLifetimeProperty>& OutLifetimeProps) const
{
Super::GetLifetimeReplicatedProps(OutLifetimeProps);
// No replicated properties — but the component must be replicated for RPCs.
}
// ─────────────────────────────────────────────────────────────────────────────
// Network relay RPCs
//
// UE5 clients can only call Server RPCs on actors they own. NPC actors are
// server-owned, so clients can't call RPCs on them. These relays run on
// the player's pawn (owned by the client) and forward to the NPC on the server.
// ─────────────────────────────────────────────────────────────────────────────
void UPS_AI_ConvAgent_InteractionComponent::ServerRelayStartConversation_Implementation(
AActor* AgentActor)
{
if (!AgentActor) return;
auto* Agent = AgentActor->FindComponentByClass<UPS_AI_ConvAgent_ElevenLabsComponent>();
if (!Agent) return;
// Resolve the requesting player from the pawn that owns this component.
APlayerController* PC = nullptr;
if (APawn* Pawn = Cast<APawn>(GetOwner()))
{
PC = Cast<APlayerController>(Pawn->GetController());
}
if (!PC) return;
// Forward to the NPC's implementation directly (we're already on the server).
Agent->ServerRequestConversation_Implementation(PC);
}
void UPS_AI_ConvAgent_InteractionComponent::ServerRelayEndConversation_Implementation(
AActor* AgentActor)
{
if (!AgentActor) return;
auto* Agent = AgentActor->FindComponentByClass<UPS_AI_ConvAgent_ElevenLabsComponent>();
if (!Agent) return;
Agent->ServerReleaseConversation_Implementation();
}
void UPS_AI_ConvAgent_InteractionComponent::ServerRelayMicAudio_Implementation(
AActor* AgentActor, const TArray<uint8>& PCMBytes)
{
if (!AgentActor) return;
auto* Agent = AgentActor->FindComponentByClass<UPS_AI_ConvAgent_ElevenLabsComponent>();
if (!Agent) return;
Agent->ServerSendMicAudio_Implementation(PCMBytes);
}
void UPS_AI_ConvAgent_InteractionComponent::ServerRelaySendText_Implementation(
AActor* AgentActor, const FString& Text)
{
if (!AgentActor) return;
auto* Agent = AgentActor->FindComponentByClass<UPS_AI_ConvAgent_ElevenLabsComponent>();
if (!Agent) return;
Agent->ServerSendTextMessage_Implementation(Text);
}
void UPS_AI_ConvAgent_InteractionComponent::ServerRelayInterrupt_Implementation(
AActor* AgentActor)
{
if (!AgentActor) return;
auto* Agent = AgentActor->FindComponentByClass<UPS_AI_ConvAgent_ElevenLabsComponent>();
if (!Agent) return;
Agent->ServerRequestInterrupt_Implementation();
}
void UPS_AI_ConvAgent_InteractionComponent::ClientRelayConversationStarted_Implementation(
AActor* AgentActor,
const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& Info)
{
if (!AgentActor) return;
auto* Agent = AgentActor->FindComponentByClass<UPS_AI_ConvAgent_ElevenLabsComponent>();
if (!Agent) return;
Agent->ClientConversationStarted_Implementation(Info);
}
void UPS_AI_ConvAgent_InteractionComponent::ClientRelayConversationFailed_Implementation(
const FString& Reason)
{
UE_LOG(LogPS_AI_ConvAgent_Select, Warning, TEXT("[NET] Conversation request denied: %s"), *Reason);
}

View File

@ -615,4 +615,6 @@ private:
bool IsLocalPlayerConversating() const;
/** Internal: performs the actual WebSocket setup (called by both local and RPC paths). */
void StartConversation_Internal();
/** Find the InteractionComponent on the local player's pawn (for relay RPCs). */
class UPS_AI_ConvAgent_InteractionComponent* FindLocalRelayComponent() const;
};

View File

@ -4,6 +4,7 @@
#include "CoreMinimal.h"
#include "Components/ActorComponent.h"
#include "PS_AI_ConvAgent_Definitions.h"
#include "PS_AI_ConvAgent_InteractionComponent.generated.h"
class UPS_AI_ConvAgent_ElevenLabsComponent;
@ -187,11 +188,47 @@ public:
UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction")
void ClearSelection();
// ── Network relay RPCs ───────────────────────────────────────────────────
// UE5 clients can only call Server RPCs on actors they own. NPC actors are
// server-owned, so clients can't call RPCs on them directly.
// These relay RPCs live on the player's pawn (which IS owned by the client),
// forwarding commands to the NPC's ElevenLabsComponent on the server.
/** Relay: request exclusive conversation with an NPC agent. */
UFUNCTION(Server, Reliable)
void ServerRelayStartConversation(AActor* AgentActor);
/** Relay: release conversation with an NPC agent. */
UFUNCTION(Server, Reliable)
void ServerRelayEndConversation(AActor* AgentActor);
/** Relay: stream mic audio to an NPC agent's WebSocket. */
UFUNCTION(Server, Unreliable)
void ServerRelayMicAudio(AActor* AgentActor, const TArray<uint8>& PCMBytes);
/** Relay: send text message to an NPC agent. */
UFUNCTION(Server, Reliable)
void ServerRelaySendText(AActor* AgentActor, const FString& Text);
/** Relay: interrupt an NPC agent's current utterance. */
UFUNCTION(Server, Reliable)
void ServerRelayInterrupt(AActor* AgentActor);
/** Relay (Client RPC): notify the requesting client that conversation started. */
UFUNCTION(Client, Reliable)
void ClientRelayConversationStarted(AActor* AgentActor,
const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& Info);
/** Relay (Client RPC): notify the requesting client that conversation failed. */
UFUNCTION(Client, Reliable)
void ClientRelayConversationFailed(const FString& Reason);
// ── UActorComponent overrides ────────────────────────────────────────────
virtual void BeginPlay() override;
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
virtual void TickComponent(float DeltaTime, ELevelTick TickType,
FActorComponentTickFunction* ThisTickFunction) override;
virtual void GetLifetimeReplicatedProps(TArray<FLifetimeProperty>& OutLifetimeProps) const override;
private:
// ── Selection logic ──────────────────────────────────────────────────────