diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp index 2c02f77..5ab4095 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp @@ -7,6 +7,7 @@ #include "PS_AI_ConvAgent_FacialExpressionComponent.h" #include "PS_AI_ConvAgent_LipSyncComponent.h" #include "PS_AI_ConvAgent_InteractionSubsystem.h" +#include "PS_AI_ConvAgent_InteractionComponent.h" #include "PS_AI_ConvAgent.h" #include "Components/AudioComponent.h" @@ -231,11 +232,17 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StartConversation() } else { - // Client: request conversation via Server RPC. - APlayerController* PC = GetWorld() ? GetWorld()->GetFirstPlayerController() : nullptr; - if (PC) + // Client: route through InteractionComponent relay (clients can't call + // Server RPCs on NPC actors — no owning connection). + if (auto* Relay = FindLocalRelayComponent()) { - ServerRequestConversation(PC); + Relay->ServerRelayStartConversation(GetOwner()); + } + else + { + // Fallback: try direct RPC (will fail with "No owning connection" warning). + APlayerController* PC = GetWorld() ? GetWorld()->GetFirstPlayerController() : nullptr; + if (PC) ServerRequestConversation(PC); } } } @@ -303,16 +310,32 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::EndConversation() } else { - // Client: request release via Server RPC. - ServerReleaseConversation(); + // Client: route through InteractionComponent relay (clients can't call + // Server RPCs on NPC actors — no owning connection). + if (auto* Relay = FindLocalRelayComponent()) + { + Relay->ServerRelayEndConversation(GetOwner()); + } + else + { + // Fallback: try direct RPC (will fail with "No owning connection" warning). + ServerReleaseConversation(); + } } } void UPS_AI_ConvAgent_ElevenLabsComponent::StartListening() { - if (!IsConnected()) + // On the server, the local WebSocket must be connected. + // On clients, there is no local WebSocket — use the replicated conversation state + // (the actual WebSocket lives on the server and audio is routed through relay RPCs). + const bool bEffectivelyConnected = IsConnected() || (GetOwnerRole() != ROLE_Authority && bNetIsConversing); + if (!bEffectivelyConnected) { - UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning, TEXT("StartListening: not connected.")); + UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning, TEXT("StartListening: not connected (IsConnected=%s bNetIsConversing=%s Role=%d)."), + IsConnected() ? TEXT("true") : TEXT("false"), + bNetIsConversing ? TEXT("true") : TEXT("false"), + static_cast(GetOwnerRole())); return; } @@ -352,7 +375,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StartListening() bIsListening = true; TurnStartTime = FPlatformTime::Seconds(); - if (TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Client) + if (TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Client && WebSocketProxy) { WebSocketProxy->SendUserTurnStart(); } @@ -438,9 +461,23 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StopListening() MicAccumulationBuffer.Num()); } } - else if (MicAccumulationBuffer.Num() > 0 && WebSocketProxy && IsConnected()) + else if (MicAccumulationBuffer.Num() > 0) { - WebSocketProxy->SendAudioChunk(MicAccumulationBuffer); + if (GetOwnerRole() == ROLE_Authority) + { + if (WebSocketProxy && IsConnected()) + { + WebSocketProxy->SendAudioChunk(MicAccumulationBuffer); + } + } + else + { + // Client: flush remaining mic audio through relay. + if (auto* Relay = FindLocalRelayComponent()) + { + Relay->ServerRelayMicAudio(GetOwner(), MicAccumulationBuffer); + } + } } MicAccumulationBuffer.Reset(); } @@ -483,12 +520,27 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StopListening() void UPS_AI_ConvAgent_ElevenLabsComponent::SendTextMessage(const FString& Text) { - if (!IsConnected()) + if (GetOwnerRole() == ROLE_Authority) { - UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning, TEXT("SendTextMessage: not connected. Call StartConversation() first.")); - return; + if (!IsConnected()) + { + UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning, TEXT("SendTextMessage: not connected. Call StartConversation() first.")); + return; + } + WebSocketProxy->SendTextMessage(Text); + } + else + { + // Client: route through InteractionComponent relay (WebSocket is on server). + if (auto* Relay = FindLocalRelayComponent()) + { + Relay->ServerRelaySendText(GetOwner(), Text); + } + else + { + ServerSendTextMessage(Text); + } } - WebSocketProxy->SendTextMessage(Text); // Enable body tracking on the sibling PostureComponent (if present). // Text input counts as conversation engagement, same as voice. @@ -505,7 +557,24 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::SendTextMessage(const FString& Text) void UPS_AI_ConvAgent_ElevenLabsComponent::InterruptAgent() { bWaitingForAgentResponse = false; // Interrupting — no response expected from previous turn. - if (WebSocketProxy) WebSocketProxy->SendInterrupt(); + + if (GetOwnerRole() == ROLE_Authority) + { + if (WebSocketProxy) WebSocketProxy->SendInterrupt(); + } + else + { + // Client: route through InteractionComponent relay (WebSocket is on server). + if (auto* Relay = FindLocalRelayComponent()) + { + Relay->ServerRelayInterrupt(GetOwner()); + } + else + { + ServerRequestInterrupt(); + } + } + StopAgentAudio(); } @@ -513,7 +582,9 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::FeedExternalAudio(const TArray { // Same logic as OnMicrophoneDataCaptured but called from an external source // (e.g. InteractionComponent on the pawn) instead of a local mic component. - if (!IsConnected() || !bIsListening) return; + // On server: check local WebSocket. On client: check replicated conversation state. + const bool bCanSend = (GetOwnerRole() == ROLE_Authority) ? IsConnected() : bNetIsConversing; + if (!bCanSend || !bIsListening) return; // Echo suppression: skip sending mic audio while the agent is speaking. if (bAgentSpeaking && !(TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server && bAllowInterruption)) @@ -534,7 +605,15 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::FeedExternalAudio(const TArray } else { - ServerSendMicAudio(MicAccumulationBuffer); + // Route through relay (clients can't call Server RPCs on NPC actors). + if (auto* Relay = FindLocalRelayComponent()) + { + Relay->ServerRelayMicAudio(GetOwner(), MicAccumulationBuffer); + } + else + { + ServerSendMicAudio(MicAccumulationBuffer); + } } MicAccumulationBuffer.Reset(); } @@ -566,9 +645,14 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleConnected(const FPS_AI_ConvAgen OnAgentConnected.Broadcast(Info); // Network: notify the requesting remote client that conversation started. - if (GetOwnerRole() == ROLE_Authority && NetConversatingPlayer) + // Client RPCs on NPC actors have no owning connection — route through the + // player pawn's InteractionComponent which IS owned by the client. + if (GetOwnerRole() == ROLE_Authority && NetConversatingPawn) { - ClientConversationStarted(Info); + if (auto* Relay = NetConversatingPawn->FindComponentByClass()) + { + Relay->ClientRelayConversationStarted(GetOwner(), Info); + } } // In Client turn mode (push-to-talk), the user controls listening manually via @@ -1139,7 +1223,9 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StopAgentAudio() // ───────────────────────────────────────────────────────────────────────────── void UPS_AI_ConvAgent_ElevenLabsComponent::OnMicrophoneDataCaptured(const TArray& FloatPCM) { - if (!IsConnected() || !bIsListening) return; + // On server: check local WebSocket. On client: check replicated conversation state. + const bool bCanSend = (GetOwnerRole() == ROLE_Authority) ? IsConnected() : bNetIsConversing; + if (!bCanSend || !bIsListening) return; // Echo suppression: skip sending mic audio while the agent is speaking. // This prevents the agent from hearing its own voice through the speakers, @@ -1170,7 +1256,15 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::OnMicrophoneDataCaptured(const TArray } else { - ServerSendMicAudio(MicAccumulationBuffer); + // Route through relay (clients can't call Server RPCs on NPC actors). + if (auto* Relay = FindLocalRelayComponent()) + { + Relay->ServerRelayMicAudio(GetOwner(), MicAccumulationBuffer); + } + else + { + ServerSendMicAudio(MicAccumulationBuffer); + } } MicAccumulationBuffer.Reset(); } @@ -1296,7 +1390,18 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ServerRequestConversation_Implementat { if (bNetIsConversing) { - ClientConversationFailed(TEXT("NPC is already in conversation with another player.")); + // Route failure notification through the player's InteractionComponent relay. + // Client RPCs on NPC actors have no owning connection. + if (RequestingPlayer) + { + if (APawn* Pawn = RequestingPlayer->GetPawn()) + { + if (auto* Relay = Pawn->FindComponentByClass()) + { + Relay->ClientRelayConversationFailed(TEXT("NPC is already in conversation with another player.")); + } + } + } return; } @@ -1538,3 +1643,18 @@ bool UPS_AI_ConvAgent_ElevenLabsComponent::IsLocalPlayerConversating() const } return false; } + +UPS_AI_ConvAgent_InteractionComponent* UPS_AI_ConvAgent_ElevenLabsComponent::FindLocalRelayComponent() const +{ + if (UWorld* World = GetWorld()) + { + if (APlayerController* PC = World->GetFirstPlayerController()) + { + if (APawn* Pawn = PC->GetPawn()) + { + return Pawn->FindComponentByClass(); + } + } + } + return nullptr; +} diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp index 6759675..485cd3e 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp @@ -10,6 +10,7 @@ #include "GameFramework/PlayerController.h" #include "Camera/PlayerCameraManager.h" #include "TimerManager.h" +#include "Net/UnrealNetwork.h" DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_Select, Log, All); @@ -20,6 +21,7 @@ UPS_AI_ConvAgent_InteractionComponent::UPS_AI_ConvAgent_InteractionComponent() { PrimaryComponentTick.bCanEverTick = true; PrimaryComponentTick.TickInterval = 1.0f / 30.0f; // 30 Hz is enough for selection evaluation. + SetIsReplicatedByDefault(true); // Required for Server/Client RPCs on this component. } // ───────────────────────────────────────────────────────────────────────────── @@ -127,8 +129,11 @@ UPS_AI_ConvAgent_ElevenLabsComponent* UPS_AI_ConvAgent_InteractionComponent::Eva UPS_AI_ConvAgent_ElevenLabsComponent* CurrentAgent = SelectedAgent.Get(); - // Get local player controller for occupied-NPC check. + // Get local player's pawn for occupied-NPC check. + // Use pawn (replicated to ALL clients) instead of PlayerController + // (only replicated to owning client due to bOnlyRelevantToOwner=true). APlayerController* LocalPC = World->GetFirstPlayerController(); + APawn* LocalPawn = LocalPC ? LocalPC->GetPawn() : nullptr; for (UPS_AI_ConvAgent_ElevenLabsComponent* Agent : Agents) { @@ -136,7 +141,9 @@ UPS_AI_ConvAgent_ElevenLabsComponent* UPS_AI_ConvAgent_InteractionComponent::Eva if (!AgentActor) continue; // Network: skip agents that are in conversation with a different player. - if (Agent->bNetIsConversing && Agent->NetConversatingPlayer != LocalPC) + // Use NetConversatingPawn (replicated to all) instead of NetConversatingPlayer + // (NULL on remote clients because APlayerController has bOnlyRelevantToOwner=true). + if (Agent->bNetIsConversing && Agent->NetConversatingPawn != LocalPawn) { continue; } @@ -256,7 +263,17 @@ void UPS_AI_ConvAgent_InteractionComponent::SetSelectedAgent(UPS_AI_ConvAgent_El // call StartConversationWithSelectedAgent() explicitly (e.g. on key press). if (bAutoStartConversation && !NewAgent->IsConnected() && !NewAgent->bNetIsConversing) { - NewAgent->StartConversation(); + // On the server, call StartConversation() directly. + // On clients, route through our relay RPC (clients can't call + // Server RPCs on NPC actors — no owning connection). + if (GetOwnerRole() == ROLE_Authority || (GetWorld() && GetWorld()->GetNetMode() == NM_Standalone)) + { + NewAgent->StartConversation(); + } + else + { + ServerRelayStartConversation(NewAgent->GetOwner()); + } // Ensure mic is capturing so we can route audio to the new agent. if (MicComponent && !MicComponent->IsCapturing()) @@ -380,7 +397,15 @@ void UPS_AI_ConvAgent_InteractionComponent::StartConversationWithSelectedAgent() Agent->GetOwner() ? *Agent->GetOwner()->GetName() : TEXT("(null)")); } - Agent->StartConversation(); + // Route through relay on clients (can't call Server RPCs on NPC actors). + if (GetOwnerRole() == ROLE_Authority || (GetWorld() && GetWorld()->GetNetMode() == NM_Standalone)) + { + Agent->StartConversation(); + } + else + { + ServerRelayStartConversation(Agent->GetOwner()); + } // Ensure mic is capturing so we can route audio to the agent. if (MicComponent && !MicComponent->IsCapturing()) @@ -467,3 +492,98 @@ void UPS_AI_ConvAgent_InteractionComponent::OnMicAudioCaptured(const TArrayFeedExternalAudio(FloatPCM); } + +// ───────────────────────────────────────────────────────────────────────────── +// Replication +// ───────────────────────────────────────────────────────────────────────────── +void UPS_AI_ConvAgent_InteractionComponent::GetLifetimeReplicatedProps( + TArray& OutLifetimeProps) const +{ + Super::GetLifetimeReplicatedProps(OutLifetimeProps); + // No replicated properties — but the component must be replicated for RPCs. +} + +// ───────────────────────────────────────────────────────────────────────────── +// Network relay RPCs +// +// UE5 clients can only call Server RPCs on actors they own. NPC actors are +// server-owned, so clients can't call RPCs on them. These relays run on +// the player's pawn (owned by the client) and forward to the NPC on the server. +// ───────────────────────────────────────────────────────────────────────────── + +void UPS_AI_ConvAgent_InteractionComponent::ServerRelayStartConversation_Implementation( + AActor* AgentActor) +{ + if (!AgentActor) return; + auto* Agent = AgentActor->FindComponentByClass(); + if (!Agent) return; + + // Resolve the requesting player from the pawn that owns this component. + APlayerController* PC = nullptr; + if (APawn* Pawn = Cast(GetOwner())) + { + PC = Cast(Pawn->GetController()); + } + + if (!PC) return; + + // Forward to the NPC's implementation directly (we're already on the server). + Agent->ServerRequestConversation_Implementation(PC); +} + +void UPS_AI_ConvAgent_InteractionComponent::ServerRelayEndConversation_Implementation( + AActor* AgentActor) +{ + if (!AgentActor) return; + auto* Agent = AgentActor->FindComponentByClass(); + if (!Agent) return; + + Agent->ServerReleaseConversation_Implementation(); +} + +void UPS_AI_ConvAgent_InteractionComponent::ServerRelayMicAudio_Implementation( + AActor* AgentActor, const TArray& PCMBytes) +{ + if (!AgentActor) return; + auto* Agent = AgentActor->FindComponentByClass(); + if (!Agent) return; + + Agent->ServerSendMicAudio_Implementation(PCMBytes); +} + +void UPS_AI_ConvAgent_InteractionComponent::ServerRelaySendText_Implementation( + AActor* AgentActor, const FString& Text) +{ + if (!AgentActor) return; + auto* Agent = AgentActor->FindComponentByClass(); + if (!Agent) return; + + Agent->ServerSendTextMessage_Implementation(Text); +} + +void UPS_AI_ConvAgent_InteractionComponent::ServerRelayInterrupt_Implementation( + AActor* AgentActor) +{ + if (!AgentActor) return; + auto* Agent = AgentActor->FindComponentByClass(); + if (!Agent) return; + + Agent->ServerRequestInterrupt_Implementation(); +} + +void UPS_AI_ConvAgent_InteractionComponent::ClientRelayConversationStarted_Implementation( + AActor* AgentActor, + const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& Info) +{ + if (!AgentActor) return; + auto* Agent = AgentActor->FindComponentByClass(); + if (!Agent) return; + + Agent->ClientConversationStarted_Implementation(Info); +} + +void UPS_AI_ConvAgent_InteractionComponent::ClientRelayConversationFailed_Implementation( + const FString& Reason) +{ + UE_LOG(LogPS_AI_ConvAgent_Select, Warning, TEXT("[NET] Conversation request denied: %s"), *Reason); +} diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h index 720691b..027b5af 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h @@ -615,4 +615,6 @@ private: bool IsLocalPlayerConversating() const; /** Internal: performs the actual WebSocket setup (called by both local and RPC paths). */ void StartConversation_Internal(); + /** Find the InteractionComponent on the local player's pawn (for relay RPCs). */ + class UPS_AI_ConvAgent_InteractionComponent* FindLocalRelayComponent() const; }; diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionComponent.h index 7214666..900e96e 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionComponent.h @@ -4,6 +4,7 @@ #include "CoreMinimal.h" #include "Components/ActorComponent.h" +#include "PS_AI_ConvAgent_Definitions.h" #include "PS_AI_ConvAgent_InteractionComponent.generated.h" class UPS_AI_ConvAgent_ElevenLabsComponent; @@ -187,11 +188,47 @@ public: UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Interaction") void ClearSelection(); + // ── Network relay RPCs ─────────────────────────────────────────────────── + // UE5 clients can only call Server RPCs on actors they own. NPC actors are + // server-owned, so clients can't call RPCs on them directly. + // These relay RPCs live on the player's pawn (which IS owned by the client), + // forwarding commands to the NPC's ElevenLabsComponent on the server. + + /** Relay: request exclusive conversation with an NPC agent. */ + UFUNCTION(Server, Reliable) + void ServerRelayStartConversation(AActor* AgentActor); + + /** Relay: release conversation with an NPC agent. */ + UFUNCTION(Server, Reliable) + void ServerRelayEndConversation(AActor* AgentActor); + + /** Relay: stream mic audio to an NPC agent's WebSocket. */ + UFUNCTION(Server, Unreliable) + void ServerRelayMicAudio(AActor* AgentActor, const TArray& PCMBytes); + + /** Relay: send text message to an NPC agent. */ + UFUNCTION(Server, Reliable) + void ServerRelaySendText(AActor* AgentActor, const FString& Text); + + /** Relay: interrupt an NPC agent's current utterance. */ + UFUNCTION(Server, Reliable) + void ServerRelayInterrupt(AActor* AgentActor); + + /** Relay (Client RPC): notify the requesting client that conversation started. */ + UFUNCTION(Client, Reliable) + void ClientRelayConversationStarted(AActor* AgentActor, + const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& Info); + + /** Relay (Client RPC): notify the requesting client that conversation failed. */ + UFUNCTION(Client, Reliable) + void ClientRelayConversationFailed(const FString& Reason); + // ── UActorComponent overrides ──────────────────────────────────────────── virtual void BeginPlay() override; virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override; virtual void TickComponent(float DeltaTime, ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction) override; + virtual void GetLifetimeReplicatedProps(TArray& OutLifetimeProps) const override; private: // ── Selection logic ──────────────────────────────────────────────────────