Multi-player shared agent: multiple players can converse with the same agent simultaneously

Replace exclusive single-player agent lock with shared multi-player model:
- NetConversatingPawn/Player → NetConnectedPawns array + NetActiveSpeakerPawn
- Server-side speaker arbitration with hysteresis (0.3s) prevents gaze ping-pong
- Speaker idle timeout (3.0s) clears active speaker after silence
- Agent gaze follows the active speaker via replicated OnRep_ActiveSpeaker
- New ServerJoinConversation/ServerLeaveConversation RPCs (idempotent join/leave)
- Backward-compatible: old ServerRequest/Release delegate to new Join/Leave
- InteractionComponent no longer skips occupied agents
- DrawDebugHUD shows connected player count and active speaker
- All mic audio paths (FeedExternalAudio, OnMicCapture, StopListening flush) route
  through speaker arbitration

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp

@@ -164,17 +164,44 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::TickComponent(float DeltaTime, ELevel
 		}
 	}
 
-	// Network: detect if the conversating player disconnected (server only).
+	// Network: detect if connected players disconnected (server only).
-	if (GetOwnerRole() == ROLE_Authority && bNetIsConversing && NetConversatingPlayer)
+	if (GetOwnerRole() == ROLE_Authority && bNetIsConversing && NetConnectedPawns.Num() > 0)
 	{
-		if (!IsValid(NetConversatingPlayer) || !NetConversatingPlayer->GetPawn())
+		for (int32 i = NetConnectedPawns.Num() - 1; i >= 0; --i)
+		{
+			APawn* Pawn = NetConnectedPawns[i];
+			if (!IsValid(Pawn))
+			{
+				UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
+					TEXT("Connected player pawn [%d] disconnected — removing."), i);
+				NetConnectedPawns.RemoveAt(i, 1, EAllowShrinking::No);
+				LastSpeakTime.Remove(Pawn);
+				if (NetActiveSpeakerPawn == Pawn)
+				{
+					SetActiveSpeaker(NetConnectedPawns.Num() > 0 ? NetConnectedPawns[0] : nullptr);
+				}
+			}
+		}
+		if (NetConnectedPawns.Num() == 0)
 		{
 			UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
-				TEXT("Conversating player disconnected — releasing NPC."));
+				TEXT("All connected players disconnected — releasing NPC."));
 			ServerReleaseConversation_Implementation();
 		}
 	}
 
+	// Speaker idle timeout: clear active speaker after silence.
+	if (GetOwnerRole() == ROLE_Authority && NetActiveSpeakerPawn && SpeakerIdleTimeout > 0.0f)
+	{
+		if (const double* LastActive = LastSpeakTime.Find(NetActiveSpeakerPawn))
+		{
+			if (FPlatformTime::Seconds() - *LastActive > SpeakerIdleTimeout)
+			{
+				SetActiveSpeaker(nullptr);
+			}
+		}
+	}
+
 	// ── Reconnection ────────────────────────────────────────────────────────
 	if (bWantsReconnect && FPlatformTime::Seconds() >= NextReconnectTime)
 	{
@@ -186,9 +213,10 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::TickComponent(float DeltaTime, ELevel
 			UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Error,
 				TEXT("Reconnection failed after %d attempts — giving up."), MaxReconnectAttempts);
 			bNetIsConversing = false;
+			NetConnectedPawns.Empty();
+			NetActiveSpeakerPawn = nullptr;
+			LastSpeakTime.Empty();
 			ApplyConversationGaze();
-			NetConversatingPlayer = nullptr;
-			NetConversatingPawn = nullptr;
 			OnAgentDisconnected.Broadcast(1006, TEXT("Reconnection failed"));
 		}
 		else
@@ -284,30 +312,11 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StartConversation()
 {
 	if (GetOwnerRole() == ROLE_Authority)
 	{
-		// Set conversation state (used by ApplyConversationGaze, gaze, LOD, etc.).
+		// Standalone / listen-server: join via the local player controller.
-		// In standalone these aren't replicated but are still needed as local state flags.
 		APlayerController* PC = GetWorld() ? GetWorld()->GetFirstPlayerController() : nullptr;
-		bNetIsConversing = true;
+		if (PC)
-		NetConversatingPlayer = PC;
-		NetConversatingPawn = PC ? PC->GetPawn() : nullptr;
-
-		// In persistent mode the WebSocket was already opened by the first StartConversation.
-		// Reuse the existing connection — only set up conversation state.
-		if (bPersistentSession && IsConnected())
 		{
-			// WebSocket already alive — just set up conversation state (gaze, etc.).
+			ServerJoinConversation_Implementation(PC);
-			ApplyConversationGaze();
-			OnAgentConnected.Broadcast(WebSocketProxy->GetConversationInfo());
-
-			// Auto-start listening if configured (same as HandleConnected).
-			if (bAutoStartListening && TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server)
-			{
-				StartListening();
-			}
-		}
-		else
-		{
-			StartConversation_Internal();
 		}
 	}
 	else
@@ -316,13 +325,13 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StartConversation()
 		// Server RPCs on NPC actors — no owning connection).
 		if (auto* Relay = FindLocalRelayComponent())
 		{
-			Relay->ServerRelayStartConversation(GetOwner());
+			Relay->ServerRelayJoinConversation(GetOwner());
 		}
 		else
 		{
 			// Fallback: try direct RPC (will fail with "No owning connection" warning).
 			APlayerController* PC = GetWorld() ? GetWorld()->GetFirstPlayerController() : nullptr;
-			if (PC) ServerRequestConversation(PC);
+			if (PC) ServerJoinConversation(PC);
 		}
 	}
 }
@@ -370,44 +379,12 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::EndConversation()
 {
 	if (GetOwnerRole() == ROLE_Authority)
 	{
-		// Cancel any pending reconnection (ephemeral mode only — persistent keeps reconnecting).
+		// Standalone / listen-server: leave via the local player controller.
-		if (!bPersistentSession)
+		APlayerController* PC = GetWorld() ? GetWorld()->GetFirstPlayerController() : nullptr;
+		if (PC)
 		{
-			bWantsReconnect = false;
+			ServerLeaveConversation_Implementation(PC);
-			ReconnectAttemptCount = 0;
 		}
-
-		StopListening();
-		// ISSUE-4: StopListening() may set bWaitingForAgentResponse=true (normal turn end path).
-		// Cancel it immediately — there is no response coming because we are ending the session.
-		// Without this, TickComponent could fire OnAgentResponseTimeout after EndConversation().
-		bWaitingForAgentResponse = false;
-		StopAgentAudio();
-
-		// In persistent mode, keep the WebSocket open — only manage conversation state.
-		if (!bPersistentSession)
-		{
-			if (WebSocketProxy)
-			{
-				bIntentionalDisconnect = true;
-				WebSocketProxy->Disconnect();
-				// OnClosed callback will fire OnAgentDisconnected.
-				WebSocketProxy = nullptr;
-			}
-		}
-		else
-		{
-			// Persistent mode: WebSocket stays alive but the interaction is over.
-			// Broadcast OnAgentDisconnected so expression components deactivate
-			// (body, facial, etc.). The WebSocket OnClosed never fires here.
-			OnAgentDisconnected.Broadcast(1000, TEXT("EndConversation (persistent)"));
-		}
-
-		// Reset replicated state so other players can talk to this NPC.
-		bNetIsConversing = false;
-		ApplyConversationGaze();
-		NetConversatingPlayer = nullptr;
-		NetConversatingPawn = nullptr;
 	}
 	else
 	{
@@ -415,7 +392,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::EndConversation()
 		// Server RPCs on NPC actors — no owning connection).
 		if (auto* Relay = FindLocalRelayComponent())
 		{
-			Relay->ServerRelayEndConversation(GetOwner());
+			Relay->ServerRelayLeaveConversation(GetOwner());
 		}
 		else
 		{
@@ -566,10 +543,14 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StopListening()
 		{
 			if (GetOwnerRole() == ROLE_Authority)
 			{
-				if (WebSocketProxy && IsConnected())
+				// Route through speaker arbitration so the correct player's
+				// final chunk is attributed properly in multi-player.
+				APawn* LocalPawn = nullptr;
+				if (const APlayerController* PC = GetWorld()->GetFirstPlayerController())
 				{
-					WebSocketProxy->SendAudioChunk(MicAccumulationBuffer);
+					LocalPawn = PC->GetPawn();
 				}
+				ServerSendMicAudioFromPlayer(LocalPawn, MicAccumulationBuffer);
 			}
 			else
 			{
@@ -723,7 +704,17 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::FeedExternalAudio(const TArray<float>
 	{
 		if (GetOwnerRole() == ROLE_Authority)
 		{
-			if (WebSocketProxy) WebSocketProxy->SendAudioChunk(MicAccumulationBuffer);
+			// On the server (listen server / standalone), route through speaker
+			// arbitration so the correct player's audio is forwarded to ElevenLabs.
+			APawn* LocalPawn = nullptr;
+			if (UWorld* World = GetWorld())
+			{
+				if (APlayerController* PC = World->GetFirstPlayerController())
+				{
+					LocalPawn = PC->GetPawn();
+				}
+			}
+			ServerSendMicAudioFromPlayer(LocalPawn, MicAccumulationBuffer);
 		}
 		else
 		{
@@ -821,14 +812,18 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleConnected(const FPS_AI_ConvAgen
 	UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log, TEXT("[T+0.00s] Agent connected. ConversationID=%s"), *Info.ConversationID);
 	OnAgentConnected.Broadcast(Info);
 
-	// Network: notify the requesting remote client that conversation started.
+	// Network: notify all connected remote clients that conversation started.
 	// Client RPCs on NPC actors have no owning connection — route through the
 	// player pawn's InteractionComponent which IS owned by the client.
-	if (GetOwnerRole() == ROLE_Authority && NetConversatingPawn)
+	if (GetOwnerRole() == ROLE_Authority)
 	{
-		if (auto* Relay = NetConversatingPawn->FindComponentByClass<UPS_AI_ConvAgent_InteractionComponent>())
+		for (APawn* Pawn : NetConnectedPawns)
 		{
-			Relay->ClientRelayConversationStarted(GetOwner(), Info);
+			if (!Pawn) continue;
+			if (auto* Relay = Pawn->FindComponentByClass<UPS_AI_ConvAgent_InteractionComponent>())
+			{
+				Relay->ClientRelayConversationStarted(GetOwner(), Info);
+			}
 		}
 	}
 
@@ -892,7 +887,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleDisconnected(int32 StatusCode,
 			TEXT("Unexpected disconnect — will attempt reconnection in %.0fs (max %d attempts)."),
 			Delay, MaxReconnectAttempts);
 		OnAgentError.Broadcast(TEXT("Connection lost — reconnecting..."));
-		// Keep bNetIsConversing / NetConversatingPawn so the NPC stays occupied.
+		// Keep bNetIsConversing / NetConnectedPawns so the NPC stays occupied.
 		return;
 	}
 
@@ -901,9 +896,10 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::HandleDisconnected(int32 StatusCode,
 	if (GetOwnerRole() == ROLE_Authority)
 	{
 		bNetIsConversing = false;
+		NetConnectedPawns.Empty();
+		NetActiveSpeakerPawn = nullptr;
+		LastSpeakTime.Empty();
 		ApplyConversationGaze();
-		NetConversatingPlayer = nullptr;
-		NetConversatingPawn = nullptr;
 	}
 
 	OnAgentDisconnected.Broadcast(StatusCode, Reason);
@@ -1527,7 +1523,16 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::OnMicrophoneDataCaptured(const TArray
 	{
 		if (GetOwnerRole() == ROLE_Authority)
 		{
-			if (WebSocketProxy) WebSocketProxy->SendAudioChunk(MicAccumulationBuffer);
+			// Route through speaker arbitration (local mic = local player).
+			APawn* LocalPawn = nullptr;
+			if (UWorld* World = GetWorld())
+			{
+				if (APlayerController* PC = World->GetFirstPlayerController())
+				{
+					LocalPawn = PC->GetPawn();
+				}
+			}
+			ServerSendMicAudioFromPlayer(LocalPawn, MicAccumulationBuffer);
 		}
 		else
 		{
@@ -1589,8 +1594,8 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::GetLifetimeReplicatedProps(
 {
 	Super::GetLifetimeReplicatedProps(OutLifetimeProps);
 	DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, bNetIsConversing);
-	DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, NetConversatingPlayer);
+	DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, NetConnectedPawns);
-	DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, NetConversatingPawn);
+	DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, NetActiveSpeakerPawn);
 	DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, CurrentEmotion);
 	DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, CurrentEmotionIntensity);
 }
@@ -1600,31 +1605,30 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::OnRep_ConversationState()
 	AActor* Owner = GetOwner();
 
 	UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
-		TEXT("[NET-REP] OnRep_ConversationState — bNetIsConversing=%s NetConversatingPawn=%s NetConversatingPlayer=%s Owner=%s Role=%d"),
+		TEXT("[NET-REP] OnRep_ConversationState — bNetIsConversing=%s ConnectedPawns=%d ActiveSpeaker=%s Owner=%s Role=%d"),
 		bNetIsConversing ? TEXT("true") : TEXT("false"),
-		NetConversatingPawn ? *NetConversatingPawn->GetName() : TEXT("NULL"),
+		NetConnectedPawns.Num(),
-		NetConversatingPlayer ? *NetConversatingPlayer->GetName() : TEXT("NULL"),
+		NetActiveSpeakerPawn ? *NetActiveSpeakerPawn->GetName() : TEXT("NULL"),
 		Owner ? *Owner->GetName() : TEXT("NULL"),
 		static_cast<int32>(GetOwnerRole()));
 
 	if (Owner)
 	{
 		// Update gaze target on all clients so the NPC head/eyes track the
-		// conversating player. TargetActor is normally set by InteractionComponent
+		// active speaker. TargetActor is normally set by InteractionComponent
 		// on the local pawn, but remote clients never run that code path.
 		if (UPS_AI_ConvAgent_GazeComponent* Gaze = Owner->FindComponentByClass<UPS_AI_ConvAgent_GazeComponent>())
 		{
-			// Use NetConversatingPawn (replicated to ALL clients) instead of
+			if (bNetIsConversing && NetConnectedPawns.Num() > 0)
-			// NetConversatingPlayer->GetPawn() — PlayerControllers are only
-			// replicated to their owning client (bOnlyRelevantToOwner=true).
-			if (bNetIsConversing && NetConversatingPawn)
 			{
 				Gaze->bActive = true;
-				Gaze->TargetActor = NetConversatingPawn;
+				// Use active speaker if set, otherwise first connected pawn.
+				APawn* GazeTarget = NetActiveSpeakerPawn ? NetActiveSpeakerPawn : NetConnectedPawns.Last();
+				Gaze->TargetActor = GazeTarget;
 				Gaze->ResetBodyTarget();
 				Gaze->bEnableBodyTracking = true;
 				UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
-					TEXT("[NET-REP] Gaze ACTIVATED, TargetActor set to %s"), *NetConversatingPawn->GetName());
+					TEXT("[NET-REP] Gaze ACTIVATED, TargetActor set to %s"), *GazeTarget->GetName());
 			}
 			else
 			{
@@ -1632,9 +1636,9 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::OnRep_ConversationState()
 				Gaze->TargetActor = nullptr;
 				Gaze->bEnableBodyTracking = false;
 				UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning,
-					TEXT("[NET-REP] Gaze TargetActor cleared — bNetIsConversing=%s Pawn=%s"),
+					TEXT("[NET-REP] Gaze TargetActor cleared — bNetIsConversing=%s ConnectedPawns=%d"),
 					bNetIsConversing ? TEXT("true") : TEXT("false"),
-					NetConversatingPawn ? TEXT("valid") : TEXT("NULL"));
+					NetConnectedPawns.Num());
 			}
 		}
 		else
@@ -1668,6 +1672,27 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::OnRep_ConversationState()
 	}
 }
 
+void UPS_AI_ConvAgent_ElevenLabsComponent::OnRep_ActiveSpeaker()
+{
+	// Update gaze on clients when the speaking player changes.
+	AActor* Owner = GetOwner();
+	if (!Owner) return;
+
+	if (auto* Gaze = Owner->FindComponentByClass<UPS_AI_ConvAgent_GazeComponent>())
+	{
+		if (NetActiveSpeakerPawn)
+		{
+			Gaze->TargetActor = NetActiveSpeakerPawn;
+			Gaze->ResetBodyTarget();
+			UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
+				TEXT("[NET-REP] ActiveSpeaker changed to %s"), *NetActiveSpeakerPawn->GetName());
+		}
+		// Don't clear gaze when null — keep looking at last speaker.
+	}
+
+	OnActiveSpeakerChanged.Broadcast(NetActiveSpeakerPawn, nullptr);
+}
+
 void UPS_AI_ConvAgent_ElevenLabsComponent::OnRep_Emotion()
 {
 	// Fire the existing delegate so FacialExpressionComponent picks it up on clients.
@@ -1677,98 +1702,211 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::OnRep_Emotion()
 // ─────────────────────────────────────────────────────────────────────────────
 // Network: Server RPCs
 // ─────────────────────────────────────────────────────────────────────────────
-void UPS_AI_ConvAgent_ElevenLabsComponent::ServerRequestConversation_Implementation(
+void UPS_AI_ConvAgent_ElevenLabsComponent::ServerJoinConversation_Implementation(
 	APlayerController* RequestingPlayer)
 {
-	if (bNetIsConversing)
+	APawn* Pawn = RequestingPlayer ? RequestingPlayer->GetPawn() : nullptr;
+	if (!Pawn) return;
+
+	// Already connected? No-op (idempotent).
+	if (NetConnectedPawns.Contains(Pawn)) return;
+
+	// Add to connected set.
+	NetConnectedPawns.Add(Pawn);
+
+	UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
+		TEXT("[NET] Player %s JOINED conversation (now %d players)."),
+		*Pawn->GetName(), NetConnectedPawns.Num());
+
+	// First player joining: open WebSocket.
+	if (!bNetIsConversing)
 	{
-		// Route failure notification through the player's InteractionComponent relay.
+		bNetIsConversing = true;
-		// Client RPCs on NPC actors have no owning connection.
+
-		if (RequestingPlayer)
+		if (bPersistentSession && IsConnected())
 		{
-			if (APawn* Pawn = RequestingPlayer->GetPawn())
+			// WebSocket already alive — set up conversation state.
-			{
+			ApplyConversationGaze();
-				if (auto* Relay = Pawn->FindComponentByClass<UPS_AI_ConvAgent_InteractionComponent>())
+			OnAgentConnected.Broadcast(WebSocketProxy->GetConversationInfo());
-				{
-					Relay->ClientRelayConversationFailed(TEXT("NPC is already in conversation with another player."));
-				}
-			}
-		}
-		return;
-	}
 
-	bNetIsConversing = true;
+			// Notify the joining player.
-	NetConversatingPlayer = RequestingPlayer;
+			if (auto* Relay = Pawn->FindComponentByClass<UPS_AI_ConvAgent_InteractionComponent>())
-	NetConversatingPawn = RequestingPlayer ? RequestingPlayer->GetPawn() : nullptr;
-
-	// Update NPC gaze on the server (OnRep never fires on Authority).
-	ApplyConversationGaze();
-
-	// In persistent mode the WebSocket is already open — skip reconnection.
-	if (bPersistentSession && IsConnected())
-	{
-		// Notify the requesting client that conversation started (normally done in HandleConnected).
-		if (NetConversatingPawn)
-		{
-			if (auto* Relay = NetConversatingPawn->FindComponentByClass<UPS_AI_ConvAgent_InteractionComponent>())
 			{
 				Relay->ClientRelayConversationStarted(GetOwner(), WebSocketProxy->GetConversationInfo());
 			}
-		}
 
-		// Auto-start listening if configured.
+			if (bAutoStartListening && TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server)
-		if (bAutoStartListening && TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server)
+			{
+				StartListening();
+			}
+		}
+		else
 		{
-			StartListening();
+			StartConversation_Internal();
 		}
 	}
 	else
 	{
-		StartConversation_Internal();
+		// Agent already in conversation — notify the joining player if WebSocket is connected.
+		if (IsConnected())
+		{
+			if (auto* Relay = Pawn->FindComponentByClass<UPS_AI_ConvAgent_InteractionComponent>())
+			{
+				Relay->ClientRelayConversationStarted(GetOwner(), WebSocketProxy->GetConversationInfo());
+			}
+		}
+		ApplyConversationGaze();
+	}
+
+	// If no active speaker yet, set gaze to the new arrival.
+	if (!NetActiveSpeakerPawn)
+	{
+		SetActiveSpeaker(Pawn);
 	}
 }
 
+// Backward compat: delegate to ServerJoinConversation.
+void UPS_AI_ConvAgent_ElevenLabsComponent::ServerRequestConversation_Implementation(
+	APlayerController* RequestingPlayer)
+{
+	ServerJoinConversation_Implementation(RequestingPlayer);
+}
+
+void UPS_AI_ConvAgent_ElevenLabsComponent::ServerLeaveConversation_Implementation(
+	APlayerController* LeavingPlayer)
+{
+	APawn* Pawn = LeavingPlayer ? LeavingPlayer->GetPawn() : nullptr;
+	if (!Pawn) return;
+
+	if (!NetConnectedPawns.Contains(Pawn)) return;
+
+	NetConnectedPawns.Remove(Pawn);
+	LastSpeakTime.Remove(Pawn);
+
+	UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
+		TEXT("[NET] Player %s LEFT conversation (now %d players)."),
+		*Pawn->GetName(), NetConnectedPawns.Num());
+
+	// If the leaving player was the active speaker, switch to another or null.
+	if (NetActiveSpeakerPawn == Pawn)
+	{
+		SetActiveSpeaker(NetConnectedPawns.Num() > 0 ? NetConnectedPawns[0] : nullptr);
+	}
+
+	// If no players left, fully end the conversation.
+	if (NetConnectedPawns.Num() == 0)
+	{
+		// Cancel any pending reconnection (ephemeral mode only).
+		if (!bPersistentSession)
+		{
+			bWantsReconnect = false;
+			ReconnectAttemptCount = 0;
+		}
+
+		StopListening();
+		bWaitingForAgentResponse = false;
+		StopAgentAudio();
+
+		// In persistent mode, keep the WebSocket open.
+		if (!bPersistentSession)
+		{
+			if (WebSocketProxy)
+			{
+				bIntentionalDisconnect = true;
+				WebSocketProxy->Disconnect();
+				WebSocketProxy = nullptr;
+			}
+		}
+		else
+		{
+			// Persistent mode: WebSocket stays alive but the interaction is over.
+			OnAgentDisconnected.Broadcast(1000, TEXT("EndConversation (persistent)"));
+		}
+
+		bNetIsConversing = false;
+		NetActiveSpeakerPawn = nullptr;
+		LastSpeakTime.Empty();
+		ApplyConversationGaze();
+	}
+	else
+	{
+		// Other players still connected — update gaze.
+		ApplyConversationGaze();
+	}
+}
+
+// Backward compat: ServerReleaseConversation has no player parameter.
+// Try to resolve the caller from the first connected player.
 void UPS_AI_ConvAgent_ElevenLabsComponent::ServerReleaseConversation_Implementation()
 {
-	// Cancel any pending reconnection (ephemeral mode only).
+	// Legacy path: find the first local player controller and leave.
-	if (!bPersistentSession)
+	APlayerController* PC = GetWorld() ? GetWorld()->GetFirstPlayerController() : nullptr;
+	if (PC)
 	{
-		bWantsReconnect = false;
+		ServerLeaveConversation_Implementation(PC);
-		ReconnectAttemptCount = 0;
 	}
-
+	else if (NetConnectedPawns.Num() > 0)
-	StopListening();
-	bWaitingForAgentResponse = false;
-	StopAgentAudio();
-
-	// In persistent mode, keep the WebSocket open.
-	if (!bPersistentSession)
 	{
-		if (WebSocketProxy)
+		// Fallback: remove all connected pawns (full release).
-		{
+		NetConnectedPawns.Empty();
-			bIntentionalDisconnect = true;
+		NetActiveSpeakerPawn = nullptr;
-			WebSocketProxy->Disconnect();
+		LastSpeakTime.Empty();
-			WebSocketProxy = nullptr;
+		bNetIsConversing = false;
-		}
+		StopListening();
+		bWaitingForAgentResponse = false;
+		StopAgentAudio();
+		ApplyConversationGaze();
 	}
-
-	// Clear gaze before nullifying the pawn pointer (ApplyConversationGaze
-	// uses NetConversatingPawn to guard against clearing someone else's target).
-	bNetIsConversing = false;
-	ApplyConversationGaze();
-	NetConversatingPlayer = nullptr;
-	NetConversatingPawn = nullptr;
 }
 
 void UPS_AI_ConvAgent_ElevenLabsComponent::ServerSendMicAudio_Implementation(
 	const TArray<uint8>& PCMBytes)
 {
+	// Legacy single-player path: forward directly (no speaker arbitration).
 	if (WebSocketProxy && WebSocketProxy->IsConnected())
 	{
 		WebSocketProxy->SendAudioChunk(PCMBytes);
 	}
 }
 
+void UPS_AI_ConvAgent_ElevenLabsComponent::ServerSendMicAudioFromPlayer(
+	APawn* SpeakerPawn, const TArray<uint8>& PCMBytes)
+{
+	if (!SpeakerPawn || !WebSocketProxy || !WebSocketProxy->IsConnected()) return;
+	if (!NetConnectedPawns.Contains(SpeakerPawn)) return;
+
+	const double Now = FPlatformTime::Seconds();
+	LastSpeakTime.FindOrAdd(SpeakerPawn) = Now;
+
+	// If this player IS the active speaker, forward immediately.
+	if (NetActiveSpeakerPawn == SpeakerPawn)
+	{
+		WebSocketProxy->SendAudioChunk(PCMBytes);
+		return;
+	}
+
+	// Speaker switch: only switch if current speaker has been silent
+	// for at least SpeakerSwitchHysteresis seconds.
+	bool bCanSwitch = true;
+	if (NetActiveSpeakerPawn)
+	{
+		if (const double* LastActive = LastSpeakTime.Find(NetActiveSpeakerPawn))
+		{
+			if (Now - *LastActive < SpeakerSwitchHysteresis)
+			{
+				bCanSwitch = false; // Current speaker still active recently.
+			}
+		}
+	}
+
+	if (bCanSwitch)
+	{
+		SetActiveSpeaker(SpeakerPawn);
+		WebSocketProxy->SendAudioChunk(PCMBytes);
+	}
+	// else: drop this audio — current speaker still has the floor.
+}
+
 void UPS_AI_ConvAgent_ElevenLabsComponent::ServerSendTextMessage_Implementation(
 	const FString& Text)
 {
@@ -1971,7 +2109,10 @@ bool UPS_AI_ConvAgent_ElevenLabsComponent::IsLocalPlayerConversating() const
 	{
 		if (APlayerController* PC = World->GetFirstPlayerController())
 		{
-			return NetConversatingPlayer == PC;
+			if (APawn* Pawn = PC->GetPawn())
+			{
+				return NetConnectedPawns.Contains(Pawn);
+			}
 		}
 	}
 	return false;
@@ -1997,8 +2138,12 @@ bool UPS_AI_ConvAgent_ElevenLabsComponent::ShouldUseExternalMic() const
 	// Network client: audio arrives via relay RPCs from InteractionComponent
 	if (GetOwnerRole() != ROLE_Authority) return true;
 
-	// Authority with a remote player: audio arrives via ServerSendMicAudio RPC
+	// Authority with remote players: audio arrives via ServerSendMicAudio RPC
-	if (NetConversatingPlayer && !NetConversatingPlayer->IsLocalController()) return true;
+	// Check if any connected pawn is NOT locally controlled.
+	for (APawn* Pawn : NetConnectedPawns)
+	{
+		if (Pawn && !Pawn->IsLocallyControlled()) return true;
+	}
 
 	// InteractionComponent on local player's pawn: it manages mic + routes audio
 	if (UWorld* World = GetWorld())
@@ -2026,18 +2171,36 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ApplyConversationGaze()
 	auto* Gaze = Owner->FindComponentByClass<UPS_AI_ConvAgent_GazeComponent>();
 	if (!Gaze) return;
 
-	if (bNetIsConversing && NetConversatingPawn)
+	if (bNetIsConversing && NetConnectedPawns.Num() > 0)
 	{
 		Gaze->bActive = true;
-		Gaze->TargetActor = NetConversatingPawn;
+		// Look at active speaker if set, otherwise last connected pawn.
-		Gaze->ResetBodyTarget();
+		AActor* GazeTarget = NetActiveSpeakerPawn
+			? static_cast<AActor*>(NetActiveSpeakerPawn)
+			: static_cast<AActor*>(NetConnectedPawns.Last());
+		if (Gaze->TargetActor != GazeTarget)
+		{
+			Gaze->TargetActor = GazeTarget;
+			Gaze->ResetBodyTarget();
+		}
 		Gaze->bEnableBodyTracking = true;
 	}
 	else
 	{
-		// Only clear if the gaze is still pointing at the departing player.
+		// Only clear if the gaze is not pointing at an unrelated target.
-		// Another InteractionComponent may have already set a new TargetActor.
+		// An InteractionComponent may have already set a new TargetActor.
-		if (!Gaze->TargetActor || Gaze->TargetActor == NetConversatingPawn)
+		bool bShouldClear = !Gaze->TargetActor;
+		if (!bShouldClear)
+		{
+			// Clear if the target is one of our (ex-)connected pawns or null.
+			for (APawn* Pawn : NetConnectedPawns)
+			{
+				if (Gaze->TargetActor == Pawn) { bShouldClear = true; break; }
+			}
+			// Also clear if the target was the active speaker.
+			if (Gaze->TargetActor == NetActiveSpeakerPawn) bShouldClear = true;
+		}
+		if (bShouldClear)
 		{
 			Gaze->bActive = false;
 			Gaze->TargetActor = nullptr;
@@ -2046,6 +2209,28 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ApplyConversationGaze()
 	}
 }
 
+void UPS_AI_ConvAgent_ElevenLabsComponent::SetActiveSpeaker(APawn* NewSpeaker)
+{
+	if (NetActiveSpeakerPawn == NewSpeaker) return;
+
+	APawn* Previous = NetActiveSpeakerPawn;
+	NetActiveSpeakerPawn = NewSpeaker;
+
+	// Update gaze on server (OnRep_ActiveSpeaker fires on clients).
+	ApplyConversationGaze();
+
+	// Broadcast locally (server).
+	OnActiveSpeakerChanged.Broadcast(NewSpeaker, Previous);
+
+	if (bDebug)
+	{
+		UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log,
+			TEXT("[NET] ActiveSpeaker changed: %s → %s"),
+			Previous ? *Previous->GetName() : TEXT("(none)"),
+			NewSpeaker ? *NewSpeaker->GetName() : TEXT("(none)"));
+	}
+}
+
 // ─────────────────────────────────────────────────────────────────────────────
 // On-screen debug display
 // ─────────────────────────────────────────────────────────────────────────────
@@ -2112,8 +2297,18 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::DrawDebugHUD() const
 		FString::Printf(TEXT("  Timing: session=%.1fs  turn=%.1fs"),
 			SessionSec, TurnSec));
 
+	// Multi-player
+	{
+		FString SpeakerName = NetActiveSpeakerPawn
+			? NetActiveSpeakerPawn->GetName()
+			: TEXT("none");
+		GEngine->AddOnScreenDebugMessage(BaseKey + 7, DisplayTime, MainColor,
+			FString::Printf(TEXT("  Players: %d  Speaker: %s"),
+				NetConnectedPawns.Num(), *SpeakerName));
+	}
+
 	// Reconnection
-	GEngine->AddOnScreenDebugMessage(BaseKey + 7, DisplayTime,
+	GEngine->AddOnScreenDebugMessage(BaseKey + 8, DisplayTime,
 		bWantsReconnect ? FColor::Red : MainColor,
 		FString::Printf(TEXT("  Reconnect: %d/%d attempts%s"),
 			ReconnectAttemptCount, MaxReconnectAttempts,

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp

@@ -209,28 +209,13 @@ UPS_AI_ConvAgent_ElevenLabsComponent* UPS_AI_ConvAgent_InteractionComponent::Eva
 		}
 	}
 
-	// Get local player's pawn for occupied-NPC check.
-	// Use pawn (replicated to ALL clients) instead of PlayerController
-	// (only replicated to owning client due to bOnlyRelevantToOwner=true).
-	APlayerController* LocalPC = World->GetFirstPlayerController();
-	APawn* LocalPawn = LocalPC ? LocalPC->GetPawn() : nullptr;
-
 	for (UPS_AI_ConvAgent_ElevenLabsComponent* Agent : Agents)
 	{
 		AActor* AgentActor = Agent->GetOwner();
 		if (!AgentActor) continue;
 
-		// Network: skip agents that are in conversation with a different player.
+		// Multi-player: do NOT skip occupied agents. Any agent in range can be
-		// Use NetConversatingPawn (replicated to all) instead of NetConversatingPlayer
+		// selected — the player will join the shared conversation via ServerJoinConversation.
-		// (NULL on remote clients because APlayerController has bOnlyRelevantToOwner=true).
-		// Null-check NetConversatingPawn: it may not have replicated yet when
-		// bNetIsConversing arrives first (OnRep ordering is not guaranteed).
-		if (Agent->bNetIsConversing
-			&& Agent->NetConversatingPawn
-			&& Agent->NetConversatingPawn != LocalPawn)
-		{
-			continue;
-		}
 
 		const FVector AgentLocation = AgentActor->GetActorLocation() + FVector(0.0f, 0.0f, AgentEyeLevelOffset);
 		const FVector ToAgent = AgentLocation - ViewLocation;
@@ -293,13 +278,26 @@ void UPS_AI_ConvAgent_InteractionComponent::SetSelectedAgent(UPS_AI_ConvAgent_El
 				OldAgent->GetOwner() ? *OldAgent->GetOwner()->GetName() : TEXT("(null)"));
 		}
 
-		// ── Conversation: end if auto-started ────────────────────────────
+		// ── Conversation: leave shared conversation if auto-started ─────
-		// If we auto-started the conversation on selection, end it now so the
+		// Use Leave instead of End so other players can keep talking to the agent.
-		// NPC becomes available for other players. EndConversation() also calls
-		// StopListening() internally, so we skip the separate StopListening below.
 		if (bAutoStartConversation && (OldAgent->IsConnected() || OldAgent->bNetIsConversing))
 		{
-			OldAgent->EndConversation();
+			if (GetOwnerRole() == ROLE_Authority || (GetWorld() && GetWorld()->GetNetMode() == NM_Standalone))
+			{
+				APlayerController* PC = nullptr;
+				if (APawn* Pawn = Cast<APawn>(GetOwner()))
+				{
+					PC = Cast<APlayerController>(Pawn->GetController());
+				}
+				if (PC)
+				{
+					OldAgent->ServerLeaveConversation_Implementation(PC);
+				}
+			}
+			else
+			{
+				ServerRelayLeaveConversation(OldAgent->GetOwner());
+			}
 		}
 		else if (bAutoManageListening)
 		{
@@ -355,24 +353,32 @@ void UPS_AI_ConvAgent_InteractionComponent::SetSelectedAgent(UPS_AI_ConvAgent_El
 				NewAgent->GetOwner() ? *NewAgent->GetOwner()->GetName() : TEXT("(null)"));
 		}
 
-		// Network: auto-start conversation if no active conversation.
+		// Multi-player: join the shared conversation (idempotent — no-op if already joined).
 		// In persistent session mode, the WebSocket stays connected but
 		// bNetIsConversing is false between interactions — we still need
-		// to call StartConversation() to re-activate gaze and mic.
+		// to call Join to re-activate gaze and mic.
 		// Only when bAutoStartConversation is true — otherwise the user must
 		// call StartConversationWithSelectedAgent() explicitly (e.g. on key press).
-		if (bAutoStartConversation && !NewAgent->bNetIsConversing)
+		if (bAutoStartConversation)
 		{
-			// On the server, call StartConversation() directly.
+			// On the server, call directly.
 			// On clients, route through our relay RPC (clients can't call
 			// Server RPCs on NPC actors — no owning connection).
 			if (GetOwnerRole() == ROLE_Authority || (GetWorld() && GetWorld()->GetNetMode() == NM_Standalone))
 			{
-				NewAgent->StartConversation();
+				APlayerController* PC = nullptr;
+				if (APawn* Pawn = Cast<APawn>(GetOwner()))
+				{
+					PC = Cast<APlayerController>(Pawn->GetController());
+				}
+				if (PC)
+				{
+					NewAgent->ServerJoinConversation_Implementation(PC);
+				}
 			}
 			else
 			{
-				ServerRelayStartConversation(NewAgent->GetOwner());
+				ServerRelayJoinConversation(NewAgent->GetOwner());
 			}
 
 			// Ensure mic is capturing so we can route audio to the new agent.
@@ -726,7 +732,7 @@ void UPS_AI_ConvAgent_InteractionComponent::GetLifetimeReplicatedProps(
 // the player's pawn (owned by the client) and forward to the NPC on the server.
 // ─────────────────────────────────────────────────────────────────────────────
 
-void UPS_AI_ConvAgent_InteractionComponent::ServerRelayStartConversation_Implementation(
+void UPS_AI_ConvAgent_InteractionComponent::ServerRelayJoinConversation_Implementation(
 	AActor* AgentActor)
 {
 	if (!AgentActor) return;
@@ -739,21 +745,39 @@ void UPS_AI_ConvAgent_InteractionComponent::ServerRelayStartConversation_Impleme
 	{
 		PC = Cast<APlayerController>(Pawn->GetController());
 	}
-
 	if (!PC) return;
 
-	// Forward to the NPC's implementation directly (we're already on the server).
+	Agent->ServerJoinConversation_Implementation(PC);
-	Agent->ServerRequestConversation_Implementation(PC);
 }
 
-void UPS_AI_ConvAgent_InteractionComponent::ServerRelayEndConversation_Implementation(
+void UPS_AI_ConvAgent_InteractionComponent::ServerRelayLeaveConversation_Implementation(
 	AActor* AgentActor)
 {
 	if (!AgentActor) return;
 	auto* Agent = AgentActor->FindComponentByClass<UPS_AI_ConvAgent_ElevenLabsComponent>();
 	if (!Agent) return;
 
-	Agent->ServerReleaseConversation_Implementation();
+	APlayerController* PC = nullptr;
+	if (APawn* Pawn = Cast<APawn>(GetOwner()))
+	{
+		PC = Cast<APlayerController>(Pawn->GetController());
+	}
+	if (!PC) return;
+
+	Agent->ServerLeaveConversation_Implementation(PC);
+}
+
+// Backward compat wrappers.
+void UPS_AI_ConvAgent_InteractionComponent::ServerRelayStartConversation_Implementation(
+	AActor* AgentActor)
+{
+	ServerRelayJoinConversation_Implementation(AgentActor);
+}
+
+void UPS_AI_ConvAgent_InteractionComponent::ServerRelayEndConversation_Implementation(
+	AActor* AgentActor)
+{
+	ServerRelayLeaveConversation_Implementation(AgentActor);
 }
 
 void UPS_AI_ConvAgent_InteractionComponent::ServerRelayMicAudio_Implementation(
@@ -767,15 +791,11 @@ void UPS_AI_ConvAgent_InteractionComponent::ServerRelayMicAudio_Implementation(
 	// instead of 3200 bytes per 100ms chunk). Decode back to raw PCM here
 	// before forwarding to the WebSocket which expects uncompressed int16.
 	TArray<uint8> DecodedPCM;
-	if (Agent->DecompressMicAudio(AudioBytes, DecodedPCM))
+	const TArray<uint8>& PCMToSend = Agent->DecompressMicAudio(AudioBytes, DecodedPCM) ? DecodedPCM : AudioBytes;
-	{
+
-		Agent->ServerSendMicAudio_Implementation(DecodedPCM);
+	// Pass the sender pawn for multi-player speaker arbitration.
-	}
+	APawn* SenderPawn = Cast<APawn>(GetOwner());
-	else
+	Agent->ServerSendMicAudioFromPlayer(SenderPawn, PCMToSend);
-	{
-		// Raw PCM fallback (no Opus or data is already uncompressed).
-		Agent->ServerSendMicAudio_Implementation(AudioBytes);
-	}
 }
 
 void UPS_AI_ConvAgent_InteractionComponent::ServerRelaySendText_Implementation(

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h

@@ -91,6 +91,13 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnAgentEmotionChanged,
 DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnAgentClientToolCall,
 	const FPS_AI_ConvAgent_ClientToolCall_ElevenLabs&, ToolCall);
 
+/**
+ * Fired when the active speaker changes in a multi-player shared conversation.
+ * Use this for UI indicators showing who is talking, or to drive camera focus.
+ */
+DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnActiveSpeakerChanged,
+	APawn*, NewSpeaker, APawn*, PreviousSpeaker);
+
 // Non-dynamic delegate for raw agent audio (high-frequency, C++ consumers only).
 // Delivers PCM chunks as int16, 16kHz mono, little-endian.
 DECLARE_MULTICAST_DELEGATE_OneParam(FOnAgentAudioData, const TArray<uint8>& /*PCMData*/);
@@ -298,6 +305,12 @@ public:
 		meta = (ToolTip = "Fires for custom client tool calls (not set_emotion).\nYou must respond via GetWebSocketProxy()->SendClientToolResult()."))
 	FOnAgentClientToolCall OnAgentClientToolCall;
 
+	/** Fired when the active speaker changes in a multi-player shared conversation.
+	 *  NewSpeaker is null when no one is speaking (idle timeout). */
+	UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|ElevenLabs|Events",
+		meta = (ToolTip = "Fires when the speaking player changes.\nNewSpeaker is null when no one is speaking."))
+	FOnActiveSpeakerChanged OnActiveSpeakerChanged;
+
 	/** The current emotion of the agent, as set by the "set_emotion" client tool. Defaults to Neutral. */
 	UPROPERTY(ReplicatedUsing = OnRep_Emotion, BlueprintReadOnly, Category = "PS AI ConvAgent|ElevenLabs")
 	EPS_AI_ConvAgent_Emotion CurrentEmotion = EPS_AI_ConvAgent_Emotion::Neutral;
@@ -313,20 +326,39 @@ public:
 
 	// ── Network state (replicated) ───────────────────────────────────────────
 
-	/** True when a player is currently in conversation with this NPC.
+	/** True when one or more players are in conversation with this NPC.
-	 *  Replicated to all clients so InteractionComponents can skip occupied NPCs. */
+	 *  Replicated to all clients for UI feedback, gaze, LOD, etc. */
 	UPROPERTY(ReplicatedUsing = OnRep_ConversationState, BlueprintReadOnly, Category = "PS AI ConvAgent|Network")
 	bool bNetIsConversing = false;
 
-	/** The player controller currently in conversation with this NPC (null if free).
+	/** All player pawns currently in conversation with this NPC.
-	 *  Only valid on server and owning client (PlayerControllers are not replicated to other clients). */
+	 *  Multiple players can share the same agent — audio is routed via speaker arbitration.
+	 *  Replicated to ALL clients for gaze target and LOD distance checks. */
 	UPROPERTY(ReplicatedUsing = OnRep_ConversationState, BlueprintReadOnly, Category = "PS AI ConvAgent|Network")
-	TObjectPtr<APlayerController> NetConversatingPlayer = nullptr;
+	TArray<TObjectPtr<APawn>> NetConnectedPawns;
 
-	/** The pawn of the conversating player. Replicated to ALL clients (unlike PlayerController).
+	/** The player currently speaking (active audio sender). Null if no one is speaking.
-	 *  Used by remote clients for gaze target (head/eye tracking) and LOD distance checks. */
+	 *  Used by GazeComponent for target switching — the NPC looks at whoever is talking.
-	UPROPERTY(ReplicatedUsing = OnRep_ConversationState, BlueprintReadOnly, Category = "PS AI ConvAgent|Network")
+	 *  Replicated to ALL clients so gaze updates everywhere. */
-	TObjectPtr<APawn> NetConversatingPawn = nullptr;
+	UPROPERTY(ReplicatedUsing = OnRep_ActiveSpeaker, BlueprintReadOnly, Category = "PS AI ConvAgent|Network")
+	TObjectPtr<APawn> NetActiveSpeakerPawn = nullptr;
+
+	// ── Multi-player speaker arbitration (server only) ──────────────────────
+
+	/** Minimum seconds of silence from the current speaker before allowing a speaker switch.
+	 *  Prevents rapid ping-pong switching when both players make brief sounds. */
+	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Network|MultiPlayer",
+		meta = (ClampMin = "0.0", ClampMax = "5.0",
+		ToolTip = "Minimum silence from current speaker before switching to another.\nPrevents rapid gaze flip-flop."))
+	float SpeakerSwitchHysteresis = 0.3f;
+
+	/** Seconds after last speech before the active speaker is cleared.
+	 *  When cleared, gaze returns to the last speaker position (or closest connected player).
+	 *  Set to 0 to never clear (last speaker stays active indefinitely). */
+	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Network|MultiPlayer",
+		meta = (ClampMin = "0.0",
+		ToolTip = "Seconds of silence before clearing the active speaker.\n0 = last speaker stays active indefinitely."))
+	float SpeakerIdleTimeout = 3.0f;
 
 	// ── Network LOD ──────────────────────────────────────────────────────────
 
@@ -345,12 +377,21 @@ public:
 
 	// ── Network RPCs ─────────────────────────────────────────────────────────
 
-	/** Request exclusive conversation with this NPC. Called by clients; the server
+	/** Join a shared conversation with this NPC. Multiple players can join.
-	 *  checks availability and opens the WebSocket connection if the NPC is free. */
+	 *  If this is the first player, the WebSocket connection is opened.
+	 *  If the agent is already in conversation, the player simply joins. */
+	UFUNCTION(Server, Reliable)
+	void ServerJoinConversation(APlayerController* RequestingPlayer);
+
+	/** Leave the shared conversation. If this is the last player, the conversation ends. */
+	UFUNCTION(Server, Reliable)
+	void ServerLeaveConversation(APlayerController* LeavingPlayer);
+
+	/** [Backward compat] Delegates to ServerJoinConversation. */
 	UFUNCTION(Server, Reliable)
 	void ServerRequestConversation(APlayerController* RequestingPlayer);
 
-	/** Release this NPC so other players can talk to it. */
+	/** [Backward compat] Delegates to ServerLeaveConversation. */
 	UFUNCTION(Server, Reliable)
 	void ServerReleaseConversation();
 
@@ -452,6 +493,10 @@ public:
 	UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|ElevenLabs")
 	void FeedExternalAudio(const TArray<float>& FloatPCM);
 
+	/** Receive mic audio from a specific player with speaker arbitration.
+	 *  Server decides whether to forward to ElevenLabs based on who is currently speaking. */
+	void ServerSendMicAudioFromPlayer(APawn* SpeakerPawn, const TArray<uint8>& PCMBytes);
+
 	// ── State queries ─────────────────────────────────────────────────────────
 
 	UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|ElevenLabs")
@@ -499,6 +544,9 @@ private:
 	UFUNCTION()
 	void OnRep_ConversationState();
 
+	UFUNCTION()
+	void OnRep_ActiveSpeaker();
+
 	UFUNCTION()
 	void OnRep_Emotion();
 
@@ -659,7 +707,7 @@ private:
 	// ── Network helpers ──────────────────────────────────────────────────────
 	/** Distance from this NPC to the local player's pawn. Returns MAX_FLT if unavailable. */
 	float GetDistanceToLocalPlayer() const;
-	/** True if the local player controller is the one currently in conversation. */
+	/** True if the local player controller is one of the connected players. */
 	bool IsLocalPlayerConversating() const;
 	/** Internal: performs the actual WebSocket setup (called by both local and RPC paths). */
 	void StartConversation_Internal();
@@ -672,10 +720,18 @@ private:
 	bool ShouldUseExternalMic() const;
 
 	/** Update the NPC's GazeComponent from the current conversation state.
-	 *  Called on the server when bNetIsConversing / NetConversatingPawn change,
+	 *  Uses NetActiveSpeakerPawn if set, otherwise the first connected pawn.
-	 *  because OnRep_ConversationState never fires on the Authority. */
+	 *  Called on the server (OnRep never fires on the Authority). */
 	void ApplyConversationGaze();
 
+	// ── Multi-player speaker arbitration (server only) ──────────────────────
+
+	/** Set the active speaker and update gaze. Skips if same as current. */
+	void SetActiveSpeaker(APawn* NewSpeaker);
+
+	/** Last non-silent audio timestamp per connected player (server only). */
+	TMap<TObjectPtr<APawn>, double> LastSpeakTime;
+
 	/** Draw on-screen debug info (called from TickComponent when bDebug). */
 	void DrawDebugHUD() const;
 };

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_InteractionComponent.h

@@ -201,11 +201,21 @@ public:
 	// These relay RPCs live on the player's pawn (which IS owned by the client),
 	// forwarding commands to the NPC's ElevenLabsComponent on the server.
 
-	/** Relay: request exclusive conversation with an NPC agent. */
+	/** Relay: join a shared conversation with an NPC agent. */
+	UFUNCTION(Server, Reliable)
+	void ServerRelayJoinConversation(AActor* AgentActor);
+
+	/** Relay: leave a shared conversation with an NPC agent. */
+	UFUNCTION(Server, Reliable)
+	void ServerRelayLeaveConversation(AActor* AgentActor);
+
+	/** Relay: [backward compat] request conversation with an NPC agent.
+	 *  Delegates to ServerRelayJoinConversation. */
 	UFUNCTION(Server, Reliable)
 	void ServerRelayStartConversation(AActor* AgentActor);
 
-	/** Relay: release conversation with an NPC agent. */
+	/** Relay: [backward compat] release conversation with an NPC agent.
+	 *  Delegates to ServerRelayLeaveConversation. */
 	UFUNCTION(Server, Reliable)
 	void ServerRelayEndConversation(AActor* AgentActor);