diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Source/PS_AI_Behavior/Private/PS_AI_Behavior_AIController.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Source/PS_AI_Behavior/Private/PS_AI_Behavior_AIController.cpp index 1648f2b..6e1e606 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Source/PS_AI_Behavior/Private/PS_AI_Behavior_AIController.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Source/PS_AI_Behavior/Private/PS_AI_Behavior_AIController.cpp @@ -730,6 +730,11 @@ void APS_AI_Behavior_AIController::TryBindGazeComponent() CachedConvAgentComponent = ConvComp; ConvProp_bNetIsConversing = CastField( ConvClass->FindPropertyByName(TEXT("bNetIsConversing"))); + // Agent-to-agent dialogue: the orchestrator pins this agent's gaze on + // the other agent via GazeOverrideTarget. Cache it so we can treat that + // as an active conversation (see IsConversationActiveOnPawn). + ConvProp_GazeOverrideTarget = CastField( + ConvClass->FindPropertyByName(TEXT("GazeOverrideTarget"))); } } @@ -797,9 +802,26 @@ void APS_AI_Behavior_AIController::ClearGazeTarget() bool APS_AI_Behavior_AIController::IsConversationActiveOnPawn() const { UActorComponent* ConvComp = CachedConvAgentComponent.Get(); - if (!ConvComp || !ConvProp_bNetIsConversing) return false; + if (!ConvComp) return false; - return ConvProp_bNetIsConversing->GetPropertyValue_InContainer(ConvComp); + // Player-driven conversation (a human joined this agent). + if (ConvProp_bNetIsConversing && + ConvProp_bNetIsConversing->GetPropertyValue_InContainer(ConvComp)) + { + return true; + } + + // Agent-to-agent dialogue: the orchestrator sets GazeOverrideTarget on this + // agent. Treat it as an active conversation so proximity gaze / wandering back + // off and the ConvAgent owns the gaze (agents look at each other, not a passing + // player). Clearing the override (StopDialogue) restores normal behavior. + if (ConvProp_GazeOverrideTarget && + ConvProp_GazeOverrideTarget->GetObjectPropertyValue_InContainer(ConvComp)) + { + return true; + } + + return false; } void APS_AI_Behavior_AIController::ApplyMovementPaused(bool bPaused) diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Source/PS_AI_Behavior/Public/PS_AI_Behavior_AIController.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Source/PS_AI_Behavior/Public/PS_AI_Behavior_AIController.h index 550be9c..b3b35c4 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Source/PS_AI_Behavior/Public/PS_AI_Behavior_AIController.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Behavior/Source/PS_AI_Behavior/Public/PS_AI_Behavior_AIController.h @@ -206,6 +206,8 @@ private: // Conversation detection cache TWeakObjectPtr CachedConvAgentComponent; FBoolProperty* ConvProp_bNetIsConversing = nullptr; + // Set by the ConvAgent agent-to-agent orchestrator while two agents converse. + FObjectProperty* ConvProp_GazeOverrideTarget = nullptr; // Mic VAD cache (local voice activity detection) TWeakObjectPtr CachedMicComponent; diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_AgentDialogue.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_AgentDialogue.cpp new file mode 100644 index 0000000..7d3102e --- /dev/null +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_AgentDialogue.cpp @@ -0,0 +1,553 @@ +// Copyright ASTERION. All Rights Reserved. + +#include "PS_AI_ConvAgent_AgentDialogue.h" +#include "PS_AI_ConvAgent_ElevenLabsComponent.h" +#include "GameFramework/Actor.h" +#include "GameFramework/Pawn.h" +#include "GameFramework/PlayerController.h" +#include "TimerManager.h" +#include "Engine/World.h" + +DEFINE_LOG_CATEGORY_STATIC(LogPS_AI_ConvAgent_Dialogue, Log, All); + +static const TCHAR* FloorName(uint8 F) +{ + switch (F) { case 1: return TEXT("A"); case 2: return TEXT("B"); default: return TEXT("None"); } +} + +APS_AI_ConvAgent_AgentDialogue::APS_AI_ConvAgent_AgentDialogue() +{ + // Ticks only while "addressing player" (auto-release watcher); disabled otherwise. + PrimaryActorTick.bCanEverTick = true; + PrimaryActorTick.bStartWithTickEnabled = false; + bReplicates = false; // server-side orchestration; agents replicate their own audio +} + +void APS_AI_ConvAgent_AgentDialogue::BeginPlay() +{ + Super::BeginPlay(); + if (bAutoStartOnBeginPlay && HasAuthority()) + { + StartDialogue(); + } +} + +void APS_AI_ConvAgent_AgentDialogue::EndPlay(const EEndPlayReason::Type Reason) +{ + StopDialogue(false); + UnbindAll(); + Super::EndPlay(Reason); +} + +UPS_AI_ConvAgent_ElevenLabsComponent* APS_AI_ConvAgent_AgentDialogue::ResolveAgentComp(AActor* Actor) const +{ + return Actor ? Actor->FindComponentByClass() : nullptr; +} + +void APS_AI_ConvAgent_AgentDialogue::StartDialogue() +{ + if (!HasAuthority()) + { + UE_LOG(LogPS_AI_ConvAgent_Dialogue, Warning, + TEXT("StartDialogue ignored — must run on the server (authority).")); + return; + } + if (bRunning) + { + UE_LOG(LogPS_AI_ConvAgent_Dialogue, Log, TEXT("StartDialogue: already running.")); + return; + } + + CompA = ResolveAgentComp(AgentA); + CompB = ResolveAgentComp(AgentB); + if (!CompA || !CompB) + { + UE_LOG(LogPS_AI_ConvAgent_Dialogue, Warning, + TEXT("StartDialogue: no ElevenLabs component on %s and/or %s."), + *GetNameSafe(AgentA), *GetNameSafe(AgentB)); + return; + } + if (CompA == CompB) + { + UE_LOG(LogPS_AI_ConvAgent_Dialogue, Warning, TEXT("StartDialogue: AgentA and AgentB are the same agent.")); + return; + } + + UnbindAll(); + BindAgent(CompA, /*bIsA=*/true); + BindAgent(CompB, /*bIsA=*/false); + + // CRITICAL: this is a TEXT-only loop. The agents must NEVER open their mics, + // otherwise each one HEARS the other's TTS and self-replies (via STT) — that + // produces the overlap / "already has the answer before the other spoke" chaos. + CompA->bAutoStartListening = false; + CompB->bAutoStartListening = false; + + // The dialogue is interrupted when the PLAYER speaks — detected via a local mic + // voice-onset (primary) and OnAgentTranscript (backup). Enable transcript, and + // force Server turn mode + interruption so per-agent editor config can't silently + // block the mic path / the player barging in. + CompA->bEnableUserTranscript = true; + CompB->bEnableUserTranscript = true; + CompA->TurnMode = EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server; + CompB->TurnMode = EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server; + CompA->bAllowInterruption = true; + CompB->bAllowInterruption = true; + + // Make the two agents look at EACH OTHER (not at a bystanding player). + // Replicated, so the audience sees it correctly on every machine. + CompA->SetGazeOverrideTarget(AgentB); + CompB->SetGazeOverrideTarget(AgentA); + + UE_LOG(LogPS_AI_ConvAgent_Dialogue, Warning, + TEXT("[Dialogue] Reminder: both agents must have an EMPTY FirstMessage in their AgentConfig " + "(a configured first message auto-greets on connect and collides with the opening).")); + + // Reset state. + Floor = EFloor::None; + TurnId = 0; + bTextReady = false; + bSpeechDone = false; + bForwardedThisTurn = false; + PendingLine.Reset(); + bOpeningSent = false; + bRunning = true; + + CompA->StartConversation(); + CompB->StartConversation(); + + TryKickoff(); + + UE_LOG(LogPS_AI_ConvAgent_Dialogue, Log, + TEXT("[Dialogue] started: %s <-> %s (starter=%s, maxTurns=%d, waitForSpeech=%d)."), + *GetNameSafe(AgentA), *GetNameSafe(AgentB), + bAgentAStarts ? *GetNameSafe(AgentA) : *GetNameSafe(AgentB), + MaxTurns, bWaitForSpeechToFinish ? 1 : 0); +} + +void APS_AI_ConvAgent_AgentDialogue::StopDialogue(bool bEndConversations) +{ + if (!bRunning && !bAddressingPlayer) { return; } + bRunning = false; + bAddressingPlayer = false; + AddressedByPlayer = nullptr; + EngagedFloor = EFloor::None; + SetActorTickEnabled(false); + + if (UWorld* W = GetWorld()) + { + W->GetTimerManager().ClearTimer(TurnTimer); + } + // Restore normal (player-tracking) gaze. + if (CompA) { CompA->SetGazeOverrideTarget(nullptr); } + if (CompB) { CompB->SetGazeOverrideTarget(nullptr); } + + if (bEndConversations) + { + if (CompA) { CompA->EndConversation(); } + if (CompB) { CompB->EndConversation(); } + } + UnbindAll(); + + UE_LOG(LogPS_AI_ConvAgent_Dialogue, Log, TEXT("[Dialogue] stopped after %d turns."), TurnId); + OnDialogueFinished.Broadcast(TurnId); +} + +void APS_AI_ConvAgent_AgentDialogue::BindAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Comp, bool bIsA) +{ + if (!Comp) { return; } + if (bIsA) + { + Comp->OnAgentTextResponse.AddDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleTextA); + Comp->OnAgentStoppedSpeaking.AddDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleStoppedA); + Comp->OnAgentConnected.AddDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleConnectedA); + // Per-agent interrupt/relay triggers (voice-onset = primary, transcript = words). + Comp->OnUserVoiceDetected.AddDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleVoiceOnsetA); + Comp->OnAgentTranscript.AddDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleTranscriptA); + } + else + { + Comp->OnAgentTextResponse.AddDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleTextB); + Comp->OnAgentStoppedSpeaking.AddDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleStoppedB); + Comp->OnAgentConnected.AddDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleConnectedB); + Comp->OnUserVoiceDetected.AddDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleVoiceOnsetB); + Comp->OnAgentTranscript.AddDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleTranscriptB); + } +} + +void APS_AI_ConvAgent_AgentDialogue::UnbindAll() +{ + if (CompA) + { + CompA->OnAgentTextResponse.RemoveDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleTextA); + CompA->OnAgentStoppedSpeaking.RemoveDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleStoppedA); + CompA->OnAgentConnected.RemoveDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleConnectedA); + CompA->OnAgentTranscript.RemoveDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleTranscriptA); + CompA->OnUserVoiceDetected.RemoveDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleVoiceOnsetA); + } + if (CompB) + { + CompB->OnAgentTextResponse.RemoveDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleTextB); + CompB->OnAgentStoppedSpeaking.RemoveDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleStoppedB); + CompB->OnAgentConnected.RemoveDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleConnectedB); + CompB->OnAgentTranscript.RemoveDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleTranscriptB); + CompB->OnUserVoiceDetected.RemoveDynamic(this, &APS_AI_ConvAgent_AgentDialogue::HandleVoiceOnsetB); + } +} + +void APS_AI_ConvAgent_AgentDialogue::HandleVoiceOnsetA() { OnPlayerSpoke(EFloor::A, FString()); } +void APS_AI_ConvAgent_AgentDialogue::HandleVoiceOnsetB() { OnPlayerSpoke(EFloor::B, FString()); } + +void APS_AI_ConvAgent_AgentDialogue::HandleTranscriptA(const FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs& Segment) +{ + if (Segment.Speaker == TEXT("user")) { OnPlayerSpoke(EFloor::A, Segment.Text); } +} + +void APS_AI_ConvAgent_AgentDialogue::HandleTranscriptB(const FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs& Segment) +{ + if (Segment.Speaker == TEXT("user")) { OnPlayerSpoke(EFloor::B, Segment.Text); } +} + +void APS_AI_ConvAgent_AgentDialogue::OnPlayerSpoke(EFloor Which, const FString& PlayerWords) +{ + if (bRunning) + { + // The player barged into the agent-to-agent dialogue: enter 3-way floor mode. + InterruptForPlayer(Which); + } + else if (bAddressingPlayer) + { + // Already talking to the player — did they turn to the OTHER agent? + if (Which != EngagedFloor) { SetEngaged(Which); } + } + else + { + return; // orchestrator idle + } + + // Relay the player's actual words (transcript path) to the WITNESS agent, so it + // "overhears" the exchange and can reference it later. Only while in floor mode. + if (bAddressingPlayer && !PlayerWords.IsEmpty() && !OverheardPrefix.IsEmpty()) + { + RelayToBystander(Which, OverheardPrefix + TEXT("(la personne) ") + PlayerWords); + } +} + +void APS_AI_ConvAgent_AgentDialogue::InterruptForPlayer(EFloor Engaged) +{ + APawn* Player = ResolveSpeakingPlayer(); + if (!Player) + { + // Couldn't identify the speaker — fall back to a plain stop. + StopDialogue(/*bEndConversations=*/false); + return; + } + + // Enter 3-way "floor" mode: stop the agent↔agent forwarding, turn BOTH agents to the + // player, and keep relaying the exchange to the witness agent. Agents stay connected; + // the orchestrator ticks to auto-release once the player leaves both (see Tick). + UE_LOG(LogPS_AI_ConvAgent_Dialogue, Log, + TEXT("[Dialogue] player '%s' spoke to %s — entering 3-way floor mode."), + *GetNameSafe(Player), FloorName((uint8)Engaged)); + + bRunning = false; + if (UWorld* W = GetWorld()) + { + W->GetTimerManager().ClearTimer(TurnTimer); + } + // Keep the agent bindings: in floor mode the text/transcript handlers RELAY to the + // witness instead of forwarding (their dialogue logic no-ops while !bRunning). + + if (CompA) { CompA->SetGazeOverrideTarget(Player); } + if (CompB) { CompB->SetGazeOverrideTarget(Player); } + + bAddressingPlayer = true; + AddressedByPlayer = Player; + EngagedFloor = EFloor::None; // force SetEngaged to send the reframe messages + SetEngaged(Engaged); + SetActorTickEnabled(true); +} + +void APS_AI_ConvAgent_AgentDialogue::SetEngaged(EFloor Which) +{ + if (Which == EFloor::None || Which == EngagedFloor) { return; } + EngagedFloor = Which; + + UPS_AI_ConvAgent_ElevenLabsComponent* Engaged = (Which == EFloor::A) ? CompA.Get() : CompB.Get(); + UPS_AI_ConvAgent_ElevenLabsComponent* Bystander = (Which == EFloor::A) ? CompB.Get() : CompA.Get(); + + UE_LOG(LogPS_AI_ConvAgent_Dialogue, Log, + TEXT("[Dialogue] player now addressing %s (the other agent becomes witness)."), + FloorName((uint8)Which)); + + // Silent reframe (contextual_update = no reply triggered). + if (Engaged && !InterruptContextMessage.IsEmpty()) + { + Engaged->SendContextualUpdate(InterruptContextMessage); + } + if (Bystander && !BystanderContextMessage.IsEmpty()) + { + Bystander->SendContextualUpdate(BystanderContextMessage); + } +} + +void APS_AI_ConvAgent_AgentDialogue::RelayToBystander(EFloor EngagedSide, const FString& FormattedLine) +{ + UPS_AI_ConvAgent_ElevenLabsComponent* Bystander = (EngagedSide == EFloor::A) ? CompB.Get() : CompA.Get(); + if (Bystander) + { + Bystander->SendContextualUpdate(FormattedLine); + } +} + +void APS_AI_ConvAgent_AgentDialogue::Tick(float DeltaSeconds) +{ + Super::Tick(DeltaSeconds); + if (!bAddressingPlayer) { return; } + + APawn* Player = AddressedByPlayer.Get(); + if (!Player) + { + EndPlayerAddress(); + return; + } + + // Release once the player has walked farther than the threshold from BOTH agents. + const FVector P = Player->GetActorLocation(); + const float Threshold = PlayerAddressReleaseDistance; + const bool bFarFromA = !AgentA || FVector::Dist(P, AgentA->GetActorLocation()) > Threshold; + const bool bFarFromB = !AgentB || FVector::Dist(P, AgentB->GetActorLocation()) > Threshold; + if (bFarFromA && bFarFromB) + { + UE_LOG(LogPS_AI_ConvAgent_Dialogue, Log, + TEXT("[Dialogue] player left both agents — releasing.")); + EndPlayerAddress(); + } +} + +void APS_AI_ConvAgent_AgentDialogue::EndPlayerAddress() +{ + bAddressingPlayer = false; + AddressedByPlayer = nullptr; + EngagedFloor = EFloor::None; + SetActorTickEnabled(false); + + // Hand the agents back to normal gaze and end their (now idle) conversations. + if (CompA) { CompA->SetGazeOverrideTarget(nullptr); CompA->EndConversation(); } + if (CompB) { CompB->SetGazeOverrideTarget(nullptr); CompB->EndConversation(); } + UnbindAll(); // floor mode kept the bindings active — release them now. + + UE_LOG(LogPS_AI_ConvAgent_Dialogue, Log, + TEXT("[Dialogue] player address ended — agents released (after %d turns)."), TurnId); + OnDialogueFinished.Broadcast(TurnId); +} + +APawn* APS_AI_ConvAgent_AgentDialogue::ResolveSpeakingPlayer() const +{ + // The player who spoke is the active speaker on one of the agents (a pawn that is + // neither AgentA nor AgentB), else a connected non-agent pawn, else the local player. + AActor* A = AgentA.Get(); + AActor* B = AgentB.Get(); + auto IsPlayer = [A, B](APawn* P) -> bool { return P && P != A && P != B; }; + + if (CompA && IsPlayer(CompA->NetActiveSpeakerPawn)) { return CompA->NetActiveSpeakerPawn; } + if (CompB && IsPlayer(CompB->NetActiveSpeakerPawn)) { return CompB->NetActiveSpeakerPawn; } + + if (CompA) { for (APawn* P : CompA->NetConnectedPawns) { if (IsPlayer(P)) { return P; } } } + if (CompB) { for (APawn* P : CompB->NetConnectedPawns) { if (IsPlayer(P)) { return P; } } } + + // Voice-onset path: NetActiveSpeakerPawn is usually unset (the player never formally + // joined the agent). Pick the player pawn NEAREST to either agent. + UWorld* W = GetWorld(); + if (!W) { return nullptr; } + + APawn* Nearest = nullptr; + double BestDistSq = TNumericLimits::Max(); + for (FConstPlayerControllerIterator It = W->GetPlayerControllerIterator(); It; ++It) + { + APlayerController* PC = It->Get(); + APawn* P = PC ? PC->GetPawn() : nullptr; + if (!IsPlayer(P)) { continue; } + const FVector L = P->GetActorLocation(); + double D = TNumericLimits::Max(); + if (A) { D = FMath::Min(D, FVector::DistSquared(L, A->GetActorLocation())); } + if (B) { D = FMath::Min(D, FVector::DistSquared(L, B->GetActorLocation())); } + if (D < BestDistSq) { BestDistSq = D; Nearest = P; } + } + if (Nearest) { return Nearest; } + + if (APlayerController* PC = W->GetFirstPlayerController()) + { + return PC->GetPawn(); + } + return nullptr; +} + +void APS_AI_ConvAgent_AgentDialogue::BeginTurn(EFloor NewFloor) +{ + Floor = NewFloor; + bTextReady = false; + bSpeechDone = false; + bForwardedThisTurn = false; + PendingLine.Reset(); +} + +bool APS_AI_ConvAgent_AgentDialogue::IsFloor(bool bFromA) const +{ + return (bFromA && Floor == EFloor::A) || (!bFromA && Floor == EFloor::B); +} + +void APS_AI_ConvAgent_AgentDialogue::TryKickoff() +{ + if (!bRunning || bOpeningSent) { return; } + + const bool bBothReady = CompA && CompB && CompA->IsConnected() && CompB->IsConnected(); + if (!bBothReady) + { + UE_LOG(LogPS_AI_ConvAgent_Dialogue, Log, TEXT("[Dialogue] Kickoff waiting: A.connected=%d B.connected=%d"), + (CompA && CompA->IsConnected()) ? 1 : 0, (CompB && CompB->IsConnected()) ? 1 : 0); + return; // wait for the other OnAgentConnected + } + + bOpeningSent = true; + BeginTurn(bAgentAStarts ? EFloor::A : EFloor::B); + UPS_AI_ConvAgent_ElevenLabsComponent* Starter = (Floor == EFloor::A) ? CompA : CompB; + + UE_LOG(LogPS_AI_ConvAgent_Dialogue, Log, TEXT("[Dialogue] FLOOR=%s SEND-OPENING '%s'"), + FloorName((uint8)Floor), *OpeningMessage); + Starter->SendTextMessage(OpeningMessage); +} + +void APS_AI_ConvAgent_AgentDialogue::TryForward() +{ + if (!bRunning || bForwardedThisTurn) { return; } + + // Need the floor-holder's full text; and, unless we're on a headless server, + // also require it to have finished speaking so the voices never overlap. + if (!bTextReady) { return; } + if (bWaitForSpeechToFinish && !bSpeechDone) { return; } + + UPS_AI_ConvAgent_ElevenLabsComponent* Listener = (Floor == EFloor::A) ? CompB : CompA; + AActor* SpeakerActor = (Floor == EFloor::A) ? AgentA : AgentB; + if (!Listener) { return; } + + if (MaxTurns > 0 && TurnId >= MaxTurns) + { + StopDialogue(false); + return; + } + + bForwardedThisTurn = true; + ++TurnId; + const FString Line = PendingLine; + + UE_LOG(LogPS_AI_ConvAgent_Dialogue, Log, TEXT("[Dialogue] turn=%d FORWARD %s->%s : '%.60s'"), + TurnId, FloorName((uint8)Floor), FloorName((uint8)(Floor == EFloor::A ? EFloor::B : EFloor::A)), *Line); + OnDialogueTurn.Broadcast(TurnId, SpeakerActor, Line); + + // Hand the floor to the listener BEFORE sending, so the listener's upcoming + // text/speech events are attributed to it (and the old speaker's late/duplicate + // events are now non-floor and get ignored). + BeginTurn(Floor == EFloor::A ? EFloor::B : EFloor::A); + + auto DoSend = [this, Line, Listener]() + { + if (bRunning && Listener && Listener->IsConnected()) + { + Listener->SendTextMessage(Line); + } + }; + + if (TurnDelaySeconds > 0.0f) + { + if (UWorld* W = GetWorld()) + { + W->GetTimerManager().SetTimer(TurnTimer, FTimerDelegate::CreateLambda(DoSend), TurnDelaySeconds, false); + } + else { DoSend(); } + } + else { DoSend(); } +} + +// ── Delegate handlers ───────────────────────────────────────────────────────── +// Every handler ignores events from the agent that does NOT hold the floor. + +void APS_AI_ConvAgent_AgentDialogue::HandleTextA(const FString& Text) +{ + if (bAddressingPlayer) + { + // Floor mode: the ENGAGED agent's spoken line is overheard by the witness. + if (EngagedFloor == EFloor::A && !Text.IsEmpty() && !OverheardPrefix.IsEmpty()) + { + RelayToBystander(EFloor::A, OverheardPrefix + TEXT("(l'autre personnage) ") + Text); + } + return; + } + if (!bRunning) { return; } + if (!IsFloor(/*bFromA=*/true)) + { + UE_LOG(LogPS_AI_ConvAgent_Dialogue, Verbose, TEXT("[Dialogue] IGNORED text from NON-FLOOR A (floor=%s)"), FloorName((uint8)Floor)); + return; + } + if (Text.IsEmpty()) { return; } + PendingLine = Text; + bTextReady = true; + TryForward(); +} + +void APS_AI_ConvAgent_AgentDialogue::HandleTextB(const FString& Text) +{ + if (bAddressingPlayer) + { + // Floor mode: the ENGAGED agent's spoken line is overheard by the witness. + if (EngagedFloor == EFloor::B && !Text.IsEmpty() && !OverheardPrefix.IsEmpty()) + { + RelayToBystander(EFloor::B, OverheardPrefix + TEXT("(l'autre personnage) ") + Text); + } + return; + } + if (!bRunning) { return; } + if (!IsFloor(/*bFromA=*/false)) + { + UE_LOG(LogPS_AI_ConvAgent_Dialogue, Verbose, TEXT("[Dialogue] IGNORED text from NON-FLOOR B (floor=%s)"), FloorName((uint8)Floor)); + return; + } + if (Text.IsEmpty()) { return; } + PendingLine = Text; + bTextReady = true; + TryForward(); +} + +void APS_AI_ConvAgent_AgentDialogue::HandleStoppedA() +{ + if (!bRunning) { return; } + if (!IsFloor(/*bFromA=*/true)) + { + UE_LOG(LogPS_AI_ConvAgent_Dialogue, Verbose, TEXT("[Dialogue] IGNORED stopped from NON-FLOOR A (floor=%s)"), FloorName((uint8)Floor)); + return; + } + bSpeechDone = true; + TryForward(); +} + +void APS_AI_ConvAgent_AgentDialogue::HandleStoppedB() +{ + if (!bRunning) { return; } + if (!IsFloor(/*bFromA=*/false)) + { + UE_LOG(LogPS_AI_ConvAgent_Dialogue, Verbose, TEXT("[Dialogue] IGNORED stopped from NON-FLOOR B (floor=%s)"), FloorName((uint8)Floor)); + return; + } + bSpeechDone = true; + TryForward(); +} + +void APS_AI_ConvAgent_AgentDialogue::HandleConnectedA(const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& Info) +{ + TryKickoff(); +} + +void APS_AI_ConvAgent_AgentDialogue::HandleConnectedB(const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& Info) +{ + TryKickoff(); +} diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp index 3314759..6e9293f 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp @@ -739,6 +739,23 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::StopListening() } } +void UPS_AI_ConvAgent_ElevenLabsComponent::SendContextualUpdate(const FString& Text) +{ + // Server-authoritative: the WebSocket lives on the server. The agent-to-agent + // orchestrator (server) uses this to reframe the agents when the player takes over. + if (GetOwnerRole() == ROLE_Authority) + { + if (IsConnected() && WebSocketProxy) + { + WebSocketProxy->SendContextualUpdate(Text); + } + else + { + UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning, TEXT("SendContextualUpdate: not connected.")); + } + } +} + void UPS_AI_ConvAgent_ElevenLabsComponent::SendTextMessage(const FString& Text) { if (GetOwnerRole() == ROLE_Authority) @@ -900,6 +917,27 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::FeedExternalAudio(const TArray // Same logic as OnMicrophoneDataCaptured but called from an external source // (e.g. InteractionComponent on the pawn) instead of a local mic component. // On server: check local WebSocket. On client: check replicated conversation state. + + // Local voice-onset detection: broadcast on the FIRST voiced frame BEFORE the + // listening/echo gates below, so an agent-to-agent dialogue can be interrupted the + // instant the player speaks — no "listening" state and no ElevenLabs STT round-trip + // needed. Debounced to fire once per utterance. (OnAudioCaptured is marshaled to the + // game thread by the mic component, so this runs on the game thread — safe to broadcast.) + { + float OnsetSumSq = 0.0f; + for (const float S : FloatPCM) { OnsetSumSq += S * S; } + const float OnsetRms = FMath::Sqrt(OnsetSumSq / FMath::Max(1, FloatPCM.Num())); + if (OnsetRms >= 0.02f) + { + const double NowSec = FPlatformTime::Seconds(); + if (NowSec - LastUserVoiceOnset > 0.5) + { + LastUserVoiceOnset = NowSec; + OnUserVoiceDetected.Broadcast(); + } + } + } + const bool bCanSend = (GetOwnerRole() == ROLE_Authority) ? IsConnected() : bNetIsConversing; if (!bCanSend || !bIsListening) return; @@ -2154,6 +2192,7 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::GetLifetimeReplicatedProps( DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, bNetIsConversing); DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, NetConnectedPawns); DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, NetActiveSpeakerPawn); + DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, GazeOverrideTarget); DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, CurrentEmotion); DOREPLIFETIME(UPS_AI_ConvAgent_ElevenLabsComponent, CurrentEmotionIntensity); } @@ -2177,7 +2216,18 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::OnRep_ConversationState() // on the local pawn, but remote clients never run that code path. if (UPS_AI_ConvAgent_GazeComponent* Gaze = Owner->FindComponentByClass()) { - if (bNetIsConversing && NetConnectedPawns.Num() > 0) + if (GazeOverrideTarget) + { + // Override (agent-to-agent dialogue): look at the fixed actor. + Gaze->bActive = true; + if (Gaze->TargetActor != GazeOverrideTarget) + { + Gaze->TargetActor = GazeOverrideTarget; + Gaze->ResetBodyTarget(); + } + Gaze->bEnableBodyTracking = true; + } + else if (bNetIsConversing && NetConnectedPawns.Num() > 0) { Gaze->bActive = true; // Use active speaker if set, otherwise first connected pawn. @@ -2261,16 +2311,22 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::OnRep_ActiveSpeaker() AActor* Owner = GetOwner(); if (!Owner) return; - if (auto* Gaze = Owner->FindComponentByClass()) + // Guard: during an agent-to-agent dialogue the gaze is pinned to the other + // agent (GazeOverrideTarget) — a replicated active-speaker change (e.g. a + // player joining/speaking) must NOT yank the eyes back onto the player. + if (!GazeOverrideTarget) { - if (NetActiveSpeakerPawn) + if (auto* Gaze = Owner->FindComponentByClass()) { - Gaze->TargetActor = NetActiveSpeakerPawn; - Gaze->ResetBodyTarget(); - UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log, - TEXT("[NET-REP] ActiveSpeaker changed to %s"), *NetActiveSpeakerPawn->GetName()); + if (NetActiveSpeakerPawn) + { + Gaze->TargetActor = NetActiveSpeakerPawn; + Gaze->ResetBodyTarget(); + UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log, + TEXT("[NET-REP] ActiveSpeaker changed to %s"), *NetActiveSpeakerPawn->GetName()); + } + // Don't clear gaze when null — keep looking at last speaker. } - // Don't clear gaze when null — keep looking at last speaker. } OnActiveSpeakerChanged.Broadcast(NetActiveSpeakerPawn, nullptr); @@ -2384,8 +2440,11 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ServerLeaveConversation_Implementatio SetActiveSpeaker(NetConnectedPawns.Num() > 0 ? NetConnectedPawns[0] : nullptr); } - // If no players left, fully end the conversation. - if (NetConnectedPawns.Num() == 0) + // If no players left, fully end the conversation — UNLESS the agent-to-agent + // orchestrator owns it (GazeOverrideTarget set). A player merely WATCHING the + // dialogue joins/leaves just by looking around; that must NOT close the agent's + // WebSocket, stop its audio, or deactivate its lip sync. Keep the dialogue alive. + if (NetConnectedPawns.Num() == 0 && !GazeOverrideTarget) { // Cancel any pending reconnection (ephemeral mode only). if (!bPersistentSession) @@ -2421,7 +2480,8 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ServerLeaveConversation_Implementatio } else { - // Other players still connected — update gaze. + // Players still connected, OR an orchestrated agent-to-agent dialogue owns + // this conversation (0 players but GazeOverrideTarget set) — keep it running. ApplyConversationGaze(); } } @@ -2851,7 +2911,18 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ApplyConversationGaze() auto* Gaze = Owner->FindComponentByClass(); if (!Gaze) return; - if (bNetIsConversing && NetConnectedPawns.Num() > 0) + // Override (agent-to-agent dialogue): look at a fixed actor, ignore players. + if (GazeOverrideTarget) + { + Gaze->bActive = true; + if (Gaze->TargetActor != GazeOverrideTarget) + { + Gaze->TargetActor = GazeOverrideTarget; + Gaze->ResetBodyTarget(); + } + Gaze->bEnableBodyTracking = true; + } + else if (bNetIsConversing && NetConnectedPawns.Num() > 0) { Gaze->bActive = true; // Look at active speaker if set, otherwise last connected pawn. @@ -2891,6 +2962,20 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ApplyConversationGaze() } } +void UPS_AI_ConvAgent_ElevenLabsComponent::SetGazeOverrideTarget(AActor* Target) +{ + if (GazeOverrideTarget == Target) return; + GazeOverrideTarget = Target; // replicates → OnRep_GazeOverrideTarget on clients + ApplyConversationGaze(); // apply immediately on the server/authority +} + +void UPS_AI_ConvAgent_ElevenLabsComponent::OnRep_GazeOverrideTarget() +{ + // Re-evaluate the gaze on clients (honours the override, or restores normal + // player-tracking when it was cleared to null). + ApplyConversationGaze(); +} + void UPS_AI_ConvAgent_ElevenLabsComponent::SetActiveSpeaker(APawn* NewSpeaker) { if (NetActiveSpeakerPawn == NewSpeaker) return; diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp index f04b8c0..1671c24 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_InteractionComponent.cpp @@ -191,7 +191,8 @@ void UPS_AI_ConvAgent_InteractionComponent::TickComponent(float DeltaTime, ELeve // Re-activate passive gaze using configured checkboxes. // ExecuteLeave killed the gaze — re-enable passively. - if (bAutoManageGaze) + // (Skip if the agent is in an agent-to-agent dialogue.) + if (bAutoManageGaze && !Pending->GazeOverrideTarget) { if (UPS_AI_ConvAgent_GazeComponent* Gaze = FindGazeOnAgent(Pending)) { @@ -301,6 +302,9 @@ void UPS_AI_ConvAgent_InteractionComponent::UpdatePassiveGaze() if (Agent == GazeRetainedAgent.Get()) continue; if (Agent == PendingLeaveAgent.Get()) continue; if (Agent->IsConversationDisabled()) continue; + // Skip agents driven by an agent-to-agent dialogue (they look at each + // other via GazeOverrideTarget — don't pull their gaze toward the player). + if (Agent->GazeOverrideTarget) continue; AActor* AgentActor = Agent->GetOwner(); if (!AgentActor) continue; @@ -600,7 +604,8 @@ void UPS_AI_ConvAgent_InteractionComponent::SetSelectedAgent(UPS_AI_ConvAgent_El // ExecuteLeave → ServerLeaveConversation → ApplyConversationGaze() // already killed bActive and cleared TargetActor. Override that: // the agent should keep looking at the player passively. - if (bAutoManageGaze) + // (Skip while an agent-to-agent dialogue owns the gaze.) + if (bAutoManageGaze && !OldAgent->GazeOverrideTarget) { if (UPS_AI_ConvAgent_GazeComponent* Gaze = FindGazeOnAgent(OldAgent)) { @@ -919,6 +924,11 @@ void UPS_AI_ConvAgent_InteractionComponent::AttachGazeTarget( // from ever firing. if (AgentPtr->IsConversationDisabled()) return; + // Agent-to-agent dialogue: this agent's gaze is pinned on the OTHER agent via + // GazeOverrideTarget. Don't attach it to the player just because the player is + // looking at / selecting it — let the override hold (the dialogue owns the gaze). + if (AgentPtr->GazeOverrideTarget) return; + if (UPS_AI_ConvAgent_GazeComponent* Gaze = FindGazeOnAgent(AgentPtr)) { Gaze->TargetActor = GetOwner(); diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_WebSocket_ElevenLabsProxy.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_WebSocket_ElevenLabsProxy.cpp index e816392..e658d31 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_WebSocket_ElevenLabsProxy.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_WebSocket_ElevenLabsProxy.cpp @@ -285,6 +285,23 @@ void UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy::SendTextMessage(const FString& SendJsonMessage(Msg); } +void UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy::SendContextualUpdate(const FString& Text) +{ + if (!IsConnected()) + { + UE_LOG(LogPS_AI_ConvAgent_WS_ElevenLabs, Warning, TEXT("SendContextualUpdate: not connected.")); + return; + } + if (Text.IsEmpty()) return; + + // API: { "type": "contextual_update", "text": "..." } — injects context WITHOUT + // triggering an agent response (unlike user_message). + TSharedPtr Msg = MakeShareable(new FJsonObject()); + Msg->SetStringField(TEXT("type"), TEXT("contextual_update")); + Msg->SetStringField(TEXT("text"), Text); + SendJsonMessage(Msg); +} + void UPS_AI_ConvAgent_WebSocket_ElevenLabsProxy::SendInterrupt() { if (!IsConnected()) return; diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_AgentDialogue.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_AgentDialogue.h new file mode 100644 index 0000000..28edf2e --- /dev/null +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_AgentDialogue.h @@ -0,0 +1,181 @@ +// Copyright ASTERION. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "GameFramework/Actor.h" +#include "PS_AI_ConvAgent_ElevenLabsComponent.h" // component type + delegates + ConversationInfo struct +#include "PS_AI_ConvAgent_AgentDialogue.generated.h" + +class UPS_AI_ConvAgent_ElevenLabsComponent; + +/** Fired each time a speaker's line is forwarded to the other agent (subtitles / logging). */ +DECLARE_DYNAMIC_MULTICAST_DELEGATE_ThreeParams(FPS_AI_ConvAgent_OnDialogueTurn, + int32, TurnIndex, AActor*, Speaker, const FString&, Text); + +/** Fired when the dialogue stops (MaxTurns reached or StopDialogue). */ +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPS_AI_ConvAgent_OnDialogueFinished, + int32, TotalTurns); + +/** + * Orchestrator actor that makes TWO ElevenLabs agents converse with each other + * over TEXT (no STT): each agent's completed text response is fed as a user + * message to the OTHER agent, which then replies. Both agents still speak aloud + * (TTS), so humans in the level hear the whole conversation. + * + * Setup: place this actor in the level, assign AgentA / AgentB to two actors that + * each already carry a PS_AI_ConvAgent_ElevenLabsComponent, set an OpeningMessage, + * then call StartDialogue() (or enable bAutoStartOnBeginPlay). + * + * Runs on the SERVER (authority) — the agents' WebSockets are server-owned. + * By default forwarding happens when a speaker FINISHES speaking so the two + * voices never overlap (needs audio playback → listen-server/host). On a + * dedicated server with no audio device, set bWaitForSpeechToFinish=false to + * drive turns from the text event instead. + */ +UCLASS(Blueprintable) +class PS_AI_CONVAGENT_API APS_AI_ConvAgent_AgentDialogue : public AActor +{ + GENERATED_BODY() + +public: + APS_AI_ConvAgent_AgentDialogue(); + + /** Actor carrying the FIRST agent's ElevenLabs component. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue") + TObjectPtr AgentA; + + /** Actor carrying the SECOND agent's ElevenLabs component. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue") + TObjectPtr AgentB; + + /** Opening line sent to the first speaker to kick off the dialogue. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue") + FString OpeningMessage = TEXT("Bonjour."); + + /** If true the opening message goes to AgentA (A replies first); else to AgentB. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue") + bool bAgentAStarts = true; + + /** Max forwarded turns before auto-stopping. 0 = unlimited. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue", meta = (ClampMin = "0")) + int32 MaxTurns = 20; + + /** Pause (s) after a speaker's turn before the other is prompted. Adds breathing room. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue", meta = (ClampMin = "0.0")) + float TurnDelaySeconds = 0.0f; + + /** True (default): forward a line when the speaker FINISHES speaking (clean, non-overlapping voices; needs audio playback). + * False: forward as soon as the text is ready (robust on dedicated servers, but voices may overlap). */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue") + bool bWaitForSpeechToFinish = true; + + /** Start the dialogue automatically on BeginPlay (authority only). */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue") + bool bAutoStartOnBeginPlay = false; + + /** After the player interrupts (by speaking), ALL dialogue agents look at the + * player until the player moves farther than this (cm) from BOTH agents, then + * the agents are released back to normal behavior and their conversations end. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue", meta = (ClampMin = "0.0")) + float PlayerAddressReleaseDistance = 500.0f; + + /** Sent to BOTH agents as a SILENT context update (ElevenLabs contextual_update — no + * reply triggered) the instant the player interrupts, so the agents know a new/real + * person is now addressing them while KEEPING the prior conversation context. + * Leave empty to inject nothing. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue", meta = (MultiLine = "true")) + FString InterruptContextMessage = TEXT("Une nouvelle personne, bien reelle, s'adresse maintenant directement a toi, ce n'est plus ton interlocuteur precedent. Garde en memoire la conversation qui vient d'avoir lieu, mais reponds desormais a cette personne."); + + /** Sent (contextual_update, no reply) to the agent the player is NOT talking to: it + * is a witness. Empty = don't inform the bystander. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue", meta = (MultiLine = "true")) + FString BystanderContextMessage = TEXT("Une nouvelle personne, bien reelle, vient d'entrer et s'adresse a ton interlocuteur, pas a toi pour l'instant. Ecoute ce qui se dit, tu pourras y faire reference plus tard quand elle se tournera vers toi."); + + /** Prefix for the "overheard" lines relayed to the witness agent during a 3-way + * exchange (contextual_update, no reply). Empty = disable passive overhearing. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "PS AI ConvAgent|Dialogue") + FString OverheardPrefix = TEXT("[Tu entends la conversation a cote] "); + + /** Fired each time a line is forwarded (TurnIndex, Speaker actor, the text). */ + UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|Dialogue") + FPS_AI_ConvAgent_OnDialogueTurn OnDialogueTurn; + + /** Fired when the dialogue ends. */ + UPROPERTY(BlueprintAssignable, Category = "PS AI ConvAgent|Dialogue") + FPS_AI_ConvAgent_OnDialogueFinished OnDialogueFinished; + + /** Open both conversations and kick off the exchange. Authority only. */ + UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Dialogue") + void StartDialogue(); + + /** Stop forwarding turns. Agents stay connected unless bEndConversations. */ + UFUNCTION(BlueprintCallable, Category = "PS AI ConvAgent|Dialogue") + void StopDialogue(bool bEndConversations = false); + + /** True while the loop is active. */ + UFUNCTION(BlueprintPure, Category = "PS AI ConvAgent|Dialogue") + bool IsRunning() const { return bRunning; } + +protected: + virtual void BeginPlay() override; + virtual void EndPlay(const EEndPlayReason::Type Reason) override; + virtual void Tick(float DeltaSeconds) override; // only active in "addressing player" mode + +private: + UPROPERTY(Transient) TObjectPtr CompA; + UPROPERTY(Transient) TObjectPtr CompB; + + // Single-floor turn state: exactly ONE agent holds the floor at a time; every + // event from the OTHER agent is ignored. A turn forwards exactly once, when the + // floor-holder's text AND its speech are both complete. + enum class EFloor : uint8 { None, A, B }; + EFloor Floor = EFloor::None; + int32 TurnId = 0; // number of forwarded turns + bool bTextReady = false; // floor-holder's full text arrived this turn + bool bSpeechDone = false; // floor-holder finished speaking this turn + bool bForwardedThisTurn = false; // exactly-one-forward guard + FString PendingLine; // floor-holder's line for this turn + bool bRunning = false; + bool bOpeningSent = false; + FTimerHandle TurnTimer; + + // "Addressing player" mode: after the player interrupts by speaking, ALL dialogue + // agents are pinned to look at the player (fast, via the gaze override) and the + // orchestrator ticks to auto-release them once the player leaves both agents. + bool bAddressingPlayer = false; + UPROPERTY(Transient) TObjectPtr AddressedByPlayer; + EFloor EngagedFloor = EFloor::None; // which agent the player is currently addressing + + APawn* ResolveSpeakingPlayer() const; // the player whose voice was just transcribed + void EndPlayerAddress(); // release agents → normal + end conversations + + UPS_AI_ConvAgent_ElevenLabsComponent* ResolveAgentComp(AActor* Actor) const; + void BindAgent(UPS_AI_ConvAgent_ElevenLabsComponent* Comp, bool bIsA); + void UnbindAll(); + void TryKickoff(); + void BeginTurn(EFloor NewFloor); // set floor + reset per-turn flags + bool IsFloor(bool bFromA) const; // is the given agent the current floor-holder? + void TryForward(); // forward once, when text AND speech done + + UFUNCTION() void HandleTextA(const FString& Text); + UFUNCTION() void HandleTextB(const FString& Text); + UFUNCTION() void HandleStoppedA(); + UFUNCTION() void HandleStoppedB(); + UFUNCTION() void HandleConnectedA(const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& Info); + UFUNCTION() void HandleConnectedB(const FPS_AI_ConvAgent_ConversationInfo_ElevenLabs& Info); + + // Per-agent interrupt/relay triggers. Knowing WHICH agent the player addressed lets + // us track the engaged agent and relay the exchange to the other ("witness") agent. + // Voice-onset = primary immediate trigger; transcript = backup + carries the words. + // Merely LOOKING at an agent never triggers these — only actually speaking does. + UFUNCTION() void HandleVoiceOnsetA(); + UFUNCTION() void HandleVoiceOnsetB(); + UFUNCTION() void HandleTranscriptA(const FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs& Segment); + UFUNCTION() void HandleTranscriptB(const FPS_AI_ConvAgent_TranscriptSegment_ElevenLabs& Segment); + + void OnPlayerSpoke(EFloor Which, const FString& PlayerWords); // player addressed agent Which + void InterruptForPlayer(EFloor Engaged); // enter 3-way floor mode (Engaged = addressee) + void SetEngaged(EFloor Which); // reframe both agents when the addressee changes + void RelayToBystander(EFloor EngagedSide, const FString& FormattedLine); // "overheard" context +}; diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h index f93dd15..75a1c9a 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h @@ -122,6 +122,11 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAgentPassiveGazeStarted); /** Fired when this agent stops passive gaze (left range or became selected). */ DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAgentPassiveGazeStopped); +/** Fired on the FIRST voiced frame of the local user's mic (via FeedExternalAudio), + * independent of the ElevenLabs STT round-trip — used to interrupt an agent-to-agent + * dialogue on voice onset. Debounced (~once per utterance). */ +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnUserVoiceDetected); + // Non-dynamic delegate for raw agent audio (high-frequency, C++ consumers only). // Delivers PCM chunks as int16, 16kHz mono, little-endian. DECLARE_MULTICAST_DELEGATE_OneParam(FOnAgentAudioData, const TArray& /*PCMData*/); @@ -324,6 +329,12 @@ public: meta = (ToolTip = "Fires when the agent is interrupted mid-speech. Audio is automatically stopped.")) FOnAgentInterrupted OnAgentInterrupted; + /** Fires on local user voice onset (first voiced mic frame via FeedExternalAudio), + * without waiting for the ElevenLabs STT round-trip. Used to interrupt an + * agent-to-agent dialogue the instant the player speaks. */ + UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events") + FOnUserVoiceDetected OnUserVoiceDetected; + /** Fired when the server starts generating a response (before any audio arrives). Use this for "thinking..." UI feedback. In push-to-talk mode, the microphone is automatically closed when this fires. */ UPROPERTY(BlueprintAssignable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs|Events", meta = (ToolTip = "Fires when the server starts generating (before audio arrives).\nUse for 'thinking...' UI. Mic is auto-closed in push-to-talk mode.")) @@ -407,6 +418,20 @@ public: UPROPERTY(ReplicatedUsing = OnRep_ActiveSpeaker, BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|Network") TObjectPtr NetActiveSpeakerPawn = nullptr; + /** When set, the conversation gaze looks at THIS actor instead of any connected + * player pawn — used by the agent-to-agent dialogue so the two agents look at + * EACH OTHER rather than at a bystander. Replicated so clients honor it too. */ + UPROPERTY(ReplicatedUsing = OnRep_GazeOverrideTarget, BlueprintReadOnly, Category = "ASTERION|PS_AI_ConvAgent|Gaze") + TObjectPtr GazeOverrideTarget = nullptr; + + UFUNCTION() + void OnRep_GazeOverrideTarget(); + + /** Force the conversation gaze to look at Target (pass null to clear and restore + * the normal player-tracking behaviour). Call on the server; replicates to clients. */ + UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|Gaze") + void SetGazeOverrideTarget(AActor* Target); + // ── Multi-player speaker arbitration (server only) ────────────────────── /** Minimum seconds of silence from the current speaker before allowing a speaker switch. @@ -568,6 +593,14 @@ public: UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs") void SendTextMessage(const FString& Text); + /** + * Inject a "silent" context update: the agent incorporates it as context but does + * NOT reply to it (unlike SendTextMessage). Used to tell an agent that a new/real + * interlocutor is now speaking while keeping the prior conversation. Authority only. + */ + UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs") + void SendContextualUpdate(const FString& Text); + /** Interrupt the agent's current utterance. */ UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs") void InterruptAgent(); @@ -664,6 +697,9 @@ public: int32 GetMicChunkMinBytesPublic() const; private: + // Debounce timestamp (seconds) for the OnUserVoiceDetected voice-onset broadcast. + double LastUserVoiceOnset = 0.0; + // ── Network OnRep handlers ─────────────────────────────────────────────── UFUNCTION() void OnRep_ConversationState(); diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_WebSocket_ElevenLabsProxy.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_WebSocket_ElevenLabsProxy.h index 1fd1af0..ae55868 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_WebSocket_ElevenLabsProxy.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_WebSocket_ElevenLabsProxy.h @@ -177,6 +177,12 @@ public: UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs") void SendTextMessage(const FString& Text); + /** + * Inject a context update WITHOUT triggering an agent response. + * Sends: { "type": "contextual_update", "text": "..." } + */ + void SendContextualUpdate(const FString& Text); + /** Ask the agent to stop the current utterance. */ UFUNCTION(BlueprintCallable, Category = "ASTERION|PS_AI_ConvAgent|ElevenLabs") void SendInterrupt();