diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp index ab73c85..4363b45 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Private/PS_AI_ConvAgent_ElevenLabsComponent.cpp @@ -901,6 +901,21 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::FeedExternalAudio(const TArray // (e.g. InteractionComponent on the pawn) instead of a local mic component. // On server: check local WebSocket. On client: check replicated conversation state. const bool bCanSend = (GetOwnerRole() == ROLE_Authority) ? IsConnected() : bNetIsConversing; + + // [MIC-DBG] TEMP diagnostic — remove once the client uplink is fixed. + { + static double GMicDbgLast = 0.0; + const double NowDbg = FPlatformTime::Seconds(); + if (NowDbg - GMicDbgLast > 1.0) + { + GMicDbgLast = NowDbg; + UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning, + TEXT("[MIC-DBG] FeedExternalAudio Role=%d bCanSend=%d bIsListening=%d bNetIsConversing=%d bAgentSpeaking=%d samples=%d"), + (int32)GetOwnerRole(), bCanSend ? 1 : 0, bIsListening ? 1 : 0, + bNetIsConversing ? 1 : 0, bAgentSpeaking.load() ? 1 : 0, FloatPCM.Num()); + } + } + if (!bCanSend || !bIsListening) return; // Echo suppression: skip sending mic audio while the agent is speaking. @@ -909,13 +924,15 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::FeedExternalAudio(const TArray return; } - // ── Network silence gate ──────────────────────────────────────────────── - // On clients the mic audio is sent via unreliable RPCs (~3200 bytes every - // 100ms = ~256 Kbits/s). Sending silence saturates the connection and - // starves movement replication, causing chaotic teleporting. - // Skip silent chunks on clients only — the server path uses a local - // WebSocket and doesn't touch the network, so it keeps the full stream - // for proper ElevenLabs VAD (voice-activity detection). + // ── Network silence gate (clients only) ───────────────────────────────── + // On clients the mic audio is sent via unreliable RPCs (~256 Kbits/s). + // Streaming silence the whole session saturates the connection and starves + // movement replication (teleporting). But ElevenLabs Server VAD needs the + // silence AFTER speech to detect end-of-turn — dropping it entirely means the + // agent hears the words but never the pause, so it never replies. + // Compromise: keep streaming a short silence "tail" after the last voiced + // chunk (MicSilenceTailSeconds), then drop silence during long idle gaps. + // The authority path is untouched — it always streams the full mic. if (GetOwnerRole() != ROLE_Authority) { float SumSq = 0.0f; @@ -924,9 +941,22 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::FeedExternalAudio(const TArray SumSq += Sample * Sample; } const float Rms = FMath::Sqrt(SumSq / FMath::Max(1, FloatPCM.Num())); - if (Rms < 0.005f) // ~-46 dBFS — well below any speech level + + const double NowSec = FPlatformTime::Seconds(); + if (Rms >= 0.005f) // ~-46 dBFS — above this is voiced speech. { - return; + LastMicVoicedTime = NowSec; // Voiced: always send, remember when. + } + else + { + // Silence: send only within the tail window after the last voiced + // chunk (so VAD sees end-of-turn); drop once the tail has elapsed. + const bool bWithinTail = (LastMicVoicedTime > 0.0) + && (NowSec - LastMicVoicedTime) <= MicSilenceTailSeconds; + if (!bWithinTail) + { + return; + } } } @@ -954,7 +984,16 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::FeedExternalAudio(const TArray else { // Route through relay (clients can't call Server RPCs on NPC actors). - if (auto* Relay = FindLocalRelayComponent()) + auto* Relay = FindLocalRelayComponent(); + + // [MIC-DBG] TEMP diagnostic — remove once the client uplink is fixed. + UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning, + TEXT("[MIC-DBG] client send %d bytes via relay=%s opus=%s"), + MicAccumulationBuffer.Num(), + Relay ? TEXT("FOUND") : TEXT("NULL (-> direct NPC RPC, will be dropped)"), + OpusEncoder.IsValid() ? TEXT("yes") : TEXT("no(raw)")); + + if (Relay) { // Opus-compress mic audio before sending over the network. // 3200 bytes raw PCM → ~200 bytes Opus (~16x reduction). @@ -2084,6 +2123,25 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::OnRep_ConversationState() } } + // ── Client: (re)start listening once the conversation is confirmed live ── + // StartListening() called early on the client defers (bPendingStartListening) + // because bNetIsConversing wasn't replicated yet. The ClientConversationStarted + // relay RPC may also race ahead of this replication. OnRep is the reliable + // "conversation is live" signal, so consume the deferred listen here — this + // closes the race regardless of RPC/replication ordering. Gated on + // IsLocalPlayerConversating() so only the joining client opens its mic path. + if (GetOwnerRole() != ROLE_Authority && bNetIsConversing && IsLocalPlayerConversating()) + { + if (bPendingStartListening + || (bAutoStartListening && TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server)) + { + bPendingStartListening = false; + UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Log, + TEXT("[NET-REP] Conversation live on client — starting listening (mic uplink).")); + StartListening(); + } + } + if (!bNetIsConversing) { // Conversation ended on server — clean up local playback. @@ -2302,6 +2360,23 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::ServerSendMicAudio_Implementation( void UPS_AI_ConvAgent_ElevenLabsComponent::ServerSendMicAudioFromPlayer( APawn* SpeakerPawn, const TArray& PCMBytes) { + // [MIC-DBG] TEMP diagnostic — remove once the client uplink is fixed. + { + static double GMicSrvDbgLast = 0.0; + const double NowDbg = FPlatformTime::Seconds(); + if (NowDbg - GMicSrvDbgLast > 1.0) + { + GMicSrvDbgLast = NowDbg; + UE_LOG(LogPS_AI_ConvAgent_ElevenLabs, Warning, + TEXT("[MIC-DBG-SRV] ServerSendMicAudioFromPlayer reached: from=%s players=%d active=%s wsConnected=%d bytes=%d"), + SpeakerPawn ? *SpeakerPawn->GetName() : TEXT("NULL"), + NetConnectedPawns.Num(), + NetActiveSpeakerPawn ? *NetActiveSpeakerPawn->GetName() : TEXT("none"), + (WebSocketProxy && WebSocketProxy->IsConnected()) ? 1 : 0, + PCMBytes.Num()); + } + } + if (!WebSocketProxy || !WebSocketProxy->IsConnected()) return; // Standalone / single-player: bypass speaker arbitration entirely. @@ -2714,12 +2789,18 @@ void UPS_AI_ConvAgent_ElevenLabsComponent::DrawDebugHUD() const const FColor GoodColor = FColor::Green; const FString OwnerName = GetOwner()->GetName(); - const bool bConnected = IsConnected(); + // On a client there is no local WebSocket (it lives on the server), so + // IsConnected() is always false. Report the effective networked state via the + // replicated bNetIsConversing flag instead, to avoid a misleading "DISCONNECTED". + const bool bIsAuthority = (GetOwnerRole() == ROLE_Authority); + const bool bConnected = bIsAuthority ? IsConnected() : bNetIsConversing; + const TCHAR* ConnStr = bConnected + ? (bIsAuthority ? TEXT("CONNECTED") : TEXT("CONNECTED (via server)")) + : TEXT("DISCONNECTED"); GEngine->AddOnScreenDebugMessage(BaseKey, DisplayTime, bConnected ? GoodColor : FColor::Red, - FString::Printf(TEXT("=== ELEVENLABS [%s]: %s ==="), *OwnerName, - bConnected ? TEXT("CONNECTED") : TEXT("DISCONNECTED"))); + FString::Printf(TEXT("=== ELEVENLABS [%s]: %s ==="), *OwnerName, ConnStr)); // Session info FString ModeStr = (TurnMode == EPS_AI_ConvAgent_TurnMode_ElevenLabs::Server) diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h index a664534..8581aaf 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_ConvAgent/Source/PS_AI_ConvAgent/Public/PS_AI_ConvAgent_ElevenLabsComponent.h @@ -439,6 +439,18 @@ public: meta = (ClampMin = "0", ToolTip = "Distance beyond which lip-sync is skipped for non-speaking players. 0 = no LOD.")) float LipSyncLODDistance = 1500.f; + // ── Network mic uplink (clients) ───────────────────────────────────────── + + /** Seconds of silence to keep streaming to the server AFTER the player's last + * voiced chunk, so ElevenLabs Server VAD can detect end-of-turn. Beyond this + * tail, silent chunks are dropped to save bandwidth (clients only — the + * authority path always streams the full mic). Set >= the agent's end-of-turn + * silence threshold (+ margin). 0 = drop all silence (breaks Server VAD). */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ASTERION|PS_AI_ConvAgent|Network|MultiPlayer", + meta = (ClampMin = "0.0", ClampMax = "5.0", + ToolTip = "Silence streamed after last voice so ElevenLabs detects end-of-turn (Server VAD).\nMust be >= the agent's end-of-turn silence. 0 = drop all silence (breaks Server VAD on clients).")) + float MicSilenceTailSeconds = 2.0f; + // ── Network RPCs ───────────────────────────────────────────────────────── /** Join a shared conversation with this NPC. Multiple players can join. @@ -843,6 +855,11 @@ private: TArray MicAccumulationBuffer; FCriticalSection MicSendLock; + // Client mic uplink: timestamp (FPlatformTime::Seconds) of the last voiced chunk. + // Used to keep streaming a short silence tail after speech so ElevenLabs Server + // VAD can detect end-of-turn (see MicSilenceTailSeconds). 0 = never voiced yet. + double LastMicVoicedTime = 0.0; + /** Compute the minimum bytes from the user-facing MicChunkDurationMs. * Formula: bytes = SampleRate * (ms / 1000) * BytesPerSample = 16000 * ms / 1000 * 2 = 32 * ms */ int32 GetMicChunkMinBytes() const { return MicChunkDurationMs * 32; }