Ajout sample CPP

This commit is contained in:
2026-02-21 20:48:10 +01:00
parent d8957625f8
commit f23acc8c1c
15 changed files with 985 additions and 4 deletions

View File

@@ -604,9 +604,15 @@ void UElevenLabsConversationalAgentComponent::OnMicrophoneDataCaptured(const TAr
{
if (!IsConnected() || !bIsListening) return;
// Echo suppression: skip sending mic audio while the agent is speaking.
// This prevents the agent from hearing its own voice through the speakers,
// which would confuse the server's VAD and STT. Matches the approach used
// in the official ElevenLabs C++ SDK (outputPlaying_ flag).
if (bAgentSpeaking) return;
// Convert this callback's samples to int16 bytes and accumulate.
// WASAPI fires every ~5ms (158 bytes at 16kHz). ElevenLabs needs ≥100ms
// (3200 bytes) per chunk for reliable VAD and STT. We hold bytes here
// WASAPI fires every ~5ms (158 bytes at 16kHz). ElevenLabs needs ≥250ms
// (8000 bytes) per chunk for reliable VAD and STT. We hold bytes here
// until we have enough, then send the whole batch in one WebSocket frame.
TArray<uint8> PCMBytes = FloatPCMToInt16Bytes(FloatPCM);
MicAccumulationBuffer.Append(PCMBytes);

View File

@@ -491,6 +491,17 @@ void UElevenLabsWebSocketProxy::HandleAudioResponse(const TSharedPtr<FJsonObject
return;
}
// Discard audio belonging to an interrupted generation (event_id approach).
// Matches the official ElevenLabs C++ and Python SDKs: only AUDIO is filtered
// by event_id — transcripts, agent_response, etc. are always processed.
int32 EventId = 0;
(*AudioEvent)->TryGetNumberField(TEXT("event_id"), EventId);
if (EventId > 0 && EventId <= LastInterruptEventId)
{
UE_LOG(LogElevenLabsWS, Verbose, TEXT("Discarding audio event_id=%d (interrupted at %d)."), EventId, LastInterruptEventId);
return;
}
FString Base64Audio;
if (!(*AudioEvent)->TryGetStringField(TEXT("audio_base_64"), Base64Audio))
{
@@ -591,7 +602,20 @@ void UElevenLabsWebSocketProxy::HandleAgentChatResponsePart(const TSharedPtr<FJs
void UElevenLabsWebSocketProxy::HandleInterruption(const TSharedPtr<FJsonObject>& Root)
{
UE_LOG(LogElevenLabsWS, Log, TEXT("Agent interrupted (server ack received)."));
// Extract the interrupt event_id so we can filter stale audio frames.
// { "type": "interruption", "interruption_event": { "event_id": 42 } }
const TSharedPtr<FJsonObject>* InterruptEvent = nullptr;
if (Root->TryGetObjectField(TEXT("interruption_event"), InterruptEvent) && InterruptEvent)
{
int32 EventId = 0;
(*InterruptEvent)->TryGetNumberField(TEXT("event_id"), EventId);
if (EventId > LastInterruptEventId)
{
LastInterruptEventId = EventId;
}
}
UE_LOG(LogElevenLabsWS, Log, TEXT("Agent interrupted (server ack, LastInterruptEventId=%d)."), LastInterruptEventId);
OnInterrupted.Broadcast();
}

View File

@@ -400,5 +400,5 @@ private:
// ElevenLabs needs at least ~100ms (3200 bytes) per chunk for reliable VAD/STT.
// We accumulate here and only call SendAudioChunk once enough bytes are ready.
TArray<uint8> MicAccumulationBuffer;
static constexpr int32 MicChunkMinBytes = 3200; // 100ms @ 16kHz 16-bit mono
static constexpr int32 MicChunkMinBytes = 8000; // 250ms @ 16kHz 16-bit mono (4000 samples, matches ElevenLabs SDK recommendation)
};

View File

@@ -226,6 +226,13 @@ private:
// Used to compute [T+Xs] session-relative timestamps in all log messages.
double SessionStartTime = 0.0;
// ── Interrupt filtering (event_id approach, matching official SDK) ────────
// When the server sends an "interruption" event it includes an event_id.
// Audio events whose event_id <= LastInterruptEventId belong to the cancelled
// generation and must be discarded. Only AUDIO is filtered — transcripts,
// agent_response, agent_chat_response_part etc. are always processed.
int32 LastInterruptEventId = 0;
public:
// Set by UElevenLabsConversationalAgentComponent before calling Connect().
// Controls turn_timeout in conversation_initiation_client_data.