From 669c503d067da2f62a69dbb6457fd4116802e8bd Mon Sep 17 00:00:00 2001
From: "j.foucher" <j.foucher@polymorph.fr>
Date: Thu, 19 Feb 2026 13:51:47 +0100
Subject: [PATCH] Fix: handle binary WebSocket frames from ElevenLabs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

ElevenLabs sends all JSON messages as binary WS frames, not text frames.
The OnRawMessage callback receives them; we were logging them as warnings
and discarding the data entirely — causing no events to fire at all.

Fix: accumulate binary frame fragments (BytesRemaining > 0 = more coming),
reassemble into a complete buffer, decode as UTF-8 JSON string, then route
through the existing OnWsMessage text handler unchanged.

Added BinaryFrameBuffer (TArray<uint8>) to proxy header for accumulation.
Compiles cleanly on UE 5.5 Win64.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../Private/ElevenLabsWebSocketProxy.cpp      | 23 ++++++++++++++++---
 .../Public/ElevenLabsWebSocketProxy.h         |  4 ++++
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsWebSocketProxy.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsWebSocketProxy.cpp
index 67e9541..356de64 100644
--- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsWebSocketProxy.cpp
+++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsWebSocketProxy.cpp
@@ -237,9 +237,26 @@ void UElevenLabsWebSocketProxy::OnWsMessage(const FString& Message)
 
 void UElevenLabsWebSocketProxy::OnWsBinaryMessage(const void* Data, SIZE_T Size, SIZE_T BytesRemaining)
 {
-	// ElevenLabs Conversational AI uses text (JSON) frames only.
-	// If binary frames arrive in future API versions, handle here.
-	UE_LOG(LogElevenLabsWS, Warning, TEXT("Received unexpected binary WebSocket frame (%llu bytes)."), (uint64)Size);
+	// ElevenLabs sends its JSON messages as binary WebSocket frames (not text frames).
+	// Accumulate fragments until BytesRemaining == 0, then parse the complete message.
+
+	const uint8* Bytes = static_cast<const uint8*>(Data);
+	BinaryFrameBuffer.Append(Bytes, Size);
+
+	if (BytesRemaining > 0)
+	{
+		// More fragments coming — wait for the rest
+		return;
+	}
+
+	// Full message received — interpret as UTF-8 JSON
+	const FString JsonString = FString(UTF8_TO_TCHAR(
+		reinterpret_cast<const char*>(BinaryFrameBuffer.GetData())));
+
+	BinaryFrameBuffer.Reset();
+
+	// Route through the existing text message handler
+	OnWsMessage(JsonString);
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsWebSocketProxy.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsWebSocketProxy.h
index f03a54d..50dec68 100644
--- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsWebSocketProxy.h
+++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsWebSocketProxy.h
@@ -179,4 +179,8 @@ private:
 	TSharedPtr<IWebSocket> WebSocket;
 	EElevenLabsConnectionState ConnectionState = EElevenLabsConnectionState::Disconnected;
 	FElevenLabsConversationInfo ConversationInfo;
+
+	// Accumulation buffer for multi-fragment binary WebSocket frames.
+	// ElevenLabs sends JSON as binary frames; large messages arrive in fragments.
+	TArray<uint8> BinaryFrameBuffer;
 };