ConvReplay: split agent utterances on silence gaps (fix reply glued to greeting)

A first-message greeting opens the agent segment but never fires OnAgentStoppedSpeaking (no agent_response), so the segment stayed open and the later reply was appended to it — inheriting the greeting's early StartTime and playing before the player's question on replay.

RecordAgentAudio now flushes the open segment and starts a fresh one when audio resumes after a >1.5s gap, so the greeting keeps its time and the reply gets its own (correct, later) timestamp. Also flush open agent segments at record stop. Added record-side diagnostic logs behind ps.convreplay.Debug (agent-START/STOP/SPLIT, player-SEG, user-TEXT).

Record-side fix: existing sidecars keep the old segmentation; re-record to benefit.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Private/PS_AI_ConvReplay_Subsystem.cpp

@@ -316,6 +316,12 @@ void UPS_AI_ConvReplay_Subsystem::EndRecordingSession()
 	}
 	RecordingMicComps.Reset();
 
+	// Flush any agent utterance still open at stop.
+	for (UPS_AI_ConvReplay_AgentTag* Tag : Tags)
+	{
+		if (Tag) FlushAgentSegment(Tag);
+	}
+
 	AgentAccum.Reset();
 	PlayerAccum.Reset();
 
@@ -357,25 +363,54 @@ void UPS_AI_ConvReplay_Subsystem::RecordAgentStarted(UPS_AI_ConvReplay_AgentTag*
 	if (!bRecording || !Tag) return;
 	FAgentAccum& A = AgentAccum.FindOrAdd(Tag);
 	A.StartTime = NowDemoSeconds();
+	A.LastAudioTime = A.StartTime;
 	A.PCM.Reset();
 	A.bOpen = true;
 	if (AActor* Owner = Tag->GetOwner())
 	{
 		A.Pos = Owner->GetActorLocation();
 	}
+	if (CVarConvReplayDebug.GetValueOnGameThread() > 0)
+	{
+		UE_LOG(LogPS_AI_ConvReplay, Log, TEXT("REC agent-START  id=%s t=%.2f"), *Tag->GetResolvedId(), A.StartTime);
+	}
 }
 
 void UPS_AI_ConvReplay_Subsystem::RecordAgentAudio(UPS_AI_ConvReplay_AgentTag* Tag, const TArray<uint8>& PCM)
 {
 	if (!bRecording || !Tag || PCM.Num() == 0) return;
 	FAgentAccum& A = AgentAccum.FindOrAdd(Tag);
+	const double Now = NowDemoSeconds();
+	const bool bDbg = CVarConvReplayDebug.GetValueOnGameThread() > 0;
 	if (!A.bOpen)
 	{
-		// Audio before an explicit start — open a segment defensively.
+		// Audio before an explicit OnAgentStartedSpeaking — open a segment defensively.
-		A.StartTime = NowDemoSeconds();
+		A.StartTime = Now;
+		A.LastAudioTime = Now;
+		A.bOpen = true;
+		if (AActor* Owner = Tag->GetOwner()) A.Pos = Owner->GetActorLocation();
+		if (bDbg) UE_LOG(LogPS_AI_ConvReplay, Warning, TEXT("REC agent-audio-NO-START (defensive open) id=%s t=%.2f"), *Tag->GetResolvedId(), Now);
+	}
+	else if (A.PCM.Num() > 0 && (Now - A.LastAudioTime) > AgentGapSplitSeconds)
+	{
+		// Audio resumed after a long gap while the segment is still open. The earlier
+		// audio was a DISTINCT utterance — typically the first-message greeting, which
+		// never fires OnAgentStoppedSpeaking (no agent_response), so the segment stayed
+		// open and the real reply would otherwise be glued onto the greeting's early
+		// StartTime. Flush the earlier utterance at ITS time, then start a fresh segment
+		// here so this audio (the reply) gets its correct, later timestamp.
+		if (bDbg)
+		{
+			UE_LOG(LogPS_AI_ConvReplay, Log,
+				TEXT("REC agent-SPLIT  id=%s gap=%.2fs → flush segStart=%.2f, new seg at %.2f"),
+				*Tag->GetResolvedId(), Now - A.LastAudioTime, A.StartTime, Now);
+		}
+		FlushAgentSegment(Tag);          // writes the earlier utterance, resets A
+		A.StartTime = Now;
 		A.bOpen = true;
 		if (AActor* Owner = Tag->GetOwner()) A.Pos = Owner->GetActorLocation();
 	}
+	A.LastAudioTime = Now;
 	A.PCM.Append(PCM);
 }
 
@@ -385,32 +420,44 @@ void UPS_AI_ConvReplay_Subsystem::RecordAgentText(UPS_AI_ConvReplay_AgentTag* Ta
 	AgentAccum.FindOrAdd(Tag).Text = Text;
 }
 
-void UPS_AI_ConvReplay_Subsystem::RecordAgentStopped(UPS_AI_ConvReplay_AgentTag* Tag)
+void UPS_AI_ConvReplay_Subsystem::FlushAgentSegment(UPS_AI_ConvReplay_AgentTag* Tag)
 {
-	if (!bRecording || !Tag) return;
+	if (!Tag) return;
 	FAgentAccum* A = AgentAccum.Find(Tag);
-	if (!A || !A->bOpen || A->PCM.Num() == 0)
+	if (!A) return;
-	{
-		if (A) { A->bOpen = false; A->PCM.Reset(); A->Text.Reset(); }
-		return;
-	}
 
-	FPS_AI_ConvReplay_Record R;
+	if (A->bOpen && A->PCM.Num() > 0)
-	R.Kind = static_cast<uint8>(EPS_AI_ConvReplay_Kind::AgentSpeech);
+	{
-	R.TimeSec = A->StartTime;
+		FPS_AI_ConvReplay_Record R;
-	R.DurationSec = static_cast<double>(A->PCM.Num() / 2) / PS_AI_ConvReplay_Audio::SampleRate;
+		R.Kind = static_cast<uint8>(EPS_AI_ConvReplay_Kind::AgentSpeech);
-	R.Id = Tag->GetResolvedId();
+		R.TimeSec = A->StartTime;
-	R.Speaker = 0;
+		R.DurationSec = static_cast<double>(A->PCM.Num() / 2) / PS_AI_ConvReplay_Audio::SampleRate;
-	R.Text = A->Text;
+		R.Id = Tag->GetResolvedId();
-	R.Position = A->Pos;
+		R.Speaker = 0;
-	EncodeUtterance(A->PCM, R.Audio);
+		R.Text = A->Text;
-	WriteRecord(R);
+		R.Position = A->Pos;
+		EncodeUtterance(A->PCM, R.Audio);
+		WriteRecord(R);
+
+		if (CVarConvReplayDebug.GetValueOnGameThread() > 0)
+		{
+			UE_LOG(LogPS_AI_ConvReplay, Log,
+				TEXT("REC agent-STOP   id=%s start=%.2f now=%.2f dur=%.2f text='%.50s'"),
+				*R.Id, R.TimeSec, NowDemoSeconds(), R.DurationSec, *R.Text);
+		}
+	}
 
 	A->bOpen = false;
 	A->PCM.Reset();
 	A->Text.Reset();
 }
 
+void UPS_AI_ConvReplay_Subsystem::RecordAgentStopped(UPS_AI_ConvReplay_AgentTag* Tag)
+{
+	if (!bRecording || !Tag) return;
+	FlushAgentSegment(Tag);
+}
+
 void UPS_AI_ConvReplay_Subsystem::RecordAction(UPS_AI_ConvReplay_AgentTag* Tag, const FString& ActionName)
 {
 	if (!bRecording || !Tag) return;
@@ -457,6 +504,10 @@ void UPS_AI_ConvReplay_Subsystem::RecordUserTranscript(UPS_AI_ConvReplay_AgentTa
 		}
 	}
 	WriteRecord(R);
+	if (CVarConvReplayDebug.GetValueOnGameThread() > 0)
+	{
+		UE_LOG(LogPS_AI_ConvReplay, Log, TEXT("REC user-TEXT    t=%.2f text='%.50s'"), R.TimeSec, *Text);
+	}
 }
 
 void UPS_AI_ConvReplay_Subsystem::RecordPlayerAudio(APawn* SpeakerPawn, const TArray<uint8>& PCM)
@@ -500,6 +551,12 @@ void UPS_AI_ConvReplay_Subsystem::FlushPlayerSegment(APawn* Pawn)
 		R.Position = A->Pos;
 		EncodeUtterance(A->PCM, R.Audio);
 		WriteRecord(R);
+		if (CVarConvReplayDebug.GetValueOnGameThread() > 0)
+		{
+			UE_LOG(LogPS_AI_ConvReplay, Log,
+				TEXT("REC player-SEG   id=%s start=%.2f now=%.2f dur=%.2f"),
+				*R.Id, R.TimeSec, NowDemoSeconds(), R.DurationSec);
+		}
 	}
 	A->bActive = false;
 	A->PCM.Reset();

Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Public/PS_AI_ConvReplay_Subsystem.h

@@ -104,6 +104,7 @@ private:
 
 	// Helpers.
 	double NowDemoSeconds() const;
+	void FlushAgentSegment(UPS_AI_ConvReplay_AgentTag* Tag);   // write + reset an agent utterance
 	void FlushPlayerSegment(APawn* Pawn);
 	static bool IsVoiced(const TArray<uint8>& PCM16);
 	static FString ResolvePlayerId(APawn* Pawn);
@@ -132,6 +133,7 @@ private:
 	struct FAgentAccum
 	{
 		double StartTime = 0.0;
+		double LastAudioTime = 0.0;  // demo time of the last appended chunk (gap detection)
 		FVector Pos = FVector::ZeroVector;
 		FString Text;
 		TArray<uint8> PCM;
@@ -159,6 +161,7 @@ private:
 	TArray<TObjectPtr<UAudioComponent>> PlayerAudioPool;
 
 	// Tunables.
+	static constexpr float AgentGapSplitSeconds = 1.5f;  // split an agent utterance if audio resumes after this gap (greeting vs reply)
 	static constexpr float PlayerSilenceFlushSec = 0.6f;
 	static constexpr float VoicedRmsThreshold = 0.02f;
 	static constexpr float ScrubBackThreshold = 0.05f;   // seconds jumped back => scrub