j.foucher/PS_AI_Agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ConvReplay: split agent utterances on silence gaps (fix reply glued to greeting)

Browse Source 
A first-message greeting opens the agent segment but never fires OnAgentStoppedSpeaking (no agent_response), so the segment stayed open and the later reply was appended to it — inheriting the greeting's early StartTime and playing before the player's question on replay.

RecordAgentAudio now flushes the open segment and starts a fresh one when audio resumes after a >1.5s gap, so the greeting keeps its time and the reply gets its own (correct, later) timestamp. Also flush open agent segments at record stop. Added record-side diagnostic logs behind ps.convreplay.Debug (agent-START/STOP/SPLIT, player-SEG, user-TEXT).

Record-side fix: existing sidecars keep the old segmentation; re-record to benefit.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
j.foucher
2026-07-06 10:06:40 +02:00
parent c783613117
commit 086311da38
2 changed files with 79 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Private/PS_AI_ConvReplay_Subsystem.cpp
View File

@@ -316,6 +316,12 @@ void UPS_AI_ConvReplay_Subsystem::EndRecordingSession()
}
RecordingMicComps.Reset();
// Flush any agent utterance still open at stop.
for (UPS_AI_ConvReplay_AgentTag* Tag : Tags)
{
if (Tag) FlushAgentSegment(Tag);
}
AgentAccum.Reset();
PlayerAccum.Reset();
@@ -357,25 +363,54 @@ void UPS_AI_ConvReplay_Subsystem::RecordAgentStarted(UPS_AI_ConvReplay_AgentTag*
if (!bRecording || !Tag) return;
FAgentAccum& A = AgentAccum.FindOrAdd(Tag);
A.StartTime = NowDemoSeconds();
A.LastAudioTime = A.StartTime;
A.PCM.Reset();
A.bOpen = true;
if (AActor* Owner = Tag->GetOwner())
{
A.Pos = Owner->GetActorLocation();
}
if (CVarConvReplayDebug.GetValueOnGameThread() > 0)
{
UE_LOG(LogPS_AI_ConvReplay, Log, TEXT("REC agent-START id=%s t=%.2f"), *Tag->GetResolvedId(), A.StartTime);
}
}
void UPS_AI_ConvReplay_Subsystem::RecordAgentAudio(UPS_AI_ConvReplay_AgentTag* Tag, const TArray<uint8>& PCM)
{
if (!bRecording || !Tag || PCM.Num() == 0) return;
FAgentAccum& A = AgentAccum.FindOrAdd(Tag);
const double Now = NowDemoSeconds();
const bool bDbg = CVarConvReplayDebug.GetValueOnGameThread() > 0;
if (!A.bOpen)
{
// Audio before an explicit start — open a segment defensively.
A.StartTime = NowDemoSeconds();
// Audio before an explicit OnAgentStartedSpeaking — open a segment defensively.
A.StartTime = Now;
A.LastAudioTime = Now;
A.bOpen = true;
if (AActor* Owner = Tag->GetOwner()) A.Pos = Owner->GetActorLocation();
if (bDbg) UE_LOG(LogPS_AI_ConvReplay, Warning, TEXT("REC agent-audio-NO-START (defensive open) id=%s t=%.2f"), *Tag->GetResolvedId(), Now);
}
else if (A.PCM.Num() > 0 && (Now - A.LastAudioTime) > AgentGapSplitSeconds)
{
// Audio resumed after a long gap while the segment is still open. The earlier
// audio was a DISTINCT utterance — typically the first-message greeting, which
// never fires OnAgentStoppedSpeaking (no agent_response), so the segment stayed
// open and the real reply would otherwise be glued onto the greeting's early
// StartTime. Flush the earlier utterance at ITS time, then start a fresh segment
// here so this audio (the reply) gets its correct, later timestamp.
if (bDbg)
{
UE_LOG(LogPS_AI_ConvReplay, Log,
TEXT("REC agent-SPLIT id=%s gap=%.2fs → flush segStart=%.2f, new seg at %.2f"),
*Tag->GetResolvedId(), Now - A.LastAudioTime, A.StartTime, Now);
}
FlushAgentSegment(Tag); // writes the earlier utterance, resets A
A.StartTime = Now;
A.bOpen = true;
if (AActor* Owner = Tag->GetOwner()) A.Pos = Owner->GetActorLocation();
}
A.LastAudioTime = Now;
A.PCM.Append(PCM);
}
@@ -385,16 +420,14 @@ void UPS_AI_ConvReplay_Subsystem::RecordAgentText(UPS_AI_ConvReplay_AgentTag* Ta
AgentAccum.FindOrAdd(Tag).Text = Text;
}
void UPS_AI_ConvReplay_Subsystem::RecordAgentStopped(UPS_AI_ConvReplay_AgentTag* Tag)
void UPS_AI_ConvReplay_Subsystem::FlushAgentSegment(UPS_AI_ConvReplay_AgentTag* Tag)
{
if (!bRecording || !Tag) return;
if (!Tag) return;
FAgentAccum* A = AgentAccum.Find(Tag);
if (!A || !A->bOpen || A->PCM.Num() == 0)
{
if (A) { A->bOpen = false; A->PCM.Reset(); A->Text.Reset(); }
return;
}
if (!A) return;
if (A->bOpen && A->PCM.Num() > 0)
{
FPS_AI_ConvReplay_Record R;
R.Kind = static_cast<uint8>(EPS_AI_ConvReplay_Kind::AgentSpeech);
R.TimeSec = A->StartTime;
@@ -406,11 +439,25 @@ void UPS_AI_ConvReplay_Subsystem::RecordAgentStopped(UPS_AI_ConvReplay_AgentTag*
EncodeUtterance(A->PCM, R.Audio);
WriteRecord(R);
if (CVarConvReplayDebug.GetValueOnGameThread() > 0)
{
UE_LOG(LogPS_AI_ConvReplay, Log,
TEXT("REC agent-STOP id=%s start=%.2f now=%.2f dur=%.2f text='%.50s'"),
*R.Id, R.TimeSec, NowDemoSeconds(), R.DurationSec, *R.Text);
}
}
A->bOpen = false;
A->PCM.Reset();
A->Text.Reset();
}
void UPS_AI_ConvReplay_Subsystem::RecordAgentStopped(UPS_AI_ConvReplay_AgentTag* Tag)
{
if (!bRecording || !Tag) return;
FlushAgentSegment(Tag);
}
void UPS_AI_ConvReplay_Subsystem::RecordAction(UPS_AI_ConvReplay_AgentTag* Tag, const FString& ActionName)
{
if (!bRecording || !Tag) return;
@@ -457,6 +504,10 @@ void UPS_AI_ConvReplay_Subsystem::RecordUserTranscript(UPS_AI_ConvReplay_AgentTa
}
}
WriteRecord(R);
if (CVarConvReplayDebug.GetValueOnGameThread() > 0)
{
UE_LOG(LogPS_AI_ConvReplay, Log, TEXT("REC user-TEXT t=%.2f text='%.50s'"), R.TimeSec, *Text);
}
}
void UPS_AI_ConvReplay_Subsystem::RecordPlayerAudio(APawn* SpeakerPawn, const TArray<uint8>& PCM)
@@ -500,6 +551,12 @@ void UPS_AI_ConvReplay_Subsystem::FlushPlayerSegment(APawn* Pawn)
R.Position = A->Pos;
EncodeUtterance(A->PCM, R.Audio);
WriteRecord(R);
if (CVarConvReplayDebug.GetValueOnGameThread() > 0)
{
UE_LOG(LogPS_AI_ConvReplay, Log,
TEXT("REC player-SEG id=%s start=%.2f now=%.2f dur=%.2f"),
*R.Id, R.TimeSec, NowDemoSeconds(), R.DurationSec);
}
}
A->bActive = false;
A->PCM.Reset();

3
Unreal/PS_AI_Agent/Plugins/PS_AI_ConvReplay/Source/PS_AI_ConvReplay/Public/PS_AI_ConvReplay_Subsystem.h
View File

@@ -104,6 +104,7 @@ private:
// Helpers.
double NowDemoSeconds() const;
void FlushAgentSegment(UPS_AI_ConvReplay_AgentTag* Tag); // write + reset an agent utterance
void FlushPlayerSegment(APawn* Pawn);
static bool IsVoiced(const TArray<uint8>& PCM16);
static FString ResolvePlayerId(APawn* Pawn);
@@ -132,6 +133,7 @@ private:
struct FAgentAccum
{
double StartTime = 0.0;
double LastAudioTime = 0.0; // demo time of the last appended chunk (gap detection)
FVector Pos = FVector::ZeroVector;
FString Text;
TArray<uint8> PCM;
@@ -159,6 +161,7 @@ private:
TArray<TObjectPtr<UAudioComponent>> PlayerAudioPool;
// Tunables.
static constexpr float AgentGapSplitSeconds = 1.5f; // split an agent utterance if audio resumes after this gap (greeting vs reply)
static constexpr float PlayerSilenceFlushSec = 0.6f;
static constexpr float VoicedRmsThreshold = 0.02f;
static constexpr float ScrubBackThreshold = 0.05f; // seconds jumped back => scrub