// Merge Whisper's transcript segments with pyannote's speaker turns into a readable conversation. // // The two inputs come from different models that never saw each other: Whisper says WHAT was said // and when, pyannote says WHO was speaking and when. The only bridge is time. Each transcript // segment is attributed to the speaker whose turns overlap it most; a segment overlapping nobody // (Whisper and pyannote disagree about where speech is — it happens at boundaries) stays with the // previous speaker rather than inventing a gap. // // Attribution is at segment granularity on purpose. Turn-taking in a real conversation happens at // the scale of sentences, and whisper.cpp's segment timestamps are plenty for that; word-level // alignment (whisperx's wav2vec2 stage) earns its cost only for interruptions mid-sentence, which // a voice memo of a conversation can afford to get slightly wrong. If boundaries feel sloppy in // practice, that stage can return as a worker endpoint — see ~/dockers/torch-worker. import type { WhisperSegment } from './transcribe'; import { appendSegment, isHallucinatedSilence } from './transcribe'; import type { SpeakerTurn } from './worker'; export type ConversationResult = { text: string; speakerCount: number; }; /** Seconds of a segment's span covered by a speaker's turns. */ function overlapWith(segment: { start: number; end: number }, turns: SpeakerTurn[], speaker: string): number { let covered = 0; for (const turn of turns) { if (turn.speaker !== speaker) continue; covered += Math.max(0, Math.min(segment.end, turn.end) - Math.max(segment.start, turn.start)); } return covered; } /** * Render transcript segments + speaker turns as a conversation. * * Speakers are renamed by order of first appearance — pyannote's SPEAKER_00/01 labels are * clustering artifacts with no stable meaning, and "Speaker 1" reads better in a transcript. * Consecutive segments by the same speaker join into one block, with the same punctuation-aware * joining and hallucination filtering the solo path uses. */ export function toConversation(segments: WhisperSegment[], turns: SpeakerTurn[]): ConversationResult { const speakerNames = new Map(); const nameFor = (label: string): string => { let name = speakerNames.get(label); if (!name) { name = `Speaker ${speakerNames.size + 1}`; speakerNames.set(label, name); } return name; }; const labels = [...new Set(turns.map((t) => t.speaker))]; const blocks: { name: string; text: string }[] = []; let currentLabel: string | null = null; for (const segment of segments) { const text = (segment.text ?? '').replace(/\s+/g, ' ').trim(); if (!text || isHallucinatedSilence(segment)) continue; let label: string | null = currentLabel; if (segment.start !== undefined && segment.end !== undefined) { const span = { start: segment.start, end: segment.end }; let best = 0; for (const candidate of labels) { const covered = overlapWith(span, turns, candidate); if (covered > best) { best = covered; label = candidate; } } // best === 0 → nobody overlaps; keep the previous speaker rather than guessing. } // Before any turn matches (e.g. transcript starts a hair before the first turn), attribute // forward to the first speaker instead of dropping words. label = label ?? labels[0] ?? 'SPEAKER_00'; const last = blocks[blocks.length - 1]; if (last && label === currentLabel) { last.text = appendSegment(last.text, text); } else { blocks.push({ name: nameFor(label), text }); currentLabel = label; } } return { text: blocks.map((b) => `${b.name}: ${b.text}`).join('\n\n'), speakerCount: speakerNames.size, }; }