pastilhasandClaude Fable 5
dd3dbb611b
A probe on the torch stage worker (~/dockers/torch-worker) runs beside every transcription and decides what the recording is: solo dictation returns as paragraphs exactly as before, several voices return speaker-labeled with conversation: true. Nobody declares which — the caller cannot know. Transcription and the worker each get a serial FIFO queue on their own resource, so the probe costs a solo memo nothing and multi-user traffic stays fair. The worker is optional and best-effort throughout: unreachable or failing, everything still transcribes, just never labeled. Speaker attribution is segment-granularity by maximal turn overlap — whisper.cpp's own timestamps, no alignment stage. If boundaries prove sloppy, wav2vec2 alignment returns as a worker endpoint. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
93 lines
3.8 KiB
TypeScript
93 lines
3.8 KiB
TypeScript
// Merge Whisper's transcript segments with pyannote's speaker turns into a readable conversation.
|
|
//
|
|
// The two inputs come from different models that never saw each other: Whisper says WHAT was said
|
|
// and when, pyannote says WHO was speaking and when. The only bridge is time. Each transcript
|
|
// segment is attributed to the speaker whose turns overlap it most; a segment overlapping nobody
|
|
// (Whisper and pyannote disagree about where speech is — it happens at boundaries) stays with the
|
|
// previous speaker rather than inventing a gap.
|
|
//
|
|
// Attribution is at segment granularity on purpose. Turn-taking in a real conversation happens at
|
|
// the scale of sentences, and whisper.cpp's segment timestamps are plenty for that; word-level
|
|
// alignment (whisperx's wav2vec2 stage) earns its cost only for interruptions mid-sentence, which
|
|
// a voice memo of a conversation can afford to get slightly wrong. If boundaries feel sloppy in
|
|
// practice, that stage can return as a worker endpoint — see ~/dockers/torch-worker.
|
|
|
|
import type { WhisperSegment } from './transcribe';
|
|
import { appendSegment, isHallucinatedSilence } from './transcribe';
|
|
import type { SpeakerTurn } from './worker';
|
|
|
|
export type ConversationResult = {
|
|
text: string;
|
|
speakerCount: number;
|
|
};
|
|
|
|
/** Seconds of a segment's span covered by a speaker's turns. */
|
|
function overlapWith(segment: { start: number; end: number }, turns: SpeakerTurn[], speaker: string): number {
|
|
let covered = 0;
|
|
for (const turn of turns) {
|
|
if (turn.speaker !== speaker) continue;
|
|
covered += Math.max(0, Math.min(segment.end, turn.end) - Math.max(segment.start, turn.start));
|
|
}
|
|
return covered;
|
|
}
|
|
|
|
/**
|
|
* Render transcript segments + speaker turns as a conversation.
|
|
*
|
|
* Speakers are renamed by order of first appearance — pyannote's SPEAKER_00/01 labels are
|
|
* clustering artifacts with no stable meaning, and "Speaker 1" reads better in a transcript.
|
|
* Consecutive segments by the same speaker join into one block, with the same punctuation-aware
|
|
* joining and hallucination filtering the solo path uses.
|
|
*/
|
|
export function toConversation(segments: WhisperSegment[], turns: SpeakerTurn[]): ConversationResult {
|
|
const speakerNames = new Map<string, string>();
|
|
const nameFor = (label: string): string => {
|
|
let name = speakerNames.get(label);
|
|
if (!name) {
|
|
name = `Speaker ${speakerNames.size + 1}`;
|
|
speakerNames.set(label, name);
|
|
}
|
|
return name;
|
|
};
|
|
|
|
const labels = [...new Set(turns.map((t) => t.speaker))];
|
|
|
|
const blocks: { name: string; text: string }[] = [];
|
|
let currentLabel: string | null = null;
|
|
|
|
for (const segment of segments) {
|
|
const text = (segment.text ?? '').replace(/\s+/g, ' ').trim();
|
|
if (!text || isHallucinatedSilence(segment)) continue;
|
|
|
|
let label: string | null = currentLabel;
|
|
if (segment.start !== undefined && segment.end !== undefined) {
|
|
const span = { start: segment.start, end: segment.end };
|
|
let best = 0;
|
|
for (const candidate of labels) {
|
|
const covered = overlapWith(span, turns, candidate);
|
|
if (covered > best) {
|
|
best = covered;
|
|
label = candidate;
|
|
}
|
|
}
|
|
// best === 0 → nobody overlaps; keep the previous speaker rather than guessing.
|
|
}
|
|
// Before any turn matches (e.g. transcript starts a hair before the first turn), attribute
|
|
// forward to the first speaker instead of dropping words.
|
|
label = label ?? labels[0] ?? 'SPEAKER_00';
|
|
|
|
const last = blocks[blocks.length - 1];
|
|
if (last && label === currentLabel) {
|
|
last.text = appendSegment(last.text, text);
|
|
} else {
|
|
blocks.push({ name: nameFor(label), text });
|
|
currentLabel = label;
|
|
}
|
|
}
|
|
|
|
return {
|
|
text: blocks.map((b) => `${b.name}: ${b.text}`).join('\n\n'),
|
|
speakerCount: speakerNames.size,
|
|
};
|
|
}
|