transcription: tune whisper for single-speaker audio, and paragraph the output

Whisper feeds its own decoded output forward as the prompt for the next 30s
window, which is what makes it loop once it slips. max_context is the budget for
that carried text; setting it to 0 stops the loops but also disables the initial
prompt, since whisper.cpp gates both on n_max_text_ctx > 0. carry_initial_prompt
puts the prompt in a static slot that is filled first and in full, so making the
prompt longer than the budget leaves nothing over for decoded text — the prompt
reaches every window and none of the model's own output does.

The prompt is a style exemplar, not an instruction: whisper imitates what it is
primed with. A first version described the format ("Commas separate clauses")
and a character surname came back as "Commas", so both prompts are now ordinary
conversational prose with no meta-language, one per language.

Also switch to verbose_json for segment timestamps and rejoin the segments into
paragraphs on pause length, rather than emitting one line per utterance with a
leading space; decode with beam search instead of greedy; widen the VAD segments
so they break at real pauses instead of at breaths; and stop verbose_json from
re-running a full language auto-detect that step 1 already answered.

Measured on a 43 minute episode against the previous output: redundant repeated
sentences 18 -> 9 (the long distinctive loops are gone, what remains is a
catchphrase), paragraphs 1 -> 93, lone-punctuation lines 3 -> 0, lines with a
leading space 1021 -> 0.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

src/servers/api/stt/transcribe.test.ts

@@ -0,0 +1,78 @@
+import { describe, expect, it } from 'bun:test';
+import { toParagraphs } from './transcribe';
+import type { WhisperSegment } from './transcribe';
+
+// Whisper's own shape: leading space on every segment, times in seconds.
+const seg = (text: string, start: number, end: number, extra: Partial<WhisperSegment> = {}): WhisperSegment => ({
+  text,
+  start,
+  end,
+  ...extra,
+});
+
+describe('toParagraphs', () => {
+  it('joins segments spoken without a real pause into one paragraph', () => {
+    const out = toParagraphs([seg(' I went to the shop.', 0, 2), seg(' It was closed.', 2.2, 4)]);
+    expect(out).toBe('I went to the shop. It was closed.');
+  });
+
+  it('starts a new paragraph after a long silence', () => {
+    const out = toParagraphs([seg(' First thought.', 0, 2), seg(' Second thought.', 5, 7)]);
+    expect(out).toBe('First thought.\n\nSecond thought.');
+  });
+
+  it('does not break on a breath-length pause', () => {
+    const out = toParagraphs([seg(' Hang on,', 0, 1), seg(' let me think.', 1.5, 3)]);
+    expect(out).toBe('Hang on, let me think.');
+  });
+
+  // The lone "." lines in the old response_format=text output came from segments like this.
+  it('attaches stranded punctuation without a space in front of it', () => {
+    const out = toParagraphs([seg(' See you tomorrow morning', 0, 2), seg(' .', 2.1, 2.2)]);
+    expect(out).toBe('See you tomorrow morning.');
+  });
+
+  it('drops segments that are probably silence decoded with low confidence', () => {
+    const out = toParagraphs([
+      seg(' Real speech here.', 0, 2),
+      seg(' Thanks for watching!', 2.1, 3, { no_speech_prob: 0.95, avg_logprob: -1.8 }),
+      seg(' More real speech.', 3.1, 5),
+    ]);
+    expect(out).toBe('Real speech here. More real speech.');
+  });
+
+  it('keeps a confident segment even when it looks like silence', () => {
+    const out = toParagraphs([seg(' Definitely said.', 0, 2, { no_speech_prob: 0.95, avg_logprob: -0.2 })]);
+    expect(out).toBe('Definitely said.');
+  });
+
+  it('keeps segments that report neither probability', () => {
+    const out = toParagraphs([seg(' No metadata at all.', 0, 2)]);
+    expect(out).toBe('No metadata at all.');
+  });
+
+  it('measures the pause from the last kept segment, not the dropped one', () => {
+    const out = toParagraphs([
+      seg(' Before.', 0, 2),
+      seg(' Hallucinated.', 2.1, 9, { no_speech_prob: 0.9, avg_logprob: -2 }),
+      seg(' After.', 9.1, 11),
+    ]);
+    // The gap that matters is 2 -> 9.1, so these are separate thoughts.
+    expect(out).toBe('Before.\n\nAfter.');
+  });
+
+  it('collapses internal whitespace and ignores empty segments', () => {
+    const out = toParagraphs([seg('  Too   many\n spaces. ', 0, 2), seg('   ', 2.1, 2.2)]);
+    expect(out).toBe('Too many spaces.');
+  });
+
+  it('returns an empty string when every segment is filtered out', () => {
+    expect(toParagraphs([])).toBe('');
+    expect(toParagraphs([seg('  ', 0, 1)])).toBe('');
+  });
+
+  it('falls back to no break when segments carry no timings', () => {
+    const out = toParagraphs([{ text: ' One.' }, { text: ' Two.' }]);
+    expect(out).toBe('One. Two.');
+  });
+});

src/servers/api/stt/transcribe.ts

@@ -7,6 +7,10 @@
 // The detection must win when Whisper is confident. `spokenLanguages` only breaks ties on clips
 // Whisper is genuinely unsure about, where its low-confidence guess defaults to English.
 //
+// The transcription pass is tuned for one person talking into a phone, which is what this is mostly
+// used for. See the constants below: the defaults whisper.cpp ships are aimed at clean dictation and
+// produce loops, clipped words and breath-length paragraph breaks on real voice memos.
+//
 // Both requests pass `timeout: false`. Bun's fetch aborts at 300s by default, and whisper.cpp runs at
 // roughly 8x realtime, so anything over ~40 minutes of audio died with "The operation timed out"
 // about 20 seconds short of the answer. AbortSignal.timeout() does not raise that ceiling — only
@@ -48,6 +52,123 @@ const LANG_NAME_TO_ISO: Record<string, string> = {
   korean: 'ko',
 };
 
+// Whisper reuses its own decoded output as the prompt for the next 30-second window, which is what
+// makes it fall into loops ("I know. I know. I know.") once it slips. `max_context` is the budget for
+// that carried text, and setting it to 0 stops the loops — but the same switch also disables the
+// initial prompt, since whisper.cpp gates both on `n_max_text_ctx > 0`.
+//
+// The way to keep the prompt and still starve the loop: `carry_initial_prompt` puts the prompt in a
+// static slot that is filled first and in full, and only the leftover budget goes to decoded text.
+// Every prompt below is comfortably longer than this budget, so there is never any leftover — the
+// vocabulary and style reach every window and none of the model's own output does.
+const PROMPT_CONTEXT_TOKENS = 96;
+
+// Whisper imitates whatever it is primed with; it does not take instructions. So each prompt is a
+// worked example of the output wanted — one person talking, punctuated the way it should come back —
+// and nothing else. An earlier version opened by *describing* the format ("Commas separate clauses,
+// and apostrophes are written properly") and that backfired: priming words which never occur in
+// speech put them in play, and a character surname came back from a Monk episode as "Commas".
+// Everything here is ordinary conversational language for that reason.
+//
+// Each is written in its own language, since an English prompt on Portuguese audio biases the
+// output, and each deliberately runs past PROMPT_CONTEXT_TOKENS — see above for why the length is
+// load-bearing. Overflow is trimmed from the front, so nothing depends on the opening words.
+//
+// Languages without an entry fall through to no prompt at all, which is handled explicitly below.
+const TRANSCRIPT_PROMPTS: Record<string, string> = {
+  en:
+    'Hey, sorry for the late reply — I was stuck in a meeting all morning and only just got out. ' +
+    "I had a look at the document you sent over, and honestly, I think it's mostly fine. There are " +
+    "a couple of things I'd change in the second half, but nothing serious. The introduction reads " +
+    'really well. Let me know when you have a few minutes and we can talk it through properly. ' +
+    "I'm around all afternoon, so whenever suits you is fine by me. Thanks again for putting it " +
+    'together, I know it took a while.',
+  pt:
+    'Olá, desculpa a demora a responder — estive numa reunião toda a manhã e só agora consegui ' +
+    'sair. Já vi o documento que enviaste e, sinceramente, acho que está quase tudo bem. Há duas ' +
+    'ou três coisas que mudava na segunda metade, mas nada de grave. A introdução está muito bem ' +
+    'escrita. Diz-me quando tiveres uns minutos e falamos com calma. Estou por aqui toda a tarde, ' +
+    'por isso quando te der jeito está bem para mim. Obrigado outra vez por teres tratado disto, ' +
+    'sei que deu trabalho.',
+};
+
+// Beam search rather than the default greedy decode (whisper.cpp ships `beam_size = -1`). It is the
+// single biggest quality lever here and it costs time, which this flow has — nothing waits on it.
+const BEAM_SIZE = 5;
+const BEST_OF = 5;
+
+// The whisper container already runs silero VAD, but its defaults assume clean dictation. Widen the
+// segments so they break at real pauses instead of at breaths, and pad the edges so word onsets are
+// not clipped — clipped onsets are what shows up in the output as a word split down the middle.
+const VAD_TUNING: Record<string, string> = {
+  vad_speech_pad_ms: '200', // default 30
+  vad_min_silence_duration_ms: '500', // default 100 — a breath is not a sentence boundary
+  vad_samples_overlap: '0.2', // default 0.1
+  vad_max_speech_duration_s: '25', // default unbounded — keep a segment inside one 30s window
+};
+
+// A silence this long reads as a new thought. Whisper reports segment times against the original
+// audio even when VAD is on (whisper.cpp maps them back through its VAD table), so these really are
+// gaps in the recording rather than artefacts of the filtering.
+const PARAGRAPH_GAP_SECONDS = 1.2;
+const MAX_PARAGRAPH_CHARS = 900;
+
+// OpenAI Whisper's own silence rule: a segment the model thinks is probably not speech *and* decodes
+// with low confidence is a hallucination rather than a transcript. whisper.cpp applies a subtly
+// different form of this internally, so it is worth re-applying to the values it reports — this is
+// what removes the stray words that appear over music and room tone.
+const NO_SPEECH_PROB = 0.6;
+const LOW_CONFIDENCE_LOGPROB = -1.0;
+
+export type WhisperSegment = {
+  text?: string;
+  start?: number;
+  end?: number;
+  no_speech_prob?: number;
+  avg_logprob?: number;
+};
+
+// Defaults are chosen so a segment missing either field is kept: dropping real speech is much worse
+// than keeping a stray word.
+const isHallucinatedSilence = (segment: WhisperSegment) =>
+  (segment.no_speech_prob ?? 0) > NO_SPEECH_PROB && (segment.avg_logprob ?? 0) < LOW_CONFIDENCE_LOGPROB;
+
+// Whisper emits each segment with a leading space and now and then strands trailing punctuation in a
+// segment of its own, which is why the old `response_format=text` output contained lone "." lines.
+const appendSegment = (paragraph: string, text: string) => {
+  if (!paragraph) return text;
+  return /^[.,;:!?…)\]]/.test(text) ? paragraph + text : `${paragraph} ${text}`;
+};
+
+// Whisper returns one segment per utterance, not per paragraph. Rejoin them into prose and start a
+// new paragraph on a long pause, so a voice memo reads as something written rather than as a list of
+// fragments.
+export function toParagraphs(segments: WhisperSegment[]): string {
+  const paragraphs: string[] = [];
+  let current = '';
+  let previousEnd: number | null = null;
+
+  for (const segment of segments) {
+    const text = (segment.text ?? '').replace(/\s+/g, ' ').trim();
+    if (!text || isHallucinatedSilence(segment)) continue;
+
+    const gap = previousEnd !== null && segment.start !== undefined ? segment.start - previousEnd : 0;
+    // Only break an over-long paragraph where a sentence actually ended, never mid-thought.
+    const overlong = current.length >= MAX_PARAGRAPH_CHARS && /[.!?…]["')\]]?$/.test(current);
+
+    if (current && (gap >= PARAGRAPH_GAP_SECONDS || overlong)) {
+      paragraphs.push(current);
+      current = '';
+    }
+
+    current = appendSegment(current, text);
+    if (segment.end !== undefined) previousEnd = segment.end;
+  }
+
+  if (current) paragraphs.push(current);
+  return paragraphs.join('\n\n');
+}
+
 export async function transcribeAudio({
   file,
   whisperUrl,
@@ -102,10 +223,31 @@ export async function transcribeAudio({
   // Step 2: transcribe in the detected language
   const transcribeForm = new FormData();
   transcribeForm.append('file', file);
+  transcribeForm.append('language', detectedLanguage);
   transcribeForm.append('temperature', '0.0');
   transcribeForm.append('temperature_inc', '0.2');
-  transcribeForm.append('response_format', 'text');
+  transcribeForm.append('beam_size', String(BEAM_SIZE));
-  transcribeForm.append('language', detectedLanguage);
+  transcribeForm.append('best_of', String(BEST_OF));
+  transcribeForm.append('suppress_nst', 'true');
+  transcribeForm.append('response_format', 'verbose_json');
+  // verbose_json re-runs a full language auto-detect unless this is set. Step 1 already answered
+  // that question, so it is pure duplicated work on every transcription.
+  transcribeForm.append('no_language_probabilities', 'true');
+  // Segment timestamps drive the paragraphing below and are kept; per-token timestamps are extra
+  // work on every segment and nothing here reads them.
+  transcribeForm.append('token_timestamps', 'false');
+
+  const prompt = TRANSCRIPT_PROMPTS[detectedLanguage];
+  if (prompt) {
+    transcribeForm.append('prompt', prompt);
+    transcribeForm.append('carry_initial_prompt', 'true');
+    transcribeForm.append('max_context', String(PROMPT_CONTEXT_TOKENS));
+  } else {
+    // Nothing worth protecting the budget for, so close the loop-feeding path outright.
+    transcribeForm.append('max_context', '0');
+  }
+
+  for (const [key, value] of Object.entries(VAD_TUNING)) transcribeForm.append(key, value);
 
   const res = await fetch(`${base}/inference`, {
     method: 'POST',
@@ -113,7 +255,11 @@ export async function transcribeAudio({
     timeout: false,
   } as BunRequestInit);
   if (!res.ok) throw new Error(`Transcription failed (${res.status})`);
-  const text = (await res.text()).trim();
+
+  const json = (await res.json()) as { text?: string; segments?: WhisperSegment[] };
+  // Fall back to the flat text for any backend that does not return segments.
+  const segments = json.segments ?? [];
+  const text = segments.length > 0 ? toParagraphs(segments) : (json.text ?? '').trim();
 
   return { text, detectedLanguage };
 }