From f4054193ae19a259fc2df21ea50872689b46a532 Mon Sep 17 00:00:00 2001 From: brunorezio Date: Sun, 26 Jul 2026 17:26:36 +0100 Subject: [PATCH] transcription: tune whisper for single-speaker audio, and paragraph the output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Whisper feeds its own decoded output forward as the prompt for the next 30s window, which is what makes it loop once it slips. max_context is the budget for that carried text; setting it to 0 stops the loops but also disables the initial prompt, since whisper.cpp gates both on n_max_text_ctx > 0. carry_initial_prompt puts the prompt in a static slot that is filled first and in full, so making the prompt longer than the budget leaves nothing over for decoded text — the prompt reaches every window and none of the model's own output does. The prompt is a style exemplar, not an instruction: whisper imitates what it is primed with. A first version described the format ("Commas separate clauses") and a character surname came back as "Commas", so both prompts are now ordinary conversational prose with no meta-language, one per language. Also switch to verbose_json for segment timestamps and rejoin the segments into paragraphs on pause length, rather than emitting one line per utterance with a leading space; decode with beam search instead of greedy; widen the VAD segments so they break at real pauses instead of at breaths; and stop verbose_json from re-running a full language auto-detect that step 1 already answered. Measured on a 43 minute episode against the previous output: redundant repeated sentences 18 -> 9 (the long distinctive loops are gone, what remains is a catchphrase), paragraphs 1 -> 93, lone-punctuation lines 3 -> 0, lines with a leading space 1021 -> 0. Co-Authored-By: Claude Opus 5 --- src/servers/api/stt/transcribe.test.ts | 78 +++++++++++++ src/servers/api/stt/transcribe.ts | 152 ++++++++++++++++++++++++- 2 files changed, 227 insertions(+), 3 deletions(-) create mode 100644 src/servers/api/stt/transcribe.test.ts diff --git a/src/servers/api/stt/transcribe.test.ts b/src/servers/api/stt/transcribe.test.ts new file mode 100644 index 00000000..6bd6d00a --- /dev/null +++ b/src/servers/api/stt/transcribe.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from 'bun:test'; +import { toParagraphs } from './transcribe'; +import type { WhisperSegment } from './transcribe'; + +// Whisper's own shape: leading space on every segment, times in seconds. +const seg = (text: string, start: number, end: number, extra: Partial = {}): WhisperSegment => ({ + text, + start, + end, + ...extra, +}); + +describe('toParagraphs', () => { + it('joins segments spoken without a real pause into one paragraph', () => { + const out = toParagraphs([seg(' I went to the shop.', 0, 2), seg(' It was closed.', 2.2, 4)]); + expect(out).toBe('I went to the shop. It was closed.'); + }); + + it('starts a new paragraph after a long silence', () => { + const out = toParagraphs([seg(' First thought.', 0, 2), seg(' Second thought.', 5, 7)]); + expect(out).toBe('First thought.\n\nSecond thought.'); + }); + + it('does not break on a breath-length pause', () => { + const out = toParagraphs([seg(' Hang on,', 0, 1), seg(' let me think.', 1.5, 3)]); + expect(out).toBe('Hang on, let me think.'); + }); + + // The lone "." lines in the old response_format=text output came from segments like this. + it('attaches stranded punctuation without a space in front of it', () => { + const out = toParagraphs([seg(' See you tomorrow morning', 0, 2), seg(' .', 2.1, 2.2)]); + expect(out).toBe('See you tomorrow morning.'); + }); + + it('drops segments that are probably silence decoded with low confidence', () => { + const out = toParagraphs([ + seg(' Real speech here.', 0, 2), + seg(' Thanks for watching!', 2.1, 3, { no_speech_prob: 0.95, avg_logprob: -1.8 }), + seg(' More real speech.', 3.1, 5), + ]); + expect(out).toBe('Real speech here. More real speech.'); + }); + + it('keeps a confident segment even when it looks like silence', () => { + const out = toParagraphs([seg(' Definitely said.', 0, 2, { no_speech_prob: 0.95, avg_logprob: -0.2 })]); + expect(out).toBe('Definitely said.'); + }); + + it('keeps segments that report neither probability', () => { + const out = toParagraphs([seg(' No metadata at all.', 0, 2)]); + expect(out).toBe('No metadata at all.'); + }); + + it('measures the pause from the last kept segment, not the dropped one', () => { + const out = toParagraphs([ + seg(' Before.', 0, 2), + seg(' Hallucinated.', 2.1, 9, { no_speech_prob: 0.9, avg_logprob: -2 }), + seg(' After.', 9.1, 11), + ]); + // The gap that matters is 2 -> 9.1, so these are separate thoughts. + expect(out).toBe('Before.\n\nAfter.'); + }); + + it('collapses internal whitespace and ignores empty segments', () => { + const out = toParagraphs([seg(' Too many\n spaces. ', 0, 2), seg(' ', 2.1, 2.2)]); + expect(out).toBe('Too many spaces.'); + }); + + it('returns an empty string when every segment is filtered out', () => { + expect(toParagraphs([])).toBe(''); + expect(toParagraphs([seg(' ', 0, 1)])).toBe(''); + }); + + it('falls back to no break when segments carry no timings', () => { + const out = toParagraphs([{ text: ' One.' }, { text: ' Two.' }]); + expect(out).toBe('One. Two.'); + }); +}); diff --git a/src/servers/api/stt/transcribe.ts b/src/servers/api/stt/transcribe.ts index 34e5a3a9..948bf752 100644 --- a/src/servers/api/stt/transcribe.ts +++ b/src/servers/api/stt/transcribe.ts @@ -7,6 +7,10 @@ // The detection must win when Whisper is confident. `spokenLanguages` only breaks ties on clips // Whisper is genuinely unsure about, where its low-confidence guess defaults to English. // +// The transcription pass is tuned for one person talking into a phone, which is what this is mostly +// used for. See the constants below: the defaults whisper.cpp ships are aimed at clean dictation and +// produce loops, clipped words and breath-length paragraph breaks on real voice memos. +// // Both requests pass `timeout: false`. Bun's fetch aborts at 300s by default, and whisper.cpp runs at // roughly 8x realtime, so anything over ~40 minutes of audio died with "The operation timed out" // about 20 seconds short of the answer. AbortSignal.timeout() does not raise that ceiling — only @@ -48,6 +52,123 @@ const LANG_NAME_TO_ISO: Record = { korean: 'ko', }; +// Whisper reuses its own decoded output as the prompt for the next 30-second window, which is what +// makes it fall into loops ("I know. I know. I know.") once it slips. `max_context` is the budget for +// that carried text, and setting it to 0 stops the loops — but the same switch also disables the +// initial prompt, since whisper.cpp gates both on `n_max_text_ctx > 0`. +// +// The way to keep the prompt and still starve the loop: `carry_initial_prompt` puts the prompt in a +// static slot that is filled first and in full, and only the leftover budget goes to decoded text. +// Every prompt below is comfortably longer than this budget, so there is never any leftover — the +// vocabulary and style reach every window and none of the model's own output does. +const PROMPT_CONTEXT_TOKENS = 96; + +// Whisper imitates whatever it is primed with; it does not take instructions. So each prompt is a +// worked example of the output wanted — one person talking, punctuated the way it should come back — +// and nothing else. An earlier version opened by *describing* the format ("Commas separate clauses, +// and apostrophes are written properly") and that backfired: priming words which never occur in +// speech put them in play, and a character surname came back from a Monk episode as "Commas". +// Everything here is ordinary conversational language for that reason. +// +// Each is written in its own language, since an English prompt on Portuguese audio biases the +// output, and each deliberately runs past PROMPT_CONTEXT_TOKENS — see above for why the length is +// load-bearing. Overflow is trimmed from the front, so nothing depends on the opening words. +// +// Languages without an entry fall through to no prompt at all, which is handled explicitly below. +const TRANSCRIPT_PROMPTS: Record = { + en: + 'Hey, sorry for the late reply — I was stuck in a meeting all morning and only just got out. ' + + "I had a look at the document you sent over, and honestly, I think it's mostly fine. There are " + + "a couple of things I'd change in the second half, but nothing serious. The introduction reads " + + 'really well. Let me know when you have a few minutes and we can talk it through properly. ' + + "I'm around all afternoon, so whenever suits you is fine by me. Thanks again for putting it " + + 'together, I know it took a while.', + pt: + 'Olá, desculpa a demora a responder — estive numa reunião toda a manhã e só agora consegui ' + + 'sair. Já vi o documento que enviaste e, sinceramente, acho que está quase tudo bem. Há duas ' + + 'ou três coisas que mudava na segunda metade, mas nada de grave. A introdução está muito bem ' + + 'escrita. Diz-me quando tiveres uns minutos e falamos com calma. Estou por aqui toda a tarde, ' + + 'por isso quando te der jeito está bem para mim. Obrigado outra vez por teres tratado disto, ' + + 'sei que deu trabalho.', +}; + +// Beam search rather than the default greedy decode (whisper.cpp ships `beam_size = -1`). It is the +// single biggest quality lever here and it costs time, which this flow has — nothing waits on it. +const BEAM_SIZE = 5; +const BEST_OF = 5; + +// The whisper container already runs silero VAD, but its defaults assume clean dictation. Widen the +// segments so they break at real pauses instead of at breaths, and pad the edges so word onsets are +// not clipped — clipped onsets are what shows up in the output as a word split down the middle. +const VAD_TUNING: Record = { + vad_speech_pad_ms: '200', // default 30 + vad_min_silence_duration_ms: '500', // default 100 — a breath is not a sentence boundary + vad_samples_overlap: '0.2', // default 0.1 + vad_max_speech_duration_s: '25', // default unbounded — keep a segment inside one 30s window +}; + +// A silence this long reads as a new thought. Whisper reports segment times against the original +// audio even when VAD is on (whisper.cpp maps them back through its VAD table), so these really are +// gaps in the recording rather than artefacts of the filtering. +const PARAGRAPH_GAP_SECONDS = 1.2; +const MAX_PARAGRAPH_CHARS = 900; + +// OpenAI Whisper's own silence rule: a segment the model thinks is probably not speech *and* decodes +// with low confidence is a hallucination rather than a transcript. whisper.cpp applies a subtly +// different form of this internally, so it is worth re-applying to the values it reports — this is +// what removes the stray words that appear over music and room tone. +const NO_SPEECH_PROB = 0.6; +const LOW_CONFIDENCE_LOGPROB = -1.0; + +export type WhisperSegment = { + text?: string; + start?: number; + end?: number; + no_speech_prob?: number; + avg_logprob?: number; +}; + +// Defaults are chosen so a segment missing either field is kept: dropping real speech is much worse +// than keeping a stray word. +const isHallucinatedSilence = (segment: WhisperSegment) => + (segment.no_speech_prob ?? 0) > NO_SPEECH_PROB && (segment.avg_logprob ?? 0) < LOW_CONFIDENCE_LOGPROB; + +// Whisper emits each segment with a leading space and now and then strands trailing punctuation in a +// segment of its own, which is why the old `response_format=text` output contained lone "." lines. +const appendSegment = (paragraph: string, text: string) => { + if (!paragraph) return text; + return /^[.,;:!?…)\]]/.test(text) ? paragraph + text : `${paragraph} ${text}`; +}; + +// Whisper returns one segment per utterance, not per paragraph. Rejoin them into prose and start a +// new paragraph on a long pause, so a voice memo reads as something written rather than as a list of +// fragments. +export function toParagraphs(segments: WhisperSegment[]): string { + const paragraphs: string[] = []; + let current = ''; + let previousEnd: number | null = null; + + for (const segment of segments) { + const text = (segment.text ?? '').replace(/\s+/g, ' ').trim(); + if (!text || isHallucinatedSilence(segment)) continue; + + const gap = previousEnd !== null && segment.start !== undefined ? segment.start - previousEnd : 0; + // Only break an over-long paragraph where a sentence actually ended, never mid-thought. + const overlong = current.length >= MAX_PARAGRAPH_CHARS && /[.!?…]["')\]]?$/.test(current); + + if (current && (gap >= PARAGRAPH_GAP_SECONDS || overlong)) { + paragraphs.push(current); + current = ''; + } + + current = appendSegment(current, text); + if (segment.end !== undefined) previousEnd = segment.end; + } + + if (current) paragraphs.push(current); + return paragraphs.join('\n\n'); +} + export async function transcribeAudio({ file, whisperUrl, @@ -102,10 +223,31 @@ export async function transcribeAudio({ // Step 2: transcribe in the detected language const transcribeForm = new FormData(); transcribeForm.append('file', file); + transcribeForm.append('language', detectedLanguage); transcribeForm.append('temperature', '0.0'); transcribeForm.append('temperature_inc', '0.2'); - transcribeForm.append('response_format', 'text'); - transcribeForm.append('language', detectedLanguage); + transcribeForm.append('beam_size', String(BEAM_SIZE)); + transcribeForm.append('best_of', String(BEST_OF)); + transcribeForm.append('suppress_nst', 'true'); + transcribeForm.append('response_format', 'verbose_json'); + // verbose_json re-runs a full language auto-detect unless this is set. Step 1 already answered + // that question, so it is pure duplicated work on every transcription. + transcribeForm.append('no_language_probabilities', 'true'); + // Segment timestamps drive the paragraphing below and are kept; per-token timestamps are extra + // work on every segment and nothing here reads them. + transcribeForm.append('token_timestamps', 'false'); + + const prompt = TRANSCRIPT_PROMPTS[detectedLanguage]; + if (prompt) { + transcribeForm.append('prompt', prompt); + transcribeForm.append('carry_initial_prompt', 'true'); + transcribeForm.append('max_context', String(PROMPT_CONTEXT_TOKENS)); + } else { + // Nothing worth protecting the budget for, so close the loop-feeding path outright. + transcribeForm.append('max_context', '0'); + } + + for (const [key, value] of Object.entries(VAD_TUNING)) transcribeForm.append(key, value); const res = await fetch(`${base}/inference`, { method: 'POST', @@ -113,7 +255,11 @@ export async function transcribeAudio({ timeout: false, } as BunRequestInit); if (!res.ok) throw new Error(`Transcription failed (${res.status})`); - const text = (await res.text()).trim(); + + const json = (await res.json()) as { text?: string; segments?: WhisperSegment[] }; + // Fall back to the flat text for any backend that does not return segments. + const segments = json.segments ?? []; + const text = segments.length > 0 ? toParagraphs(segments) : (json.text ?? '').trim(); return { text, detectedLanguage }; }