diff --git a/src/servers/api/stt/transcribe.ts b/src/servers/api/stt/transcribe.ts index 841c8684..a93e187e 100644 --- a/src/servers/api/stt/transcribe.ts +++ b/src/servers/api/stt/transcribe.ts @@ -4,6 +4,10 @@ // Only ask Whisper to translate (which is English-only) when the detected // language is NOT one the user speaks — i.e. they wouldn't understand the // raw transcript anyway. +// +// The detection must win when Whisper is confident. Biasing toward the user's spoken languages +// unconditionally means a user who speaks only English forces `language=en` onto every recording, +// and Whisper answers non-English audio with a rough, unmarked translation instead of a transcript. type TranscribeArgs = { file: File | Blob; @@ -17,6 +21,10 @@ export type TranscribeResult = { translated: boolean; }; +// Above this probability Whisper's own detection is taken as authoritative. Below it the clip is +// treated as ambiguous and the user's spoken languages break the tie. +const CONFIDENT_DETECTION = 0.5; + // whisper.cpp's "language" field uses full names ("english"); ISO codes live in // `language_probabilities`. Map covers what the user can pick in Profile settings. const LANG_NAME_TO_ISO: Record = { @@ -33,11 +41,14 @@ const LANG_NAME_TO_ISO: Record = { korean: 'ko', }; -export async function transcribeAudio({ file, whisperUrl, spokenLanguages }: TranscribeArgs): Promise { +export async function transcribeAudio({ + file, + whisperUrl, + spokenLanguages, +}: TranscribeArgs): Promise { const base = whisperUrl.replace(/\/+$/, ''); - // Step 1: detect language. Bias toward the user's spoken languages — Whisper's - // raw auto-detect is unreliable on short clips and often picks English by default. + // Step 1: detect language, and only fall back to the spoken-language bias when Whisper is unsure. const detectForm = new FormData(); detectForm.append('file', file); detectForm.append('temperature', '0.0'); @@ -48,24 +59,32 @@ export async function transcribeAudio({ file, whisperUrl, spokenLanguages }: Tra if (!detectRes.ok) throw new Error(`Language detection failed (${detectRes.status})`); const detectJson = (await detectRes.json()) as { language?: string; + detected_language?: string; + detected_language_probability?: number; language_probabilities?: Record; }; const probs = detectJson.language_probabilities; + const ranked = probs ? Object.entries(probs).sort((a, b) => b[1] - a[1]) : []; + const [topCode, topProb] = ranked[0] ?? []; + let detectedLanguage: string; - if (probs && spokenLanguages.length > 0) { - // Pick the spoken language with the highest probability - const ranked = spokenLanguages - .map((code) => [code, probs[code] ?? 0] as const) - .sort((a, b) => b[1] - a[1]); - detectedLanguage = ranked[0]?.[0] ?? spokenLanguages[0]!; - } else if (probs) { - // No spoken-language hint: take the highest-probability ISO code - const top = Object.entries(probs).sort((a, b) => b[1] - a[1])[0]; - detectedLanguage = top?.[0] ?? 'en'; + if (topCode && (topProb ?? 0) >= CONFIDENT_DETECTION) { + // Whisper is sure. Trust it even when it is not a language the user speaks — that is exactly the + // case translation exists for. + detectedLanguage = topCode; + } else if (ranked.length > 0 && spokenLanguages.length > 0) { + // Genuinely ambiguous (usually a short clip). Break the tie toward a language the user speaks, + // since Whisper's low-confidence guess defaults to English regardless of the audio. + const bestSpoken = spokenLanguages + .map((code) => [code, probs?.[code] ?? 0] as const) + .sort((a, b) => b[1] - a[1])[0]; + detectedLanguage = bestSpoken?.[0] ?? topCode ?? 'en'; + } else if (topCode) { + detectedLanguage = topCode; } else { - // No probabilities field (non-whisper.cpp backend) — try to map the name - const raw = (detectJson.language ?? 'en').toLowerCase(); + // No probabilities field (non-whisper.cpp backend) — try to map the reported name. + const raw = (detectJson.detected_language ?? detectJson.language ?? 'en').toLowerCase(); detectedLanguage = LANG_NAME_TO_ISO[raw] ?? raw; }