let Whisper's language detection win when it is confident

Language selection ranked only the user's spoken languages, so a user who speaks
one language always got that language. With spoken=["en"], Portuguese audio
detected at 0.968 pt was sent as language=en, and Whisper answered with a rough
English rendering rather than a transcript. `translated` stayed false, so nothing
downstream could tell a translation had happened, and detectedLanguage reported
"en" for audio that was not English.

Take Whisper's top language when it clears 0.5, and fall back to the spoken
languages only when the clip really is ambiguous — which is the case the bias was
written for.

Verified against the local whisper server with pt/en/es recordings:
  spoken=["en"]      pt -> translated English, es -> translated English, en -> as-is
  spoken=["en","pt"] pt -> Portuguese transcript, no translation
  spoken=[]          pt -> Portuguese transcript

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

src/servers/api/stt/transcribe.ts

@@ -4,6 +4,10 @@
 // Only ask Whisper to translate (which is English-only) when the detected
 // language is NOT one the user speaks — i.e. they wouldn't understand the
 // raw transcript anyway.
+//
+// The detection must win when Whisper is confident. Biasing toward the user's spoken languages
+// unconditionally means a user who speaks only English forces `language=en` onto every recording,
+// and Whisper answers non-English audio with a rough, unmarked translation instead of a transcript.
 
 type TranscribeArgs = {
   file: File | Blob;
@@ -17,6 +21,10 @@ export type TranscribeResult = {
   translated: boolean;
 };
 
+// Above this probability Whisper's own detection is taken as authoritative. Below it the clip is
+// treated as ambiguous and the user's spoken languages break the tie.
+const CONFIDENT_DETECTION = 0.5;
+
 // whisper.cpp's "language" field uses full names ("english"); ISO codes live in
 // `language_probabilities`. Map covers what the user can pick in Profile settings.
 const LANG_NAME_TO_ISO: Record<string, string> = {
@@ -33,11 +41,14 @@ const LANG_NAME_TO_ISO: Record<string, string> = {
   korean: 'ko',
 };
 
-export async function transcribeAudio({ file, whisperUrl, spokenLanguages }: TranscribeArgs): Promise<TranscribeResult> {
+export async function transcribeAudio({
+  file,
+  whisperUrl,
+  spokenLanguages,
+}: TranscribeArgs): Promise<TranscribeResult> {
   const base = whisperUrl.replace(/\/+$/, '');
 
-  // Step 1: detect language. Bias toward the user's spoken languages — Whisper's
+  // Step 1: detect language, and only fall back to the spoken-language bias when Whisper is unsure.
-  // raw auto-detect is unreliable on short clips and often picks English by default.
   const detectForm = new FormData();
   detectForm.append('file', file);
   detectForm.append('temperature', '0.0');
@@ -48,24 +59,32 @@ export async function transcribeAudio({ file, whisperUrl, spokenLanguages }: Tra
   if (!detectRes.ok) throw new Error(`Language detection failed (${detectRes.status})`);
   const detectJson = (await detectRes.json()) as {
     language?: string;
+    detected_language?: string;
+    detected_language_probability?: number;
     language_probabilities?: Record<string, number>;
   };
 
   const probs = detectJson.language_probabilities;
+  const ranked = probs ? Object.entries(probs).sort((a, b) => b[1] - a[1]) : [];
+  const [topCode, topProb] = ranked[0] ?? [];
+
   let detectedLanguage: string;
-  if (probs && spokenLanguages.length > 0) {
+  if (topCode && (topProb ?? 0) >= CONFIDENT_DETECTION) {
-    // Pick the spoken language with the highest probability
+    // Whisper is sure. Trust it even when it is not a language the user speaks — that is exactly the
-    const ranked = spokenLanguages
+    // case translation exists for.
-      .map((code) => [code, probs[code] ?? 0] as const)
+    detectedLanguage = topCode;
-      .sort((a, b) => b[1] - a[1]);
+  } else if (ranked.length > 0 && spokenLanguages.length > 0) {
-    detectedLanguage = ranked[0]?.[0] ?? spokenLanguages[0]!;
+    // Genuinely ambiguous (usually a short clip). Break the tie toward a language the user speaks,
-  } else if (probs) {
+    // since Whisper's low-confidence guess defaults to English regardless of the audio.
-    // No spoken-language hint: take the highest-probability ISO code
+    const bestSpoken = spokenLanguages
-    const top = Object.entries(probs).sort((a, b) => b[1] - a[1])[0];
+      .map((code) => [code, probs?.[code] ?? 0] as const)
-    detectedLanguage = top?.[0] ?? 'en';
+      .sort((a, b) => b[1] - a[1])[0];
+    detectedLanguage = bestSpoken?.[0] ?? topCode ?? 'en';
+  } else if (topCode) {
+    detectedLanguage = topCode;
   } else {
-    // No probabilities field (non-whisper.cpp backend) — try to map the name
+    // No probabilities field (non-whisper.cpp backend) — try to map the reported name.
-    const raw = (detectJson.language ?? 'en').toLowerCase();
+    const raw = (detectJson.detected_language ?? detectJson.language ?? 'en').toLowerCase();
     detectedLanguage = LANG_NAME_TO_ISO[raw] ?? raw;
   }