the language-detection pass is optional now
every transcription on this machine failed with "Language detection failed (400)" before it started. the server was fine — it had answered /inference correctly minutes earlier — and the platform was fine. the plugin's FIRST call sends detect_language=true, which is a whisper.cpp parameter, and whisperx answers 400 to it. so a pass that exists to HELP became the only thing that could fail, on a backend that does its own detection anyway. whisperx logs "No language specified, language will be detected for each audio file" and gets it right per file. a backend refusing that request is telling us it detects for itself. skip the pass, send no `language`, and read back whatever the transcription reports. the two-pass path is untouched for whisper.cpp, which needs it, and the spoken-language tie-break still applies wherever detection runs. no prompt priming when the language is unknown either — TRANSCRIPT_PROMPTS is keyed by language, and priming with the wrong one is worse than not priming.
This commit is contained in:
+44
-16
@@ -330,7 +330,17 @@ export async function transcribeAudio({
|
||||
}: TranscribeArgs): Promise<TranscribeResult> {
|
||||
const base = whisperUrl.replace(/\/+$/, '');
|
||||
|
||||
// Step 1: detect language, and only fall back to the spoken-language bias when Whisper is unsure.
|
||||
/*
|
||||
Step 1: detect the language, and only fall back to the spoken-language bias when Whisper is unsure.
|
||||
|
||||
OPTIONAL, because `detect_language` is a whisper.cpp parameter and not every backend has it. WhisperX
|
||||
answers 400 to this request and detects per file on its own — so a pass that exists to help became the
|
||||
thing that failed every transcription before it started, on a server that was working perfectly.
|
||||
|
||||
A backend that refuses it is telling us it does its own detection. Skip the pass, send no `language`,
|
||||
and read back whatever the transcription reports. The two-pass path is unchanged for whisper.cpp,
|
||||
which needs it.
|
||||
*/
|
||||
const detectForm = new FormData();
|
||||
detectForm.append('file', file);
|
||||
detectForm.append('temperature', '0.0');
|
||||
@@ -341,21 +351,33 @@ export async function transcribeAudio({
|
||||
method: 'POST',
|
||||
body: detectForm,
|
||||
timeout: false,
|
||||
} as BunRequestInit);
|
||||
if (!detectRes.ok) throw new Error(`Language detection failed (${detectRes.status})`);
|
||||
const detectJson = (await detectRes.json()) as {
|
||||
language?: string;
|
||||
detected_language?: string;
|
||||
detected_language_probability?: number;
|
||||
language_probabilities?: Record<string, number>;
|
||||
};
|
||||
} as BunRequestInit).catch(() => null);
|
||||
|
||||
const probs = detectJson.language_probabilities;
|
||||
const detectJson =
|
||||
detectRes && detectRes.ok
|
||||
? ((await detectRes.json()) as {
|
||||
language?: string;
|
||||
detected_language?: string;
|
||||
detected_language_probability?: number;
|
||||
language_probabilities?: Record<string, number>;
|
||||
})
|
||||
: null;
|
||||
|
||||
if (!detectJson) {
|
||||
console.log(
|
||||
`[transcribe] no separate language detection (${detectRes ? `HTTP ${detectRes.status}` : 'unreachable'}) — letting the server detect`,
|
||||
);
|
||||
}
|
||||
|
||||
const probs = detectJson?.language_probabilities;
|
||||
const ranked = probs ? Object.entries(probs).sort((a, b) => b[1] - a[1]) : [];
|
||||
const [topCode, topProb] = ranked[0] ?? [];
|
||||
|
||||
let detectedLanguage: string;
|
||||
if (topCode && (topProb ?? 0) >= CONFIDENT_DETECTION) {
|
||||
let detectedLanguage: string | null;
|
||||
if (!detectJson) {
|
||||
// The server detects for itself. Nothing to bias and nothing to send.
|
||||
detectedLanguage = null;
|
||||
} else if (topCode && (topProb ?? 0) >= CONFIDENT_DETECTION) {
|
||||
// Whisper is sure. Trust it even when it is not a language the user speaks — that is exactly the
|
||||
// case translation exists for.
|
||||
detectedLanguage = topCode;
|
||||
@@ -377,7 +399,9 @@ export async function transcribeAudio({
|
||||
// Step 2: transcribe in the detected language
|
||||
const transcribeForm = new FormData();
|
||||
transcribeForm.append('file', file);
|
||||
transcribeForm.append('language', detectedLanguage);
|
||||
// Absent when step 1 was skipped: the server detects, and telling it a language we did not detect
|
||||
// would be a guess dressed as an instruction.
|
||||
if (detectedLanguage) transcribeForm.append('language', detectedLanguage);
|
||||
transcribeForm.append('temperature', '0.0');
|
||||
transcribeForm.append('temperature_inc', '0.2');
|
||||
transcribeForm.append('beam_size', String(BEAM_SIZE));
|
||||
@@ -391,7 +415,7 @@ export async function transcribeAudio({
|
||||
// work on every segment and nothing here reads them.
|
||||
transcribeForm.append('token_timestamps', 'false');
|
||||
|
||||
const prompt = TRANSCRIPT_PROMPTS[detectedLanguage];
|
||||
const prompt = detectedLanguage ? TRANSCRIPT_PROMPTS[detectedLanguage] : undefined;
|
||||
if (prompt) {
|
||||
transcribeForm.append('prompt', prompt);
|
||||
transcribeForm.append('carry_initial_prompt', 'true');
|
||||
@@ -410,10 +434,14 @@ export async function transcribeAudio({
|
||||
} as BunRequestInit);
|
||||
if (!res.ok) throw new Error(`Transcription failed (${res.status})`);
|
||||
|
||||
const json = (await res.json()) as { text?: string; segments?: WhisperSegment[] };
|
||||
const json = (await res.json()) as { text?: string; segments?: WhisperSegment[]; language?: string };
|
||||
// Fall back to the flat text for any backend that does not return segments.
|
||||
const segments = json.segments ?? [];
|
||||
const text = segments.length > 0 ? toParagraphs(segments) : (json.text ?? '').trim();
|
||||
|
||||
return { text, detectedLanguage, segments };
|
||||
// When step 1 was skipped, the transcription response is where the language comes from.
|
||||
const reported = (json.language ?? '').toLowerCase();
|
||||
const language = detectedLanguage ?? (LANG_NAME_TO_ISO[reported] ?? reported) ?? 'unknown';
|
||||
|
||||
return { text, detectedLanguage: language || 'unknown', segments };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user