the language-detection pass is optional now
every transcription on this machine failed with "Language detection failed (400)" before it started. the server was fine — it had answered /inference correctly minutes earlier — and the platform was fine. the plugin's FIRST call sends detect_language=true, which is a whisper.cpp parameter, and whisperx answers 400 to it. so a pass that exists to HELP became the only thing that could fail, on a backend that does its own detection anyway. whisperx logs "No language specified, language will be detected for each audio file" and gets it right per file. a backend refusing that request is telling us it detects for itself. skip the pass, send no `language`, and read back whatever the transcription reports. the two-pass path is untouched for whisper.cpp, which needs it, and the spoken-language tie-break still applies wherever detection runs. no prompt priming when the language is unknown either — TRANSCRIPT_PROMPTS is keyed by language, and priming with the wrong one is worse than not priming.
This commit is contained in:
+44
-16
@@ -330,7 +330,17 @@ export async function transcribeAudio({
|
|||||||
}: TranscribeArgs): Promise<TranscribeResult> {
|
}: TranscribeArgs): Promise<TranscribeResult> {
|
||||||
const base = whisperUrl.replace(/\/+$/, '');
|
const base = whisperUrl.replace(/\/+$/, '');
|
||||||
|
|
||||||
// Step 1: detect language, and only fall back to the spoken-language bias when Whisper is unsure.
|
/*
|
||||||
|
Step 1: detect the language, and only fall back to the spoken-language bias when Whisper is unsure.
|
||||||
|
|
||||||
|
OPTIONAL, because `detect_language` is a whisper.cpp parameter and not every backend has it. WhisperX
|
||||||
|
answers 400 to this request and detects per file on its own — so a pass that exists to help became the
|
||||||
|
thing that failed every transcription before it started, on a server that was working perfectly.
|
||||||
|
|
||||||
|
A backend that refuses it is telling us it does its own detection. Skip the pass, send no `language`,
|
||||||
|
and read back whatever the transcription reports. The two-pass path is unchanged for whisper.cpp,
|
||||||
|
which needs it.
|
||||||
|
*/
|
||||||
const detectForm = new FormData();
|
const detectForm = new FormData();
|
||||||
detectForm.append('file', file);
|
detectForm.append('file', file);
|
||||||
detectForm.append('temperature', '0.0');
|
detectForm.append('temperature', '0.0');
|
||||||
@@ -341,21 +351,33 @@ export async function transcribeAudio({
|
|||||||
method: 'POST',
|
method: 'POST',
|
||||||
body: detectForm,
|
body: detectForm,
|
||||||
timeout: false,
|
timeout: false,
|
||||||
} as BunRequestInit);
|
} as BunRequestInit).catch(() => null);
|
||||||
if (!detectRes.ok) throw new Error(`Language detection failed (${detectRes.status})`);
|
|
||||||
const detectJson = (await detectRes.json()) as {
|
|
||||||
language?: string;
|
|
||||||
detected_language?: string;
|
|
||||||
detected_language_probability?: number;
|
|
||||||
language_probabilities?: Record<string, number>;
|
|
||||||
};
|
|
||||||
|
|
||||||
const probs = detectJson.language_probabilities;
|
const detectJson =
|
||||||
|
detectRes && detectRes.ok
|
||||||
|
? ((await detectRes.json()) as {
|
||||||
|
language?: string;
|
||||||
|
detected_language?: string;
|
||||||
|
detected_language_probability?: number;
|
||||||
|
language_probabilities?: Record<string, number>;
|
||||||
|
})
|
||||||
|
: null;
|
||||||
|
|
||||||
|
if (!detectJson) {
|
||||||
|
console.log(
|
||||||
|
`[transcribe] no separate language detection (${detectRes ? `HTTP ${detectRes.status}` : 'unreachable'}) — letting the server detect`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const probs = detectJson?.language_probabilities;
|
||||||
const ranked = probs ? Object.entries(probs).sort((a, b) => b[1] - a[1]) : [];
|
const ranked = probs ? Object.entries(probs).sort((a, b) => b[1] - a[1]) : [];
|
||||||
const [topCode, topProb] = ranked[0] ?? [];
|
const [topCode, topProb] = ranked[0] ?? [];
|
||||||
|
|
||||||
let detectedLanguage: string;
|
let detectedLanguage: string | null;
|
||||||
if (topCode && (topProb ?? 0) >= CONFIDENT_DETECTION) {
|
if (!detectJson) {
|
||||||
|
// The server detects for itself. Nothing to bias and nothing to send.
|
||||||
|
detectedLanguage = null;
|
||||||
|
} else if (topCode && (topProb ?? 0) >= CONFIDENT_DETECTION) {
|
||||||
// Whisper is sure. Trust it even when it is not a language the user speaks — that is exactly the
|
// Whisper is sure. Trust it even when it is not a language the user speaks — that is exactly the
|
||||||
// case translation exists for.
|
// case translation exists for.
|
||||||
detectedLanguage = topCode;
|
detectedLanguage = topCode;
|
||||||
@@ -377,7 +399,9 @@ export async function transcribeAudio({
|
|||||||
// Step 2: transcribe in the detected language
|
// Step 2: transcribe in the detected language
|
||||||
const transcribeForm = new FormData();
|
const transcribeForm = new FormData();
|
||||||
transcribeForm.append('file', file);
|
transcribeForm.append('file', file);
|
||||||
transcribeForm.append('language', detectedLanguage);
|
// Absent when step 1 was skipped: the server detects, and telling it a language we did not detect
|
||||||
|
// would be a guess dressed as an instruction.
|
||||||
|
if (detectedLanguage) transcribeForm.append('language', detectedLanguage);
|
||||||
transcribeForm.append('temperature', '0.0');
|
transcribeForm.append('temperature', '0.0');
|
||||||
transcribeForm.append('temperature_inc', '0.2');
|
transcribeForm.append('temperature_inc', '0.2');
|
||||||
transcribeForm.append('beam_size', String(BEAM_SIZE));
|
transcribeForm.append('beam_size', String(BEAM_SIZE));
|
||||||
@@ -391,7 +415,7 @@ export async function transcribeAudio({
|
|||||||
// work on every segment and nothing here reads them.
|
// work on every segment and nothing here reads them.
|
||||||
transcribeForm.append('token_timestamps', 'false');
|
transcribeForm.append('token_timestamps', 'false');
|
||||||
|
|
||||||
const prompt = TRANSCRIPT_PROMPTS[detectedLanguage];
|
const prompt = detectedLanguage ? TRANSCRIPT_PROMPTS[detectedLanguage] : undefined;
|
||||||
if (prompt) {
|
if (prompt) {
|
||||||
transcribeForm.append('prompt', prompt);
|
transcribeForm.append('prompt', prompt);
|
||||||
transcribeForm.append('carry_initial_prompt', 'true');
|
transcribeForm.append('carry_initial_prompt', 'true');
|
||||||
@@ -410,10 +434,14 @@ export async function transcribeAudio({
|
|||||||
} as BunRequestInit);
|
} as BunRequestInit);
|
||||||
if (!res.ok) throw new Error(`Transcription failed (${res.status})`);
|
if (!res.ok) throw new Error(`Transcription failed (${res.status})`);
|
||||||
|
|
||||||
const json = (await res.json()) as { text?: string; segments?: WhisperSegment[] };
|
const json = (await res.json()) as { text?: string; segments?: WhisperSegment[]; language?: string };
|
||||||
// Fall back to the flat text for any backend that does not return segments.
|
// Fall back to the flat text for any backend that does not return segments.
|
||||||
const segments = json.segments ?? [];
|
const segments = json.segments ?? [];
|
||||||
const text = segments.length > 0 ? toParagraphs(segments) : (json.text ?? '').trim();
|
const text = segments.length > 0 ? toParagraphs(segments) : (json.text ?? '').trim();
|
||||||
|
|
||||||
return { text, detectedLanguage, segments };
|
// When step 1 was skipped, the transcription response is where the language comes from.
|
||||||
|
const reported = (json.language ?? '').toLowerCase();
|
||||||
|
const language = detectedLanguage ?? (LANG_NAME_TO_ISO[reported] ?? reported) ?? 'unknown';
|
||||||
|
|
||||||
|
return { text, detectedLanguage: language || 'unknown', segments };
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user