always transcribe in the source language, and let tasks call the API
Three changes that Transcribe Audio needs. Whisper's translate mode only ever outputs English, so it cannot honour "translate into <language>" for anything else — it answered Portuguese audio with a rough English rendering instead of a transcript. The translate decision is gone and transcription is always faithful to the detected language; spokenLanguages now only breaks ties on clips Whisper is unsure about. Script tasks get OFFICER_API_URL / OFFICER_API_HOST / OFFICER_AUTH_TOKEN so they can call Officer's own endpoints rather than reimplementing server-side work. Requests go to 127.0.0.1 so nothing depends on DNS or the proxy, but origin validation matches Host against PUBLIC_URL, hence the separate host variable. `inline` accepts "ask", which offers both affordances in the runner — Run here streams into the modal, Run as job queues it. Useful when the same task can take a second or an hour depending on whether it was pointed at a file or a library. Existing true/false values behave exactly as before. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
5f7d574dec
commit
06aac5478f
@@ -1,13 +1,11 @@
|
|||||||
// Whisper transcription with language-aware translation.
|
// Whisper transcription, always in the audio's own language.
|
||||||
//
|
//
|
||||||
// Strategy: detect the audio's language, then transcribe in that language.
|
// Detect the language first, then transcribe in it. Whisper's translate mode is English-only, so it
|
||||||
// Only ask Whisper to translate (which is English-only) when the detected
|
// cannot honour "translate into <language>" for anything else — rather than silently returning a
|
||||||
// language is NOT one the user speaks — i.e. they wouldn't understand the
|
// rough English rendering of Portuguese audio, transcription is always faithful to the source.
|
||||||
// raw transcript anyway.
|
|
||||||
//
|
//
|
||||||
// The detection must win when Whisper is confident. Biasing toward the user's spoken languages
|
// The detection must win when Whisper is confident. `spokenLanguages` only breaks ties on clips
|
||||||
// unconditionally means a user who speaks only English forces `language=en` onto every recording,
|
// Whisper is genuinely unsure about, where its low-confidence guess defaults to English.
|
||||||
// and Whisper answers non-English audio with a rough, unmarked translation instead of a transcript.
|
|
||||||
|
|
||||||
type TranscribeArgs = {
|
type TranscribeArgs = {
|
||||||
file: File | Blob;
|
file: File | Blob;
|
||||||
@@ -18,7 +16,6 @@ type TranscribeArgs = {
|
|||||||
export type TranscribeResult = {
|
export type TranscribeResult = {
|
||||||
text: string;
|
text: string;
|
||||||
detectedLanguage: string;
|
detectedLanguage: string;
|
||||||
translated: boolean;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// Above this probability Whisper's own detection is taken as authoritative. Below it the clip is
|
// Above this probability Whisper's own detection is taken as authoritative. Below it the clip is
|
||||||
@@ -88,21 +85,17 @@ export async function transcribeAudio({
|
|||||||
detectedLanguage = LANG_NAME_TO_ISO[raw] ?? raw;
|
detectedLanguage = LANG_NAME_TO_ISO[raw] ?? raw;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Step 2: translate only when the user doesn't speak the detected language
|
// Step 2: transcribe in the detected language
|
||||||
const translated = spokenLanguages.length > 0 && !spokenLanguages.includes(detectedLanguage);
|
|
||||||
|
|
||||||
// Step 3: full transcription
|
|
||||||
const transcribeForm = new FormData();
|
const transcribeForm = new FormData();
|
||||||
transcribeForm.append('file', file);
|
transcribeForm.append('file', file);
|
||||||
transcribeForm.append('temperature', '0.0');
|
transcribeForm.append('temperature', '0.0');
|
||||||
transcribeForm.append('temperature_inc', '0.2');
|
transcribeForm.append('temperature_inc', '0.2');
|
||||||
transcribeForm.append('response_format', 'text');
|
transcribeForm.append('response_format', 'text');
|
||||||
transcribeForm.append('language', detectedLanguage);
|
transcribeForm.append('language', detectedLanguage);
|
||||||
if (translated) transcribeForm.append('translate', 'true');
|
|
||||||
|
|
||||||
const res = await fetch(`${base}/inference`, { method: 'POST', body: transcribeForm });
|
const res = await fetch(`${base}/inference`, { method: 'POST', body: transcribeForm });
|
||||||
if (!res.ok) throw new Error(`Transcription failed (${res.status})`);
|
if (!res.ok) throw new Error(`Transcription failed (${res.status})`);
|
||||||
const text = (await res.text()).trim();
|
const text = (await res.text()).trim();
|
||||||
|
|
||||||
return { text, detectedLanguage, translated };
|
return { text, detectedLanguage };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import { tmpdir } from 'node:os';
|
|||||||
import { getTaskByDirName } from './task-files';
|
import { getTaskByDirName } from './task-files';
|
||||||
import { getOwnerHomeDir, DATA_PATH } from '../../data-path';
|
import { getOwnerHomeDir, DATA_PATH } from '../../data-path';
|
||||||
import { killTree } from './process-tree';
|
import { killTree } from './process-tree';
|
||||||
|
import { buildTaskApiEnv } from './task-api-env';
|
||||||
|
|
||||||
// Script-job event stream. `stdout`/`stderr` are high-frequency (live-only + persisted to the log
|
// Script-job event stream. `stdout`/`stderr` are high-frequency (live-only + persisted to the log
|
||||||
// file); `started`/`exit`/`error` are structural and get buffered for viewer replay.
|
// file); `started`/`exit`/`error` are structural and get buffered for viewer replay.
|
||||||
@@ -16,7 +17,9 @@ export type ScriptEvent =
|
|||||||
|
|
||||||
export type ExecuteScriptParams = {
|
export type ExecuteScriptParams = {
|
||||||
jobId: string;
|
jobId: string;
|
||||||
|
userId: number;
|
||||||
email: string;
|
email: string;
|
||||||
|
username?: string | null;
|
||||||
taskDirName: string;
|
taskDirName: string;
|
||||||
inputs: Record<string, string>;
|
inputs: Record<string, string>;
|
||||||
cwd?: string;
|
cwd?: string;
|
||||||
@@ -66,7 +69,7 @@ export const jobLogPath = (jobId: string) => join(DATA_PATH, 'jobs', `${jobId}.l
|
|||||||
// output to a durable log file. Resolves with the process exit code; throws only on spawn failure or
|
// output to a durable log file. Resolves with the process exit code; throws only on spawn failure or
|
||||||
// when aborted (the manager maps those to failed/stopped).
|
// when aborted (the manager maps those to failed/stopped).
|
||||||
export async function executeScript(params: ExecuteScriptParams): Promise<{ exitCode: number }> {
|
export async function executeScript(params: ExecuteScriptParams): Promise<{ exitCode: number }> {
|
||||||
const { jobId, email, inputs, abortSignal, emit } = params;
|
const { jobId, userId, email, username, inputs, abortSignal, emit } = params;
|
||||||
|
|
||||||
const task = await getTaskByDirName(params.taskDirName);
|
const task = await getTaskByDirName(params.taskDirName);
|
||||||
if (!task) throw new Error(`Task not found: ${params.taskDirName}`);
|
if (!task) throw new Error(`Task not found: ${params.taskDirName}`);
|
||||||
@@ -83,7 +86,8 @@ export async function executeScript(params: ExecuteScriptParams): Promise<{ exit
|
|||||||
const cwd = params.cwd ? (isAbsolute(params.cwd) ? params.cwd : join(homeDir, params.cwd)) : homeDir;
|
const cwd = params.cwd ? (isAbsolute(params.cwd) ? params.cwd : join(homeDir, params.cwd)) : homeDir;
|
||||||
|
|
||||||
const spawnCmd = cmd;
|
const spawnCmd = cmd;
|
||||||
const spawnEnv = { ...(process.env as Record<string, string>), ...inputEnv };
|
const apiEnv = await buildTaskApiEnv({ id: userId, email, username });
|
||||||
|
const spawnEnv = { ...(process.env as Record<string, string>), ...apiEnv, ...inputEnv };
|
||||||
const spawnCwd = cwd;
|
const spawnCwd = cwd;
|
||||||
|
|
||||||
mkdirSync(join(DATA_PATH, 'jobs'), { recursive: true });
|
mkdirSync(join(DATA_PATH, 'jobs'), { recursive: true });
|
||||||
|
|||||||
@@ -179,7 +179,9 @@ function launch(jobId: string, mode: JobMode, params: StartJobParams) {
|
|||||||
mode === 'script'
|
mode === 'script'
|
||||||
? executeScript({
|
? executeScript({
|
||||||
jobId,
|
jobId,
|
||||||
|
userId: params.userId,
|
||||||
email: params.email,
|
email: params.email,
|
||||||
|
username: params.username,
|
||||||
taskDirName: params.taskDirName,
|
taskDirName: params.taskDirName,
|
||||||
inputs: params.inputs,
|
inputs: params.inputs,
|
||||||
cwd: params.cwd,
|
cwd: params.cwd,
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
import { sign } from '@@/jwt';
|
||||||
|
|
||||||
|
const { PORT = '5000', PUBLIC_URL } = process.env;
|
||||||
|
|
||||||
|
const PUBLIC_HOST = (() => {
|
||||||
|
try {
|
||||||
|
return PUBLIC_URL ? new URL(PUBLIC_URL).host : undefined;
|
||||||
|
} catch {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
|
||||||
|
type TaskApiUser = { id: number; email: string; username?: string | null };
|
||||||
|
|
||||||
|
// Environment that lets a script task call Officer's own API instead of reimplementing server-side
|
||||||
|
// work (transcription, for one — the endpoint already handles language detection and caching).
|
||||||
|
//
|
||||||
|
// Requests go to 127.0.0.1 so a task never depends on DNS or the reverse proxy being up. Origin
|
||||||
|
// validation matches the Host header against PUBLIC_URL though, so a bare localhost request is
|
||||||
|
// rejected — hence OFFICER_API_HOST, which scripts must pass as an explicit Host header:
|
||||||
|
//
|
||||||
|
// curl -H "Authorization: Bearer $OFFICER_AUTH_TOKEN" -H "Host: $OFFICER_API_HOST" \
|
||||||
|
// "$OFFICER_API_URL/api/file-browser/transcribe"
|
||||||
|
export async function buildTaskApiEnv(user: TaskApiUser): Promise<Record<string, string>> {
|
||||||
|
const token = await sign({ id: user.id, email: user.email, username: user.username ?? '' }, '12h');
|
||||||
|
return {
|
||||||
|
OFFICER_API_URL: `http://127.0.0.1:${PORT}`,
|
||||||
|
OFFICER_API_HOST: PUBLIC_HOST ?? `127.0.0.1:${PORT}`,
|
||||||
|
OFFICER_AUTH_TOKEN: token,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -3,6 +3,7 @@ import { join, isAbsolute } from 'node:path';
|
|||||||
import { mkdirSync, writeFileSync, chmodSync, rmSync, readdirSync, readFileSync } from 'node:fs';
|
import { mkdirSync, writeFileSync, chmodSync, rmSync, readdirSync, readFileSync } from 'node:fs';
|
||||||
import { getTaskByDirName } from './task-files';
|
import { getTaskByDirName } from './task-files';
|
||||||
import { getOwnerHomeDir } from '../../data-path';
|
import { getOwnerHomeDir } from '../../data-path';
|
||||||
|
import { buildTaskApiEnv } from './task-api-env';
|
||||||
|
|
||||||
type WSData = {
|
type WSData = {
|
||||||
userId: number;
|
userId: number;
|
||||||
@@ -157,7 +158,7 @@ function buildArgs(inputs: Record<string, string>, argsOrder?: string[] | null):
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function handleRun(ws: ServerWebSocket<WSData>, msg: RunMessage) {
|
async function handleRun(ws: ServerWebSocket<WSData>, msg: RunMessage) {
|
||||||
const { email } = ws.data;
|
const { userId, email, username } = ws.data;
|
||||||
|
|
||||||
// Resolve task from the file-backed store
|
// Resolve task from the file-backed store
|
||||||
const task = await getTaskByDirName(msg.taskDirName);
|
const task = await getTaskByDirName(msg.taskDirName);
|
||||||
@@ -197,7 +198,8 @@ async function handleRun(ws: ServerWebSocket<WSData>, msg: RunMessage) {
|
|||||||
const cwd = msg.cwd ? (isAbsolute(msg.cwd) ? msg.cwd : join(homeDir, msg.cwd)) : homeDir;
|
const cwd = msg.cwd ? (isAbsolute(msg.cwd) ? msg.cwd : join(homeDir, msg.cwd)) : homeDir;
|
||||||
|
|
||||||
const spawnCmd = cmd;
|
const spawnCmd = cmd;
|
||||||
const spawnEnv = { ...(process.env as Record<string, string>), ...inputEnv };
|
const apiEnv = await buildTaskApiEnv({ id: userId, email, username });
|
||||||
|
const spawnEnv = { ...(process.env as Record<string, string>), ...apiEnv, ...inputEnv };
|
||||||
const spawnCwd = cwd;
|
const spawnCwd = cwd;
|
||||||
|
|
||||||
const cleanup = () => {
|
const cleanup = () => {
|
||||||
|
|||||||
@@ -14,7 +14,9 @@ export type TaskFrontmatter = {
|
|||||||
category: string | null; // groups the task into a context-menu submenu; null falls back to 'Other'
|
category: string | null; // groups the task into a context-menu submenu; null falls back to 'Other'
|
||||||
version: number;
|
version: number;
|
||||||
mode: string;
|
mode: string;
|
||||||
inline: boolean; // quick/interactive tasks run in the modal instead of as a background job
|
// How the task runs: false = background job, true = ephemerally in the modal, 'ask' = the modal
|
||||||
|
// offers the choice per run.
|
||||||
|
inline: boolean | 'ask';
|
||||||
language: string | null;
|
language: string | null;
|
||||||
args: string[] | null;
|
args: string[] | null;
|
||||||
tags: string[] | null;
|
tags: string[] | null;
|
||||||
@@ -40,7 +42,7 @@ export type TaskSummary = {
|
|||||||
description: string | null;
|
description: string | null;
|
||||||
category: string | null;
|
category: string | null;
|
||||||
mode: string;
|
mode: string;
|
||||||
inline: boolean;
|
inline: boolean | 'ask';
|
||||||
version: number;
|
version: number;
|
||||||
trigger: unknown;
|
trigger: unknown;
|
||||||
};
|
};
|
||||||
@@ -112,7 +114,7 @@ function parseTaskMd(raw: string): { fm: TaskFrontmatter; body: string } {
|
|||||||
category: parsed.category == null ? null : String(parsed.category),
|
category: parsed.category == null ? null : String(parsed.category),
|
||||||
version,
|
version,
|
||||||
mode: parsed.mode == null ? 'agentic' : String(parsed.mode),
|
mode: parsed.mode == null ? 'agentic' : String(parsed.mode),
|
||||||
inline: parsed.inline === true,
|
inline: parsed.inline === 'ask' ? 'ask' : parsed.inline === true,
|
||||||
language: parsed.language == null ? null : String(parsed.language),
|
language: parsed.language == null ? null : String(parsed.language),
|
||||||
args: asArray(parsed.args),
|
args: asArray(parsed.args),
|
||||||
tags: asArray(parsed.tags),
|
tags: asArray(parsed.tags),
|
||||||
|
|||||||
+22
-14
@@ -869,8 +869,11 @@ const ScriptRunner = ({
|
|||||||
const client = useClient();
|
const client = useClient();
|
||||||
const navigate = useNavigate();
|
const navigate = useNavigate();
|
||||||
const files = useFilesAPI('home');
|
const files = useFilesAPI('home');
|
||||||
// Inline tasks (quick/interactive) run ephemerally in this modal; everything else becomes a job.
|
// How this task runs: false = job, true = ephemerally in this modal, 'ask' = the task leaves the
|
||||||
const [inline, setInline] = useState(false);
|
// choice to the user and both affordances are offered.
|
||||||
|
const [inlineMode, setInlineMode] = useState<boolean | 'ask'>(false);
|
||||||
|
const alwaysInline = inlineMode === true;
|
||||||
|
const canChooseRunMode = inlineMode === 'ask';
|
||||||
// Is a job already running? (drives the Run vs Queue affordance).
|
// Is a job already running? (drives the Run vs Queue affordance).
|
||||||
const [jobRunning, setJobRunning] = useState(false);
|
const [jobRunning, setJobRunning] = useState(false);
|
||||||
// After a job is created, show a confirmation (View / Close) rather than navigating away.
|
// After a job is created, show a confirmation (View / Close) rather than navigating away.
|
||||||
@@ -901,13 +904,13 @@ const ScriptRunner = ({
|
|||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
client
|
client
|
||||||
.get<{
|
.get<{
|
||||||
inline?: boolean;
|
inline?: boolean | 'ask';
|
||||||
inputs?: Record<string, TaskInputDef>;
|
inputs?: Record<string, TaskInputDef>;
|
||||||
config?: { folderKeepAll?: boolean; perGroupTracks?: boolean; perGroupAllFiles?: boolean };
|
config?: { folderKeepAll?: boolean; perGroupTracks?: boolean; perGroupAllFiles?: boolean };
|
||||||
}>(`/tasks/${taskDirName}`)
|
}>(`/tasks/${taskDirName}`)
|
||||||
.then((task) => {
|
.then((task) => {
|
||||||
const defs: Record<string, TaskInputDef> = task.inputs ?? {};
|
const defs: Record<string, TaskInputDef> = task.inputs ?? {};
|
||||||
setInline(task.inline === true);
|
setInlineMode(task.inline === 'ask' ? 'ask' : task.inline === true);
|
||||||
setFolderKeepAll(task.config?.folderKeepAll === true);
|
setFolderKeepAll(task.config?.folderKeepAll === true);
|
||||||
setPerGroupTracks(task.config?.perGroupTracks === true);
|
setPerGroupTracks(task.config?.perGroupTracks === true);
|
||||||
setPerGroupAllFiles(task.config?.perGroupAllFiles === true);
|
setPerGroupAllFiles(task.config?.perGroupAllFiles === true);
|
||||||
@@ -1036,12 +1039,12 @@ const ScriptRunner = ({
|
|||||||
|
|
||||||
// Non-inline tasks become jobs — check if one is already running so we can offer Queue.
|
// Non-inline tasks become jobs — check if one is already running so we can offer Queue.
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (inline) return;
|
if (alwaysInline) return;
|
||||||
client
|
client
|
||||||
.get<Array<unknown>>('/jobs?live=1')
|
.get<Array<unknown>>('/jobs?live=1')
|
||||||
.then((live) => setJobRunning(live.length > 0))
|
.then((live) => setJobRunning(live.length > 0))
|
||||||
.catch(() => {});
|
.catch(() => {});
|
||||||
}, [inline]);
|
}, [alwaysInline]);
|
||||||
|
|
||||||
// Which per-group pickers this task declares, and whether the current selection is real work.
|
// Which per-group pickers this task declares, and whether the current selection is real work.
|
||||||
const has = pickerKinds(inputDefs);
|
const has = pickerKinds(inputDefs);
|
||||||
@@ -1166,20 +1169,24 @@ const ScriptRunner = ({
|
|||||||
const noWork = perGroupTracks && entryType === 'directory' && !perGroupHasWork;
|
const noWork = perGroupTracks && entryType === 'directory' && !perGroupHasWork;
|
||||||
const base =
|
const base =
|
||||||
'flex items-center gap-2 px-6 py-2.5 rounded-lg font-medium text-sm transition-colors disabled:opacity-50 disabled:cursor-not-allowed cursor-pointer';
|
'flex items-center gap-2 px-6 py-2.5 rounded-lg font-medium text-sm transition-colors disabled:opacity-50 disabled:cursor-not-allowed cursor-pointer';
|
||||||
if (inline) {
|
const jobDisabled = !inputDefs || probing || noWork;
|
||||||
return (
|
const runHere = (
|
||||||
<button
|
<button
|
||||||
onClick={runInline}
|
onClick={runInline}
|
||||||
disabled={!runner.isConnected || !inputDefs || probing || noWork}
|
disabled={!runner.isConnected || jobDisabled}
|
||||||
className={`${base} bg-duck-teal text-white hover:bg-duck-teal/90`}
|
className={`${base} ${
|
||||||
|
canChooseRunMode
|
||||||
|
? 'bg-duck-dark/10 dark:bg-foreground/10 text-duck-dark dark:text-foreground hover:bg-duck-dark/15 dark:hover:bg-foreground/15'
|
||||||
|
: 'bg-duck-teal text-white hover:bg-duck-teal/90'
|
||||||
|
}`}
|
||||||
>
|
>
|
||||||
<Play className="h-4 w-4" /> Run
|
<Play className="h-4 w-4" /> {canChooseRunMode ? 'Run here' : 'Run'}
|
||||||
</button>
|
</button>
|
||||||
);
|
);
|
||||||
}
|
if (alwaysInline) return runHere;
|
||||||
const jobDisabled = !inputDefs || probing || noWork;
|
|
||||||
return (
|
return (
|
||||||
<>
|
<>
|
||||||
|
{canChooseRunMode && runHere}
|
||||||
{jobRunning && (
|
{jobRunning && (
|
||||||
<button
|
<button
|
||||||
onClick={() => submitJob('queue')}
|
onClick={() => submitJob('queue')}
|
||||||
@@ -1194,7 +1201,8 @@ const ScriptRunner = ({
|
|||||||
disabled={jobDisabled}
|
disabled={jobDisabled}
|
||||||
className={`${base} text-white ${jobRunning ? 'bg-red-500 hover:bg-red-500/90' : 'bg-duck-teal hover:bg-duck-teal/90'}`}
|
className={`${base} text-white ${jobRunning ? 'bg-red-500 hover:bg-red-500/90' : 'bg-duck-teal hover:bg-duck-teal/90'}`}
|
||||||
>
|
>
|
||||||
<Play className="h-4 w-4" /> {jobRunning ? 'Run now' : 'Run'}
|
<Play className="h-4 w-4" />{' '}
|
||||||
|
{canChooseRunMode ? (jobRunning ? 'Run now as job' : 'Run as job') : jobRunning ? 'Run now' : 'Run'}
|
||||||
</button>
|
</button>
|
||||||
</>
|
</>
|
||||||
);
|
);
|
||||||
|
|||||||
Reference in New Issue
Block a user