From 3da6d11c00f9eda110fdbcb46a9ae1feb0cecd74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Padez?= Date: Mon, 24 Aug 2026 03:28:04 +0000 Subject: [PATCH] extract speech-to-text into its own plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moves platform STT out into a plugin that owns the whole whisper.cpp contract: sidecar, config (a JSON file under DATA_PATH/transcribe/, validated against the server before storing), a thin proxy router, a skill so an agent knows the contract without being told, and a script-mode file-browser task (not agentic — no model turn per audio file). sidecar/transcribe.ts and its test moved unchanged from src/servers/api/stt/ — every constant in it has a reason attached and none of it was touched. The platform-side removal (routes, dead useAudioRecording path, Settings screen, locale keys) lands in a separate, uncommitted change in the platform repo for review. Not yet reachable from the file browser: the platform does not load tasks from a plugin's directory yet (skills/affordances/notifications do). Ships anyway per instructions. Officer-Session: 656f5c85-e49a-4f1f-b9f3-042a04b4d043 --- PLUGIN.md | 77 +++++ README.md | 46 +++ api/router.ts | 22 ++ manifest.ts | 51 ++++ sidecar/config.ts | 87 ++++++ sidecar/index.ts | 170 +++++++++++ sidecar/transcribe.test.ts | 78 +++++ sidecar/transcribe.ts | 416 +++++++++++++++++++++++++++ skills/transcribe/SKILL.md | 69 +++++ tasks/transcribe-audio-file/TASK.md | 58 ++++ tasks/transcribe-audio-file/index.ts | 118 ++++++++ 11 files changed, 1192 insertions(+) create mode 100644 PLUGIN.md create mode 100644 README.md create mode 100644 api/router.ts create mode 100644 manifest.ts create mode 100644 sidecar/config.ts create mode 100644 sidecar/index.ts create mode 100644 sidecar/transcribe.test.ts create mode 100644 sidecar/transcribe.ts create mode 100644 skills/transcribe/SKILL.md create mode 100644 tasks/transcribe-audio-file/TASK.md create mode 100644 tasks/transcribe-audio-file/index.ts diff --git a/PLUGIN.md b/PLUGIN.md new file mode 100644 index 0000000..7a077e7 --- /dev/null +++ b/PLUGIN.md @@ -0,0 +1,77 @@ +# Transcribe — the extraction record + +Extracted from the platform on **2026-08-24**. `plugins/EXTRACTING-A-PLUGIN.md` is the runbook; this +file is what it cost. + +## What moved + +| Was | Is now | +| --------------------------------------------- | ---------------------------------------- | +| `src/servers/api/stt/transcribe.ts` | `sidecar/transcribe.ts` — unchanged | +| `src/servers/api/stt/transcribe.test.ts` | `sidecar/transcribe.test.ts` — unchanged | +| `src/servers/api/server-settings/stt.ts` | `sidecar/config.ts` — rewritten | + +`sidecar/transcribe.ts` is the reason this extraction exists at all: 416 lines of tuning — the two-pass +detect-then-transcribe, 25 language priming prompts, the VAD widening, the paragraphing — each with a +reason in its own comment, and none of it touched. Moving it, not rewriting it, was the instruction and +the only defensible one: every constant in that file was chosen against a real recording at some point, +and there is no way to re-derive that from reading the code cold. + +## What did not move unchanged: config + +The platform kept a single admin-only setting, `server-settings/stt` — one row in a JSON blob +(`readServerSettings`/`writeServerSettings`), gated by the `server-admin` permission and reachable only +by the owner. That shape does not exist here to move into: a plugin's config is its own file, not a slot +in the platform's settings blob, and this plugin's own permission (`transcribe`) is grantable to anyone, +not owner-only. + +So `sidecar/config.ts` is new, not moved: a JSON file under `DATA_PATH/transcribe/`, following the +`DATA_PATH//` convention `plugins/music/sidecar/indexer.ts` established, with the same +validate-before-store behaviour the old `/server-settings/stt/test` route had (an empty multipart POST to +`/inference` — Whisper always rejects it, but any response at all means the server exists). + +**Deliberately not owner-only.** Photos' `/_config` holds a real credential (an Immich API key) and is +still reachable by any account holding the permission at write; jellyfin's manifest, despite the +`ownerOnly` field existing for exactly this shape, does not set it either. A Whisper URL is not a +credential — there is nothing to steal by reading it, and nothing more dangerous than "the transcriptions +go to a different server" by changing it — so this follows the same precedent rather than inventing a +narrower one. + +## What the platform lost + +Deleted, not commented — see the platform-side diff for the full list: `src/servers/api/stt/`, the +`server-settings/stt` router and its mount, `POST /chat/stt`, the dead `useAudioRecording` STT path +(`blobToWav` stays; `DictateDialog.tsx` still needs it for something that was never STT — see below), +`POST /file-browser/transcribe` and the FileViewer handler behind the button that was already hidden, the +Settings → Speech to Text screen, and the orphaned locale keys across all six `officer-web` locales. + +**One thing found and left alone.** `DictateDialog.tsx` in the file browser imports `blobToWav` from the +same file `useAudioRecording`'s dead STT path lived in. It records audio and uploads the raw `.wav` — it +never called Whisper, was never part of this feature, and stays in the platform exactly as it is. Deleting +the whole file would have taken it down as collateral. + +## The task: script, not agentic + +The worked reference this shipped against (`~/dockers/officer.dev/seed-archive/tasks/transcribe-audio-file/` +on alpha) is agentic: it runs its own detect-language pass before transcribing, and generates an +archive-folder title from the transcription's contents. Both are wrong for a script task and were dropped +on purpose rather than carried over — the detect step duplicates what the plugin's route already does +server-side, and a generated title needs a model turn, which is the entire thing "script, not agentic" was +asked for to avoid spending per file. The filing behaviour (a dated, slugged sibling directory, original +moved in, transcript saved beside it) is the part worth keeping, so it survived; the slug now comes from +the original filename instead of a generated title. + +## What is NOT wired up yet + +**The platform does not load tasks from a plugin's directory.** Skills, affordances and notifications are +read from `plugins//` at runtime (`docs/plugin-capabilities.md`); tasks are not — `task-files.ts` +only reads `$OFFICER_ITEMS_DIR/tasks/`. So `tasks/transcribe-audio-file/` ships here, is correct, and will +not appear in the file browser's right-click menu until that loader lands. Not a defect in this plugin; +recorded so nobody spends an evening looking for why it is missing. + +## Still open + +- **No `assets/icon.png`.** Falls back to the `AudioLines` lucide glyph. +- **Single Whisper server, not a registry.** Unlike `photos_config` or `jellyfin_servers`, there is + exactly one URL and no way to keep a second on hand. Revisit if that ever turns out to be wanted — + nothing here forecloses it, but nothing asked for it either. diff --git a/README.md b/README.md new file mode 100644 index 0000000..b28c468 --- /dev/null +++ b/README.md @@ -0,0 +1,46 @@ +# Transcribe + +Officer's speech-to-text, talking to a self-hosted whisper.cpp server. + +Transcribe does not run Whisper and ships no model. You point it at a whisper.cpp `/inference` server +you already run and it forwards to it, tuned for one person talking into a phone: detect the spoken +language, then transcribe faithfully in it — never translated, never guessed at in English when the +audio is not. + +## What you get + +- **`POST /api/transcribe/transcribe`** — send an audio file, get back its text and the language Whisper + detected. Two-pass under the hood (detect, then transcribe in that language) so a low-confidence guess + never quietly defaults to English. +- **A file-browser task.** Right-click an `.mp3`, `.wav` or `.m4a` and transcribe it without a model + turn — the task is a plain script that calls this plugin's own route and files the result next to the + original. +- **A skill**, so an agent asked to transcribe something knows the route, the shape, and what to say when + nothing is configured, without being told. + +## What you need + +**A whisper.cpp server you already run**, reachable from this machine, answering `POST /inference` — +the same container the platform used to talk to directly. Nothing else. No system packages, no host +binaries; the sidecar reaches it with `fetch`. + +## Setup + +``` +PUT /api/transcribe/config +{ "url": "http://192.168.1.50:64201" } +``` + +Validated against the server before it is stored — a wrong URL is a 400 with the reason, not a saved +value that fails on the first real transcription. + +## What this plugin deliberately does not have + +- **No dashboard route, no dock tile.** There is nothing to look at — this is reached through the file + browser's task menu and by an agent that has read the skill, not through a screen. Shipping no `web/` + is a supported shape, not an oversight. +- **No streaming.** One file in, one answer back. If you want a live microphone, that is a different + feature built on top of this one, not something this plugin does today. +- **No translation.** It transcribes in whatever language was spoken. Whisper's own translate mode is + English-only, so honouring "translate into <language>" for anything else would mean silently + handing back a rough English rendering of, say, Portuguese audio — worse than refusing. diff --git a/api/router.ts b/api/router.ts new file mode 100644 index 0000000..a8d4f77 --- /dev/null +++ b/api/router.ts @@ -0,0 +1,22 @@ +import { createSidecarProxy } from '@@/sidecar/create-proxy'; +import { mountPrefix } from '@@/plugins/manifest'; +import { manifest } from '../manifest'; + +// /api/transcribe/* — auth, then forward to officer-transcribe. No routes of its own and no Whisper +// knowledge: the whole contract lives in ../sidecar/index.ts. +// +// ── The prefix is DERIVED, not written ── +// +// `mountPrefix` returns `/transcribe` for this first-party publisher; a literal '/api/transcribe' would +// be right only by coincidence and wrong the moment anyone else publishes this plugin. Same reasoning as +// `plugins/photos/api/router.ts` and `plugins/music/api/router.ts`. +const proxy = createSidecarProxy({ + name: 'transcribe', + prefix: `/api${mountPrefix({ appName: 'transcribe', manifest })}`, + // whisper.cpp runs at roughly 8x realtime, so a long voice memo (the moved `transcribe.ts` was tuned + // for recordings past 40 minutes, and disables Bun's own 300s fetch ceiling for exactly that reason) + // can hold this hop open for minutes. 1800s gives it the same headroom on the platform->sidecar leg. + timeoutSeconds: 1800, +}); + +export const router = proxy.router; diff --git a/manifest.ts b/manifest.ts new file mode 100644 index 0000000..3829b80 --- /dev/null +++ b/manifest.ts @@ -0,0 +1,51 @@ +import type { PluginManifest } from '@@/plugins/manifest'; + +/** + * Transcribe — Officer's speech-to-text, talking to a self-hosted whisper.cpp server. + * + * ── What this plugin is, and what it is not ── + * + * It does NOT run Whisper, and it does not ship a model. You point it at a whisper.cpp `/inference` + * server you already run (the same container the platform used to talk to directly) and it forwards to + * it. Nothing is transcribed here — the sidecar's whole job is the request shape: detect the language, + * then transcribe faithfully in it, tuned for one person talking into a phone. + * + * Named for the feature, not the implementation — the same reasoning as `photos` not `immich`. Swapping + * the backend for a different `/inference`-compatible server would not move the mount point or the route + * shape. + * + * ── Where the connection lives ── + * + * A single URL, in a JSON file under `DATA_PATH/transcribe/`, not in Postgres and not in this + * repository. There is only ever one Whisper server for the whole install — unlike `photos_config` or + * `jellyfin_servers`, nobody has a reason to keep more than one — so a file the owner can also edit by + * hand is simpler than a table with one row. See `sidecar/config.ts`. + * + * ── No `osDependencies` ── + * + * Nothing shells out. The sidecar reaches whisper.cpp with `fetch`, the same as the code it replaces did. + * + * ── No `web/` ── + * + * This plugin ships no dashboard route and no dock tile — `plugin.web` gates the tile, and a + * backend-only plugin has nothing to pin one to. It is reached from a task's right-click menu and by an + * agent that has read its skill, not by a screen. + */ +export const manifest: PluginManifest = { + publisher: 'officerdev', + version: '1.0.0', + platform: '>=1.0.0', + + label: 'Transcribe', + summary: 'Speech-to-text over your own whisper.cpp server', + icon: 'AudioLines', + color: '#f97316', + + permissions: [ + { + key: 'transcribe', + label: 'Transcribe', + description: 'Turn audio into text over your own whisper.cpp server', + }, + ], +}; diff --git a/sidecar/config.ts b/sidecar/config.ts new file mode 100644 index 0000000..6dc2bdd --- /dev/null +++ b/sidecar/config.ts @@ -0,0 +1,87 @@ +import { join } from 'node:path'; +import { mkdir } from 'node:fs/promises'; +import { DATA_PATH } from '@@/data-path'; + +// The Whisper connection — a single URL, kept in a JSON file rather than a database row. +// +// There is only ever one whisper.cpp server for the whole install: unlike `photos_config` or +// `jellyfin_servers`, nobody has a reason to keep more than one on hand, so a file the owner can also +// edit by hand (or drop in with a provisioning script) is simpler than a table that will only ever hold +// one row. See `plugins/music/sidecar/indexer.ts` for the same `DATA_PATH//` convention. +// +// Not committed anywhere: this file lives under DATA_PATH, outside both the platform's and this plugin's +// git repositories. Config is data, not definition. + +const CONFIG_DIR = join(DATA_PATH, 'transcribe'); +const CONFIG_PATH = join(CONFIG_DIR, 'config.json'); + +export type TranscribeConfig = { url: string }; + +// Cached across requests; invalidated only by `setConfig`. A single small file re-read on every +// transcription would be a needless stat + parse on the hot path. +let cache: TranscribeConfig | null | undefined; + +async function loadConfig(): Promise { + const file = Bun.file(CONFIG_PATH); + if (!(await file.exists())) return null; + try { + const parsed = (await file.json()) as Partial; + return typeof parsed.url === 'string' && parsed.url ? { url: parsed.url } : null; + } catch { + return null; + } +} + +export async function getConfig(): Promise { + if (cache === undefined) cache = await loadConfig(); + return cache; +} + +export async function setConfig(config: TranscribeConfig): Promise { + await mkdir(CONFIG_DIR, { recursive: true }); + await Bun.write(CONFIG_PATH, JSON.stringify(config, null, 2)); + cache = config; +} + +/** + * The URL as the browser is allowed to see it. + * + * A whisper.cpp URL is never expected to carry credentials — the server has no auth of its own — but if + * an owner points this at something behind a reverse proxy with basic auth baked into the URL, userinfo + * (`user:pass@`) must not round-trip back out to `GET /config`. Stripped rather than trusted to stay + * absent. + */ +export function redactUrl(url: string): string { + try { + const parsed = new URL(url); + parsed.username = ''; + parsed.password = ''; + return parsed.toString(); + } catch { + return url; + } +} + +export type ProbeResult = { ok: true } | { ok: false; error: string }; + +/** + * Ask a whisper.cpp server whether it is really there. + * + * Mirrors the platform's old `POST /server-settings/stt/test`: an empty multipart body to `/inference` + * always fails on the Whisper side, but ANY response — including its error — means the server exists and + * speaks the expected protocol. A network-level failure (wrong host, nothing listening, timeout) is the + * only thing that means "not reachable". + */ +export async function probeWhisper(url: string): Promise { + const base = url.replace(/\/+$/, ''); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 5000); + try { + await fetch(`${base}/inference`, { method: 'POST', body: new FormData(), signal: controller.signal }); + return { ok: true }; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : 'could not reach the server' }; + } finally { + clearTimeout(timeout); + } +} diff --git a/sidecar/index.ts b/sidecar/index.ts new file mode 100644 index 0000000..159e8a4 --- /dev/null +++ b/sidecar/index.ts @@ -0,0 +1,170 @@ +import type { SidecarCommand, SidecarEvent } from '@@/sidecar/protocol'; +import { createSidecarConnector } from '@@/sidecar/connect'; +import { API_URL } from '@@/officer-url.mjs'; +import { getConfig, setConfig, redactUrl, probeWhisper } from './config'; +import { transcribeAudio } from './transcribe'; + +// The officer-transcribe sidecar. Owns the whole whisper.cpp contract for Officer: the server URL and +// the request tuning that makes it usable for a person talking into a phone (see transcribe.ts — moved +// here unchanged from the platform, and every constant in it has a reason attached in its own comments). +// +// The platform API is a thin auth-gated forwarder (../api/router.ts) holding no Whisper knowledge at all. +// +// ───────────────────────────────────────────────────────────────────────────────────────────────── +// HTTP CONTRACT — the platform strips its /api/transcribe mount prefix before forwarding. +// +// GET /_health { ok, configured, url? } +// GET /config { configured, url } — url has its userinfo stripped, never a secret +// PUT /config { url } — validated against the server before storing +// POST /transcribe multipart: file, spokenLanguages[] → { text, detectedLanguage } +// +// anything else 404 +// +// Every route needs `X-Officer-User`, which the platform proxy sets after authenticating the caller. We +// bind loopback only, so its presence is the trust signal — a request without it did not come through +// the platform. +// ───────────────────────────────────────────────────────────────────────────────────────────────── + +/** Grab an ephemeral free port by briefly binding one and releasing it. */ +function getFreePort(): number { + const probeServer = Bun.serve({ port: 0, hostname: '127.0.0.1', fetch: () => new Response('') }); + const p = probeServer.port; + probeServer.stop(true); + if (p == null) throw new Error('failed to acquire a free port'); + return p; +} + +const port = getFreePort(); + +async function handleConfigGet(): Promise { + const cfg = await getConfig(); + return Response.json({ configured: !!cfg, url: cfg ? redactUrl(cfg.url) : null }); +} + +function isHttpUrl(url: string): boolean { + return /^https?:\/\//i.test(url); +} + +async function handleConfigPut(req: Request): Promise { + const body = ((await req.json().catch(() => null)) as { url?: unknown } | null) ?? {}; + const url = typeof body.url === 'string' ? body.url.trim() : ''; + if (!url) return Response.json({ error: 'url is required' }, { status: 400 }); + if (!isHttpUrl(url)) return Response.json({ error: 'url must start with http:// or https://' }, { status: 400 }); + + const probe = await probeWhisper(url); + if (!probe.ok) return Response.json({ error: probe.error }, { status: 400 }); + + await setConfig({ url }); + return Response.json({ configured: true, url: redactUrl(url) }); +} + +async function handleTranscribe(req: Request): Promise { + const cfg = await getConfig(); + if (!cfg) { + return Response.json({ error: 'Whisper not configured — PUT /api/transcribe/config first' }, { status: 400 }); + } + + const body = await req.formData().catch(() => null); + const file = body?.get('file'); + if (!file || !(file instanceof File)) return Response.json({ error: 'file is required' }, { status: 400 }); + + const spokenLanguages = (body?.getAll('spokenLanguages') ?? []).map(String); + + try { + const result = await transcribeAudio({ file, whisperUrl: cfg.url, spokenLanguages }); + return Response.json(result); + } catch (err) { + console.error('[transcribe] transcription failed', err); + return Response.json( + { error: err instanceof Error ? err.message : 'Transcription failed' }, + { status: 502 }, + ); + } +} + +const server = Bun.serve({ + port, + hostname: '127.0.0.1', + // A phone's voice memo is small; this just needs to be well above any single recording. + maxRequestBodySize: 1024 * 1024 * 1024, + async fetch(req) { + const url = new URL(req.url); + + const officerUser = req.headers.get('X-Officer-User'); + const userId = Number(officerUser); + if (!officerUser || !Number.isInteger(userId) || userId <= 0) { + return Response.json({ error: 'missing or invalid X-Officer-User' }, { status: 401 }); + } + + if (url.pathname === '/_health') { + const cfg = await getConfig(); + if (!cfg) return Response.json({ ok: false, configured: false }, { status: 503 }); + const probe = await probeWhisper(cfg.url); + if (!probe.ok) return Response.json({ ok: false, configured: true, error: probe.error }, { status: 502 }); + return Response.json({ ok: true, configured: true, url: redactUrl(cfg.url) }); + } + + if (url.pathname === '/config') { + try { + if (req.method === 'GET') return await handleConfigGet(); + if (req.method === 'PUT' || req.method === 'POST') return await handleConfigPut(req); + return Response.json({ error: 'method not allowed' }, { status: 405 }); + } catch (err) { + console.error(`[transcribe] ${req.method} /config failed`, err); + return Response.json({ error: 'internal error' }, { status: 500 }); + } + } + + if (url.pathname === '/transcribe') { + if (req.method !== 'POST') return Response.json({ error: 'method not allowed' }, { status: 405 }); + return handleTranscribe(req); + } + + return Response.json({ error: 'not found' }, { status: 404 }); + }, +}); + +console.log(`[transcribe] listening on 127.0.0.1:${port} (whisper.cpp URL configured from PUT /config)`); + +type ReplyFn = (msg: SidecarEvent) => void; + +function handleCommand(cmd: SidecarCommand, reply: ReplyFn) { + switch (cmd.type) { + case 'ping': + reply({ type: 'pong', id: cmd.id }); + break; + default: + reply({ + type: 'error', + id: (cmd as SidecarCommand).id, + error: `Unknown command type: ${(cmd as Record).type}`, + }); + } +} + +const connection = createSidecarConnector({ + apiUrl: `${API_URL}/api/sidecar/register`, + name: 'transcribe', + handles: ['transcribe'], + onCommand(cmd, reply) { + handleCommand(cmd as SidecarCommand, reply as ReplyFn); + }, + onConnected() { + connection.send({ type: 'transcribe:server', port }); + console.log(`[transcribe] reported server port ${port} to API`); + }, +}); + +function shutdown(signal: string) { + console.log(`[transcribe] ${signal} received, shutting down...`); + try { + server.stop(true); + } catch { + /* already stopped */ + } + connection.destroy(); + process.exit(0); +} + +process.on('SIGTERM', () => shutdown('SIGTERM')); +process.on('SIGINT', () => shutdown('SIGINT')); diff --git a/sidecar/transcribe.test.ts b/sidecar/transcribe.test.ts new file mode 100644 index 0000000..6bd6d00 --- /dev/null +++ b/sidecar/transcribe.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from 'bun:test'; +import { toParagraphs } from './transcribe'; +import type { WhisperSegment } from './transcribe'; + +// Whisper's own shape: leading space on every segment, times in seconds. +const seg = (text: string, start: number, end: number, extra: Partial = {}): WhisperSegment => ({ + text, + start, + end, + ...extra, +}); + +describe('toParagraphs', () => { + it('joins segments spoken without a real pause into one paragraph', () => { + const out = toParagraphs([seg(' I went to the shop.', 0, 2), seg(' It was closed.', 2.2, 4)]); + expect(out).toBe('I went to the shop. It was closed.'); + }); + + it('starts a new paragraph after a long silence', () => { + const out = toParagraphs([seg(' First thought.', 0, 2), seg(' Second thought.', 5, 7)]); + expect(out).toBe('First thought.\n\nSecond thought.'); + }); + + it('does not break on a breath-length pause', () => { + const out = toParagraphs([seg(' Hang on,', 0, 1), seg(' let me think.', 1.5, 3)]); + expect(out).toBe('Hang on, let me think.'); + }); + + // The lone "." lines in the old response_format=text output came from segments like this. + it('attaches stranded punctuation without a space in front of it', () => { + const out = toParagraphs([seg(' See you tomorrow morning', 0, 2), seg(' .', 2.1, 2.2)]); + expect(out).toBe('See you tomorrow morning.'); + }); + + it('drops segments that are probably silence decoded with low confidence', () => { + const out = toParagraphs([ + seg(' Real speech here.', 0, 2), + seg(' Thanks for watching!', 2.1, 3, { no_speech_prob: 0.95, avg_logprob: -1.8 }), + seg(' More real speech.', 3.1, 5), + ]); + expect(out).toBe('Real speech here. More real speech.'); + }); + + it('keeps a confident segment even when it looks like silence', () => { + const out = toParagraphs([seg(' Definitely said.', 0, 2, { no_speech_prob: 0.95, avg_logprob: -0.2 })]); + expect(out).toBe('Definitely said.'); + }); + + it('keeps segments that report neither probability', () => { + const out = toParagraphs([seg(' No metadata at all.', 0, 2)]); + expect(out).toBe('No metadata at all.'); + }); + + it('measures the pause from the last kept segment, not the dropped one', () => { + const out = toParagraphs([ + seg(' Before.', 0, 2), + seg(' Hallucinated.', 2.1, 9, { no_speech_prob: 0.9, avg_logprob: -2 }), + seg(' After.', 9.1, 11), + ]); + // The gap that matters is 2 -> 9.1, so these are separate thoughts. + expect(out).toBe('Before.\n\nAfter.'); + }); + + it('collapses internal whitespace and ignores empty segments', () => { + const out = toParagraphs([seg(' Too many\n spaces. ', 0, 2), seg(' ', 2.1, 2.2)]); + expect(out).toBe('Too many spaces.'); + }); + + it('returns an empty string when every segment is filtered out', () => { + expect(toParagraphs([])).toBe(''); + expect(toParagraphs([seg(' ', 0, 1)])).toBe(''); + }); + + it('falls back to no break when segments carry no timings', () => { + const out = toParagraphs([{ text: ' One.' }, { text: ' Two.' }]); + expect(out).toBe('One. Two.'); + }); +}); diff --git a/sidecar/transcribe.ts b/sidecar/transcribe.ts new file mode 100644 index 0000000..09dd337 --- /dev/null +++ b/sidecar/transcribe.ts @@ -0,0 +1,416 @@ +// Whisper transcription, always in the audio's own language. +// +// Detect the language first, then transcribe in it. Whisper's translate mode is English-only, so it +// cannot honour "translate into " for anything else — rather than silently returning a +// rough English rendering of Portuguese audio, transcription is always faithful to the source. +// +// The detection must win when Whisper is confident. `spokenLanguages` only breaks ties on clips +// Whisper is genuinely unsure about, where its low-confidence guess defaults to English. +// +// The transcription pass is tuned for one person talking into a phone, which is what this is mostly +// used for. See the constants below: the defaults whisper.cpp ships are aimed at clean dictation and +// produce loops, clipped words and breath-length paragraph breaks on real voice memos. +// +// Both requests pass `timeout: false`. Bun's fetch aborts at 300s by default, and whisper.cpp runs at +// roughly 8x realtime, so anything over ~40 minutes of audio died with "The operation timed out" +// about 20 seconds short of the answer. AbortSignal.timeout() does not raise that ceiling — only +// disabling the built-in timeout does. + +// Bun's fetch accepts `timeout: false` to disable its built-in 300s ceiling. The DOM RequestInit +// type this project compiles against does not declare it, so widen it here rather than casting at +// each call site. +type BunRequestInit = RequestInit & { timeout?: boolean | number }; + +type TranscribeArgs = { + file: File | Blob; + whisperUrl: string; + spokenLanguages: string[]; +}; + +export type TranscribeResult = { + text: string; + detectedLanguage: string; +}; + +// Above this probability Whisper's own detection is taken as authoritative. Below it the clip is +// treated as ambiguous and the user's spoken languages break the tie. +const CONFIDENT_DETECTION = 0.5; + +// whisper.cpp's "language" field uses full names ("english"); ISO codes live in +// `language_probabilities`. Map covers what the user can pick in Profile settings. +const LANG_NAME_TO_ISO: Record = { + english: 'en', + portuguese: 'pt', + spanish: 'es', + french: 'fr', + german: 'de', + italian: 'it', + dutch: 'nl', + russian: 'ru', + chinese: 'zh', + japanese: 'ja', + korean: 'ko', +}; + +// Whisper reuses its own decoded output as the prompt for the next 30-second window, which is what +// makes it fall into loops ("I know. I know. I know.") once it slips. `max_context` is the budget for +// that carried text, and setting it to 0 stops the loops — but the same switch also disables the +// initial prompt, since whisper.cpp gates both on `n_max_text_ctx > 0`. +// +// The way to keep the prompt and still starve the loop: `carry_initial_prompt` puts the prompt in a +// static slot that is filled first and in full, and only the leftover budget goes to decoded text. +// Every prompt below is comfortably longer than this budget, so there is never any leftover — the +// vocabulary and style reach every window and none of the model's own output does. +const PROMPT_CONTEXT_TOKENS = 96; + +// Whisper imitates whatever it is primed with; it does not take instructions. So each prompt is a +// worked example of the output wanted — one person talking, punctuated the way it should come back — +// and nothing else. An earlier version opened by *describing* the format ("Commas separate clauses, +// and apostrophes are written properly") and that backfired: priming words which never occur in +// speech put them in play, and a character surname came back from a Monk episode as "Commas". +// Everything here is ordinary conversational language for that reason. +// +// Each is written in its own language, since an English prompt on Portuguese audio biases the +// output, and each deliberately runs past PROMPT_CONTEXT_TOKENS — see above for why the length is +// load-bearing. Overflow is trimmed from the front, so nothing depends on the opening words. +// +// Languages without an entry fall through to no prompt at all, which is handled explicitly below. +// +// Every prompt uses its language's full diacritic set (and, for a non-Latin script, its own +// punctuation — Japanese 。/、, Greek ; as a question mark, Arabic ، and ؟) in ordinary words, never +// forced in. Omitting one, e.g. writing Swedish without å ä ö, teaches Whisper to drop it too. +const TRANSCRIPT_PROMPTS: Record = { + en: + 'Hey, sorry for the late reply — I was stuck in a meeting all morning and only just got out. ' + + "I had a look at the document you sent over, and honestly, I think it's mostly fine. There are " + + "a couple of things I'd change in the second half, but nothing serious. The introduction reads " + + 'really well. Let me know when you have a few minutes and we can talk it through properly. ' + + "I'm around all afternoon, so whenever suits you is fine by me. Thanks again for putting it " + + 'together, I know it took a while.', + pt: + 'Olá, desculpa a demora a responder — estive numa reunião toda a manhã e só agora consegui ' + + 'sair. Já vi o documento que enviaste e, sinceramente, acho que está quase tudo bem. Há duas ' + + 'ou três coisas que mudava na segunda metade, mas nada de grave. A introdução está muito bem ' + + 'escrita. Diz-me quando tiveres uns minutos e falamos com calma. Estou por aqui toda a tarde, ' + + 'por isso quando te der jeito está bem para mim. Obrigado outra vez por teres tratado disto, ' + + 'sei que deu trabalho.', + es: + 'Hola, perdona la tardanza en responder — he estado toda la mañana en una reunión y recién ahora ' + + 'he podido salir. Ya vi el documento que me enviaste y, la verdad, creo que está bastante bien. ' + + 'Hay un par de cosas que cambiaría en la segunda mitad, pero nada grave. La introducción está muy ' + + 'bien escrita. Avísame cuando tengas unos minutos y lo hablamos con calma. Estoy disponible toda ' + + 'la tarde, así que cuando te venga bien está perfecto para mí. Gracias otra vez por armarlo, sé ' + + 'que llevó tiempo.', + de: + 'Hallo, entschuldige die späte Antwort — ich saß den ganzen Vormittag in einer Besprechung und ' + + 'bin gerade erst rausgekommen. Ich habe mir das Dokument angeschaut, das du geschickt hast, und ' + + 'ehrlich gesagt finde ich es größtenteils gut. In der zweiten Hälfte würde ich ein paar ' + + 'Kleinigkeiten ändern, aber nichts Ernstes. Die Einleitung liest sich wirklich gut. Sag mir ' + + 'Bescheid, wenn du später ein paar Minuten hast, dann können wir in Ruhe darüber reden. Ich bin ' + + 'den ganzen Nachmittag über da, also wann immer es dir passt, ist mir recht. Danke nochmal, dass ' + + 'du dir die Mühe gemacht hast, ich weiß, das hat Zeit gekostet.', + fr: + "Salut, désolé pour la réponse tardive — j'étais coincé en réunion toute la matinée et je viens " + + "tout juste d'en sortir. J'ai regardé le document que tu as envoyé, et honnêtement, je trouve que " + + "c'est plutôt bien. Il y a deux ou trois choses que je changerais dans la deuxième moitié, mais " + + "rien de grave. L'introduction est vraiment bien écrite. Dis-moi quand tu as quelques minutes et " + + "on en parle tranquillement. Je suis disponible tout l'après-midi, donc quand ça t'arrange, ça me " + + "va. Merci encore de l'avoir préparé, je sais que ça a pris du temps.", + sv: + 'Hej, förlåt att jag inte svarat tidigare — jag satt i möte hela förmiddagen och kom precis loss. ' + + 'Jag har tittat på dokumentet du skickade, och ärligt talat tycker jag att det mesta är bra. Det ' + + 'är några saker jag skulle ändra i andra halvan, men inget allvarligt. Inledningen är verkligen ' + + 'bra skriven. Säg till när du har några minuter så pratar vi igenom det ordentligt. Jag är ledig ' + + 'hela eftermiddagen, så när det passar dig passar det mig. Tack igen för att du satte ihop det, ' + + 'jag vet att det tog ett tag.', + it: + 'Ciao, scusa il ritardo nella risposta — sono stato in riunione tutta la mattina e sono appena ' + + "uscito. Ho dato un'occhiata al documento che mi hai mandato e, sinceramente, penso che vada bene " + + 'quasi tutto. Ci sono un paio di cose che cambierei nella seconda metà, ma niente di grave. ' + + "L'introduzione è scritta davvero bene. Fammi sapere quando hai qualche minuto così ne parliamo " + + 'con calma. Sono libero tutto il pomeriggio, quindi va bene quando ti fa comodo. Grazie ancora ' + + 'per averlo messo insieme, so che ci è voluto tempo.', + nl: + 'Hoi, sorry voor het late antwoord — ik zat de hele ochtend in een vergadering en ben er net pas ' + + 'uit. Ik heb naar het document gekeken dat je hebt gestuurd, en eerlijk gezegd vind ik het ' + + 'grotendeels goed. Er zijn een paar dingen die ik in de tweede helft zou veranderen, maar niets ' + + 'ergs. De inleiding leest heel prettig. Laat het me weten wanneer je een paar minuten hebt, dan ' + + 'bespreken we het rustig. Ik ben de hele middag beschikbaar, dus wanneer het jou uitkomt is voor ' + + 'mij prima. Bedankt nogmaals dat je het hebt samengesteld, ik weet dat het tijd heeft gekost.', + pl: + 'Cześć, przepraszam za późną odpowiedź — siedziałem cały ranek na spotkaniu i w końcu udało mi ' + + 'się z niego wyrwać. Zerknąłem na dokument, który przesłałeś, i szczerze mówiąc, myślę, że ' + + 'większość jest w porządku. Jest kilka rzeczy, które zmieniłbym w drugiej połowie, ale nic ' + + 'poważnego. Wstęp jest naprawdę dobrze napisany. Daj znać, kiedy będziesz mieć chwilę, możemy to ' + + 'spokojnie omówić. Jestem wolny całe popołudnie, więc kiedy ci pasuje, mi też pasuje. Dzięki ' + + 'jeszcze raz za to, że to zebrałeś, wiem, że zajęło to trochę czasu.', + da: + 'Hej, undskyld det sene svar — jeg sad i møde hele formiddagen og er lige kommet ud. Jeg har ' + + 'kigget på dokumentet, du sendte, og ærligt talt synes jeg det meste er fint. Der er et par ting, ' + + 'jeg ville ændre i anden halvdel, men ikke noget alvorligt. Indledningen er skrevet rigtig godt. ' + + 'Sig til, når du har et par minutter, så snakker vi ordentligt om det. Jeg er ledig hele ' + + 'eftermiddagen, så det passer mig, når det passer dig. Tak igen fordi du satte det sammen, jeg ' + + 'ved det tog tid.', + no: + 'Hei, beklager det sene svaret — jeg satt i møte hele formiddagen og er akkurat ferdig. Jeg har ' + + 'sett på dokumentet du sendte, og ærlig talt synes jeg det meste er bra. Det er et par ting jeg ' + + 'ville endret i andre halvdel, men ingenting alvorlig. Innledningen er skrevet veldig bra. Si ' + + 'ifra når du har noen minutter, så tar vi en skikkelig prat om det. Jeg er ledig hele ' + + 'ettermiddagen, så det som passer deg passer meg. Takk igjen for at du satte det sammen, jeg vet ' + + 'det tok tid.', + fi: + 'Hei, anteeksi myöhäinen vastaus — istuin koko aamupäivän palaverissa ja pääsin juuri äsken pois. ' + + 'Katsoin dokumentin, jonka lähetit, ja rehellisesti sanottuna suurin osa siitä on ihan hyvä. ' + + 'Loppupuoliskolla on pari asiaa, jotka muuttaisin, mutta ei mitään vakavaa. Johdanto on ' + + 'kirjoitettu tosi hyvin. Sano, kun sinulla on hetki aikaa, niin jutellaan siitä kunnolla. Olen ' + + 'vapaana koko iltapäivän, joten mikä tahansa sopii sinulle, sopii minulle. Kiitos vielä kerran, ' + + 'kun kokosit sen, tiedän että siihen meni aikaa.', + tr: + 'Selam, geç cevap verdiğim için kusura bakma — bütün sabah toplantıdaydım ve az önce çıkabildim. ' + + 'Gönderdiğin dokümana baktım ve açıkçası çoğunlukla iyi olduğunu düşünüyorum. İkinci yarıda ' + + 'değiştireceğim birkaç şey var ama ciddi bir şey değil. Giriş kısmı gerçekten iyi yazılmış. ' + + 'Birkaç dakikan olduğunda haber ver, düzgünce konuşuruz. Bütün öğleden sonra müsaitim, yani sana ' + + 'uyan zaman bana da uyar. Bunu hazırladığın için tekrar teşekkürler, biliyorum zaman almıştır.', + cs: + 'Ahoj, promiň za pozdní odpověď — celé dopoledne jsem byl na poradě a teprve teď jsem se dostal ' + + 'ven. Podíval jsem se na dokument, cos poslal, a upřímně řečeno, myslím, že většina je v pořádku. ' + + 'V druhé polovině bych pár věcí změnil, ale nic vážného. Úvod je napsaný fakt dobře. Dej vědět, ' + + 'až budeš mít pár minut, můžeme to pořádně probrat. Celé odpoledne mám volno, takže kdy se ti to ' + + 'bude hodit, mně to vyhovuje. Díky ještě jednou, žes to dal dohromady, vím, že to zabralo čas.', + el: + 'Γεια σου, συγγνώμη για την καθυστερημένη απάντηση — ήμουν σε συνάντηση όλο το πρωί και μόλις ' + + 'τώρα βγήκα. Είδα το έγγραφο που έστειλες και, ειλικρινά, νομίζω πως είναι σχεδόν όλο εντάξει. ' + + 'Υπάρχουν δύο τρία πράγματα που θα άλλαζα στο δεύτερο μισό, αλλά τίποτα σοβαρό. Η εισαγωγή είναι ' + + 'γραμμένη πολύ καλά. Έχεις λίγα λεπτά να τα πούμε με την ησυχία μας; Είμαι ελεύθερος όλο το ' + + 'απόγευμα, οπότε όποτε σε βολεύει εμένα με βολεύει. Ευχαριστώ ξανά που το ετοίμασες, ξέρω πως ' + + 'πήρε χρόνο.', + uk: + 'Привіт, вибач за пізню відповідь — я весь ранок сидів на нараді і щойно звідти вийшов. Подивився ' + + 'документ, який ти надіслав, і чесно кажучи, мої думки такі — майже все добре. У мене є пара ' + + 'речей, які я б змінив у другій половині, але нічого серйозного. Вступ написаний дуже добре. ' + + 'Напиши, коли матимеш кілька хвилин, і спокійно все обговоримо. Я вільний увесь день, тобто коли тобі зручно, ' + + 'мені теж підходить. Дякую ще раз, що зібрав це докупи, знаю, що це зайняло час.', + ru: + 'Привет, извини, что долго не отвечал — всё утро просидел на встрече и только что освободился. ' + + 'Посмотрел документ, который ты прислал, и, честно говоря, думаю, что почти всё нормально. Есть ' + + 'пара моментов, которые я бы изменил во второй половине, но ничего серьёзного. Введение написано ' + + 'правда хорошо. Напиши, когда будет пара свободных минут, спокойно всё обсудим. Я свободен весь ' + + 'день, так что когда тебе удобно, мне тоже подходит. Спасибо ещё раз, что всё это собрал, знаю, ' + + 'что это заняло время.', + ro: + 'Salut, scuze pentru răspunsul întârziat — am fost toată dimineața într-o ședință și abia acum am ' + + 'scăpat. M-am uitat pe documentul pe care mi l-ai trimis și, sincer, cred că e aproape totul ' + + 'bine. Sunt vreo două-trei lucruri pe care le-aș schimba în a doua jumătate, dar nimic grav. ' + + 'Introducerea e scrisă chiar bine. Spune-mi când ai câteva minute și vorbim pe îndelete despre ' + + 'asta. Sunt liber toată după-amiaza, deci oricând îți convine mie îmi convine. Mulțumesc încă o ' + + 'dată că ai pus totul cap la cap, știu că a durat.', + hu: + 'Szia, bocs a késői válaszért — egész délelőtt gyűlésen ültem, és csak most sikerült ' + + 'kiszabadulnom. Megnéztem a dokumentumot, amit küldtél, és őszintén szólva szerintem szinte ' + + 'minden rendben van. Van pár dolog, amit megváltoztatnék a második felében, de semmi komoly. A ' + + 'bevezető nagyon jól van megírva. Szólj, ha van pár szabad perced, és nyugodtan átbeszéljük. ' + + 'Egész délután érek rá, szóval amikor neked jó, nekem is megfelel. Köszönöm még egyszer, hogy ' + + 'összeraktad, tudom, hogy hosszú időbe telt.', + ja: + '返信が遅くなってごめん。朝からずっと会議で、今やっと抜けられたところ。送ってもらった資料、見たよ。' + + '正直、ほとんど問題ないと思う。後半のところで直したい部分が二つ三つあるけど、大したことじゃない。' + + '導入の部分はすごくよく書けてると思う。時間があるときに教えて、ゆっくり話そう。' + + '午後はずっと空いてるから、都合のいいときでいいよ。まとめてくれてありがとう、時間かかったよね。', + zh: + '抱歉这么晚才回你——我一早上都在开会，刚出来。你发的那份文件我看过了，说实话，大部分都挺好的。' + + '后半部分有几个地方我想改一下，但都不是什么大问题。开头写得真的很不错。你什么时候有空跟我说一声，我们好好聊聊。' + + '我整个下午都有空，所以你什么时候方便都行。再次谢谢你把这个整理出来，我知道挺花时间的。', + ko: + '답장이 늦어서 미안해. 오전 내내 회의하다가 방금 겨우 빠져나왔어. 보내준 문서 봤는데, 솔직히 거의 다 괜찮은 것 같아. 후반부에 고치고 싶은 부분이 두어 개 있긴 한데, ' + + '심각한 건 아니야. 서론은 진짜 잘 쓴 것 같아. 시간 될 때 얘기해줘, 그럼 차분히 얘기해보자. 오후 내내 시간 있으니까 편한 때 아무 때나 괜찮아. 정리해줘서 다시 한번 ' + + '고마워, 시간 좀 걸렸을 텐데.', + ar: + 'مرحبًا، آسف على تأخري في الرد — كنت في اجتماع طوال الصباح ولتوّي خرجت منه. اطّلعت على المستند ' + + 'الذي أرسلته، وبصراحة أعتقد أن معظمه جيد. هناك بعض الأشياء التي أود تغييرها في النصف الثاني، لكن ' + + 'لا شيء خطير. المقدمة مكتوبة بشكل جيد جدًا. هل يمكن أن تخبرني متى تكون لديك بضع دقائق لنتحدث ' + + 'بهدوء؟ أنا متفرغ طوال بعد الظهر، فأي وقت يناسبك يناسبني. شكرًا مرة أخرى على إعدادك لهذا، أعرف أن ' + + 'الأمر استغرق وقتًا.', + he: + 'היי, סליחה על התגובה המאוחרת — הייתי בישיבה כל הבוקר ורק עכשיו יצאתי. הסתכלתי על המסמך ששלחת, ' + + 'ובכנות, אני חושב שרוב הדברים בסדר. יש כמה דברים שהייתי משנה בחצי השני, אבל שום דבר רציני. הפתיחה ' + + 'כתובה ממש טוב. תגיד לי מתי יש לך כמה דקות ונדבר על זה בנחת. אני פנוי כל אחר הצהריים, אז מתי שנוח ' + + 'לך נוח גם לי. תודה שוב שסידרת את זה, אני יודע שזה לקח זמן.', + hi: + 'हाय, देर से जवाब देने के लिए माफ़ करना — मैं पूरी सुबह मीटिंग में था और अभी-अभी निकल पाया। तुमने ' + + 'जो डॉक्यूमेंट भेजा था, वो देख लिया, और सच कहूं तो ज़्यादातर हिस्सा ठीक लग रहा है। दूसरे हिस्से ' + + 'में कुछ चीज़ें हैं जो मैं बदलना चाहूंगा, पर कुछ गंभीर नहीं है। शुरुआत बहुत अच्छी लिखी गई है। जब ' + + 'भी कुछ मिनट मिलें बता देना, फिर आराम से बात करते हैं। पूरी दोपहर मैं फ्री हूं, तो जब भी तुम्हें ' + + 'सही लगे मुझे ठीक है। इसे तैयार करने के लिए फिर से शुक्रिया, पता है इसमें वक़्त लगा होगा।', +}; + +// Beam search rather than the default greedy decode (whisper.cpp ships `beam_size = -1`). It is the +// single biggest quality lever here and it costs time, which this flow has — nothing waits on it. +const BEAM_SIZE = 5; +const BEST_OF = 5; + +// The whisper container already runs silero VAD, but its defaults assume clean dictation. Widen the +// segments so they break at real pauses instead of at breaths, and pad the edges so word onsets are +// not clipped — clipped onsets are what shows up in the output as a word split down the middle. +const VAD_TUNING: Record = { + vad_speech_pad_ms: '200', // default 30 + vad_min_silence_duration_ms: '500', // default 100 — a breath is not a sentence boundary + vad_samples_overlap: '0.2', // default 0.1 + vad_max_speech_duration_s: '25', // default unbounded — keep a segment inside one 30s window +}; + +// A silence this long reads as a new thought. Whisper reports segment times against the original +// audio even when VAD is on (whisper.cpp maps them back through its VAD table), so these really are +// gaps in the recording rather than artefacts of the filtering. +const PARAGRAPH_GAP_SECONDS = 1.2; +const MAX_PARAGRAPH_CHARS = 900; + +// OpenAI Whisper's own silence rule: a segment the model thinks is probably not speech *and* decodes +// with low confidence is a hallucination rather than a transcript. whisper.cpp applies a subtly +// different form of this internally, so it is worth re-applying to the values it reports — this is +// what removes the stray words that appear over music and room tone. +const NO_SPEECH_PROB = 0.6; +const LOW_CONFIDENCE_LOGPROB = -1.0; + +export type WhisperSegment = { + text?: string; + start?: number; + end?: number; + no_speech_prob?: number; + avg_logprob?: number; +}; + +// Defaults are chosen so a segment missing either field is kept: dropping real speech is much worse +// than keeping a stray word. +const isHallucinatedSilence = (segment: WhisperSegment) => + (segment.no_speech_prob ?? 0) > NO_SPEECH_PROB && (segment.avg_logprob ?? 0) < LOW_CONFIDENCE_LOGPROB; + +// Whisper emits each segment with a leading space and now and then strands trailing punctuation in a +// segment of its own, which is why the old `response_format=text` output contained lone "." lines. +const appendSegment = (paragraph: string, text: string) => { + if (!paragraph) return text; + return /^[.,;:!?…)\]]/.test(text) ? paragraph + text : `${paragraph} ${text}`; +}; + +// Whisper returns one segment per utterance, not per paragraph. Rejoin them into prose and start a +// new paragraph on a long pause, so a voice memo reads as something written rather than as a list of +// fragments. +export function toParagraphs(segments: WhisperSegment[]): string { + const paragraphs: string[] = []; + let current = ''; + let previousEnd: number | null = null; + + for (const segment of segments) { + const text = (segment.text ?? '').replace(/\s+/g, ' ').trim(); + if (!text || isHallucinatedSilence(segment)) continue; + + const gap = previousEnd !== null && segment.start !== undefined ? segment.start - previousEnd : 0; + // Only break an over-long paragraph where a sentence actually ended, never mid-thought. + const overlong = current.length >= MAX_PARAGRAPH_CHARS && /[.!?…]["')\]]?$/.test(current); + + if (current && (gap >= PARAGRAPH_GAP_SECONDS || overlong)) { + paragraphs.push(current); + current = ''; + } + + current = appendSegment(current, text); + if (segment.end !== undefined) previousEnd = segment.end; + } + + if (current) paragraphs.push(current); + return paragraphs.join('\n\n'); +} + +export async function transcribeAudio({ + file, + whisperUrl, + spokenLanguages, +}: TranscribeArgs): Promise { + const base = whisperUrl.replace(/\/+$/, ''); + + // Step 1: detect language, and only fall back to the spoken-language bias when Whisper is unsure. + const detectForm = new FormData(); + detectForm.append('file', file); + detectForm.append('temperature', '0.0'); + detectForm.append('response_format', 'verbose_json'); + detectForm.append('detect_language', 'true'); + + const detectRes = await fetch(`${base}/inference`, { + method: 'POST', + body: detectForm, + timeout: false, + } as BunRequestInit); + if (!detectRes.ok) throw new Error(`Language detection failed (${detectRes.status})`); + const detectJson = (await detectRes.json()) as { + language?: string; + detected_language?: string; + detected_language_probability?: number; + language_probabilities?: Record; + }; + + const probs = detectJson.language_probabilities; + const ranked = probs ? Object.entries(probs).sort((a, b) => b[1] - a[1]) : []; + const [topCode, topProb] = ranked[0] ?? []; + + let detectedLanguage: string; + if (topCode && (topProb ?? 0) >= CONFIDENT_DETECTION) { + // Whisper is sure. Trust it even when it is not a language the user speaks — that is exactly the + // case translation exists for. + detectedLanguage = topCode; + } else if (ranked.length > 0 && spokenLanguages.length > 0) { + // Genuinely ambiguous (usually a short clip). Break the tie toward a language the user speaks, + // since Whisper's low-confidence guess defaults to English regardless of the audio. + const bestSpoken = spokenLanguages + .map((code) => [code, probs?.[code] ?? 0] as const) + .sort((a, b) => b[1] - a[1])[0]; + detectedLanguage = bestSpoken?.[0] ?? topCode ?? 'en'; + } else if (topCode) { + detectedLanguage = topCode; + } else { + // No probabilities field (non-whisper.cpp backend) — try to map the reported name. + const raw = (detectJson.detected_language ?? detectJson.language ?? 'en').toLowerCase(); + detectedLanguage = LANG_NAME_TO_ISO[raw] ?? raw; + } + + // Step 2: transcribe in the detected language + const transcribeForm = new FormData(); + transcribeForm.append('file', file); + transcribeForm.append('language', detectedLanguage); + transcribeForm.append('temperature', '0.0'); + transcribeForm.append('temperature_inc', '0.2'); + transcribeForm.append('beam_size', String(BEAM_SIZE)); + transcribeForm.append('best_of', String(BEST_OF)); + transcribeForm.append('suppress_nst', 'true'); + transcribeForm.append('response_format', 'verbose_json'); + // verbose_json re-runs a full language auto-detect unless this is set. Step 1 already answered + // that question, so it is pure duplicated work on every transcription. + transcribeForm.append('no_language_probabilities', 'true'); + // Segment timestamps drive the paragraphing below and are kept; per-token timestamps are extra + // work on every segment and nothing here reads them. + transcribeForm.append('token_timestamps', 'false'); + + const prompt = TRANSCRIPT_PROMPTS[detectedLanguage]; + if (prompt) { + transcribeForm.append('prompt', prompt); + transcribeForm.append('carry_initial_prompt', 'true'); + transcribeForm.append('max_context', String(PROMPT_CONTEXT_TOKENS)); + } else { + // Nothing worth protecting the budget for, so close the loop-feeding path outright. + transcribeForm.append('max_context', '0'); + } + + for (const [key, value] of Object.entries(VAD_TUNING)) transcribeForm.append(key, value); + + const res = await fetch(`${base}/inference`, { + method: 'POST', + body: transcribeForm, + timeout: false, + } as BunRequestInit); + if (!res.ok) throw new Error(`Transcription failed (${res.status})`); + + const json = (await res.json()) as { text?: string; segments?: WhisperSegment[] }; + // Fall back to the flat text for any backend that does not return segments. + const segments = json.segments ?? []; + const text = segments.length > 0 ? toParagraphs(segments) : (json.text ?? '').trim(); + + return { text, detectedLanguage }; +} diff --git a/skills/transcribe/SKILL.md b/skills/transcribe/SKILL.md new file mode 100644 index 0000000..7016e95 --- /dev/null +++ b/skills/transcribe/SKILL.md @@ -0,0 +1,69 @@ +--- +name: Transcribe +description: Use when you need to turn a recording into text — a voice memo, a meeting, dictation on disk. Carries the route, the request shape, and what to do when nothing is configured. +version: 1 +--- + +Officer's speech-to-text runs entirely inside the `transcribe` plugin, over your own whisper.cpp server. +Nothing else on the platform reads audio for you — a request that mentions a recording, a voice memo, a +call, a lecture, means this plugin. + +**There is no model turn here.** The plugin does the whole job itself: detect the spoken language, then +transcribe faithfully in it. Do not narrate the audio yourself, do not summarize it in place of +transcribing, and do not chain a translation step — the plugin is already faithful to the source language, +and Whisper's translate mode is English-only, so anything past it would be a rough loss rather than a +feature. + +## Is it set up? + +``` +GET /api/transcribe/config +→ { "configured": true, "url": "http://192.168.1.50:64201" } +→ { "configured": false, "url": null } +``` + +If `configured` is `false`, say so plainly and stop — "speech-to-text isn't set up yet; point it at a +whisper.cpp server with `PUT /api/transcribe/config`" — rather than attempting the transcription and +reporting a generic failure. Setting it up is a one-time, deliberate act (it validates the URL against the +server before storing it), not something to do silently on the caller's behalf without being asked. + +``` +PUT /api/transcribe/config +{ "url": "http://192.168.1.50:64201" } +→ { "configured": true, "url": "http://192.168.1.50:64201" } +→ 400 { "error": "…" } — url missing, malformed, or the server did not answer +``` + +## Transcribing + +``` +POST /api/transcribe/transcribe +multipart/form-data: + file — the audio file (mp3, wav, m4a, whatever whisper.cpp accepts) + spokenLanguages — zero or more, e.g. spokenLanguages=en&spokenLanguages=pt + +→ { "text": "…", "detectedLanguage": "en" } +→ 400 { "error": "Whisper not configured — PUT /api/transcribe/config first" } +→ 502 { "error": "…" } — the whisper.cpp server rejected or failed the request +``` + +`spokenLanguages` only matters when Whisper's own detection is genuinely unsure — it breaks the tie +toward a language you already know the speaker uses. Pass what you know; pass nothing and detection still +works, it is just less informed on an ambiguous clip. + +**This can take minutes** on a long recording — whisper.cpp runs at roughly 8x realtime, so a 40-minute +memo is a couple of minutes of waiting, not a couple of seconds. Do not treat a slow response as a failure +and retry; wait for it. + +## The file-browser task + +`tasks/transcribe-audio-file` in this plugin ships a right-click task for `.mp3`/`.wav`/`.m4a` files that +calls this same route and files the result next to the original — a **script** task, deliberately, with no +model turn per audio file. If you are asked to transcribe something already sitting in the file browser, +prefer pointing the user at that over doing it yourself turn-by-turn. + +## What this is not + +Not a live dictation microphone — there is no streaming endpoint, only a file upload. Not a translator — +it transcribes in whatever language was spoken, never into a different one. And it holds no memory of past +transcriptions; every call is independent. diff --git a/tasks/transcribe-audio-file/TASK.md b/tasks/transcribe-audio-file/TASK.md new file mode 100644 index 0000000..497de9c --- /dev/null +++ b/tasks/transcribe-audio-file/TASK.md @@ -0,0 +1,58 @@ +--- +name: Transcribe Audio File +description: Transcribe an audio file to text over your own whisper.cpp server, no model turn per file. +category: Media +version: 1 +mode: script +inline: false +language: typescript +tags: + - audio + - transcription +skills: + - transcribe +inputs: + - name: file_path + description: Path to the audio file to transcribe. Auto-filled from the file browser. + required: true +trigger: + - type: file + extensions: + - mp3 + - wav + - m4a +--- + +# Transcribe Audio File + +Sends the audio file to the `transcribe` plugin's own route — `POST /api/transcribe/transcribe` — and +files the result next to the original. + +A **script** task, deliberately: the plugin already does its own language detection and transcription +server-side, so there is nothing here worth a model turn for. Running one per audio file would be a +model call spent re-deriving what the plugin already returns. + +## What it does + +1. Read the audio file at `file_path` and the caller's spoken-language preferences + (`GET /api/user/settings`). +2. `POST` the file to `/api/transcribe/transcribe`. The plugin detects the language and transcribes + faithfully in it — see `plugins/transcribe/skills/transcribe/SKILL.md` for the contract. +3. Create a sibling directory named `_`, where `` is today in `YYYYMMDD` and `` + is derived from the original file's own name (not a generated title — a script task gets no model + turn to ask for one). +4. Move the original audio file into that directory and save the transcription beside it as a + Markdown file with the same base name. + +## Before it runs + +The plugin has to be configured — `PUT /api/transcribe/config` with a reachable whisper.cpp URL. If it +is not, this fails with that told plainly rather than a generic error. + +## What it deliberately does not do + +The worked example this was adapted from (an agentic task) ran its own detect-language pass before +transcribing, and generated a title from the transcription's contents. Both are dropped here: the +detect-then-transcribe step is already what the plugin's route does internally, so repeating it would be +redundant work against the same server. And a generated title needs a model turn, which is exactly what +this task exists to avoid spending per file. diff --git a/tasks/transcribe-audio-file/index.ts b/tasks/transcribe-audio-file/index.ts new file mode 100644 index 0000000..c1ebe6a --- /dev/null +++ b/tasks/transcribe-audio-file/index.ts @@ -0,0 +1,118 @@ +// Transcribe Audio File — a SCRIPT task, so this is a plain program, not an agent. It is materialized +// alone into a temp dir and run with `bun run index.ts` (see platform/CLAUDE.md → "Task-runner data flow"), +// which means it must be fully self-contained: no sibling imports, everything inline. +// +// Calls Officer's own API rather than talking to whisper.cpp directly — the platform hands every script +// task OFFICER_API_URL / OFFICER_API_HOST / OFFICER_AUTH_TOKEN for exactly this (see +// src/servers/api/tasks/task-api-env.ts). The transcribe plugin owns the Whisper contract; this script +// only owns filing the result. + +import { rename, mkdir } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; +import { basename, dirname, join, extname } from 'node:path'; + +const filePath = process.env.INPUT_FILE_PATH; +if (!filePath) { + console.error('file_path is required'); + process.exit(1); +} + +const apiUrl = process.env.OFFICER_API_URL; +const apiHost = process.env.OFFICER_API_HOST; +const authToken = process.env.OFFICER_AUTH_TOKEN; +if (!apiUrl || !apiHost || !authToken) { + console.error('missing OFFICER_API_URL / OFFICER_API_HOST / OFFICER_AUTH_TOKEN — not running as a task?'); + process.exit(1); +} + +const authHeaders = { Authorization: `Bearer ${authToken}`, Host: apiHost }; + +const slugify = (name: string) => + name + .trim() + .toLowerCase() + .replace(/\s+/g, '-') + .replace(/[^a-z0-9-]/g, '') + .replace(/-+/g, '-') + .replace(/^-|-$/g, ''); + +async function getSpokenLanguages(): Promise { + try { + const res = await fetch(`${apiUrl}/api/user/settings`, { headers: authHeaders }); + if (!res.ok) return []; + const settings = (await res.json()) as { languages?: { spoken?: string[] } }; + return settings.languages?.spoken ?? []; + } catch { + // Not fatal — the plugin's own detection still runs, spoken languages only break a genuine tie. + return []; + } +} + +async function transcribe(path: string, spokenLanguages: string[]): Promise<{ text: string; detectedLanguage: string }> { + const form = new FormData(); + form.append('file', Bun.file(path), basename(path)); + for (const lang of spokenLanguages) form.append('spokenLanguages', lang); + + const res = await fetch(`${apiUrl}/api/transcribe/transcribe`, { + method: 'POST', + headers: authHeaders, + body: form, + }); + + const json = (await res.json().catch(() => null)) as { text?: string; detectedLanguage?: string; error?: string } | null; + if (!res.ok || !json) { + throw new Error(json?.error ?? `transcription failed (${res.status})`); + } + return { text: json.text ?? '', detectedLanguage: json.detectedLanguage ?? 'unknown' }; +} + +/** `/`, walking `-2`, `-3`, … past a collision. Same shape as the file browser's own. */ +async function uniqueDir(parentDir: string, name: string): Promise { + let candidate = join(parentDir, name); + let n = 2; + while (existsSync(candidate)) { + candidate = join(parentDir, `${name}-${n}`); + n += 1; + } + return candidate; +} + +async function main() { + const path = filePath!; + if (!existsSync(path)) { + console.error(`no such file: ${path}`); + process.exit(1); + } + + const spokenLanguages = await getSpokenLanguages(); + + console.log(`Transcribing ${basename(path)}…`); + const { text, detectedLanguage } = await transcribe(path, spokenLanguages); + console.log(`Detected language: ${detectedLanguage}`); + + const parentDir = dirname(path); + const ext = extname(path); + const base = basename(path, ext); + + const now = new Date(); + const date = `${now.getFullYear()}${String(now.getMonth() + 1).padStart(2, '0')}${String(now.getDate()).padStart(2, '0')}`; + // No model turn to ask for a title (see TASK.md), so the slug comes from the original filename rather + // than the transcription's contents. + const slug = slugify(base) || 'recording'; + + const archiveDir = await uniqueDir(parentDir, `${date}_${slug}`); + await mkdir(archiveDir, { recursive: true }); + + const archivedAudioPath = join(archiveDir, basename(path)); + await rename(path, archivedAudioPath); + + const transcriptPath = join(archiveDir, `${base}.md`); + await Bun.write(transcriptPath, text); + + console.log(`Filed under ${archiveDir}`); +} + +main().catch((err) => { + console.error(err instanceof Error ? err.message : String(err)); + process.exit(1); +});