diff --git a/src/apps/officer-web/Screens/Dashboard/Layout/Spotlight/useSpotlightItems.ts b/src/apps/officer-web/Screens/Dashboard/Layout/Spotlight/useSpotlightItems.ts index 20c53368..2b73e324 100644 --- a/src/apps/officer-web/Screens/Dashboard/Layout/Spotlight/useSpotlightItems.ts +++ b/src/apps/officer-web/Screens/Dashboard/Layout/Spotlight/useSpotlightItems.ts @@ -9,7 +9,6 @@ import { LayoutGrid, Lock, Mail, - Mic, ScanText, ShieldCheck, Sparkles, @@ -156,7 +155,6 @@ const SETTINGS_ITEMS: SettingsItem[] = [ }, // System — SystemSettings.tsx { id: 'set:tts', labelKey: 'settings.sections.tts.label', to: '/settings/system/tts', icon: Volume2 }, - { id: 'set:stt', labelKey: 'settings.sections.stt.label', to: '/settings/system/stt', icon: Mic }, { id: 'set:ocr', labelKey: 'settings.sections.ocr.label', to: '/settings/system/ocr', icon: ScanText }, { id: 'set:maintenance', diff --git a/src/apps/officer-web/Screens/Dashboard/Settings/ServerSettings/STTSection.tsx b/src/apps/officer-web/Screens/Dashboard/Settings/ServerSettings/STTSection.tsx deleted file mode 100644 index 511d0de8..00000000 --- a/src/apps/officer-web/Screens/Dashboard/Settings/ServerSettings/STTSection.tsx +++ /dev/null @@ -1,107 +0,0 @@ -import type { FormEvent } from 'react'; -import { useState, useEffect } from 'react'; -import { useQuery, useQueryClient } from '@tanstack/react-query'; -import { toast } from 'sonner'; -import { Button } from '@/components/ui/button'; -import { Input } from '@/components/ui/input'; -import { Label } from '@/components/ui/label'; -import { useClient } from 'hooks/useClient'; -import { useTranslation } from '@/lib/i18n'; - -type SttConfig = { - url: string; -}; - -export const STTSection = () => { - const { t } = useTranslation(); - const client = useClient(); - const queryClient = useQueryClient(); - - const { data: config, isLoading } = useQuery({ - queryKey: ['STT_CONFIG'], - queryFn: () => client.get('/server-settings/stt'), - }); - - const [url, setUrl] = useState('http://localhost:64201'); - const [isSaving, setIsSaving] = useState(false); - const [isTesting, setIsTesting] = useState(false); - - useEffect(() => { - if (!config) return; - setUrl(config.url); - }, [config]); - - const handleSave = async (ev?: FormEvent) => { - ev?.preventDefault(); - if (isSaving) return; - setIsSaving(true); - try { - await client.put('/server-settings/stt', { url }); - queryClient.invalidateQueries({ queryKey: ['STT_CONFIG'] }); - toast.success(t('settings.server.stt.saved')); - } catch { - toast.error(t('settings.server.stt.saveFailed')); - } finally { - setIsSaving(false); - } - }; - - const handleTest = async () => { - if (isTesting) return; - setIsTesting(true); - try { - const res = await client.post<{ success?: boolean; error?: string }>('/server-settings/stt/test', { url }); - if (res.error) { - toast.error(res.error); - } else { - toast.success(t('settings.server.stt.reachable')); - } - } catch (err: unknown) { - const raw = (err as { message?: string })?.message; - let msg = 'Connection failed'; - try { - if (raw) msg = JSON.parse(raw).error ?? msg; - } catch { - /* ignore */ - } - toast.error(msg); - } finally { - setIsTesting(false); - } - }; - - if (isLoading) return

{t('settings.server.loading')}

; - - return ( -
- - -
- - -
-
- ); -}; diff --git a/src/apps/officer-web/Screens/Dashboard/Settings/SystemSettings.tsx b/src/apps/officer-web/Screens/Dashboard/Settings/SystemSettings.tsx index d6a249b4..2a1e4a13 100644 --- a/src/apps/officer-web/Screens/Dashboard/Settings/SystemSettings.tsx +++ b/src/apps/officer-web/Screens/Dashboard/Settings/SystemSettings.tsx @@ -1,5 +1,5 @@ import { useMemo } from 'react'; -import { Bot, Hammer, Mail, Mic, ScanText, Settings, Volume2 } from 'lucide-react'; +import { Bot, Hammer, Mail, ScanText, Settings, Volume2 } from 'lucide-react'; import type { LayoutNode, PanelComponents } from 'officerdev'; import { WorkspaceLayout } from 'officerdev'; @@ -7,7 +7,6 @@ import { SettingsRoute, createSettingsPanelComponents, type SettingsSectionGroup import { SMTPSection } from './ServerSettings/SMTPSection'; import { MaintenanceSection } from './ServerSettings/MaintenanceSection'; import { TTSSection } from './ServerSettings/TTSSection'; -import { STTSection } from './ServerSettings/STTSection'; import { OCRSection } from './ServerSettings/OCRSection'; const groups: SettingsSectionGroup[] = [ @@ -22,13 +21,6 @@ const groups: SettingsSectionGroup[] = [ descriptionKey: 'settings.sections.tts.description', content: , }, - { - key: 'stt', - icon: Mic, - titleKey: 'settings.sections.stt.label', - descriptionKey: 'settings.sections.stt.description', - content: , - }, { key: 'ocr', icon: ScanText, diff --git a/src/apps/officer-web/locales/de-DE.json b/src/apps/officer-web/locales/de-DE.json index 502b9119..3a8197b8 100644 --- a/src/apps/officer-web/locales/de-DE.json +++ b/src/apps/officer-web/locales/de-DE.json @@ -145,10 +145,6 @@ "label": "Text-zu-Sprache", "description": "TTS-Anbieter und Stimme" }, - "stt": { - "label": "Sprache-zu-Text", - "description": "Whisper-Server-URL" - }, "ocr": { "label": "OCR", "description": "Vision-Modell für Textextraktion" @@ -234,12 +230,6 @@ "saveFailed": "OCR-Einstellungen konnten nicht gespeichert werden", "reachable": "Vision-Modell-Server ist erreichbar" }, - "stt": { - "serverUrl": "Whisper-Server-URL", - "saved": "STT-Einstellungen gespeichert", - "saveFailed": "STT-Einstellungen konnten nicht gespeichert werden", - "reachable": "Whisper-Server ist erreichbar" - }, "tts": { "openaiCompatible": "OpenAI-kompatibel (Kokoro, OpenAI usw.)", "elevenLabs": "ElevenLabs", diff --git a/src/apps/officer-web/locales/en-GB.json b/src/apps/officer-web/locales/en-GB.json index f6d1e724..9efa42df 100644 --- a/src/apps/officer-web/locales/en-GB.json +++ b/src/apps/officer-web/locales/en-GB.json @@ -145,10 +145,6 @@ "label": "Text to Speech", "description": "TTS provider and voice" }, - "stt": { - "label": "Speech to Text", - "description": "Whisper server URL" - }, "ocr": { "label": "OCR", "description": "Vision model for text extraction" @@ -234,12 +230,6 @@ "saveFailed": "Failed to save OCR settings", "reachable": "Vision model server is reachable" }, - "stt": { - "serverUrl": "Whisper Server URL", - "saved": "STT settings saved", - "saveFailed": "Failed to save STT settings", - "reachable": "Whisper server is reachable" - }, "tts": { "openaiCompatible": "OpenAI-compatible (Kokoro, OpenAI, etc.)", "elevenLabs": "ElevenLabs", diff --git a/src/apps/officer-web/locales/es-ES.json b/src/apps/officer-web/locales/es-ES.json index 407d35c1..c0662222 100644 --- a/src/apps/officer-web/locales/es-ES.json +++ b/src/apps/officer-web/locales/es-ES.json @@ -145,10 +145,6 @@ "label": "Texto a voz", "description": "Proveedor de TTS y voz" }, - "stt": { - "label": "Voz a texto", - "description": "URL del servidor Whisper" - }, "ocr": { "label": "OCR", "description": "Modelo de visión para extracción de texto" @@ -234,12 +230,6 @@ "saveFailed": "No se pudieron guardar los ajustes de OCR", "reachable": "El servidor del modelo de visión está accesible" }, - "stt": { - "serverUrl": "URL del servidor Whisper", - "saved": "Ajustes de STT guardados", - "saveFailed": "No se pudieron guardar los ajustes de STT", - "reachable": "El servidor Whisper está accesible" - }, "tts": { "openaiCompatible": "Compatible con OpenAI (Kokoro, OpenAI, etc.)", "elevenLabs": "ElevenLabs", diff --git a/src/apps/officer-web/locales/fr-FR.json b/src/apps/officer-web/locales/fr-FR.json index 700adf39..c8a9b531 100644 --- a/src/apps/officer-web/locales/fr-FR.json +++ b/src/apps/officer-web/locales/fr-FR.json @@ -145,10 +145,6 @@ "label": "Synthèse vocale", "description": "Fournisseur et voix pour la synthèse vocale" }, - "stt": { - "label": "Reconnaissance vocale", - "description": "URL du serveur Whisper" - }, "ocr": { "label": "OCR", "description": "Modèle de vision pour l'extraction de texte" @@ -234,12 +230,6 @@ "saveFailed": "Échec de l'enregistrement des paramètres OCR", "reachable": "Le serveur du modèle de vision est accessible" }, - "stt": { - "serverUrl": "URL du serveur Whisper", - "saved": "Paramètres de reconnaissance vocale enregistrés", - "saveFailed": "Échec de l'enregistrement des paramètres de reconnaissance vocale", - "reachable": "Le serveur Whisper est accessible" - }, "tts": { "openaiCompatible": "Compatible OpenAI (Kokoro, OpenAI, etc.)", "elevenLabs": "ElevenLabs", diff --git a/src/apps/officer-web/locales/pt-PT.json b/src/apps/officer-web/locales/pt-PT.json index 53027a41..a1d147d1 100644 --- a/src/apps/officer-web/locales/pt-PT.json +++ b/src/apps/officer-web/locales/pt-PT.json @@ -145,10 +145,6 @@ "label": "Texto para fala", "description": "Fornecedor de TTS e voz" }, - "stt": { - "label": "Fala para texto", - "description": "URL do servidor Whisper" - }, "ocr": { "label": "OCR", "description": "Modelo de visão para extração de texto" @@ -234,12 +230,6 @@ "saveFailed": "Falha ao guardar as definições de OCR", "reachable": "O servidor do modelo de visão está acessível" }, - "stt": { - "serverUrl": "URL do servidor Whisper", - "saved": "Definições de STT guardadas", - "saveFailed": "Falha ao guardar as definições de STT", - "reachable": "O servidor Whisper está acessível" - }, "tts": { "openaiCompatible": "Compatível com OpenAI (Kokoro, OpenAI, etc.)", "elevenLabs": "ElevenLabs", diff --git a/src/apps/officer-web/locales/sv-SE.json b/src/apps/officer-web/locales/sv-SE.json index b11662d2..63908e5a 100644 --- a/src/apps/officer-web/locales/sv-SE.json +++ b/src/apps/officer-web/locales/sv-SE.json @@ -145,10 +145,6 @@ "label": "Text till tal", "description": "TTS-leverantör och röst" }, - "stt": { - "label": "Tal till text", - "description": "Whisper-serverns URL" - }, "ocr": { "label": "OCR", "description": "Synmodell för textextrahering" @@ -234,12 +230,6 @@ "saveFailed": "Det gick inte att spara OCR-inställningarna", "reachable": "Synmodellsservern är nåbar" }, - "stt": { - "serverUrl": "URL till Whisper-server", - "saved": "STT-inställningarna sparade", - "saveFailed": "Det gick inte att spara STT-inställningarna", - "reachable": "Whisper-servern är nåbar" - }, "tts": { "openaiCompatible": "OpenAI-kompatibel (Kokoro, OpenAI, m.fl.)", "elevenLabs": "ElevenLabs", diff --git a/src/servers/api/chat/chat.ts b/src/servers/api/chat/chat.ts index 6d4f3401..f04a262d 100644 --- a/src/servers/api/chat/chat.ts +++ b/src/servers/api/chat/chat.ts @@ -5,7 +5,6 @@ import { createRouter } from '../../create-router'; import * as errors from '@@/custom-errors'; import * as sidecar from '@@/sidecar-registry'; import { - getUserSettings, getChatSessionModel, getChatSessionModels, listCommandHqSessionIds, @@ -45,7 +44,6 @@ import type { ChatIdentity } from './claude-sessions'; import { officerModelForApiModel } from './model-aliases'; import { readAccountEmail } from './claude-usage'; import { listStoredUsage, mergeUsage, snapshotClaudeUsage } from './usage-store'; -import { readSttConfig } from '../server-settings/stt'; /** * Whose transcripts a request may read. @@ -81,7 +79,6 @@ export async function chatIdentity(user: { id: number; email: string }): Promise // identity the way `spawnClaudeAsMember` does. Restrict here rather than at the permission layer because // `chat` is one permission covering both harnesses, and splitting it would strand the grants already issued. // The matching refusal on the execution path is in `websocket.ts` → `handleChat`. -import { transcribeAudio } from '../stt/transcribe'; import { registerAgentPanelRoutes } from './agent-panels-routes'; export const chatRouter = createRouter(); @@ -629,32 +626,6 @@ chatRouter.get('/models', async (ctx: Context) => { } }); -// POST /chat/stt — proxy an audio clip to the configured Whisper server. -chatRouter.post('/stt', async (ctx: Context) => { - const sttConfig = await readSttConfig(); - if (!sttConfig?.url) { - return ctx.json({ error: 'Whisper not configured — set it up in Settings → Speech to Text' }, 400); - } - - const user = ctx.get('user'); - const body = await ctx.req.parseBody(); - const file = body['file']; - if (!file || !(file instanceof File)) { - return ctx.json({ error: 'file is required' }, 400); - } - - const settings = (await getUserSettings(user.id)) as { languages?: { spoken?: string[] } }; - const spokenLanguages = settings.languages?.spoken ?? []; - - try { - const result = await transcribeAudio({ file, whisperUrl: sttConfig.url, spokenLanguages }); - return ctx.json(result); - } catch (err) { - logger.error('STT proxy failed', { error: String(err) }); - return ctx.json({ error: err instanceof Error ? err.message : 'Failed to reach Whisper server' }, 502); - } -}); - // The agent address book — naming a chat panel so other agents on the same dashboard can reach it. // Lives on this router because it is the same authority `chat` already grants: creating and naming // Claude sessions. See servers/api/chat/agent-panels-routes.ts and docs/agent-coordination.md. diff --git a/src/servers/api/file-browser/router.ts b/src/servers/api/file-browser/router.ts index 75fef4b5..04c70797 100644 --- a/src/servers/api/file-browser/router.ts +++ b/src/servers/api/file-browser/router.ts @@ -6,9 +6,7 @@ import { getOwnerHomeDir, DATA_PATH } from '@@/data-path'; import { resolveHomeDir } from '@@/user-home'; import * as errors from '@@/custom-errors'; import { readTtsConfig } from '@@/api/server-settings/tts'; -import { readSttConfig } from '@@/api/server-settings/stt'; import { readOcrConfig } from '@@/api/server-settings/ocr'; -import { transcribeAudio } from '@@/api/stt/transcribe'; import { getUserSettings } from 'officerdb'; async function getUserTtsVoice(userId: number): Promise { @@ -1104,46 +1102,6 @@ router.post('/extract', async (ctx) => { return ctx.json({ extractedPath }); }); -// Transcribe audio via Whisper with caching -// Workflow: detect language → check user's spoken languages → translate if needed → transcribe -router.post('/transcribe', async (ctx) => { - const user = ctx.get('user'); - const { path: filePath, root, saveNextTo } = ctx.get('body') as { path: string; root?: string; saveNextTo?: boolean }; - if (!filePath) throw errors.BAD_REQUEST('path is required'); - - const rootDir = getRootDir(user, root); - const absPath = resolveUserPath(rootDir, filePath); - const s = await stat(absPath); - if (s.isDirectory()) throw errors.BAD_REQUEST('Cannot transcribe a directory'); - - // No caching: a transcription is always produced fresh and overwrites any previous one. - const { dir, name } = parsePath(filePath.replace(/^\/+/, '')); - - const sttConfig = await readSttConfig(); - const whisperUrl = sttConfig?.url; - if (!whisperUrl) throw errors.BAD_REQUEST('Whisper not configured — set it up in Settings → Speech to Text'); - - const settings = await getUserSettings(user.id); - const spokenLanguages = (settings.languages as { spoken?: string[] })?.spoken ?? []; - - let text: string; - try { - const result = await transcribeAudio({ file: Bun.file(absPath), whisperUrl, spokenLanguages }); - text = result.text; - } catch (err) { - throw errors.BAD_REQUEST(err instanceof Error ? err.message : 'Transcription failed'); - } - - if (saveNextTo) { - const siblingRel = dir ? `${dir}/${name}.md` : `${name}.md`; - const siblingAbs = resolveUserPath(rootDir, siblingRel); - await Bun.write(siblingAbs, text); - return ctx.json({ transcriptionPath: `/${siblingRel}`, transcriptionRoot: root ?? 'home', cached: false }); - } - - return ctx.json({ text, cached: false }); -}); - // Search files by name router.get('/search', async (ctx) => { const user = ctx.get('user'); diff --git a/src/servers/api/server-settings/server-settings.ts b/src/servers/api/server-settings/server-settings.ts index 224a5607..f1132b39 100644 --- a/src/servers/api/server-settings/server-settings.ts +++ b/src/servers/api/server-settings/server-settings.ts @@ -9,7 +9,6 @@ import { readServerSettings, writeServerSettings } from 'officerdb'; import { chatProvidersRouter } from './chat-providers'; import { smtpRouter } from './smtp'; import { ttsRouter } from './tts'; -import { sttRouter } from './stt'; import { ocrRouter } from './ocr'; export const serverSettingsRouter = createRouter(); @@ -17,7 +16,6 @@ export const serverSettingsRouter = createRouter(); serverSettingsRouter.route('/chat-providers', chatProvidersRouter); serverSettingsRouter.route('/smtp', smtpRouter); serverSettingsRouter.route('/tts', ttsRouter); -serverSettingsRouter.route('/stt', sttRouter); serverSettingsRouter.route('/ocr', ocrRouter); export { readServerSettings as readSettings }; diff --git a/src/servers/api/server-settings/stt.ts b/src/servers/api/server-settings/stt.ts deleted file mode 100644 index 1d8b9910..00000000 --- a/src/servers/api/server-settings/stt.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { createRouter } from '../../create-router'; -import { readServerSettings, writeServerSettings } from 'officerdb'; - -type SttConfig = { - url: string; -}; - -export async function readSttConfig(): Promise { - const settings = await readServerSettings(); - return settings.stt as SttConfig | undefined; -} - -export const sttRouter = createRouter(); - -sttRouter.get('/', async (ctx) => { - const stt = await readSttConfig(); - if (!stt) return ctx.json(null); - return ctx.json(stt); -}); - -sttRouter.put('/', async (ctx) => { - const body = await ctx.req.json(); - const settings = await readServerSettings(); - settings.stt = body; - await writeServerSettings(settings); - return ctx.json({ success: true }); -}); - -sttRouter.post('/test', async (ctx) => { - const body = await ctx.req.json<{ url: string }>(); - if (!body.url) return ctx.json({ error: 'URL required' }, 400); - - try { - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), 5000); - const res = await fetch(`${body.url.replace(/\/+$/, '')}/inference`, { - method: 'POST', - body: new FormData(), - signal: controller.signal, - }); - clearTimeout(timeout); - // Whisper will return an error for empty form, but a response means it's reachable - return ctx.json({ success: true, status: res.status }); - } catch (err) { - const message = err instanceof Error ? err.message : 'Unknown error'; - return ctx.json({ error: message }, 500); - } -}); diff --git a/src/servers/api/stt/transcribe.test.ts b/src/servers/api/stt/transcribe.test.ts deleted file mode 100644 index 6bd6d00a..00000000 --- a/src/servers/api/stt/transcribe.test.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { describe, expect, it } from 'bun:test'; -import { toParagraphs } from './transcribe'; -import type { WhisperSegment } from './transcribe'; - -// Whisper's own shape: leading space on every segment, times in seconds. -const seg = (text: string, start: number, end: number, extra: Partial = {}): WhisperSegment => ({ - text, - start, - end, - ...extra, -}); - -describe('toParagraphs', () => { - it('joins segments spoken without a real pause into one paragraph', () => { - const out = toParagraphs([seg(' I went to the shop.', 0, 2), seg(' It was closed.', 2.2, 4)]); - expect(out).toBe('I went to the shop. It was closed.'); - }); - - it('starts a new paragraph after a long silence', () => { - const out = toParagraphs([seg(' First thought.', 0, 2), seg(' Second thought.', 5, 7)]); - expect(out).toBe('First thought.\n\nSecond thought.'); - }); - - it('does not break on a breath-length pause', () => { - const out = toParagraphs([seg(' Hang on,', 0, 1), seg(' let me think.', 1.5, 3)]); - expect(out).toBe('Hang on, let me think.'); - }); - - // The lone "." lines in the old response_format=text output came from segments like this. - it('attaches stranded punctuation without a space in front of it', () => { - const out = toParagraphs([seg(' See you tomorrow morning', 0, 2), seg(' .', 2.1, 2.2)]); - expect(out).toBe('See you tomorrow morning.'); - }); - - it('drops segments that are probably silence decoded with low confidence', () => { - const out = toParagraphs([ - seg(' Real speech here.', 0, 2), - seg(' Thanks for watching!', 2.1, 3, { no_speech_prob: 0.95, avg_logprob: -1.8 }), - seg(' More real speech.', 3.1, 5), - ]); - expect(out).toBe('Real speech here. More real speech.'); - }); - - it('keeps a confident segment even when it looks like silence', () => { - const out = toParagraphs([seg(' Definitely said.', 0, 2, { no_speech_prob: 0.95, avg_logprob: -0.2 })]); - expect(out).toBe('Definitely said.'); - }); - - it('keeps segments that report neither probability', () => { - const out = toParagraphs([seg(' No metadata at all.', 0, 2)]); - expect(out).toBe('No metadata at all.'); - }); - - it('measures the pause from the last kept segment, not the dropped one', () => { - const out = toParagraphs([ - seg(' Before.', 0, 2), - seg(' Hallucinated.', 2.1, 9, { no_speech_prob: 0.9, avg_logprob: -2 }), - seg(' After.', 9.1, 11), - ]); - // The gap that matters is 2 -> 9.1, so these are separate thoughts. - expect(out).toBe('Before.\n\nAfter.'); - }); - - it('collapses internal whitespace and ignores empty segments', () => { - const out = toParagraphs([seg(' Too many\n spaces. ', 0, 2), seg(' ', 2.1, 2.2)]); - expect(out).toBe('Too many spaces.'); - }); - - it('returns an empty string when every segment is filtered out', () => { - expect(toParagraphs([])).toBe(''); - expect(toParagraphs([seg(' ', 0, 1)])).toBe(''); - }); - - it('falls back to no break when segments carry no timings', () => { - const out = toParagraphs([{ text: ' One.' }, { text: ' Two.' }]); - expect(out).toBe('One. Two.'); - }); -}); diff --git a/src/servers/api/stt/transcribe.ts b/src/servers/api/stt/transcribe.ts deleted file mode 100644 index 09dd337c..00000000 --- a/src/servers/api/stt/transcribe.ts +++ /dev/null @@ -1,416 +0,0 @@ -// Whisper transcription, always in the audio's own language. -// -// Detect the language first, then transcribe in it. Whisper's translate mode is English-only, so it -// cannot honour "translate into " for anything else — rather than silently returning a -// rough English rendering of Portuguese audio, transcription is always faithful to the source. -// -// The detection must win when Whisper is confident. `spokenLanguages` only breaks ties on clips -// Whisper is genuinely unsure about, where its low-confidence guess defaults to English. -// -// The transcription pass is tuned for one person talking into a phone, which is what this is mostly -// used for. See the constants below: the defaults whisper.cpp ships are aimed at clean dictation and -// produce loops, clipped words and breath-length paragraph breaks on real voice memos. -// -// Both requests pass `timeout: false`. Bun's fetch aborts at 300s by default, and whisper.cpp runs at -// roughly 8x realtime, so anything over ~40 minutes of audio died with "The operation timed out" -// about 20 seconds short of the answer. AbortSignal.timeout() does not raise that ceiling — only -// disabling the built-in timeout does. - -// Bun's fetch accepts `timeout: false` to disable its built-in 300s ceiling. The DOM RequestInit -// type this project compiles against does not declare it, so widen it here rather than casting at -// each call site. -type BunRequestInit = RequestInit & { timeout?: boolean | number }; - -type TranscribeArgs = { - file: File | Blob; - whisperUrl: string; - spokenLanguages: string[]; -}; - -export type TranscribeResult = { - text: string; - detectedLanguage: string; -}; - -// Above this probability Whisper's own detection is taken as authoritative. Below it the clip is -// treated as ambiguous and the user's spoken languages break the tie. -const CONFIDENT_DETECTION = 0.5; - -// whisper.cpp's "language" field uses full names ("english"); ISO codes live in -// `language_probabilities`. Map covers what the user can pick in Profile settings. -const LANG_NAME_TO_ISO: Record = { - english: 'en', - portuguese: 'pt', - spanish: 'es', - french: 'fr', - german: 'de', - italian: 'it', - dutch: 'nl', - russian: 'ru', - chinese: 'zh', - japanese: 'ja', - korean: 'ko', -}; - -// Whisper reuses its own decoded output as the prompt for the next 30-second window, which is what -// makes it fall into loops ("I know. I know. I know.") once it slips. `max_context` is the budget for -// that carried text, and setting it to 0 stops the loops — but the same switch also disables the -// initial prompt, since whisper.cpp gates both on `n_max_text_ctx > 0`. -// -// The way to keep the prompt and still starve the loop: `carry_initial_prompt` puts the prompt in a -// static slot that is filled first and in full, and only the leftover budget goes to decoded text. -// Every prompt below is comfortably longer than this budget, so there is never any leftover — the -// vocabulary and style reach every window and none of the model's own output does. -const PROMPT_CONTEXT_TOKENS = 96; - -// Whisper imitates whatever it is primed with; it does not take instructions. So each prompt is a -// worked example of the output wanted — one person talking, punctuated the way it should come back — -// and nothing else. An earlier version opened by *describing* the format ("Commas separate clauses, -// and apostrophes are written properly") and that backfired: priming words which never occur in -// speech put them in play, and a character surname came back from a Monk episode as "Commas". -// Everything here is ordinary conversational language for that reason. -// -// Each is written in its own language, since an English prompt on Portuguese audio biases the -// output, and each deliberately runs past PROMPT_CONTEXT_TOKENS — see above for why the length is -// load-bearing. Overflow is trimmed from the front, so nothing depends on the opening words. -// -// Languages without an entry fall through to no prompt at all, which is handled explicitly below. -// -// Every prompt uses its language's full diacritic set (and, for a non-Latin script, its own -// punctuation — Japanese 。/、, Greek ; as a question mark, Arabic ، and ؟) in ordinary words, never -// forced in. Omitting one, e.g. writing Swedish without å ä ö, teaches Whisper to drop it too. -const TRANSCRIPT_PROMPTS: Record = { - en: - 'Hey, sorry for the late reply — I was stuck in a meeting all morning and only just got out. ' + - "I had a look at the document you sent over, and honestly, I think it's mostly fine. There are " + - "a couple of things I'd change in the second half, but nothing serious. The introduction reads " + - 'really well. Let me know when you have a few minutes and we can talk it through properly. ' + - "I'm around all afternoon, so whenever suits you is fine by me. Thanks again for putting it " + - 'together, I know it took a while.', - pt: - 'Olá, desculpa a demora a responder — estive numa reunião toda a manhã e só agora consegui ' + - 'sair. Já vi o documento que enviaste e, sinceramente, acho que está quase tudo bem. Há duas ' + - 'ou três coisas que mudava na segunda metade, mas nada de grave. A introdução está muito bem ' + - 'escrita. Diz-me quando tiveres uns minutos e falamos com calma. Estou por aqui toda a tarde, ' + - 'por isso quando te der jeito está bem para mim. Obrigado outra vez por teres tratado disto, ' + - 'sei que deu trabalho.', - es: - 'Hola, perdona la tardanza en responder — he estado toda la mañana en una reunión y recién ahora ' + - 'he podido salir. Ya vi el documento que me enviaste y, la verdad, creo que está bastante bien. ' + - 'Hay un par de cosas que cambiaría en la segunda mitad, pero nada grave. La introducción está muy ' + - 'bien escrita. Avísame cuando tengas unos minutos y lo hablamos con calma. Estoy disponible toda ' + - 'la tarde, así que cuando te venga bien está perfecto para mí. Gracias otra vez por armarlo, sé ' + - 'que llevó tiempo.', - de: - 'Hallo, entschuldige die späte Antwort — ich saß den ganzen Vormittag in einer Besprechung und ' + - 'bin gerade erst rausgekommen. Ich habe mir das Dokument angeschaut, das du geschickt hast, und ' + - 'ehrlich gesagt finde ich es größtenteils gut. In der zweiten Hälfte würde ich ein paar ' + - 'Kleinigkeiten ändern, aber nichts Ernstes. Die Einleitung liest sich wirklich gut. Sag mir ' + - 'Bescheid, wenn du später ein paar Minuten hast, dann können wir in Ruhe darüber reden. Ich bin ' + - 'den ganzen Nachmittag über da, also wann immer es dir passt, ist mir recht. Danke nochmal, dass ' + - 'du dir die Mühe gemacht hast, ich weiß, das hat Zeit gekostet.', - fr: - "Salut, désolé pour la réponse tardive — j'étais coincé en réunion toute la matinée et je viens " + - "tout juste d'en sortir. J'ai regardé le document que tu as envoyé, et honnêtement, je trouve que " + - "c'est plutôt bien. Il y a deux ou trois choses que je changerais dans la deuxième moitié, mais " + - "rien de grave. L'introduction est vraiment bien écrite. Dis-moi quand tu as quelques minutes et " + - "on en parle tranquillement. Je suis disponible tout l'après-midi, donc quand ça t'arrange, ça me " + - "va. Merci encore de l'avoir préparé, je sais que ça a pris du temps.", - sv: - 'Hej, förlåt att jag inte svarat tidigare — jag satt i möte hela förmiddagen och kom precis loss. ' + - 'Jag har tittat på dokumentet du skickade, och ärligt talat tycker jag att det mesta är bra. Det ' + - 'är några saker jag skulle ändra i andra halvan, men inget allvarligt. Inledningen är verkligen ' + - 'bra skriven. Säg till när du har några minuter så pratar vi igenom det ordentligt. Jag är ledig ' + - 'hela eftermiddagen, så när det passar dig passar det mig. Tack igen för att du satte ihop det, ' + - 'jag vet att det tog ett tag.', - it: - 'Ciao, scusa il ritardo nella risposta — sono stato in riunione tutta la mattina e sono appena ' + - "uscito. Ho dato un'occhiata al documento che mi hai mandato e, sinceramente, penso che vada bene " + - 'quasi tutto. Ci sono un paio di cose che cambierei nella seconda metà, ma niente di grave. ' + - "L'introduzione è scritta davvero bene. Fammi sapere quando hai qualche minuto così ne parliamo " + - 'con calma. Sono libero tutto il pomeriggio, quindi va bene quando ti fa comodo. Grazie ancora ' + - 'per averlo messo insieme, so che ci è voluto tempo.', - nl: - 'Hoi, sorry voor het late antwoord — ik zat de hele ochtend in een vergadering en ben er net pas ' + - 'uit. Ik heb naar het document gekeken dat je hebt gestuurd, en eerlijk gezegd vind ik het ' + - 'grotendeels goed. Er zijn een paar dingen die ik in de tweede helft zou veranderen, maar niets ' + - 'ergs. De inleiding leest heel prettig. Laat het me weten wanneer je een paar minuten hebt, dan ' + - 'bespreken we het rustig. Ik ben de hele middag beschikbaar, dus wanneer het jou uitkomt is voor ' + - 'mij prima. Bedankt nogmaals dat je het hebt samengesteld, ik weet dat het tijd heeft gekost.', - pl: - 'Cześć, przepraszam za późną odpowiedź — siedziałem cały ranek na spotkaniu i w końcu udało mi ' + - 'się z niego wyrwać. Zerknąłem na dokument, który przesłałeś, i szczerze mówiąc, myślę, że ' + - 'większość jest w porządku. Jest kilka rzeczy, które zmieniłbym w drugiej połowie, ale nic ' + - 'poważnego. Wstęp jest naprawdę dobrze napisany. Daj znać, kiedy będziesz mieć chwilę, możemy to ' + - 'spokojnie omówić. Jestem wolny całe popołudnie, więc kiedy ci pasuje, mi też pasuje. Dzięki ' + - 'jeszcze raz za to, że to zebrałeś, wiem, że zajęło to trochę czasu.', - da: - 'Hej, undskyld det sene svar — jeg sad i møde hele formiddagen og er lige kommet ud. Jeg har ' + - 'kigget på dokumentet, du sendte, og ærligt talt synes jeg det meste er fint. Der er et par ting, ' + - 'jeg ville ændre i anden halvdel, men ikke noget alvorligt. Indledningen er skrevet rigtig godt. ' + - 'Sig til, når du har et par minutter, så snakker vi ordentligt om det. Jeg er ledig hele ' + - 'eftermiddagen, så det passer mig, når det passer dig. Tak igen fordi du satte det sammen, jeg ' + - 'ved det tog tid.', - no: - 'Hei, beklager det sene svaret — jeg satt i møte hele formiddagen og er akkurat ferdig. Jeg har ' + - 'sett på dokumentet du sendte, og ærlig talt synes jeg det meste er bra. Det er et par ting jeg ' + - 'ville endret i andre halvdel, men ingenting alvorlig. Innledningen er skrevet veldig bra. Si ' + - 'ifra når du har noen minutter, så tar vi en skikkelig prat om det. Jeg er ledig hele ' + - 'ettermiddagen, så det som passer deg passer meg. Takk igjen for at du satte det sammen, jeg vet ' + - 'det tok tid.', - fi: - 'Hei, anteeksi myöhäinen vastaus — istuin koko aamupäivän palaverissa ja pääsin juuri äsken pois. ' + - 'Katsoin dokumentin, jonka lähetit, ja rehellisesti sanottuna suurin osa siitä on ihan hyvä. ' + - 'Loppupuoliskolla on pari asiaa, jotka muuttaisin, mutta ei mitään vakavaa. Johdanto on ' + - 'kirjoitettu tosi hyvin. Sano, kun sinulla on hetki aikaa, niin jutellaan siitä kunnolla. Olen ' + - 'vapaana koko iltapäivän, joten mikä tahansa sopii sinulle, sopii minulle. Kiitos vielä kerran, ' + - 'kun kokosit sen, tiedän että siihen meni aikaa.', - tr: - 'Selam, geç cevap verdiğim için kusura bakma — bütün sabah toplantıdaydım ve az önce çıkabildim. ' + - 'Gönderdiğin dokümana baktım ve açıkçası çoğunlukla iyi olduğunu düşünüyorum. İkinci yarıda ' + - 'değiştireceğim birkaç şey var ama ciddi bir şey değil. Giriş kısmı gerçekten iyi yazılmış. ' + - 'Birkaç dakikan olduğunda haber ver, düzgünce konuşuruz. Bütün öğleden sonra müsaitim, yani sana ' + - 'uyan zaman bana da uyar. Bunu hazırladığın için tekrar teşekkürler, biliyorum zaman almıştır.', - cs: - 'Ahoj, promiň za pozdní odpověď — celé dopoledne jsem byl na poradě a teprve teď jsem se dostal ' + - 'ven. Podíval jsem se na dokument, cos poslal, a upřímně řečeno, myslím, že většina je v pořádku. ' + - 'V druhé polovině bych pár věcí změnil, ale nic vážného. Úvod je napsaný fakt dobře. Dej vědět, ' + - 'až budeš mít pár minut, můžeme to pořádně probrat. Celé odpoledne mám volno, takže kdy se ti to ' + - 'bude hodit, mně to vyhovuje. Díky ještě jednou, žes to dal dohromady, vím, že to zabralo čas.', - el: - 'Γεια σου, συγγνώμη για την καθυστερημένη απάντηση — ήμουν σε συνάντηση όλο το πρωί και μόλις ' + - 'τώρα βγήκα. Είδα το έγγραφο που έστειλες και, ειλικρινά, νομίζω πως είναι σχεδόν όλο εντάξει. ' + - 'Υπάρχουν δύο τρία πράγματα που θα άλλαζα στο δεύτερο μισό, αλλά τίποτα σοβαρό. Η εισαγωγή είναι ' + - 'γραμμένη πολύ καλά. Έχεις λίγα λεπτά να τα πούμε με την ησυχία μας; Είμαι ελεύθερος όλο το ' + - 'απόγευμα, οπότε όποτε σε βολεύει εμένα με βολεύει. Ευχαριστώ ξανά που το ετοίμασες, ξέρω πως ' + - 'πήρε χρόνο.', - uk: - 'Привіт, вибач за пізню відповідь — я весь ранок сидів на нараді і щойно звідти вийшов. Подивився ' + - 'документ, який ти надіслав, і чесно кажучи, мої думки такі — майже все добре. У мене є пара ' + - 'речей, які я б змінив у другій половині, але нічого серйозного. Вступ написаний дуже добре. ' + - 'Напиши, коли матимеш кілька хвилин, і спокійно все обговоримо. Я вільний увесь день, тобто коли тобі зручно, ' + - 'мені теж підходить. Дякую ще раз, що зібрав це докупи, знаю, що це зайняло час.', - ru: - 'Привет, извини, что долго не отвечал — всё утро просидел на встрече и только что освободился. ' + - 'Посмотрел документ, который ты прислал, и, честно говоря, думаю, что почти всё нормально. Есть ' + - 'пара моментов, которые я бы изменил во второй половине, но ничего серьёзного. Введение написано ' + - 'правда хорошо. Напиши, когда будет пара свободных минут, спокойно всё обсудим. Я свободен весь ' + - 'день, так что когда тебе удобно, мне тоже подходит. Спасибо ещё раз, что всё это собрал, знаю, ' + - 'что это заняло время.', - ro: - 'Salut, scuze pentru răspunsul întârziat — am fost toată dimineața într-o ședință și abia acum am ' + - 'scăpat. M-am uitat pe documentul pe care mi l-ai trimis și, sincer, cred că e aproape totul ' + - 'bine. Sunt vreo două-trei lucruri pe care le-aș schimba în a doua jumătate, dar nimic grav. ' + - 'Introducerea e scrisă chiar bine. Spune-mi când ai câteva minute și vorbim pe îndelete despre ' + - 'asta. Sunt liber toată după-amiaza, deci oricând îți convine mie îmi convine. Mulțumesc încă o ' + - 'dată că ai pus totul cap la cap, știu că a durat.', - hu: - 'Szia, bocs a késői válaszért — egész délelőtt gyűlésen ültem, és csak most sikerült ' + - 'kiszabadulnom. Megnéztem a dokumentumot, amit küldtél, és őszintén szólva szerintem szinte ' + - 'minden rendben van. Van pár dolog, amit megváltoztatnék a második felében, de semmi komoly. A ' + - 'bevezető nagyon jól van megírva. Szólj, ha van pár szabad perced, és nyugodtan átbeszéljük. ' + - 'Egész délután érek rá, szóval amikor neked jó, nekem is megfelel. Köszönöm még egyszer, hogy ' + - 'összeraktad, tudom, hogy hosszú időbe telt.', - ja: - '返信が遅くなってごめん。朝からずっと会議で、今やっと抜けられたところ。送ってもらった資料、見たよ。' + - '正直、ほとんど問題ないと思う。後半のところで直したい部分が二つ三つあるけど、大したことじゃない。' + - '導入の部分はすごくよく書けてると思う。時間があるときに教えて、ゆっくり話そう。' + - '午後はずっと空いてるから、都合のいいときでいいよ。まとめてくれてありがとう、時間かかったよね。', - zh: - '抱歉这么晚才回你——我一早上都在开会，刚出来。你发的那份文件我看过了，说实话，大部分都挺好的。' + - '后半部分有几个地方我想改一下，但都不是什么大问题。开头写得真的很不错。你什么时候有空跟我说一声，我们好好聊聊。' + - '我整个下午都有空，所以你什么时候方便都行。再次谢谢你把这个整理出来，我知道挺花时间的。', - ko: - '답장이 늦어서 미안해. 오전 내내 회의하다가 방금 겨우 빠져나왔어. 보내준 문서 봤는데, 솔직히 거의 다 괜찮은 것 같아. 후반부에 고치고 싶은 부분이 두어 개 있긴 한데, ' + - '심각한 건 아니야. 서론은 진짜 잘 쓴 것 같아. 시간 될 때 얘기해줘, 그럼 차분히 얘기해보자. 오후 내내 시간 있으니까 편한 때 아무 때나 괜찮아. 정리해줘서 다시 한번 ' + - '고마워, 시간 좀 걸렸을 텐데.', - ar: - 'مرحبًا، آسف على تأخري في الرد — كنت في اجتماع طوال الصباح ولتوّي خرجت منه. اطّلعت على المستند ' + - 'الذي أرسلته، وبصراحة أعتقد أن معظمه جيد. هناك بعض الأشياء التي أود تغييرها في النصف الثاني، لكن ' + - 'لا شيء خطير. المقدمة مكتوبة بشكل جيد جدًا. هل يمكن أن تخبرني متى تكون لديك بضع دقائق لنتحدث ' + - 'بهدوء؟ أنا متفرغ طوال بعد الظهر، فأي وقت يناسبك يناسبني. شكرًا مرة أخرى على إعدادك لهذا، أعرف أن ' + - 'الأمر استغرق وقتًا.', - he: - 'היי, סליחה על התגובה המאוחרת — הייתי בישיבה כל הבוקר ורק עכשיו יצאתי. הסתכלתי על המסמך ששלחת, ' + - 'ובכנות, אני חושב שרוב הדברים בסדר. יש כמה דברים שהייתי משנה בחצי השני, אבל שום דבר רציני. הפתיחה ' + - 'כתובה ממש טוב. תגיד לי מתי יש לך כמה דקות ונדבר על זה בנחת. אני פנוי כל אחר הצהריים, אז מתי שנוח ' + - 'לך נוח גם לי. תודה שוב שסידרת את זה, אני יודע שזה לקח זמן.', - hi: - 'हाय, देर से जवाब देने के लिए माफ़ करना — मैं पूरी सुबह मीटिंग में था और अभी-अभी निकल पाया। तुमने ' + - 'जो डॉक्यूमेंट भेजा था, वो देख लिया, और सच कहूं तो ज़्यादातर हिस्सा ठीक लग रहा है। दूसरे हिस्से ' + - 'में कुछ चीज़ें हैं जो मैं बदलना चाहूंगा, पर कुछ गंभीर नहीं है। शुरुआत बहुत अच्छी लिखी गई है। जब ' + - 'भी कुछ मिनट मिलें बता देना, फिर आराम से बात करते हैं। पूरी दोपहर मैं फ्री हूं, तो जब भी तुम्हें ' + - 'सही लगे मुझे ठीक है। इसे तैयार करने के लिए फिर से शुक्रिया, पता है इसमें वक़्त लगा होगा।', -}; - -// Beam search rather than the default greedy decode (whisper.cpp ships `beam_size = -1`). It is the -// single biggest quality lever here and it costs time, which this flow has — nothing waits on it. -const BEAM_SIZE = 5; -const BEST_OF = 5; - -// The whisper container already runs silero VAD, but its defaults assume clean dictation. Widen the -// segments so they break at real pauses instead of at breaths, and pad the edges so word onsets are -// not clipped — clipped onsets are what shows up in the output as a word split down the middle. -const VAD_TUNING: Record = { - vad_speech_pad_ms: '200', // default 30 - vad_min_silence_duration_ms: '500', // default 100 — a breath is not a sentence boundary - vad_samples_overlap: '0.2', // default 0.1 - vad_max_speech_duration_s: '25', // default unbounded — keep a segment inside one 30s window -}; - -// A silence this long reads as a new thought. Whisper reports segment times against the original -// audio even when VAD is on (whisper.cpp maps them back through its VAD table), so these really are -// gaps in the recording rather than artefacts of the filtering. -const PARAGRAPH_GAP_SECONDS = 1.2; -const MAX_PARAGRAPH_CHARS = 900; - -// OpenAI Whisper's own silence rule: a segment the model thinks is probably not speech *and* decodes -// with low confidence is a hallucination rather than a transcript. whisper.cpp applies a subtly -// different form of this internally, so it is worth re-applying to the values it reports — this is -// what removes the stray words that appear over music and room tone. -const NO_SPEECH_PROB = 0.6; -const LOW_CONFIDENCE_LOGPROB = -1.0; - -export type WhisperSegment = { - text?: string; - start?: number; - end?: number; - no_speech_prob?: number; - avg_logprob?: number; -}; - -// Defaults are chosen so a segment missing either field is kept: dropping real speech is much worse -// than keeping a stray word. -const isHallucinatedSilence = (segment: WhisperSegment) => - (segment.no_speech_prob ?? 0) > NO_SPEECH_PROB && (segment.avg_logprob ?? 0) < LOW_CONFIDENCE_LOGPROB; - -// Whisper emits each segment with a leading space and now and then strands trailing punctuation in a -// segment of its own, which is why the old `response_format=text` output contained lone "." lines. -const appendSegment = (paragraph: string, text: string) => { - if (!paragraph) return text; - return /^[.,;:!?…)\]]/.test(text) ? paragraph + text : `${paragraph} ${text}`; -}; - -// Whisper returns one segment per utterance, not per paragraph. Rejoin them into prose and start a -// new paragraph on a long pause, so a voice memo reads as something written rather than as a list of -// fragments. -export function toParagraphs(segments: WhisperSegment[]): string { - const paragraphs: string[] = []; - let current = ''; - let previousEnd: number | null = null; - - for (const segment of segments) { - const text = (segment.text ?? '').replace(/\s+/g, ' ').trim(); - if (!text || isHallucinatedSilence(segment)) continue; - - const gap = previousEnd !== null && segment.start !== undefined ? segment.start - previousEnd : 0; - // Only break an over-long paragraph where a sentence actually ended, never mid-thought. - const overlong = current.length >= MAX_PARAGRAPH_CHARS && /[.!?…]["')\]]?$/.test(current); - - if (current && (gap >= PARAGRAPH_GAP_SECONDS || overlong)) { - paragraphs.push(current); - current = ''; - } - - current = appendSegment(current, text); - if (segment.end !== undefined) previousEnd = segment.end; - } - - if (current) paragraphs.push(current); - return paragraphs.join('\n\n'); -} - -export async function transcribeAudio({ - file, - whisperUrl, - spokenLanguages, -}: TranscribeArgs): Promise { - const base = whisperUrl.replace(/\/+$/, ''); - - // Step 1: detect language, and only fall back to the spoken-language bias when Whisper is unsure. - const detectForm = new FormData(); - detectForm.append('file', file); - detectForm.append('temperature', '0.0'); - detectForm.append('response_format', 'verbose_json'); - detectForm.append('detect_language', 'true'); - - const detectRes = await fetch(`${base}/inference`, { - method: 'POST', - body: detectForm, - timeout: false, - } as BunRequestInit); - if (!detectRes.ok) throw new Error(`Language detection failed (${detectRes.status})`); - const detectJson = (await detectRes.json()) as { - language?: string; - detected_language?: string; - detected_language_probability?: number; - language_probabilities?: Record; - }; - - const probs = detectJson.language_probabilities; - const ranked = probs ? Object.entries(probs).sort((a, b) => b[1] - a[1]) : []; - const [topCode, topProb] = ranked[0] ?? []; - - let detectedLanguage: string; - if (topCode && (topProb ?? 0) >= CONFIDENT_DETECTION) { - // Whisper is sure. Trust it even when it is not a language the user speaks — that is exactly the - // case translation exists for. - detectedLanguage = topCode; - } else if (ranked.length > 0 && spokenLanguages.length > 0) { - // Genuinely ambiguous (usually a short clip). Break the tie toward a language the user speaks, - // since Whisper's low-confidence guess defaults to English regardless of the audio. - const bestSpoken = spokenLanguages - .map((code) => [code, probs?.[code] ?? 0] as const) - .sort((a, b) => b[1] - a[1])[0]; - detectedLanguage = bestSpoken?.[0] ?? topCode ?? 'en'; - } else if (topCode) { - detectedLanguage = topCode; - } else { - // No probabilities field (non-whisper.cpp backend) — try to map the reported name. - const raw = (detectJson.detected_language ?? detectJson.language ?? 'en').toLowerCase(); - detectedLanguage = LANG_NAME_TO_ISO[raw] ?? raw; - } - - // Step 2: transcribe in the detected language - const transcribeForm = new FormData(); - transcribeForm.append('file', file); - transcribeForm.append('language', detectedLanguage); - transcribeForm.append('temperature', '0.0'); - transcribeForm.append('temperature_inc', '0.2'); - transcribeForm.append('beam_size', String(BEAM_SIZE)); - transcribeForm.append('best_of', String(BEST_OF)); - transcribeForm.append('suppress_nst', 'true'); - transcribeForm.append('response_format', 'verbose_json'); - // verbose_json re-runs a full language auto-detect unless this is set. Step 1 already answered - // that question, so it is pure duplicated work on every transcription. - transcribeForm.append('no_language_probabilities', 'true'); - // Segment timestamps drive the paragraphing below and are kept; per-token timestamps are extra - // work on every segment and nothing here reads them. - transcribeForm.append('token_timestamps', 'false'); - - const prompt = TRANSCRIPT_PROMPTS[detectedLanguage]; - if (prompt) { - transcribeForm.append('prompt', prompt); - transcribeForm.append('carry_initial_prompt', 'true'); - transcribeForm.append('max_context', String(PROMPT_CONTEXT_TOKENS)); - } else { - // Nothing worth protecting the budget for, so close the loop-feeding path outright. - transcribeForm.append('max_context', '0'); - } - - for (const [key, value] of Object.entries(VAD_TUNING)) transcribeForm.append(key, value); - - const res = await fetch(`${base}/inference`, { - method: 'POST', - body: transcribeForm, - timeout: false, - } as BunRequestInit); - if (!res.ok) throw new Error(`Transcription failed (${res.status})`); - - const json = (await res.json()) as { text?: string; segments?: WhisperSegment[] }; - // Fall back to the flat text for any backend that does not return segments. - const segments = json.segments ?? []; - const text = segments.length > 0 ? toParagraphs(segments) : (json.text ?? '').trim(); - - return { text, detectedLanguage }; -} diff --git a/src/servers/api/tasks/task-api-env.ts b/src/servers/api/tasks/task-api-env.ts index 363441e3..fb7f51b9 100644 --- a/src/servers/api/tasks/task-api-env.ts +++ b/src/servers/api/tasks/task-api-env.ts @@ -14,14 +14,15 @@ const PUBLIC_HOST = (() => { type TaskApiUser = { id: number; email: string; username?: string | null }; // Environment that lets a script task call Officer's own API instead of reimplementing server-side -// work (transcription, for one — the endpoint already handles language detection and caching). +// work (transcription, for one — `plugins/transcribe/tasks/transcribe-audio-file/index.ts` uses exactly +// this to reach the transcribe plugin's own route rather than talking to Whisper itself). // // Requests go to 127.0.0.1 so a task never depends on DNS or the reverse proxy being up. Origin // validation matches the Host header against PUBLIC_URL though, so a bare localhost request is // rejected — hence OFFICER_API_HOST, which scripts must pass as an explicit Host header: // // curl -H "Authorization: Bearer $OFFICER_AUTH_TOKEN" -H "Host: $OFFICER_API_HOST" \ -// "$OFFICER_API_URL/api/file-browser/transcribe" +// "$OFFICER_API_URL/api/transcribe/transcribe" export async function buildTaskApiEnv(user: TaskApiUser): Promise> { const token = await sign({ id: user.id, email: user.email, username: user.username ?? '' }, '12h'); return { diff --git a/src/servers/permissions/registry.ts b/src/servers/permissions/registry.ts index a58684e9..5ae186bf 100644 --- a/src/servers/permissions/registry.ts +++ b/src/servers/permissions/registry.ts @@ -5,9 +5,11 @@ // // The obvious model is "list the routes a role may call". It does not survive contact with this codebase. // A sweep of all 100 mutating platform routes on 2026-08-06 found reads permanently stuck on POST for two -// reasons that are not going away: bodies GET cannot carry (`/stt` multipart audio, `/tts`, `/ocr`, -// `/transcribe`), and credentials that must not sit in a query string where access logs, shell history and -// Referer headers capture them (`/tts/voices` apiKey, the four `/test` endpoints, `/local-providers/probe`). +// reasons that are not going away: bodies GET cannot carry (`/tts`, `/ocr` multipart audio and images — +// `/stt` and `/transcribe` were two more of these until 2026-08-24, when speech-to-text left for the +// `transcribe` plugin), and credentials that must not sit in a query string where access logs, shell +// history and Referer headers capture them (`/tts/voices` apiKey, the four `/test` endpoints, +// `/local-providers/probe`). // Five genuinely free conversions were done in e54d71d; the rest are staying. So the METHOD alone cannot // carry the read/write distinction — hence `readOnlyWrites` below, declared per permission. // diff --git a/src/workspaces/officerdev/src/apps/Chat/index.ts b/src/workspaces/officerdev/src/apps/Chat/index.ts index 1895f14c..8be87046 100644 --- a/src/workspaces/officerdev/src/apps/Chat/index.ts +++ b/src/workspaces/officerdev/src/apps/Chat/index.ts @@ -20,7 +20,6 @@ export { EmbeddableChat, type UseEmbeddableChatType } from './EmbeddableChat'; export { useChat, type UseChatType } from '../../hooks/useChat'; export { useSlashCommands } from './useSlashCommands'; export { useAttachments, type UseAttachmentsType } from './useAttachments'; -export { useAudioRecording, type UseAudioRecordingType } from './useAudioRecording'; export * from './types'; diff --git a/src/workspaces/officerdev/src/apps/Chat/useAudioRecording.test.tsx b/src/workspaces/officerdev/src/apps/Chat/useAudioRecording.test.tsx deleted file mode 100644 index ef12456b..00000000 --- a/src/workspaces/officerdev/src/apps/Chat/useAudioRecording.test.tsx +++ /dev/null @@ -1,83 +0,0 @@ -import { afterEach, describe, expect, test } from 'bun:test'; -import { act, cleanup, render } from '@testing-library/react'; -import { useAudioRecording } from './useAudioRecording'; - -/** - * The microphone is the one resource in this app the browser tells the user about: a recording indicator - * stays lit in the tab strip until every track is stopped. Nothing here released it on unmount, so closing - * a Chat panel mid-recording left the light on for the life of the tab with no way to switch it off. - * - * The stubs are the smallest thing that can answer "was the track stopped": a stream that counts stops and - * a recorder that reports its state. Neither the real MediaRecorder nor a real device is involved. - */ - -const stopped: string[] = []; - -const fakeStream = (label: string) => ({ - getTracks: () => [{ stop: () => stopped.push(label) }], -}); - -class FakeMediaRecorder { - state: 'recording' | 'inactive' = 'inactive'; - stream = fakeStream('track'); - mimeType = 'audio/webm'; - ondataavailable: ((ev: { data: Blob }) => void) | null = null; - onstop: (() => void) | null = null; - start() { - this.state = 'recording'; - } - stop() { - this.state = 'inactive'; - this.onstop?.(); - } -} - -const install = () => { - stopped.length = 0; - Object.assign(globalThis, { MediaRecorder: FakeMediaRecorder }); - Object.defineProperty(globalThis.navigator, 'mediaDevices', { - configurable: true, - value: { getUserMedia: async () => fakeStream('track') }, - }); -}; - -type Handle = { toggle: () => Promise; recording: boolean }; - -const handle: Handle = { toggle: async () => {}, recording: false }; - -const Probe = () => { - const runner = useAudioRecording(() => {}); - handle.toggle = runner.toggleRecording; - handle.recording = runner.recording; - return null; -}; - -afterEach(cleanup); - -describe('useAudioRecording', () => { - test('releases the microphone when its component goes away mid-recording', async () => { - install(); - const view = render(); - - await act(async () => { - await handle.toggle(); - }); - expect(handle.recording).toBe(true); - expect(stopped).toEqual([]); - - view.unmount(); - - // Not "the user pressed stop" — the panel was closed, swapped or remounted while still recording, and - // that is the case that used to leak. - expect(stopped).toEqual(['track']); - }); - - test('unmounting when nothing was ever recorded does nothing', () => { - install(); - const view = render(); - - view.unmount(); - - expect(stopped).toEqual([]); - }); -}); diff --git a/src/workspaces/officerdev/src/apps/Chat/useAudioRecording.ts b/src/workspaces/officerdev/src/apps/Chat/useAudioRecording.ts index 1602af9b..dbbf453f 100644 --- a/src/workspaces/officerdev/src/apps/Chat/useAudioRecording.ts +++ b/src/workspaces/officerdev/src/apps/Chat/useAudioRecording.ts @@ -1,114 +1,7 @@ -import { useState, useRef, useEffect } from 'react'; -import { toast } from 'sonner'; -import { useTranslation } from '../../locales'; - -export function useAudioRecording(onTranscription: (text: string) => void) { - const { t } = useTranslation(); - const [recording, setRecording] = useState(false); - const [transcribing, setTranscribing] = useState(false); - const mediaRecorderRef = useRef(null); - const chunksRef = useRef([]); - - const toggleRecording = async () => { - if (recording) { - const recorder = mediaRecorderRef.current; - if (!recorder) return; - - setRecording(false); - - try { - if (recorder.state === 'inactive') { - recorder.stream.getTracks().forEach((track) => track.stop()); - return; - } - - const blob = await new Promise((resolve, reject) => { - const timeout = setTimeout(() => reject(new Error('Recording stop timed out')), 5000); - recorder.onstop = () => { - clearTimeout(timeout); - resolve(new Blob(chunksRef.current, { type: recorder.mimeType })); - chunksRef.current = []; - }; - recorder.stop(); - }); - - recorder.stream.getTracks().forEach((track) => track.stop()); - - if (blob.size === 0) { - toast.error(t('chat.composer.noAudio')); - return; - } - - setTranscribing(true); - try { - const wav = await blobToWav(blob); - const formData = new FormData(); - formData.append('file', wav, 'recording.wav'); - formData.append('temperature', '0.0'); - formData.append('temperature_inc', '0.2'); - formData.append('response_format', 'json'); - - const token = localStorage.getItem('BEARER_TOKEN') ?? sessionStorage.getItem('BEARER_TOKEN'); - const res = await fetch('/api/chat/stt', { - method: 'POST', - body: formData, - headers: token ? { Authorization: `Bearer ${token}` } : {}, - }); - if (!res.ok) throw new Error(`Whisper returned ${res.status}`); - const json = await res.json(); - if (json.error) throw new Error(json.error); - const text = (json.text ?? '').trim(); - if (text) onTranscription(text); - } finally { - setTranscribing(false); - } - } catch (err) { - recorder.stream?.getTracks().forEach((track) => track.stop()); - chunksRef.current = []; - toast.error(err instanceof Error ? err.message : 'Recording failed'); - } - return; - } - - try { - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); - const recorder = new MediaRecorder(stream); - mediaRecorderRef.current = recorder; - chunksRef.current = []; - - recorder.ondataavailable = (ev) => { - if (ev.data.size > 0) chunksRef.current.push(ev.data); - }; - - recorder.start(250); - setRecording(true); - } catch (err) { - toast.error(err instanceof Error ? err.message : 'Could not access microphone'); - } - }; - - // Nothing here released the microphone on unmount, and a panel can go away mid-recording — closing the - // Chat panel, switching dashboards, a layout change that remounts the slot. The recorder is only ever - // stopped by the user pressing the button again, so without this the stream outlives the component and - // the browser keeps showing the recording indicator for the life of the tab. - useEffect( - () => () => { - const recorder = mediaRecorderRef.current; - if (!recorder) return; - if (recorder.state !== 'inactive') recorder.stop(); - recorder.stream?.getTracks().forEach((track) => track.stop()); - mediaRecorderRef.current = null; - chunksRef.current = []; - }, - [], - ); - - return { recording, transcribing, toggleRecording }; -} - -export type UseAudioRecordingType = ReturnType; - -// ── Helpers ── +// The chat composer's own STT-driven recording hook lived here until 2026-08-24, when speech-to-text +// left the platform for the `transcribe` plugin. It had no caller — the composer never mounted it — so +// it went rather than being ported. `blobToWav` stays: `DictateDialog.tsx` in the file browser uses it +// for something that was never STT, just recording a voice memo straight to a `.wav` upload. export const blobToWav = async (blob: Blob): Promise => { const ctx = new AudioContext(); diff --git a/src/workspaces/officerdev/src/apps/FileViewer/FileViewerContext.tsx b/src/workspaces/officerdev/src/apps/FileViewer/FileViewerContext.tsx index 65892954..e245b31d 100644 --- a/src/workspaces/officerdev/src/apps/FileViewer/FileViewerContext.tsx +++ b/src/workspaces/officerdev/src/apps/FileViewer/FileViewerContext.tsx @@ -12,7 +12,6 @@ export type FileViewerContextValue = { directContent: boolean; ttsLoading: boolean; ocrLoading: boolean; - transcribeLoading: boolean; extractAudioLoading: boolean; extractLoading: boolean; autoPlay: boolean; @@ -22,7 +21,6 @@ export type FileViewerContextValue = { setContent: (content: string) => void; handleReadAloud: () => void; handleOcr: () => void; - handleTranscribe: () => void; handleExtractAudio: () => void; handleExtract: () => void; handleDownload: () => void; diff --git a/src/workspaces/officerdev/src/apps/FileViewer/FileViewerHeader.tsx b/src/workspaces/officerdev/src/apps/FileViewer/FileViewerHeader.tsx index 2de7c55d..1deea4b9 100644 --- a/src/workspaces/officerdev/src/apps/FileViewer/FileViewerHeader.tsx +++ b/src/workspaces/officerdev/src/apps/FileViewer/FileViewerHeader.tsx @@ -7,7 +7,6 @@ import { FileType2, Volume2, ScanText, - FileText, AudioLines, FolderArchive, Save, @@ -28,12 +27,10 @@ export const FileViewerHeader = () => { directContent, ttsLoading, ocrLoading, - transcribeLoading, extractAudioLoading, extractLoading, handleReadAloud, handleOcr, - handleTranscribe, handleExtractAudio, handleExtract, handleDownload, @@ -98,31 +95,6 @@ export const FileViewerHeader = () => { {ocrLoading ? : } )} - {/* - TRANSCRIBE — hidden, not deleted, and the distinction is deliberate. - - Speech-to-text is leaving the platform for a plugin of its own. Until that lands this button calls - `POST /file-browser/transcribe`, which refuses on every server here because no STT endpoint is - configured anywhere — so it is a control that can only fail, and one of those teaches people to - distrust the controls beside it. - - Kept because the wiring is the useful part. When the plugin exists this becomes a call to it, and - the handler, the loading state and the sibling-markdown behaviour in FileViewerProvider are all - still correct; deleting would mean rediscovering them. - - It may not come back here at all. The other route to the same job is the file browser's context - menu, which a plugin reaches with a task — and that path needs no button in this header. - */} - {false && fileType === 'audio' && ( - - )} {fileType === 'video' && (