From e35ff5c7ad8b2ed5ab3a23cb35aa05110459e9c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Padez?= Date: Mon, 24 Aug 2026 03:29:58 +0000 Subject: [PATCH] stt leaves the platform for the transcribe plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Speech-to-text is now plugins/transcribe — sidecar, its own config, its own skill and a task for the file browser's context menu. The transcription logic moved BYTE FOR BYTE: 416 lines whose every decision has a reason attached, from the two-pass detect-then-transcribe to the 25 language priming prompts to timeout:false. Verified identical rather than assumed. Removed here: api/stt/, server-settings/stt.ts and its mount, POST /chat/stt, POST /file-browser/transcribe, the FileViewer handler behind the button hidden last release, the Settings screen, the dead useAudioRecording hook and the orphaned locale keys across all six locales. blobToWav stayed. It lived in the same file as the dead hook and is still used by DictateDialog — unrelated to STT, and deleting by file rather than by use is how that would have gone. No permission entry to remove: STT never had one, living under server-admin, chat and files. The registry count is unchanged and docs:check is green. The plugin ships a task that will not appear in the menu yet — the platform loads a plugin's skills, affordances and notifications from its own directory but not its tasks. That loader is the remaining piece of docs/plugin-capabilities.md and is deliberately not built here. 1426 pass, tsgo clean, 18 plugins discovered with none broken. Co-Authored-By: Claude Opus 5 Officer-Session: 656f5c85-e49a-4f1f-b9f3-042a04b4d043 --- .../Layout/Spotlight/useSpotlightItems.ts | 2 - .../Settings/ServerSettings/STTSection.tsx | 107 ----- .../Dashboard/Settings/SystemSettings.tsx | 10 +- src/apps/officer-web/locales/de-DE.json | 10 - src/apps/officer-web/locales/en-GB.json | 10 - src/apps/officer-web/locales/es-ES.json | 10 - src/apps/officer-web/locales/fr-FR.json | 10 - src/apps/officer-web/locales/pt-PT.json | 10 - src/apps/officer-web/locales/sv-SE.json | 10 - src/servers/api/chat/chat.ts | 29 -- src/servers/api/file-browser/router.ts | 42 -- .../api/server-settings/server-settings.ts | 2 - src/servers/api/server-settings/stt.ts | 48 -- src/servers/api/stt/transcribe.test.ts | 78 ---- src/servers/api/stt/transcribe.ts | 416 ------------------ src/servers/api/tasks/task-api-env.ts | 5 +- src/servers/permissions/registry.ts | 8 +- .../officerdev/src/apps/Chat/index.ts | 1 - .../src/apps/Chat/useAudioRecording.test.tsx | 83 ---- .../src/apps/Chat/useAudioRecording.ts | 115 +---- .../src/apps/FileViewer/FileViewerContext.tsx | 2 - .../src/apps/FileViewer/FileViewerHeader.tsx | 28 -- .../apps/FileViewer/FileViewerProvider.tsx | 16 - .../officerdev/src/hooks/useFilesAPI.ts | 7 - src/workspaces/officerdev/src/index.ts | 3 +- .../officerdev/src/locales/de-DE.json | 5 +- .../officerdev/src/locales/en-GB.json | 5 +- .../officerdev/src/locales/es-ES.json | 5 +- .../officerdev/src/locales/fr-FR.json | 5 +- .../officerdev/src/locales/pt-PT.json | 5 +- .../officerdev/src/locales/sv-SE.json | 5 +- 31 files changed, 20 insertions(+), 1072 deletions(-) delete mode 100644 src/apps/officer-web/Screens/Dashboard/Settings/ServerSettings/STTSection.tsx delete mode 100644 src/servers/api/server-settings/stt.ts delete mode 100644 src/servers/api/stt/transcribe.test.ts delete mode 100644 src/servers/api/stt/transcribe.ts delete mode 100644 src/workspaces/officerdev/src/apps/Chat/useAudioRecording.test.tsx diff --git a/src/apps/officer-web/Screens/Dashboard/Layout/Spotlight/useSpotlightItems.ts b/src/apps/officer-web/Screens/Dashboard/Layout/Spotlight/useSpotlightItems.ts index 20c53368..2b73e324 100644 --- a/src/apps/officer-web/Screens/Dashboard/Layout/Spotlight/useSpotlightItems.ts +++ b/src/apps/officer-web/Screens/Dashboard/Layout/Spotlight/useSpotlightItems.ts @@ -9,7 +9,6 @@ import { LayoutGrid, Lock, Mail, - Mic, ScanText, ShieldCheck, Sparkles, @@ -156,7 +155,6 @@ const SETTINGS_ITEMS: SettingsItem[] = [ }, // System — SystemSettings.tsx { id: 'set:tts', labelKey: 'settings.sections.tts.label', to: '/settings/system/tts', icon: Volume2 }, - { id: 'set:stt', labelKey: 'settings.sections.stt.label', to: '/settings/system/stt', icon: Mic }, { id: 'set:ocr', labelKey: 'settings.sections.ocr.label', to: '/settings/system/ocr', icon: ScanText }, { id: 'set:maintenance', diff --git a/src/apps/officer-web/Screens/Dashboard/Settings/ServerSettings/STTSection.tsx b/src/apps/officer-web/Screens/Dashboard/Settings/ServerSettings/STTSection.tsx deleted file mode 100644 index 511d0de8..00000000 --- a/src/apps/officer-web/Screens/Dashboard/Settings/ServerSettings/STTSection.tsx +++ /dev/null @@ -1,107 +0,0 @@ -import type { FormEvent } from 'react'; -import { useState, useEffect } from 'react'; -import { useQuery, useQueryClient } from '@tanstack/react-query'; -import { toast } from 'sonner'; -import { Button } from '@/components/ui/button'; -import { Input } from '@/components/ui/input'; -import { Label } from '@/components/ui/label'; -import { useClient } from 'hooks/useClient'; -import { useTranslation } from '@/lib/i18n'; - -type SttConfig = { - url: string; -}; - -export const STTSection = () => { - const { t } = useTranslation(); - const client = useClient(); - const queryClient = useQueryClient(); - - const { data: config, isLoading } = useQuery({ - queryKey: ['STT_CONFIG'], - queryFn: () => client.get('/server-settings/stt'), - }); - - const [url, setUrl] = useState('http://localhost:64201'); - const [isSaving, setIsSaving] = useState(false); - const [isTesting, setIsTesting] = useState(false); - - useEffect(() => { - if (!config) return; - setUrl(config.url); - }, [config]); - - const handleSave = async (ev?: FormEvent) => { - ev?.preventDefault(); - if (isSaving) return; - setIsSaving(true); - try { - await client.put('/server-settings/stt', { url }); - queryClient.invalidateQueries({ queryKey: ['STT_CONFIG'] }); - toast.success(t('settings.server.stt.saved')); - } catch { - toast.error(t('settings.server.stt.saveFailed')); - } finally { - setIsSaving(false); - } - }; - - const handleTest = async () => { - if (isTesting) return; - setIsTesting(true); - try { - const res = await client.post<{ success?: boolean; error?: string }>('/server-settings/stt/test', { url }); - if (res.error) { - toast.error(res.error); - } else { - toast.success(t('settings.server.stt.reachable')); - } - } catch (err: unknown) { - const raw = (err as { message?: string })?.message; - let msg = 'Connection failed'; - try { - if (raw) msg = JSON.parse(raw).error ?? msg; - } catch { - /* ignore */ - } - toast.error(msg); - } finally { - setIsTesting(false); - } - }; - - if (isLoading) return

{t('settings.server.loading')}

; - - return ( -
- - -
- - -
-
- ); -}; diff --git a/src/apps/officer-web/Screens/Dashboard/Settings/SystemSettings.tsx b/src/apps/officer-web/Screens/Dashboard/Settings/SystemSettings.tsx index d6a249b4..2a1e4a13 100644 --- a/src/apps/officer-web/Screens/Dashboard/Settings/SystemSettings.tsx +++ b/src/apps/officer-web/Screens/Dashboard/Settings/SystemSettings.tsx @@ -1,5 +1,5 @@ import { useMemo } from 'react'; -import { Bot, Hammer, Mail, Mic, ScanText, Settings, Volume2 } from 'lucide-react'; +import { Bot, Hammer, Mail, ScanText, Settings, Volume2 } from 'lucide-react'; import type { LayoutNode, PanelComponents } from 'officerdev'; import { WorkspaceLayout } from 'officerdev'; @@ -7,7 +7,6 @@ import { SettingsRoute, createSettingsPanelComponents, type SettingsSectionGroup import { SMTPSection } from './ServerSettings/SMTPSection'; import { MaintenanceSection } from './ServerSettings/MaintenanceSection'; import { TTSSection } from './ServerSettings/TTSSection'; -import { STTSection } from './ServerSettings/STTSection'; import { OCRSection } from './ServerSettings/OCRSection'; const groups: SettingsSectionGroup[] = [ @@ -22,13 +21,6 @@ const groups: SettingsSectionGroup[] = [ descriptionKey: 'settings.sections.tts.description', content: , }, - { - key: 'stt', - icon: Mic, - titleKey: 'settings.sections.stt.label', - descriptionKey: 'settings.sections.stt.description', - content: , - }, { key: 'ocr', icon: ScanText, diff --git a/src/apps/officer-web/locales/de-DE.json b/src/apps/officer-web/locales/de-DE.json index 502b9119..3a8197b8 100644 --- a/src/apps/officer-web/locales/de-DE.json +++ b/src/apps/officer-web/locales/de-DE.json @@ -145,10 +145,6 @@ "label": "Text-zu-Sprache", "description": "TTS-Anbieter und Stimme" }, - "stt": { - "label": "Sprache-zu-Text", - "description": "Whisper-Server-URL" - }, "ocr": { "label": "OCR", "description": "Vision-Modell für Textextraktion" @@ -234,12 +230,6 @@ "saveFailed": "OCR-Einstellungen konnten nicht gespeichert werden", "reachable": "Vision-Modell-Server ist erreichbar" }, - "stt": { - "serverUrl": "Whisper-Server-URL", - "saved": "STT-Einstellungen gespeichert", - "saveFailed": "STT-Einstellungen konnten nicht gespeichert werden", - "reachable": "Whisper-Server ist erreichbar" - }, "tts": { "openaiCompatible": "OpenAI-kompatibel (Kokoro, OpenAI usw.)", "elevenLabs": "ElevenLabs", diff --git a/src/apps/officer-web/locales/en-GB.json b/src/apps/officer-web/locales/en-GB.json index f6d1e724..9efa42df 100644 --- a/src/apps/officer-web/locales/en-GB.json +++ b/src/apps/officer-web/locales/en-GB.json @@ -145,10 +145,6 @@ "label": "Text to Speech", "description": "TTS provider and voice" }, - "stt": { - "label": "Speech to Text", - "description": "Whisper server URL" - }, "ocr": { "label": "OCR", "description": "Vision model for text extraction" @@ -234,12 +230,6 @@ "saveFailed": "Failed to save OCR settings", "reachable": "Vision model server is reachable" }, - "stt": { - "serverUrl": "Whisper Server URL", - "saved": "STT settings saved", - "saveFailed": "Failed to save STT settings", - "reachable": "Whisper server is reachable" - }, "tts": { "openaiCompatible": "OpenAI-compatible (Kokoro, OpenAI, etc.)", "elevenLabs": "ElevenLabs", diff --git a/src/apps/officer-web/locales/es-ES.json b/src/apps/officer-web/locales/es-ES.json index 407d35c1..c0662222 100644 --- a/src/apps/officer-web/locales/es-ES.json +++ b/src/apps/officer-web/locales/es-ES.json @@ -145,10 +145,6 @@ "label": "Texto a voz", "description": "Proveedor de TTS y voz" }, - "stt": { - "label": "Voz a texto", - "description": "URL del servidor Whisper" - }, "ocr": { "label": "OCR", "description": "Modelo de visión para extracción de texto" @@ -234,12 +230,6 @@ "saveFailed": "No se pudieron guardar los ajustes de OCR", "reachable": "El servidor del modelo de visión está accesible" }, - "stt": { - "serverUrl": "URL del servidor Whisper", - "saved": "Ajustes de STT guardados", - "saveFailed": "No se pudieron guardar los ajustes de STT", - "reachable": "El servidor Whisper está accesible" - }, "tts": { "openaiCompatible": "Compatible con OpenAI (Kokoro, OpenAI, etc.)", "elevenLabs": "ElevenLabs", diff --git a/src/apps/officer-web/locales/fr-FR.json b/src/apps/officer-web/locales/fr-FR.json index 700adf39..c8a9b531 100644 --- a/src/apps/officer-web/locales/fr-FR.json +++ b/src/apps/officer-web/locales/fr-FR.json @@ -145,10 +145,6 @@ "label": "Synthèse vocale", "description": "Fournisseur et voix pour la synthèse vocale" }, - "stt": { - "label": "Reconnaissance vocale", - "description": "URL du serveur Whisper" - }, "ocr": { "label": "OCR", "description": "Modèle de vision pour l'extraction de texte" @@ -234,12 +230,6 @@ "saveFailed": "Échec de l'enregistrement des paramètres OCR", "reachable": "Le serveur du modèle de vision est accessible" }, - "stt": { - "serverUrl": "URL du serveur Whisper", - "saved": "Paramètres de reconnaissance vocale enregistrés", - "saveFailed": "Échec de l'enregistrement des paramètres de reconnaissance vocale", - "reachable": "Le serveur Whisper est accessible" - }, "tts": { "openaiCompatible": "Compatible OpenAI (Kokoro, OpenAI, etc.)", "elevenLabs": "ElevenLabs", diff --git a/src/apps/officer-web/locales/pt-PT.json b/src/apps/officer-web/locales/pt-PT.json index 53027a41..a1d147d1 100644 --- a/src/apps/officer-web/locales/pt-PT.json +++ b/src/apps/officer-web/locales/pt-PT.json @@ -145,10 +145,6 @@ "label": "Texto para fala", "description": "Fornecedor de TTS e voz" }, - "stt": { - "label": "Fala para texto", - "description": "URL do servidor Whisper" - }, "ocr": { "label": "OCR", "description": "Modelo de visão para extração de texto" @@ -234,12 +230,6 @@ "saveFailed": "Falha ao guardar as definições de OCR", "reachable": "O servidor do modelo de visão está acessível" }, - "stt": { - "serverUrl": "URL do servidor Whisper", - "saved": "Definições de STT guardadas", - "saveFailed": "Falha ao guardar as definições de STT", - "reachable": "O servidor Whisper está acessível" - }, "tts": { "openaiCompatible": "Compatível com OpenAI (Kokoro, OpenAI, etc.)", "elevenLabs": "ElevenLabs", diff --git a/src/apps/officer-web/locales/sv-SE.json b/src/apps/officer-web/locales/sv-SE.json index b11662d2..63908e5a 100644 --- a/src/apps/officer-web/locales/sv-SE.json +++ b/src/apps/officer-web/locales/sv-SE.json @@ -145,10 +145,6 @@ "label": "Text till tal", "description": "TTS-leverantör och röst" }, - "stt": { - "label": "Tal till text", - "description": "Whisper-serverns URL" - }, "ocr": { "label": "OCR", "description": "Synmodell för textextrahering" @@ -234,12 +230,6 @@ "saveFailed": "Det gick inte att spara OCR-inställningarna", "reachable": "Synmodellsservern är nåbar" }, - "stt": { - "serverUrl": "URL till Whisper-server", - "saved": "STT-inställningarna sparade", - "saveFailed": "Det gick inte att spara STT-inställningarna", - "reachable": "Whisper-servern är nåbar" - }, "tts": { "openaiCompatible": "OpenAI-kompatibel (Kokoro, OpenAI, m.fl.)", "elevenLabs": "ElevenLabs", diff --git a/src/servers/api/chat/chat.ts b/src/servers/api/chat/chat.ts index 6d4f3401..f04a262d 100644 --- a/src/servers/api/chat/chat.ts +++ b/src/servers/api/chat/chat.ts @@ -5,7 +5,6 @@ import { createRouter } from '../../create-router'; import * as errors from '@@/custom-errors'; import * as sidecar from '@@/sidecar-registry'; import { - getUserSettings, getChatSessionModel, getChatSessionModels, listCommandHqSessionIds, @@ -45,7 +44,6 @@ import type { ChatIdentity } from './claude-sessions'; import { officerModelForApiModel } from './model-aliases'; import { readAccountEmail } from './claude-usage'; import { listStoredUsage, mergeUsage, snapshotClaudeUsage } from './usage-store'; -import { readSttConfig } from '../server-settings/stt'; /** * Whose transcripts a request may read. @@ -81,7 +79,6 @@ export async function chatIdentity(user: { id: number; email: string }): Promise // identity the way `spawnClaudeAsMember` does. Restrict here rather than at the permission layer because // `chat` is one permission covering both harnesses, and splitting it would strand the grants already issued. // The matching refusal on the execution path is in `websocket.ts` → `handleChat`. -import { transcribeAudio } from '../stt/transcribe'; import { registerAgentPanelRoutes } from './agent-panels-routes'; export const chatRouter = createRouter(); @@ -629,32 +626,6 @@ chatRouter.get('/models', async (ctx: Context) => { } }); -// POST /chat/stt — proxy an audio clip to the configured Whisper server. -chatRouter.post('/stt', async (ctx: Context) => { - const sttConfig = await readSttConfig(); - if (!sttConfig?.url) { - return ctx.json({ error: 'Whisper not configured — set it up in Settings → Speech to Text' }, 400); - } - - const user = ctx.get('user'); - const body = await ctx.req.parseBody(); - const file = body['file']; - if (!file || !(file instanceof File)) { - return ctx.json({ error: 'file is required' }, 400); - } - - const settings = (await getUserSettings(user.id)) as { languages?: { spoken?: string[] } }; - const spokenLanguages = settings.languages?.spoken ?? []; - - try { - const result = await transcribeAudio({ file, whisperUrl: sttConfig.url, spokenLanguages }); - return ctx.json(result); - } catch (err) { - logger.error('STT proxy failed', { error: String(err) }); - return ctx.json({ error: err instanceof Error ? err.message : 'Failed to reach Whisper server' }, 502); - } -}); - // The agent address book — naming a chat panel so other agents on the same dashboard can reach it. // Lives on this router because it is the same authority `chat` already grants: creating and naming // Claude sessions. See servers/api/chat/agent-panels-routes.ts and docs/agent-coordination.md. diff --git a/src/servers/api/file-browser/router.ts b/src/servers/api/file-browser/router.ts index 75fef4b5..04c70797 100644 --- a/src/servers/api/file-browser/router.ts +++ b/src/servers/api/file-browser/router.ts @@ -6,9 +6,7 @@ import { getOwnerHomeDir, DATA_PATH } from '@@/data-path'; import { resolveHomeDir } from '@@/user-home'; import * as errors from '@@/custom-errors'; import { readTtsConfig } from '@@/api/server-settings/tts'; -import { readSttConfig } from '@@/api/server-settings/stt'; import { readOcrConfig } from '@@/api/server-settings/ocr'; -import { transcribeAudio } from '@@/api/stt/transcribe'; import { getUserSettings } from 'officerdb'; async function getUserTtsVoice(userId: number): Promise { @@ -1104,46 +1102,6 @@ router.post('/extract', async (ctx) => { return ctx.json({ extractedPath }); }); -// Transcribe audio via Whisper with caching -// Workflow: detect language → check user's spoken languages → translate if needed → transcribe -router.post('/transcribe', async (ctx) => { - const user = ctx.get('user'); - const { path: filePath, root, saveNextTo } = ctx.get('body') as { path: string; root?: string; saveNextTo?: boolean }; - if (!filePath) throw errors.BAD_REQUEST('path is required'); - - const rootDir = getRootDir(user, root); - const absPath = resolveUserPath(rootDir, filePath); - const s = await stat(absPath); - if (s.isDirectory()) throw errors.BAD_REQUEST('Cannot transcribe a directory'); - - // No caching: a transcription is always produced fresh and overwrites any previous one. - const { dir, name } = parsePath(filePath.replace(/^\/+/, '')); - - const sttConfig = await readSttConfig(); - const whisperUrl = sttConfig?.url; - if (!whisperUrl) throw errors.BAD_REQUEST('Whisper not configured — set it up in Settings → Speech to Text'); - - const settings = await getUserSettings(user.id); - const spokenLanguages = (settings.languages as { spoken?: string[] })?.spoken ?? []; - - let text: string; - try { - const result = await transcribeAudio({ file: Bun.file(absPath), whisperUrl, spokenLanguages }); - text = result.text; - } catch (err) { - throw errors.BAD_REQUEST(err instanceof Error ? err.message : 'Transcription failed'); - } - - if (saveNextTo) { - const siblingRel = dir ? `${dir}/${name}.md` : `${name}.md`; - const siblingAbs = resolveUserPath(rootDir, siblingRel); - await Bun.write(siblingAbs, text); - return ctx.json({ transcriptionPath: `/${siblingRel}`, transcriptionRoot: root ?? 'home', cached: false }); - } - - return ctx.json({ text, cached: false }); -}); - // Search files by name router.get('/search', async (ctx) => { const user = ctx.get('user'); diff --git a/src/servers/api/server-settings/server-settings.ts b/src/servers/api/server-settings/server-settings.ts index 224a5607..f1132b39 100644 --- a/src/servers/api/server-settings/server-settings.ts +++ b/src/servers/api/server-settings/server-settings.ts @@ -9,7 +9,6 @@ import { readServerSettings, writeServerSettings } from 'officerdb'; import { chatProvidersRouter } from './chat-providers'; import { smtpRouter } from './smtp'; import { ttsRouter } from './tts'; -import { sttRouter } from './stt'; import { ocrRouter } from './ocr'; export const serverSettingsRouter = createRouter(); @@ -17,7 +16,6 @@ export const serverSettingsRouter = createRouter(); serverSettingsRouter.route('/chat-providers', chatProvidersRouter); serverSettingsRouter.route('/smtp', smtpRouter); serverSettingsRouter.route('/tts', ttsRouter); -serverSettingsRouter.route('/stt', sttRouter); serverSettingsRouter.route('/ocr', ocrRouter); export { readServerSettings as readSettings }; diff --git a/src/servers/api/server-settings/stt.ts b/src/servers/api/server-settings/stt.ts deleted file mode 100644 index 1d8b9910..00000000 --- a/src/servers/api/server-settings/stt.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { createRouter } from '../../create-router'; -import { readServerSettings, writeServerSettings } from 'officerdb'; - -type SttConfig = { - url: string; -}; - -export async function readSttConfig(): Promise { - const settings = await readServerSettings(); - return settings.stt as SttConfig | undefined; -} - -export const sttRouter = createRouter(); - -sttRouter.get('/', async (ctx) => { - const stt = await readSttConfig(); - if (!stt) return ctx.json(null); - return ctx.json(stt); -}); - -sttRouter.put('/', async (ctx) => { - const body = await ctx.req.json(); - const settings = await readServerSettings(); - settings.stt = body; - await writeServerSettings(settings); - return ctx.json({ success: true }); -}); - -sttRouter.post('/test', async (ctx) => { - const body = await ctx.req.json<{ url: string }>(); - if (!body.url) return ctx.json({ error: 'URL required' }, 400); - - try { - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), 5000); - const res = await fetch(`${body.url.replace(/\/+$/, '')}/inference`, { - method: 'POST', - body: new FormData(), - signal: controller.signal, - }); - clearTimeout(timeout); - // Whisper will return an error for empty form, but a response means it's reachable - return ctx.json({ success: true, status: res.status }); - } catch (err) { - const message = err instanceof Error ? err.message : 'Unknown error'; - return ctx.json({ error: message }, 500); - } -}); diff --git a/src/servers/api/stt/transcribe.test.ts b/src/servers/api/stt/transcribe.test.ts deleted file mode 100644 index 6bd6d00a..00000000 --- a/src/servers/api/stt/transcribe.test.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { describe, expect, it } from 'bun:test'; -import { toParagraphs } from './transcribe'; -import type { WhisperSegment } from './transcribe'; - -// Whisper's own shape: leading space on every segment, times in seconds. -const seg = (text: string, start: number, end: number, extra: Partial = {}): WhisperSegment => ({ - text, - start, - end, - ...extra, -}); - -describe('toParagraphs', () => { - it('joins segments spoken without a real pause into one paragraph', () => { - const out = toParagraphs([seg(' I went to the shop.', 0, 2), seg(' It was closed.', 2.2, 4)]); - expect(out).toBe('I went to the shop. It was closed.'); - }); - - it('starts a new paragraph after a long silence', () => { - const out = toParagraphs([seg(' First thought.', 0, 2), seg(' Second thought.', 5, 7)]); - expect(out).toBe('First thought.\n\nSecond thought.'); - }); - - it('does not break on a breath-length pause', () => { - const out = toParagraphs([seg(' Hang on,', 0, 1), seg(' let me think.', 1.5, 3)]); - expect(out).toBe('Hang on, let me think.'); - }); - - // The lone "." lines in the old response_format=text output came from segments like this. - it('attaches stranded punctuation without a space in front of it', () => { - const out = toParagraphs([seg(' See you tomorrow morning', 0, 2), seg(' .', 2.1, 2.2)]); - expect(out).toBe('See you tomorrow morning.'); - }); - - it('drops segments that are probably silence decoded with low confidence', () => { - const out = toParagraphs([ - seg(' Real speech here.', 0, 2), - seg(' Thanks for watching!', 2.1, 3, { no_speech_prob: 0.95, avg_logprob: -1.8 }), - seg(' More real speech.', 3.1, 5), - ]); - expect(out).toBe('Real speech here. More real speech.'); - }); - - it('keeps a confident segment even when it looks like silence', () => { - const out = toParagraphs([seg(' Definitely said.', 0, 2, { no_speech_prob: 0.95, avg_logprob: -0.2 })]); - expect(out).toBe('Definitely said.'); - }); - - it('keeps segments that report neither probability', () => { - const out = toParagraphs([seg(' No metadata at all.', 0, 2)]); - expect(out).toBe('No metadata at all.'); - }); - - it('measures the pause from the last kept segment, not the dropped one', () => { - const out = toParagraphs([ - seg(' Before.', 0, 2), - seg(' Hallucinated.', 2.1, 9, { no_speech_prob: 0.9, avg_logprob: -2 }), - seg(' After.', 9.1, 11), - ]); - // The gap that matters is 2 -> 9.1, so these are separate thoughts. - expect(out).toBe('Before.\n\nAfter.'); - }); - - it('collapses internal whitespace and ignores empty segments', () => { - const out = toParagraphs([seg(' Too many\n spaces. ', 0, 2), seg(' ', 2.1, 2.2)]); - expect(out).toBe('Too many spaces.'); - }); - - it('returns an empty string when every segment is filtered out', () => { - expect(toParagraphs([])).toBe(''); - expect(toParagraphs([seg(' ', 0, 1)])).toBe(''); - }); - - it('falls back to no break when segments carry no timings', () => { - const out = toParagraphs([{ text: ' One.' }, { text: ' Two.' }]); - expect(out).toBe('One. Two.'); - }); -}); diff --git a/src/servers/api/stt/transcribe.ts b/src/servers/api/stt/transcribe.ts deleted file mode 100644 index 09dd337c..00000000 --- a/src/servers/api/stt/transcribe.ts +++ /dev/null @@ -1,416 +0,0 @@ -// Whisper transcription, always in the audio's own language. -// -// Detect the language first, then transcribe in it. Whisper's translate mode is English-only, so it -// cannot honour "translate into " for anything else — rather than silently returning a -// rough English rendering of Portuguese audio, transcription is always faithful to the source. -// -// The detection must win when Whisper is confident. `spokenLanguages` only breaks ties on clips -// Whisper is genuinely unsure about, where its low-confidence guess defaults to English. -// -// The transcription pass is tuned for one person talking into a phone, which is what this is mostly -// used for. See the constants below: the defaults whisper.cpp ships are aimed at clean dictation and -// produce loops, clipped words and breath-length paragraph breaks on real voice memos. -// -// Both requests pass `timeout: false`. Bun's fetch aborts at 300s by default, and whisper.cpp runs at -// roughly 8x realtime, so anything over ~40 minutes of audio died with "The operation timed out" -// about 20 seconds short of the answer. AbortSignal.timeout() does not raise that ceiling — only -// disabling the built-in timeout does. - -// Bun's fetch accepts `timeout: false` to disable its built-in 300s ceiling. The DOM RequestInit -// type this project compiles against does not declare it, so widen it here rather than casting at -// each call site. -type BunRequestInit = RequestInit & { timeout?: boolean | number }; - -type TranscribeArgs = { - file: File | Blob; - whisperUrl: string; - spokenLanguages: string[]; -}; - -export type TranscribeResult = { - text: string; - detectedLanguage: string; -}; - -// Above this probability Whisper's own detection is taken as authoritative. Below it the clip is -// treated as ambiguous and the user's spoken languages break the tie. -const CONFIDENT_DETECTION = 0.5; - -// whisper.cpp's "language" field uses full names ("english"); ISO codes live in -// `language_probabilities`. Map covers what the user can pick in Profile settings. -const LANG_NAME_TO_ISO: Record = { - english: 'en', - portuguese: 'pt', - spanish: 'es', - french: 'fr', - german: 'de', - italian: 'it', - dutch: 'nl', - russian: 'ru', - chinese: 'zh', - japanese: 'ja', - korean: 'ko', -}; - -// Whisper reuses its own decoded output as the prompt for the next 30-second window, which is what -// makes it fall into loops ("I know. I know. I know.") once it slips. `max_context` is the budget for -// that carried text, and setting it to 0 stops the loops — but the same switch also disables the -// initial prompt, since whisper.cpp gates both on `n_max_text_ctx > 0`. -// -// The way to keep the prompt and still starve the loop: `carry_initial_prompt` puts the prompt in a -// static slot that is filled first and in full, and only the leftover budget goes to decoded text. -// Every prompt below is comfortably longer than this budget, so there is never any leftover — the -// vocabulary and style reach every window and none of the model's own output does. -const PROMPT_CONTEXT_TOKENS = 96; - -// Whisper imitates whatever it is primed with; it does not take instructions. So each prompt is a -// worked example of the output wanted — one person talking, punctuated the way it should come back — -// and nothing else. An earlier version opened by *describing* the format ("Commas separate clauses, -// and apostrophes are written properly") and that backfired: priming words which never occur in -// speech put them in play, and a character surname came back from a Monk episode as "Commas". -// Everything here is ordinary conversational language for that reason. -// -// Each is written in its own language, since an English prompt on Portuguese audio biases the -// output, and each deliberately runs past PROMPT_CONTEXT_TOKENS — see above for why the length is -// load-bearing. Overflow is trimmed from the front, so nothing depends on the opening words. -// -// Languages without an entry fall through to no prompt at all, which is handled explicitly below. -// -// Every prompt uses its language's full diacritic set (and, for a non-Latin script, its own -// punctuation — Japanese 。/、, Greek ; as a question mark, Arabic ، and ؟) in ordinary words, never -// forced in. Omitting one, e.g. writing Swedish without å ä ö, teaches Whisper to drop it too. -const TRANSCRIPT_PROMPTS: Record = { - en: - 'Hey, sorry for the late reply — I was stuck in a meeting all morning and only just got out. ' + - "I had a look at the document you sent over, and honestly, I think it's mostly fine. There are " + - "a couple of things I'd change in the second half, but nothing serious. The introduction reads " + - 'really well. Let me know when you have a few minutes and we can talk it through properly. ' + - "I'm around all afternoon, so whenever suits you is fine by me. Thanks again for putting it " + - 'together, I know it took a while.', - pt: - 'Olá, desculpa a demora a responder — estive numa reunião toda a manhã e só agora consegui ' + - 'sair. Já vi o documento que enviaste e, sinceramente, acho que está quase tudo bem. Há duas ' + - 'ou três coisas que mudava na segunda metade, mas nada de grave. A introdução está muito bem ' + - 'escrita. Diz-me quando tiveres uns minutos e falamos com calma. Estou por aqui toda a tarde, ' + - 'por isso quando te der jeito está bem para mim. Obrigado outra vez por teres tratado disto, ' + - 'sei que deu trabalho.', - es: - 'Hola, perdona la tardanza en responder — he estado toda la mañana en una reunión y recién ahora ' + - 'he podido salir. Ya vi el documento que me enviaste y, la verdad, creo que está bastante bien. ' + - 'Hay un par de cosas que cambiaría en la segunda mitad, pero nada grave. La introducción está muy ' + - 'bien escrita. Avísame cuando tengas unos minutos y lo hablamos con calma. Estoy disponible toda ' + - 'la tarde, así que cuando te venga bien está perfecto para mí. Gracias otra vez por armarlo, sé ' + - 'que llevó tiempo.', - de: - 'Hallo, entschuldige die späte Antwort — ich saß den ganzen Vormittag in einer Besprechung und ' + - 'bin gerade erst rausgekommen. Ich habe mir das Dokument angeschaut, das du geschickt hast, und ' + - 'ehrlich gesagt finde ich es größtenteils gut. In der zweiten Hälfte würde ich ein paar ' + - 'Kleinigkeiten ändern, aber nichts Ernstes. Die Einleitung liest sich wirklich gut. Sag mir ' + - 'Bescheid, wenn du später ein paar Minuten hast, dann können wir in Ruhe darüber reden. Ich bin ' + - 'den ganzen Nachmittag über da, also wann immer es dir passt, ist mir recht. Danke nochmal, dass ' + - 'du dir die Mühe gemacht hast, ich weiß, das hat Zeit gekostet.', - fr: - "Salut, désolé pour la réponse tardive — j'étais coincé en réunion toute la matinée et je viens " + - "tout juste d'en sortir. J'ai regardé le document que tu as envoyé, et honnêtement, je trouve que " + - "c'est plutôt bien. Il y a deux ou trois choses que je changerais dans la deuxième moitié, mais " + - "rien de grave. L'introduction est vraiment bien écrite. Dis-moi quand tu as quelques minutes et " + - "on en parle tranquillement. Je suis disponible tout l'après-midi, donc quand ça t'arrange, ça me " + - "va. Merci encore de l'avoir préparé, je sais que ça a pris du temps.", - sv: - 'Hej, förlåt att jag inte svarat tidigare — jag satt i möte hela förmiddagen och kom precis loss. ' + - 'Jag har tittat på dokumentet du skickade, och ärligt talat tycker jag att det mesta är bra. Det ' + - 'är några saker jag skulle ändra i andra halvan, men inget allvarligt. Inledningen är verkligen ' + - 'bra skriven. Säg till när du har några minuter så pratar vi igenom det ordentligt. Jag är ledig ' + - 'hela eftermiddagen, så när det passar dig passar det mig. Tack igen för att du satte ihop det, ' + - 'jag vet att det tog ett tag.', - it: - 'Ciao, scusa il ritardo nella risposta — sono stato in riunione tutta la mattina e sono appena ' + - "uscito. Ho dato un'occhiata al documento che mi hai mandato e, sinceramente, penso che vada bene " + - 'quasi tutto. Ci sono un paio di cose che cambierei nella seconda metà, ma niente di grave. ' + - "L'introduzione è scritta davvero bene. Fammi sapere quando hai qualche minuto così ne parliamo " + - 'con calma. Sono libero tutto il pomeriggio, quindi va bene quando ti fa comodo. Grazie ancora ' + - 'per averlo messo insieme, so che ci è voluto tempo.', - nl: - 'Hoi, sorry voor het late antwoord — ik zat de hele ochtend in een vergadering en ben er net pas ' + - 'uit. Ik heb naar het document gekeken dat je hebt gestuurd, en eerlijk gezegd vind ik het ' + - 'grotendeels goed. Er zijn een paar dingen die ik in de tweede helft zou veranderen, maar niets ' + - 'ergs. De inleiding leest heel prettig. Laat het me weten wanneer je een paar minuten hebt, dan ' + - 'bespreken we het rustig. Ik ben de hele middag beschikbaar, dus wanneer het jou uitkomt is voor ' + - 'mij prima. Bedankt nogmaals dat je het hebt samengesteld, ik weet dat het tijd heeft gekost.', - pl: - 'Cześć, przepraszam za późną odpowiedź — siedziałem cały ranek na spotkaniu i w końcu udało mi ' + - 'się z niego wyrwać. Zerknąłem na dokument, który przesłałeś, i szczerze mówiąc, myślę, że ' + - 'większość jest w porządku. Jest kilka rzeczy, które zmieniłbym w drugiej połowie, ale nic ' + - 'poważnego. Wstęp jest naprawdę dobrze napisany. Daj znać, kiedy będziesz mieć chwilę, możemy to ' + - 'spokojnie omówić. Jestem wolny całe popołudnie, więc kiedy ci pasuje, mi też pasuje. Dzięki ' + - 'jeszcze raz za to, że to zebrałeś, wiem, że zajęło to trochę czasu.', - da: - 'Hej, undskyld det sene svar — jeg sad i møde hele formiddagen og er lige kommet ud. Jeg har ' + - 'kigget på dokumentet, du sendte, og ærligt talt synes jeg det meste er fint. Der er et par ting, ' + - 'jeg ville ændre i anden halvdel, men ikke noget alvorligt. Indledningen er skrevet rigtig godt. ' + - 'Sig til, når du har et par minutter, så snakker vi ordentligt om det. Jeg er ledig hele ' + - 'eftermiddagen, så det passer mig, når det passer dig. Tak igen fordi du satte det sammen, jeg ' + - 'ved det tog tid.', - no: - 'Hei, beklager det sene svaret — jeg satt i møte hele formiddagen og er akkurat ferdig. Jeg har ' + - 'sett på dokumentet du sendte, og ærlig talt synes jeg det meste er bra. Det er et par ting jeg ' + - 'ville endret i andre halvdel, men ingenting alvorlig. Innledningen er skrevet veldig bra. Si ' + - 'ifra når du har noen minutter, så tar vi en skikkelig prat om det. Jeg er ledig hele ' + - 'ettermiddagen, så det som passer deg passer meg. Takk igjen for at du satte det sammen, jeg vet ' + - 'det tok tid.', - fi: - 'Hei, anteeksi myöhäinen vastaus — istuin koko aamupäivän palaverissa ja pääsin juuri äsken pois. ' + - 'Katsoin dokumentin, jonka lähetit, ja rehellisesti sanottuna suurin osa siitä on ihan hyvä. ' + - 'Loppupuoliskolla on pari asiaa, jotka muuttaisin, mutta ei mitään vakavaa. Johdanto on ' + - 'kirjoitettu tosi hyvin. Sano, kun sinulla on hetki aikaa, niin jutellaan siitä kunnolla. Olen ' + - 'vapaana koko iltapäivän, joten mikä tahansa sopii sinulle, sopii minulle. Kiitos vielä kerran, ' + - 'kun kokosit sen, tiedän että siihen meni aikaa.', - tr: - 'Selam, geç cevap verdiğim için kusura bakma — bütün sabah toplantıdaydım ve az önce çıkabildim. ' + - 'Gönderdiğin dokümana baktım ve açıkçası çoğunlukla iyi olduğunu düşünüyorum. İkinci yarıda ' + - 'değiştireceğim birkaç şey var ama ciddi bir şey değil. Giriş kısmı gerçekten iyi yazılmış. ' + - 'Birkaç dakikan olduğunda haber ver, düzgünce konuşuruz. Bütün öğleden sonra müsaitim, yani sana ' + - 'uyan zaman bana da uyar. Bunu hazırladığın için tekrar teşekkürler, biliyorum zaman almıştır.', - cs: - 'Ahoj, promiň za pozdní odpověď — celé dopoledne jsem byl na poradě a teprve teď jsem se dostal ' + - 'ven. Podíval jsem se na dokument, cos poslal, a upřímně řečeno, myslím, že většina je v pořádku. ' + - 'V druhé polovině bych pár věcí změnil, ale nic vážného. Úvod je napsaný fakt dobře. Dej vědět, ' + - 'až budeš mít pár minut, můžeme to pořádně probrat. Celé odpoledne mám volno, takže kdy se ti to ' + - 'bude hodit, mně to vyhovuje. Díky ještě jednou, žes to dal dohromady, vím, že to zabralo čas.', - el: - 'Γεια σου, συγγνώμη για την καθυστερημένη απάντηση — ήμουν σε συνάντηση όλο το πρωί και μόλις ' + - 'τώρα βγήκα. Είδα το έγγραφο που έστειλες και, ειλικρινά, νομίζω πως είναι σχεδόν όλο εντάξει. ' + - 'Υπάρχουν δύο τρία πράγματα που θα άλλαζα στο δεύτερο μισό, αλλά τίποτα σοβαρό. Η εισαγωγή είναι ' + - 'γραμμένη πολύ καλά. Έχεις λίγα λεπτά να τα πούμε με την ησυχία μας; Είμαι ελεύθερος όλο το ' + - 'απόγευμα, οπότε όποτε σε βολεύει εμένα με βολεύει. Ευχαριστώ ξανά που το ετοίμασες, ξέρω πως ' + - 'πήρε χρόνο.', - uk: - 'Привіт, вибач за пізню відповідь — я весь ранок сидів на нараді і щойно звідти вийшов. Подивився ' + - 'документ, який ти надіслав, і чесно кажучи, мої думки такі — майже все добре. У мене є пара ' + - 'речей, які я б змінив у другій половині, але нічого серйозного. Вступ написаний дуже добре. ' + - 'Напиши, коли матимеш кілька хвилин, і спокійно все обговоримо. Я вільний увесь день, тобто коли тобі зручно, ' + - 'мені теж підходить. Дякую ще раз, що зібрав це докупи, знаю, що це зайняло час.', - ru: - 'Привет, извини, что долго не отвечал — всё утро просидел на встрече и только что освободился. ' + - 'Посмотрел документ, который ты прислал, и, честно говоря, думаю, что почти всё нормально. Есть ' + - 'пара моментов, которые я бы изменил во второй половине, но ничего серьёзного. Введение написано ' + - 'правда хорошо. Напиши, когда будет пара свободных минут, спокойно всё обсудим. Я свободен весь ' + - 'день, так что когда тебе удобно, мне тоже подходит. Спасибо ещё раз, что всё это собрал, знаю, ' + - 'что это заняло время.', - ro: - 'Salut, scuze pentru răspunsul întârziat — am fost toată dimineața într-o ședință și abia acum am ' + - 'scăpat. M-am uitat pe documentul pe care mi l-ai trimis și, sincer, cred că e aproape totul ' + - 'bine. Sunt vreo două-trei lucruri pe care le-aș schimba în a doua jumătate, dar nimic grav. ' + - 'Introducerea e scrisă chiar bine. Spune-mi când ai câteva minute și vorbim pe îndelete despre ' + - 'asta. Sunt liber toată după-amiaza, deci oricând îți convine mie îmi convine. Mulțumesc încă o ' + - 'dată că ai pus totul cap la cap, știu că a durat.', - hu: - 'Szia, bocs a késői válaszért — egész délelőtt gyűlésen ültem, és csak most sikerült ' + - 'kiszabadulnom. Megnéztem a dokumentumot, amit küldtél, és őszintén szólva szerintem szinte ' + - 'minden rendben van. Van pár dolog, amit megváltoztatnék a második felében, de semmi komoly. A ' + - 'bevezető nagyon jól van megírva. Szólj, ha van pár szabad perced, és nyugodtan átbeszéljük. ' + - 'Egész délután érek rá, szóval amikor neked jó, nekem is megfelel. Köszönöm még egyszer, hogy ' + - 'összeraktad, tudom, hogy hosszú időbe telt.', - ja: - '返信が遅くなってごめん。朝からずっと会議で、今やっと抜けられたところ。送ってもらった資料、見たよ。' + - '正直、ほとんど問題ないと思う。後半のところで直したい部分が二つ三つあるけど、大したことじゃない。' + - '導入の部分はすごくよく書けてると思う。時間があるときに教えて、ゆっくり話そう。' + - '午後はずっと空いてるから、都合のいいときでいいよ。まとめてくれてありがとう、時間かかったよね。', - zh: - '抱歉这么晚才回你——我一早上都在开会，刚出来。你发的那份文件我看过了，说实话，大部分都挺好的。' + - '后半部分有几个地方我想改一下，但都不是什么大问题。开头写得真的很不错。你什么时候有空跟我说一声，我们好好聊聊。' + - '我整个下午都有空，所以你什么时候方便都行。再次谢谢你把这个整理出来，我知道挺花时间的。', - ko: - '답장이 늦어서 미안해. 오전 내내 회의하다가 방금 겨우 빠져나왔어. 보내준 문서 봤는데, 솔직히 거의 다 괜찮은 것 같아. 후반부에 고치고 싶은 부분이 두어 개 있긴 한데, ' + - '심각한 건 아니야. 서론은 진짜 잘 쓴 것 같아. 시간 될 때 얘기해줘, 그럼 차분히 얘기해보자. 오후 내내 시간 있으니까 편한 때 아무 때나 괜찮아. 정리해줘서 다시 한번 ' + - '고마워, 시간 좀 걸렸을 텐데.', - ar: - 'مرحبًا، آسف على تأخري في الرد — كنت في اجتماع طوال الصباح ولتوّي خرجت منه. اطّلعت على المستند ' + - 'الذي أرسلته، وبصراحة أعتقد أن معظمه جيد. هناك بعض الأشياء التي أود تغييرها في النصف الثاني، لكن ' + - 'لا شيء خطير. المقدمة مكتوبة بشكل جيد جدًا. هل يمكن أن تخبرني متى تكون لديك بضع دقائق لنتحدث ' + - 'بهدوء؟ أنا متفرغ طوال بعد الظهر، فأي وقت يناسبك يناسبني. شكرًا مرة أخرى على إعدادك لهذا، أعرف أن ' + - 'الأمر استغرق وقتًا.', - he: - 'היי, סליחה על התגובה המאוחרת — הייתי בישיבה כל הבוקר ורק עכשיו יצאתי. הסתכלתי על המסמך ששלחת, ' + - 'ובכנות, אני חושב שרוב הדברים בסדר. יש כמה דברים שהייתי משנה בחצי השני, אבל שום דבר רציני. הפתיחה ' + - 'כתובה ממש טוב. תגיד לי מתי יש לך כמה דקות ונדבר על זה בנחת. אני פנוי כל אחר הצהריים, אז מתי שנוח ' + - 'לך נוח גם לי. תודה שוב שסידרת את זה, אני יודע שזה לקח זמן.', - hi: - 'हाय, देर से जवाब देने के लिए माफ़ करना — मैं पूरी सुबह मीटिंग में था और अभी-अभी निकल पाया। तुमने ' + - 'जो डॉक्यूमेंट भेजा था, वो देख लिया, और सच कहूं तो ज़्यादातर हिस्सा ठीक लग रहा है। दूसरे हिस्से ' + - 'में कुछ चीज़ें हैं जो मैं बदलना चाहूंगा, पर कुछ गंभीर नहीं है। शुरुआत बहुत अच्छी लिखी गई है। जब ' + - 'भी कुछ मिनट मिलें बता देना, फिर आराम से बात करते हैं। पूरी दोपहर मैं फ्री हूं, तो जब भी तुम्हें ' + - 'सही लगे मुझे ठीक है। इसे तैयार करने के लिए फिर से शुक्रिया, पता है इसमें वक़्त लगा होगा।', -}; - -// Beam search rather than the default greedy decode (whisper.cpp ships `beam_size = -1`). It is the -// single biggest quality lever here and it costs time, which this flow has — nothing waits on it. -const BEAM_SIZE = 5; -const BEST_OF = 5; - -// The whisper container already runs silero VAD, but its defaults assume clean dictation. Widen the -// segments so they break at real pauses instead of at breaths, and pad the edges so word onsets are -// not clipped — clipped onsets are what shows up in the output as a word split down the middle. -const VAD_TUNING: Record = { - vad_speech_pad_ms: '200', // default 30 - vad_min_silence_duration_ms: '500', // default 100 — a breath is not a sentence boundary - vad_samples_overlap: '0.2', // default 0.1 - vad_max_speech_duration_s: '25', // default unbounded — keep a segment inside one 30s window -}; - -// A silence this long reads as a new thought. Whisper reports segment times against the original -// audio even when VAD is on (whisper.cpp maps them back through its VAD table), so these really are -// gaps in the recording rather than artefacts of the filtering. -const PARAGRAPH_GAP_SECONDS = 1.2; -const MAX_PARAGRAPH_CHARS = 900; - -// OpenAI Whisper's own silence rule: a segment the model thinks is probably not speech *and* decodes -// with low confidence is a hallucination rather than a transcript. whisper.cpp applies a subtly -// different form of this internally, so it is worth re-applying to the values it reports — this is -// what removes the stray words that appear over music and room tone. -const NO_SPEECH_PROB = 0.6; -const LOW_CONFIDENCE_LOGPROB = -1.0; - -export type WhisperSegment = { - text?: string; - start?: number; - end?: number; - no_speech_prob?: number; - avg_logprob?: number; -}; - -// Defaults are chosen so a segment missing either field is kept: dropping real speech is much worse -// than keeping a stray word. -const isHallucinatedSilence = (segment: WhisperSegment) => - (segment.no_speech_prob ?? 0) > NO_SPEECH_PROB && (segment.avg_logprob ?? 0) < LOW_CONFIDENCE_LOGPROB; - -// Whisper emits each segment with a leading space and now and then strands trailing punctuation in a -// segment of its own, which is why the old `response_format=text` output contained lone "." lines. -const appendSegment = (paragraph: string, text: string) => { - if (!paragraph) return text; - return /^[.,;:!?…)\]]/.test(text) ? paragraph + text : `${paragraph} ${text}`; -}; - -// Whisper returns one segment per utterance, not per paragraph. Rejoin them into prose and start a -// new paragraph on a long pause, so a voice memo reads as something written rather than as a list of -// fragments. -export function toParagraphs(segments: WhisperSegment[]): string { - const paragraphs: string[] = []; - let current = ''; - let previousEnd: number | null = null; - - for (const segment of segments) { - const text = (segment.text ?? '').replace(/\s+/g, ' ').trim(); - if (!text || isHallucinatedSilence(segment)) continue; - - const gap = previousEnd !== null && segment.start !== undefined ? segment.start - previousEnd : 0; - // Only break an over-long paragraph where a sentence actually ended, never mid-thought. - const overlong = current.length >= MAX_PARAGRAPH_CHARS && /[.!?…]["')\]]?$/.test(current); - - if (current && (gap >= PARAGRAPH_GAP_SECONDS || overlong)) { - paragraphs.push(current); - current = ''; - } - - current = appendSegment(current, text); - if (segment.end !== undefined) previousEnd = segment.end; - } - - if (current) paragraphs.push(current); - return paragraphs.join('\n\n'); -} - -export async function transcribeAudio({ - file, - whisperUrl, - spokenLanguages, -}: TranscribeArgs): Promise { - const base = whisperUrl.replace(/\/+$/, ''); - - // Step 1: detect language, and only fall back to the spoken-language bias when Whisper is unsure. - const detectForm = new FormData(); - detectForm.append('file', file); - detectForm.append('temperature', '0.0'); - detectForm.append('response_format', 'verbose_json'); - detectForm.append('detect_language', 'true'); - - const detectRes = await fetch(`${base}/inference`, { - method: 'POST', - body: detectForm, - timeout: false, - } as BunRequestInit); - if (!detectRes.ok) throw new Error(`Language detection failed (${detectRes.status})`); - const detectJson = (await detectRes.json()) as { - language?: string; - detected_language?: string; - detected_language_probability?: number; - language_probabilities?: Record; - }; - - const probs = detectJson.language_probabilities; - const ranked = probs ? Object.entries(probs).sort((a, b) => b[1] - a[1]) : []; - const [topCode, topProb] = ranked[0] ?? []; - - let detectedLanguage: string; - if (topCode && (topProb ?? 0) >= CONFIDENT_DETECTION) { - // Whisper is sure. Trust it even when it is not a language the user speaks — that is exactly the - // case translation exists for. - detectedLanguage = topCode; - } else if (ranked.length > 0 && spokenLanguages.length > 0) { - // Genuinely ambiguous (usually a short clip). Break the tie toward a language the user speaks, - // since Whisper's low-confidence guess defaults to English regardless of the audio. - const bestSpoken = spokenLanguages - .map((code) => [code, probs?.[code] ?? 0] as const) - .sort((a, b) => b[1] - a[1])[0]; - detectedLanguage = bestSpoken?.[0] ?? topCode ?? 'en'; - } else if (topCode) { - detectedLanguage = topCode; - } else { - // No probabilities field (non-whisper.cpp backend) — try to map the reported name. - const raw = (detectJson.detected_language ?? detectJson.language ?? 'en').toLowerCase(); - detectedLanguage = LANG_NAME_TO_ISO[raw] ?? raw; - } - - // Step 2: transcribe in the detected language - const transcribeForm = new FormData(); - transcribeForm.append('file', file); - transcribeForm.append('language', detectedLanguage); - transcribeForm.append('temperature', '0.0'); - transcribeForm.append('temperature_inc', '0.2'); - transcribeForm.append('beam_size', String(BEAM_SIZE)); - transcribeForm.append('best_of', String(BEST_OF)); - transcribeForm.append('suppress_nst', 'true'); - transcribeForm.append('response_format', 'verbose_json'); - // verbose_json re-runs a full language auto-detect unless this is set. Step 1 already answered - // that question, so it is pure duplicated work on every transcription. - transcribeForm.append('no_language_probabilities', 'true'); - // Segment timestamps drive the paragraphing below and are kept; per-token timestamps are extra - // work on every segment and nothing here reads them. - transcribeForm.append('token_timestamps', 'false'); - - const prompt = TRANSCRIPT_PROMPTS[detectedLanguage]; - if (prompt) { - transcribeForm.append('prompt', prompt); - transcribeForm.append('carry_initial_prompt', 'true'); - transcribeForm.append('max_context', String(PROMPT_CONTEXT_TOKENS)); - } else { - // Nothing worth protecting the budget for, so close the loop-feeding path outright. - transcribeForm.append('max_context', '0'); - } - - for (const [key, value] of Object.entries(VAD_TUNING)) transcribeForm.append(key, value); - - const res = await fetch(`${base}/inference`, { - method: 'POST', - body: transcribeForm, - timeout: false, - } as BunRequestInit); - if (!res.ok) throw new Error(`Transcription failed (${res.status})`); - - const json = (await res.json()) as { text?: string; segments?: WhisperSegment[] }; - // Fall back to the flat text for any backend that does not return segments. - const segments = json.segments ?? []; - const text = segments.length > 0 ? toParagraphs(segments) : (json.text ?? '').trim(); - - return { text, detectedLanguage }; -} diff --git a/src/servers/api/tasks/task-api-env.ts b/src/servers/api/tasks/task-api-env.ts index 363441e3..fb7f51b9 100644 --- a/src/servers/api/tasks/task-api-env.ts +++ b/src/servers/api/tasks/task-api-env.ts @@ -14,14 +14,15 @@ const PUBLIC_HOST = (() => { type TaskApiUser = { id: number; email: string; username?: string | null }; // Environment that lets a script task call Officer's own API instead of reimplementing server-side -// work (transcription, for one — the endpoint already handles language detection and caching). +// work (transcription, for one — `plugins/transcribe/tasks/transcribe-audio-file/index.ts` uses exactly +// this to reach the transcribe plugin's own route rather than talking to Whisper itself). // // Requests go to 127.0.0.1 so a task never depends on DNS or the reverse proxy being up. Origin // validation matches the Host header against PUBLIC_URL though, so a bare localhost request is // rejected — hence OFFICER_API_HOST, which scripts must pass as an explicit Host header: // // curl -H "Authorization: Bearer $OFFICER_AUTH_TOKEN" -H "Host: $OFFICER_API_HOST" \ -// "$OFFICER_API_URL/api/file-browser/transcribe" +// "$OFFICER_API_URL/api/transcribe/transcribe" export async function buildTaskApiEnv(user: TaskApiUser): Promise> { const token = await sign({ id: user.id, email: user.email, username: user.username ?? '' }, '12h'); return { diff --git a/src/servers/permissions/registry.ts b/src/servers/permissions/registry.ts index a58684e9..5ae186bf 100644 --- a/src/servers/permissions/registry.ts +++ b/src/servers/permissions/registry.ts @@ -5,9 +5,11 @@ // // The obvious model is "list the routes a role may call". It does not survive contact with this codebase. // A sweep of all 100 mutating platform routes on 2026-08-06 found reads permanently stuck on POST for two -// reasons that are not going away: bodies GET cannot carry (`/stt` multipart audio, `/tts`, `/ocr`, -// `/transcribe`), and credentials that must not sit in a query string where access logs, shell history and -// Referer headers capture them (`/tts/voices` apiKey, the four `/test` endpoints, `/local-providers/probe`). +// reasons that are not going away: bodies GET cannot carry (`/tts`, `/ocr` multipart audio and images — +// `/stt` and `/transcribe` were two more of these until 2026-08-24, when speech-to-text left for the +// `transcribe` plugin), and credentials that must not sit in a query string where access logs, shell +// history and Referer headers capture them (`/tts/voices` apiKey, the four `/test` endpoints, +// `/local-providers/probe`). // Five genuinely free conversions were done in e54d71d; the rest are staying. So the METHOD alone cannot // carry the read/write distinction — hence `readOnlyWrites` below, declared per permission. // diff --git a/src/workspaces/officerdev/src/apps/Chat/index.ts b/src/workspaces/officerdev/src/apps/Chat/index.ts index 1895f14c..8be87046 100644 --- a/src/workspaces/officerdev/src/apps/Chat/index.ts +++ b/src/workspaces/officerdev/src/apps/Chat/index.ts @@ -20,7 +20,6 @@ export { EmbeddableChat, type UseEmbeddableChatType } from './EmbeddableChat'; export { useChat, type UseChatType } from '../../hooks/useChat'; export { useSlashCommands } from './useSlashCommands'; export { useAttachments, type UseAttachmentsType } from './useAttachments'; -export { useAudioRecording, type UseAudioRecordingType } from './useAudioRecording'; export * from './types'; diff --git a/src/workspaces/officerdev/src/apps/Chat/useAudioRecording.test.tsx b/src/workspaces/officerdev/src/apps/Chat/useAudioRecording.test.tsx deleted file mode 100644 index ef12456b..00000000 --- a/src/workspaces/officerdev/src/apps/Chat/useAudioRecording.test.tsx +++ /dev/null @@ -1,83 +0,0 @@ -import { afterEach, describe, expect, test } from 'bun:test'; -import { act, cleanup, render } from '@testing-library/react'; -import { useAudioRecording } from './useAudioRecording'; - -/** - * The microphone is the one resource in this app the browser tells the user about: a recording indicator - * stays lit in the tab strip until every track is stopped. Nothing here released it on unmount, so closing - * a Chat panel mid-recording left the light on for the life of the tab with no way to switch it off. - * - * The stubs are the smallest thing that can answer "was the track stopped": a stream that counts stops and - * a recorder that reports its state. Neither the real MediaRecorder nor a real device is involved. - */ - -const stopped: string[] = []; - -const fakeStream = (label: string) => ({ - getTracks: () => [{ stop: () => stopped.push(label) }], -}); - -class FakeMediaRecorder { - state: 'recording' | 'inactive' = 'inactive'; - stream = fakeStream('track'); - mimeType = 'audio/webm'; - ondataavailable: ((ev: { data: Blob }) => void) | null = null; - onstop: (() => void) | null = null; - start() { - this.state = 'recording'; - } - stop() { - this.state = 'inactive'; - this.onstop?.(); - } -} - -const install = () => { - stopped.length = 0; - Object.assign(globalThis, { MediaRecorder: FakeMediaRecorder }); - Object.defineProperty(globalThis.navigator, 'mediaDevices', { - configurable: true, - value: { getUserMedia: async () => fakeStream('track') }, - }); -}; - -type Handle = { toggle: () => Promise; recording: boolean }; - -const handle: Handle = { toggle: async () => {}, recording: false }; - -const Probe = () => { - const runner = useAudioRecording(() => {}); - handle.toggle = runner.toggleRecording; - handle.recording = runner.recording; - return null; -}; - -afterEach(cleanup); - -describe('useAudioRecording', () => { - test('releases the microphone when its component goes away mid-recording', async () => { - install(); - const view = render(); - - await act(async () => { - await handle.toggle(); - }); - expect(handle.recording).toBe(true); - expect(stopped).toEqual([]); - - view.unmount(); - - // Not "the user pressed stop" — the panel was closed, swapped or remounted while still recording, and - // that is the case that used to leak. - expect(stopped).toEqual(['track']); - }); - - test('unmounting when nothing was ever recorded does nothing', () => { - install(); - const view = render(); - - view.unmount(); - - expect(stopped).toEqual([]); - }); -}); diff --git a/src/workspaces/officerdev/src/apps/Chat/useAudioRecording.ts b/src/workspaces/officerdev/src/apps/Chat/useAudioRecording.ts index 1602af9b..dbbf453f 100644 --- a/src/workspaces/officerdev/src/apps/Chat/useAudioRecording.ts +++ b/src/workspaces/officerdev/src/apps/Chat/useAudioRecording.ts @@ -1,114 +1,7 @@ -import { useState, useRef, useEffect } from 'react'; -import { toast } from 'sonner'; -import { useTranslation } from '../../locales'; - -export function useAudioRecording(onTranscription: (text: string) => void) { - const { t } = useTranslation(); - const [recording, setRecording] = useState(false); - const [transcribing, setTranscribing] = useState(false); - const mediaRecorderRef = useRef(null); - const chunksRef = useRef([]); - - const toggleRecording = async () => { - if (recording) { - const recorder = mediaRecorderRef.current; - if (!recorder) return; - - setRecording(false); - - try { - if (recorder.state === 'inactive') { - recorder.stream.getTracks().forEach((track) => track.stop()); - return; - } - - const blob = await new Promise((resolve, reject) => { - const timeout = setTimeout(() => reject(new Error('Recording stop timed out')), 5000); - recorder.onstop = () => { - clearTimeout(timeout); - resolve(new Blob(chunksRef.current, { type: recorder.mimeType })); - chunksRef.current = []; - }; - recorder.stop(); - }); - - recorder.stream.getTracks().forEach((track) => track.stop()); - - if (blob.size === 0) { - toast.error(t('chat.composer.noAudio')); - return; - } - - setTranscribing(true); - try { - const wav = await blobToWav(blob); - const formData = new FormData(); - formData.append('file', wav, 'recording.wav'); - formData.append('temperature', '0.0'); - formData.append('temperature_inc', '0.2'); - formData.append('response_format', 'json'); - - const token = localStorage.getItem('BEARER_TOKEN') ?? sessionStorage.getItem('BEARER_TOKEN'); - const res = await fetch('/api/chat/stt', { - method: 'POST', - body: formData, - headers: token ? { Authorization: `Bearer ${token}` } : {}, - }); - if (!res.ok) throw new Error(`Whisper returned ${res.status}`); - const json = await res.json(); - if (json.error) throw new Error(json.error); - const text = (json.text ?? '').trim(); - if (text) onTranscription(text); - } finally { - setTranscribing(false); - } - } catch (err) { - recorder.stream?.getTracks().forEach((track) => track.stop()); - chunksRef.current = []; - toast.error(err instanceof Error ? err.message : 'Recording failed'); - } - return; - } - - try { - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); - const recorder = new MediaRecorder(stream); - mediaRecorderRef.current = recorder; - chunksRef.current = []; - - recorder.ondataavailable = (ev) => { - if (ev.data.size > 0) chunksRef.current.push(ev.data); - }; - - recorder.start(250); - setRecording(true); - } catch (err) { - toast.error(err instanceof Error ? err.message : 'Could not access microphone'); - } - }; - - // Nothing here released the microphone on unmount, and a panel can go away mid-recording — closing the - // Chat panel, switching dashboards, a layout change that remounts the slot. The recorder is only ever - // stopped by the user pressing the button again, so without this the stream outlives the component and - // the browser keeps showing the recording indicator for the life of the tab. - useEffect( - () => () => { - const recorder = mediaRecorderRef.current; - if (!recorder) return; - if (recorder.state !== 'inactive') recorder.stop(); - recorder.stream?.getTracks().forEach((track) => track.stop()); - mediaRecorderRef.current = null; - chunksRef.current = []; - }, - [], - ); - - return { recording, transcribing, toggleRecording }; -} - -export type UseAudioRecordingType = ReturnType; - -// ── Helpers ── +// The chat composer's own STT-driven recording hook lived here until 2026-08-24, when speech-to-text +// left the platform for the `transcribe` plugin. It had no caller — the composer never mounted it — so +// it went rather than being ported. `blobToWav` stays: `DictateDialog.tsx` in the file browser uses it +// for something that was never STT, just recording a voice memo straight to a `.wav` upload. export const blobToWav = async (blob: Blob): Promise => { const ctx = new AudioContext(); diff --git a/src/workspaces/officerdev/src/apps/FileViewer/FileViewerContext.tsx b/src/workspaces/officerdev/src/apps/FileViewer/FileViewerContext.tsx index 65892954..e245b31d 100644 --- a/src/workspaces/officerdev/src/apps/FileViewer/FileViewerContext.tsx +++ b/src/workspaces/officerdev/src/apps/FileViewer/FileViewerContext.tsx @@ -12,7 +12,6 @@ export type FileViewerContextValue = { directContent: boolean; ttsLoading: boolean; ocrLoading: boolean; - transcribeLoading: boolean; extractAudioLoading: boolean; extractLoading: boolean; autoPlay: boolean; @@ -22,7 +21,6 @@ export type FileViewerContextValue = { setContent: (content: string) => void; handleReadAloud: () => void; handleOcr: () => void; - handleTranscribe: () => void; handleExtractAudio: () => void; handleExtract: () => void; handleDownload: () => void; diff --git a/src/workspaces/officerdev/src/apps/FileViewer/FileViewerHeader.tsx b/src/workspaces/officerdev/src/apps/FileViewer/FileViewerHeader.tsx index 2de7c55d..1deea4b9 100644 --- a/src/workspaces/officerdev/src/apps/FileViewer/FileViewerHeader.tsx +++ b/src/workspaces/officerdev/src/apps/FileViewer/FileViewerHeader.tsx @@ -7,7 +7,6 @@ import { FileType2, Volume2, ScanText, - FileText, AudioLines, FolderArchive, Save, @@ -28,12 +27,10 @@ export const FileViewerHeader = () => { directContent, ttsLoading, ocrLoading, - transcribeLoading, extractAudioLoading, extractLoading, handleReadAloud, handleOcr, - handleTranscribe, handleExtractAudio, handleExtract, handleDownload, @@ -98,31 +95,6 @@ export const FileViewerHeader = () => { {ocrLoading ? : } )} - {/* - TRANSCRIBE — hidden, not deleted, and the distinction is deliberate. - - Speech-to-text is leaving the platform for a plugin of its own. Until that lands this button calls - `POST /file-browser/transcribe`, which refuses on every server here because no STT endpoint is - configured anywhere — so it is a control that can only fail, and one of those teaches people to - distrust the controls beside it. - - Kept because the wiring is the useful part. When the plugin exists this becomes a call to it, and - the handler, the loading state and the sibling-markdown behaviour in FileViewerProvider are all - still correct; deleting would mean rediscovering them. - - It may not come back here at all. The other route to the same job is the file browser's context - menu, which a plugin reaches with a task — and that path needs no button in this header. - */} - {false && fileType === 'audio' && ( - - )} {fileType === 'video' && (