diff --git a/README.md b/README.md index b28c468..057b537 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,15 @@ audio is not. - **`POST /api/transcribe/transcribe`** — send an audio file, get back its text and the language Whisper detected. Two-pass under the hood (detect, then transcribe in that language) so a low-confidence guess never quietly defaults to English. +- **Conversations come back as conversations.** Nobody declares what a recording is — a probe on the + diarization worker runs alongside every transcription and finds out. Solo dictation returns as + paragraphs, exactly as before; a recording with several voices returns speaker-labeled + (`Speaker 1: …`) with `conversation: true` and a speaker count. The worker is optional: without it, + or if it fails mid-request, everything still transcribes — just never labeled. See + `~/dockers/torch-worker` for the worker itself. +- **Fair queueing for a multi-user server.** Transcription and diarization each run one job at a time + behind their own queue, in arrival order — and on different hardware, so the conversation probe costs + a solo memo nothing. - **A file-browser task.** Right-click an `.mp3`, `.wav` or `.m4a` and transcribe it without a model turn — the task is a plain script that calls this plugin's own route and files the result next to the original. diff --git a/manifest.ts b/manifest.ts index aa82704..cf7676f 100644 --- a/manifest.ts +++ b/manifest.ts @@ -6,9 +6,11 @@ import type { PluginManifest } from '@@/plugins/manifest'; * ── What this plugin is, and what it is not ── * * It does NOT run Whisper, and it does not ship a model. You point it at a whisper.cpp `/inference` - * server you already run (the same container the platform used to talk to directly) and it forwards to - * it. Nothing is transcribed here — the sidecar's whole job is the request shape: detect the language, - * then transcribe faithfully in it, tuned for one person talking into a phone. + * server you already run (the same container the platform used to talk to directly) and, optionally, at + * a diarization worker beside it. Nothing is transcribed here — the sidecar's job is the request shape + * (detect the language, then transcribe faithfully in it, tuned for one person talking into a phone) + * and the pipeline around it: probe whether the recording is a conversation, label the speakers when it + * is, and queue multi-user traffic fairly across the two servers. * * Named for the feature, not the implementation — the same reasoning as `photos` not `immich`. Swapping * the backend for a different `/inference`-compatible server would not move the mount point or the route @@ -58,6 +60,16 @@ export const manifest: PluginManifest = { required: true, placeholder: 'http://127.0.0.1:8178', }, + { + key: 'workerUrl', + label: 'Diarization worker', + hint: + 'The torch stage worker that detects conversations and labels speakers. Optional — ' + + 'without it, recordings still transcribe but nothing is speaker-labeled.', + type: 'url', + required: false, + placeholder: 'http://127.0.0.1:8183', + }, ], permissions: [ diff --git a/sidecar/config.ts b/sidecar/config.ts index 6dc2bdd..843bd00 100644 --- a/sidecar/config.ts +++ b/sidecar/config.ts @@ -15,7 +15,12 @@ import { DATA_PATH } from '@@/data-path'; const CONFIG_DIR = join(DATA_PATH, 'transcribe'); const CONFIG_PATH = join(CONFIG_DIR, 'config.json'); -export type TranscribeConfig = { url: string }; +export type TranscribeConfig = { + url: string; + /** The torch stage worker (diarization + conversation probe). Optional: without it the plugin + * still transcribes, it just never labels speakers — degradation, not failure. */ + workerUrl?: string; +}; // Cached across requests; invalidated only by `setConfig`. A single small file re-read on every // transcription would be a needless stat + parse on the hot path. @@ -26,7 +31,9 @@ async function loadConfig(): Promise { if (!(await file.exists())) return null; try { const parsed = (await file.json()) as Partial; - return typeof parsed.url === 'string' && parsed.url ? { url: parsed.url } : null; + if (typeof parsed.url !== 'string' || !parsed.url) return null; + const workerUrl = typeof parsed.workerUrl === 'string' && parsed.workerUrl ? parsed.workerUrl : undefined; + return { url: parsed.url, ...(workerUrl ? { workerUrl } : {}) }; } catch { return null; } diff --git a/sidecar/conversation.test.ts b/sidecar/conversation.test.ts new file mode 100644 index 0000000..2a0ae7b --- /dev/null +++ b/sidecar/conversation.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from 'bun:test'; +import { toConversation } from './conversation'; +import type { WhisperSegment } from './transcribe'; +import type { SpeakerTurn } from './worker'; + +const seg = (text: string, start: number, end: number, extra: Partial = {}): WhisperSegment => ({ + text, + start, + end, + ...extra, +}); + +const turn = (start: number, end: number, speaker: string): SpeakerTurn => ({ start, end, speaker }); + +describe('toConversation', () => { + it('attributes each segment to the speaker whose turns cover it most', () => { + const out = toConversation( + [seg(' How are you?', 0, 2), seg(' Fine, thanks.', 2.5, 4)], + [turn(0, 2.2, 'SPEAKER_00'), turn(2.3, 4.5, 'SPEAKER_01')], + ); + expect(out.text).toBe('Speaker 1: How are you?\n\nSpeaker 2: Fine, thanks.'); + expect(out.speakerCount).toBe(2); + }); + + it('numbers speakers by first appearance, not by pyannote label order', () => { + // SPEAKER_01 talks first — they are "Speaker 1" in the transcript. + const out = toConversation( + [seg(' Me first.', 0, 2), seg(' Me second.', 3, 5)], + [turn(0, 2, 'SPEAKER_01'), turn(3, 5, 'SPEAKER_00')], + ); + expect(out.text).toBe('Speaker 1: Me first.\n\nSpeaker 2: Me second.'); + }); + + it('joins consecutive segments by the same speaker into one block', () => { + const out = toConversation( + [seg(' First sentence.', 0, 2), seg(' Second sentence.', 2.2, 4), seg(' A reply.', 5, 6)], + [turn(0, 4.5, 'SPEAKER_00'), turn(4.8, 6.5, 'SPEAKER_01')], + ); + expect(out.text).toBe('Speaker 1: First sentence. Second sentence.\n\nSpeaker 2: A reply.'); + }); + + // Whisper and pyannote disagree about boundaries all the time; a segment neither turn covers + // must stay with whoever was already talking, not open a phantom block. + it('keeps an uncovered segment with the previous speaker', () => { + const out = toConversation( + [seg(' Covered.', 0, 2), seg(' In the gap.', 2.5, 3), seg(' Reply.', 4, 6)], + [turn(0, 2.2, 'SPEAKER_00'), turn(3.8, 6, 'SPEAKER_01')], + ); + expect(out.text).toBe('Speaker 1: Covered. In the gap.\n\nSpeaker 2: Reply.'); + }); + + it('attributes a transcript that starts before the first turn to the first speaker', () => { + const out = toConversation([seg(' Early words.', 0, 1)], [turn(1.5, 3, 'SPEAKER_00')]); + expect(out.text).toBe('Speaker 1: Early words.'); + expect(out.speakerCount).toBe(1); + }); + + it('drops hallucinated-silence segments like the solo path does', () => { + const out = toConversation( + [seg(' Real.', 0, 2), seg(' Thanks for watching!', 2.1, 3, { no_speech_prob: 0.95, avg_logprob: -1.8 })], + [turn(0, 3, 'SPEAKER_00')], + ); + expect(out.text).toBe('Speaker 1: Real.'); + }); + + it('attaches stranded punctuation without a space, inside a block', () => { + const out = toConversation( + [seg(' See you tomorrow', 0, 2), seg(' .', 2.1, 2.2)], + [turn(0, 3, 'SPEAKER_00')], + ); + expect(out.text).toBe('Speaker 1: See you tomorrow.'); + }); + + it('reports one speaker for a monologue so the caller can fall back to paragraphs', () => { + const out = toConversation( + [seg(' Just me talking.', 0, 2)], + [turn(0, 2, 'SPEAKER_00'), turn(3, 4, 'SPEAKER_00')], + ); + expect(out.speakerCount).toBe(1); + }); +}); diff --git a/sidecar/conversation.ts b/sidecar/conversation.ts new file mode 100644 index 0000000..8cd7927 --- /dev/null +++ b/sidecar/conversation.ts @@ -0,0 +1,92 @@ +// Merge Whisper's transcript segments with pyannote's speaker turns into a readable conversation. +// +// The two inputs come from different models that never saw each other: Whisper says WHAT was said +// and when, pyannote says WHO was speaking and when. The only bridge is time. Each transcript +// segment is attributed to the speaker whose turns overlap it most; a segment overlapping nobody +// (Whisper and pyannote disagree about where speech is — it happens at boundaries) stays with the +// previous speaker rather than inventing a gap. +// +// Attribution is at segment granularity on purpose. Turn-taking in a real conversation happens at +// the scale of sentences, and whisper.cpp's segment timestamps are plenty for that; word-level +// alignment (whisperx's wav2vec2 stage) earns its cost only for interruptions mid-sentence, which +// a voice memo of a conversation can afford to get slightly wrong. If boundaries feel sloppy in +// practice, that stage can return as a worker endpoint — see ~/dockers/torch-worker. + +import type { WhisperSegment } from './transcribe'; +import { appendSegment, isHallucinatedSilence } from './transcribe'; +import type { SpeakerTurn } from './worker'; + +export type ConversationResult = { + text: string; + speakerCount: number; +}; + +/** Seconds of a segment's span covered by a speaker's turns. */ +function overlapWith(segment: { start: number; end: number }, turns: SpeakerTurn[], speaker: string): number { + let covered = 0; + for (const turn of turns) { + if (turn.speaker !== speaker) continue; + covered += Math.max(0, Math.min(segment.end, turn.end) - Math.max(segment.start, turn.start)); + } + return covered; +} + +/** + * Render transcript segments + speaker turns as a conversation. + * + * Speakers are renamed by order of first appearance — pyannote's SPEAKER_00/01 labels are + * clustering artifacts with no stable meaning, and "Speaker 1" reads better in a transcript. + * Consecutive segments by the same speaker join into one block, with the same punctuation-aware + * joining and hallucination filtering the solo path uses. + */ +export function toConversation(segments: WhisperSegment[], turns: SpeakerTurn[]): ConversationResult { + const speakerNames = new Map(); + const nameFor = (label: string): string => { + let name = speakerNames.get(label); + if (!name) { + name = `Speaker ${speakerNames.size + 1}`; + speakerNames.set(label, name); + } + return name; + }; + + const labels = [...new Set(turns.map((t) => t.speaker))]; + + const blocks: { name: string; text: string }[] = []; + let currentLabel: string | null = null; + + for (const segment of segments) { + const text = (segment.text ?? '').replace(/\s+/g, ' ').trim(); + if (!text || isHallucinatedSilence(segment)) continue; + + let label: string | null = currentLabel; + if (segment.start !== undefined && segment.end !== undefined) { + const span = { start: segment.start, end: segment.end }; + let best = 0; + for (const candidate of labels) { + const covered = overlapWith(span, turns, candidate); + if (covered > best) { + best = covered; + label = candidate; + } + } + // best === 0 → nobody overlaps; keep the previous speaker rather than guessing. + } + // Before any turn matches (e.g. transcript starts a hair before the first turn), attribute + // forward to the first speaker instead of dropping words. + label = label ?? labels[0] ?? 'SPEAKER_00'; + + const last = blocks[blocks.length - 1]; + if (last && label === currentLabel) { + last.text = appendSegment(last.text, text); + } else { + blocks.push({ name: nameFor(label), text }); + currentLabel = label; + } + } + + return { + text: blocks.map((b) => `${b.name}: ${b.text}`).join('\n\n'), + speakerCount: speakerNames.size, + }; +} diff --git a/sidecar/index.ts b/sidecar/index.ts index 9456794..79d9bb1 100644 --- a/sidecar/index.ts +++ b/sidecar/index.ts @@ -3,6 +3,9 @@ import { createSidecarConnector } from '@@/sidecar/connect'; import { API_URL } from '@@/officer-url.mjs'; import { getConfig, setConfig, redactUrl, probeWhisper } from './config'; import { transcribeAudio } from './transcribe'; +import { toConversation } from './conversation'; +import { whisperQueue, workerQueue } from './queue'; +import { diarizeAudio, probeAudio, probeWorker, type SpeakerTurn } from './worker'; // The officer-transcribe sidecar. Owns the whole whisper.cpp contract for Officer: the server URL and // the request tuning that makes it usable for a person talking into a phone (see transcribe.ts — moved @@ -13,10 +16,15 @@ import { transcribeAudio } from './transcribe'; // ───────────────────────────────────────────────────────────────────────────────────────────────── // HTTP CONTRACT — the platform strips its /api/transcribe mount prefix before forwarding. // -// GET /_health { ok, configured, url? } -// GET /config { configured, url } — url has its userinfo stripped, never a secret -// PUT /config { url } — validated against the server before storing -// POST /transcribe multipart: file, spokenLanguages[] → { text, detectedLanguage } +// GET /_health { ok, configured, url?, worker, queues } +// GET /config { configured, url, workerUrl } — urls have userinfo stripped, never a secret +// PUT /config { url, workerUrl? } — both validated against their servers first +// POST /transcribe multipart: file, spokenLanguages[] +// → { text, detectedLanguage, conversation: false } +// → { text: "Speaker 1: …", detectedLanguage, conversation: true, speakers } +// The conversation probe and diarization run on the torch worker (workerUrl, +// ~/dockers/torch-worker) behind their own queue; any worker failure degrades +// to the plain transcript rather than failing the request. // // anything else 404 // @@ -38,7 +46,11 @@ const port = getFreePort(); async function handleConfigGet(): Promise { const cfg = await getConfig(); - return Response.json({ configured: !!cfg, url: cfg ? redactUrl(cfg.url) : null }); + return Response.json({ + configured: !!cfg, + url: cfg ? redactUrl(cfg.url) : null, + workerUrl: cfg?.workerUrl ? redactUrl(cfg.workerUrl) : null, + }); } function isHttpUrl(url: string): boolean { @@ -46,7 +58,7 @@ function isHttpUrl(url: string): boolean { } async function handleConfigPut(req: Request): Promise { - const body = ((await req.json().catch(() => null)) as { url?: unknown } | null) ?? {}; + const body = ((await req.json().catch(() => null)) as { url?: unknown; workerUrl?: unknown } | null) ?? {}; const url = typeof body.url === 'string' ? body.url.trim() : ''; if (!url) return Response.json({ error: 'url is required' }, { status: 400 }); if (!isHttpUrl(url)) return Response.json({ error: 'url must start with http:// or https://' }, { status: 400 }); @@ -54,10 +66,31 @@ async function handleConfigPut(req: Request): Promise { const probe = await probeWhisper(url); if (!probe.ok) return Response.json({ error: probe.error }, { status: 400 }); - await setConfig({ url }); - return Response.json({ configured: true, url: redactUrl(url) }); + // The worker is optional, but a URL that is GIVEN must answer — storing a dead URL silently + // would turn every future conversation back into an unlabeled wall of text with no error. + const workerUrl = typeof body.workerUrl === 'string' ? body.workerUrl.trim() : ''; + if (workerUrl) { + if (!isHttpUrl(workerUrl)) { + return Response.json({ error: 'workerUrl must start with http:// or https://' }, { status: 400 }); + } + const workerProbe = await probeWorker(workerUrl); + if (!workerProbe.ok) return Response.json({ error: `diarization worker: ${workerProbe.error}` }, { status: 400 }); + } + + await setConfig({ url, ...(workerUrl ? { workerUrl } : {}) }); + return Response.json({ + configured: true, + url: redactUrl(url), + workerUrl: workerUrl ? redactUrl(workerUrl) : null, + }); } +// The pipeline. Transcription and the conversation probe run CONCURRENTLY on their own +// resources (whisper.cpp on its GPU, the torch worker on CPU cores), each behind its own serial +// queue, so the probe costs a solo memo nothing — it finishes long before the transcript does. +// +// The probe and diarization are strictly best-effort: any worker failure logs and degrades to +// the plain transcript. A memo must never be lost to a speaker-labeling problem. async function handleTranscribe(req: Request): Promise { const cfg = await getConfig(); if (!cfg) { @@ -69,10 +102,47 @@ async function handleTranscribe(req: Request): Promise { if (!file || !(file instanceof File)) return Response.json({ error: 'file is required' }, { status: 400 }); const spokenLanguages = (body?.getAll('spokenLanguages') ?? []).map(String); + const workerUrl = cfg.workerUrl; + + const probePromise = workerUrl + ? workerQueue.run(() => probeAudio(file, workerUrl)).catch((err) => { + console.error('[transcribe] conversation probe failed — returning plain transcript', err); + return null; + }) + : Promise.resolve(null); try { - const result = await transcribeAudio({ file, whisperUrl: cfg.url, spokenLanguages }); - return Response.json(result); + const result = await whisperQueue.run(() => transcribeAudio({ file, whisperUrl: cfg.url, spokenLanguages })); + const probe = await probePromise; + + if (probe && probe.verdict !== 'solo' && workerUrl) { + // "unsure" diarizes too: a mislabeled monologue reads fine, a missed conversation is + // information lost for good. Reuse the probe's turns when it already covered the file. + const turns: SpeakerTurn[] | null = probe.turns + ?? (await workerQueue.run(() => diarizeAudio(file, workerUrl)).then( + (d) => d.turns, + (err) => { + console.error('[transcribe] diarization failed — returning plain transcript', err); + return null; + }, + )); + + if (turns && turns.length > 0) { + const conversation = toConversation(result.segments, turns); + // A "conversation" that clusters into one voice end-to-end is a monologue after all — + // the paragraph rendering reads better than a wall of "Speaker 1:" prefixes. + if (conversation.speakerCount > 1) { + return Response.json({ + text: conversation.text, + detectedLanguage: result.detectedLanguage, + conversation: true, + speakers: conversation.speakerCount, + }); + } + } + } + + return Response.json({ text: result.text, detectedLanguage: result.detectedLanguage, conversation: false }); } catch (err) { console.error('[transcribe] transcription failed', err); return Response.json( @@ -115,7 +185,16 @@ const server = Bun.serve({ if (!cfg) return Response.json({ ok: false, configured: false }, { status: 503 }); const probe = await probeWhisper(cfg.url); if (!probe.ok) return Response.json({ ok: false, configured: true, error: probe.error }, { status: 502 }); - return Response.json({ ok: true, configured: true, url: redactUrl(cfg.url) }); + // The worker degrades rather than fails: an unreachable worker is reported but does not + // cost the health check — transcription still works without it. + const worker = cfg.workerUrl ? await probeWorker(cfg.workerUrl) : null; + return Response.json({ + ok: true, + configured: true, + url: redactUrl(cfg.url), + worker: worker === null ? 'not configured' : worker.ok ? 'ok' : `unreachable: ${worker.error}`, + queues: { whisper: whisperQueue.pending, worker: workerQueue.pending }, + }); } if (url.pathname === '/config') { diff --git a/sidecar/queue.ts b/sidecar/queue.ts new file mode 100644 index 0000000..f11b54d --- /dev/null +++ b/sidecar/queue.ts @@ -0,0 +1,38 @@ +// Two serial queues, one per compute resource, shared by every user of the platform. +// +// The whisper.cpp server and the torch worker each run one job at a time — both saturate their +// device, so concurrency inside them buys nothing. What the platform owes its users is fairness +// and calm under simultaneous requests: jobs enter here and run strictly in arrival order, and a +// slow episode ahead of you delays you but never breaks you. +// +// The two queues are deliberately independent: a probe or diarization runs on the torch worker +// WHILE someone else's transcription runs on Whisper. That is what makes the conversation check +// free for solo memos — it happens on the other resource's time. +// +// v2 (planned, not built): feature-aware ordering — short memos ahead of hour-long episodes, +// probes ahead of full diarizations. The queue object is the seam where that goes; callers will +// not change. + +export class SerialQueue { + private tail: Promise = Promise.resolve(); + private depth = 0; + + /** Jobs currently running or waiting. */ + get pending(): number { + return this.depth; + } + + run(job: () => Promise): Promise { + this.depth += 1; + const result = this.tail.then(job).finally(() => { + this.depth -= 1; + }); + // The chain itself must survive a failed job — the failure belongs to the caller who + // submitted it, not to everyone queued behind them. + this.tail = result.catch(() => undefined); + return result; + } +} + +export const whisperQueue = new SerialQueue(); +export const workerQueue = new SerialQueue(); diff --git a/sidecar/transcribe.ts b/sidecar/transcribe.ts index 09dd337..66adc5a 100644 --- a/sidecar/transcribe.ts +++ b/sidecar/transcribe.ts @@ -30,6 +30,9 @@ type TranscribeArgs = { export type TranscribeResult = { text: string; detectedLanguage: string; + /** Raw Whisper segments, kept for speaker attribution (conversation.ts). Not part of the HTTP + * response — index.ts decides what leaves the sidecar. */ + segments: WhisperSegment[]; }; // Above this probability Whisper's own detection is taken as authoritative. Below it the clip is @@ -281,12 +284,12 @@ export type WhisperSegment = { // Defaults are chosen so a segment missing either field is kept: dropping real speech is much worse // than keeping a stray word. -const isHallucinatedSilence = (segment: WhisperSegment) => +export const isHallucinatedSilence = (segment: WhisperSegment) => (segment.no_speech_prob ?? 0) > NO_SPEECH_PROB && (segment.avg_logprob ?? 0) < LOW_CONFIDENCE_LOGPROB; // Whisper emits each segment with a leading space and now and then strands trailing punctuation in a // segment of its own, which is why the old `response_format=text` output contained lone "." lines. -const appendSegment = (paragraph: string, text: string) => { +export const appendSegment = (paragraph: string, text: string) => { if (!paragraph) return text; return /^[.,;:!?…)\]]/.test(text) ? paragraph + text : `${paragraph} ${text}`; }; @@ -412,5 +415,5 @@ export async function transcribeAudio({ const segments = json.segments ?? []; const text = segments.length > 0 ? toParagraphs(segments) : (json.text ?? '').trim(); - return { text, detectedLanguage }; + return { text, detectedLanguage, segments }; } diff --git a/sidecar/worker.ts b/sidecar/worker.ts new file mode 100644 index 0000000..d7e7127 --- /dev/null +++ b/sidecar/worker.ts @@ -0,0 +1,68 @@ +// Client for the torch stage worker (~/dockers/torch-worker) — speaker diarization and the +// "is this a conversation?" probe. The worker is deliberately dumb: it answers acoustic +// questions about audio it is handed and holds no policy. Deciding what to do with a verdict +// happens in index.ts. +// +// Every call passes `timeout: false`: a full diarization runs at roughly half realtime on CPU, +// so an hour-long recording legitimately holds this connection for tens of minutes. The 255s +// lessons of 2026-08-24 are not getting relearned here. + +type BunRequestInit = RequestInit & { timeout?: boolean | number }; + +export type SpeakerTurn = { start: number; end: number; speaker: string }; + +export type ProbeVerdict = 'solo' | 'conversation' | 'unsure'; + +export type ProbeResult = { + duration: number; + analyzed: { start: number; end: number; whole_file: boolean }; + speakers_found: number; + overlap_ratio: number; + verdict: ProbeVerdict; + /** Complete diarization when the probe covered the whole file — reuse it, skip /diarize. */ + turns: SpeakerTurn[] | null; +}; + +export type DiarizeResult = { + duration: number; + speakers: string[]; + turns: SpeakerTurn[]; +}; + +async function post(url: string, form: FormData): Promise { + const res = await fetch(url, { method: 'POST', body: form, timeout: false } as BunRequestInit); + if (!res.ok) { + const detail = await res.text().catch(() => ''); + throw new Error(`worker ${res.status}: ${detail.slice(0, 300)}`); + } + return (await res.json()) as T; +} + +export async function probeAudio(file: File | Blob, workerUrl: string): Promise { + const form = new FormData(); + form.append('file', file); + return post(`${workerUrl.replace(/\/+$/, '')}/probe`, form); +} + +export async function diarizeAudio(file: File | Blob, workerUrl: string): Promise { + const form = new FormData(); + form.append('file', file); + return post(`${workerUrl.replace(/\/+$/, '')}/diarize`, form); +} + +export type WorkerProbe = { ok: true } | { ok: false; error: string }; + +/** Reachability check for PUT /config validation and /_health, mirroring probeWhisper. */ +export async function probeWorker(workerUrl: string): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 5000); + try { + const res = await fetch(`${workerUrl.replace(/\/+$/, '')}/health`, { signal: controller.signal }); + if (!res.ok) return { ok: false, error: `worker answered ${res.status}` }; + return { ok: true }; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : 'could not reach the worker' }; + } finally { + clearTimeout(timer); + } +} diff --git a/skills/transcribe/SKILL.md b/skills/transcribe/SKILL.md index 7016e95..0aecbf1 100644 --- a/skills/transcribe/SKILL.md +++ b/skills/transcribe/SKILL.md @@ -42,11 +42,18 @@ multipart/form-data: file — the audio file (mp3, wav, m4a, whatever whisper.cpp accepts) spokenLanguages — zero or more, e.g. spokenLanguages=en&spokenLanguages=pt -→ { "text": "…", "detectedLanguage": "en" } +→ { "text": "…", "detectedLanguage": "en", "conversation": false } +→ { "text": "Speaker 1: …\n\nSpeaker 2: …", "detectedLanguage": "en", "conversation": true, "speakers": 2 } → 400 { "error": "Whisper not configured — PUT /api/transcribe/config first" } → 502 { "error": "…" } — the whisper.cpp server rejected or failed the request ``` +Nobody declares whether a recording is a conversation — the pipeline finds out. A probe on the +diarization worker runs alongside transcription; solo dictation comes back as paragraphs exactly as +before, and a recording with several voices comes back speaker-labeled with `conversation: true`. +Expect diarized recordings to take longer (roughly half realtime on top of transcription). If the +worker is down or not configured, everything still transcribes — just never labeled. + `spokenLanguages` only matters when Whisper's own detection is genuinely unsure — it breaks the tie toward a language you already know the speaker uses. Pass what you know; pass nothing and detection still works, it is just less informed on an ambiguous clip.