From 7bdeecd7f0de2efe3082a4d5254e7967c1c614a9 Mon Sep 17 00:00:00 2001 From: Andre Padez Date: Wed, 25 Feb 2026 05:11:09 +0000 Subject: [PATCH] read alout chat messages --- src/servers/api/file-browser/router.ts | 45 ++++++++++++++ src/workspaces/emailer/emails/UserInvite.tsx | 2 +- .../apps/Chat/components/MessageBubble.tsx | 62 +++++++++++++++++-- .../officerdev/src/apps/Chat/types.ts | 2 +- .../components/TaskRunnerModal.tsx | 2 +- .../officerdev/src/hooks/useFilesAPI.ts | 3 + .../officerdev/src/hooks/usePiChat.ts | 10 +-- 7 files changed, 113 insertions(+), 13 deletions(-) diff --git a/src/servers/api/file-browser/router.ts b/src/servers/api/file-browser/router.ts index be2c1ecc..3052c187 100644 --- a/src/servers/api/file-browser/router.ts +++ b/src/servers/api/file-browser/router.ts @@ -414,6 +414,51 @@ router.post('/tts', async (ctx) => { return ctx.json({ audioPath: cacheRel, audioRoot: 'user-data' }); }); +// Text-to-speech from raw text with caching +router.post('/tts-text', async (ctx) => { + const user = ctx.get('user'); + const { text, id } = ctx.get('body') as { text: string; id: string }; + if (!text) throw errors.BAD_REQUEST('text is required'); + if (!id) throw errors.BAD_REQUEST('id is required'); + + const userDataDir = getUserDataDir(user.email); + const cacheRel = `cache/tts/chat/${id}.mp3`; + const cacheAbs = resolve(userDataDir, cacheRel); + + if (existsSync(cacheAbs)) { + return ctx.json({ audioPath: cacheRel, audioRoot: 'user-data' }); + } + + const ttsConfig = await readTtsConfig(); + if (!ttsConfig) throw errors.BAD_REQUEST('TTS not configured — set it up in Settings → Text to Speech'); + + const headers: Record = { 'Content-Type': 'application/json' }; + let res: Response; + + if (ttsConfig.provider === 'elevenlabs') { + if (!ttsConfig.apiKey) throw errors.BAD_REQUEST('ElevenLabs API key not configured'); + res = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${encodeURIComponent(ttsConfig.voice)}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'xi-api-key': ttsConfig.apiKey }, + body: JSON.stringify({ text, model_id: ttsConfig.model }), + }); + } else { + if (ttsConfig.apiKey) headers['Authorization'] = `Bearer ${ttsConfig.apiKey}`; + res = await fetch(`${ttsConfig.url.replace(/\/+$/, '')}/v1/audio/speech`, { + method: 'POST', + headers, + body: JSON.stringify({ model: ttsConfig.model, input: text, voice: ttsConfig.voice, response_format: 'mp3' }), + }); + } + if (!res.ok) throw errors.BAD_REQUEST('TTS request failed'); + + await mkdir(dirname(cacheAbs), { recursive: true }); + const buffer = await res.arrayBuffer(); + await Bun.write(cacheAbs, buffer); + + return ctx.json({ audioPath: cacheRel, audioRoot: 'user-data' }); +}); + // OCR image via vision model with caching router.post('/ocr', async (ctx) => { const user = ctx.get('user'); diff --git a/src/workspaces/emailer/emails/UserInvite.tsx b/src/workspaces/emailer/emails/UserInvite.tsx index 7e7faa9b..dde700b5 100644 --- a/src/workspaces/emailer/emails/UserInvite.tsx +++ b/src/workspaces/emailer/emails/UserInvite.tsx @@ -15,7 +15,7 @@ const Email = ({ invitedBy, url }: EmailProps) => { officer.dev diff --git a/src/workspaces/officerdev/src/apps/Chat/components/MessageBubble.tsx b/src/workspaces/officerdev/src/apps/Chat/components/MessageBubble.tsx index dd3e4814..d23e3733 100644 --- a/src/workspaces/officerdev/src/apps/Chat/components/MessageBubble.tsx +++ b/src/workspaces/officerdev/src/apps/Chat/components/MessageBubble.tsx @@ -1,10 +1,13 @@ +import { useState, useRef } from 'react'; import ReactMarkdown from 'react-markdown'; import remarkGfm from 'remark-gfm'; import rehypeRaw from 'rehype-raw'; +import { Volume2, Loader2, Square } from 'lucide-react'; import type { ChatMessage } from '../types'; import { ToolActivity } from './ToolActivity'; import { QuestionActivity } from './QuestionActivity'; import { getRawUrl } from '../../FileViewer/file-types'; +import { useFilesAPI } from '../../../hooks/useFilesAPI'; // Matches absolute image file paths, e.g. /home/user/pic.png or /tmp/photo.jpg const IMAGE_PATH_RE = /(\/(?:home\/[^/\s]+\/)?[^\s`"'<>\n\r[\]()]+\.(?:png|jpg|jpeg|gif|webp|svg|bmp|ico))/gi; @@ -44,6 +47,50 @@ const CollapsibleBlock = ({ label, content }: { label: string; content: string } ); +type TtsState = 'idle' | 'loading' | 'playing'; + +const ReadAloudButton = ({ id, text }: { id: string; text: string }) => { + const [state, setState] = useState('idle'); + const audioRef = useRef(null); + const api = useFilesAPI('user-data'); + + const handleClick = async () => { + if (state === 'playing') { + audioRef.current?.pause(); + audioRef.current = null; + setState('idle'); + return; + } + if (state === 'loading') return; + + setState('loading'); + try { + const { audioPath, audioRoot } = await api.ttsText(id, text); + const url = getRawUrl(audioPath, audioRoot); + const audio = new Audio(url); + audioRef.current = audio; + audio.onended = () => { audioRef.current = null; setState('idle'); }; + audio.onerror = () => { audioRef.current = null; setState('idle'); }; + await audio.play(); + setState('playing'); + } catch { + setState('idle'); + } + }; + + return ( + + ); +}; + type MessageBubbleProps = { message: ChatMessage; onAnswer?: (text: string) => void; @@ -77,11 +124,16 @@ export const MessageBubble = ({ message, onAnswer }: MessageBubbleProps) => { if (!message.text) return null; return (
-
-
- - {injectImages(message.text)} - +
+
+
+ + {injectImages(message.text)} + +
+
+
+
diff --git a/src/workspaces/officerdev/src/apps/Chat/types.ts b/src/workspaces/officerdev/src/apps/Chat/types.ts index 7ab6489b..907889fb 100644 --- a/src/workspaces/officerdev/src/apps/Chat/types.ts +++ b/src/workspaces/officerdev/src/apps/Chat/types.ts @@ -41,7 +41,7 @@ export type GroupEntry = { export type ChatMessage = | { role: 'user'; text: string; images?: { filename: string; dataUrl: string }[] } - | { role: 'assistant'; text: string } + | { role: 'assistant'; id: string; text: string } | { role: 'system'; text: string } | { role: 'tool'; diff --git a/src/workspaces/officerdev/src/apps/FileBrowser/FileBrowserApp/components/TaskRunnerModal.tsx b/src/workspaces/officerdev/src/apps/FileBrowser/FileBrowserApp/components/TaskRunnerModal.tsx index 7ea3ec89..1e42263b 100644 --- a/src/workspaces/officerdev/src/apps/FileBrowser/FileBrowserApp/components/TaskRunnerModal.tsx +++ b/src/workspaces/officerdev/src/apps/FileBrowser/FileBrowserApp/components/TaskRunnerModal.tsx @@ -81,7 +81,7 @@ const PiMonoInner = ({ defaultInput, cwd, initialModel, taskInfo, sandboxed }: P const text = lastStreamRef.current; const isDuplicate = accRef.current.some((a) => a.role === 'assistant' && 'text' in a && a.text === text); if (!isDuplicate) { - accRef.current.push({ role: 'assistant', text }); + accRef.current.push({ role: 'assistant', id: crypto.randomUUID(), text }); bump((n) => n + 1); } lastStreamRef.current = ''; diff --git a/src/workspaces/officerdev/src/hooks/useFilesAPI.ts b/src/workspaces/officerdev/src/hooks/useFilesAPI.ts index a68cfbf5..66d74b8c 100644 --- a/src/workspaces/officerdev/src/hooks/useFilesAPI.ts +++ b/src/workspaces/officerdev/src/hooks/useFilesAPI.ts @@ -51,6 +51,9 @@ export const useFilesAPI = (root: string = 'home') => { tts: (path: string) => client.post<{ audioPath: string; audioRoot: string }>('/file-browser/tts', { path, root }), + ttsText: (id: string, text: string) => + client.post<{ audioPath: string; audioRoot: string }>('/file-browser/tts-text', { id, text }), + ocr: (path: string) => client.post<{ ocrPath: string; ocrRoot: string }>('/file-browser/ocr', { path, root }), diff --git a/src/workspaces/officerdev/src/hooks/usePiChat.ts b/src/workspaces/officerdev/src/hooks/usePiChat.ts index 383433fc..3e6b00f3 100644 --- a/src/workspaces/officerdev/src/hooks/usePiChat.ts +++ b/src/workspaces/officerdev/src/hooks/usePiChat.ts @@ -69,7 +69,7 @@ export function usePiChat(initialSessionId?: string, initialModel?: string | nul function commitStreaming() { if (!streamingRef.current) return; - setMessages((prev) => [...prev, { role: 'assistant', text: streamingRef.current }]); + setMessages((prev) => [...prev, { role: 'assistant', id: crypto.randomUUID(), text: streamingRef.current }]); streamingRef.current = ''; setStreamingText(''); } @@ -95,7 +95,7 @@ export function usePiChat(initialSessionId?: string, initialModel?: string | nul if (streamingRef.current) { commitStreaming(); } else { - setMessages((prev) => [...prev, { role: 'assistant', text: msg.text }]); + setMessages((prev) => [...prev, { role: 'assistant', id: crypto.randomUUID(), text: msg.text }]); } break; @@ -142,7 +142,7 @@ export function usePiChat(initialSessionId?: string, initialModel?: string | nul if (m.role === 'user') { return { role: 'user', text: m.text || '' }; } else if (m.role === 'assistant') { - return { role: 'assistant', text: m.text || '' }; + return { role: 'assistant', id: m.id ?? crypto.randomUUID(), text: m.text || '' }; } else if (m.role === 'tool') { return { role: 'tool', @@ -206,7 +206,7 @@ export function usePiChat(initialSessionId?: string, initialModel?: string | nul if (m.role === 'user') { return { role: 'user', text: m.text || '' }; } else if (m.role === 'assistant') { - return { role: 'assistant', text: m.text || '' }; + return { role: 'assistant', id: m.id ?? crypto.randomUUID(), text: m.text || '' }; } else if (m.role === 'tool') { return { role: 'tool', @@ -217,7 +217,7 @@ export function usePiChat(initialSessionId?: string, initialModel?: string | nul isError: m.isError, }; } - return { role: 'assistant', text: '' }; // Fallback + return { role: 'assistant', id: crypto.randomUUID(), text: '' }; // Fallback }); setMessages(chatMessages); setHasStarted(true);