read alout chat messages

This commit is contained in:
2026-02-25 05:11:09 +00:00
parent 0720f25ed5
commit 7bdeecd7f0
7 changed files with 113 additions and 13 deletions
+45
View File
@@ -414,6 +414,51 @@ router.post('/tts', async (ctx) => {
return ctx.json({ audioPath: cacheRel, audioRoot: 'user-data' });
});
// Text-to-speech from raw text with caching
router.post('/tts-text', async (ctx) => {
const user = ctx.get('user');
const { text, id } = ctx.get('body') as { text: string; id: string };
if (!text) throw errors.BAD_REQUEST('text is required');
if (!id) throw errors.BAD_REQUEST('id is required');
const userDataDir = getUserDataDir(user.email);
const cacheRel = `cache/tts/chat/${id}.mp3`;
const cacheAbs = resolve(userDataDir, cacheRel);
if (existsSync(cacheAbs)) {
return ctx.json({ audioPath: cacheRel, audioRoot: 'user-data' });
}
const ttsConfig = await readTtsConfig();
if (!ttsConfig) throw errors.BAD_REQUEST('TTS not configured — set it up in Settings → Text to Speech');
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
let res: Response;
if (ttsConfig.provider === 'elevenlabs') {
if (!ttsConfig.apiKey) throw errors.BAD_REQUEST('ElevenLabs API key not configured');
res = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${encodeURIComponent(ttsConfig.voice)}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'xi-api-key': ttsConfig.apiKey },
body: JSON.stringify({ text, model_id: ttsConfig.model }),
});
} else {
if (ttsConfig.apiKey) headers['Authorization'] = `Bearer ${ttsConfig.apiKey}`;
res = await fetch(`${ttsConfig.url.replace(/\/+$/, '')}/v1/audio/speech`, {
method: 'POST',
headers,
body: JSON.stringify({ model: ttsConfig.model, input: text, voice: ttsConfig.voice, response_format: 'mp3' }),
});
}
if (!res.ok) throw errors.BAD_REQUEST('TTS request failed');
await mkdir(dirname(cacheAbs), { recursive: true });
const buffer = await res.arrayBuffer();
await Bun.write(cacheAbs, buffer);
return ctx.json({ audioPath: cacheRel, audioRoot: 'user-data' });
});
// OCR image via vision model with caching
router.post('/ocr', async (ctx) => {
const user = ctx.get('user');
+1 -1
View File
@@ -15,7 +15,7 @@ const Email = ({ invitedBy, url }: EmailProps) => {
<Container className="rounded-lg bg-white p-8 shadow-lg">
<Img
className="mx-auto block"
src="https://static.officer.dev/og-image.png"
src="https://static.officer.dev/og-image.jpg"
width="480"
alt="officer.dev"
/>
@@ -1,10 +1,13 @@
import { useState, useRef } from 'react';
import ReactMarkdown from 'react-markdown';
import remarkGfm from 'remark-gfm';
import rehypeRaw from 'rehype-raw';
import { Volume2, Loader2, Square } from 'lucide-react';
import type { ChatMessage } from '../types';
import { ToolActivity } from './ToolActivity';
import { QuestionActivity } from './QuestionActivity';
import { getRawUrl } from '../../FileViewer/file-types';
import { useFilesAPI } from '../../../hooks/useFilesAPI';
// Matches absolute image file paths, e.g. /home/user/pic.png or /tmp/photo.jpg
const IMAGE_PATH_RE = /(\/(?:home\/[^/\s]+\/)?[^\s`"'<>\n\r[\]()]+\.(?:png|jpg|jpeg|gif|webp|svg|bmp|ico))/gi;
@@ -44,6 +47,50 @@ const CollapsibleBlock = ({ label, content }: { label: string; content: string }
</div>
);
type TtsState = 'idle' | 'loading' | 'playing';
const ReadAloudButton = ({ id, text }: { id: string; text: string }) => {
const [state, setState] = useState<TtsState>('idle');
const audioRef = useRef<HTMLAudioElement | null>(null);
const api = useFilesAPI('user-data');
const handleClick = async () => {
if (state === 'playing') {
audioRef.current?.pause();
audioRef.current = null;
setState('idle');
return;
}
if (state === 'loading') return;
setState('loading');
try {
const { audioPath, audioRoot } = await api.ttsText(id, text);
const url = getRawUrl(audioPath, audioRoot);
const audio = new Audio(url);
audioRef.current = audio;
audio.onended = () => { audioRef.current = null; setState('idle'); };
audio.onerror = () => { audioRef.current = null; setState('idle'); };
await audio.play();
setState('playing');
} catch {
setState('idle');
}
};
return (
<button
type="button"
onClick={handleClick}
className="p-1 rounded text-duck-dark/30 hover:text-duck-dark/60 transition-colors cursor-pointer"
>
{state === 'loading' && <Loader2 className="h-3.5 w-3.5 animate-spin" />}
{state === 'playing' && <Square className="h-3.5 w-3.5" />}
{state === 'idle' && <Volume2 className="h-3.5 w-3.5" />}
</button>
);
};
type MessageBubbleProps = {
message: ChatMessage;
onAnswer?: (text: string) => void;
@@ -77,11 +124,16 @@ export const MessageBubble = ({ message, onAnswer }: MessageBubbleProps) => {
if (!message.text) return null;
return (
<div className="flex justify-start">
<div className="max-w-[85%] rounded-2xl rounded-tl-sm bg-muted/50 border border-border/50 px-4 py-2.5">
<div className="chat-md">
<ReactMarkdown remarkPlugins={[remarkGfm]} rehypePlugins={[rehypeRaw]}>
{injectImages(message.text)}
</ReactMarkdown>
<div className="max-w-[85%]">
<div className="rounded-2xl rounded-tl-sm bg-muted/50 border border-border/50 px-4 py-2.5">
<div className="chat-md">
<ReactMarkdown remarkPlugins={[remarkGfm]} rehypePlugins={[rehypeRaw]}>
{injectImages(message.text)}
</ReactMarkdown>
</div>
</div>
<div className="flex justify-end mt-0.5">
<ReadAloudButton id={message.id} text={message.text} />
</div>
</div>
</div>
@@ -41,7 +41,7 @@ export type GroupEntry = {
export type ChatMessage =
| { role: 'user'; text: string; images?: { filename: string; dataUrl: string }[] }
| { role: 'assistant'; text: string }
| { role: 'assistant'; id: string; text: string }
| { role: 'system'; text: string }
| {
role: 'tool';
@@ -81,7 +81,7 @@ const PiMonoInner = ({ defaultInput, cwd, initialModel, taskInfo, sandboxed }: P
const text = lastStreamRef.current;
const isDuplicate = accRef.current.some((a) => a.role === 'assistant' && 'text' in a && a.text === text);
if (!isDuplicate) {
accRef.current.push({ role: 'assistant', text });
accRef.current.push({ role: 'assistant', id: crypto.randomUUID(), text });
bump((n) => n + 1);
}
lastStreamRef.current = '';
@@ -51,6 +51,9 @@ export const useFilesAPI = (root: string = 'home') => {
tts: (path: string) =>
client.post<{ audioPath: string; audioRoot: string }>('/file-browser/tts', { path, root }),
ttsText: (id: string, text: string) =>
client.post<{ audioPath: string; audioRoot: string }>('/file-browser/tts-text', { id, text }),
ocr: (path: string) =>
client.post<{ ocrPath: string; ocrRoot: string }>('/file-browser/ocr', { path, root }),
@@ -69,7 +69,7 @@ export function usePiChat(initialSessionId?: string, initialModel?: string | nul
function commitStreaming() {
if (!streamingRef.current) return;
setMessages((prev) => [...prev, { role: 'assistant', text: streamingRef.current }]);
setMessages((prev) => [...prev, { role: 'assistant', id: crypto.randomUUID(), text: streamingRef.current }]);
streamingRef.current = '';
setStreamingText('');
}
@@ -95,7 +95,7 @@ export function usePiChat(initialSessionId?: string, initialModel?: string | nul
if (streamingRef.current) {
commitStreaming();
} else {
setMessages((prev) => [...prev, { role: 'assistant', text: msg.text }]);
setMessages((prev) => [...prev, { role: 'assistant', id: crypto.randomUUID(), text: msg.text }]);
}
break;
@@ -142,7 +142,7 @@ export function usePiChat(initialSessionId?: string, initialModel?: string | nul
if (m.role === 'user') {
return { role: 'user', text: m.text || '' };
} else if (m.role === 'assistant') {
return { role: 'assistant', text: m.text || '' };
return { role: 'assistant', id: m.id ?? crypto.randomUUID(), text: m.text || '' };
} else if (m.role === 'tool') {
return {
role: 'tool',
@@ -206,7 +206,7 @@ export function usePiChat(initialSessionId?: string, initialModel?: string | nul
if (m.role === 'user') {
return { role: 'user', text: m.text || '' };
} else if (m.role === 'assistant') {
return { role: 'assistant', text: m.text || '' };
return { role: 'assistant', id: m.id ?? crypto.randomUUID(), text: m.text || '' };
} else if (m.role === 'tool') {
return {
role: 'tool',
@@ -217,7 +217,7 @@ export function usePiChat(initialSessionId?: string, initialModel?: string | nul
isError: m.isError,
};
}
return { role: 'assistant', text: '' }; // Fallback
return { role: 'assistant', id: crypto.randomUUID(), text: '' }; // Fallback
});
setMessages(chatMessages);
setHasStarted(true);