read alout chat messages
This commit is contained in:
@@ -414,6 +414,51 @@ router.post('/tts', async (ctx) => {
|
||||
return ctx.json({ audioPath: cacheRel, audioRoot: 'user-data' });
|
||||
});
|
||||
|
||||
// Text-to-speech from raw text with caching
|
||||
router.post('/tts-text', async (ctx) => {
|
||||
const user = ctx.get('user');
|
||||
const { text, id } = ctx.get('body') as { text: string; id: string };
|
||||
if (!text) throw errors.BAD_REQUEST('text is required');
|
||||
if (!id) throw errors.BAD_REQUEST('id is required');
|
||||
|
||||
const userDataDir = getUserDataDir(user.email);
|
||||
const cacheRel = `cache/tts/chat/${id}.mp3`;
|
||||
const cacheAbs = resolve(userDataDir, cacheRel);
|
||||
|
||||
if (existsSync(cacheAbs)) {
|
||||
return ctx.json({ audioPath: cacheRel, audioRoot: 'user-data' });
|
||||
}
|
||||
|
||||
const ttsConfig = await readTtsConfig();
|
||||
if (!ttsConfig) throw errors.BAD_REQUEST('TTS not configured — set it up in Settings → Text to Speech');
|
||||
|
||||
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
||||
let res: Response;
|
||||
|
||||
if (ttsConfig.provider === 'elevenlabs') {
|
||||
if (!ttsConfig.apiKey) throw errors.BAD_REQUEST('ElevenLabs API key not configured');
|
||||
res = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${encodeURIComponent(ttsConfig.voice)}`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'xi-api-key': ttsConfig.apiKey },
|
||||
body: JSON.stringify({ text, model_id: ttsConfig.model }),
|
||||
});
|
||||
} else {
|
||||
if (ttsConfig.apiKey) headers['Authorization'] = `Bearer ${ttsConfig.apiKey}`;
|
||||
res = await fetch(`${ttsConfig.url.replace(/\/+$/, '')}/v1/audio/speech`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({ model: ttsConfig.model, input: text, voice: ttsConfig.voice, response_format: 'mp3' }),
|
||||
});
|
||||
}
|
||||
if (!res.ok) throw errors.BAD_REQUEST('TTS request failed');
|
||||
|
||||
await mkdir(dirname(cacheAbs), { recursive: true });
|
||||
const buffer = await res.arrayBuffer();
|
||||
await Bun.write(cacheAbs, buffer);
|
||||
|
||||
return ctx.json({ audioPath: cacheRel, audioRoot: 'user-data' });
|
||||
});
|
||||
|
||||
// OCR image via vision model with caching
|
||||
router.post('/ocr', async (ctx) => {
|
||||
const user = ctx.get('user');
|
||||
|
||||
@@ -15,7 +15,7 @@ const Email = ({ invitedBy, url }: EmailProps) => {
|
||||
<Container className="rounded-lg bg-white p-8 shadow-lg">
|
||||
<Img
|
||||
className="mx-auto block"
|
||||
src="https://static.officer.dev/og-image.png"
|
||||
src="https://static.officer.dev/og-image.jpg"
|
||||
width="480"
|
||||
alt="officer.dev"
|
||||
/>
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
import { useState, useRef } from 'react';
|
||||
import ReactMarkdown from 'react-markdown';
|
||||
import remarkGfm from 'remark-gfm';
|
||||
import rehypeRaw from 'rehype-raw';
|
||||
import { Volume2, Loader2, Square } from 'lucide-react';
|
||||
import type { ChatMessage } from '../types';
|
||||
import { ToolActivity } from './ToolActivity';
|
||||
import { QuestionActivity } from './QuestionActivity';
|
||||
import { getRawUrl } from '../../FileViewer/file-types';
|
||||
import { useFilesAPI } from '../../../hooks/useFilesAPI';
|
||||
|
||||
// Matches absolute image file paths, e.g. /home/user/pic.png or /tmp/photo.jpg
|
||||
const IMAGE_PATH_RE = /(\/(?:home\/[^/\s]+\/)?[^\s`"'<>\n\r[\]()]+\.(?:png|jpg|jpeg|gif|webp|svg|bmp|ico))/gi;
|
||||
@@ -44,6 +47,50 @@ const CollapsibleBlock = ({ label, content }: { label: string; content: string }
|
||||
</div>
|
||||
);
|
||||
|
||||
type TtsState = 'idle' | 'loading' | 'playing';
|
||||
|
||||
const ReadAloudButton = ({ id, text }: { id: string; text: string }) => {
|
||||
const [state, setState] = useState<TtsState>('idle');
|
||||
const audioRef = useRef<HTMLAudioElement | null>(null);
|
||||
const api = useFilesAPI('user-data');
|
||||
|
||||
const handleClick = async () => {
|
||||
if (state === 'playing') {
|
||||
audioRef.current?.pause();
|
||||
audioRef.current = null;
|
||||
setState('idle');
|
||||
return;
|
||||
}
|
||||
if (state === 'loading') return;
|
||||
|
||||
setState('loading');
|
||||
try {
|
||||
const { audioPath, audioRoot } = await api.ttsText(id, text);
|
||||
const url = getRawUrl(audioPath, audioRoot);
|
||||
const audio = new Audio(url);
|
||||
audioRef.current = audio;
|
||||
audio.onended = () => { audioRef.current = null; setState('idle'); };
|
||||
audio.onerror = () => { audioRef.current = null; setState('idle'); };
|
||||
await audio.play();
|
||||
setState('playing');
|
||||
} catch {
|
||||
setState('idle');
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleClick}
|
||||
className="p-1 rounded text-duck-dark/30 hover:text-duck-dark/60 transition-colors cursor-pointer"
|
||||
>
|
||||
{state === 'loading' && <Loader2 className="h-3.5 w-3.5 animate-spin" />}
|
||||
{state === 'playing' && <Square className="h-3.5 w-3.5" />}
|
||||
{state === 'idle' && <Volume2 className="h-3.5 w-3.5" />}
|
||||
</button>
|
||||
);
|
||||
};
|
||||
|
||||
type MessageBubbleProps = {
|
||||
message: ChatMessage;
|
||||
onAnswer?: (text: string) => void;
|
||||
@@ -77,11 +124,16 @@ export const MessageBubble = ({ message, onAnswer }: MessageBubbleProps) => {
|
||||
if (!message.text) return null;
|
||||
return (
|
||||
<div className="flex justify-start">
|
||||
<div className="max-w-[85%] rounded-2xl rounded-tl-sm bg-muted/50 border border-border/50 px-4 py-2.5">
|
||||
<div className="chat-md">
|
||||
<ReactMarkdown remarkPlugins={[remarkGfm]} rehypePlugins={[rehypeRaw]}>
|
||||
{injectImages(message.text)}
|
||||
</ReactMarkdown>
|
||||
<div className="max-w-[85%]">
|
||||
<div className="rounded-2xl rounded-tl-sm bg-muted/50 border border-border/50 px-4 py-2.5">
|
||||
<div className="chat-md">
|
||||
<ReactMarkdown remarkPlugins={[remarkGfm]} rehypePlugins={[rehypeRaw]}>
|
||||
{injectImages(message.text)}
|
||||
</ReactMarkdown>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex justify-end mt-0.5">
|
||||
<ReadAloudButton id={message.id} text={message.text} />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -41,7 +41,7 @@ export type GroupEntry = {
|
||||
|
||||
export type ChatMessage =
|
||||
| { role: 'user'; text: string; images?: { filename: string; dataUrl: string }[] }
|
||||
| { role: 'assistant'; text: string }
|
||||
| { role: 'assistant'; id: string; text: string }
|
||||
| { role: 'system'; text: string }
|
||||
| {
|
||||
role: 'tool';
|
||||
|
||||
+1
-1
@@ -81,7 +81,7 @@ const PiMonoInner = ({ defaultInput, cwd, initialModel, taskInfo, sandboxed }: P
|
||||
const text = lastStreamRef.current;
|
||||
const isDuplicate = accRef.current.some((a) => a.role === 'assistant' && 'text' in a && a.text === text);
|
||||
if (!isDuplicate) {
|
||||
accRef.current.push({ role: 'assistant', text });
|
||||
accRef.current.push({ role: 'assistant', id: crypto.randomUUID(), text });
|
||||
bump((n) => n + 1);
|
||||
}
|
||||
lastStreamRef.current = '';
|
||||
|
||||
@@ -51,6 +51,9 @@ export const useFilesAPI = (root: string = 'home') => {
|
||||
tts: (path: string) =>
|
||||
client.post<{ audioPath: string; audioRoot: string }>('/file-browser/tts', { path, root }),
|
||||
|
||||
ttsText: (id: string, text: string) =>
|
||||
client.post<{ audioPath: string; audioRoot: string }>('/file-browser/tts-text', { id, text }),
|
||||
|
||||
ocr: (path: string) =>
|
||||
client.post<{ ocrPath: string; ocrRoot: string }>('/file-browser/ocr', { path, root }),
|
||||
|
||||
|
||||
@@ -69,7 +69,7 @@ export function usePiChat(initialSessionId?: string, initialModel?: string | nul
|
||||
|
||||
function commitStreaming() {
|
||||
if (!streamingRef.current) return;
|
||||
setMessages((prev) => [...prev, { role: 'assistant', text: streamingRef.current }]);
|
||||
setMessages((prev) => [...prev, { role: 'assistant', id: crypto.randomUUID(), text: streamingRef.current }]);
|
||||
streamingRef.current = '';
|
||||
setStreamingText('');
|
||||
}
|
||||
@@ -95,7 +95,7 @@ export function usePiChat(initialSessionId?: string, initialModel?: string | nul
|
||||
if (streamingRef.current) {
|
||||
commitStreaming();
|
||||
} else {
|
||||
setMessages((prev) => [...prev, { role: 'assistant', text: msg.text }]);
|
||||
setMessages((prev) => [...prev, { role: 'assistant', id: crypto.randomUUID(), text: msg.text }]);
|
||||
}
|
||||
break;
|
||||
|
||||
@@ -142,7 +142,7 @@ export function usePiChat(initialSessionId?: string, initialModel?: string | nul
|
||||
if (m.role === 'user') {
|
||||
return { role: 'user', text: m.text || '' };
|
||||
} else if (m.role === 'assistant') {
|
||||
return { role: 'assistant', text: m.text || '' };
|
||||
return { role: 'assistant', id: m.id ?? crypto.randomUUID(), text: m.text || '' };
|
||||
} else if (m.role === 'tool') {
|
||||
return {
|
||||
role: 'tool',
|
||||
@@ -206,7 +206,7 @@ export function usePiChat(initialSessionId?: string, initialModel?: string | nul
|
||||
if (m.role === 'user') {
|
||||
return { role: 'user', text: m.text || '' };
|
||||
} else if (m.role === 'assistant') {
|
||||
return { role: 'assistant', text: m.text || '' };
|
||||
return { role: 'assistant', id: m.id ?? crypto.randomUUID(), text: m.text || '' };
|
||||
} else if (m.role === 'tool') {
|
||||
return {
|
||||
role: 'tool',
|
||||
@@ -217,7 +217,7 @@ export function usePiChat(initialSessionId?: string, initialModel?: string | nul
|
||||
isError: m.isError,
|
||||
};
|
||||
}
|
||||
return { role: 'assistant', text: '' }; // Fallback
|
||||
return { role: 'assistant', id: crypto.randomUUID(), text: '' }; // Fallback
|
||||
});
|
||||
setMessages(chatMessages);
|
||||
setHasStarted(true);
|
||||
|
||||
Reference in New Issue
Block a user