From fee9682fcaa071b46ee5e48d5f3eb5d90c2b67dc Mon Sep 17 00:00:00 2001 From: Andre Padez Date: Wed, 18 Feb 2026 03:17:21 +0000 Subject: [PATCH] chat whisper --- .../Screens/Dashboard/Chat/InputArea.tsx | 145 +++++++++++++++++- .../Dashboard/Files/Screen/FileViewer.tsx | 80 ++++++++++ 2 files changed, 218 insertions(+), 7 deletions(-) diff --git a/src/apps/officer-web/Screens/Dashboard/Chat/InputArea.tsx b/src/apps/officer-web/Screens/Dashboard/Chat/InputArea.tsx index b11b649b..c4495c81 100644 --- a/src/apps/officer-web/Screens/Dashboard/Chat/InputArea.tsx +++ b/src/apps/officer-web/Screens/Dashboard/Chat/InputArea.tsx @@ -1,6 +1,7 @@ import type { KeyboardEvent, RefObject } from 'react'; import { useState, useRef } from 'react'; -import { FileText, Image, Link, Loader2, Paperclip, Send, Square, X } from 'lucide-react'; +import { FileText, Image, Link, Loader2, Mic, Paperclip, Send, Square, X } from 'lucide-react'; +import { toast } from 'sonner'; import { Button } from '@/components/ui/button'; import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogDescription } from '@/components/ui/dialog'; import { @@ -14,6 +15,42 @@ import type { ChatMessage } from 'widgets/Chat'; import type { Attachment } from './EmbeddableChat'; import { Settings } from './Settings'; +const blobToWav = async (blob: Blob): Promise => { + const ctx = new AudioContext(); + const buf = await ctx.decodeAudioData(await blob.arrayBuffer()); + await ctx.close(); + + const samples = buf.getChannelData(0); + const len = samples.length; + const sr = buf.sampleRate; + const ab = new ArrayBuffer(44 + len * 2); + const v = new DataView(ab); + + const s = (o: number, str: string) => { + for (let i = 0; i < str.length; i++) v.setUint8(o + i, str.charCodeAt(i)); + }; + s(0, 'RIFF'); + v.setUint32(4, 36 + len * 2, true); + s(8, 'WAVE'); + s(12, 'fmt '); + v.setUint32(16, 16, true); + v.setUint16(20, 1, true); + v.setUint16(22, 1, true); + v.setUint32(24, sr, true); + v.setUint32(28, sr * 2, true); + v.setUint16(32, 2, true); + v.setUint16(34, 16, true); + s(36, 'data'); + v.setUint32(40, len * 2, true); + + for (let i = 0; i < len; i++) { + const val = Math.max(-1, Math.min(1, samples[i]!)); + v.setInt16(44 + i * 2, val < 0 ? val * 0x8000 : val * 0x7fff, true); + } + + return new Blob([ab], { type: 'audio/wav' }); +}; + type InputAreaProps = { input: string; onInputChange: (value: string) => void; @@ -61,7 +98,11 @@ export const InputArea = ({ }: InputAreaProps) => { const [urlDialogOpen, setUrlDialogOpen] = useState(false); const [urlInput, setUrlInput] = useState(''); + const [recording, setRecording] = useState(false); + const [transcribing, setTranscribing] = useState(false); const imageInputRef = useRef(null); + const mediaRecorderRef = useRef(null); + const chunksRef = useRef([]); const handleUrlSubmit = () => { const url = urlInput.trim(); @@ -71,8 +112,81 @@ export const InputArea = ({ setUrlDialogOpen(false); }; + const handleMicClick = async () => { + if (recording) { + const recorder = mediaRecorderRef.current; + if (!recorder) return; + + setRecording(false); + + try { + if (recorder.state === 'inactive') { + recorder.stream.getTracks().forEach((t) => t.stop()); + return; + } + + const blob = await new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error('Recording stop timed out')), 5000); + recorder.onstop = () => { + clearTimeout(timeout); + resolve(new Blob(chunksRef.current, { type: recorder.mimeType })); + chunksRef.current = []; + }; + recorder.stop(); + }); + + recorder.stream.getTracks().forEach((t) => t.stop()); + + if (blob.size === 0) { + toast.error('No audio was captured'); + return; + } + + setTranscribing(true); + try { + const wav = await blobToWav(blob); + const formData = new FormData(); + formData.append('file', wav, 'recording.wav'); + formData.append('temperature', '0.0'); + formData.append('temperature_inc', '0.2'); + formData.append('response_format', 'json'); + + const res = await fetch('http://macmini:8178/inference', { method: 'POST', body: formData }); + if (!res.ok) throw new Error(`Whisper returned ${res.status}`); + const json = await res.json(); + if (json.error) throw new Error(json.error); + const text = (json.text ?? '').trim(); + if (text) onInputChange(input + (input.length > 0 ? ' ' : '') + text); + } finally { + setTranscribing(false); + } + } catch (err) { + recorder.stream?.getTracks().forEach((t) => t.stop()); + chunksRef.current = []; + toast.error(err instanceof Error ? err.message : 'Recording failed'); + } + return; + } + + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const recorder = new MediaRecorder(stream); + mediaRecorderRef.current = recorder; + chunksRef.current = []; + + recorder.ondataavailable = (ev) => { + if (ev.data.size > 0) chunksRef.current.push(ev.data); + }; + + recorder.start(250); + setRecording(true); + } catch (err) { + toast.error(err instanceof Error ? err.message : 'Could not access microphone'); + } + }; + return ( -
+
{commandFeedback && (
{commandFeedback}
)} @@ -106,12 +220,12 @@ export const InputArea = ({
)} -
+
@@ -135,6 +249,23 @@ export const InputArea = ({ + {isGenerating ? ( - ) : ( @@ -176,7 +307,7 @@ export const InputArea = ({ onClick={onSend} disabled={!input.trim() || !isConnected} size="icon" - className="shrink-0 h-9 w-9 bg-duck-teal hover:bg-duck-teal/90 cursor-pointer disabled:opacity-40" + className="shrink-0 h-7 w-7 md:h-9 md:w-9 bg-duck-teal hover:bg-duck-teal/90 cursor-pointer disabled:opacity-40" > diff --git a/src/apps/officer-web/Screens/Dashboard/Files/Screen/FileViewer.tsx b/src/apps/officer-web/Screens/Dashboard/Files/Screen/FileViewer.tsx index a686de71..e361d8ef 100644 --- a/src/apps/officer-web/Screens/Dashboard/Files/Screen/FileViewer.tsx +++ b/src/apps/officer-web/Screens/Dashboard/Files/Screen/FileViewer.tsx @@ -9,6 +9,7 @@ import { FileType2, Maximize2, Minimize2, + Square, Play, Pause, Volume2, @@ -28,6 +29,7 @@ import { cardStyle } from '@/components/Card'; import { useFiles } from 'widgets/FileBrowser'; import { getHeaders } from 'hooks/useClient'; import { config } from 'config'; +import { toast } from 'sonner'; import { getIcon } from 'material-file-icons'; type FileViewerProps = { @@ -922,6 +924,10 @@ export const FileViewer = ({ open, onOpenChange, filePath, fileName, root }: Fil const [loading, setLoading] = useState(false); const [error, setError] = useState(null); const [expanded, setExpanded] = useState(false); + const [ttsLoading, setTtsLoading] = useState(false); + const [ttsPlaying, setTtsPlaying] = useState(false); + const ttsAudioRef = useRef(null); + const ttsUrlRef = useRef(null); const scrollRef = useRef(null); const files = useFiles(root); const fileType = getFileType(fileName); @@ -931,6 +937,16 @@ export const FileViewer = ({ open, onOpenChange, filePath, fileName, root }: Fil setContent(null); setError(null); setExpanded(false); + if (ttsAudioRef.current) { + ttsAudioRef.current.pause(); + ttsAudioRef.current = null; + } + if (ttsUrlRef.current) { + URL.revokeObjectURL(ttsUrlRef.current); + ttsUrlRef.current = null; + } + setTtsLoading(false); + setTtsPlaying(false); return; } @@ -957,6 +973,54 @@ export const FileViewer = ({ open, onOpenChange, filePath, fileName, root }: Fil a.click(); }; + const handleReadAloud = async () => { + if (ttsPlaying) { + if (ttsAudioRef.current) { + ttsAudioRef.current.pause(); + ttsAudioRef.current = null; + } + if (ttsUrlRef.current) { + URL.revokeObjectURL(ttsUrlRef.current); + ttsUrlRef.current = null; + } + setTtsPlaying(false); + return; + } + + setTtsLoading(true); + try { + const res = await fetch('http://alpha:9051/v1/audio/speech', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ model: 'kokoro', input: content, voice: 'af_heart', response_format: 'mp3' }), + }); + if (!res.ok) throw new Error('TTS request failed'); + const blob = await res.blob(); + const url = URL.createObjectURL(blob); + ttsUrlRef.current = url; + const audio = new Audio(url); + ttsAudioRef.current = audio; + audio.addEventListener('ended', () => { + setTtsPlaying(false); + ttsAudioRef.current = null; + URL.revokeObjectURL(url); + ttsUrlRef.current = null; + }); + await audio.play(); + setTtsPlaying(true); + } catch { + toast.error('Failed to generate speech audio'); + if (ttsUrlRef.current) { + URL.revokeObjectURL(ttsUrlRef.current); + ttsUrlRef.current = null; + } + ttsAudioRef.current = null; + setTtsPlaying(false); + } finally { + setTtsLoading(false); + } + }; + const headerIcon = fileType === 'audio' ? ( @@ -1019,6 +1083,22 @@ export const FileViewer = ({ open, onOpenChange, filePath, fileName, root }: Fil > {fileType === 'code' ? getLang(fileName) : fileType} + {(fileType === 'markdown' || fileType === 'code' || fileType === 'text') && content && ( + + )}