chat whisper

src/apps/officer-web/Screens/Dashboard/Chat/InputArea.tsx

@@ -1,6 +1,7 @@
 import type { KeyboardEvent, RefObject } from 'react';
 import { useState, useRef } from 'react';
-import { FileText, Image, Link, Loader2, Paperclip, Send, Square, X } from 'lucide-react';
+import { FileText, Image, Link, Loader2, Mic, Paperclip, Send, Square, X } from 'lucide-react';
+import { toast } from 'sonner';
 import { Button } from '@/components/ui/button';
 import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogDescription } from '@/components/ui/dialog';
 import {
@@ -14,6 +15,42 @@ import type { ChatMessage } from 'widgets/Chat';
 import type { Attachment } from './EmbeddableChat';
 import { Settings } from './Settings';
 
+const blobToWav = async (blob: Blob): Promise<Blob> => {
+  const ctx = new AudioContext();
+  const buf = await ctx.decodeAudioData(await blob.arrayBuffer());
+  await ctx.close();
+
+  const samples = buf.getChannelData(0);
+  const len = samples.length;
+  const sr = buf.sampleRate;
+  const ab = new ArrayBuffer(44 + len * 2);
+  const v = new DataView(ab);
+
+  const s = (o: number, str: string) => {
+    for (let i = 0; i < str.length; i++) v.setUint8(o + i, str.charCodeAt(i));
+  };
+  s(0, 'RIFF');
+  v.setUint32(4, 36 + len * 2, true);
+  s(8, 'WAVE');
+  s(12, 'fmt ');
+  v.setUint32(16, 16, true);
+  v.setUint16(20, 1, true);
+  v.setUint16(22, 1, true);
+  v.setUint32(24, sr, true);
+  v.setUint32(28, sr * 2, true);
+  v.setUint16(32, 2, true);
+  v.setUint16(34, 16, true);
+  s(36, 'data');
+  v.setUint32(40, len * 2, true);
+
+  for (let i = 0; i < len; i++) {
+    const val = Math.max(-1, Math.min(1, samples[i]!));
+    v.setInt16(44 + i * 2, val < 0 ? val * 0x8000 : val * 0x7fff, true);
+  }
+
+  return new Blob([ab], { type: 'audio/wav' });
+};
+
 type InputAreaProps = {
   input: string;
   onInputChange: (value: string) => void;
@@ -61,7 +98,11 @@ export const InputArea = ({
 }: InputAreaProps) => {
   const [urlDialogOpen, setUrlDialogOpen] = useState(false);
   const [urlInput, setUrlInput] = useState('');
+  const [recording, setRecording] = useState(false);
+  const [transcribing, setTranscribing] = useState(false);
   const imageInputRef = useRef<HTMLInputElement>(null);
+  const mediaRecorderRef = useRef<MediaRecorder | null>(null);
+  const chunksRef = useRef<Blob[]>([]);
 
   const handleUrlSubmit = () => {
     const url = urlInput.trim();
@@ -71,8 +112,81 @@ export const InputArea = ({
     setUrlDialogOpen(false);
   };
 
+  const handleMicClick = async () => {
+    if (recording) {
+      const recorder = mediaRecorderRef.current;
+      if (!recorder) return;
+
+      setRecording(false);
+
+      try {
+        if (recorder.state === 'inactive') {
+          recorder.stream.getTracks().forEach((t) => t.stop());
+          return;
+        }
+
+        const blob = await new Promise<Blob>((resolve, reject) => {
+          const timeout = setTimeout(() => reject(new Error('Recording stop timed out')), 5000);
+          recorder.onstop = () => {
+            clearTimeout(timeout);
+            resolve(new Blob(chunksRef.current, { type: recorder.mimeType }));
+            chunksRef.current = [];
+          };
+          recorder.stop();
+        });
+
+        recorder.stream.getTracks().forEach((t) => t.stop());
+
+        if (blob.size === 0) {
+          toast.error('No audio was captured');
+          return;
+        }
+
+        setTranscribing(true);
+        try {
+          const wav = await blobToWav(blob);
+          const formData = new FormData();
+          formData.append('file', wav, 'recording.wav');
+          formData.append('temperature', '0.0');
+          formData.append('temperature_inc', '0.2');
+          formData.append('response_format', 'json');
+
+          const res = await fetch('http://macmini:8178/inference', { method: 'POST', body: formData });
+          if (!res.ok) throw new Error(`Whisper returned ${res.status}`);
+          const json = await res.json();
+          if (json.error) throw new Error(json.error);
+          const text = (json.text ?? '').trim();
+          if (text) onInputChange(input + (input.length > 0 ? ' ' : '') + text);
+        } finally {
+          setTranscribing(false);
+        }
+      } catch (err) {
+        recorder.stream?.getTracks().forEach((t) => t.stop());
+        chunksRef.current = [];
+        toast.error(err instanceof Error ? err.message : 'Recording failed');
+      }
+      return;
+    }
+
+    try {
+      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      const recorder = new MediaRecorder(stream);
+      mediaRecorderRef.current = recorder;
+      chunksRef.current = [];
+
+      recorder.ondataavailable = (ev) => {
+        if (ev.data.size > 0) chunksRef.current.push(ev.data);
+      };
+
+      recorder.start(250);
+      setRecording(true);
+    } catch (err) {
+      toast.error(err instanceof Error ? err.message : 'Could not access microphone');
+    }
+  };
+
   return (
-    <div className="shrink-0 border-t border-duck-dark/10 bg-white/60 p-3">
+    <div className="shrink-0 border-t border-duck-dark/10 bg-white/60 p-2 md:p-3">
       {commandFeedback && (
         <div className="mb-2 px-3 py-1.5 text-xs text-duck-teal bg-duck-teal/10 rounded-md">{commandFeedback}</div>
       )}
@@ -106,12 +220,12 @@ export const InputArea = ({
         </div>
       )}
 
-      <div className="flex items-end gap-2">
+      <div className="flex items-end gap-1 md:gap-2">
         <DropdownMenu>
           <DropdownMenuTrigger asChild>
             <button
               type="button"
-              className="shrink-0 h-9 w-9 flex items-center justify-center rounded-lg text-duck-dark/40 hover:text-duck-dark/70 hover:bg-duck-dark/5 transition-colors cursor-pointer"
+              className="shrink-0 h-7 w-7 md:h-9 md:w-9 flex items-center justify-center rounded-lg text-duck-dark/40 hover:text-duck-dark/70 hover:bg-duck-dark/5 transition-colors cursor-pointer"
             >
               <Paperclip className="h-4 w-4" />
             </button>
@@ -135,6 +249,23 @@ export const InputArea = ({
             </DropdownMenuItem>
           </DropdownMenuContent>
         </DropdownMenu>
+        <button
+          type="button"
+          disabled={transcribing}
+          onClick={handleMicClick}
+          className="relative shrink-0 h-7 w-7 md:h-9 md:w-9 flex items-center justify-center rounded-lg text-duck-dark/40 hover:text-duck-dark/70 hover:bg-duck-dark/5 transition-colors cursor-pointer disabled:opacity-40 disabled:cursor-not-allowed"
+        >
+          {transcribing ? (
+            <Loader2 className="h-4 w-4 animate-spin" />
+          ) : recording ? (
+            <>
+              <span className="absolute inset-0 rounded-lg animate-ping bg-red-400/30" />
+              <Square className="h-3.5 w-3.5 text-red-500" />
+            </>
+          ) : (
+            <Mic className="h-4 w-4" />
+          )}
+        </button>
         <input
           ref={imageInputRef}
           type="file"
@@ -165,10 +296,10 @@ export const InputArea = ({
           }}
           placeholder="Type a message..."
           rows={1}
-          className="flex-1 resize-none rounded-lg border border-duck-dark/20 bg-white/80 px-3 py-2 text-base md:text-sm text-duck-dark placeholder:text-duck-dark/30 focus:outline-none focus:ring-2 focus:ring-duck-teal/30 focus:border-duck-teal/50"
+          className="min-w-0 flex-1 resize-none rounded-lg border border-duck-dark/20 bg-white/80 px-2 py-1.5 md:px-3 md:py-2 text-base md:text-sm text-duck-dark placeholder:text-duck-dark/30 focus:outline-none focus:ring-2 focus:ring-duck-teal/30 focus:border-duck-teal/50"
         />
         {isGenerating ? (
-          <Button onClick={onStop} variant="destructive" size="icon" className="shrink-0 h-9 w-9 cursor-pointer">
+          <Button onClick={onStop} variant="destructive" size="icon" className="shrink-0 h-7 w-7 md:h-9 md:w-9 cursor-pointer">
             <Square className="h-4 w-4" />
           </Button>
         ) : (
@@ -176,7 +307,7 @@ export const InputArea = ({
             onClick={onSend}
             disabled={!input.trim() || !isConnected}
             size="icon"
-            className="shrink-0 h-9 w-9 bg-duck-teal hover:bg-duck-teal/90 cursor-pointer disabled:opacity-40"
+            className="shrink-0 h-7 w-7 md:h-9 md:w-9 bg-duck-teal hover:bg-duck-teal/90 cursor-pointer disabled:opacity-40"
           >
             <Send className="h-4 w-4" />
           </Button>

src/apps/officer-web/Screens/Dashboard/Files/Screen/FileViewer.tsx

@@ -9,6 +9,7 @@ import {
   FileType2,
   Maximize2,
   Minimize2,
+  Square,
   Play,
   Pause,
   Volume2,
@@ -28,6 +29,7 @@ import { cardStyle } from '@/components/Card';
 import { useFiles } from 'widgets/FileBrowser';
 import { getHeaders } from 'hooks/useClient';
 import { config } from 'config';
+import { toast } from 'sonner';
 import { getIcon } from 'material-file-icons';
 
 type FileViewerProps = {
@@ -922,6 +924,10 @@ export const FileViewer = ({ open, onOpenChange, filePath, fileName, root }: Fil
   const [loading, setLoading] = useState(false);
   const [error, setError] = useState<string | null>(null);
   const [expanded, setExpanded] = useState(false);
+  const [ttsLoading, setTtsLoading] = useState(false);
+  const [ttsPlaying, setTtsPlaying] = useState(false);
+  const ttsAudioRef = useRef<HTMLAudioElement | null>(null);
+  const ttsUrlRef = useRef<string | null>(null);
   const scrollRef = useRef<HTMLDivElement>(null);
   const files = useFiles(root);
   const fileType = getFileType(fileName);
@@ -931,6 +937,16 @@ export const FileViewer = ({ open, onOpenChange, filePath, fileName, root }: Fil
       setContent(null);
       setError(null);
       setExpanded(false);
+      if (ttsAudioRef.current) {
+        ttsAudioRef.current.pause();
+        ttsAudioRef.current = null;
+      }
+      if (ttsUrlRef.current) {
+        URL.revokeObjectURL(ttsUrlRef.current);
+        ttsUrlRef.current = null;
+      }
+      setTtsLoading(false);
+      setTtsPlaying(false);
       return;
     }
 
@@ -957,6 +973,54 @@ export const FileViewer = ({ open, onOpenChange, filePath, fileName, root }: Fil
     a.click();
   };
 
+  const handleReadAloud = async () => {
+    if (ttsPlaying) {
+      if (ttsAudioRef.current) {
+        ttsAudioRef.current.pause();
+        ttsAudioRef.current = null;
+      }
+      if (ttsUrlRef.current) {
+        URL.revokeObjectURL(ttsUrlRef.current);
+        ttsUrlRef.current = null;
+      }
+      setTtsPlaying(false);
+      return;
+    }
+
+    setTtsLoading(true);
+    try {
+      const res = await fetch('http://alpha:9051/v1/audio/speech', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ model: 'kokoro', input: content, voice: 'af_heart', response_format: 'mp3' }),
+      });
+      if (!res.ok) throw new Error('TTS request failed');
+      const blob = await res.blob();
+      const url = URL.createObjectURL(blob);
+      ttsUrlRef.current = url;
+      const audio = new Audio(url);
+      ttsAudioRef.current = audio;
+      audio.addEventListener('ended', () => {
+        setTtsPlaying(false);
+        ttsAudioRef.current = null;
+        URL.revokeObjectURL(url);
+        ttsUrlRef.current = null;
+      });
+      await audio.play();
+      setTtsPlaying(true);
+    } catch {
+      toast.error('Failed to generate speech audio');
+      if (ttsUrlRef.current) {
+        URL.revokeObjectURL(ttsUrlRef.current);
+        ttsUrlRef.current = null;
+      }
+      ttsAudioRef.current = null;
+      setTtsPlaying(false);
+    } finally {
+      setTtsLoading(false);
+    }
+  };
+
   const headerIcon =
     fileType === 'audio' ? (
       <Music className="h-4 w-4 text-duck-teal shrink-0" />
@@ -1019,6 +1083,22 @@ export const FileViewer = ({ open, onOpenChange, filePath, fileName, root }: Fil
             >
               {fileType === 'code' ? getLang(fileName) : fileType}
             </span>
+            {(fileType === 'markdown' || fileType === 'code' || fileType === 'text') && content && (
+              <button
+                onClick={handleReadAloud}
+                disabled={ttsLoading}
+                className="p-1.5 rounded-md text-duck-dark/40 hover:text-duck-dark hover:bg-duck-dark/5 disabled:opacity-40 transition-colors cursor-pointer"
+                title="Read Aloud"
+              >
+                {ttsLoading ? (
+                  <Loader2 className="h-4 w-4 animate-spin" />
+                ) : ttsPlaying ? (
+                  <Square className="h-4 w-4" />
+                ) : (
+                  <Volume2 className="h-4 w-4" />
+                )}
+              </button>
+            )}
             <button
               onClick={handleDownload}
               className={`p-1.5 rounded-md transition-colors cursor-pointer ${