officerdev/platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chat whisper

Browse Source
This commit is contained in:
pastilhas
2026-02-18 03:17:21 +00:00
parent 826904d74f
commit fee9682fca
2 changed files with 218 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/apps/officer-web/Screens/Dashboard/Chat/InputArea.tsx
+138 -7
View File
@@ -1,6 +1,7 @@
import type { KeyboardEvent, RefObject } from 'react';
import { useState, useRef } from 'react';
import { FileText, Image, Link, Loader2, Paperclip, Send, Square, X } from 'lucide-react';
import { FileText, Image, Link, Loader2, Mic, Paperclip, Send, Square, X } from 'lucide-react';
import { toast } from 'sonner';
import { Button } from '@/components/ui/button';
import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogDescription } from '@/components/ui/dialog';
import {
@@ -14,6 +15,42 @@ import type { ChatMessage } from 'widgets/Chat';
import type { Attachment } from './EmbeddableChat';
import { Settings } from './Settings';
const blobToWav = async (blob: Blob): Promise<Blob> => {
const ctx = new AudioContext();
const buf = await ctx.decodeAudioData(await blob.arrayBuffer());
await ctx.close();
const samples = buf.getChannelData(0);
const len = samples.length;
const sr = buf.sampleRate;
const ab = new ArrayBuffer(44 + len * 2);
const v = new DataView(ab);
const s = (o: number, str: string) => {
for (let i = 0; i < str.length; i++) v.setUint8(o + i, str.charCodeAt(i));
};
s(0, 'RIFF');
v.setUint32(4, 36 + len * 2, true);
s(8, 'WAVE');
s(12, 'fmt ');
v.setUint32(16, 16, true);
v.setUint16(20, 1, true);
v.setUint16(22, 1, true);
v.setUint32(24, sr, true);
v.setUint32(28, sr * 2, true);
v.setUint16(32, 2, true);
v.setUint16(34, 16, true);
s(36, 'data');
v.setUint32(40, len * 2, true);
for (let i = 0; i < len; i++) {
const val = Math.max(-1, Math.min(1, samples[i]!));
v.setInt16(44 + i * 2, val < 0 ? val * 0x8000 : val * 0x7fff, true);
}
return new Blob([ab], { type: 'audio/wav' });
};
type InputAreaProps = {
input: string;
onInputChange: (value: string) => void;
@@ -61,7 +98,11 @@ export const InputArea = ({
}: InputAreaProps) => {
const [urlDialogOpen, setUrlDialogOpen] = useState(false);
const [urlInput, setUrlInput] = useState('');
const [recording, setRecording] = useState(false);
const [transcribing, setTranscribing] = useState(false);
const imageInputRef = useRef<HTMLInputElement>(null);
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
const chunksRef = useRef<Blob[]>([]);
const handleUrlSubmit = () => {
const url = urlInput.trim();
@@ -71,8 +112,81 @@ export const InputArea = ({
setUrlDialogOpen(false);
};
const handleMicClick = async () => {
if (recording) {
const recorder = mediaRecorderRef.current;
if (!recorder) return;
setRecording(false);
try {
if (recorder.state === 'inactive') {
recorder.stream.getTracks().forEach((t) => t.stop());
return;
}
const blob = await new Promise<Blob>((resolve, reject) => {
const timeout = setTimeout(() => reject(new Error('Recording stop timed out')), 5000);
recorder.onstop = () => {
clearTimeout(timeout);
resolve(new Blob(chunksRef.current, { type: recorder.mimeType }));
chunksRef.current = [];
};
recorder.stop();
});
recorder.stream.getTracks().forEach((t) => t.stop());
if (blob.size === 0) {
toast.error('No audio was captured');
return;
}
setTranscribing(true);
try {
const wav = await blobToWav(blob);
const formData = new FormData();
formData.append('file', wav, 'recording.wav');
formData.append('temperature', '0.0');
formData.append('temperature_inc', '0.2');
formData.append('response_format', 'json');
const res = await fetch('http://macmini:8178/inference', { method: 'POST', body: formData });
if (!res.ok) throw new Error(`Whisper returned ${res.status}`);
const json = await res.json();
if (json.error) throw new Error(json.error);
const text = (json.text ?? '').trim();
if (text) onInputChange(input + (input.length > 0 ? ' ' : '') + text);
} finally {
setTranscribing(false);
}
} catch (err) {
recorder.stream?.getTracks().forEach((t) => t.stop());
chunksRef.current = [];
toast.error(err instanceof Error ? err.message : 'Recording failed');
}
return;
}
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
const recorder = new MediaRecorder(stream);
mediaRecorderRef.current = recorder;
chunksRef.current = [];
recorder.ondataavailable = (ev) => {
if (ev.data.size > 0) chunksRef.current.push(ev.data);
};
recorder.start(250);
setRecording(true);
} catch (err) {
toast.error(err instanceof Error ? err.message : 'Could not access microphone');
}
};
return (
<div className="shrink-0 border-t border-duck-dark/10 bg-white/60 p-3">
<div className="shrink-0 border-t border-duck-dark/10 bg-white/60 p-2 md:p-3">
{commandFeedback && (
<div className="mb-2 px-3 py-1.5 text-xs text-duck-teal bg-duck-teal/10 rounded-md">{commandFeedback}</div>
)}
@@ -106,12 +220,12 @@ export const InputArea = ({
</div>
)}
<div className="flex items-end gap-2">
<div className="flex items-end gap-1 md:gap-2">
<DropdownMenu>
<DropdownMenuTrigger asChild>
<button
type="button"
className="shrink-0 h-9 w-9 flex items-center justify-center rounded-lg text-duck-dark/40 hover:text-duck-dark/70 hover:bg-duck-dark/5 transition-colors cursor-pointer"
className="shrink-0 h-7 w-7 md:h-9 md:w-9 flex items-center justify-center rounded-lg text-duck-dark/40 hover:text-duck-dark/70 hover:bg-duck-dark/5 transition-colors cursor-pointer"
>
<Paperclip className="h-4 w-4" />
</button>
@@ -135,6 +249,23 @@ export const InputArea = ({
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
<button
type="button"
disabled={transcribing}
onClick={handleMicClick}
className="relative shrink-0 h-7 w-7 md:h-9 md:w-9 flex items-center justify-center rounded-lg text-duck-dark/40 hover:text-duck-dark/70 hover:bg-duck-dark/5 transition-colors cursor-pointer disabled:opacity-40 disabled:cursor-not-allowed"
>
{transcribing ? (
<Loader2 className="h-4 w-4 animate-spin" />
) : recording ? (
<>
<span className="absolute inset-0 rounded-lg animate-ping bg-red-400/30" />
<Square className="h-3.5 w-3.5 text-red-500" />
</>
) : (
<Mic className="h-4 w-4" />
)}
</button>
<input
ref={imageInputRef}
type="file"
@@ -165,10 +296,10 @@ export const InputArea = ({
}}
placeholder="Type a message..."
rows={1}
className="flex-1 resize-none rounded-lg border border-duck-dark/20 bg-white/80 px-3 py-2 text-base md:text-sm text-duck-dark placeholder:text-duck-dark/30 focus:outline-none focus:ring-2 focus:ring-duck-teal/30 focus:border-duck-teal/50"
className="min-w-0 flex-1 resize-none rounded-lg border border-duck-dark/20 bg-white/80 px-2 py-1.5 md:px-3 md:py-2 text-base md:text-sm text-duck-dark placeholder:text-duck-dark/30 focus:outline-none focus:ring-2 focus:ring-duck-teal/30 focus:border-duck-teal/50"
/>
{isGenerating ? (
<Button onClick={onStop} variant="destructive" size="icon" className="shrink-0 h-9 w-9 cursor-pointer">
<Button onClick={onStop} variant="destructive" size="icon" className="shrink-0 h-7 w-7 md:h-9 md:w-9 cursor-pointer">
<Square className="h-4 w-4" />
</Button>
) : (
@@ -176,7 +307,7 @@ export const InputArea = ({
onClick={onSend}
disabled={!input.trim() || !isConnected}
size="icon"
className="shrink-0 h-9 w-9 bg-duck-teal hover:bg-duck-teal/90 cursor-pointer disabled:opacity-40"
className="shrink-0 h-7 w-7 md:h-9 md:w-9 bg-duck-teal hover:bg-duck-teal/90 cursor-pointer disabled:opacity-40"
>
<Send className="h-4 w-4" />
</Button>
src/apps/officer-web/Screens/Dashboard/Files/Screen/FileViewer.tsx
+80
View File
@@ -9,6 +9,7 @@ import {
FileType2,
Maximize2,
Minimize2,
Square,
Play,
Pause,
Volume2,
@@ -28,6 +29,7 @@ import { cardStyle } from '@/components/Card';
import { useFiles } from 'widgets/FileBrowser';
import { getHeaders } from 'hooks/useClient';
import { config } from 'config';
import { toast } from 'sonner';
import { getIcon } from 'material-file-icons';
type FileViewerProps = {
@@ -922,6 +924,10 @@ export const FileViewer = ({ open, onOpenChange, filePath, fileName, root }: Fil
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [expanded, setExpanded] = useState(false);
const [ttsLoading, setTtsLoading] = useState(false);
const [ttsPlaying, setTtsPlaying] = useState(false);
const ttsAudioRef = useRef<HTMLAudioElement | null>(null);
const ttsUrlRef = useRef<string | null>(null);
const scrollRef = useRef<HTMLDivElement>(null);
const files = useFiles(root);
const fileType = getFileType(fileName);
@@ -931,6 +937,16 @@ export const FileViewer = ({ open, onOpenChange, filePath, fileName, root }: Fil
setContent(null);
setError(null);
setExpanded(false);
if (ttsAudioRef.current) {
ttsAudioRef.current.pause();
ttsAudioRef.current = null;
}
if (ttsUrlRef.current) {
URL.revokeObjectURL(ttsUrlRef.current);
ttsUrlRef.current = null;
}
setTtsLoading(false);
setTtsPlaying(false);
return;
}
@@ -957,6 +973,54 @@ export const FileViewer = ({ open, onOpenChange, filePath, fileName, root }: Fil
a.click();
};
const handleReadAloud = async () => {
if (ttsPlaying) {
if (ttsAudioRef.current) {
ttsAudioRef.current.pause();
ttsAudioRef.current = null;
}
if (ttsUrlRef.current) {
URL.revokeObjectURL(ttsUrlRef.current);
ttsUrlRef.current = null;
}
setTtsPlaying(false);
return;
}
setTtsLoading(true);
try {
const res = await fetch('http://alpha:9051/v1/audio/speech', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ model: 'kokoro', input: content, voice: 'af_heart', response_format: 'mp3' }),
});
if (!res.ok) throw new Error('TTS request failed');
const blob = await res.blob();
const url = URL.createObjectURL(blob);
ttsUrlRef.current = url;
const audio = new Audio(url);
ttsAudioRef.current = audio;
audio.addEventListener('ended', () => {
setTtsPlaying(false);
ttsAudioRef.current = null;
URL.revokeObjectURL(url);
ttsUrlRef.current = null;
});
await audio.play();
setTtsPlaying(true);
} catch {
toast.error('Failed to generate speech audio');
if (ttsUrlRef.current) {
URL.revokeObjectURL(ttsUrlRef.current);
ttsUrlRef.current = null;
}
ttsAudioRef.current = null;
setTtsPlaying(false);
} finally {
setTtsLoading(false);
}
};
const headerIcon =
fileType === 'audio' ? (
<Music className="h-4 w-4 text-duck-teal shrink-0" />
@@ -1019,6 +1083,22 @@ export const FileViewer = ({ open, onOpenChange, filePath, fileName, root }: Fil
>
{fileType === 'code' ? getLang(fileName) : fileType}
</span>
{(fileType === 'markdown' || fileType === 'code' || fileType === 'text') && content && (
<button
onClick={handleReadAloud}
disabled={ttsLoading}
className="p-1.5 rounded-md text-duck-dark/40 hover:text-duck-dark hover:bg-duck-dark/5 disabled:opacity-40 transition-colors cursor-pointer"
title="Read Aloud"
>
{ttsLoading ? (
<Loader2 className="h-4 w-4 animate-spin" />
) : ttsPlaying ? (
<Square className="h-4 w-4" />
) : (
<Volume2 className="h-4 w-4" />
)}
</button>
)}
<button
onClick={handleDownload}
className={`p-1.5 rounded-md transition-colors cursor-pointer ${