cliamp music player integration with browser audio streaming

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-02-27 17:23:52 +00:00
co-authored by Claude Opus 4.6
parent fd4d77a389
commit 30f06e91e4
16 changed files with 1051 additions and 18 deletions
+2 -1
View File
@@ -25,7 +25,8 @@
"dev:emailer": "cd src/workspaces/emailer && bun run dev", "dev:emailer": "cd src/workspaces/emailer && bun run dev",
"format": "{ git diff --name-only HEAD -- 'src/**/*.ts' 'src/**/*.tsx'; git ls-files --others --exclude-standard -- 'src/**/*.ts' 'src/**/*.tsx'; } | xargs -r prettier --write", "format": "{ git diff --name-only HEAD -- 'src/**/*.ts' 'src/**/*.tsx'; git ls-files --others --exclude-standard -- 'src/**/*.ts' 'src/**/*.tsx'; } | xargs -r prettier --write",
"format:all": "prettier --write \"src/**/*.{ts,tsx}\"", "format:all": "prettier --write \"src/**/*.{ts,tsx}\"",
"format:check": "prettier --check \"src/**/*.{ts,tsx}\"" "format:check": "prettier --check \"src/**/*.{ts,tsx}\"",
"setup": "bash scripts/setup.sh"
}, },
"dependencies": { "dependencies": {
"@anthropic-ai/claude-agent-sdk": "^0.2.41", "@anthropic-ai/claude-agent-sdk": "^0.2.41",
+363
View File
@@ -0,0 +1,363 @@
#!/bin/bash
# Officer — full host dependency setup
# Run once on a fresh Ubuntu/Debian host before launching the server.
# Usage: bash scripts/setup.sh
set -e
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
NC='\033[0m'
ok() { echo -e " ${GREEN}${NC} $1"; }
warn() { echo -e " ${YELLOW}!${NC} $1"; }
fail() { echo -e " ${RED}${NC} $1"; }
skip() { echo -e " - $1 (already installed)"; }
has() { command -v "$1" &>/dev/null; }
# ─── detect package manager ────────────────────────────────────────────────────
if has apt; then
PM=apt
elif has pacman; then
PM=pacman
elif has brew; then
PM=brew
else
fail "No supported package manager found (apt, pacman, brew)"
exit 1
fi
install_pkg() {
case $PM in
apt) sudo apt install -y "$@" ;;
pacman) sudo pacman -S --noconfirm "$@" ;;
brew) brew install "$@" ;;
esac
}
echo ""
echo "═══════════════════════════════════════════"
echo " Officer — dependency setup ($PM)"
echo "═══════════════════════════════════════════"
# ─── 1. core system packages ───────────────────────────────────────────────────
echo ""
echo "── Core system packages ──"
CORE_PKGS=()
# git
if has git; then skip "git"; else CORE_PKGS+=(git); fi
# zip / unzip
if has zip; then skip "zip"; else CORE_PKGS+=(zip); fi
if has unzip; then skip "unzip"; else CORE_PKGS+=(unzip); fi
# curl / wget (usually present but just in case)
if has curl; then skip "curl"; else CORE_PKGS+=(curl); fi
# psmisc (fuser) and procps (pgrep)
if has fuser; then skip "fuser (psmisc)"; else
case $PM in
apt|pacman) CORE_PKGS+=(psmisc) ;;
brew) skip "fuser (not needed on macOS)" ;;
esac
fi
if has pgrep; then skip "pgrep (procps)"; else
case $PM in
apt) CORE_PKGS+=(procps) ;;
pacman) CORE_PKGS+=(procps-ng) ;;
brew) skip "pgrep (built-in on macOS)" ;;
esac
fi
# script (bsdutils on apt, util-linux on pacman, built-in on macOS)
if has script; then skip "script (bsdutils)"; else
case $PM in
apt) CORE_PKGS+=(bsdutils) ;;
pacman) CORE_PKGS+=(util-linux) ;;
brew) skip "script (built-in on macOS)" ;;
esac
fi
if [ ${#CORE_PKGS[@]} -gt 0 ]; then
install_pkg "${CORE_PKGS[@]}"
ok "Installed: ${CORE_PKGS[*]}"
fi
# ─── 2. archive extras (optional but useful) ──────────────────────────────────
echo ""
echo "── Archive utilities (optional) ──"
ARCHIVE_PKGS=()
# p7zip
if has 7z; then skip "7z (p7zip)"; else
case $PM in
apt) ARCHIVE_PKGS+=(p7zip-full) ;;
pacman) ARCHIVE_PKGS+=(p7zip) ;;
brew) ARCHIVE_PKGS+=(p7zip) ;;
esac
fi
# unrar
if has unrar; then skip "unrar"; else
case $PM in
apt) ARCHIVE_PKGS+=(unrar) ;;
pacman) ARCHIVE_PKGS+=(unrar) ;;
brew) ARCHIVE_PKGS+=(unrar) ;;
esac
fi
if [ ${#ARCHIVE_PKGS[@]} -gt 0 ]; then
install_pkg "${ARCHIVE_PKGS[@]}" || warn "Some archive packages may need non-free repos"
ok "Installed: ${ARCHIVE_PKGS[*]}"
fi
# ─── 3. ffmpeg ─────────────────────────────────────────────────────────────────
echo ""
echo "── FFmpeg ──"
if has ffmpeg; then
skip "ffmpeg ($(ffmpeg -version 2>&1 | head -1 | awk '{print $3}'))"
else
install_pkg ffmpeg
ok "ffmpeg installed"
fi
# ─── 4. docker ─────────────────────────────────────────────────────────────────
echo ""
echo "── Docker ──"
if has docker; then
skip "docker ($(docker --version 2>/dev/null | awk '{print $3}' | tr -d ','))"
else
case $PM in
apt)
warn "Installing docker.io from apt"
sudo apt install -y docker.io
sudo usermod -aG docker "$USER"
warn "You may need to log out/in for docker group to take effect"
;;
pacman)
sudo pacman -S --noconfirm docker
sudo systemctl enable --now docker
sudo usermod -aG docker "$USER"
;;
brew)
warn "Install Docker Desktop from https://www.docker.com/products/docker-desktop"
;;
esac
if has docker; then ok "docker installed"; else warn "docker not found — install manually"; fi
fi
# ─── 5. Node.js 22 ────────────────────────────────────────────────────────────
echo ""
echo "── Node.js 22 ──"
if has node; then
NODE_VER=$(node -v 2>/dev/null | tr -d 'v')
NODE_MAJOR=$(echo "$NODE_VER" | cut -d. -f1)
if [ "$NODE_MAJOR" = "22" ]; then
skip "node v$NODE_VER"
else
warn "Node $NODE_VER found but v22 is required"
warn "Use nvm: nvm install 22 && nvm use 22"
fi
else
warn "Node.js not found — install v22 via nvm:"
warn " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash"
warn " nvm install 22"
fi
# ─── 6. Bun ───────────────────────────────────────────────────────────────────
echo ""
echo "── Bun ──"
if has bun; then
skip "bun ($(bun --version 2>/dev/null))"
else
curl -fsSL https://bun.sh/install | bash
export BUN_INSTALL="$HOME/.bun"
export PATH="$BUN_INSTALL/bin:$PATH"
if has bun; then ok "bun installed"; else fail "bun install failed"; fi
fi
# ─── 7. Go ─────────────────────────────────────────────────────────────────────
echo ""
echo "── Go ──"
if has go; then
skip "go ($(go version 2>/dev/null | awk '{print $3}'))"
else
install_pkg golang-go 2>/dev/null || install_pkg go 2>/dev/null || install_pkg golang 2>/dev/null
if has go; then ok "go installed"; else warn "go not found — install manually from https://go.dev/dl/"; fi
fi
# ─── 8. PulseAudio (headless audio for cliamp) ────────────────────────────────
echo ""
echo "── PulseAudio (headless audio) ──"
PULSE_PKGS=()
if has pulseaudio; then skip "pulseaudio"; else
case $PM in
apt) PULSE_PKGS+=(pulseaudio) ;;
pacman) PULSE_PKGS+=(pulseaudio) ;;
brew) warn "PulseAudio: brew install pulseaudio (cliamp audio won't work without it)" ;;
esac
fi
# pulseaudio-utils provides parec and pactl
if has parec && has pactl; then skip "pulseaudio-utils (parec, pactl)"; else
case $PM in
apt) PULSE_PKGS+=(pulseaudio-utils) ;;
pacman) ;; # included in pulseaudio package
brew) ;; # included in pulseaudio formula
esac
fi
# ALSA dev headers (needed to compile cliamp's Go audio library)
case $PM in
apt)
if dpkg -s libasound2-dev &>/dev/null 2>&1; then skip "libasound2-dev"; else PULSE_PKGS+=(libasound2-dev); fi
;;
pacman)
if pacman -Qi alsa-lib &>/dev/null 2>&1; then skip "alsa-lib"; else PULSE_PKGS+=(alsa-lib); fi
;;
brew) ;; # not needed on macOS
esac
if [ ${#PULSE_PKGS[@]} -gt 0 ]; then
install_pkg "${PULSE_PKGS[@]}"
ok "Installed: ${PULSE_PKGS[*]}"
fi
# ─── 9. cliamp (music player) ─────────────────────────────────────────────────
echo ""
echo "── cliamp ──"
GOPATH_BIN="${GOPATH:-$HOME/go}/bin"
export PATH="$GOPATH_BIN:$PATH"
if has cliamp; then
skip "cliamp ($(which cliamp))"
else
if ! has go; then
warn "Go not installed — skipping cliamp build"
else
echo " Building cliamp from source..."
TMPDIR=$(mktemp -d)
git clone --depth=1 https://github.com/bjarneo/cliamp.git "$TMPDIR/cliamp"
(cd "$TMPDIR/cliamp" && go install .)
rm -rf "$TMPDIR"
if [ -f "$GOPATH_BIN/cliamp" ]; then
ok "cliamp installed at $GOPATH_BIN/cliamp"
else
fail "cliamp build failed"
fi
fi
fi
# ─── 10. yt-dlp (optional — video/audio download) ─────────────────────────────
echo ""
echo "── yt-dlp (optional) ──"
if has yt-dlp; then
skip "yt-dlp"
else
case $PM in
apt) install_pkg yt-dlp 2>/dev/null && ok "yt-dlp installed" || warn "yt-dlp not in apt repos — install via pip or GitHub" ;;
pacman) install_pkg yt-dlp && ok "yt-dlp installed" ;;
brew) install_pkg yt-dlp && ok "yt-dlp installed" ;;
esac
fi
# ─── 11. npm dependencies ─────────────────────────────────────────────────────
echo ""
echo "── npm global packages ──"
if has npm; then
if has pi; then
skip "pi (@mariozechner/pi-coding-agent)"
else
npm install -g @mariozechner/pi-coding-agent
if has pi; then ok "pi installed"; else warn "pi install failed"; fi
fi
else
warn "npm not found — skipping global package installs"
fi
# ─── 12. bun install (project dependencies) ───────────────────────────────────
echo ""
echo "── Project dependencies ──"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
if has bun && [ -f "$PROJECT_DIR/package.json" ]; then
echo " Running bun install..."
(cd "$PROJECT_DIR" && bun install)
ok "Project dependencies installed"
else
warn "Skipping bun install (bun not found or not in project dir)"
fi
# ─── verification ─────────────────────────────────────────────────────────────
echo ""
echo "═══════════════════════════════════════════"
echo " Verification"
echo "═══════════════════════════════════════════"
echo ""
check() {
if has "$1"; then ok "$1"; else fail "$1 — NOT FOUND"; fi
}
echo "Required:"
check git
check node
check bun
check npm
check docker
check ffmpeg
check zip
check script
echo ""
echo "Audio (cliamp):"
check go
check pulseaudio
check parec
check pactl
check cliamp
echo ""
echo "Optional:"
check unzip
check 7z
check unrar
check pgrep
check fuser
check yt-dlp
check pi
echo ""
echo "Installable from Settings UI:"
for tool in claude opencode; do
if has "$tool"; then ok "$tool"; else echo " - $tool (install from Settings > Applications)"; fi
done
echo ""
echo "═══════════════════════════════════════════"
echo " Setup complete!"
echo "═══════════════════════════════════════════"
echo ""
echo "Notes:"
echo " • PulseAudio null sink starts automatically with the server"
echo " • Make sure ~/go/bin is in your PATH for cliamp"
echo " • Claude, opencode, sharp, whisper-cpp, mlx-audio"
echo " can be installed from the Settings > Applications page"
echo ""
+52 -3
View File
@@ -6,6 +6,8 @@ import { verify } from './servers/jwt';
import { isTokenBlacklisted } from 'officerdb'; import { isTokenBlacklisted } from 'officerdb';
import { terminalWebsocket, initTerminalSidecars } from './servers/api/terminal/websocket'; import { terminalWebsocket, initTerminalSidecars } from './servers/api/terminal/websocket';
import { piWebsocket } from './servers/api/pi/websocket'; import { piWebsocket } from './servers/api/pi/websocket';
import { cliampWebsocket } from './servers/api/cliamp/websocket';
import { cliampAudioWebsocket } from './servers/api/cliamp/audio-ws';
import { findEntryByProxyId, touchEntry } from './servers/api/dev-server/router'; import { findEntryByProxyId, touchEntry } from './servers/api/dev-server/router';
import officerWeb from './apps/officer-web/index.html'; import officerWeb from './apps/officer-web/index.html';
import { startBrowserRelay } from './servers/api/browser/relay'; import { startBrowserRelay } from './servers/api/browser/relay';
@@ -17,13 +19,14 @@ type WSData = {
email: string; email: string;
username: string; username: string;
role: string; role: string;
provider: 'terminal' | 'pi' | 'dev-server'; provider: 'terminal' | 'pi' | 'dev-server' | 'cliamp' | 'cliamp-audio';
sandboxed: boolean; sandboxed: boolean;
sessionId?: string; sessionId?: string;
cwd?: string; cwd?: string;
command?: string; command?: string;
cols?: number; cols?: number;
rows?: number; rows?: number;
files?: string;
devServerPort?: number; devServerPort?: number;
devServerSlug?: string; devServerSlug?: string;
wsProxyPath?: string; wsProxyPath?: string;
@@ -33,6 +36,8 @@ type WSData = {
const handlers: Record<string, any> = { const handlers: Record<string, any> = {
terminal: terminalWebsocket, terminal: terminalWebsocket,
pi: piWebsocket, pi: piWebsocket,
cliamp: cliampWebsocket,
'cliamp-audio': cliampAudioWebsocket,
}; };
// Dev-server WebSocket proxy: bridges client WS ↔ upstream dev server WS (for HMR etc.) // Dev-server WebSocket proxy: bridges client WS ↔ upstream dev server WS (for HMR etc.)
@@ -100,7 +105,7 @@ const devServerWebsocket = {
}; };
handlers['dev-server'] = devServerWebsocket; handlers['dev-server'] = devServerWebsocket;
async function upgradeWs(req: Request, server: any, provider: 'terminal' | 'pi') { async function upgradeWs(req: Request, server: any, provider: 'terminal' | 'pi' | 'cliamp' | 'cliamp-audio') {
const token = new URL(req.url).searchParams.get('token'); const token = new URL(req.url).searchParams.get('token');
if (!token) return new Response('Unauthorized', { status: 401 }); if (!token) return new Response('Unauthorized', { status: 401 });
@@ -119,8 +124,9 @@ async function upgradeWs(req: Request, server: any, provider: 'terminal' | 'pi')
const command = url.searchParams.get('command') ?? undefined; const command = url.searchParams.get('command') ?? undefined;
const cols = url.searchParams.get('cols') ? Number(url.searchParams.get('cols')) : undefined; const cols = url.searchParams.get('cols') ? Number(url.searchParams.get('cols')) : undefined;
const rows = url.searchParams.get('rows') ? Number(url.searchParams.get('rows')) : undefined; const rows = url.searchParams.get('rows') ? Number(url.searchParams.get('rows')) : undefined;
const files = url.searchParams.get('files') ?? undefined;
const ok = server.upgrade(req, { const ok = server.upgrade(req, {
data: { userId: user.id, email: user.email, username: user.username || user.email.split('@')[0]!, role: user.role, provider, sandboxed, sessionId, cwd, command, cols, rows }, data: { userId: user.id, email: user.email, username: user.username || user.email.split('@')[0]!, role: user.role, provider, sandboxed, sessionId, cwd, command, cols, rows, files },
}); });
if (!ok) return new Response('Upgrade failed', { status: 500 }); if (!ok) return new Response('Upgrade failed', { status: 500 });
} catch { } catch {
@@ -171,6 +177,8 @@ const server = serve({
}, },
'/api/terminal/ws': (req, server) => upgradeWs(req, server, 'terminal'), '/api/terminal/ws': (req, server) => upgradeWs(req, server, 'terminal'),
'/api/pi/chat/ws': (req, server) => upgradeWs(req, server, 'pi'), '/api/pi/chat/ws': (req, server) => upgradeWs(req, server, 'pi'),
'/api/cliamp/ws': (req, server) => upgradeWs(req, server, 'cliamp'),
'/api/cliamp/audio/ws': (req, server) => upgradeWs(req, server, 'cliamp-audio'),
'/': officerWeb, '/': officerWeb,
'/*': officerWeb, '/*': officerWeb,
'/api': honoServer.fetch, '/api': honoServer.fetch,
@@ -211,6 +219,47 @@ try {
void initTerminalSidecars(); void initTerminalSidecars();
// Ensure PulseAudio is running with virtual sink for cliamp audio streaming
(async () => {
const pulseaudio = Bun.which('pulseaudio');
const pactl = Bun.which('pactl');
if (!pulseaudio || !pactl) {
console.log('[cliamp] pulseaudio not installed, skipping audio setup');
return;
}
// Start PulseAudio daemon if not running
const check = Bun.spawnSync({ cmd: [pulseaudio, '--check'], stdout: 'ignore', stderr: 'ignore' });
if (check.exitCode !== 0) {
const start = Bun.spawnSync({ cmd: [pulseaudio, '--start', '-D'], stdout: 'ignore', stderr: 'ignore' });
if (start.exitCode !== 0) {
console.error('[cliamp] failed to start pulseaudio');
return;
}
console.log('[cliamp] pulseaudio started');
} else {
console.log('[cliamp] pulseaudio already running');
}
// Load null sink if not already loaded
const sinks = Bun.spawnSync({ cmd: [pactl, 'list', 'short', 'sinks'], stdout: 'pipe', stderr: 'ignore' });
const sinkList = sinks.stdout.toString();
if (!sinkList.includes('virtual_out')) {
const load = Bun.spawnSync({
cmd: [pactl, 'load-module', 'module-null-sink', 'sink_name=virtual_out', 'sink_properties=device.description=Virtual_Output'],
stdout: 'pipe',
stderr: 'pipe',
});
if (load.exitCode !== 0) {
console.error('[cliamp] failed to load null sink:', load.stderr.toString().trim());
} else {
console.log('[cliamp] virtual_out null sink loaded');
}
} else {
console.log('[cliamp] virtual_out sink already exists');
}
})();
// Ensure pi is installed // Ensure pi is installed
(async () => { (async () => {
try { try {
+9
View File
@@ -0,0 +1,9 @@
pcm.!default {
type pulse
fallback "sysdefault"
}
ctl.!default {
type pulse
fallback "sysdefault"
}
+91
View File
@@ -0,0 +1,91 @@
import type { ServerWebSocket } from 'bun';
import { spawn, type Subprocess } from 'bun';
type WSData = {
userId: number;
email: string;
username: string;
role: string;
};
type AudioSession = {
proc: Subprocess;
closed: boolean;
};
const sessions = new Map<ServerWebSocket<WSData>, AudioSession>();
export const cliampAudioWebsocket = {
async open(ws: ServerWebSocket<WSData>) {
const parecPath = Bun.which('parec');
if (!parecPath) {
ws.close(4000, 'parec not found on host');
return;
}
let proc: Subprocess<'ignore', 'pipe', 'ignore'>;
try {
proc = spawn({
cmd: [parecPath, '--format=s16le', '--rate=44100', '--channels=2', '-d', 'virtual_out.monitor'],
stdout: 'pipe',
stderr: 'ignore',
stdin: 'ignore',
});
} catch {
ws.close(4000, 'Failed to start audio capture');
return;
}
const session: AudioSession = { proc, closed: false };
sessions.set(ws, session);
console.log('[cliamp-audio] parec started, streaming to WS');
// Stream stdout chunks as binary WS frames
const reader = proc.stdout.getReader();
let totalBytes = 0;
const pump = async () => {
try {
while (!session.closed) {
const { done, value } = await reader.read();
if (done) break;
if (value && !session.closed) {
totalBytes += value.byteLength;
if (totalBytes <= value.byteLength) {
console.log(`[cliamp-audio] first chunk: ${value.byteLength} bytes`);
}
try {
ws.sendBinary(value);
} catch {
break;
}
}
}
} catch {
// stream ended or error
} finally {
if (!session.closed) {
session.closed = true;
try { ws.close(); } catch { /* ignore */ }
}
}
};
pump();
},
message() {
// No client-to-server messages expected
},
close(ws: ServerWebSocket<WSData>) {
console.log('[cliamp-audio] WS closed');
const session = sessions.get(ws);
if (session) {
session.closed = true;
try { session.proc.kill(); } catch { /* ignore */ }
sessions.delete(ws);
}
},
drain() {},
};
+190
View File
@@ -0,0 +1,190 @@
import type { ServerWebSocket } from 'bun';
import { spawn, type Subprocess } from 'bun';
import { resolve, normalize, dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { getHomeDir } from '@@/data-path';
const __dirname = dirname(fileURLToPath(import.meta.url));
const ASOUNDRC_PATH = join(__dirname, 'asoundrc');
type WSData = {
userId: number;
email: string;
username: string;
role: string;
files: string;
};
type CliampSession = {
proc: Subprocess<'pipe', 'pipe', 'pipe'>;
closed: boolean;
};
const sessions = new Map<ServerWebSocket<WSData>, CliampSession>();
const sendOutput = (ws: ServerWebSocket<WSData>, data: string) => {
try {
ws.send(JSON.stringify({ type: 'output', data }));
} catch {
// ws already closed
}
};
const sendExit = (ws: ServerWebSocket<WSData>) => {
try {
ws.send(JSON.stringify({ type: 'exit' }));
} catch {
// ws already closed
}
};
const validateFilePaths = (files: string[], homeDir: string): string[] | null => {
const resolved = files.map((f) => normalize(resolve(homeDir, f)));
for (const p of resolved) {
if (!p.startsWith(homeDir)) return null;
}
return resolved;
};
const findCliamp = (): string | null => {
const which = Bun.which('cliamp');
if (which) return which;
const goPath = process.env.GOPATH ?? `${process.env.HOME}/go`;
const goBin = `${goPath}/bin/cliamp`;
try {
const stat = Bun.spawnSync({ cmd: ['test', '-x', goBin], stdout: 'ignore', stderr: 'ignore' });
if (stat.exitCode === 0) return goBin;
} catch { /* ignore */ }
return null;
};
const shellEscape = (s: string) => `'${s.replace(/'/g, "'\\''")}'`;
export const cliampWebsocket = {
async open(ws: ServerWebSocket<WSData>) {
const { email, files: filesParam } = ws.data;
if (!filesParam) {
sendOutput(ws, '\r\n[Error] No files specified.\r\n');
return;
}
const cliampPath = findCliamp();
if (!cliampPath) {
sendOutput(ws, '\r\n[Error] cliamp not found on host.\r\n');
return;
}
const homeDir = getHomeDir(email);
const rawFiles = [filesParam];
// Resolve paths relative to user home dir
const absoluteFiles = rawFiles.map((f) => {
if (f.startsWith('/')) return `${homeDir}${f}`;
return `${homeDir}/${f}`;
});
const validated = validateFilePaths(absoluteFiles, homeDir);
if (!validated) {
sendOutput(ws, '\r\n[Error] Invalid file path.\r\n');
return;
}
// Use `script` to allocate a PTY for cliamp (avoids node-pty dependency)
// script -qfc '<command> <args...>' /dev/null
const cliampCmd = [shellEscape(cliampPath), ...validated.map(shellEscape)].join(' ');
console.log(`[cliamp] spawning: ${cliampCmd}`);
let proc: Subprocess<'pipe', 'pipe', 'pipe'>;
try {
proc = spawn({
cmd: ['script', '-qfc', cliampCmd, '/dev/null'],
stdin: 'pipe',
stdout: 'pipe',
stderr: 'pipe',
cwd: homeDir,
env: { ...process.env, TERM: 'xterm-256color', PULSE_SINK: 'virtual_out', ALSA_CONFIG_PATH: ASOUNDRC_PATH },
});
} catch (err) {
const message = err instanceof Error ? err.message : 'Failed to start cliamp';
sendOutput(ws, `\r\n[Error] ${message}\r\n`);
return;
}
const session: CliampSession = { proc, closed: false };
sessions.set(ws, session);
// Pump stdout → WS
const reader = proc.stdout.getReader();
const pump = async () => {
try {
while (!session.closed) {
const { done, value } = await reader.read();
if (done) break;
if (value && !session.closed) {
sendOutput(ws, new TextDecoder().decode(value));
}
}
} catch {
// stream ended
} finally {
if (!session.closed) {
session.closed = true;
sendExit(ws);
}
}
};
pump();
// Also read stderr (cliamp may write there)
const stderrReader = proc.stderr.getReader();
const pumpStderr = async () => {
try {
while (!session.closed) {
const { done, value } = await stderrReader.read();
if (done) break;
if (value && !session.closed) {
sendOutput(ws, new TextDecoder().decode(value));
}
}
} catch {
// stream ended
}
};
pumpStderr();
// On process exit → notify client
proc.exited.then((code) => {
console.log(`[cliamp] process exited code=${code}`);
if (!session.closed) {
session.closed = true;
sendExit(ws);
}
sessions.delete(ws);
});
},
message(ws: ServerWebSocket<WSData>, raw: string | Buffer) {
const session = sessions.get(ws);
if (!session || session.closed) return;
try {
const msg = JSON.parse(typeof raw === 'string' ? raw : raw.toString());
if (msg.type === 'input' && msg.data) {
session.proc.stdin.write(msg.data);
}
} catch {
// ignore
}
},
close(ws: ServerWebSocket<WSData>) {
const session = sessions.get(ws);
if (session) {
session.closed = true;
try { session.proc.kill(); } catch { /* ignore */ }
sessions.delete(ws);
}
},
drain() {},
};
+1 -1
View File
@@ -151,7 +151,7 @@ wss.on('connection', (ws) => {
let ptyEnv; let ptyEnv;
if (isHost) { if (isHost) {
ptyEnv = { ...process.env, TERM: 'xterm-256color' }; ptyEnv = { ...process.env, TERM: 'xterm-256color', ...(msg.env ?? {}) };
} else { } else {
const prompt = `${userLabel} in %~ %# `; const prompt = `${userLabel} in %~ %# `;
const bashPrompt = `${userLabel} \\w \\$ `; const bashPrompt = `${userLabel} \\w \\$ `;
+13
View File
@@ -344,6 +344,19 @@ const startHostSidecar = async () => {
console.log(`[terminal] host sidecar started on port ${HOST_SIDECAR_PORT}`); console.log(`[terminal] host sidecar started on port ${HOST_SIDECAR_PORT}`);
}; };
export const ensureHostSidecar = async () => {
const alive = await sidecarAlive(HOST_SIDECAR_PORT);
if (!alive) {
await startHostSidecar();
// Wait for it to come up
for (let i = 0; i < 10; i++) {
await new Promise((r) => setTimeout(r, 300));
if (await sidecarAlive(HOST_SIDECAR_PORT)) return;
}
throw new Error('Host sidecar failed to start');
}
};
export const initTerminalSidecars = async () => { export const initTerminalSidecars = async () => {
await startHostSidecar(); await startHostSidecar();
ensureDockerImage(); ensureDockerImage();
@@ -0,0 +1,164 @@
import { useEffect, useRef, useState } from 'react';
import { Volume2, VolumeX } from 'lucide-react';
type AudioStreamPlayerProps = {
wsUrl: string;
onError?: (message: string) => void;
};
const SAMPLE_RATE = 44100;
const CHANNELS = 2;
const buildWsUrl = (wsPath: string) => {
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
const token = localStorage.getItem('BEARER_TOKEN') ?? '';
const separator = wsPath.includes('?') ? '&' : '?';
return `${protocol}//${window.location.host}${wsPath}${separator}token=${encodeURIComponent(token)}`;
};
const WORKLET_CODE = `
class PCMProcessor extends AudioWorkletProcessor {
constructor() {
super();
this.buffer = new Float32Array(0);
this.port.onmessage = (e) => {
const incoming = e.data;
const merged = new Float32Array(this.buffer.length + incoming.length);
merged.set(this.buffer);
merged.set(incoming, this.buffer.length);
this.buffer = merged;
const max = ${SAMPLE_RATE * CHANNELS * 2};
if (this.buffer.length > max) {
this.buffer = this.buffer.slice(this.buffer.length - max);
}
};
}
process(inputs, outputs) {
const output = outputs[0];
if (!output || output.length === 0) return true;
const channels = output.length;
const frameSize = output[0].length;
const samplesNeeded = frameSize * channels;
if (this.buffer.length >= samplesNeeded) {
for (let i = 0; i < frameSize; i++) {
for (let ch = 0; ch < channels; ch++) {
output[ch][i] = this.buffer[i * channels + ch];
}
}
this.buffer = this.buffer.slice(samplesNeeded);
} else {
for (let ch = 0; ch < channels; ch++) {
output[ch].fill(0);
}
}
return true;
}
}
registerProcessor('pcm-processor', PCMProcessor);
`;
const workletBlobUrl = URL.createObjectURL(new Blob([WORKLET_CODE], { type: 'application/javascript' }));
export const AudioStreamPlayer = ({ wsUrl, onError }: AudioStreamPlayerProps) => {
const [muted, setMuted] = useState(false);
const [started, setStarted] = useState(false);
const ctxRef = useRef<AudioContext | null>(null);
const nodeRef = useRef<AudioWorkletNode | null>(null);
const wsRef = useRef<WebSocket | null>(null);
const gainRef = useRef<GainNode | null>(null);
useEffect(() => {
let disposed = false;
let audioCtx: AudioContext | null = null;
const init = async () => {
try {
audioCtx = new AudioContext({ sampleRate: SAMPLE_RATE });
ctxRef.current = audioCtx;
await audioCtx.audioWorklet.addModule(workletBlobUrl);
if (disposed) { audioCtx.close(); return; }
const workletNode = new AudioWorkletNode(audioCtx, 'pcm-processor', {
outputChannelCount: [CHANNELS],
});
nodeRef.current = workletNode;
const gainNode = audioCtx.createGain();
gainRef.current = gainNode;
workletNode.connect(gainNode);
gainNode.connect(audioCtx.destination);
const ws = new WebSocket(buildWsUrl(wsUrl));
ws.binaryType = 'arraybuffer';
wsRef.current = ws;
ws.addEventListener('open', () => {
if (!disposed) setStarted(true);
});
ws.addEventListener('message', (ev) => {
if (disposed || !(ev.data instanceof ArrayBuffer)) return;
// Resume context if suspended (autoplay policy — will unlock on user gesture)
if (audioCtx && audioCtx.state === 'suspended') {
audioCtx.resume();
}
const int16 = new Int16Array(ev.data);
const float32 = new Float32Array(int16.length);
for (let i = 0; i < int16.length; i++) {
float32[i] = int16[i]! / 32768;
}
workletNode.port.postMessage(float32);
});
ws.addEventListener('error', () => {
if (!disposed) onError?.('Audio stream connection failed');
});
ws.addEventListener('close', () => {
if (!disposed) setStarted(false);
});
} catch (err) {
if (!disposed) {
onError?.(err instanceof Error ? err.message : 'Audio playback failed');
}
}
};
init();
return () => {
disposed = true;
try { wsRef.current?.close(); } catch { /* ignore */ }
wsRef.current = null;
try { nodeRef.current?.disconnect(); } catch { /* ignore */ }
nodeRef.current = null;
try { audioCtx?.close(); } catch { /* ignore */ }
ctxRef.current = null;
gainRef.current = null;
};
}, [wsUrl]);
useEffect(() => {
if (gainRef.current) {
gainRef.current.gain.value = muted ? 0 : 1;
}
}, [muted]);
return (
<button
onClick={() => setMuted((m) => !m)}
className="p-1.5 rounded hover:bg-duck-dark/10 transition-colors cursor-pointer"
title={muted ? 'Unmute' : 'Mute'}
>
{muted ? (
<VolumeX className={`h-4 w-4 ${started ? 'text-red-500' : 'text-duck-dark/40'}`} />
) : (
<Volume2 className={`h-4 w-4 ${started ? 'text-duck-teal' : 'text-duck-dark/40'}`} />
)}
</button>
);
};
@@ -0,0 +1,58 @@
import { useCallback } from 'react';
import { useSearchParams } from 'react-router';
import { X } from 'lucide-react';
import { TerminalView } from '../Terminal/Terminal';
import { AudioStreamPlayer } from './AudioStreamPlayer';
export const CliampPanelHeader = () => {
const [searchParams, setSearchParams] = useSearchParams();
const playPath = searchParams.get('play') ?? '';
const fileName = playPath.split('/').pop() ?? 'cliamp';
const handleClose = useCallback(() => {
setSearchParams((prev) => {
const next = new URLSearchParams(prev);
next.delete('play');
return next;
});
}, [setSearchParams]);
return (
<div className="flex items-center gap-2 px-3 py-1.5 border-b border-duck-dark/10 bg-background/95">
<span className="text-sm font-medium text-duck-dark truncate flex-1">{fileName}</span>
<AudioStreamPlayer wsUrl="/api/cliamp/audio/ws" />
<button
onClick={handleClose}
className="p-1 rounded hover:bg-duck-dark/10 transition-colors cursor-pointer"
title="Close player"
>
<X className="h-4 w-4 text-duck-dark/60" />
</button>
</div>
);
};
export const CliampPanelBody = () => {
const [searchParams, setSearchParams] = useSearchParams();
const playPath = searchParams.get('play') ?? '';
const wsPath = `/api/cliamp/ws?files=${encodeURIComponent(playPath)}`;
const handleExit = useCallback(() => {
setSearchParams((prev) => {
const next = new URLSearchParams(prev);
next.delete('play');
return next;
});
}, [setSearchParams]);
return (
<TerminalView
className="h-full w-full"
wsPath={wsPath}
sandboxed={false}
onExit={handleExit}
autoFocus
/>
);
};
@@ -65,6 +65,7 @@ export const FileGrid = ({ fileBrowserManager }: FileGridProps) => {
handleTranscribe, handleTranscribe,
handleExtractAudio, handleExtractAudio,
handleExtract, handleExtract,
handlePlay,
getMatchingTasks, getMatchingTasks,
handleRunTask, handleRunTask,
handleCreateWorkspace, handleCreateWorkspace,
@@ -183,6 +184,7 @@ export const FileGrid = ({ fileBrowserManager }: FileGridProps) => {
onTranscribe={handleTranscribe} onTranscribe={handleTranscribe}
onExtractAudio={handleExtractAudio} onExtractAudio={handleExtractAudio}
onExtract={handleExtract} onExtract={handleExtract}
onPlay={handlePlay}
matchingTasks={getMatchingTasks(entry.name, entry.type)} matchingTasks={getMatchingTasks(entry.name, entry.type)}
onRunTask={handleRunTask} onRunTask={handleRunTask}
onCreateWorkspace={handleCreateWorkspace} onCreateWorkspace={handleCreateWorkspace}
@@ -1,5 +1,5 @@
import { useState, useEffect, useRef } from 'react'; import { useState, useEffect, useRef } from 'react';
import { Folder, Trash2, Pencil, MoreVertical, MessageSquare, Scissors, Copy, Check, Play, Download, LayoutGrid, Volume2, ScanText, FileText, AudioLines, FolderArchive, ClipboardCopy } from 'lucide-react'; import { Folder, Trash2, Pencil, MoreVertical, MessageSquare, Scissors, Copy, Check, Play, Download, LayoutGrid, Volume2, ScanText, FileText, AudioLines, FolderArchive, ClipboardCopy, Music } from 'lucide-react';
import { getIcon } from 'material-file-icons'; import { getIcon } from 'material-file-icons';
import { import {
DropdownMenu, DropdownMenu,
@@ -49,6 +49,7 @@ export type FileItemProps = {
onTranscribe: (entry: DirEntry) => void; onTranscribe: (entry: DirEntry) => void;
onExtractAudio: (entry: DirEntry) => void; onExtractAudio: (entry: DirEntry) => void;
onExtract: (entry: DirEntry) => void; onExtract: (entry: DirEntry) => void;
onPlay: (entry: DirEntry) => void;
matchingTasks: TaskSummary[]; matchingTasks: TaskSummary[];
onRunTask: (task: TaskSummary, entry: DirEntry) => void; onRunTask: (task: TaskSummary, entry: DirEntry) => void;
onCreateWorkspace: (entry: DirEntry) => void; onCreateWorkspace: (entry: DirEntry) => void;
@@ -81,6 +82,7 @@ type MenuItemsProps = {
onTranscribe: (e: DirEntry) => void; onTranscribe: (e: DirEntry) => void;
onExtractAudio: (e: DirEntry) => void; onExtractAudio: (e: DirEntry) => void;
onExtract: (e: DirEntry) => void; onExtract: (e: DirEntry) => void;
onPlay: (e: DirEntry) => void;
onCut: () => void; onCut: () => void;
onCopy: () => void; onCopy: () => void;
matchingTasks: TaskSummary[]; matchingTasks: TaskSummary[];
@@ -101,6 +103,7 @@ const DropdownMenuItems = ({
onTranscribe, onTranscribe,
onExtractAudio, onExtractAudio,
onExtract, onExtract,
onPlay,
onCut, onCut,
onCopy, onCopy,
matchingTasks, matchingTasks,
@@ -113,9 +116,16 @@ const DropdownMenuItems = ({
const showTranscribe = fileType === 'audio'; const showTranscribe = fileType === 'audio';
const showExtractAudio = fileType === 'video'; const showExtractAudio = fileType === 'video';
const showExtract = fileType === 'archive'; const showExtract = fileType === 'archive';
const showPlay = fileType === 'audio' || entry.type === 'directory';
return ( return (
<> <>
{showPlay && (
<DropdownMenuItem onClick={() => onPlay(entry)} className="cursor-pointer">
<Music className="mr-2 h-4 w-4" />
Play
</DropdownMenuItem>
)}
<DropdownMenuItem onClick={() => onChat(entry)} className="cursor-pointer"> <DropdownMenuItem onClick={() => onChat(entry)} className="cursor-pointer">
<MessageSquare className="mr-2 h-4 w-4" /> <MessageSquare className="mr-2 h-4 w-4" />
Chat... Chat...
@@ -219,6 +229,7 @@ const ContextMenuItems = ({
onTranscribe, onTranscribe,
onExtractAudio, onExtractAudio,
onExtract, onExtract,
onPlay,
onCut, onCut,
onCopy, onCopy,
matchingTasks, matchingTasks,
@@ -231,9 +242,16 @@ const ContextMenuItems = ({
const showTranscribe = fileType === 'audio'; const showTranscribe = fileType === 'audio';
const showExtractAudio = fileType === 'video'; const showExtractAudio = fileType === 'video';
const showExtract = fileType === 'archive'; const showExtract = fileType === 'archive';
const showPlay = fileType === 'audio' || entry.type === 'directory';
return ( return (
<> <>
{showPlay && (
<ContextMenuItem onClick={() => onPlay(entry)} className="cursor-pointer">
<Music className="mr-2 h-4 w-4" />
Play
</ContextMenuItem>
)}
<ContextMenuItem onClick={() => onChat(entry)} className="cursor-pointer"> <ContextMenuItem onClick={() => onChat(entry)} className="cursor-pointer">
<MessageSquare className="mr-2 h-4 w-4" /> <MessageSquare className="mr-2 h-4 w-4" />
Chat... Chat...
@@ -438,6 +456,7 @@ export const FileItem = ({
onTranscribe, onTranscribe,
onExtractAudio, onExtractAudio,
onExtract, onExtract,
onPlay,
matchingTasks, matchingTasks,
onRunTask, onRunTask,
onCreateWorkspace, onCreateWorkspace,
@@ -519,6 +538,7 @@ export const FileItem = ({
onTranscribe, onTranscribe,
onExtractAudio, onExtractAudio,
onExtract, onExtract,
onPlay,
onCut, onCut,
onCopy, onCopy,
matchingTasks, matchingTasks,
@@ -457,6 +457,11 @@ export const useFileBrowserApp = (basePath: string, rootOverride?: string, initi
} }
}; };
const handlePlay = (entry: DirEntry) => {
const filePath = entryPath(entry.name);
setSearchParams({ play: filePath });
};
const handleExtract = async (entry: DirEntry) => { const handleExtract = async (entry: DirEntry) => {
const filePath = entryPath(entry.name); const filePath = entryPath(entry.name);
const toastId = toast.loading('Extracting archive...'); const toastId = toast.loading('Extracting archive...');
@@ -705,6 +710,7 @@ export const useFileBrowserApp = (basePath: string, rootOverride?: string, initi
handleTranscribe, handleTranscribe,
handleExtractAudio, handleExtractAudio,
handleExtract, handleExtract,
handlePlay,
handleGitClone, handleGitClone,
handleVideoDownload, handleVideoDownload,
handleCut, handleCut,
@@ -0,0 +1,41 @@
class PCMProcessor extends AudioWorkletProcessor {
constructor() {
super();
this.buffer = new Float32Array(0);
this.port.onmessage = (e) => {
const incoming = e.data;
const merged = new Float32Array(this.buffer.length + incoming.length);
merged.set(this.buffer);
merged.set(incoming, this.buffer.length);
this.buffer = merged;
};
}
process(_inputs, outputs) {
const output = outputs[0];
if (!output || output.length === 0) return true;
const channels = output.length;
const frameSize = output[0].length;
const samplesNeeded = frameSize * channels;
if (this.buffer.length >= samplesNeeded) {
// Deinterleave: buffer is interleaved L R L R ...
for (let i = 0; i < frameSize; i++) {
for (let ch = 0; ch < channels; ch++) {
output[ch][i] = this.buffer[i * channels + ch];
}
}
this.buffer = this.buffer.slice(samplesNeeded);
} else {
// Not enough data — output silence
for (let ch = 0; ch < channels; ch++) {
output[ch].fill(0);
}
}
return true;
}
}
registerProcessor('pcm-processor', PCMProcessor);
@@ -6,6 +6,12 @@ export const singleViewerLayout: LayoutNode = {
appType: null, appType: null,
}; };
export const singleCliampLayout: LayoutNode = {
type: 'panel',
id: 'files-cliamp',
appType: null,
};
export const viewerWithEphemeralLayout: LayoutNode = { export const viewerWithEphemeralLayout: LayoutNode = {
type: 'group', type: 'group',
id: 'files-viewer-group', id: 'files-viewer-group',
@@ -2,10 +2,11 @@ import { useCallback, useEffect, useMemo, useRef } from 'react';
import { useSearchParams } from 'react-router'; import { useSearchParams } from 'react-router';
import type { EphemeralPanels } from '../../components/Workspace'; import type { EphemeralPanels } from '../../components/Workspace';
import { FileViewerHeader, FileViewerBody } from '../../apps/FileViewer'; import { FileViewerHeader, FileViewerBody } from '../../apps/FileViewer';
import { singleViewerLayout, viewerWithEphemeralLayout, viewerWithEphemeralSplitLayout, singleChatLayout } from './layouts'; import { singleViewerLayout, singleCliampLayout, viewerWithEphemeralLayout, viewerWithEphemeralSplitLayout, singleChatLayout } from './layouts';
import { ViewerProvider, EphemeralProvider, Ephemeral2Provider, ChatEphemeralBody } from './Providers'; import { ViewerProvider, EphemeralProvider, Ephemeral2Provider, ChatEphemeralBody } from './Providers';
import { CliampPanelHeader, CliampPanelBody } from '../../apps/FileBrowser/CliampPanel';
const EPHEMERAL_KEYS = ['view', 'ephemeral', 'ephemeralRoot', 'ephemeral2', 'ephemeral2Root', 'ephemeral2Auto', 'chatContext', 'chatType']; const EPHEMERAL_KEYS = ['view', 'ephemeral', 'ephemeralRoot', 'ephemeral2', 'ephemeral2Root', 'ephemeral2Auto', 'chatContext', 'chatType', 'play'];
export const useFileViewerPanels = (): EphemeralPanels | null => { export const useFileViewerPanels = (): EphemeralPanels | null => {
const [searchParams, setSearchParams] = useSearchParams(); const [searchParams, setSearchParams] = useSearchParams();
@@ -28,8 +29,11 @@ export const useFileViewerPanels = (): EphemeralPanels | null => {
const ephemeralPath = searchParams.get('ephemeral'); const ephemeralPath = searchParams.get('ephemeral');
const ephemeral2Path = searchParams.get('ephemeral2'); const ephemeral2Path = searchParams.get('ephemeral2');
const chatContext = searchParams.get('chatContext'); const chatContext = searchParams.get('chatContext');
const playPath = searchParams.get('play');
const layout = chatContext const layout = playPath
? singleCliampLayout
: chatContext
? singleChatLayout ? singleChatLayout
: viewPath && ephemeralPath && ephemeral2Path : viewPath && ephemeralPath && ephemeral2Path
? viewerWithEphemeralSplitLayout ? viewerWithEphemeralSplitLayout
@@ -76,8 +80,23 @@ export const useFileViewerPanels = (): EphemeralPanels | null => {
[setSearchParams], [setSearchParams],
); );
const onClosePlay = useCallback(
() =>
setSearchParams((prev) => {
const next = new URLSearchParams(prev);
next.delete('play');
return next;
}),
[setSearchParams],
);
const components = useMemo( const components = useMemo(
() => ({ () => ({
'files-cliamp': {
header: CliampPanelHeader,
component: CliampPanelBody,
onClose: onClosePlay,
},
'files-viewer': { 'files-viewer': {
provider: ViewerProvider, provider: ViewerProvider,
header: FileViewerHeader, header: FileViewerHeader,
@@ -101,11 +120,12 @@ export const useFileViewerPanels = (): EphemeralPanels | null => {
onClose: onCloseChat, onClose: onCloseChat,
}, },
}), }),
[onCloseViewer, onCloseEphemeral, onCloseEphemeral2, onCloseChat], [onCloseViewer, onCloseEphemeral, onCloseEphemeral2, onCloseChat, onClosePlay],
); );
if (!viewPath && !chatContext) return null; if (!viewPath && !chatContext && !playPath) return null;
return { layout, components, defaultBaseSize: 40, onClose: onCloseViewer }; const onClose = playPath ? onClosePlay : onCloseViewer;
return { layout, components, defaultBaseSize: 40, onClose };
}; };
export type UseFileViewerPanelsType = ReturnType<typeof useFileViewerPanels>; export type UseFileViewerPanelsType = ReturnType<typeof useFileViewerPanels>;