browser relay: multi-session override, stale session recovery, better error feedback

- relay accepts new extension connections by closing old one (last wins, code 4000) - extension recognizes code 4000 and stops auto-reconnect (shows "replaced" badge) - fix ping interval race where old WS close handler killed new connection's pings - retry CDP commands on "Session with given id not found" by re-attaching debugger - describeError() maps known errors to user-friendly badge tooltips - persisted relay tokens restored on server start - user-scoped token salts, token regeneration/deletion endpoints - extension download endpoint, integrations UI for browser relay setup - browser relay env vars passed to pi-bridge sandboxes - browser screen layout with chat panel and prompt prefix Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-27 15:22:15 +00:00
parent 8f1963fedb
commit 4d30672adb
16 changed files with 696 additions and 89 deletions
@@ -0,0 +1,43 @@
+---
+name: browser
+label: Browser Control
+description: Control connected Chrome browser tabs via the Officer Browser Relay. Use this tool to list tabs, take screenshots, navigate to URLs, evaluate JavaScript, get page info, activate (focus) tabs, or close tabs. Requires the user to have connected the Browser Relay extension in Settings → Integrations.
+language: typescript
+inputs:
+  action:
+    type: string
+    description: "Action to perform: list_tabs, screenshot, navigate, evaluate, page_info, activate, close"
+  tab_id:
+    type: string
+    description: Target tab ID. Optional — defaults to the first connected tab.
+    optional: true
+  url:
+    type: string
+    description: URL to navigate to (required for navigate action)
+    optional: true
+  expression:
+    type: string
+    description: JavaScript expression to evaluate in the tab (required for evaluate action)
+    optional: true
+---
+
+# Browser Tool
+
+Control the user's Chrome browser tabs through the Officer Browser Relay and Chrome DevTools Protocol.
+
+## Available Actions
+
+- **list_tabs**: List all connected tabs with their title, URL, and ID.
+- **screenshot**: Capture a screenshot of a tab. Returns the image directly. Use this when asked about page content.
+- **navigate**: Navigate a tab to a URL. Requires `url`.
+- **evaluate**: Run JavaScript in a tab and return the result. Requires `expression`.
+- **page_info**: Get the title and URL of a tab.
+- **activate**: Bring a tab to the foreground (focus it).
+- **close**: Close a tab.
+
+## Tips
+
+- When asked about what's on a page, take a screenshot first.
+- Use `evaluate` for extracting structured data from pages (DOM queries, reading text content, etc.).
+- If no `tab_id` is provided, the first connected tab is used.
+- Tab IDs can be obtained from `list_tabs`.
@@ -0,0 +1,222 @@
+type ToolResult = {
+  content: Array<{ type: string; text?: string; source?: { type: string; media_type: string; data: string } }>;
+  isError?: boolean;
+};
+
+type Params = {
+  action: string;
+  tab_id?: string;
+  url?: string;
+  expression?: string;
+};
+
+type TabInfo = {
+  id: string;
+  title: string;
+  url: string;
+};
+
+const RELAY_AUTH_HEADER = 'x-officer-relay-token';
+
+function getConfig(): { port: number; token: string } | null {
+  const port = process.env.OFFICER_BROWSER_RELAY_PORT;
+  const token = process.env.OFFICER_BROWSER_RELAY_TOKEN;
+  if (!port || !token) return null;
+  return { port: Number(port), token };
+}
+
+async function listTabs(port: number, token: string): Promise<TabInfo[]> {
+  const res = await fetch(`http://127.0.0.1:${port}/json/list`, {
+    headers: { [RELAY_AUTH_HEADER]: token },
+  });
+  if (!res.ok) throw new Error(`Failed to list tabs (${res.status})`);
+  const tabs = (await res.json()) as Array<{ id: string; title: string; url: string }>;
+  return tabs.map((t) => ({ id: t.id, title: t.title, url: t.url }));
+}
+
+async function resolveTab(port: number, token: string, tabId?: string): Promise<TabInfo> {
+  const tabs = await listTabs(port, token);
+  if (tabs.length === 0) throw new Error('No browser tabs connected. The user needs to attach tabs via the Browser Relay extension.');
+  if (tabId) {
+    const tab = tabs.find((t) => t.id === tabId);
+    if (!tab) throw new Error(`Tab ${tabId} not found. Available tabs: ${tabs.map((t) => t.id).join(', ')}`);
+    return tab;
+  }
+  return tabs[0]!;
+}
+
+async function sendCdpCommand(port: number, token: string, tabId: string, method: string, params?: unknown): Promise<unknown> {
+  const url = `ws://127.0.0.1:${port}/cdp?token=${encodeURIComponent(token)}`;
+
+  return await new Promise<unknown>((resolve, reject) => {
+    const ws = new WebSocket(url);
+    let settled = false;
+    const timeout = setTimeout(() => {
+      if (settled) return;
+      settled = true;
+      ws.close();
+      reject(new Error(`CDP command timeout: ${method}`));
+    }, 30_000);
+
+    ws.addEventListener('open', () => {
+      const cmd: Record<string, unknown> = { id: 1, method };
+      if (params) cmd.params = params;
+      cmd.sessionId = tabId;
+      ws.send(JSON.stringify(cmd));
+    });
+
+    ws.addEventListener('message', (event) => {
+      if (settled) return;
+      try {
+        const msg = JSON.parse(String(event.data)) as { id?: number; result?: unknown; error?: { message: string } };
+        if (msg.id === 1) {
+          settled = true;
+          clearTimeout(timeout);
+          ws.close();
+          if (msg.error) reject(new Error(msg.error.message));
+          else resolve(msg.result);
+        }
+      } catch {
+        // ignore parse errors, wait for correct message
+      }
+    });
+
+    ws.addEventListener('error', () => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      reject(new Error('CDP WebSocket connection failed — is the Browser Relay running?'));
+    });
+
+    ws.addEventListener('close', () => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      reject(new Error('CDP WebSocket closed before response'));
+    });
+  });
+}
+
+// --- Actions ---
+
+async function actionListTabs(port: number, token: string): Promise<ToolResult> {
+  const tabs = await listTabs(port, token);
+  if (tabs.length === 0) {
+    return { content: [{ type: 'text', text: 'No browser tabs connected. The user needs to attach tabs via the Browser Relay extension.' }] };
+  }
+  const lines = tabs.map((t, i) => `${i + 1}. ${t.title}\n   URL: ${t.url}\n   ID: ${t.id}`);
+  return { content: [{ type: 'text', text: `Connected tabs (${tabs.length}):\n\n${lines.join('\n\n')}` }] };
+}
+
+async function actionScreenshot(port: number, token: string, tabId?: string): Promise<ToolResult> {
+  const tab = await resolveTab(port, token, tabId);
+  const result = (await sendCdpCommand(port, token, tab.id, 'Page.captureScreenshot', { format: 'png' })) as { data: string };
+  return {
+    content: [
+      { type: 'text', text: `Screenshot of "${tab.title}" (${tab.url})` },
+      { type: 'image', source: { type: 'base64', media_type: 'image/png', data: result.data } },
+    ],
+  };
+}
+
+async function actionNavigate(port: number, token: string, url: string, tabId?: string): Promise<ToolResult> {
+  const tab = await resolveTab(port, token, tabId);
+  await sendCdpCommand(port, token, tab.id, 'Page.navigate', { url });
+  return { content: [{ type: 'text', text: `Navigated tab "${tab.title}" to ${url}` }] };
+}
+
+async function actionEvaluate(port: number, token: string, expression: string, tabId?: string): Promise<ToolResult> {
+  const tab = await resolveTab(port, token, tabId);
+  const result = (await sendCdpCommand(port, token, tab.id, 'Runtime.evaluate', {
+    expression,
+    returnByValue: true,
+    awaitPromise: true,
+  })) as { result?: { value?: unknown; description?: string }; exceptionDetails?: { text?: string } };
+
+  if (result.exceptionDetails) {
+    return { content: [{ type: 'text', text: `Error evaluating JS: ${result.exceptionDetails.text ?? 'Evaluation failed'}` }], isError: true };
+  }
+
+  const value = result.result?.value;
+  const text = typeof value === 'string' ? value : JSON.stringify(value, null, 2);
+  return { content: [{ type: 'text', text: `Result from "${tab.title}":\n${text}` }] };
+}
+
+async function actionPageInfo(port: number, token: string, tabId?: string): Promise<ToolResult> {
+  const tab = await resolveTab(port, token, tabId);
+  const result = (await sendCdpCommand(port, token, tab.id, 'Runtime.evaluate', {
+    expression: 'JSON.stringify({ title: document.title, url: location.href })',
+    returnByValue: true,
+  })) as { result?: { value?: string } };
+
+  let info: { title: string; url: string };
+  try {
+    info = JSON.parse(result.result?.value ?? '{}');
+  } catch {
+    info = { title: tab.title, url: tab.url };
+  }
+
+  return { content: [{ type: 'text', text: `Title: ${info.title}\nURL: ${info.url}\nTab ID: ${tab.id}` }] };
+}
+
+async function actionActivate(port: number, token: string, tabId?: string): Promise<ToolResult> {
+  const tab = await resolveTab(port, token, tabId);
+  const res = await fetch(`http://127.0.0.1:${port}/json/activate/${encodeURIComponent(tab.id)}`, {
+    headers: { [RELAY_AUTH_HEADER]: token },
+  });
+  if (!res.ok) throw new Error(`Failed to activate tab (${res.status})`);
+  return { content: [{ type: 'text', text: `Activated tab "${tab.title}"` }] };
+}
+
+async function actionClose(port: number, token: string, tabId?: string): Promise<ToolResult> {
+  const tab = await resolveTab(port, token, tabId);
+  const res = await fetch(`http://127.0.0.1:${port}/json/close/${encodeURIComponent(tab.id)}`, {
+    headers: { [RELAY_AUTH_HEADER]: token },
+  });
+  if (!res.ok) throw new Error(`Failed to close tab (${res.status})`);
+  return { content: [{ type: 'text', text: `Closed tab "${tab.title}"` }] };
+}
+
+// --- Main ---
+
+export async function execute(_toolCallId: string, params: Params): Promise<ToolResult> {
+  const config = getConfig();
+  if (!config) {
+    return {
+      content: [{ type: 'text', text: 'Browser Relay is not available. The user needs to connect the Browser Relay extension in Settings → Integrations.' }],
+      isError: true,
+    };
+  }
+
+  const { port, token } = config;
+  const { action, tab_id, url, expression } = params;
+
+  try {
+    switch (action) {
+      case 'list_tabs':
+        return await actionListTabs(port, token);
+      case 'screenshot':
+        return await actionScreenshot(port, token, tab_id);
+      case 'navigate':
+        if (!url) return { content: [{ type: 'text', text: 'url is required for navigate action' }], isError: true };
+        return await actionNavigate(port, token, url, tab_id);
+      case 'evaluate':
+        if (!expression) return { content: [{ type: 'text', text: 'expression is required for evaluate action' }], isError: true };
+        return await actionEvaluate(port, token, expression, tab_id);
+      case 'page_info':
+        return await actionPageInfo(port, token, tab_id);
+      case 'activate':
+        return await actionActivate(port, token, tab_id);
+      case 'close':
+        return await actionClose(port, token, tab_id);
+      default:
+        return {
+          content: [{ type: 'text', text: `Unknown action: ${action}. Use list_tabs, screenshot, navigate, evaluate, page_info, activate, or close.` }],
+          isError: true,
+        };
+    }
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    return { content: [{ type: 'text', text: `Browser error: ${message}` }], isError: true };
+  }
+}