From 6d0d103c785aee2f013e5fccfb8cad1e5c00d08f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Padez?= Date: Fri, 7 Aug 2026 02:54:39 +0000 Subject: [PATCH] carry attached images through to the model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The composer already uploaded an image, split its data URL and put the bytes on the wire as `images`. Nothing on the server read them. The `chat` ClientMessage had no such field, and the prompt reached the sidecar as a bare string, so all the model ever saw was the client-generated `[Attached image: …]` placeholder — a label describing a picture it was never shown. The transport was never the obstacle: `query()` consumes an async iterable of user messages whose `content` is an Anthropic `MessageParam`, and only `pushTurn` hardcoding a string kept it to text. So `images` is threaded through the four hops that dropped it and turned into native image content blocks at the end, renaming `mediaType` to the API's `media_type` at that last step. Co-Authored-By: Claude Opus 5 --- src/servers/api/chat/types.ts | 10 ++++++ src/servers/api/chat/websocket.ts | 6 +++- src/servers/channels/send-claude-code.ts | 3 +- src/servers/sidecar/claude/claude-manager.ts | 33 ++++++++++++++++---- src/servers/sidecar/protocol.ts | 3 +- 5 files changed, 46 insertions(+), 9 deletions(-) diff --git a/src/servers/api/chat/types.ts b/src/servers/api/chat/types.ts index 948d7942..405db5d6 100644 --- a/src/servers/api/chat/types.ts +++ b/src/servers/api/chat/types.ts @@ -45,6 +45,15 @@ export type GroupMeta = { export type ThinkingLevel = 'off' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh'; +/** + * An image pasted or attached in the composer, already split out of its data URL by the client. + * + * `/upload` also writes the file to disk, but the on-disk copy is for the attachment list and history — + * what the model actually reads is these bytes, carried through to a native image content block. Naming + * is camelCase on the wire and renamed to the Anthropic `media_type` at the last step. + */ +export type PromptImage = { mediaType: string; data: string }; + export type ClientMessage = | { type: 'chat'; @@ -56,6 +65,7 @@ export type ClientMessage = cwdRoot?: string; groupSlug?: string; attachmentIds?: string[]; + images?: PromptImage[]; thinking?: ThinkingLevel; context?: string; contextId?: string; diff --git a/src/servers/api/chat/websocket.ts b/src/servers/api/chat/websocket.ts index 7fd52e9e..d73b28b6 100644 --- a/src/servers/api/chat/websocket.ts +++ b/src/servers/api/chat/websocket.ts @@ -1,6 +1,6 @@ import type { ServerWebSocket } from 'bun'; import { randomUUID } from 'crypto'; -import type { ClientMessage, ServerMessage, Message, TurnMessage, UserSession } from './types'; +import type { ClientMessage, ServerMessage, Message, PromptImage, TurnMessage, UserSession } from './types'; import { sessionManager } from './session-manager'; import { sendClaudeCodeStreaming } from '@@/channels/send-claude-code'; import { sendOpenCodeStreaming } from '@@/channels/send-opencode'; @@ -248,6 +248,7 @@ async function handleChat( cwdRoot?: string; groupSlug?: string; attachmentIds?: string[]; + images?: PromptImage[]; thinking?: string; context?: string; contextId?: string; @@ -286,6 +287,7 @@ async function handleClaudeCodeChat( contextId?: string; cwd?: string; cwdRoot?: string; + images?: PromptImage[]; resumeSessionId?: string; }, effectivePrompt: string, @@ -338,6 +340,7 @@ async function handleClaudeCodeChat( email, username, prompt: effectivePrompt, + images: msg.images, sessionKey: sessionId, cwd, model, @@ -353,6 +356,7 @@ async function handleClaudeCodeChat( email, username, prompt: effectivePrompt, + images: msg.images, sessionKey: sessionId, cwd, model, diff --git a/src/servers/channels/send-claude-code.ts b/src/servers/channels/send-claude-code.ts index 7f54883c..3b2c5296 100644 --- a/src/servers/channels/send-claude-code.ts +++ b/src/servers/channels/send-claude-code.ts @@ -1,5 +1,5 @@ import { logger } from '@@/api/chat/logger'; -import type { MessageCost, TurnMessage } from '@@/api/chat/types'; +import type { MessageCost, PromptImage, TurnMessage } from '@@/api/chat/types'; import * as sidecar from '@@/sidecar-registry'; type ClaudeCodeParams = { @@ -34,6 +34,7 @@ type ClaudeCodeStreamingParams = { email: string; username: string; prompt: string; + images?: PromptImage[]; sessionKey: string; cwd?: string; model?: string; diff --git a/src/servers/sidecar/claude/claude-manager.ts b/src/servers/sidecar/claude/claude-manager.ts index 3e5d412d..d2259619 100644 --- a/src/servers/sidecar/claude/claude-manager.ts +++ b/src/servers/sidecar/claude/claude-manager.ts @@ -2,7 +2,7 @@ import { existsSync } from 'node:fs'; import { homedir } from 'node:os'; import { join } from 'node:path'; import { query, type Query } from '@anthropic-ai/claude-agent-sdk'; -import type { ChatEvent } from '../../api/chat/types'; +import type { ChatEvent, PromptImage } from '../../api/chat/types'; import type { ClaudeSpawnParams, ClaudeSpawnStreamingParams, ClaudeCodeResult } from '../protocol'; import { setClaudeSession, clearClaudeSession, getClaudeSession } from './state'; import { createParseState, processMessage } from './stream-parser'; @@ -147,9 +147,18 @@ export async function spawnClaude(params: ClaudeSpawnParams): Promise void; + pushTurn: (prompt: string, images?: PromptImage[]) => void; closeInput: () => void; abort: AbortController; emit: (event: ChatEvent) => void; @@ -264,12 +273,24 @@ function createSession(params: ClaudeSpawnStreamingParams, onEvent: (event: Chat }); session.query = q; - session.pushTurn = (prompt: string) => { + session.pushTurn = (prompt: string, images?: PromptImage[]) => { if (session.idleTimer) clearTimeout(session.idleTimer); session.isGenerating = true; + // Images first, then the text: the model reads what it is looking at before what to do about it. + const content: string | ContentBlock[] = images?.length + ? [ + ...images.map( + (img): ContentBlock => ({ + type: 'image', + source: { type: 'base64', media_type: img.mediaType, data: img.data }, + }), + ), + { type: 'text', text: prompt }, + ] + : prompt; input.push({ type: 'user', - message: { role: 'user', content: prompt }, + message: { role: 'user', content }, parent_tool_use_id: null, session_id: sessionKey, }); @@ -333,7 +354,7 @@ export async function spawnClaudeStreaming( } else { session = createSession(params, onEvent); } - session.pushTurn(params.prompt); + session.pushTurn(params.prompt, params.images); } /** Interrupt the current turn but KEEP the session alive (the "stop" button). */ diff --git a/src/servers/sidecar/protocol.ts b/src/servers/sidecar/protocol.ts index 7ee07bdb..129bda53 100644 --- a/src/servers/sidecar/protocol.ts +++ b/src/servers/sidecar/protocol.ts @@ -1,4 +1,4 @@ -import type { MessageCost, TurnMessage } from '../api/chat/types'; +import type { MessageCost, PromptImage, TurnMessage } from '../api/chat/types'; // ── Envelope ── @@ -121,6 +121,7 @@ export type ClaudeSpawnStreamingParams = { email: string; username: string; prompt: string; + images?: PromptImage[]; // sent as native image content blocks alongside the prompt text sessionKey: string; cwd?: string; model?: string;