carry attached images through to the model

The composer already uploaded an image, split its data URL and put the bytes on the wire as
`images`. Nothing on the server read them. The `chat` ClientMessage had no such field, and the
prompt reached the sidecar as a bare string, so all the model ever saw was the client-generated
`[Attached image: …]` placeholder — a label describing a picture it was never shown.

The transport was never the obstacle: `query()` consumes an async iterable of user messages whose
`content` is an Anthropic `MessageParam`, and only `pushTurn` hardcoding a string kept it to text.
So `images` is threaded through the four hops that dropped it and turned into native image content
blocks at the end, renaming `mediaType` to the API's `media_type` at that last step.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-07 02:54:39 +00:00
co-authored by Claude Opus 5
parent 5134501a2f
commit 6d0d103c78
5 changed files with 46 additions and 9 deletions
+27 -6
View File
@@ -2,7 +2,7 @@ import { existsSync } from 'node:fs';
import { homedir } from 'node:os';
import { join } from 'node:path';
import { query, type Query } from '@anthropic-ai/claude-agent-sdk';
import type { ChatEvent } from '../../api/chat/types';
import type { ChatEvent, PromptImage } from '../../api/chat/types';
import type { ClaudeSpawnParams, ClaudeSpawnStreamingParams, ClaudeCodeResult } from '../protocol';
import { setClaudeSession, clearClaudeSession, getClaudeSession } from './state';
import { createParseState, processMessage } from './stream-parser';
@@ -147,9 +147,18 @@ export async function spawnClaude(params: ClaudeSpawnParams): Promise<ClaudeCode
// ChatEvent and forwards it. Idle policy: torn down after IDLE_TIMEOUT_MS with no new turn, or on an
// explicit kill (user "disconnect"). "Stop" is interrupt() — it ends the turn but keeps the session.
/**
* `content` is an Anthropic `MessageParam` content: a bare string, or the block array we build when the
* turn carries images. Blocks are what make an attached screenshot actually reach the model — the string
* form can only carry the client's `[Attached image: …]` placeholder, which is a label, not the picture.
*/
type ContentBlock =
| { type: 'text'; text: string }
| { type: 'image'; source: { type: 'base64'; media_type: string; data: string } };
type SdkUserMessage = {
type: 'user';
message: { role: 'user'; content: string };
message: { role: 'user'; content: string | ContentBlock[] };
parent_tool_use_id: null;
session_id: string;
};
@@ -157,7 +166,7 @@ type SdkUserMessage = {
type PersistentSession = {
sessionKey: string;
query: Query;
pushTurn: (prompt: string) => void;
pushTurn: (prompt: string, images?: PromptImage[]) => void;
closeInput: () => void;
abort: AbortController;
emit: (event: ChatEvent) => void;
@@ -264,12 +273,24 @@ function createSession(params: ClaudeSpawnStreamingParams, onEvent: (event: Chat
});
session.query = q;
session.pushTurn = (prompt: string) => {
session.pushTurn = (prompt: string, images?: PromptImage[]) => {
if (session.idleTimer) clearTimeout(session.idleTimer);
session.isGenerating = true;
// Images first, then the text: the model reads what it is looking at before what to do about it.
const content: string | ContentBlock[] = images?.length
? [
...images.map(
(img): ContentBlock => ({
type: 'image',
source: { type: 'base64', media_type: img.mediaType, data: img.data },
}),
),
{ type: 'text', text: prompt },
]
: prompt;
input.push({
type: 'user',
message: { role: 'user', content: prompt },
message: { role: 'user', content },
parent_tool_use_id: null,
session_id: sessionKey,
});
@@ -333,7 +354,7 @@ export async function spawnClaudeStreaming(
} else {
session = createSession(params, onEvent);
}
session.pushTurn(params.prompt);
session.pushTurn(params.prompt, params.images);
}
/** Interrupt the current turn but KEEP the session alive (the "stop" button). */
+2 -1
View File
@@ -1,4 +1,4 @@
import type { MessageCost, TurnMessage } from '../api/chat/types';
import type { MessageCost, PromptImage, TurnMessage } from '../api/chat/types';
// ── Envelope ──
@@ -121,6 +121,7 @@ export type ClaudeSpawnStreamingParams = {
email: string;
username: string;
prompt: string;
images?: PromptImage[]; // sent as native image content blocks alongside the prompt text
sessionKey: string;
cwd?: string;
model?: string;