carry attached images through to the model
The composer already uploaded an image, split its data URL and put the bytes on the wire as `images`. Nothing on the server read them. The `chat` ClientMessage had no such field, and the prompt reached the sidecar as a bare string, so all the model ever saw was the client-generated `[Attached image: …]` placeholder — a label describing a picture it was never shown. The transport was never the obstacle: `query()` consumes an async iterable of user messages whose `content` is an Anthropic `MessageParam`, and only `pushTurn` hardcoding a string kept it to text. So `images` is threaded through the four hops that dropped it and turned into native image content blocks at the end, renaming `mediaType` to the API's `media_type` at that last step. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -45,6 +45,15 @@ export type GroupMeta = {
|
|||||||
|
|
||||||
export type ThinkingLevel = 'off' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh';
|
export type ThinkingLevel = 'off' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* An image pasted or attached in the composer, already split out of its data URL by the client.
|
||||||
|
*
|
||||||
|
* `/upload` also writes the file to disk, but the on-disk copy is for the attachment list and history —
|
||||||
|
* what the model actually reads is these bytes, carried through to a native image content block. Naming
|
||||||
|
* is camelCase on the wire and renamed to the Anthropic `media_type` at the last step.
|
||||||
|
*/
|
||||||
|
export type PromptImage = { mediaType: string; data: string };
|
||||||
|
|
||||||
export type ClientMessage =
|
export type ClientMessage =
|
||||||
| {
|
| {
|
||||||
type: 'chat';
|
type: 'chat';
|
||||||
@@ -56,6 +65,7 @@ export type ClientMessage =
|
|||||||
cwdRoot?: string;
|
cwdRoot?: string;
|
||||||
groupSlug?: string;
|
groupSlug?: string;
|
||||||
attachmentIds?: string[];
|
attachmentIds?: string[];
|
||||||
|
images?: PromptImage[];
|
||||||
thinking?: ThinkingLevel;
|
thinking?: ThinkingLevel;
|
||||||
context?: string;
|
context?: string;
|
||||||
contextId?: string;
|
contextId?: string;
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import type { ServerWebSocket } from 'bun';
|
import type { ServerWebSocket } from 'bun';
|
||||||
import { randomUUID } from 'crypto';
|
import { randomUUID } from 'crypto';
|
||||||
import type { ClientMessage, ServerMessage, Message, TurnMessage, UserSession } from './types';
|
import type { ClientMessage, ServerMessage, Message, PromptImage, TurnMessage, UserSession } from './types';
|
||||||
import { sessionManager } from './session-manager';
|
import { sessionManager } from './session-manager';
|
||||||
import { sendClaudeCodeStreaming } from '@@/channels/send-claude-code';
|
import { sendClaudeCodeStreaming } from '@@/channels/send-claude-code';
|
||||||
import { sendOpenCodeStreaming } from '@@/channels/send-opencode';
|
import { sendOpenCodeStreaming } from '@@/channels/send-opencode';
|
||||||
@@ -248,6 +248,7 @@ async function handleChat(
|
|||||||
cwdRoot?: string;
|
cwdRoot?: string;
|
||||||
groupSlug?: string;
|
groupSlug?: string;
|
||||||
attachmentIds?: string[];
|
attachmentIds?: string[];
|
||||||
|
images?: PromptImage[];
|
||||||
thinking?: string;
|
thinking?: string;
|
||||||
context?: string;
|
context?: string;
|
||||||
contextId?: string;
|
contextId?: string;
|
||||||
@@ -286,6 +287,7 @@ async function handleClaudeCodeChat(
|
|||||||
contextId?: string;
|
contextId?: string;
|
||||||
cwd?: string;
|
cwd?: string;
|
||||||
cwdRoot?: string;
|
cwdRoot?: string;
|
||||||
|
images?: PromptImage[];
|
||||||
resumeSessionId?: string;
|
resumeSessionId?: string;
|
||||||
},
|
},
|
||||||
effectivePrompt: string,
|
effectivePrompt: string,
|
||||||
@@ -338,6 +340,7 @@ async function handleClaudeCodeChat(
|
|||||||
email,
|
email,
|
||||||
username,
|
username,
|
||||||
prompt: effectivePrompt,
|
prompt: effectivePrompt,
|
||||||
|
images: msg.images,
|
||||||
sessionKey: sessionId,
|
sessionKey: sessionId,
|
||||||
cwd,
|
cwd,
|
||||||
model,
|
model,
|
||||||
@@ -353,6 +356,7 @@ async function handleClaudeCodeChat(
|
|||||||
email,
|
email,
|
||||||
username,
|
username,
|
||||||
prompt: effectivePrompt,
|
prompt: effectivePrompt,
|
||||||
|
images: msg.images,
|
||||||
sessionKey: sessionId,
|
sessionKey: sessionId,
|
||||||
cwd,
|
cwd,
|
||||||
model,
|
model,
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { logger } from '@@/api/chat/logger';
|
import { logger } from '@@/api/chat/logger';
|
||||||
import type { MessageCost, TurnMessage } from '@@/api/chat/types';
|
import type { MessageCost, PromptImage, TurnMessage } from '@@/api/chat/types';
|
||||||
import * as sidecar from '@@/sidecar-registry';
|
import * as sidecar from '@@/sidecar-registry';
|
||||||
|
|
||||||
type ClaudeCodeParams = {
|
type ClaudeCodeParams = {
|
||||||
@@ -34,6 +34,7 @@ type ClaudeCodeStreamingParams = {
|
|||||||
email: string;
|
email: string;
|
||||||
username: string;
|
username: string;
|
||||||
prompt: string;
|
prompt: string;
|
||||||
|
images?: PromptImage[];
|
||||||
sessionKey: string;
|
sessionKey: string;
|
||||||
cwd?: string;
|
cwd?: string;
|
||||||
model?: string;
|
model?: string;
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import { existsSync } from 'node:fs';
|
|||||||
import { homedir } from 'node:os';
|
import { homedir } from 'node:os';
|
||||||
import { join } from 'node:path';
|
import { join } from 'node:path';
|
||||||
import { query, type Query } from '@anthropic-ai/claude-agent-sdk';
|
import { query, type Query } from '@anthropic-ai/claude-agent-sdk';
|
||||||
import type { ChatEvent } from '../../api/chat/types';
|
import type { ChatEvent, PromptImage } from '../../api/chat/types';
|
||||||
import type { ClaudeSpawnParams, ClaudeSpawnStreamingParams, ClaudeCodeResult } from '../protocol';
|
import type { ClaudeSpawnParams, ClaudeSpawnStreamingParams, ClaudeCodeResult } from '../protocol';
|
||||||
import { setClaudeSession, clearClaudeSession, getClaudeSession } from './state';
|
import { setClaudeSession, clearClaudeSession, getClaudeSession } from './state';
|
||||||
import { createParseState, processMessage } from './stream-parser';
|
import { createParseState, processMessage } from './stream-parser';
|
||||||
@@ -147,9 +147,18 @@ export async function spawnClaude(params: ClaudeSpawnParams): Promise<ClaudeCode
|
|||||||
// ChatEvent and forwards it. Idle policy: torn down after IDLE_TIMEOUT_MS with no new turn, or on an
|
// ChatEvent and forwards it. Idle policy: torn down after IDLE_TIMEOUT_MS with no new turn, or on an
|
||||||
// explicit kill (user "disconnect"). "Stop" is interrupt() — it ends the turn but keeps the session.
|
// explicit kill (user "disconnect"). "Stop" is interrupt() — it ends the turn but keeps the session.
|
||||||
|
|
||||||
|
/**
|
||||||
|
* `content` is an Anthropic `MessageParam` content: a bare string, or the block array we build when the
|
||||||
|
* turn carries images. Blocks are what make an attached screenshot actually reach the model — the string
|
||||||
|
* form can only carry the client's `[Attached image: …]` placeholder, which is a label, not the picture.
|
||||||
|
*/
|
||||||
|
type ContentBlock =
|
||||||
|
| { type: 'text'; text: string }
|
||||||
|
| { type: 'image'; source: { type: 'base64'; media_type: string; data: string } };
|
||||||
|
|
||||||
type SdkUserMessage = {
|
type SdkUserMessage = {
|
||||||
type: 'user';
|
type: 'user';
|
||||||
message: { role: 'user'; content: string };
|
message: { role: 'user'; content: string | ContentBlock[] };
|
||||||
parent_tool_use_id: null;
|
parent_tool_use_id: null;
|
||||||
session_id: string;
|
session_id: string;
|
||||||
};
|
};
|
||||||
@@ -157,7 +166,7 @@ type SdkUserMessage = {
|
|||||||
type PersistentSession = {
|
type PersistentSession = {
|
||||||
sessionKey: string;
|
sessionKey: string;
|
||||||
query: Query;
|
query: Query;
|
||||||
pushTurn: (prompt: string) => void;
|
pushTurn: (prompt: string, images?: PromptImage[]) => void;
|
||||||
closeInput: () => void;
|
closeInput: () => void;
|
||||||
abort: AbortController;
|
abort: AbortController;
|
||||||
emit: (event: ChatEvent) => void;
|
emit: (event: ChatEvent) => void;
|
||||||
@@ -264,12 +273,24 @@ function createSession(params: ClaudeSpawnStreamingParams, onEvent: (event: Chat
|
|||||||
});
|
});
|
||||||
|
|
||||||
session.query = q;
|
session.query = q;
|
||||||
session.pushTurn = (prompt: string) => {
|
session.pushTurn = (prompt: string, images?: PromptImage[]) => {
|
||||||
if (session.idleTimer) clearTimeout(session.idleTimer);
|
if (session.idleTimer) clearTimeout(session.idleTimer);
|
||||||
session.isGenerating = true;
|
session.isGenerating = true;
|
||||||
|
// Images first, then the text: the model reads what it is looking at before what to do about it.
|
||||||
|
const content: string | ContentBlock[] = images?.length
|
||||||
|
? [
|
||||||
|
...images.map(
|
||||||
|
(img): ContentBlock => ({
|
||||||
|
type: 'image',
|
||||||
|
source: { type: 'base64', media_type: img.mediaType, data: img.data },
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
{ type: 'text', text: prompt },
|
||||||
|
]
|
||||||
|
: prompt;
|
||||||
input.push({
|
input.push({
|
||||||
type: 'user',
|
type: 'user',
|
||||||
message: { role: 'user', content: prompt },
|
message: { role: 'user', content },
|
||||||
parent_tool_use_id: null,
|
parent_tool_use_id: null,
|
||||||
session_id: sessionKey,
|
session_id: sessionKey,
|
||||||
});
|
});
|
||||||
@@ -333,7 +354,7 @@ export async function spawnClaudeStreaming(
|
|||||||
} else {
|
} else {
|
||||||
session = createSession(params, onEvent);
|
session = createSession(params, onEvent);
|
||||||
}
|
}
|
||||||
session.pushTurn(params.prompt);
|
session.pushTurn(params.prompt, params.images);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Interrupt the current turn but KEEP the session alive (the "stop" button). */
|
/** Interrupt the current turn but KEEP the session alive (the "stop" button). */
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import type { MessageCost, TurnMessage } from '../api/chat/types';
|
import type { MessageCost, PromptImage, TurnMessage } from '../api/chat/types';
|
||||||
|
|
||||||
// ── Envelope ──
|
// ── Envelope ──
|
||||||
|
|
||||||
@@ -121,6 +121,7 @@ export type ClaudeSpawnStreamingParams = {
|
|||||||
email: string;
|
email: string;
|
||||||
username: string;
|
username: string;
|
||||||
prompt: string;
|
prompt: string;
|
||||||
|
images?: PromptImage[]; // sent as native image content blocks alongside the prompt text
|
||||||
sessionKey: string;
|
sessionKey: string;
|
||||||
cwd?: string;
|
cwd?: string;
|
||||||
model?: string;
|
model?: string;
|
||||||
|
|||||||
Reference in New Issue
Block a user