diff --git a/src/workspaces/officerdev/src/apps/Chat/components/downscale.ts b/src/workspaces/officerdev/src/apps/Chat/components/downscale.ts new file mode 100644 index 00000000..d384210f --- /dev/null +++ b/src/workspaces/officerdev/src/apps/Chat/components/downscale.ts @@ -0,0 +1,74 @@ +/** + * A screenshot, shrunk to the largest size the model will actually look at. + * + * ── Why there is a ceiling at all ── + * + * An attached image goes to the model as base64 INSIDE the message, and it stays there: every later + * turn in the conversation re-sends every earlier picture. So a screenshot is not paid for once, it is + * paid for on every turn for the rest of the session — in request bytes and in tokens. + * + * ── Why 1568 specifically ── + * + * It is the point past which Anthropic downscales server-side. A 2539px-wide screenshot is resized to + * 1568 before the model sees a pixel of it, so the extra width was never read — it was carried up the + * wire, counted, and thrown away. Sending it already small changes nothing about what the model can + * see and stops paying for what it cannot. + * + * Measured on one day of this session: 25 screenshots, 10 of them over the line, 53,300 tokens of + * image data — 23% of which bought nothing. + * + * ── The ORIGINAL is still uploaded ── + * + * This makes the copy that travels in the prompt. The file that lands on disk is untouched, so an agent + * that needs the detail can `Read` it at full size — which is the right split: the conversation carries + * what is worth re-reading every turn, and the disk keeps what was actually captured. + */ +const MAX_EDGE = 1568; + +/** JPEG for photographs would be smaller, and a screenshot of text is exactly where it looks worst. */ +const TYPE = 'image/png'; + +export async function downscaleForPrompt(file: File): Promise { + const bitmap = await createImageBitmap(file).catch(() => null); + if (!bitmap) return readAsDataUrl(file); + + const scale = Math.min(1, MAX_EDGE / Math.max(bitmap.width, bitmap.height)); + // Already small enough. Re-encoding it would cost quality and could cost SIZE — a PNG round-tripped + // through a canvas is often bigger than the one the screenshotter wrote. + if (scale === 1) { + bitmap.close(); + return readAsDataUrl(file); + } + + const width = Math.round(bitmap.width * scale); + const height = Math.round(bitmap.height * scale); + + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + + const context = canvas.getContext('2d'); + if (!context) { + bitmap.close(); + return readAsDataUrl(file); + } + + // `high` because this is text: the default is a box filter, and screenshot text resampled with one + // comes out fringed in a way that is harder to read than the same text smaller. + context.imageSmoothingEnabled = true; + context.imageSmoothingQuality = 'high'; + context.drawImage(bitmap, 0, 0, width, height); + bitmap.close(); + + return canvas.toDataURL(TYPE); +} + +/** The file as it is, for anything this cannot or should not shrink. */ +function readAsDataUrl(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(String(reader.result)); + reader.onerror = () => reject(reader.error); + reader.readAsDataURL(file); + }); +} diff --git a/src/workspaces/officerdev/src/apps/Chat/useAttachments.ts b/src/workspaces/officerdev/src/apps/Chat/useAttachments.ts index 4f59bb26..c68cafa8 100644 --- a/src/workspaces/officerdev/src/apps/Chat/useAttachments.ts +++ b/src/workspaces/officerdev/src/apps/Chat/useAttachments.ts @@ -1,4 +1,5 @@ import { useState } from 'react'; +import { downscaleForPrompt } from './components/downscale'; import { toast } from 'sonner'; import { useClient } from 'hooks/useClient'; import type { Attachment } from './types'; @@ -52,11 +53,22 @@ export function useAttachments(params?: UseAttachmentsParams) { if (params?.sessionId) formData.append('sessionId', params.sessionId); formData.append('provider', 'chat'); - const res = await client.post<{ filename: string; dataUrl: string; attachmentId: string }>('/upload', formData); + /* + The original is uploaded and a SMALLER copy is what travels in the prompt. + + The server's `dataUrl` is the file byte for byte, and that copy is re-sent on every turn for the + rest of the conversation. Past 1568px on the long edge the model never sees the extra pixels — + they are downscaled server-side before it looks — so the width was being carried and counted and + then thrown away. See downscale.ts. + + Both, deliberately: the disk keeps what was captured, so `Read` can still open it at full size. + */ + const [res, dataUrl] = await Promise.all([ + client.post<{ filename: string; dataUrl: string; attachmentId: string }>('/upload', formData), + downscaleForPrompt(file), + ]); setAttachments((prev) => - prev.map((a, i) => - i === idx ? { ...a, dataUrl: res.dataUrl, attachmentId: res.attachmentId, loading: false } : a, - ), + prev.map((a, i) => (i === idx ? { ...a, dataUrl, attachmentId: res.attachmentId, loading: false } : a)), ); } catch { setAttachments((prev) => prev.filter((_, i) => i !== idx));