diff --git a/sidecar/catalogue/shipped.ts b/sidecar/catalogue/shipped.ts index b2fae83..c47a944 100644 --- a/sidecar/catalogue/shipped.ts +++ b/sidecar/catalogue/shipped.ts @@ -83,5 +83,49 @@ export const SHIPPED_CATALOGUE: Catalogue = { }, ], }, + { + /** + * The first LOCAL provider: sd-server on this machine's GPU, not a vendor. + * + * No `envVar`, because there is no key and never will be — see `keyless` on the adapter. It is + * `configured` by existing. + * + * The id is `sd-cpp-local` and not `qwen-image-2.1` because that is what `GET /v1/models` + * returns, hardcoded in sd-server with no override. The catalogue is the only side that can + * move, so it moves: the id agrees with the server, and the LABEL carries what the thing + * actually is. + */ + name: 'qwen-local', + label: 'Qwen-Image (local)', + defaultModel: 'sd-cpp-local', + models: [ + { + id: 'sd-cpp-local', + /** + * The timings are in the label because there is nowhere else for them, and a user choosing + * 1024 should know before they wait rather than after. Measured from the server's own log on + * this machine: 512 completes in ~185s; 1024 samples at ~41.4s/step over 20 steps. + * + * A `note` field on CatalogueModel would be the cleaner home for this. Not added on a guess — + * it needs the UI to render it, and a field nothing displays is worse than a long label. + */ + label: 'Qwen-Image 2.1 (local) — 512 takes ~3 min, 1024 ~14 min', + kind: 'image', + /** + * 512 first, and the adapter substitutes it for `auto` or for nothing. 1024 is offered + * because it is genuinely better and sometimes worth the wait — but a default nobody chose + * should not spend a quarter of an hour of the only GPU there is. + */ + sizes: ['512x512', '768x768', '1024x1024'], + transparency: false, + /** No `edit()` in this version — same GPU, same queue, and the generate path is unproven. */ + edit: false, + /** Zero is the true answer, not a missing one: it runs on hardware we already own. */ + pricing: { unit: 'image', perImage: 0 }, + status: 'current', + source: 'shipped', + }, + ], + }, ], }; diff --git a/sidecar/catalogue/types.ts b/sidecar/catalogue/types.ts index e76613a..6e6087b 100644 --- a/sidecar/catalogue/types.ts +++ b/sidecar/catalogue/types.ts @@ -71,8 +71,14 @@ export type CatalogueProvider = { /** Matches an adapter's `name` when code for it exists. A provider with no adapter is a listing, not an offer. */ name: string; label: string; - /** The bootstrap credential's variable, for a connection with no stored key. */ - envVar: string; + /** + * The bootstrap credential's variable, for a connection with no stored key. + * + * Optional since the first LOCAL provider: a service of ours, on this machine, authenticates with + * nothing, and naming an environment variable for a key that does not exist would describe a way to + * configure it that does not work. See `keyless` on `ImageAdapter`. + */ + envVar?: string; defaultModel?: string; models: CatalogueModel[]; }; diff --git a/sidecar/providers/credentials.ts b/sidecar/providers/credentials.ts index da17c34..536e57d 100644 --- a/sidecar/providers/credentials.ts +++ b/sidecar/providers/credentials.ts @@ -23,6 +23,19 @@ import type { ImageAdapter } from './types'; * The catalogue says which providers exist; the secret store says which of them you can use. */ +/** + * Does this provider authenticate with anything at all? + * + * Asked of the ADAPTER, because it is a fact about the code rather than about configuration: a local + * service of ours takes no key and never will, so there is nothing a user could set or forget. Every + * "is it configured?" test below has to consult this FIRST — otherwise a keyless provider reads as + * unconfigured forever and can never be selected. + */ +export const isKeyless = (provider: string): boolean => adapterByName(provider)?.keyless === true; + +/** Usable means: no key needed, or a key we hold. The single test the resolver and the UI both want. */ +export const isUsable = (provider: string): boolean => isKeyless(provider) || Boolean(keyFor(provider)); + /** `openai` -> `atelier-openai-key`. The store creates purposes on first use, so nothing is provisioned. */ export const secretPurposeFor = (provider: string) => `atelier-${provider}-key`; @@ -47,7 +60,8 @@ export function keyFor(provider: string): string | undefined { } /** Where a key came from, for a UI that should say "from the environment" rather than show an empty field. */ -export function keySource(provider: string): 'store' | 'environment' | null { +export function keySource(provider: string): 'store' | 'environment' | 'keyless' | null { + if (isKeyless(provider)) return 'keyless'; try { if (hasKey(secretPurposeFor(provider))) return 'store'; } catch { @@ -101,16 +115,21 @@ export async function resolveForRequest(options: { // A model nobody in the catalogue claims is not refused: the catalogue is advisory, and a model // released today should work before we have shipped a file that mentions it. The first provider we // hold a key for gets it, and the vendor gives the real answer. - if (!name) name = catalogue.providers.find((entry) => keyFor(entry.name))?.name; + if (!name) name = catalogue.providers.find((entry) => isUsable(entry.name))?.name; } else { - name = catalogue.providers.find((entry) => keyFor(entry.name))?.name; + name = catalogue.providers.find((entry) => isUsable(entry.name))?.name; } - if (!name) return { error: 'No provider has an API key yet. Add one in Atelier.', status: 503 }; + if (!name) return { error: 'No provider is usable yet. Add an API key in Atelier.', status: 503 }; const adapter = adapterByName(name); if (!adapter) return { error: `Provider "${name}" has no adapter in this build.`, status: 400 }; + // A keyless adapter must not reach `keyFor` at all. The comment on it is the reason: it uses + // `getSecret` rather than `getKey` precisely so a missing FOREIGN credential is never minted, and a + // provider that has no credential by design has no business in that code path. + if (adapter.keyless) return { providerName: name, adapter, apiKey: '' }; + const apiKey = keyFor(name); if (!apiKey) return { error: `Provider "${name}" has no API key.`, status: 503 }; @@ -133,12 +152,17 @@ export async function providerStatuses() { const models = provider.models.filter((model) => model.status !== 'deprecated'); const source = keySource(provider.name); + const keyless = isKeyless(provider.name); + return { name: provider.name, label: provider.label, - configured: Boolean(keyFor(provider.name)), + /** Keyless is configured by existing. There is nothing to add, so nothing can be missing. */ + configured: keyless || Boolean(keyFor(provider.name)), + keyless, key_source: source, - env_var: envVarFor(provider.name), + /** Absent for a keyless provider: naming a variable nobody can usefully set would be a wrong hint. */ + env_var: keyless ? null : envVarFor(provider.name), /** False for a catalogue entry we have no code for — listed, but not yet usable. */ usable: Boolean(adapter), supports_edit: typeof adapter?.edit === 'function', diff --git a/sidecar/providers/index.ts b/sidecar/providers/index.ts index 58a762e..66a9829 100644 --- a/sidecar/providers/index.ts +++ b/sidecar/providers/index.ts @@ -1,4 +1,5 @@ import { openaiAdapter } from './openai'; +import { qwenLocalAdapter } from './qwen-local'; import type { ImageAdapter } from './types'; export * from './types'; @@ -14,7 +15,7 @@ export * from './types'; * A list in one place, the way `hono.ts` keeps its mounts: adding an adapter is a new file plus a * line here. */ -export const ADAPTERS: ImageAdapter[] = [openaiAdapter]; +export const ADAPTERS: ImageAdapter[] = [openaiAdapter, qwenLocalAdapter]; export function adapterByName(name: string): ImageAdapter | undefined { return ADAPTERS.find((adapter) => adapter.name === name); diff --git a/sidecar/providers/qwen-local.ts b/sidecar/providers/qwen-local.ts new file mode 100644 index 0000000..92e0040 --- /dev/null +++ b/sidecar/providers/qwen-local.ts @@ -0,0 +1,135 @@ +import type { ImageAdapter, ImageRequest, ImageUsage, ProviderResult } from './types'; + +/** + * Qwen-Image 2.1 on our own GPU, via `sd-server` from stable-diffusion.cpp with the Vulkan backend. + * + * It already speaks OpenAI's image API, which is the vocabulary `ImageRequest` is written in, so there + * is no translation layer — only the things below that are true of a local GPU and not of a hosted API. + * + * ── It is slow, and the number matters ── + * + * Measured on this machine from the server's own log, not estimated: 512x512 completes in ~185s, and + * 1024x1024 samples at ~41.4s per step over 20 steps, so about 14 minutes. The catalogue therefore + * defaults to 512 and offers 1024 labelled for what it costs — a default nobody chose should not spend a + * quarter of an hour of the only GPU there is. + * + * ── `timeout: false`, and it is not caution ── + * + * The server serialises strictly: a second caller waits for the first to finish ENTIRELY before its own + * first step. So a queued 512 request can wait its own 3 minutes plus a 14-minute job ahead of it — 17 + * minutes before a single byte comes back. Bun's fetch has a default timeout, and every default anyone + * would reach for is far under that. The caller's `signal` is still honoured, because a user cancelling + * is a different thing from a client giving up. + * + * ── No `edit()` in this version ── + * + * Deliberate. It is the same GPU and the same serialisation, so it doubles the ways the queue can be + * blocked while the generate path is still unproven in anger. `ImageAdapter` makes absence the way to + * say a provider cannot do something, and the route answers 400 rather than the queue finding out late. + */ + +/** + * Where the server is. Overridable because the same service answers on the tailnet + * (`http://100.64.0.1:8188`), and a second machine should not need a code change to reach it. + */ +const BASE = process.env.ATELIER_QWEN_LOCAL_URL ?? 'http://127.0.0.1:8188'; + +/** + * What `GET /v1/models` actually returns, which is not what the model IS. + * + * sd-server hardcodes this id with no override, so the catalogue is the only side that can move: the id + * stays `sd-cpp-local` so the two agree, and the catalogue carries "Qwen-Image 2.1 (local)" as the + * label so the UI still says something meaningful. A catalogue that disagreed with `/v1/models` would + * make every request name a model the server denies knowing. + */ +const MODEL_ID = 'sd-cpp-local'; + +/** Sent when the caller names no size. See the timings above for why it is not 1024. */ +const DEFAULT_SIZE = '512x512'; + +type OpenAiImagesResponse = { + data?: { b64_json?: string; revised_prompt?: string }[]; + usage?: ImageUsage; + size?: string; + quality?: string; + background?: string; +}; + +/** + * Only what this server understands. + * + * `quality`, `style`, `moderation` and `background` are OpenAI concepts with no Qwen equivalent — they + * are dropped rather than passed, because sd-server ignores unknown fields silently and a parameter + * that appears to have been honoured is worse than one that was visibly not offered. What was actually + * used comes back in `echoed`. + */ +function body(request: ImageRequest): Record { + return { + prompt: request.prompt, + model: MODEL_ID, + n: 1, + size: request.size && request.size !== 'auto' ? request.size : DEFAULT_SIZE, + response_format: 'b64_json', + ...(request.output_format ? { output_format: request.output_format } : {}), + }; +} + +export const qwenLocalAdapter: ImageAdapter = { + name: 'qwen-local', + label: 'Qwen-Image (local)', + models: [MODEL_ID], + defaultModel: MODEL_ID, + keyless: true, + + async generate(request, _apiKey, signal): Promise { + const sent = body(request); + + const response = await fetch(`${BASE}/v1/images/generations`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(sent), + signal, + // Bun extension. See the block comment: the honest worst case is 17 minutes. + timeout: false, + } as RequestInit); + + if (!response.ok) { + const detail = await response.text().catch(() => ''); + throw new Error(`qwen-local ${response.status}: ${detail.slice(0, 400) || response.statusText}`); + } + + const payload = (await response.json()) as OpenAiImagesResponse; + const entries = payload.data ?? []; + if (entries.length === 0) throw new Error('qwen-local returned no images'); + + const format = request.output_format ?? 'png'; + const images = entries + .filter((entry) => typeof entry.b64_json === 'string') + .map((entry) => ({ + bytes: Uint8Array.from(atob(entry.b64_json!), (char) => char.charCodeAt(0)), + format, + ...(entry.revised_prompt ? { revisedPrompt: entry.revised_prompt } : {}), + })); + + if (images.length === 0) throw new Error('qwen-local returned images with no b64_json'); + + return { + images, + usage: payload.usage, + // What was actually used, including the size we substituted — the caller asked for `auto` or for + // nothing and got 512, and the row should record that rather than the absence. + echoed: { model: MODEL_ID, size: sent.size, provider: 'qwen-local', ...(payload.quality ? { quality: payload.quality } : {}) }, + }; + }, + + /** + * Zero, and that is a fact rather than a gap. + * + * `null` here means "this model's price is not known", which is what the OpenAI adapter returns for + * models it has no row for. Our own GPU costs no dollars per image, so zero is the true answer and + * saying "unknown" would put this provider in the same bucket as one we simply failed to price. + */ + costUsd(_model: string, _usage: ImageUsage | undefined): string { + return '0.00'; + }, +}; diff --git a/sidecar/providers/types.ts b/sidecar/providers/types.ts index f22f20c..ab32f32 100644 --- a/sidecar/providers/types.ts +++ b/sidecar/providers/types.ts @@ -85,6 +85,17 @@ export type ImageAdapter = { /** Models this adapter is known to serve. Advisory — an unknown model is passed through. */ readonly models: readonly string[]; readonly defaultModel: string; + /** + * This adapter authenticates with nothing, because it talks to a service of OURS rather than a + * vendor's. Availability is then a fact about the code, not about what is in the secret store. + * + * The alternative was storing a dummy secret so the existing "has a key?" test would pass. That would + * make the store lie about what is configured, and the store is the one place that is supposed to be + * the truth about credentials. A flag that says "this question does not apply" is smaller and honest. + * + * A category rather than an exception: whisper, OCR and TTS already run on this machine too. + */ + readonly keyless?: boolean; /** * The key is passed in rather than read here: it can come from the database or the environment, and * which one is a question about configuration, not about how to talk to this provider. diff --git a/web/Generate/ProvidersBar.tsx b/web/Generate/ProvidersBar.tsx index 60267ca..09b5e65 100644 --- a/web/Generate/ProvidersBar.tsx +++ b/web/Generate/ProvidersBar.tsx @@ -74,7 +74,11 @@ export const ProvidersBar = ({ {entry.label} {/* Said here rather than discovered at Generate: a vendor with no key is listed precisely so you can add one, so the state has to be visible in the list. */} - {!entry.configured && no key} + {entry.keyless ? ( + local + ) : !entry.configured ? ( + no key + ) : null} ))} @@ -103,10 +107,16 @@ export const ProvidersBar = ({
- + {/* Hidden rather than disabled for a keyless provider. A greyed-out "Add key" invites the + question of what would enable it, and the answer is nothing — there is no key to add. */} + {provider?.keyless ? ( + Runs on this machine — no key needed + ) : ( + + )}