From 0de38f0b1330a5a66cc6c7acc46785a55dc46ad0 Mon Sep 17 00:00:00 2001 From: Andre Padez Date: Thu, 20 Aug 2026 22:15:35 +0100 Subject: [PATCH] feat(catalogue): a curated file of providers and models, refreshed daily MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The five connections in the database were never configured by anyone — a migration minted one per model from the adapter's own list, so the table held five decisions nobody made. What that list actually is, is a CATALOGUE: what you can choose from, which is not the same kind of thing as what you chose. So it is a file. `sidecar/catalogue/shipped.ts` is the curated baseline, in git, reviewed; `$OFFICER_ROOT/data/atelier/catalogue.json` is what a refresh writes, and it wins where it has something to say. Data dir, not the plugin dir — a plugin directory is a clone an update replaces, which is the rule env.ts already states for images. Refresh runs in the sidecar, daily, and only on boot when the file is actually stale — this process has restarted twenty times in an afternoon and should not hit OpenAI twenty times for it. Two sources, unequal, and the code says so. `GET /v1/models` is authoritative and thin: ids and nothing else, so a model matching /^(gpt-image|dall-e)/ appears the day it ships without a release. Prices come off an HTML page because there is no pricing endpoint for any plan, and that parser is written to give up rather than guess — a guess gets written to disk and then believed. When it gives up it says so in the log; the durable replacement is handing the page to a model for structured output, which Officer can already do locally. Nothing a refresh cannot establish overwrites a curated value: fields merge one by one, not objects wholesale, so an id-only listing cannot wipe the sizes and prices that were written by hand. Retiring is decided by the merge, which can tell a complete listing from a request that failed halfway — and it marks `deprecated` rather than deleting, because the gallery labels 88 historical rows from this list. Per-model sizes, qualities and transparency now travel with each entry, which is what will let the form stop offering dall-e-2 a transparent background it cannot do. The catalogue is advisory throughout: validate.ts still refuses to enforce per-model rules, exactly as before. --- sidecar/api/providers.ts | 15 ++- sidecar/catalogue/index.ts | 4 + sidecar/catalogue/merge.test.ts | 41 ++++++++ sidecar/catalogue/openai-refresh.ts | 156 ++++++++++++++++++++++++++++ sidecar/catalogue/refresh.ts | 83 +++++++++++++++ sidecar/catalogue/shipped.ts | 87 ++++++++++++++++ sidecar/catalogue/store.ts | 104 +++++++++++++++++++ sidecar/catalogue/types.ts | 88 ++++++++++++++++ sidecar/index.ts | 5 + sidecar/providers/credentials.ts | 46 ++++++-- web/lib/api.ts | 24 ++++- 11 files changed, 641 insertions(+), 12 deletions(-) create mode 100644 sidecar/catalogue/index.ts create mode 100644 sidecar/catalogue/merge.test.ts create mode 100644 sidecar/catalogue/openai-refresh.ts create mode 100644 sidecar/catalogue/refresh.ts create mode 100644 sidecar/catalogue/shipped.ts create mode 100644 sidecar/catalogue/store.ts create mode 100644 sidecar/catalogue/types.ts diff --git a/sidecar/api/providers.ts b/sidecar/api/providers.ts index 2fdc1e2..070d0d9 100644 --- a/sidecar/api/providers.ts +++ b/sidecar/api/providers.ts @@ -3,6 +3,7 @@ import { createRouter } from '../create-router'; import { encrypt } from '../crypto'; import { adapterByName } from '../providers'; import { adapterCatalogue, providerStatuses } from '../providers/credentials'; +import { refreshCatalogue } from '../catalogue/refresh'; import { openaiError, UUID_PATTERN } from '../errors'; export const providersRouter = createRouter(); @@ -20,7 +21,19 @@ const notFound = (id: string) => openaiError(`No provider with id "${id}".`, 'no */ providersRouter.get('/', async (ctx) => ctx.json({ data: await providerStatuses() })); -providersRouter.get('/adapters', (ctx) => ctx.json({ data: adapterCatalogue() })); +providersRouter.get('/adapters', async (ctx) => ctx.json({ data: await adapterCatalogue() })); + +/** + * Refresh the catalogue now, rather than waiting for the daily pass. + * + * Here rather than on a timer alone because the first thing anyone does after adding a key is expect + * the model list to reflect it — the daily refresh has no key to work with until one exists, so the + * pass that matters most is the one immediately after. + */ +providersRouter.post('/catalogue/refresh', async (ctx) => { + const catalogue = await refreshCatalogue(); + return ctx.json({ data: { updated_at: catalogue.updatedAt, providers: catalogue.providers.length } }); +}); providersRouter.post('/', async (ctx) => { const body = (await ctx.req.json().catch(() => null)) as Record | null; diff --git a/sidecar/catalogue/index.ts b/sidecar/catalogue/index.ts new file mode 100644 index 0000000..c636810 --- /dev/null +++ b/sidecar/catalogue/index.ts @@ -0,0 +1,4 @@ +export * from './types'; +export { SHIPPED_CATALOGUE } from './shipped'; +export { getCatalogue, mergeCatalogue, invalidateCatalogue, CATALOGUE_PATH } from './store'; +export { refreshCatalogue, startCatalogueRefresh } from './refresh'; diff --git a/sidecar/catalogue/merge.test.ts b/sidecar/catalogue/merge.test.ts new file mode 100644 index 0000000..29128d4 --- /dev/null +++ b/sidecar/catalogue/merge.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, test } from 'bun:test'; +import { mergeCatalogue } from './store'; +import { SHIPPED_CATALOGUE } from './shipped'; +import type { Catalogue } from './types'; + +const price = (c: Catalogue, id: string) => + c.providers[0]!.models.find((m) => m.id === id)?.pricing as { imageOutput?: number } | undefined; + +describe('catalogue merge', () => { + test('a refresh that establishes nothing changes nothing', () => { + expect(mergeCatalogue(SHIPPED_CATALOGUE, null)).toEqual(SHIPPED_CATALOGUE); + }); + + test('an id-only refresh never wipes curated facts', () => { + const merged = mergeCatalogue(SHIPPED_CATALOGUE, { + revision: 1, + providers: [{ name: 'openai', label: 'OpenAI', envVar: 'OPENAI_API_KEY', models: [{ id: 'gpt-image-1', kind: 'image', source: 'api' }] }], + }); + const model = merged.providers[0]!.models.find((m) => m.id === 'gpt-image-1')!; + expect(model.sizes).toHaveLength(4); + expect(price(merged, 'gpt-image-1')?.imageOutput).toBe(40); + expect(model.source).toBe('api'); + }); + + test('a model we have never shipped is added, without an invented price', () => { + const merged = mergeCatalogue(SHIPPED_CATALOGUE, { + revision: 1, + providers: [{ name: 'openai', label: 'OpenAI', envVar: 'OPENAI_API_KEY', models: [{ id: 'gpt-image-2', kind: 'image', source: 'api' }] }], + }); + expect(merged.providers[0]!.models.some((m) => m.id === 'gpt-image-2')).toBe(true); + expect(price(merged, 'gpt-image-2')).toBeUndefined(); + }); + + test('a file from another revision is ignored entirely', () => { + const merged = mergeCatalogue(SHIPPED_CATALOGUE, { + revision: 99, + providers: [{ name: 'openai', label: 'nonsense', envVar: '', models: [] }], + }); + expect(merged.providers[0]!.label).toBe('OpenAI'); + }); +}); diff --git a/sidecar/catalogue/openai-refresh.ts b/sidecar/catalogue/openai-refresh.ts new file mode 100644 index 0000000..98722f2 --- /dev/null +++ b/sidecar/catalogue/openai-refresh.ts @@ -0,0 +1,156 @@ +import { listProviders } from '../db'; +import { keyForProvider } from '../providers/credentials'; +import { projectEnv } from '../env'; +import type { Catalogue, CatalogueModel } from './types'; +import { CATALOGUE_REVISION } from './types'; + +/** + * What a refresh can establish about OpenAI, from OpenAI. + * + * Two sources, and they are not equal: + * + * `GET /v1/models` authoritative, and thin. Ids, an owner, a created timestamp. Nothing else. + * the pricing page everything that actually matters, and it is HTML written for people. + * + * There is no pricing endpoint — none, for any plan — so the second source is a page and parsing it is + * inherently fragile. Which is why nothing here is allowed to overwrite a curated value with a guess: + * a field this cannot establish is returned `undefined`, and `mergeCatalogue` leaves the shipped value + * standing. + */ + +const MODELS_ENDPOINT = 'https://api.openai.com/v1/models'; + +/** Overridable, because the page this parses is the part most likely to move. */ +const PRICING_URL = projectEnv('ATELIER_OPENAI_PRICING_URL') ?? 'https://platform.openai.com/docs/pricing'; + +/** + * Which of OpenAI's models make pictures. + * + * A pattern, not a list, and that is the point of asking the API at all: `gpt-image-2` on release day + * matches this and appears in the add dialog without a plugin update. The cost of a pattern is the + * occasional false positive, which surfaces as a model that fails at the vendor rather than one that + * silently does the wrong thing. + */ +const IMAGE_MODEL = /^(gpt-image|dall-e)/i; + +/** A key from any connection, or the bootstrap environment variable. Refresh is read-only, so any will do. */ +async function anyKey(): Promise { + for (const provider of await listProviders()) { + if (provider.adapter !== 'openai') continue; + const key = await keyForProvider(provider); + if (key) return key; + } + return projectEnv('OPENAI_API_KEY'); +} + +async function fetchModelIds(apiKey: string): Promise { + try { + const response = await fetch(MODELS_ENDPOINT, { + headers: { authorization: `Bearer ${apiKey}` }, + signal: AbortSignal.timeout(30_000), + }); + if (!response.ok) { + console.warn(`[atelier:catalogue] models endpoint answered ${response.status}`); + return null; + } + const body = (await response.json()) as { data?: { id?: unknown }[] }; + return (body.data ?? []) + .map((entry) => entry.id) + .filter((id): id is string => typeof id === 'string' && IMAGE_MODEL.test(id)); + } catch (err) { + console.warn('[atelier:catalogue] could not list models:', err); + return null; + } +} + +/** + * Prices, off the pricing page. + * + * Deliberately narrow: it looks for a model id and reads the dollar figures that follow it in the same + * chunk of markup, and returns nothing at all when the page does not look the way it expects. A parser + * that guesses on an unfamiliar page is worse than one that gives up, because the guess is written to + * disk and then believed. + * + * This will fail the first time the page is restyled, and it is meant to fail LOUDLY — the log line is + * the signal to replace this function. The durable version of it is not a better regex: it is handing + * the page to a model and asking for structured output, which Officer can already do locally through + * its own Claude sidecar. That is the next step, not a different design. + */ +async function fetchPricing(modelIds: string[]): Promise> { + const prices = new Map(); + + try { + const response = await fetch(PRICING_URL, { signal: AbortSignal.timeout(30_000) }); + if (!response.ok) { + console.warn(`[atelier:catalogue] pricing page answered ${response.status} — keeping shipped prices`); + return prices; + } + + const html = await response.text(); + for (const id of modelIds) { + const at = html.indexOf(id); + if (at === -1) continue; + // The figures that follow the id, within one row's worth of markup. `$5.00`, `$0.01`, `5.00`. + const window = html.slice(at, at + 1200); + const found = [...window.matchAll(/\$\s?(\d+(?:\.\d+)?)/g)].map((match) => Number(match[1])); + if (found.length > 0) prices.set(id, found); + } + + if (prices.size === 0) { + console.warn( + `[atelier:catalogue] found no prices on ${PRICING_URL} — the page has changed shape, or renders its ` + + 'prices client-side. Shipped prices kept; this extractor needs replacing.', + ); + } + } catch (err) { + console.warn('[atelier:catalogue] could not read the pricing page:', err); + } + + return prices; +} + +/** + * One refresh pass for OpenAI. + * + * Returns only what it established. Every model it did not see is simply absent from the result, and + * absent means "unchanged" — this function cannot retire a model, because "the endpoint did not list + * it" and "the request failed halfway" look identical from here. Retiring is done by the merge, which + * can tell a complete listing from a missing one. + */ +export async function refreshOpenAI(): Promise<{ catalogue: Catalogue | null; listedIds: string[] | null }> { + const apiKey = await anyKey(); + if (!apiKey) { + console.log('[atelier:catalogue] no OpenAI key available — nothing to refresh, shipped catalogue stands'); + return { catalogue: null, listedIds: null }; + } + + const listedIds = await fetchModelIds(apiKey); + if (!listedIds) return { catalogue: null, listedIds: null }; + + const prices = await fetchPricing(listedIds); + + const models: CatalogueModel[] = listedIds.map((id) => { + const figures = prices.get(id); + return { + id, + kind: 'image', + status: 'current', + source: figures ? 'scraped' : 'api', + // Three figures in published order: text in, image in, image out. Fewer than three is not a + // price we understand, so it is left undefined rather than guessed into the wrong slots. + pricing: + figures && figures.length >= 3 + ? { unit: 'token', textInput: figures[0], imageInput: figures[1], imageOutput: figures[2] } + : undefined, + }; + }); + + return { + catalogue: { + revision: CATALOGUE_REVISION, + updatedAt: new Date().toISOString(), + providers: [{ name: 'openai', label: 'OpenAI', envVar: 'OPENAI_API_KEY', models }], + }, + listedIds, + }; +} diff --git a/sidecar/catalogue/refresh.ts b/sidecar/catalogue/refresh.ts new file mode 100644 index 0000000..a62c49b --- /dev/null +++ b/sidecar/catalogue/refresh.ts @@ -0,0 +1,83 @@ +import { projectEnv } from '../env'; +import { refreshOpenAI } from './openai-refresh'; +import { SHIPPED_CATALOGUE } from './shipped'; +import { catalogueAge, getCatalogue, mergeCatalogue, writeCatalogue } from './store'; +import type { Catalogue } from './types'; + +/** + * Keeping the catalogue current — the mechanism, with OpenAI as its first and only implementation. + * + * A provider is refreshed by one function that returns what it could establish; this file decides + * when they run, merges the results, and writes the file. Adding a second provider is a second + * function and a line in `REFRESHERS` — the same restraint hono.ts keeps on its mounts. + */ + +/** Once a day. Model lists and prices move on the order of weeks; this is already generous. */ +const INTERVAL_MS = Number(projectEnv('ATELIER_CATALOGUE_INTERVAL_MS') ?? 24 * 60 * 60 * 1000); + +const REFRESHERS = [refreshOpenAI]; + +/** + * Run every refresher and write the result. + * + * Retiring happens HERE rather than in a refresher, because it is the one judgement that needs the + * whole picture: a model we ship and the vendor no longer lists is deprecated, but only when the + * listing actually succeeded. "The endpoint returned three models" and "the request failed after + * three" are indistinguishable inside the refresher and completely different here. + */ +export async function refreshCatalogue(): Promise { + const results = await Promise.all(REFRESHERS.map((run) => run())); + + let merged = SHIPPED_CATALOGUE; + for (const { catalogue, listedIds } of results) { + if (!catalogue) continue; + merged = mergeCatalogue(merged, catalogue); + + if (listedIds) { + const listed = new Set(listedIds); + for (const provider of merged.providers) { + if (!catalogue.providers.some((entry) => entry.name === provider.name)) continue; + for (const model of provider.models) { + // Deprecated, not deleted: the gallery labels historical rows from this list, and 88 images + // name the model that made them. + if (!listed.has(model.id)) model.status = 'deprecated'; + } + } + } + } + + const written = { ...merged, updatedAt: new Date().toISOString() }; + await writeCatalogue(written); + + const count = written.providers.reduce((sum, provider) => sum + provider.models.length, 0); + console.log(`[atelier:catalogue] refreshed — ${written.providers.length} provider(s), ${count} model(s)`); + return written; +} + +/** + * Start the schedule. + * + * Refreshes on boot only when the file is genuinely stale, so a sidecar that restarts twenty times in + * an afternoon — which this one has — does not hit OpenAI twenty times. `unref` so a pending timer + * never holds the process open at shutdown. + */ +export async function startCatalogueRefresh(): Promise { + const globalForCatalogue = globalThis as unknown as { __atelierCatalogueStarted?: boolean }; + if (globalForCatalogue.__atelierCatalogueStarted) return; + globalForCatalogue.__atelierCatalogueStarted = true; + + const age = await catalogueAge(); + if (age === null || age > INTERVAL_MS) { + // Not awaited: a slow or unreachable vendor must not delay the sidecar's own startup, and every + // failure path here already degrades to the shipped catalogue. + void refreshCatalogue().catch((err: unknown) => console.error('[atelier:catalogue] refresh failed:', err)); + } else { + console.log(`[atelier:catalogue] ${Math.round(age / 3_600_000)}h old — next refresh on schedule`); + } + + setInterval(() => { + void refreshCatalogue().catch((err: unknown) => console.error('[atelier:catalogue] refresh failed:', err)); + }, INTERVAL_MS).unref(); +} + +export { getCatalogue }; diff --git a/sidecar/catalogue/shipped.ts b/sidecar/catalogue/shipped.ts new file mode 100644 index 0000000..b2fae83 --- /dev/null +++ b/sidecar/catalogue/shipped.ts @@ -0,0 +1,87 @@ +import { CATALOGUE_REVISION, type Catalogue } from './types'; + +/** + * The catalogue as it ships with the plugin — the floor a refresh builds on, and the whole of it when + * no refresh has ever run or every refresh has failed. + * + * Written by hand and reviewed in git, because none of it is fetchable: `GET /v1/models` returns ids, + * an owner and a timestamp, and says nothing about price, sizes, transparency, or whether a model is + * one you would still choose. That is the entire reason this file exists rather than a nightly fetch. + * + * The prices are USD per million tokens, from OpenAI's published pricing, and three of the five models + * have none — deliberately. `PRICING` in providers/openai.ts had the same gap and the same reason: a + * guessed price produces a number that looks authoritative and sums into a wrong total. + */ +export const SHIPPED_CATALOGUE: Catalogue = { + revision: CATALOGUE_REVISION, + providers: [ + { + name: 'openai', + label: 'OpenAI', + envVar: 'OPENAI_API_KEY', + defaultModel: 'gpt-image-1', + models: [ + { + id: 'gpt-image-1.5', + label: 'GPT Image 1.5', + kind: 'image', + sizes: ['auto', '1024x1024', '1536x1024', '1024x1536'], + qualities: ['auto', 'low', 'medium', 'high'], + transparency: true, + edit: true, + pricing: { unit: 'token', textInput: 5, imageInput: 8, imageOutput: 32 }, + status: 'current', + source: 'shipped', + }, + { + id: 'gpt-image-1', + label: 'GPT Image 1', + kind: 'image', + sizes: ['auto', '1024x1024', '1536x1024', '1024x1536'], + qualities: ['auto', 'low', 'medium', 'high'], + transparency: true, + edit: true, + pricing: { unit: 'token', textInput: 5, imageInput: 10, imageOutput: 40 }, + status: 'current', + source: 'shipped', + }, + { + id: 'gpt-image-1-mini', + label: 'GPT Image 1 mini', + kind: 'image', + sizes: ['auto', '1024x1024', '1536x1024', '1024x1536'], + qualities: ['auto', 'low', 'medium', 'high'], + transparency: true, + edit: true, + pricing: { unit: 'token', textInput: 2, imageInput: 2.5, imageOutput: 8 }, + status: 'current', + source: 'shipped', + }, + { + // No transparency and a different size set — which is exactly what the form was getting + // wrong by offering one hard-coded list of sizes and qualities for every model. + id: 'dall-e-3', + label: 'DALL·E 3', + kind: 'image', + sizes: ['1024x1024', '1792x1024', '1024x1792'], + qualities: ['standard', 'hd'], + transparency: false, + edit: false, + status: 'current', + source: 'shipped', + }, + { + id: 'dall-e-2', + label: 'DALL·E 2', + kind: 'image', + sizes: ['256x256', '512x512', '1024x1024'], + qualities: ['standard'], + transparency: false, + edit: true, + status: 'current', + source: 'shipped', + }, + ], + }, + ], +}; diff --git a/sidecar/catalogue/store.ts b/sidecar/catalogue/store.ts new file mode 100644 index 0000000..3cda016 --- /dev/null +++ b/sidecar/catalogue/store.ts @@ -0,0 +1,104 @@ +import { join } from 'node:path'; +import { DATA_DIR } from '../env'; +import { SHIPPED_CATALOGUE } from './shipped'; +import { CATALOGUE_REVISION, type Catalogue, type CatalogueModel, type CatalogueProvider } from './types'; + +/** + * Where a refreshed catalogue lives: beside the images, NOT in the plugin directory. + * + * A plugin directory is a git clone that an update replaces, so a file written there disappears on + * upgrade with nothing logging it — the same rule env.ts states for generated images, and the reason + * `$OFFICER_ROOT/data/atelier` exists at all. + */ +export const CATALOGUE_PATH = join(DATA_DIR, 'catalogue.json'); + +/** In-memory copy, so every add-connection dialog does not read the disk. Cleared by a refresh. */ +let cached: Catalogue | null = null; + +/** + * Merge a refreshed catalogue over the shipped one. + * + * Shipped is the FLOOR, never the ceiling: a refresh may correct a price, add a model or retire one, + * and it may also fail halfway. Merging rather than replacing means a refresh that returned three + * models out of five leaves the other two intact instead of deleting them, and a refresh that + * returned nothing at all changes nothing. + * + * Per model, the refreshed FIELDS win one by one rather than the refreshed OBJECT winning whole — the + * models endpoint knows ids and nothing else, so an entry built from it would otherwise wipe the + * sizes, qualities and price that were curated by hand. + */ +export function mergeCatalogue(base: Catalogue, refreshed: Catalogue | null): Catalogue { + if (!refreshed || refreshed.revision !== base.revision) return base; + + const providers: CatalogueProvider[] = base.providers.map((provider) => { + const incoming = refreshed.providers.find((entry) => entry.name === provider.name); + if (!incoming) return provider; + + const models: CatalogueModel[] = provider.models.map((model) => { + const update = incoming.models.find((entry) => entry.id === model.id); + return update ? { ...model, ...definedOnly(update) } : model; + }); + + // Models the refresh found that we have never shipped: kept, with whatever it established and + // nothing invented. They are usable — a request names a model by string — and honest about cost. + for (const model of incoming.models) { + if (!models.some((entry) => entry.id === model.id)) models.push(model); + } + + return { ...provider, ...definedOnly(incoming), models }; + }); + + // A provider the refresh knows about and we do not ship. Same reasoning as a new model. + for (const provider of refreshed.providers) { + if (!providers.some((entry) => entry.name === provider.name)) providers.push(provider); + } + + return { revision: base.revision, updatedAt: refreshed.updatedAt, providers }; +} + +/** `{...a, ...b}` with b's `undefined`s dropped, so "not established" never overwrites "curated". */ +function definedOnly(value: T): Partial { + return Object.fromEntries(Object.entries(value).filter(([, v]) => v !== undefined)) as Partial; +} + +async function readRefreshed(): Promise { + try { + const file = Bun.file(CATALOGUE_PATH); + if (!(await file.exists())) return null; + const parsed = (await file.json()) as Catalogue; + // A file written by a plugin with a different shape is ignored rather than half-read. It is not + // corruption — it is last month's format, and the shipped catalogue is a better answer than a + // partially-understood one. + if (parsed?.revision !== CATALOGUE_REVISION || !Array.isArray(parsed.providers)) { + console.warn(`[atelier:catalogue] ignoring ${CATALOGUE_PATH}: revision ${parsed?.revision} is not ${CATALOGUE_REVISION}`); + return null; + } + return parsed; + } catch (err) { + console.error('[atelier:catalogue] could not read the refreshed catalogue:', err); + return null; + } +} + +/** The catalogue as everything else should see it: shipped, with any successful refresh merged over. */ +export async function getCatalogue(): Promise { + if (cached) return cached; + cached = mergeCatalogue(SHIPPED_CATALOGUE, await readRefreshed()); + return cached; +} + +export async function writeCatalogue(catalogue: Catalogue): Promise { + await Bun.write(CATALOGUE_PATH, `${JSON.stringify(catalogue, null, 2)}\n`); + cached = null; +} + +/** When the refreshed file was last written, or null if there has never been one. */ +export async function catalogueAge(): Promise { + const file = Bun.file(CATALOGUE_PATH); + if (!(await file.exists())) return null; + return Date.now() - file.lastModified; +} + +export function invalidateCatalogue(): void { + cached = null; +} diff --git a/sidecar/catalogue/types.ts b/sidecar/catalogue/types.ts new file mode 100644 index 0000000..e76613a --- /dev/null +++ b/sidecar/catalogue/types.ts @@ -0,0 +1,88 @@ +/** + * The catalogue: what Officer says you can connect to, and what each model can do. + * + * ── Catalogue, connections, adapters ── + * + * Three words that must never be swapped for one another: + * + * catalogue THIS. A curated file. "OpenAI exists, it has these models, they cost this." + * connection A row in `atelier_providers`. "MY OpenAI, on gpt-image-1, with this key." + * adapter Code that speaks a vendor's HTTP API. Written once, only developers say it. + * + * The catalogue is what you choose FROM; a connection is what you chose. Five rows were once seeded + * into the table from the adapter's own model list, which made five decisions nobody had made — the + * catalogue exists so that the list of possibilities never again has to be stored as if it were a + * set of choices. + * + * ── Advisory, never authoritative ── + * + * The catalogue shapes what the UI OFFERS. It never decides what the server ACCEPTS: `validate.ts` + * deliberately refuses to enforce per-model rules because a stale copy of them would reject valid + * combinations, and the vendor is the one who should answer. So a model absent from here is still + * usable if the caller names it, and a size not listed is still sent. + * + * ── Allowed to be incomplete ── + * + * Every field a refresh could not establish stays absent, and absence degrades: no pricing means a + * null cost, which is honest, where an invented price sums into a total that is wrong. + */ + +/** What a model produces. Video is not implemented yet; the field exists so the file never has to change shape for it. */ +export type MediaKind = 'image' | 'video'; + +/** + * What a model costs. + * + * A shape per unit rather than one number, because images are priced per million tokens and video is + * priced per second of output — and a single `price` field would have to lie about one of them. + */ +export type Pricing = + | { unit: 'token'; textInput?: number; imageInput?: number; imageOutput?: number } + | { unit: 'second'; perSecond: number } + | { unit: 'image'; perImage: number }; + +export type CatalogueModel = { + /** What the vendor calls it, and what a request names — `gpt-image-1`. */ + id: string; + /** What a person calls it. Falls back to the id when we have nothing better. */ + label?: string; + kind: MediaKind; + /** Sizes the UI offers. `auto` first where the vendor supports it. */ + sizes?: string[]; + qualities?: string[]; + /** Whether `background: transparent` is meaningful here. dall-e-* cannot do it. */ + transparency?: boolean; + /** Whether this model can be used for edits, not only generations. */ + edit?: boolean; + pricing?: Pricing; + /** + * `deprecated` keeps a retired model in the file instead of removing it. + * + * The gallery labels historical rows from this list — 88 images name the model that made them — so + * deleting an entry rewrites the past as "unknown model". Deprecated is hidden from the add dialog + * and still resolves everywhere else. + */ + status?: 'current' | 'deprecated'; + /** Where this entry's facts came from, so a bad refresh is legible rather than mysterious. */ + source?: 'shipped' | 'api' | 'scraped'; +}; + +export type CatalogueProvider = { + /** Matches an adapter's `name` when code for it exists. A provider with no adapter is a listing, not an offer. */ + name: string; + label: string; + /** The bootstrap credential's variable, for a connection with no stored key. */ + envVar: string; + defaultModel?: string; + models: CatalogueModel[]; +}; + +export type Catalogue = { + /** Bumped when the SHAPE changes, so a file written by an older plugin can be recognised and ignored. */ + revision: number; + /** When the refresh that produced this ran. Absent on the shipped file — it was written by hand. */ + updatedAt?: string; + providers: CatalogueProvider[]; +}; + +export const CATALOGUE_REVISION = 1; diff --git a/sidecar/index.ts b/sidecar/index.ts index 1b0741c..922538d 100644 --- a/sidecar/index.ts +++ b/sidecar/index.ts @@ -4,6 +4,7 @@ import { API_URL } from '@@/officer-url.mjs'; import { buildHonoApp } from './hono'; import { waitForDatabase } from './db'; import { startQueue } from './generations/queue'; +import { startCatalogueRefresh } from './catalogue/refresh'; // The officer-atelier sidecar. Owns the whole image-generation contract: providers, the generation // queue, and the `atelier` Postgres database (kept separate from officer_db — see ../PLUGIN.md). @@ -79,6 +80,10 @@ const connection = createSidecarConnector({ void (async () => { if (await waitForDatabase()) await startQueue(); else console.error('[atelier:queue] not started: no database'); + + // After the database, because a refresh reads any connection's key before falling back to the + // environment — and before it, there is nothing to read. + await startCatalogueRefresh(); })(); function shutdown(signal: string) { diff --git a/sidecar/providers/credentials.ts b/sidecar/providers/credentials.ts index d52a432..b333eb2 100644 --- a/sidecar/providers/credentials.ts +++ b/sidecar/providers/credentials.ts @@ -3,6 +3,7 @@ import { getProvider, getProviderByName, listProviders, resolveProviderForModel import type { Provider } from '../db'; import { decrypt } from '../crypto'; import { ADAPTERS, adapterByName } from './index'; +import { getCatalogue } from '../catalogue/store'; import type { ImageAdapter } from './types'; /** @@ -115,13 +116,38 @@ export async function providerStatuses() { ); } -/** The adapters a provider can be built on, for the "add provider" form. */ -export const adapterCatalogue = () => - ADAPTERS.map((adapter) => ({ - name: adapter.name, - label: adapter.label, - models: adapter.models, - default_model: adapter.defaultModel, - env_var: envVarFor(adapter.name), - supports_edit: typeof adapter.edit === 'function', - })); +/** + * What a new connection can be built on: the CATALOGUE, filtered to adapters that exist as code. + * + * The model list used to come off the adapter itself — a hard-coded array beside the HTTP client — + * which meant adding a model was a code change and a release. It comes from the curated catalogue + * now, refreshed daily, and the adapter keeps the one thing only it knows: whether it can edit. + * + * A catalogue provider with no adapter is skipped rather than offered. Listing a vendor we cannot + * actually call is an offer that fails at the moment of use, which is worse than not being offered. + */ +export const adapterCatalogue = async () => { + const catalogue = await getCatalogue(); + + return catalogue.providers.flatMap((provider) => { + const adapter = adapterByName(provider.name); + if (!adapter) return []; + + // Deprecated models stay in the catalogue so old rows keep their labels; they are not offered. + const models = provider.models.filter((model) => model.status !== 'deprecated'); + + return [ + { + name: provider.name, + label: provider.label, + models: models.map((model) => model.id), + /** The full records, for a UI that offers per-model sizes and qualities rather than one list. */ + model_details: models, + default_model: provider.defaultModel ?? adapter.defaultModel, + env_var: provider.envVar || envVarFor(provider.name), + supports_edit: typeof adapter.edit === 'function', + updated_at: catalogue.updatedAt ?? null, + }, + ]; + }); +}; diff --git a/web/lib/api.ts b/web/lib/api.ts index abd1e40..2889f72 100644 --- a/web/lib/api.ts +++ b/web/lib/api.ts @@ -22,14 +22,36 @@ export type ProviderStatus = { updated_at: string; }; -/** The code side — what a new provider can be built on. */ +/** One model in the catalogue — what it makes, what it accepts, what it costs. */ +export type CatalogueModel = { + id: string; + label?: string; + kind: 'image' | 'video'; + sizes?: string[]; + qualities?: string[]; + transparency?: boolean; + edit?: boolean; + pricing?: Record; + status?: 'current' | 'deprecated'; + source?: 'shipped' | 'api' | 'scraped'; +}; + +/** + * What a new connection can be built on: a catalogue provider that has an adapter behind it. + * + * `models` is the flat list of ids; `model_details` carries the per-model facts, which is what lets a + * form offer dall-e-2's three sizes rather than gpt-image-1's four. The catalogue is refreshed daily + * in the sidecar — `updated_at` says when, and is null while only the shipped file has been read. + */ export type Adapter = { name: string; label: string; models: string[]; + model_details: CatalogueModel[]; default_model: string; env_var: string; supports_edit: boolean; + updated_at: string | null; }; /** Snake case, matching the server — one spelling per field across every endpoint. */