atelier/sidecar/catalogue/openai-refresh.ts

import { listProviders } from '../db';
import { keyForProvider } from '../providers/credentials';
import { projectEnv } from '../env';
import type { Catalogue, CatalogueModel } from './types';
import { CATALOGUE_REVISION } from './types';

/**
 * What a refresh can establish about OpenAI, from OpenAI.
 *
 * Two sources, and they are not equal:
 *
 *   `GET /v1/models`   authoritative, and thin. Ids, an owner, a created timestamp. Nothing else.
 *   the pricing page   everything that actually matters, and it is HTML written for people.
 *
 * There is no pricing endpoint — none, for any plan — so the second source is a page and parsing it is
 * inherently fragile. Which is why nothing here is allowed to overwrite a curated value with a guess:
 * a field this cannot establish is returned `undefined`, and `mergeCatalogue` leaves the shipped value
 * standing.
 */

const MODELS_ENDPOINT = 'https://api.openai.com/v1/models';

/** Overridable, because the page this parses is the part most likely to move. */
const PRICING_URL = projectEnv('ATELIER_OPENAI_PRICING_URL') ?? 'https://platform.openai.com/docs/pricing';

/**
 * Which of OpenAI's models make pictures.
 *
 * A pattern, not a list, and that is the point of asking the API at all: `gpt-image-2` on release day
 * matches this and appears in the add dialog without a plugin update. The cost of a pattern is the
 * occasional false positive, which surfaces as a model that fails at the vendor rather than one that
 * silently does the wrong thing.
 */
const IMAGE_MODEL = /^(gpt-image|dall-e)/i;

/** A key from any connection, or the bootstrap environment variable. Refresh is read-only, so any will do. */
async function anyKey(): Promise<string | undefined> {
  for (const provider of await listProviders()) {
    if (provider.adapter !== 'openai') continue;
    const key = await keyForProvider(provider);
    if (key) return key;
  }
  return projectEnv('OPENAI_API_KEY');
}

async function fetchModelIds(apiKey: string): Promise<string[] | null> {
  try {
    const response = await fetch(MODELS_ENDPOINT, {
      headers: { authorization: `Bearer ${apiKey}` },
      signal: AbortSignal.timeout(30_000),
    });
    if (!response.ok) {
      console.warn(`[atelier:catalogue] models endpoint answered ${response.status}`);
      return null;
    }
    const body = (await response.json()) as { data?: { id?: unknown }[] };
    return (body.data ?? [])
      .map((entry) => entry.id)
      .filter((id): id is string => typeof id === 'string' && IMAGE_MODEL.test(id));
  } catch (err) {
    console.warn('[atelier:catalogue] could not list models:', err);
    return null;
  }
}

/**
 * Prices, off the pricing page.
 *
 * Deliberately narrow: it looks for a model id and reads the dollar figures that follow it in the same
 * chunk of markup, and returns nothing at all when the page does not look the way it expects. A parser
 * that guesses on an unfamiliar page is worse than one that gives up, because the guess is written to
 * disk and then believed.
 *
 * This will fail the first time the page is restyled, and it is meant to fail LOUDLY — the log line is
 * the signal to replace this function. The durable version of it is not a better regex: it is handing
 * the page to a model and asking for structured output, which Officer can already do locally through
 * its own Claude sidecar. That is the next step, not a different design.
 */
async function fetchPricing(modelIds: string[]): Promise<Map<string, number[]>> {
  const prices = new Map<string, number[]>();

  try {
    const response = await fetch(PRICING_URL, { signal: AbortSignal.timeout(30_000) });
    if (!response.ok) {
      console.warn(`[atelier:catalogue] pricing page answered ${response.status} — keeping shipped prices`);
      return prices;
    }

    const html = await response.text();
    for (const id of modelIds) {
      const at = html.indexOf(id);
      if (at === -1) continue;
      // The figures that follow the id, within one row's worth of markup. `$5.00`, `$0.01`, `5.00`.
      const window = html.slice(at, at + 1200);
      const found = [...window.matchAll(/\$\s?(\d+(?:\.\d+)?)/g)].map((match) => Number(match[1]));
      if (found.length > 0) prices.set(id, found);
    }

    if (prices.size === 0) {
      console.warn(
        `[atelier:catalogue] found no prices on ${PRICING_URL} — the page has changed shape, or renders its ` +
          'prices client-side. Shipped prices kept; this extractor needs replacing.',
      );
    }
  } catch (err) {
    console.warn('[atelier:catalogue] could not read the pricing page:', err);
  }

  return prices;
}

/**
 * One refresh pass for OpenAI.
 *
 * Returns only what it established. Every model it did not see is simply absent from the result, and
 * absent means "unchanged" — this function cannot retire a model, because "the endpoint did not list
 * it" and "the request failed halfway" look identical from here. Retiring is done by the merge, which
 * can tell a complete listing from a missing one.
 */
export async function refreshOpenAI(): Promise<{ catalogue: Catalogue | null; listedIds: string[] | null }> {
  const apiKey = await anyKey();
  if (!apiKey) {
    console.log('[atelier:catalogue] no OpenAI key available — nothing to refresh, shipped catalogue stands');
    return { catalogue: null, listedIds: null };
  }

  const listedIds = await fetchModelIds(apiKey);
  if (!listedIds) return { catalogue: null, listedIds: null };

  const prices = await fetchPricing(listedIds);

  const models: CatalogueModel[] = listedIds.map((id) => {
    const figures = prices.get(id);
    return {
      id,
      kind: 'image',
      status: 'current',
      source: figures ? 'scraped' : 'api',
      // Three figures in published order: text in, image in, image out. Fewer than three is not a
      // price we understand, so it is left undefined rather than guessed into the wrong slots.
      pricing:
        figures && figures.length >= 3
          ? { unit: 'token', textInput: figures[0], imageInput: figures[1], imageOutput: figures[2] }
          : undefined,
    };
  });

  return {
    catalogue: {
      revision: CATALOGUE_REVISION,
      updatedAt: new Date().toISOString(),
      providers: [{ name: 'openai', label: 'OpenAI', envVar: 'OPENAI_API_KEY', models }],
    },
    listedIds,
  };
}