notify: the APNs channel

Step 3. Apple directly over HTTP/2 — no Expo, no library, just node:http2 and node:crypto.

Three things here are the difference between working and a silent failure:

- The provider JWT must be signed with dsaEncoding 'ieee-p1363'. Node's default is DER, which
  is a perfectly valid ECDSA signature that Apple rejects, and the rejection says nothing
  about why. A test asserts the signature is 64 bytes rather than trusting the flag.
- Apple rejects a token minted more than once per 20 minutes and any token older than 60, so
  it is cached and refreshed at 40 — between the two walls, not per request.
- APNs expects ONE long-lived HTTP/2 session carrying many requests. A session per push gets
  throttled, so sessions are kept per host and re-created only when they die, with an error
  handler so a transport failure cannot take the sidecar down as an unhandled rejection.

Sandbox and production are separate hosts and separate token namespaces, so devices are sent
per their stored `environment` — a debug-build token against production fails with
BadDeviceToken and no other symptom.

Dead tokens (BadDeviceToken, Unregistered, DeviceTokenNotForTopic) delete their row
immediately; everything else counts a strike. Going direct means Apple answers inline, so
none of Expo's deferred receipt-polling is needed.

The doorbell rule is now enforced by a test, not just by convention: a producer that passes
subject/from/body — which the type forbids but JavaScript permits — cannot get any of it into
the serialized payload. `aps.alert` carries a generic title composed from the category, and
the custom `officer` key carries ids the app fetches by.

12 tests, 100% of the JWT and payload paths. Verified against the real endpoint too: a
throwaway key gets 403 InvalidProviderToken from api.sandbox.push.apple.com, which means the
connection, path, apns-topic, push-type and JWT structure are all accepted and only the
credential is missing.

Still needed for a real send: APNS_KEY_P8, APNS_KEY_ID, APNS_TEAM_ID, and a device token
from the app.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

src/servers/sidecar/notify/apns-jwt.ts

@@ -0,0 +1,54 @@
+import { createSign, createPrivateKey, type KeyObject } from 'node:crypto';
+
+// The APNs authorization token: an ES256 JWT signed with the .p8 auth key.
+//
+// Two details here are the difference between working and a 403 with no useful message:
+//
+// 1. The signature must be raw r||s (64 bytes), NOT DER. Node signs ECDSA as DER by default, which is
+//    a valid signature that Apple rejects — `dsaEncoding: 'ieee-p1363'` is what produces the JOSE form.
+// 2. Apple rejects a token minted more than once every 20 minutes (TooManyProviderTokenUpdates) and any
+//    token older than 60 minutes (ExpiredProviderToken). So it is cached and refreshed on a schedule that
+//    sits between those two walls, not per request.
+
+const REFRESH_AFTER_MS = 40 * 60 * 1000; // comfortably inside 60, comfortably outside 20
+
+export type ApnsAuthConfig = {
+  /** Contents of the .p8 file — PEM, PKCS#8, EC P-256. */
+  keyP8: string;
+  /** The 10-character Key ID shown beside the key in the Apple Developer portal. */
+  keyId: string;
+  /** The 10-character Team ID. */
+  teamId: string;
+};
+
+const b64url = (input: object | Buffer): string =>
+  (Buffer.isBuffer(input) ? input : Buffer.from(JSON.stringify(input))).toString('base64url');
+
+export class ApnsAuthToken {
+  private key: KeyObject;
+  private cached: { token: string; mintedAt: number } | null = null;
+
+  constructor(private config: ApnsAuthConfig) {
+    // Parsed once, at construction, so a malformed .p8 fails at startup rather than on the first push.
+    // `\n` is unescaped because the key usually arrives through an env var, where real newlines do not survive.
+    this.key = createPrivateKey({ key: config.keyP8.replace(/\\n/g, '\n'), format: 'pem' });
+  }
+
+  /** A valid provider token, minted only when the cached one is old enough to warrant it. */
+  get(): string {
+    const now = Date.now();
+    if (this.cached && now - this.cached.mintedAt < REFRESH_AFTER_MS) return this.cached.token;
+
+    const header = b64url({ alg: 'ES256', kid: this.config.keyId });
+    const claims = b64url({ iss: this.config.teamId, iat: Math.floor(now / 1000) });
+    const signingInput = `${header}.${claims}`;
+
+    const signer = createSign('SHA256');
+    signer.update(signingInput);
+    const signature = signer.sign({ key: this.key, dsaEncoding: 'ieee-p1363' });
+
+    const token = `${signingInput}.${b64url(signature)}`;
+    this.cached = { token, mintedAt: now };
+    return token;
+  }
+}

src/servers/sidecar/notify/apns-payload.ts

@@ -0,0 +1,54 @@
+import type { Notification } from './types';
+import { renderTitle, renderBody } from './text';
+
+// The APNs payload.
+//
+// `aps` is Apple's reserved key and controls what the system does; everything beside it is ours and
+// arrives in the app untouched. The doorbell rule lives in what we put in each: the alert carries only a
+// generic title composed from the category, and the custom keys carry ids the app uses to FETCH the real
+// content over the tailnet. Nothing here should ever be readable as the thing itself.
+
+export type ApnsPayload = {
+  aps: {
+    alert: { title: string; body?: string };
+    sound: string;
+    /** Drives the red badge. Only meaningful where a count is, so it is set only then. */
+    badge?: number;
+    /** Lets the app group/replace banners rather than stacking one per event. */
+    'thread-id'?: string;
+  };
+  /** Ours. The app routes on these; they are ids, never content. */
+  officer: {
+    type: string;
+    id?: string | number;
+    ok?: boolean;
+  };
+};
+
+export function buildApnsPayload(n: Notification): ApnsPayload {
+  const body = renderBody(n);
+
+  return {
+    aps: {
+      alert: body ? { title: renderTitle(n), body } : { title: renderTitle(n) },
+      sound: 'default',
+      ...(typeof n.count === 'number' ? { badge: n.count } : {}),
+      // One thread per category: three job notifications collapse instead of filling the shade.
+      'thread-id': n.type,
+    },
+    officer: {
+      type: n.type,
+      ...(n.id !== undefined ? { id: n.id } : {}),
+      ...(n.ok !== undefined ? { ok: n.ok } : {}),
+    },
+  };
+}
+
+/**
+ * `apns-collapse-id` — Apple replaces an undelivered notification with the same id rather than queueing
+ * both. Right for "3 new emails" superseding "1 new email"; wrong for jobs, where each completion is its
+ * own event, so those collapse per id instead of per category.
+ */
+export function collapseId(n: Notification): string {
+  return n.id !== undefined ? `${n.type}-${n.id}` : n.type;
+}

src/servers/sidecar/notify/apns.test.ts

@@ -0,0 +1,111 @@
+import { describe, test, expect } from 'bun:test';
+import { generateKeyPairSync, createVerify } from 'node:crypto';
+import { ApnsAuthToken } from './apns-jwt';
+import { buildApnsPayload, collapseId } from './apns-payload';
+import { renderTitle } from './text';
+import type { Notification } from './types';
+
+// What can be proven without Apple: the token is a well-formed ES256 JWT with the encoding Apple
+// actually requires, and the payload never carries content.
+
+const { privateKey, publicKey } = generateKeyPairSync('ec', { namedCurve: 'P-256' });
+const keyP8 = privateKey.export({ format: 'pem', type: 'pkcs8' }) as string;
+const auth = () => new ApnsAuthToken({ keyP8, keyId: 'ABCDE12345', teamId: 'TEAM123456' });
+
+const decode = (part: string) => JSON.parse(Buffer.from(part, 'base64url').toString());
+
+describe('APNs provider token', () => {
+  test('is a three-part JWT with the header Apple expects', () => {
+    const [h, c, s] = auth().get().split('.');
+    expect(decode(h!)).toEqual({ alg: 'ES256', kid: 'ABCDE12345' });
+    expect(decode(c!).iss).toBe('TEAM123456');
+    expect(typeof decode(c!).iat).toBe('number');
+    expect(s!.length).toBeGreaterThan(0);
+  });
+
+  // The whole point of dsaEncoding: 'ieee-p1363'. Node's default DER encoding is a *valid* ECDSA
+  // signature that Apple rejects, and the failure gives no hint as to why — so assert the length.
+  test('signs with raw r||s (64 bytes), not DER', () => {
+    const sig = Buffer.from(auth().get().split('.')[2]!, 'base64url');
+    expect(sig.length).toBe(64);
+  });
+
+  test('the signature actually verifies over header.claims', () => {
+    const [h, c, s] = auth().get().split('.');
+    const v = createVerify('SHA256');
+    v.update(`${h}.${c}`);
+    expect(v.verify({ key: publicKey, dsaEncoding: 'ieee-p1363' }, Buffer.from(s!, 'base64url'))).toBe(true);
+  });
+
+  // Apple rejects tokens minted more than once per 20 minutes (TooManyProviderTokenUpdates), so a fresh
+  // JWT per push would eventually lock us out.
+  test('is cached rather than minted per call', () => {
+    const a = auth();
+    expect(a.get()).toBe(a.get());
+  });
+
+  test('accepts a key whose newlines were escaped by the environment', () => {
+    const escaped = keyP8.replace(/\n/g, '\\n');
+    expect(() => new ApnsAuthToken({ keyP8: escaped, keyId: 'K', teamId: 'T' }).get()).not.toThrow();
+  });
+});
+
+describe('APNs payload', () => {
+  const mail: Notification = { type: 'mail', userId: 1, count: 3 };
+
+  test('carries a generic title and no content', () => {
+    const p = buildApnsPayload(mail);
+    expect(p.aps.alert.title).toBe('3 new emails');
+    expect(p.aps.badge).toBe(3);
+    expect(p.officer).toEqual({ type: 'mail' });
+  });
+
+  // The doorbell rule, asserted rather than trusted: whatever a producer sends, nothing resembling
+  // content may appear in the serialized payload that leaves for Apple.
+  test('a producer cannot smuggle content through', () => {
+    const sneaky = {
+      type: 'mail',
+      userId: 1,
+      count: 1,
+      // none of these are part of Notification, but a caller in JS could still pass them
+      subject: 'Re: your invoice',
+      from: 'accounts@example.com',
+      body: 'please pay',
+    } as unknown as Notification;
+
+    const json = JSON.stringify(buildApnsPayload(sneaky));
+    expect(json).not.toContain('invoice');
+    expect(json).not.toContain('example.com');
+    expect(json).not.toContain('please pay');
+  });
+
+  test('ids travel, so the app can fetch the real thing', () => {
+    const p = buildApnsPayload({ type: 'job', userId: 1, id: 412, ok: false });
+    expect(p.officer).toEqual({ type: 'job', id: 412, ok: false });
+    expect(p.aps.alert.title).toBe('Job failed');
+    expect(p.aps.badge).toBeUndefined();
+  });
+
+  test('threads by category so banners group instead of stacking', () => {
+    expect(buildApnsPayload(mail).aps['thread-id']).toBe('mail');
+  });
+
+  // Collapse replaces an undelivered notification. Right for mail (one count supersedes the last),
+  // wrong for jobs, where each completion is its own event.
+  test('collapses mail by category and jobs by id', () => {
+    expect(collapseId(mail)).toBe('mail');
+    expect(collapseId({ type: 'job', userId: 1, id: 412 })).toBe('job-412');
+  });
+});
+
+describe('rendered text', () => {
+  test('singular and plural mail read correctly', () => {
+    expect(renderTitle({ type: 'mail', userId: 1, count: 1 })).toBe('New email');
+    expect(renderTitle({ type: 'mail', userId: 1, count: 5 })).toBe('5 new emails');
+  });
+
+  test('job wording follows ok', () => {
+    expect(renderTitle({ type: 'job', userId: 1, ok: true })).toBe('Job finished');
+    expect(renderTitle({ type: 'job', userId: 1, ok: false })).toBe('Job failed');
+  });
+});

src/servers/sidecar/notify/apns.ts

@@ -0,0 +1,155 @@
+import http2 from 'node:http2';
+import { getPushDevices, deletePushDevice, recordPushFailure, markPushDeviceSeen } from 'officerdb';
+import type { Channel, DeliveryResult, Notification } from './types';
+import { ApnsAuthToken } from './apns-jwt';
+import { buildApnsPayload, collapseId } from './apns-payload';
+
+// The Apple channel. Talks to APNs directly — no Expo, no library.
+//
+// APNs is HTTP/2 only and expects ONE long-lived connection carrying many requests. Opening a session
+// per push is the classic mistake: Apple throttles it, and the TLS handshake dwarfs the send. So the
+// session is created once, reused, and re-created only when it dies.
+//
+// Sandbox and production are different hosts AND different token namespaces. A token minted by a debug
+// build fails against production with BadDeviceToken and no other symptom, which is why `environment` is
+// a stored column rather than a guess — and why devices are grouped by it here and sent in two batches.
+
+const HOSTS = {
+  production: 'https://api.push.apple.com',
+  sandbox: 'https://api.sandbox.push.apple.com',
+} as const;
+
+/** Apple's reasons for "this token is dead" — the row goes immediately rather than after three strikes. */
+const DEAD_TOKEN_REASONS = new Set(['BadDeviceToken', 'Unregistered', 'DeviceTokenNotForTopic']);
+
+const { APNS_KEY_P8, APNS_KEY_ID, APNS_TEAM_ID } = process.env;
+
+const auth =
+  APNS_KEY_P8 && APNS_KEY_ID && APNS_TEAM_ID
+    ? new ApnsAuthToken({ keyP8: APNS_KEY_P8, keyId: APNS_KEY_ID, teamId: APNS_TEAM_ID })
+    : null;
+
+// One session per host, lazily created and reused.
+const sessions = new Map<string, http2.ClientHttp2Session>();
+
+function session(host: string): http2.ClientHttp2Session {
+  const existing = sessions.get(host);
+  if (existing && !existing.closed && !existing.destroyed) return existing;
+
+  const s = http2.connect(host);
+  // Without this a transport error becomes an unhandled rejection and takes the sidecar down.
+  s.on('error', (err) => {
+    console.error(`[notify:apns] session error (${host}):`, err.message);
+    sessions.delete(host);
+  });
+  s.on('close', () => sessions.delete(host));
+  sessions.set(host, s);
+  return s;
+}
+
+type SendOutcome = { ok: boolean; dead: boolean; reason?: string };
+
+function sendOne(host: string, token: string, bundleId: string, n: Notification): Promise<SendOutcome> {
+  return new Promise((resolve) => {
+    const body = Buffer.from(JSON.stringify(buildApnsPayload(n)));
+
+    const req = session(host).request({
+      ':method': 'POST',
+      ':path': `/3/device/${token}`,
+      authorization: `bearer ${auth!.get()}`,
+      'apns-topic': bundleId,
+      // `alert` because these are user-visible banners. A silent/background push would be `background`
+      // with priority 5, which is a different feature and not what this channel is for.
+      'apns-push-type': 'alert',
+      'apns-priority': '10',
+      'apns-collapse-id': collapseId(n),
+      'content-type': 'application/json',
+      'content-length': body.length,
+    });
+
+    let status = 0;
+    let raw = '';
+
+    req.on('response', (headers) => {
+      status = Number(headers[':status'] ?? 0);
+    });
+    req.on('data', (chunk) => {
+      raw += chunk;
+    });
+    req.on('end', () => {
+      if (status === 200) return resolve({ ok: true, dead: false });
+      // Apple answers failures with { "reason": "BadDeviceToken" } and nothing else useful.
+      let reason = `status ${status}`;
+      try {
+        reason = (JSON.parse(raw) as { reason?: string }).reason ?? reason;
+      } catch {
+        /* keep the status */
+      }
+      resolve({ ok: false, dead: DEAD_TOKEN_REASONS.has(reason), reason });
+    });
+    req.on('error', (err) => resolve({ ok: false, dead: false, reason: err.message }));
+
+    req.setTimeout(10_000, () => {
+      req.close();
+      resolve({ ok: false, dead: false, reason: 'timeout' });
+    });
+
+    req.end(body);
+  });
+}
+
+export const apnsChannel: Channel = {
+  name: 'apns',
+
+  isConfigured: () => auth !== null,
+
+  async send(n: Notification): Promise<DeliveryResult> {
+    if (!auth) return { channel: 'apns', sent: 0, failed: 0 };
+
+    const devices = (await getPushDevices(n.userId, n.appSlug)).filter((d) => d.platform === 'ios');
+    if (devices.length === 0) return { channel: 'apns', sent: 0, failed: 0 };
+
+    const dead: string[] = [];
+    let sent = 0;
+    let failed = 0;
+
+    await Promise.all(
+      devices.map(async (d) => {
+        const host = HOSTS[d.environment as keyof typeof HOSTS] ?? HOSTS.production;
+        const outcome = await sendOne(host, d.token, d.bundleId, n);
+
+        if (outcome.ok) {
+          sent++;
+          await markPushDeviceSeen(d.token);
+          return;
+        }
+
+        failed++;
+        console.error(`[notify:apns] ${d.bundleId} ${d.environment}: ${outcome.reason}`);
+
+        if (outcome.dead) {
+          // An uninstalled app or a rotated token. Delete now — retrying cannot help, and a registry
+          // full of dead tokens is how delivery quietly degrades.
+          dead.push(d.token);
+          await deletePushDevice(d.token);
+        } else {
+          await recordPushFailure(d.token);
+        }
+      }),
+    );
+
+    return { channel: 'apns', sent, failed, ...(dead.length ? { dead } : {}) };
+  },
+};
+
+/** Close the shared sessions on shutdown so the process can exit promptly. */
+export function closeApnsSessions(): void {
+  for (const s of sessions.values()) {
+    try {
+      s.close();
+    } catch {
+      /* already gone */
+    }
+  }
+  sessions.clear();
+}

src/servers/sidecar/notify/dispatch.ts

@@ -1,5 +1,6 @@
 import type { Channel, DeliveryResult, Notification } from './types';
 import { discordChannel } from './discord';
+import { apnsChannel } from './apns';
 
 // Fan a notification out to every configured channel.
 //
@@ -7,7 +8,7 @@ import { discordChannel } from './discord';
 // whether or not a banner appeared. So every channel is awaited with its own error boundary, and the
 // dispatcher always resolves.
 
-const channels: Channel[] = [discordChannel];
+const channels: Channel[] = [apnsChannel, discordChannel];
 
 /** Registered here rather than imported at the top so channels can be added without touching producers. */
 export function registerChannel(channel: Channel): void {

src/servers/sidecar/notify/index.ts

@@ -2,6 +2,7 @@ import type { SidecarCommand, SidecarEvent } from '../protocol';
 import { createSidecarConnector } from '../connect';
 import { dispatch, configuredChannels } from './dispatch';
 import { handleDeviceRoute } from './devices';
+import { closeApnsSessions } from './apns';
 import type { Notification, NotifyType } from './types';
 
 // The officer-notify sidecar. The one place anything leaves this machine to tell the owner something.
@@ -106,6 +107,7 @@ const connection = createSidecarConnector({
 
 function shutdown(signal: string) {
   console.log(`[notify] ${signal} received, shutting down...`);
+  closeApnsSessions();
   connection.destroy();
   process.exit(0);
 }