notify: only "the app is gone" deletes a device row

reported from the field on the first real push: it failed with BadDeviceToken
and the server deleted the registration. the build was production-signed and
the row said sandbox — a configuration mismatch, with the device and the app
both perfectly fine.

pruning there destroys the evidence. the phone re-registers on next launch,
the row comes back looking healthy, and the next send fails identically; you
can lose a day to that loop without ever seeing what is actually wrong.

so the one set becomes three. Unregistered/UNREGISTERED/NOT_FOUND prune —
apple and google are saying the app is uninstalled. BadDeviceToken,
DeviceTokenNotForTopic, TopicDisallowed, INVALID_ARGUMENT and
SENDER_ID_MISMATCH are OURS: the row is kept and the log says what to check.
everything else is transient and still counts toward three strikes.

INVALID_ARGUMENT was the sharpest one. it is how fcm reports a malformed
MESSAGE — including a non-string in `data`, which is the mistake this codebase
is most likely to make and already has a test for. deleting on it meant one
bug in the payload builder would wipe every android device from the registry
on a single send, presenting as "android push stopped working" with nothing
left to explain why.

misconfigured is also not counted as a strike. not deleting is not enough on
its own — a persistent mismatch would reach three failures and delete the row
anyway, which is the same bug with a slower fuse.

the apns log now names the mismatch rather than the reason code, since the
code alone sends people to look at the device: it prints the row's environment
and topic and says to check the SHIPPED artefact rather than the intermediate
build, which is exactly where this one hid.

src/servers/sidecar/notify/apns.test.ts

@@ -2,6 +2,7 @@ import { describe, test, expect } from 'bun:test';
 import { generateKeyPairSync, createVerify } from 'node:crypto';
 import { ApnsAuthToken } from './apns-jwt';
 import { buildApnsPayload, collapseId } from './apns-payload';
+import { __testing } from './apns';
 import { renderNotification, FALLBACK_TITLE } from './render';
 import type { Notification } from './types';
 import type { Catalogue } from '../../notifications/catalogue';
@@ -127,3 +128,32 @@ describe('APNs payload', () => {
     expect(renderNotification({ code: 'nope/nope:x', userId: 1 }, catalogue).title).toBe(FALLBACK_TITLE);
   });
 });
+
+describe('what a failure does to the device row', () => {
+  // Reported from the field 2026-08-19: a first real push failed with BadDeviceToken and the server
+  // deleted the registration. The build was production-signed and the row said sandbox — a configuration
+  // mismatch, with the device and the app both perfectly fine.
+  //
+  // Pruning there destroys the evidence: the phone re-registers on next launch, the row returns looking
+  // healthy, and the next send fails identically. You can lose a day to that loop.
+  test('only Unregistered means the app is gone', () => {
+    expect(__testing.dispositionFor('Unregistered')).toBe('gone');
+  });
+
+  test('an environment mismatch keeps the row so it can be diagnosed', () => {
+    expect(__testing.dispositionFor('BadDeviceToken')).toBe('misconfigured');
+    expect(__testing.dispositionFor('DeviceTokenNotForTopic')).toBe('misconfigured');
+  });
+
+  // Not merely "not deleted" — misconfigured must not count as a strike either, or a persistent mismatch
+  // deletes the row on the third send. Same bug, slower fuse.
+  test('misconfigured is neither pruned nor a strike', () => {
+    expect(__testing.dispositionFor('BadDeviceToken')).not.toBe('gone');
+    expect(__testing.dispositionFor('BadDeviceToken')).not.toBe('transient');
+  });
+
+  test('anything unrecognised is transient, so it counts toward three strikes', () => {
+    expect(__testing.dispositionFor('ServiceUnavailable')).toBe('transient');
+    expect(__testing.dispositionFor('timeout')).toBe('transient');
+  });
+});

src/servers/sidecar/notify/apns.ts

@@ -20,8 +20,36 @@ const HOSTS = {
   sandbox: 'https://api.sandbox.push.apple.com',
 } as const;
 
-/** Apple's reasons for "this token is dead" — the row goes immediately rather than after three strikes. */
-const DEAD_TOKEN_REASONS = new Set(['BadDeviceToken', 'Unregistered', 'DeviceTokenNotForTopic']);
+// ── Telling "the app is gone" apart from "we are misconfigured" ──
+//
+// These used to be one set, and every one of them deleted the row. That is right for exactly one reason
+// and actively harmful for the others.
+//
+// `Unregistered` (410) means Apple knows the app was uninstalled. The token will never work again, so
+// pruning is correct and a registry full of those is how delivery quietly degrades.
+//
+// `BadDeviceToken` and `DeviceTokenNotForTopic` are CONFIGURATION problems, and almost always the same
+// one: a token minted under `aps-environment: development` sent to the production host, or the reverse.
+// The device is fine, the app is installed, the registration is good — the server dialled the wrong
+// Apple. Deleting the row there destroys the evidence: the phone re-registers on next launch, the row
+// comes back looking healthy, and the next send fails identically. You can lose a day to that loop
+// without ever seeing the thing that is actually wrong.
+//
+// So: only `Unregistered` prunes.
+const GONE_REASONS = new Set(['Unregistered']);
+
+/**
+ * Wrong on OUR side, not the device's. Never pruned, and deliberately not counted toward the
+ * three-strikes rule either — a persistent environment mismatch would otherwise delete the row on the
+ * third send, which is the same bug with a slower fuse.
+ */
+const MISCONFIGURED_REASONS = new Set(['BadDeviceToken', 'DeviceTokenNotForTopic', 'TopicDisallowed']);
+
+/** What to do about one failed send. */
+type Disposition = 'gone' | 'misconfigured' | 'transient';
+
+const dispositionFor = (reason: string): Disposition =>
+  GONE_REASONS.has(reason) ? 'gone' : MISCONFIGURED_REASONS.has(reason) ? 'misconfigured' : 'transient';
 
 // Secret store first, env second.
 //
@@ -73,7 +101,7 @@ function session(host: string): http2.ClientHttp2Session {
   return s;
 }
 
-type SendOutcome = { ok: boolean; dead: boolean; reason?: string };
+type SendOutcome = { ok: boolean; disposition?: Disposition; reason?: string };
 
 function sendOne(
   host: string,
@@ -109,7 +137,7 @@ function sendOne(
       raw += chunk;
     });
     req.on('end', () => {
-      if (status === 200) return resolve({ ok: true, dead: false });
+      if (status === 200) return resolve({ ok: true });
       // Apple answers failures with { "reason": "BadDeviceToken" } and nothing else useful.
       let reason = `status ${status}`;
       try {
@@ -117,13 +145,13 @@ function sendOne(
       } catch {
         /* keep the status */
       }
-      resolve({ ok: false, dead: DEAD_TOKEN_REASONS.has(reason), reason });
+      resolve({ ok: false, disposition: dispositionFor(reason), reason });
     });
-    req.on('error', (err) => resolve({ ok: false, dead: false, reason: err.message }));
+    req.on('error', (err) => resolve({ ok: false, disposition: 'transient', reason: err.message }));
 
     req.setTimeout(10_000, () => {
       req.close();
-      resolve({ ok: false, dead: false, reason: 'timeout' });
+      resolve({ ok: false, disposition: 'transient', reason: 'timeout' });
     });
 
     req.end(body);
@@ -159,14 +187,27 @@ export const apnsChannel: Channel = {
         failed++;
         console.error(`[notify:apns] ${d.bundleId} ${d.environment}: ${outcome.reason}`);
 
-        if (outcome.dead) {
-          // An uninstalled app or a rotated token. Delete now — retrying cannot help, and a registry
-          // full of dead tokens is how delivery quietly degrades.
+        if (outcome.disposition === 'gone') {
+          // Apple says the app is uninstalled. Retrying cannot help.
           dead.push(d.token);
           await deletePushDevice(d.token);
-        } else {
-          await recordPushFailure(d.token);
+          return;
         }
+
+        if (outcome.disposition === 'misconfigured') {
+          // Left ALONE — not deleted, and not counted as a strike. Say what is actually wrong instead,
+          // because the reason code alone sends people to look at the device.
+          const other = d.environment === 'production' ? 'sandbox' : 'production';
+          console.error(
+            `[notify:apns] '${outcome.reason}' is a configuration mismatch, not a dead device. This row ` +
+              `says environment='${d.environment}' and topic='${d.bundleId}'. If the installed build is ` +
+              `signed for '${other}', the registration is wrong and the device row is kept so it can be ` +
+              `corrected — check aps-environment in the SHIPPED artefact, not the intermediate build.`,
+          );
+          return;
+        }
+
+        await recordPushFailure(d.token);
       }),
     );
 
@@ -185,3 +226,6 @@ export function closeApnsSessions(): void {
   }
   sessions.clear();
 }
+
+/** Exposed for tests: classifying a failure is the whole of the pruning policy. */
+export const __testing = { dispositionFor };

src/servers/sidecar/notify/fcm.ts

@@ -15,8 +15,27 @@ import { FcmAccessToken, parseServiceAccount } from './fcm-auth';
 
 const ENDPOINT = (projectId: string) => `https://fcm.googleapis.com/v1/projects/${projectId}/messages:send`;
 
-/** Google's ways of saying "this token is dead" — the row goes immediately. */
-const DEAD_TOKEN_CODES = new Set(['UNREGISTERED', 'INVALID_ARGUMENT', 'NOT_FOUND']);
+// ── "The app is gone" versus "we sent something wrong" ──
+//
+// `UNREGISTERED` and `NOT_FOUND` mean the registration is dead: uninstalled, or a token that has been
+// replaced. Pruning is correct.
+//
+// `INVALID_ARGUMENT` is NOT that, and it used to be in the same set. It is how FCM reports a malformed
+// MESSAGE — including the one this codebase is most likely to produce, a non-string value in `data`,
+// which has its own test for exactly that reason. Deleting on it means a single bug in the payload
+// builder wipes every Android device from the registry on one send, and the symptom is "Android push
+// stopped working" with no trace of why.
+//
+// `SENDER_ID_MISMATCH` is a wrong service account, i.e. also ours.
+const GONE_CODES = new Set(['UNREGISTERED', 'NOT_FOUND']);
+
+/** Wrong on OUR side. Never pruned, and not counted toward three strikes either. */
+const MISCONFIGURED_CODES = new Set(['INVALID_ARGUMENT', 'SENDER_ID_MISMATCH', 'THIRD_PARTY_AUTH_ERROR']);
+
+type Disposition = 'gone' | 'misconfigured' | 'transient';
+
+const dispositionFor = (code: string): Disposition =>
+  GONE_CODES.has(code) ? 'gone' : MISCONFIGURED_CODES.has(code) ? 'misconfigured' : 'transient';
 
 // Store first, env second — same reasoning as apns.ts.
 const raw = (() => {
@@ -60,7 +79,7 @@ function buildMessage(token: string, n: Notification, ctx: SendContext): FcmMess
   return { message: { token, data, android: { priority: 'high' } } };
 }
 
-type SendOutcome = { ok: boolean; dead: boolean; reason?: string };
+type SendOutcome = { ok: boolean; disposition?: Disposition; reason?: string };
 
 async function sendOne(
   accessToken: string,
@@ -77,7 +96,7 @@ async function sendOne(
       signal: AbortSignal.timeout(10_000),
     });
 
-    if (res.ok) return { ok: true, dead: false };
+    if (res.ok) return { ok: true };
 
     // FCM errors look like { error: { status: 'UNREGISTERED', message: '…' } }. The message can echo the
     // token, so only the status is logged.
@@ -88,9 +107,9 @@ async function sendOne(
     } catch {
       /* keep the http status */
     }
-    return { ok: false, dead: DEAD_TOKEN_CODES.has(status), reason: status };
+    return { ok: false, disposition: dispositionFor(status), reason: status };
   } catch (err) {
-    return { ok: false, dead: false, reason: err instanceof Error ? err.message : 'send failed' };
+    return { ok: false, disposition: 'transient', reason: err instanceof Error ? err.message : 'send failed' };
   }
 }
 
@@ -131,12 +150,21 @@ export const fcmChannel: Channel = {
         failed++;
         console.error(`[notify:fcm] ${d.bundleId}: ${outcome.reason}`);
 
-        if (outcome.dead) {
+        if (outcome.disposition === 'gone') {
           dead.push(d.token);
           await deletePushDevice(d.token);
-        } else {
-          await recordPushFailure(d.token);
+          return;
         }
+
+        if (outcome.disposition === 'misconfigured') {
+          console.error(
+            `[notify:fcm] '${outcome.reason}' is a problem with what WE sent, not with this device. ` +
+              `The row is kept so it can be diagnosed — check the message shape and the service account.`,
+          );
+          return;
+        }
+
+        await recordPushFailure(d.token);
       }),
     );