recover an imported wallet's coins with an nbxplorer utxo scan

registering an xpub only indexes it from that moment on, so an imported
seed with history read as a confident zero: every call succeeded, the
coins were simply absent. scantxoutset walks the node's current utxo set
directly and finds them regardless of when the account was registered.

runs all four script variants sequentially — the funds could be on any
one — and surfaces progress through the existing SyncState channel so
the balance says "scanning" rather than nothing. auto-fires on an
imported mnemonic only; a generated seed has no history to look for.

recovers spendable coins, not spent history.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

src/servers/sidecar/wallet/backends/onchain.ts

@@ -61,6 +61,7 @@ import {
   type NewAddressRequest,
   type NodeInfo,
   type OnchainTx,
+  type RescanState,
   type SendCoinsRequest,
   type SendCoinsResult,
   type SignMessageResult,
@@ -312,9 +313,31 @@ export class OnchainBackend extends BaseBackend {
       syncedAt: this.current?.at ?? null,
       stale: this.current !== null && Date.now() - this.current.at >= SCAN_TTL_MS,
       lastError: this.lastError,
+      rescan: this.chain.rescanState?.() ?? null,
     };
   }
 
+  /**
+   * Ask the chain source to go looking for this wallet's coins from scratch.
+   *
+   * Returns as soon as the search is queued — it takes minutes, and the caller is an HTTP route. The
+   * refresh chained onto its completion is the part that matters: a rescan that found coins has changed
+   * nothing visible until the snapshot in front of it is rebuilt, and without this the owner would be
+   * staring at the same zero until the TTL happened to expire.
+   */
+  async startRescan(): Promise<RescanState> {
+    const start = this.chain.startRescan?.bind(this.chain);
+    if (!start) this.notSupported('rescanning the chain');
+    const { state, done } = start(this.scanContext().accounts);
+    void done
+      .then(() => this.refresh())
+      // The rescan's own failure is already on its state and reported through getSyncState; a failed
+      // refresh behind it lands on lastError the same way. Neither should surface as an unhandled
+      // rejection in the sidecar's log.
+      .catch(() => {});
+    return state;
+  }
+
   /**
    * The wallet's view of the chain, stale-while-revalidate.
    *

src/servers/sidecar/wallet/chain-source-nbxplorer.ts

@@ -6,16 +6,21 @@
 // many addresses have been used. What that costs instead is reassembly — NBXplorer answers wallet-wide,
 // so the per-address view the rest of the wallet expects has to be rebuilt from UTXO key paths.
 //
-// PRUNED-NODE CAVEAT, and it is a real one: this node is pruned to 25 GB, so a newly tracked xpub only
-// picks up activity from the moment it is registered. History older than the prune horizon will not
-// backfill and this source will report it as absent, not as an error. That is fine for a wallet created
-// here and wrong for one being recovered from an old seed — which is exactly why Esplora stays available
-// rather than being replaced.
+// REGISTRATION IS NOT RECOVERY, and this is the trap: a newly tracked xpub only picks up activity from
+// the moment it is registered. An imported seed therefore reads as a real, quiet, empty wallet — every
+// call succeeds, nothing errors, and the coins are simply not in the index. `startRescan` below is the
+// answer for balance and spending: `scantxoutset` walks the node's current UTXO set directly, so it finds
+// coins regardless of when the account was registered or how far back the node is pruned.
+//
+// What a rescan does NOT recover is spent-transaction history predating registration; that is a block
+// rescan, a much heavier thing, and this source will keep reporting it as absent rather than as an error.
+// Esplora stays available for the owner who wants the full history back.
 
 import * as bitcoin from 'bitcoinjs-lib';
 import {
   type AddressEntry,
   type ChainIndex,
+  type RescanHandle,
   type ScanAccount,
   type ScanContext,
   type ScannedAddress,
@@ -25,11 +30,29 @@ import {
 import { derivationScheme, type NbxplorerChain, type NbxTransaction, type NbxUtxo } from './nbxplorer';
 import type { SpendableUtxo } from './psbt';
 import { networkFor } from './psbt';
-import { BackendError, type AddressType, type BitcoinNetwork, type FeeEstimates, type OnchainTx } from './types';
+import {
+  BackendError,
+  type AddressType,
+  type BitcoinNetwork,
+  type FeeEstimates,
+  type OnchainTx,
+  type RescanState,
+} from './types';
 
 /** Confirmation targets, in blocks, behind the five fee tiers the wallet reports. */
 const FEE_TARGETS = { fastestFee: 1, halfHourFee: 3, hourFee: 6, economyFee: 25, minimumFee: 144 } as const;
 
+/** How often to ask how a `scantxoutset` is going. Frequent enough that a finished result is never missed. */
+const SCAN_POLL_MS = 2_000;
+
+/** Per-variant ceiling. A scan of a large UTXO set is minutes; anything past this is a wedged node. */
+const SCAN_TIMEOUT_MS = 10 * 60_000;
+
+/** Consecutive 404s tolerated before a variant has ever reported a status — see `scanVariant`. */
+const SCAN_NULL_GRACE = 5;
+
+const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
+
 export type NbxplorerChainSourceOptions = { chain: NbxplorerChain; network: BitcoinNetwork; label: string };
 
 export class NbxplorerChainSource implements WalletChainSource {
@@ -47,6 +70,10 @@ export class NbxplorerChainSource implements WalletChainSource {
    */
   private accounts: readonly ScanAccount[] = [];
 
+  /** The rescan in flight, or the last one's outcome. Mutated in place as the scan progresses. */
+  private rescan: RescanState | null = null;
+  private rescanRun: Promise<RescanState> | null = null;
+
   constructor(opts: NbxplorerChainSourceOptions) {
     this.chain = opts.chain;
     this.btcNetwork = networkFor(opts.network);
@@ -132,6 +159,95 @@ export class NbxplorerChainSource implements WalletChainSource {
     return { tipHeight, addresses, utxos, txs };
   }
 
+  rescanState(): RescanState | null {
+    return this.rescan;
+  }
+
+  /**
+   * Run `scantxoutset` over every account variant, one at a time.
+   *
+   * ALL FOUR VARIANTS, ALWAYS. A recovered seed's coins can sit on any script type, and nothing here can
+   * tell which until the node has looked — the p2sh account of a wallet that only ever used native segwit
+   * costs one wasted scan, whereas skipping it costs the owner their money. The node serialises
+   * `scantxoutset` anyway, so scanning them sequentially is what happens regardless of what we ask for;
+   * doing it explicitly is what makes `done`/`total` mean something on screen.
+   *
+   * A second call while one is running returns the same handle rather than queueing a duplicate.
+   */
+  startRescan(accounts: readonly ScanAccount[]): RescanHandle {
+    const existing = this.rescan;
+    if (existing?.running && this.rescanRun) return { state: existing, done: this.rescanRun };
+
+    const state: RescanState = {
+      running: true,
+      done: 0,
+      total: accounts.length,
+      current: accounts[0]?.type ?? null,
+      found: 0,
+      startedAt: Date.now(),
+      finishedAt: null,
+      error: null,
+    };
+    this.rescan = state;
+    this.rescanRun = this.runRescan(accounts, state);
+    return { state, done: this.rescanRun };
+  }
+
+  private async runRescan(accounts: readonly ScanAccount[], state: RescanState): Promise<RescanState> {
+    try {
+      for (const account of accounts) {
+        state.current = account.type;
+        state.found += await this.scanVariant(account);
+        state.done += 1;
+      }
+    } catch (err) {
+      // One variant failing stops the run, and the partial `found` stays on the state: coins already
+      // pulled in by an earlier variant are really there, and reporting zero would understate the wallet.
+      state.error = err instanceof Error ? err.message : String(err);
+    } finally {
+      state.running = false;
+      state.current = null;
+      state.finishedAt = Date.now();
+    }
+    return state;
+  }
+
+  /** Queue one variant's scan and wait it out. Resolves with how many UTXOs it pulled in. */
+  private async scanVariant(account: ScanAccount): Promise<number> {
+    const { accountXpub, type } = account;
+    await this.chain.startUtxoScan(accountXpub, type);
+
+    const deadline = Date.now() + SCAN_TIMEOUT_MS;
+    let found = 0;
+    let seen = false;
+    let nulls = 0;
+
+    while (Date.now() < deadline) {
+      await sleep(SCAN_POLL_MS);
+      const status = await this.chain.getUtxoScanStatus(accountXpub, type);
+
+      // 404 means "no scan is running" — which before the first sighting is a race with the queue, and
+      // after it means the result has expired. Only the second reading is a finish, hence the flag; the
+      // grace bounds the first so a scan the node silently dropped does not hold the run for ten minutes.
+      if (!status) {
+        if (seen) return found;
+        if (++nulls >= SCAN_NULL_GRACE) return found;
+        continue;
+      }
+
+      seen = true;
+      nulls = 0;
+      found = status.progress?.found ?? found;
+
+      if (status.status === 'Complete') return found;
+      if (status.status === 'Error') {
+        throw new BackendError(`utxo scan of the ${type} account failed: ${status.error ?? 'unknown'}`, 502, 'SCAN');
+      }
+    }
+
+    throw new BackendError(`utxo scan of the ${type} account did not finish in time`, 504, 'SCAN_TIMEOUT');
+  }
+
   private async scanAccount(ctx: ScanContext, account: ScanAccount) {
     const { accountXpub, type } = account;
     // track() is idempotent and every one of these calls makes it first, so registration cannot be

src/servers/sidecar/wallet/chain-source.ts

@@ -16,7 +16,7 @@
 // about which addresses are the wallet's, which is the kind of bug that loses coins rather than failing.
 
 import type { SpendableUtxo } from './psbt';
-import type { AddressType, FeeEstimates, OnchainTx } from './types';
+import type { AddressType, FeeEstimates, OnchainTx, RescanState } from './types';
 
 /** BIP44 chain index: 0 is the receive chain, 1 the internal (change) chain. */
 export type ChainIndex = 0 | 1;
@@ -69,6 +69,15 @@ export type ScanContext = {
   derive: (type: AddressType, chain: ChainIndex, index: number) => AddressEntry;
 };
 
+/**
+ * A rescan in flight.
+ *
+ * `state` is the SAME object the source keeps and mutates as the scan progresses, so a caller that holds
+ * it sees the counters move without asking again. `done` is how the wallet knows to re-read the chain:
+ * a rescan that found coins has changed nothing until the snapshot behind it is rebuilt.
+ */
+export type RescanHandle = { state: RescanState; done: Promise<RescanState> };
+
 /** One coherent read of the chain — balances, coins and history as of the same moment. */
 export type ScanResult = {
   tipHeight: number;
@@ -103,6 +112,19 @@ export interface WalletChainSource {
 
   /** Read the whole wallet off the chain. */
   scan(ctx: ScanContext): Promise<ScanResult>;
+
+  /**
+   * Search the chain for this wallet's coins from scratch, rather than from whenever the upstream
+   * started watching it. Returns immediately — a rescan takes minutes, and the caller is an HTTP route.
+   *
+   * Optional, and Esplora does not implement it — it has nothing to rescan, because a gap-limit walk
+   * already asks about every address every time. This exists for an upstream that *indexes*, where a
+   * newly registered account starts empty and stays empty until told to go and look.
+   */
+  startRescan?(accounts: readonly ScanAccount[]): RescanHandle;
+
+  /** The rescan in flight, or the last one's outcome. Null when this source has never run one. */
+  rescanState?(): RescanState | null;
 }
 
 /** Bounded-concurrency map that preserves input order. Address-level sources fan out wide. */

src/servers/sidecar/wallet/nbxplorer.ts

@@ -113,6 +113,40 @@ export type NbxAddress = {
   redeem?: string | null;
 };
 
+/**
+ * `GET …/utxos/scan` — how a `scantxoutset` sweep is going.
+ *
+ * Every field below `status` is optional on purpose: NBXplorer fills `progress` only once the node has
+ * actually started, and a queued scan reports nothing but its place in the queue.
+ */
+export type NbxScanStatus = {
+  status: 'Queued' | 'Pending' | 'Complete' | 'Error';
+  error?: string | null;
+  queuedAt?: string;
+  progress?: {
+    startedAt?: string;
+    completedAt?: string | null;
+    /** UTXOs pulled in. This is the number that answers "did the scan find my coins". */
+    found?: number;
+    batchNumber?: number;
+    remainingBatches?: number;
+    currentBatchProgress?: number;
+    overallProgress?: number;
+    remainingSeconds?: number;
+    highestKeyIndexFound?: Partial<Record<DerivationFeature, number | null>>;
+  } | null;
+};
+
+/**
+ * `POST …/utxos/scan` parameters.
+ *
+ * The defaults are the values the node operator verified live. `gapLimit` 1000 is generous for any
+ * wallet that has not used addresses beyond index 1000; raising it costs node CPU, not correctness.
+ */
+export type UtxoScanOptions = { batchSize?: number; gapLimit?: number; from?: number };
+
+const SCAN_DEFAULTS = { batchSize: 1000, gapLimit: 1000, from: 0 } as const;
+
 /** A failed broadcast comes back as HTTP 200 with `success: false` — never as an HTTP error. */
 export type NbxBroadcastResult = {
   success: boolean;
@@ -153,7 +187,7 @@ export class NbxplorerChain {
   }
 
   /** Every request funnels through here, so a timeout and an upstream failure share one error shape. */
-  private async raw(path: string, init: RequestInitLite = {}): Promise<string> {
+  private async send(path: string, init: RequestInitLite): Promise<{ status: number; body: string }> {
     const url = `${this.baseUrl}${path.startsWith('/') ? path : `/${path}`}`;
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), this.timeoutMs);
@@ -173,23 +207,48 @@ export class NbxplorerChain {
       clearTimeout(timer);
     }
 
-    const body = await res.text();
-    if (!res.ok) {
-      // NBXplorer answers with {"code","message"} where it can, and bare text where it cannot.
-      let message = body.slice(0, 300);
-      try {
-        const parsed = JSON.parse(body) as { message?: unknown };
-        if (typeof parsed.message === 'string' && parsed.message) message = parsed.message;
-      } catch {
-        /* not its JSON envelope */
-      }
-      throw new BackendError(`nbxplorer ${res.status} on ${path}: ${message}`, res.status, 'NBXPLORER_ERROR');
+    return { status: res.status, body: await res.text() };
+  }
+
+  private fail(path: string, status: number, body: string): never {
+    // NBXplorer answers with {"code","message"} where it can, and bare text where it cannot.
+    let message = body.slice(0, 300);
+    try {
+      const parsed = JSON.parse(body) as { message?: unknown };
+      if (typeof parsed.message === 'string' && parsed.message) message = parsed.message;
+    } catch {
+      /* not its JSON envelope */
     }
+    throw new BackendError(`nbxplorer ${status} on ${path}: ${message}`, status, 'NBXPLORER_ERROR');
+  }
+
+  private async raw(path: string, init: RequestInitLite = {}): Promise<string> {
+    const { status, body } = await this.send(path, init);
+    if (status < 200 || status >= 300) this.fail(path, status, body);
     return body;
   }
 
   private async json<T>(path: string, init?: RequestInitLite): Promise<T> {
     const body = await this.raw(path, init);
+    return this.parse<T>(path, body);
+  }
+
+  /**
+   * Like `json`, but a 404 is an answer rather than a failure.
+   *
+   * Only the scan-status endpoint needs this, and it needs it badly: NBXplorer 404s both when no scan is
+   * running and once a finished scan's result has expired. Treating that as an error would turn the two
+   * most ordinary moments of a rescan — before it starts, and a while after it ends — into upstream
+   * failures on the wallet screen.
+   */
+  private async jsonOrNull<T>(path: string, init?: RequestInitLite): Promise<T | null> {
+    const { status, body } = await this.send(path, init ?? {});
+    if (status === 404) return null;
+    if (status < 200 || status >= 300) this.fail(path, status, body);
+    return this.parse<T>(path, body);
+  }
+
+  private parse<T>(path: string, body: string): T {
     try {
       return JSON.parse(body) as T;
     } catch {
@@ -277,6 +336,42 @@ export class NbxplorerChain {
     return hex;
   }
 
+  /**
+   * `POST …/utxos/scan` — sweep the node's whole UTXO set for this account's coins.
+   *
+   * WHY THIS EXISTS AT ALL. Registering a scheme only makes NBXplorer index it *from now on*. An
+   * imported xpub with a history therefore reads as a real, quiet, zero-balance wallet: every call
+   * succeeds, nothing errors, and the coins are simply not there. This endpoint is the fix — it runs
+   * bitcoind's `scantxoutset`, which walks the current UTXO set directly and finds coins regardless of
+   * when the account was registered or how far back the node is pruned.
+   *
+   * It finds spendable coins, NOT history. Spent-transaction history predating registration stays
+   * missing; recovering that is a block rescan, which is a different and much heavier thing.
+   *
+   * Returns as soon as the scan is queued. `scantxoutset` is single-threaded and IO-heavy on the node,
+   * so concurrent scans queue and run one after another — poll `getUtxoScanStatus` for the outcome.
+   */
+  async startUtxoScan(accountXpub: string, type: AddressType, opts: UtxoScanOptions = {}): Promise<void> {
+    await this.track(accountXpub, type);
+    const q = new URLSearchParams({
+      batchSize: String(opts.batchSize ?? SCAN_DEFAULTS.batchSize),
+      gapLimit: String(opts.gapLimit ?? SCAN_DEFAULTS.gapLimit),
+      from: String(opts.from ?? SCAN_DEFAULTS.from),
+    });
+    await this.raw(`${this.root}/derivations/${this.scheme(accountXpub, type)}/utxos/scan?${q}`, { method: 'POST' });
+  }
+
+  /**
+   * `GET …/utxos/scan` — the running scan, or null.
+   *
+   * Null means "nothing to report": no scan is running, or one finished long enough ago that NBXplorer
+   * has dropped the result. Both are 404s and neither is a failure, so poll promptly and read a null
+   * after a `Complete` as "it is over", not as "it vanished".
+   */
+  getUtxoScanStatus(accountXpub: string, type: AddressType): Promise<NbxScanStatus | null> {
+    return this.jsonOrNull<NbxScanStatus>(`${this.root}/derivations/${this.scheme(accountXpub, type)}/utxos/scan`);
+  }
+
   /** `GET /v1/cryptos/BTC/fees/{blockCount}` — one target per call, sat/vB as a float. */
   async getFeeRate(blockCount: number): Promise<number> {
     const res = await this.json<{ feeRate: number; blockCount: number }>(`${this.root}/fees/${blockCount}`);

src/servers/sidecar/wallet/routes.ts

@@ -273,6 +273,18 @@ async function handleWallets(ctx: OfficerContext, seg: string[]): Promise<Respon
     case 'utxos':
       return await utxosRoute(ctx, walletId, backend, rest.slice(1));
 
+    // POST starts a deep rescan, GET reads how it is going. Both answer with the same `rescan` block that
+    // rides on balances/transactions/utxos, so the UI has one shape to render and can poll whichever it
+    // was already polling. A backend with nothing to rescan is a clean 501 rather than a silent no-op.
+    case 'rescan': {
+      if (ctx.req.method === 'GET') return json({ rescan: syncOf(backend)?.rescan ?? null });
+      if (ctx.req.method !== 'POST') return json({ error: 'method not allowed' }, 405);
+      if (!backend.startRescan) {
+        throw new BackendError('this wallet has nothing to rescan', 501, 'NOT_SUPPORTED');
+      }
+      return json({ rescan: await backend.startRescan() });
+    }
+
     case 'fees':
       return json({ fees: await backend.estimateFees() });
 
@@ -430,12 +442,33 @@ async function createWalletRoute(ctx: OfficerContext): Promise<Response> {
     makeActive: b.makeActive ?? true,
   });
 
+  // An IMPORTED seed is the one case that needs the chain searched from scratch: an indexing upstream
+  // only watches an account from the moment it is registered, so a wallet with a history would otherwise
+  // show a confident, wrong zero. A freshly generated seed has no history to find, and a rescan for it
+  // would burn several minutes of the node's CPU to confirm nothing.
+  if (b.mnemonic) void kickOffRescan(ctx.userId, wallet.id);
+
   // Return the mnemonic exactly once, and ONLY when we generated it — the owner has to write it down and
   // has no other chance to see it without re-entering the passphrase. An imported mnemonic is never
   // echoed back: the caller already has it, and echoing would put it in a response log for no reason.
   return json({ wallet, mnemonic: b.mnemonic ? undefined : mnemonic }, 201);
 }
 
+/**
+ * Start a rescan behind the response that created the wallet.
+ *
+ * Never throws: a chain source that cannot rescan, or an upstream that is down, must not turn a
+ * successful wallet import into a failed one. The owner can trigger it by hand from Settings either way.
+ */
+async function kickOffRescan(userId: number, walletId: number): Promise<void> {
+  try {
+    const { backend } = await resolveBackend(userId, walletId);
+    await backend.startRescan?.();
+  } catch (err) {
+    console.error('[wallet] rescan after import failed to start:', err instanceof Error ? err.message : err);
+  }
+}
+
 async function deleteWalletRoute(ctx: OfficerContext, walletId: number): Promise<Response> {
   const wallet = await getWallet(ctx.userId, walletId);
   if (!wallet) return json({ error: 'wallet not found' }, 404);

src/servers/sidecar/wallet/types.ts

@@ -298,6 +298,15 @@ export interface WalletBackend {
    * be inventing one.
    */
   getSyncState?(): SyncState;
+
+  /**
+   * Ask the chain to search for this wallet's coins from scratch. Resolves as soon as the search is
+   * queued, not when it finishes — progress is read back through `getSyncState().rescan`.
+   *
+   * Optional because only a wallet-level chain source can do it. Absent means the backend has nothing to
+   * rescan (a node backend already knows its own coins) or the source has no such endpoint.
+   */
+  startRescan?(): Promise<RescanState>;
 }
 
 /**
@@ -312,6 +321,34 @@ export type SyncState = {
   stale: boolean;
   /** The last refresh failure, still reported while the previous good data is being served. */
   lastError: string | null;
+  /** A deep rescan in flight, or the outcome of the last one. Null from a source that cannot rescan. */
+  rescan: RescanState | null;
+};
+
+/**
+ * A deep chain rescan — the upstream searching the whole UTXO set for a wallet's coins, rather than
+ * indexing forward from the moment the wallet was registered.
+ *
+ * This exists for exactly one situation, and it is not a rare one: an imported seed on a node that only
+ * started watching the xpub today. Every read succeeds, nothing errors, and the balance is zero — which
+ * is indistinguishable from an empty wallet unless the UI can say "still looking". So the state is
+ * carried on SyncState beside `syncedAt`, on the same responses, for the same reason.
+ *
+ * It runs one account variant at a time because the node runs `scantxoutset` one at a time; four
+ * variants is minutes, not seconds.
+ */
+export type RescanState = {
+  running: boolean;
+  /** Account variants finished, out of how many were queued. */
+  done: number;
+  total: number;
+  /** The variant the node is chewing on right now. Null when nothing is running. */
+  current: AddressType | null;
+  /** UTXOs the finished variants pulled in. */
+  found: number;
+  startedAt: number;
+  finishedAt: number | null;
+  error: string | null;
 };
 
 // ── errors ───────────────────────────────────────────────────────────────────────────────────────