officerdev/platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

split the wallet's chain reads from its signing half

Browse Source 
OnchainBackend depended on the concrete EsploraChain class, so the only wallet
it could ever have was an Esplora-backed one. The seam is now WalletChainSource,
and it is drawn at the scan rather than at the HTTP client: Esplora is
address-level and has to walk the gap limit, NBXplorer is wallet-level and has
no per-address endpoint at all, so there is nothing to share one level down.

The backend keeps the keys and the money — derivation, snapshot cache, coin
selection, PSBT construction, signing — and owns no HTTP. Which indexer answers
is a constructor argument.

Also adds the NBXplorer implementation of the seam, verified end to end against
the owner's own pruned node, and the first tests over any of this: a stub
Esplora drives a real backend through the gap-limit walk, balance summation,
UTXO mapping, transaction scoring and address issuance. Nothing covered the
scan before it was moved, which is the wrong time to have no tests.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
pastilhas
2026-08-03 23:50:53 +00:00
co-authored by Claude Opus 5
parent 2d71424d5a
commit 5c38236b39
6 changed files with 944 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/servers/sidecar/wallet/backends/onchain.ts
+41 -230
View File
@@ -4,8 +4,13 @@
// (backends/EmbeddedLND.ts) and LdkNode (backends/LdkNode.ts) are thin JS shims over React Native
// native modules — `lndmobile`, `ldk-node-rn` — that bundle an actual node into the app process.
// Neither exists off a phone, so neither can be ported into a Bun sidecar. Instead this backend is a
// real wallet in its own right: it derives addresses from a BIP32 account xpub, reads the chain from an
// Esplora HTTP API (chain.ts), and builds/signs its own transactions with bitcoinjs-lib (psbt.ts).
// real wallet in its own right: it derives addresses from a BIP32 account xpub, reads the chain through
// an injected WalletChainSource (chain-source.ts), and builds/signs its own transactions with
// bitcoinjs-lib (psbt.ts).
//
// It owns the KEYS and the MONEY: derivation, the snapshot cache, coin selection, PSBT construction,
// signing. It owns no HTTP at all. Which indexer answers — a public Esplora, the owner's own node behind
// NBXplorer — is a constructor argument, and nothing below this line knows the difference.
//
// WATCH-ONLY WHILE LOCKED is the load-bearing design property. Everything a UI polls —
// getInfo / getBalances / getTransactions / getNewAddress / getUtxos / estimateFees — is derived from
@@ -21,7 +26,16 @@ import { HDKey } from '@scure/bip32';
import * as bitcoin from 'bitcoinjs-lib';
import { toXOnly } from 'bitcoinjs-lib/src/psbt/bip371';
import * as ecc from '@bitcoinerlab/secp256k1';
import type { EsploraAddress, EsploraChain, EsploraTx } from '../chain';
import {
mapLimit,
MAX_SCAN_INDEX,
REQUEST_CONCURRENCY,
type AddressEntry,
type ChainIndex,
type ScanContext,
type ScannedAddress,
type WalletChainSource,
} from '../chain-source';
import {
buildPsbt,
coinTypeFor,
@@ -130,37 +144,15 @@ const PURPOSE: Record<AddressType, number> = {
/** Preference order when the caller does not name a script type. Native segwit first. */
const TYPE_PREFERENCE: readonly AddressType[] = ['p2wpkh', 'p2tr', 'p2sh-p2wpkh', 'p2pkh'];
/** BIP44 chain index: 0 is the receive chain, 1 the internal (change) chain. */
type ChainIndex = 0 | 1;
type Account = {
type: AddressType;
node: HDKey;
/** The extended key exactly as it was given. Wallet-level chain sources index by this string. */
accountXpub: string;
/** Account-level path from the wallet root, e.g. `m/84'/0'/0'`. */
basePath: string;
};
type AddressEntry = {
type: AddressType;
chain: ChainIndex;
index: number;
address: string;
/** Compressed 33-byte pubkey, hex. */
pubkeyHex: string;
scriptPubKeyHex: string;
/** Full path from the wallet root — what the signer derives with. */
path: string;
/** Path relative to the account xpub, which is what the public Utxo type carries. */
relPath: string;
};
type ScannedAddress = AddressEntry & {
confirmedSats: number;
unconfirmedSats: number;
txCount: number;
used: boolean;
};
/**
* One coherent read of the chain: the address scan and everything derived from it, taken together.
*
@@ -175,35 +167,21 @@ type Snapshot = {
at: number;
tipHeight: number;
addresses: ScannedAddress[];
/** scriptPubKey hex → the entry that owns it. Used to score transactions as ours. Never persisted. */
byScript: Map<string, ScannedAddress>;
utxos: SpendableUtxo[];
txs: OnchainTx[];
};
// ── tuning ───────────────────────────────────────────────────────────────────────────────────────
/** BIP44's standard gap limit: 20 consecutive unused addresses ends the scan for a chain. */
const GAP_LIMIT = 20;
/** How long a snapshot is served without kicking off a refresh behind it. */
const SCAN_TTL_MS = 30_000;
/** Cap on the persisted history. A wallet with years of activity should not grow an unbounded jsonb blob. */
const MAX_CACHED_TXS = 500;
/** Parallel Esplora requests. Public instances rate-limit, so this stays modest. */
const REQUEST_CONCURRENCY = 6;
/** Hard stop on a runaway scan — a misconfigured xpub against a busy chain must not loop forever. */
const MAX_SCAN_INDEX = 1_000;
const DEFAULT_TX_LIMIT = 100;
// ── the backend ──────────────────────────────────────────────────────────────────────────────────
export type OnchainBackendOptions = {
chain: EsploraChain;
chain: WalletChainSource;
network: BitcoinNetwork;
/**
* The account-level extended public key. A bare string is taken as the BIP84 (p2wpkh) account; pass a
@@ -227,7 +205,7 @@ export class OnchainBackend extends BaseBackend {
'signMessage',
]);
private readonly chain: EsploraChain;
private readonly chain: WalletChainSource;
private readonly network: BitcoinNetwork;
private readonly btcNetwork: bitcoin.Network;
private readonly signer: WalletSigner;
@@ -269,6 +247,7 @@ export class OnchainBackend extends BaseBackend {
this.accounts.set(type, {
type,
node: parseAccountXpub(xpub),
accountXpub: xpub,
basePath: `m/${PURPOSE[type]}'/${coin}'/0'`,
});
}
@@ -457,10 +436,7 @@ export class OnchainBackend extends BaseBackend {
});
}
const byScript = new Map<string, ScannedAddress>();
for (const entry of addresses) byScript.set(entry.scriptPubKeyHex, entry);
return { at, tipHeight: stored.tipHeight, addresses, byScript, utxos, txs: stored.transactions };
return { at, tipHeight: stored.tipHeight, addresses, utxos, txs: stored.transactions };
}
/** Derive from untrusted stored coordinates — returns null rather than throwing on anything unusable. */
@@ -475,12 +451,25 @@ export class OnchainBackend extends BaseBackend {
}
}
/** One full read of the chain: scan, then everything derived from it, all at the same moment. */
/**
* What the source is allowed to know: the account keys, which type is ours, and how to derive.
*
* `derive` is bound to this backend on purpose — a source that derived addresses itself could disagree
* with the wallet about which addresses are the wallet's, and the resulting bug would lose coins rather
* than fail loudly.
*/
private scanContext(): ScanContext {
return {
accounts: [...this.accounts.values()].map((a) => ({ type: a.type, accountXpub: a.accountXpub })),
defaultType: this.defaultType,
derive: (type, chain, index) => this.derive(type, chain, index),
};
}
/** One full read of the chain, stamped with the moment it was taken. */
private async buildSnapshot(): Promise<Snapshot> {
const scan = await this.runScan();
// Both fan out over the scanned address set and neither depends on the other, so they overlap.
const [utxos, txs] = await Promise.all([this.collectUtxos(scan), this.collectTxs(scan)]);
return { ...scan, utxos, txs };
const scan = await this.chain.scan(this.scanContext());
return { at: Date.now(), ...scan };
}
/** Drop everything after a send, so the spent coins cannot reappear from cache on the next read. */
@@ -488,64 +477,6 @@ export class OnchainBackend extends BaseBackend {
this.current = null;
}
private async runScan(): Promise<Omit<Snapshot, 'utxos' | 'txs'>> {
const tipHeight = await this.chain.getTipHeight();
const addresses: ScannedAddress[] = [];
// A wallet holds an account xpub for all four BIP purposes, but a full gap-limit walk of every one
// costs 4 types x 2 chains x 20 addresses = 160 requests, which trips the rate limit on every public
// Esplora instance. Only the wallet's own script type is walked unconditionally; the others are
// probed at receive index 0 first (one request each) and walked only if that address has ever been
// used. A freshly generated seed therefore costs 43 requests instead of 160, while a seed recovered
// from a wallet that used a different script type is still found rather than silently reported empty.
for (const type of this.accounts.keys()) {
if (type !== this.defaultType && !(await this.hasHistory(type))) continue;
for (const chain of [0, 1] as ChainIndex[]) {
addresses.push(...(await this.scanChain(type, chain)));
}
}
const byScript = new Map<string, ScannedAddress>();
for (const entry of addresses) byScript.set(entry.scriptPubKeyHex, entry);
return { at: Date.now(), tipHeight, addresses, byScript };
}
/**
* Has this script type ever been used at all? One request against receive index 0, which is the
* address any wallet hands out first — so a used account is essentially never missed, and an unused
* one costs a single call instead of a forty-address walk.
*/
private async hasHistory(type: AddressType): Promise<boolean> {
const stat = await this.chain.getAddress(this.derive(type, 0, 0).address);
return toScannedAddress(this.derive(type, 0, 0), stat).used;
}
/** Walk one (type, chain) pair a gap-limit window at a time until GAP_LIMIT consecutive misses. */
private async scanChain(type: AddressType, chain: ChainIndex): Promise<ScannedAddress[]> {
const found: ScannedAddress[] = [];
let index = 0;
let gap = 0;
while (gap < GAP_LIMIT && index < MAX_SCAN_INDEX) {
const window = Array.from({ length: GAP_LIMIT }, (_, i) => this.derive(type, chain, index + i));
const stats = await mapLimit(window, REQUEST_CONCURRENCY, (entry) => this.chain.getAddress(entry.address));
for (let i = 0; i < window.length; i++) {
const entry = window[i];
const stat = stats[i];
if (!entry || !stat) continue;
const scanned = toScannedAddress(entry, stat);
found.push(scanned);
gap = scanned.used ? 0 : gap + 1;
if (gap >= GAP_LIMIT) break;
}
index += window.length;
}
return found;
}
// ── watch-only reads (no key material required) ────────────────────────────────────────────────
async getInfo(): Promise<NodeInfo> {
@@ -614,69 +545,6 @@ export class OnchainBackend extends BaseBackend {
}
/** History for every address the scan found had been touched. Only those can appear in it. */
private async collectTxs(scan: Omit<Snapshot, 'utxos' | 'txs'>): Promise<OnchainTx[]> {
const touched = scan.addresses.filter((a) => a.used);
const pages = await mapLimit(touched, REQUEST_CONCURRENCY, (a) => this.chain.getAddressTxs(a.address));
const seen = new Map<string, EsploraTx>();
for (const page of pages) {
for (const tx of page) if (!seen.has(tx.txid)) seen.set(tx.txid, tx);
}
const txs = [...seen.values()].map((tx) => this.toOnchainTx(tx, scan));
txs.sort((a, b) => {
// Unconfirmed first (they have no height), then newest block, then newest timestamp.
const ah = a.blockHeight ?? Number.MAX_SAFE_INTEGER;
const bh = b.blockHeight ?? Number.MAX_SAFE_INTEGER;
if (ah !== bh) return bh - ah;
return (b.timestamp ?? 0) - (a.timestamp ?? 0);
});
return txs.slice(0, MAX_CACHED_TXS);
}
/** Score one Esplora transaction against the wallet's own scripts. */
private toOnchainTx(tx: EsploraTx, scan: Omit<Snapshot, 'utxos' | 'txs'>): OnchainTx {
let credit = 0;
let debit = 0;
const ours: string[] = [];
const theirs: string[] = [];
for (const out of tx.vout) {
const mine = scan.byScript.get(out.scriptpubkey);
if (mine) {
credit += out.value;
ours.push(mine.address);
} else if (out.scriptpubkey_address) {
theirs.push(out.scriptpubkey_address);
}
}
for (const input of tx.vin) {
const prevout = input.prevout;
if (prevout && scan.byScript.has(prevout.scriptpubkey)) debit += prevout.value;
}
const height = tx.status.confirmed ? (tx.status.block_height ?? null) : null;
const confirmations = height === null ? 0 : Math.max(0, scan.tipHeight - height + 1);
const amount = credit - debit;
return {
txid: tx.txid,
amount,
// The fee is only ours to report when we funded an input; for an incoming payment the sender
// paid it and attributing it to this wallet would be a lie.
feeSats: debit > 0 ? tx.fee : null,
blockHeight: height,
timestamp: tx.status.block_time ?? null,
confirmations,
// No label store in this backend; SendCoinsRequest.label is accepted and dropped.
label: null,
destAddresses: amount < 0 ? (theirs.length > 0 ? theirs : ours) : ours,
// Fetching /tx/{txid}/hex per transaction would double the request count for a list view. The
// contract permits null, and the raw hex is fetched on demand where it is actually needed.
rawHex: null,
};
}
// ── spending (requires the root key) ───────────────────────────────────────────────────────────
override async sendCoins(req: SendCoinsRequest): Promise<SendCoinsResult> {
@@ -793,31 +661,6 @@ export class OnchainBackend extends BaseBackend {
// ── shared internals ───────────────────────────────────────────────────────────────────────────
/** Fetch UTXOs for every scanned address that still holds a balance. */
private async collectUtxos(scan: Omit<Snapshot, 'utxos' | 'txs'>): Promise<SpendableUtxo[]> {
// An address whose funded and spent counts match holds nothing; asking Esplora about it is a
// wasted round trip, and on a wallet with long history that is most of the address set.
const funded = scan.addresses.filter((a) => a.confirmedSats + a.unconfirmedSats > 0);
const sets = await mapLimit(funded, REQUEST_CONCURRENCY, async (entry) => {
const utxos = await this.chain.getAddressUtxos(entry.address);
return utxos.map<SpendableUtxo>((u) => ({
txid: u.txid,
vout: u.vout,
amountSats: u.value,
address: entry.address,
addressType: entry.type,
confirmations:
u.status.confirmed && u.status.block_height !== undefined
? Math.max(0, scan.tipHeight - u.status.block_height + 1)
: 0,
derivationPath: entry.path,
frozen: false,
scriptPubKeyHex: entry.scriptPubKeyHex,
pubkeyHex: entry.pubkeyHex,
}));
});
return sets.flat();
}
/**
* First address on a chain that the blockchain has never seen, at or beyond the in-memory issuance
* mark. `peek` reads without consuming; otherwise the mark advances so the next call hands out a
@@ -891,21 +734,6 @@ function toPersisted(snap: Snapshot): WalletChainSnapshot {
};
}
function toScannedAddress(entry: AddressEntry, stat: EsploraAddress): ScannedAddress {
const chainBalance = stat.chain_stats.funded_txo_sum - stat.chain_stats.spent_txo_sum;
// The mempool delta is signed: an unconfirmed spend of a confirmed coin reads negative here, which is
// exactly what the Balances contract wants in onchainUnconfirmed.
const mempoolBalance = stat.mempool_stats.funded_txo_sum - stat.mempool_stats.spent_txo_sum;
const txCount = stat.chain_stats.tx_count + stat.mempool_stats.tx_count;
return {
...entry,
confirmedSats: chainBalance,
unconfirmedSats: mempoolBalance,
txCount,
used: txCount > 0,
};
}
/** Varint, for the length prefix in the Bitcoin signed-message preimage. */
function varint(n: number): Buffer {
if (n < 0xfd) return Buffer.from([n]);
@@ -929,20 +757,3 @@ function bitcoinMessageHash(message: string, network: bitcoin.Network): Buffer {
const body = Buffer.from(message, 'utf8');
return bitcoin.crypto.hash256(Buffer.concat([prefix, varint(body.length), body]));
}
/** Bounded-concurrency map that preserves input order. Esplora is per-address, so scans fan out wide. */
async function mapLimit<T, R>(items: T[], limit: number, fn: (item: T) => Promise<R>): Promise<R[]> {
const out = new Array<R>(items.length);
let cursor = 0;
const worker = async (): Promise<void> => {
for (;;) {
const i = cursor++;
if (i >= items.length) return;
const item = items[i];
if (item === undefined) continue;
out[i] = await fn(item);
}
};
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
return out;
}