headscale: diagnostics from the officer companion

Reads the per-server Officer Companion (COMMS/HEADSCALE_COMPANION_API.md): container health with its
own diagnosis, a log snapshot, a live SSE tail, and restart/stop/start.

The companion sits at ${server.url}/officer-api behind the same admin key we already store, so there
is nothing new to register — but only the sidecar can decrypt that key, so the sidecar proxies it and
the browser never talks to the companion directly. That also rules out EventSource for the stream
(no Authorization header), which is why the tail is fetch() + a hand-rolled SSE reader.

Two inversions the code is built around: /health is always HTTP 200 and must be read by verdict, and
an absent companion is a state to render rather than an error — an HTML 502 is nginx, a JSON 502 is
the companion reporting a failed docker op, and the admin API on the same domain is independent
either way.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-05 16:17:01 +00:00
co-authored by Claude Opus 5
parent 208f26ad89
commit d6d405100c
7 changed files with 748 additions and 1 deletions
@@ -0,0 +1,326 @@
import { useEffect, useRef, useState } from 'react';
import {
Activity,
AlertTriangle,
Loader2,
Play,
PlugZap,
RotateCw,
ScrollText,
Square,
Trash2,
Unplug,
} from 'lucide-react';
import type { CompanionAction, CompanionContainer, CompanionHealthBody, CompanionVerdict } from './shared';
import { timeAgo } from './format';
import { useHeadscaleServers } from './useHeadscaleServers';
import {
useCompanionAction,
useCompanionHealth,
useCompanionLogStream,
useCompanionLogs,
} from './useHeadscaleCompanion';
import { Card, SectionHeader, Button, Dot, ErrorNote } from './Cards';
import { ViewShell } from './ViewShell';
// What the admin API structurally cannot tell you: is the container running, what did it log on the way
// down, and can we bring it back. All of it comes from the Officer Companion deployed alongside the server.
//
// The companion is optional and per-server, so "not deployed" is the ordinary case for a server that has
// never had one, and is rendered as an explanation rather than an error. Note the two inversions this
// section is built around: the companion's /health is ALWAYS HTTP 200 (read `verdict`, never the status),
// and an unavailable companion says nothing about Headscale itself — the admin API on the same domain is
// independent and the other sections may be working perfectly.
const VERDICTS: Record<CompanionVerdict, { tone: 'ok' | 'warn' | 'bad' | 'idle'; label: string; blurb: string }> = {
ok: { tone: 'ok', label: 'Healthy', blurb: 'The container is running and Headscale is answering.' },
degraded: {
tone: 'warn',
label: 'Degraded',
blurb: 'The container is running, but Headscale is not answering properly.',
},
down: { tone: 'bad', label: 'Down', blurb: 'The container is not running.' },
unknown: { tone: 'idle', label: 'Unknown', blurb: 'Docker does not know this container.' },
};
const ACTIONS: { id: CompanionAction; label: string; icon: typeof RotateCw; confirm: string }[] = [
{
id: 'restart',
label: 'Restart',
icon: RotateCw,
confirm: 'Restart the Headscale container? Every node loses its control-plane connection until it is back.',
},
{
id: 'stop',
label: 'Stop',
icon: Square,
confirm: 'Stop the Headscale container? Every node stays disconnected until you start it again.',
},
{ id: 'start', label: 'Start', icon: Play, confirm: 'Start the Headscale container?' },
];
/** The RFC3339 zero date the companion passes through from docker for a container that never finished. */
const isZeroDate = (iso: string) => iso.startsWith('0001-');
const Row = ({ label, value }: { label: string; value: React.ReactNode }) => (
<div className="flex items-baseline justify-between gap-4 py-1.5 text-xs">
<span className="shrink-0 text-zinc-500">{label}</span>
<span className="min-w-0 truncate text-right text-zinc-300">{value}</span>
</div>
);
const ContainerFacts = ({ container }: { container: CompanionContainer }) => (
<div className="divide-y divide-white/5">
<Row label="Container" value={container.status} />
<Row label="Healthcheck" value={container.healthcheck ?? 'none defined'} />
<Row label="Started" value={timeAgo(container.startedAt)} />
{!container.running && !isZeroDate(container.finishedAt) && (
<Row label="Exited" value={`${timeAgo(container.finishedAt)} · code ${container.exitCode}`} />
)}
<Row label="Restarts" value={container.restartCount === 0 ? 'none' : `${container.restartCount} by docker`} />
</div>
);
/** Evidence, shown only when the verdict is not ok — likely causes first, then the raw tail behind them. */
const Evidence = ({ health }: { health: CompanionHealthBody }) => {
const causes = health.likelyCauses ?? [];
const recent = health.recentLogs ?? [];
if (causes.length === 0 && recent.length === 0 && !health.healthcheckOutput) return null;
return (
<div className="flex flex-col gap-3 border-t border-white/10 p-4">
{causes.length > 0 && (
<div>
<div className="mb-1.5 flex items-center gap-1.5 text-xs font-medium text-amber-300">
<AlertTriangle className="h-3.5 w-3.5" />
Likely causes
</div>
<ul className="flex flex-col gap-1">
{causes.map((cause) => (
<li key={cause} className="rounded-md bg-amber-500/10 px-2.5 py-1.5 text-xs text-amber-200/90">
{cause}
</li>
))}
</ul>
{/* The companion reads these out of the logs heuristically. Saying so is the difference between a
hint the owner checks and a diagnosis they trust and then chase down the wrong hole. */}
<p className="mt-1 text-[11px] text-zinc-600">Guessed from the logs treat them as leads, not answers.</p>
</div>
)}
{health.healthcheckOutput && (
<div>
<div className="mb-1.5 text-xs font-medium text-zinc-400">Healthcheck output</div>
<pre className="overflow-x-auto rounded-md bg-black/50 p-2.5 font-mono text-[11px] whitespace-pre-wrap text-zinc-400">
{health.healthcheckOutput}
</pre>
</div>
)}
{recent.length > 0 && (
<div>
<div className="mb-1.5 text-xs font-medium text-zinc-400">Last lines before now</div>
<pre className="max-h-48 overflow-auto rounded-md bg-black/50 p-2.5 font-mono text-[11px] text-zinc-400">
{recent.join('\n')}
</pre>
</div>
)}
</div>
);
};
const TAILS = [100, 500, 2000];
const LogViewer = () => {
const [tail, setTail] = useState(200);
const [follow, setFollow] = useState(false);
const snapshot = useCompanionLogs(tail, !follow);
const stream = useCompanionLogStream(follow, tail);
const boxRef = useRef<HTMLPreElement>(null);
const lines = follow ? stream.lines : snapshot.data?.available ? snapshot.data.lines : [];
// Pin to the bottom while following. Only while following: scrolling a snapshot back to the top and having
// it yanked down again would be the viewer fighting the reader.
useEffect(() => {
if (follow && boxRef.current) boxRef.current.scrollTop = boxRef.current.scrollHeight;
}, [follow, lines.length]);
const unavailable = !follow && snapshot.data && !snapshot.data.available ? snapshot.data.reason : null;
return (
<Card>
<div className="flex flex-wrap items-center gap-2 border-b border-white/10 px-3 py-2">
<div className="mr-auto flex items-center gap-1.5 text-xs font-medium text-zinc-300">
<ScrollText className="h-3.5 w-3.5" />
Logs
{follow && (
<span className="ml-1 inline-flex items-center gap-1 text-[11px] text-zinc-500">
<Dot tone={stream.live ? 'ok' : 'idle'} />
{stream.live ? 'live' : 'stopped'}
</span>
)}
</div>
<div className="flex items-center gap-1">
{TAILS.map((n) => (
<Button key={n} variant={tail === n ? 'primary' : 'ghost'} onClick={() => setTail(n)}>
{n}
</Button>
))}
</div>
<Button variant={follow ? 'primary' : 'ghost'} onClick={() => setFollow((on) => !on)}>
{follow ? <Unplug className="h-3.5 w-3.5" /> : <PlugZap className="h-3.5 w-3.5" />}
{follow ? 'Stop' : 'Follow'}
</Button>
{follow ? (
<Button onClick={stream.clear} title="Clear what has been received">
<Trash2 className="h-3.5 w-3.5" />
</Button>
) : (
<Button onClick={() => void snapshot.refetch()} disabled={snapshot.isFetching}>
{snapshot.isFetching ? (
<Loader2 className="h-3.5 w-3.5 animate-spin" />
) : (
<RotateCw className="h-3.5 w-3.5" />
)}
Refresh
</Button>
)}
</div>
{stream.error && (
<div className="border-b border-white/10 px-3 py-2 text-[11px] text-red-300">{stream.error}</div>
)}
{unavailable && <div className="border-b border-white/10 px-3 py-2 text-[11px] text-zinc-500">{unavailable}</div>}
<pre
ref={boxRef}
className="max-h-[26rem] min-h-[12rem] overflow-auto bg-black/40 p-3 font-mono text-[11px] leading-relaxed text-zinc-400"
>
{lines.length > 0
? lines.join('\n')
: snapshot.isLoading
? 'Loading…'
: follow
? 'Waiting for output…'
: 'No log lines.'}
</pre>
</Card>
);
};
const Lifecycle = ({ running }: { running: boolean | null }) => {
const action = useCompanionAction();
const [pending, setPending] = useState<CompanionAction | null>(null);
const run = async (id: CompanionAction, confirm: string) => {
if (!window.confirm(confirm)) return;
setPending(id);
try {
await action.mutateAsync(id);
} catch {
/* surfaced from action.error below */
} finally {
setPending(null);
}
};
const result = action.data;
return (
<div className="flex flex-col gap-2 border-t border-white/10 p-4">
<div className="flex flex-wrap items-center gap-2">
{ACTIONS.map(({ id, label, icon: Icon, confirm }) => (
<Button
key={id}
variant={id === 'stop' ? 'danger' : 'ghost'}
disabled={pending !== null || (running !== null && (id === 'start' ? running : !running))}
onClick={() => void run(id, confirm)}
title={id === 'start' && running ? 'Already running' : undefined}
>
{pending === id ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <Icon className="h-3.5 w-3.5" />}
{label}
</Button>
))}
<span className="text-[11px] text-zinc-600">Acts on the container, not on Officer.</span>
</div>
{action.error != null && <ErrorNote>The action could not be sent.</ErrorNote>}
{result && !result.available && <ErrorNote>{result.reason}</ErrorNote>}
{result && result.available && !result.ok && (
<ErrorNote>{result.error ?? 'Docker refused the action.'}</ErrorNote>
)}
{result && result.available && result.ok && (
<div className="text-[11px] text-emerald-300">
{result.action}: {result.result}
</div>
)}
</div>
);
};
export const DiagnosticsView = () => {
const { active } = useHeadscaleServers();
const query = useCompanionHealth();
const result = query.data;
return (
<ViewShell isLoading={query.isLoading} error={query.error} label="diagnostics">
<div className="mx-auto flex max-w-3xl flex-col gap-4">
<SectionHeader
title="Diagnostics"
subtitle={active ? `The container behind ${active.name}, as seen from the machine it runs on.` : undefined}
action={query.isFetching ? <Loader2 className="h-3.5 w-3.5 animate-spin text-zinc-600" /> : undefined}
/>
{result && !result.available ? (
<Card>
<div className="flex flex-col gap-2 p-4">
<div className="flex items-center gap-2 text-sm font-semibold text-zinc-100">
<Activity className="h-4 w-4 text-zinc-500" />
No companion on this server
</div>
<p className="text-xs leading-relaxed text-zinc-500">
{result.reason}. The Officer Companion is a small service deployed next to Headscale that can see its
container it is what makes health, logs and restart possible from here.
</p>
<p className="text-xs leading-relaxed text-zinc-600">
This says nothing about Headscale itself: it is served by the same domain but a different process, so
the other sections may be working normally. When the companion is missing, the Console section is the
way in.
</p>
</div>
</Card>
) : result ? (
<>
<Card>
<div className="flex items-start gap-3 p-4">
<div className="mt-1">
<Dot tone={VERDICTS[result.health.verdict].tone} />
</div>
<div className="min-w-0 flex-1">
<div className="text-sm font-semibold text-zinc-100">{VERDICTS[result.health.verdict].label}</div>
<p className="mt-0.5 text-xs text-zinc-500">
{result.health.reason ?? VERDICTS[result.health.verdict].blurb}
</p>
</div>
</div>
<div className="border-t border-white/10 px-4 py-2">
<Row label="Control plane" value={result.health.connected ? 'answering' : 'not answering'} />
{result.health.probe && <Row label="Probe" value={result.health.probe} />}
{result.health.container && <ContainerFacts container={result.health.container} />}
</div>
<Evidence health={result.health} />
<Lifecycle running={result.health.container?.running ?? null} />
</Card>
<LogViewer />
</>
) : null}
</div>
</ViewShell>
);
};
@@ -1,6 +1,6 @@
import type { LucideIcon } from 'lucide-react';
import { NavLink } from 'react-router';
import { Network, Server, Laptop, Users, KeyRound, TerminalSquare, Check } from 'lucide-react';
import { Network, Server, Laptop, Users, KeyRound, Activity, TerminalSquare, Check } from 'lucide-react';
import { HEADSCALE_SECTIONS, headscaleSectionPath, type HeadscaleSectionId } from './shared';
import { useHeadscaleServers } from './useHeadscaleServers';
@@ -18,6 +18,7 @@ const ICONS: Record<HeadscaleSectionId, LucideIcon> = {
nodes: Laptop,
users: Users,
keys: KeyRound,
diagnostics: Activity,
console: TerminalSquare,
};
@@ -3,6 +3,7 @@ import { ServersView } from './ServersView';
import { NodesView } from './NodesView';
import { UsersView } from './UsersView';
import { KeysView } from './KeysView';
import { DiagnosticsView } from './DiagnosticsView';
import { ConsoleView } from './ConsoleView';
// Right panel of the /headscale workspace — renders the section named by the URL.
@@ -20,6 +21,8 @@ export const HeadscaleView = () => {
return <UsersView />;
case 'keys':
return <KeysView />;
case 'diagnostics':
return <DiagnosticsView />;
case 'console':
return <ConsoleView />;
default:
@@ -9,6 +9,7 @@ export const HEADSCALE_SECTIONS = [
{ id: 'nodes', label: 'Nodes' },
{ id: 'users', label: 'Users' },
{ id: 'keys', label: 'Pre-auth keys' },
{ id: 'diagnostics', label: 'Diagnostics' },
{ id: 'console', label: 'Console' },
] as const;
@@ -56,6 +57,54 @@ export type HeadscaleSshTest = { ok: boolean; error?: string; ms: number };
/** Officer's supported floor, restated for UI copy. The sidecar is the enforcer; this is only a label. */
export const MIN_HEADSCALE_VERSION = '0.29';
// ── Companion API ─────────────────────────────────────────────────────────────────────────────────
// The Officer Companion is a service deployed next to a Headscale server that can see the container the
// admin API is served from: whether it is running, what it logged, and start/stop/restart. It is optional
// and per-server, so `available: false` is a first-class state rather than an error — the admin API on the
// same domain is independent and may still work. Contract: COMMS/HEADSCALE_COMPANION_API.md.
/** Never available for a companion that is missing — the reason says which flavour of missing. */
type Unavailable = { available: false; reason: string };
export type CompanionVerdict = 'ok' | 'degraded' | 'down' | 'unknown';
export type CompanionContainer = {
status: string;
running: boolean;
exitCode: number;
restartCount: number;
startedAt: string;
/** The RFC3339 zero date (`0001-…`) while the container is running. */
finishedAt: string;
/** Null when the image defines no healthcheck. */
healthcheck: string | null;
};
export type CompanionHealthBody = {
verdict: CompanionVerdict;
/** Whether Headscale's own HTTP is answering — the "is the control plane serving?" signal. */
connected: boolean;
container?: CompanionContainer;
/** Human string for the probe outcome, e.g. `GET /health -> 200`, `unreachable`, `container not running`. */
probe?: string;
/** Only on `unknown`: docker has no container by that name. */
reason?: string;
/** Only when not ok — best-effort guesses read out of the logs. May be empty. */
likelyCauses?: string[];
healthcheckOutput?: string | null;
recentLogs?: string[];
};
export type CompanionHealthResult = ({ available: true } & { health: CompanionHealthBody }) | Unavailable;
export type CompanionLogsResult = { available: true; lines: string[] } | Unavailable;
/** The three lifecycle verbs. `stop`/`start` are what the companion calls `disconnect`/`reconnect`. */
export type CompanionAction = 'restart' | 'stop' | 'start';
export type CompanionActionResult =
| { available: true; ok: boolean; action?: string; result?: string; error?: string }
| Unavailable;
// ── Domain objects ────────────────────────────────────────────────────────────────────────────────
// Ids are strings because Headscale's are uint64 — never parse them to numbers.
@@ -0,0 +1,165 @@
import { useEffect, useRef, useState } from 'react';
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
import { useClient, getHeaders } from 'hooks/useClient';
import type { CompanionAction, CompanionActionResult, CompanionHealthResult, CompanionLogsResult } from './shared';
// Client for the active server's Officer Companion. Everything here goes through the headscale sidecar,
// because the companion authenticates with the Headscale admin key — which is encrypted in Postgres and
// decryptable only there. The browser never sees it and never talks to the companion directly.
const BASE = '/headscale/_officer/companion';
const HEALTH_KEY = ['headscale', 'companion', 'health'] as const;
/**
* The container's health, polled.
*
* Polling is the point rather than a convenience: this section is what you have open while waiting for a
* restart to take, so it has to move on its own. 10s is fast enough to watch a container come back and slow
* enough that a degraded server isn't being hammered while it struggles.
*/
export function useCompanionHealth() {
const { get } = useClient();
return useQuery({
queryKey: HEALTH_KEY,
queryFn: () => get<CompanionHealthResult>(`${BASE}/health`),
refetchInterval: 10_000,
// A verdict from ten seconds ago is stale by definition; never show one on remount without refetching.
staleTime: 0,
});
}
/** A snapshot of the last N lines. The live tail is a separate thing — see useCompanionLogStream. */
export function useCompanionLogs(tail: number, enabled: boolean) {
const { get } = useClient();
return useQuery({
queryKey: ['headscale', 'companion', 'logs', tail],
queryFn: () => get<CompanionLogsResult>(`${BASE}/logs?tail=${tail}`),
enabled,
staleTime: 0,
});
}
/**
* Restart / stop / start the Headscale container.
*
* Every one of these drops every node's control-plane connection, so nothing here retries and nothing here
* fires without the owner having confirmed. The health query is invalidated on settle — including on
* failure, where "did it happen anyway?" is exactly the question.
*/
export function useCompanionAction() {
const { post } = useClient();
const qc = useQueryClient();
return useMutation({
mutationFn: (action: CompanionAction) => post<CompanionActionResult>(`${BASE}/${action}`),
onSettled: () => qc.invalidateQueries({ queryKey: HEALTH_KEY }),
});
}
/** How many lines the viewer keeps. Beyond this the browser, not the server, becomes the bottleneck. */
const MAX_LINES = 5000;
export type LogStream = {
lines: string[];
/** Set when the stream ended badly — the companion's own `event: error` frame, or a dropped connection. */
error: string | null;
/** True between opening the request and the stream ending, however it ends. */
live: boolean;
clear: () => void;
};
/**
* The live log tail, over `fetch()` rather than `EventSource`.
*
* `EventSource` cannot send an `Authorization` header and every hop of this chain needs one — Officer's own
* bearer token to reach the platform, and the Headscale admin key from there on. So the SSE framing is
* parsed by hand: split on blank lines, read `data:` and `event:`. It is a small parser and it only has to
* handle what the companion emits (one line per frame, an `event: error` frame before a fatal close).
*
* `follow` changing off aborts mid-stream; the abort is deliberately not reported as an error, since it is
* the owner switching the toggle rather than anything going wrong.
*/
export function useCompanionLogStream(follow: boolean, tail: number): LogStream {
const [lines, setLines] = useState<string[]>([]);
const [error, setError] = useState<string | null>(null);
const [live, setLive] = useState(false);
// Batched through a ref: a busy container emits faster than React can render, and one setState per line
// would spend the whole frame budget on log output.
const pending = useRef<string[]>([]);
useEffect(() => {
if (!follow) return;
const controller = new AbortController();
let flushTimer: ReturnType<typeof setInterval> | null = null;
setError(null);
setLive(true);
const flush = () => {
if (pending.current.length === 0) return;
const batch = pending.current;
pending.current = [];
setLines((prev) => {
const next = prev.concat(batch);
return next.length > MAX_LINES ? next.slice(next.length - MAX_LINES) : next;
});
};
const run = async () => {
try {
const res = await fetch(`/api${BASE}/logs/stream?tail=${tail}`, {
headers: getHeaders(),
signal: controller.signal,
});
if (!res.ok || !res.body) {
setError(`the log stream returned ${res.status}`);
return;
}
flushTimer = setInterval(flush, 200);
const reader = res.body.getReader();
const decoder = new TextDecoder();
let buffer = '';
for (;;) {
const { done, value } = await reader.read();
if (done) break;
buffer += decoder.decode(value, { stream: true });
// Frames are separated by a blank line; anything after the last one is a partial frame and waits.
const frames = buffer.split('\n\n');
buffer = frames.pop() ?? '';
for (const frame of frames) {
let event = 'message';
const data: string[] = [];
for (const rawLine of frame.split('\n')) {
if (rawLine.startsWith('event:')) event = rawLine.slice(6).trim();
else if (rawLine.startsWith('data:')) data.push(rawLine.slice(5).replace(/^ /, ''));
}
if (data.length === 0) continue;
const text = data.join('\n');
if (event === 'error') setError(text);
else pending.current.push(text);
}
}
} catch (err) {
if ((err as Error | null)?.name !== 'AbortError') setError('the log stream disconnected');
} finally {
if (flushTimer) clearInterval(flushTimer);
flush();
// An aborted stream is already being torn down by the effect that replaced this one; letting it set
// state here would flash "not live" onto a stream that is about to reopen.
if (!controller.signal.aborted) setLive(false);
}
};
void run();
return () => {
controller.abort();
if (flushTimer) clearInterval(flushTimer);
setLive(false);
};
}, [follow, tail]);
return { lines, error, live, clear: () => setLines([]) };
}