Three more same-level grid cards (bTop now 7 cards; Temperature stays under Memory via natural 3-col flow): - GPU: gpu_busy_percent + VRAM used/total from /sys/class/drm (instant). - Network: ↓/↑ throughput (bytes/sec) from /proc/net/dev deltas between /stats calls, aggregate + top interfaces. - Power: CPU package watts via RAPL energy delta + GPU watts (amdgpu hwmon). Note: RAPL energy_uj is root-only by default (Spectre-era lockdown), so CPU package power shows "—" unless made readable (a udev rule); GPU watts work. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
501 lines
17 KiB
TypeScript
501 lines
17 KiB
TypeScript
import os from 'node:os';
|
|
import { readFile, readdir } from 'node:fs/promises';
|
|
import { execFile } from 'node:child_process';
|
|
import { promisify } from 'node:util';
|
|
import { createRouter } from '../../create-router';
|
|
|
|
// First-stab system monitor: a single /stats snapshot the frontend polls. Linux-only (reads /proc +
|
|
// shells out to df/ps); each piece degrades to empty/zero rather than failing the whole response.
|
|
|
|
const exec = promisify(execFile);
|
|
|
|
export const systemMonitorRouter = createRouter();
|
|
|
|
type CpuSample = { total: number; busy: number; perCore: { total: number; busy: number }[] };
|
|
|
|
async function readCpuSample(): Promise<CpuSample> {
|
|
const data = await readFile('/proc/stat', 'utf8');
|
|
const lines = data.split('\n').filter((l) => l.startsWith('cpu'));
|
|
const parse = (line: string) => {
|
|
const n = line.trim().split(/\s+/).slice(1).map(Number);
|
|
const total = n.reduce((a, b) => a + b, 0);
|
|
const idle = (n[3] ?? 0) + (n[4] ?? 0); // idle + iowait
|
|
return { total, busy: total - idle };
|
|
};
|
|
const agg = parse(lines[0] ?? 'cpu 0 0 0 0');
|
|
const perCore = lines.slice(1).map(parse);
|
|
return { total: agg.total, busy: agg.busy, perCore };
|
|
}
|
|
|
|
const pct = (busyDelta: number, totalDelta: number) =>
|
|
totalDelta > 0 ? Math.round((busyDelta / totalDelta) * 1000) / 10 : 0;
|
|
|
|
async function readMem() {
|
|
const data = await readFile('/proc/meminfo', 'utf8');
|
|
const map: Record<string, number> = {};
|
|
for (const line of data.split('\n')) {
|
|
const m = line.match(/^(\w+):\s+(\d+)/);
|
|
if (m) map[m[1]!] = Number(m[2]) * 1024; // kB → bytes
|
|
}
|
|
const total = map.MemTotal ?? 0;
|
|
const available = map.MemAvailable ?? map.MemFree ?? 0;
|
|
const used = Math.max(0, total - available);
|
|
const swapTotal = map.SwapTotal ?? 0;
|
|
const swapUsed = Math.max(0, swapTotal - (map.SwapFree ?? 0));
|
|
return {
|
|
totalBytes: total,
|
|
usedBytes: used,
|
|
freeBytes: available,
|
|
usedPct: total ? Math.round((used / total) * 1000) / 10 : 0,
|
|
swapTotalBytes: swapTotal,
|
|
swapUsedBytes: swapUsed,
|
|
};
|
|
}
|
|
|
|
async function readDisks() {
|
|
try {
|
|
const { stdout } = await exec('df', [
|
|
'-B1',
|
|
'--output=target,fstype,size,used,pcent',
|
|
'-x', 'tmpfs', '-x', 'devtmpfs', '-x', 'squashfs', '-x', 'overlay', '-x', 'efivarfs',
|
|
]);
|
|
return stdout
|
|
.trim()
|
|
.split('\n')
|
|
.slice(1)
|
|
.map((l) => {
|
|
const [mount, fsType, size, used, pcent] = l.trim().split(/\s+/);
|
|
return {
|
|
mount: mount ?? '',
|
|
fsType: fsType ?? '',
|
|
totalBytes: Number(size ?? 0),
|
|
usedBytes: Number(used ?? 0),
|
|
usedPct: Number((pcent ?? '0').replace('%', '')),
|
|
};
|
|
})
|
|
.filter((d) => d.totalBytes > 0);
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
async function readProcesses() {
|
|
try {
|
|
const { stdout } = await exec('ps', ['-eo', 'pid,user:16,pcpu,pmem,comm', '--sort=-pcpu']);
|
|
return stdout
|
|
.trim()
|
|
.split('\n')
|
|
.slice(1, 21)
|
|
.map((l) => {
|
|
const [pid, user, cpu, mem, ...comm] = l.trim().split(/\s+/);
|
|
return {
|
|
pid: Number(pid ?? 0),
|
|
user: user ?? '',
|
|
cpuPct: Number(cpu ?? 0),
|
|
memPct: Number(mem ?? 0),
|
|
command: comm.join(' '),
|
|
};
|
|
});
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
// GET /stats — one snapshot. CPU% comes from two /proc/stat samples ~120ms apart.
|
|
// Read every hwmon temperature sensor (name/label/°C) and pick the CPU one (k10temp/coretemp/…).
|
|
async function readTemps() {
|
|
const sensors: { name: string; label: string; celsius: number }[] = [];
|
|
let dirs: string[];
|
|
try {
|
|
dirs = await readdir('/sys/class/hwmon');
|
|
} catch {
|
|
return { cpuC: null as number | null, cpuLabel: null as string | null, sensors };
|
|
}
|
|
for (const d of dirs) {
|
|
const base = `/sys/class/hwmon/${d}`;
|
|
const name = (await readFile(`${base}/name`, 'utf8').catch(() => '')).trim();
|
|
let files: string[];
|
|
try {
|
|
files = await readdir(base);
|
|
} catch {
|
|
continue;
|
|
}
|
|
for (const f of files) {
|
|
const m = f.match(/^temp(\d+)_input$/);
|
|
if (!m) continue;
|
|
const milli = Number.parseInt((await readFile(`${base}/${f}`, 'utf8').catch(() => '')).trim(), 10);
|
|
if (!Number.isFinite(milli)) continue;
|
|
const label = (await readFile(`${base}/temp${m[1]}_label`, 'utf8').catch(() => '')).trim();
|
|
sensors.push({ name, label: label || `temp${m[1]}`, celsius: Math.round(milli / 100) / 10 });
|
|
}
|
|
}
|
|
const CPU_DRIVERS = ['k10temp', 'zenpower', 'coretemp', 'k8temp', 'cpu_thermal'];
|
|
const cpu =
|
|
sensors.find((s) => CPU_DRIVERS.includes(s.name.toLowerCase()) && /tctl|tdie|package|composite|core 0/i.test(s.label)) ??
|
|
sensors.find((s) => CPU_DRIVERS.includes(s.name.toLowerCase())) ??
|
|
null;
|
|
return { cpuC: cpu?.celsius ?? null, cpuLabel: cpu ? `${cpu.name} · ${cpu.label}` : null, sensors };
|
|
}
|
|
|
|
// GPU (amdgpu/intel): busy% + VRAM, read straight from /sys/class/drm (instant, no sampling).
|
|
async function readGpu() {
|
|
let cards: string[];
|
|
try {
|
|
cards = await readdir('/sys/class/drm');
|
|
} catch {
|
|
return null;
|
|
}
|
|
for (const c of cards.sort()) {
|
|
if (!/^card\d+$/.test(c)) continue;
|
|
const dev = `/sys/class/drm/${c}/device`;
|
|
const busyRaw = await readFile(`${dev}/gpu_busy_percent`, 'utf8').catch(() => null);
|
|
if (busyRaw == null) continue;
|
|
const busyPct = Number.parseInt(busyRaw.trim(), 10);
|
|
const vramUsed = Number.parseInt((await readFile(`${dev}/mem_info_vram_used`, 'utf8').catch(() => '0')).trim(), 10) || 0;
|
|
const vramTotal = Number.parseInt((await readFile(`${dev}/mem_info_vram_total`, 'utf8').catch(() => '0')).trim(), 10) || 0;
|
|
return { busyPct: Number.isFinite(busyPct) ? busyPct : 0, vramUsedBytes: vramUsed, vramTotalBytes: vramTotal };
|
|
}
|
|
return null;
|
|
}
|
|
|
|
// Network throughput — bytes/sec computed from the delta since the previous /stats call (~2s window).
|
|
let lastNet: { total: { rx: number; tx: number }; per: Record<string, { rx: number; tx: number }>; ts: number } | null = null;
|
|
async function readNet() {
|
|
const now = Date.now();
|
|
let data: string;
|
|
try {
|
|
data = await readFile('/proc/net/dev', 'utf8');
|
|
} catch {
|
|
return null;
|
|
}
|
|
const per: Record<string, { rx: number; tx: number }> = {};
|
|
let totRx = 0;
|
|
let totTx = 0;
|
|
for (const line of data.split('\n')) {
|
|
const m = line.match(/^\s*([^:]+):\s*(.+)$/);
|
|
if (!m) continue;
|
|
const iface = m[1]!.trim();
|
|
if (iface === 'lo') continue;
|
|
const cols = m[2]!.trim().split(/\s+/).map(Number);
|
|
const rx = cols[0] ?? 0;
|
|
const tx = cols[8] ?? 0;
|
|
per[iface] = { rx, tx };
|
|
totRx += rx;
|
|
totTx += tx;
|
|
}
|
|
const prev = lastNet;
|
|
lastNet = { total: { rx: totRx, tx: totTx }, per, ts: now };
|
|
if (!prev || now <= prev.ts) return { rxBytesPerSec: 0, txBytesPerSec: 0, interfaces: [] as { name: string; rxBytesPerSec: number; txBytesPerSec: number }[] };
|
|
const dt = (now - prev.ts) / 1000;
|
|
const rate = (cur: number, old: number) => Math.max(0, Math.round((cur - old) / dt));
|
|
const interfaces = Object.entries(per)
|
|
.map(([name, v]) => ({ name, rxBytesPerSec: rate(v.rx, prev.per[name]?.rx ?? v.rx), txBytesPerSec: rate(v.tx, prev.per[name]?.tx ?? v.tx) }))
|
|
.filter((i) => i.rxBytesPerSec > 0 || i.txBytesPerSec > 0)
|
|
.sort((a, b) => b.rxBytesPerSec + b.txBytesPerSec - (a.rxBytesPerSec + a.txBytesPerSec));
|
|
return { rxBytesPerSec: rate(totRx, prev.total.rx), txBytesPerSec: rate(totTx, prev.total.tx), interfaces };
|
|
}
|
|
|
|
// Power draw — CPU package watts from the RAPL energy counter (delta since last call) + GPU watts
|
|
// (amdgpu hwmon, instant).
|
|
let lastRapl: { uj: number; ts: number } | null = null;
|
|
async function readPower() {
|
|
const now = Date.now();
|
|
let cpuWatts: number | null = null;
|
|
const ujRaw = await readFile('/sys/class/powercap/intel-rapl:0/energy_uj', 'utf8').catch(() => null);
|
|
if (ujRaw != null) {
|
|
const uj = Number.parseInt(ujRaw.trim(), 10);
|
|
const prev = lastRapl;
|
|
lastRapl = { uj, ts: now };
|
|
if (prev && now > prev.ts && uj >= prev.uj) {
|
|
cpuWatts = Math.round(((uj - prev.uj) / 1e6 / ((now - prev.ts) / 1000)) * 10) / 10;
|
|
}
|
|
}
|
|
let gpuWatts: number | null = null;
|
|
try {
|
|
for (const d of await readdir('/sys/class/hwmon')) {
|
|
const base = `/sys/class/hwmon/${d}`;
|
|
if ((await readFile(`${base}/name`, 'utf8').catch(() => '')).trim() !== 'amdgpu') continue;
|
|
const p = (await readFile(`${base}/power1_average`, 'utf8').catch(() => null)) ?? (await readFile(`${base}/power1_input`, 'utf8').catch(() => null));
|
|
if (p != null) {
|
|
const uw = Number.parseInt(p.trim(), 10);
|
|
if (Number.isFinite(uw)) gpuWatts = Math.round((uw / 1e6) * 10) / 10;
|
|
}
|
|
break;
|
|
}
|
|
} catch {
|
|
/* no amdgpu */
|
|
}
|
|
return { cpuWatts, gpuWatts };
|
|
}
|
|
|
|
systemMonitorRouter.get('/stats', async (ctx) => {
|
|
const first = await readCpuSample().catch(() => null);
|
|
await new Promise((r) => setTimeout(r, 120));
|
|
const second = await readCpuSample().catch(() => null);
|
|
|
|
const cpuUsage =
|
|
first && second ? pct(second.busy - first.busy, second.total - first.total) : 0;
|
|
const perCore =
|
|
first && second
|
|
? second.perCore.map((c, i) => {
|
|
const a = first.perCore[i];
|
|
return a ? pct(c.busy - a.busy, c.total - a.total) : 0;
|
|
})
|
|
: [];
|
|
|
|
const [mem, disks, processes, temp, gpu, net, power] = await Promise.all([
|
|
readMem().catch(() => null),
|
|
readDisks(),
|
|
readProcesses(),
|
|
readTemps().catch(() => null),
|
|
readGpu().catch(() => null),
|
|
readNet().catch(() => null),
|
|
readPower().catch(() => null),
|
|
]);
|
|
|
|
const cpus = os.cpus();
|
|
return ctx.json({
|
|
hostname: os.hostname(),
|
|
platform: `${os.type()} ${os.release()}`,
|
|
uptimeSec: Math.round(os.uptime()),
|
|
loadavg: os.loadavg(),
|
|
cpu: {
|
|
model: cpus[0]?.model?.trim() ?? 'unknown',
|
|
cores: cpus.length,
|
|
usagePct: cpuUsage,
|
|
perCore,
|
|
},
|
|
mem,
|
|
disks,
|
|
processes,
|
|
temp,
|
|
gpu,
|
|
net,
|
|
power,
|
|
timestamp: Date.now(),
|
|
});
|
|
});
|
|
|
|
// GET /pm2 — pm2 process list (the "pm2 processes" scope).
|
|
systemMonitorRouter.get('/pm2', async (ctx) => {
|
|
try {
|
|
const { stdout } = await exec('pm2', ['jlist'], { maxBuffer: 8 * 1024 * 1024 });
|
|
const list = JSON.parse(stdout) as Array<{
|
|
pm_id?: number;
|
|
name?: string;
|
|
pid?: number;
|
|
monit?: { cpu?: number; memory?: number };
|
|
pm2_env?: { status?: string; restart_time?: number; pm_uptime?: number };
|
|
}>;
|
|
const now = Date.now();
|
|
return ctx.json({
|
|
processes: list.map((p) => {
|
|
const env = p.pm2_env ?? {};
|
|
return {
|
|
id: p.pm_id ?? -1,
|
|
name: p.name ?? '?',
|
|
status: env.status ?? 'unknown',
|
|
pid: p.pid ?? null,
|
|
cpuPct: p.monit?.cpu ?? 0,
|
|
memBytes: p.monit?.memory ?? 0,
|
|
restarts: env.restart_time ?? 0,
|
|
uptimeMs: env.status === 'online' && env.pm_uptime ? now - env.pm_uptime : 0,
|
|
};
|
|
}),
|
|
});
|
|
} catch (err) {
|
|
return ctx.json({ processes: [], error: err instanceof Error ? err.message : String(err) });
|
|
}
|
|
});
|
|
|
|
// GET /docker — running docker containers (the "dockers" scope).
|
|
systemMonitorRouter.get('/docker', async (ctx) => {
|
|
try {
|
|
const { stdout } = await exec('docker', ['ps', '--no-trunc', '--format', '{{json .}}'], { maxBuffer: 8 * 1024 * 1024 });
|
|
return ctx.json({
|
|
containers: stdout
|
|
.trim()
|
|
.split('\n')
|
|
.filter(Boolean)
|
|
.map((line) => {
|
|
const c = JSON.parse(line) as Record<string, string>;
|
|
return {
|
|
id: (c.ID ?? '').slice(0, 12),
|
|
name: c.Names ?? '',
|
|
image: c.Image ?? '',
|
|
state: c.State ?? '',
|
|
status: c.Status ?? '',
|
|
ports: c.Ports ?? '',
|
|
};
|
|
}),
|
|
});
|
|
} catch (err) {
|
|
return ctx.json({ containers: [], error: err instanceof Error ? err.message : String(err) });
|
|
}
|
|
});
|
|
|
|
// GET /pm2/logs?id=<pm_id>&lines=<n> — SSE stream of a pm2 process's live logs (combined out+err via
|
|
// `pm2 logs <id> --raw`). id is validated numeric and passed as a spawn arg (no shell) — no injection.
|
|
systemMonitorRouter.get('/pm2/logs', (ctx) => {
|
|
const id = ctx.req.query('id') ?? '';
|
|
if (!/^\d+$/.test(id)) return ctx.text('numeric pm2 id required', 400);
|
|
const lines = Math.min(1000, Math.max(0, Number(ctx.req.query('lines')) || 100));
|
|
|
|
const enc = new TextEncoder();
|
|
const stream = new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
let proc: ReturnType<typeof Bun.spawn> | null = null;
|
|
let closed = false;
|
|
const send = (line: string) => {
|
|
try {
|
|
controller.enqueue(enc.encode(`data: ${line}\n\n`));
|
|
} catch {
|
|
/* closed */
|
|
}
|
|
};
|
|
const cleanup = () => {
|
|
if (closed) return;
|
|
closed = true;
|
|
clearInterval(hb);
|
|
try {
|
|
proc?.kill();
|
|
} catch {
|
|
/* already gone */
|
|
}
|
|
try {
|
|
controller.close();
|
|
} catch {
|
|
/* already closed */
|
|
}
|
|
};
|
|
|
|
try {
|
|
proc = Bun.spawn(['pm2', 'logs', id, '--raw', '--lines', String(lines)], { stdout: 'pipe', stderr: 'pipe' });
|
|
} catch (err) {
|
|
send(`[error spawning pm2 logs: ${String(err)}]`);
|
|
controller.close();
|
|
return;
|
|
}
|
|
|
|
void (async () => {
|
|
const reader = (proc!.stdout as ReadableStream<Uint8Array>).getReader();
|
|
const dec = new TextDecoder();
|
|
let buf = '';
|
|
try {
|
|
while (true) {
|
|
const { done, value } = await reader.read();
|
|
if (done) break;
|
|
buf += dec.decode(value, { stream: true });
|
|
const parts = buf.split('\n');
|
|
buf = parts.pop() ?? '';
|
|
for (const l of parts) send(l);
|
|
}
|
|
} catch {
|
|
/* stream ended */
|
|
} finally {
|
|
cleanup();
|
|
}
|
|
})();
|
|
|
|
const hb = setInterval(() => {
|
|
try {
|
|
controller.enqueue(enc.encode(': hb\n\n'));
|
|
} catch {
|
|
/* closed */
|
|
}
|
|
}, 15_000);
|
|
ctx.req.raw.signal.addEventListener('abort', cleanup);
|
|
setTimeout(cleanup, 60 * 60 * 1000);
|
|
},
|
|
});
|
|
|
|
return new Response(stream, {
|
|
headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', Connection: 'keep-alive' },
|
|
});
|
|
});
|
|
|
|
// GET /docker/logs?id=<container>&lines=<n> — SSE stream of a container's live logs (docker logs -f,
|
|
// combined stdout+stderr). id validated to a docker id/name charset + passed as a spawn arg (no shell).
|
|
systemMonitorRouter.get('/docker/logs', (ctx) => {
|
|
const id = ctx.req.query('id') ?? '';
|
|
if (!/^[a-zA-Z0-9][\w.-]{0,127}$/.test(id)) return ctx.text('valid container id/name required', 400);
|
|
const lines = Math.min(1000, Math.max(0, Number(ctx.req.query('lines')) || 100));
|
|
|
|
const enc = new TextEncoder();
|
|
const stream = new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
let proc: ReturnType<typeof Bun.spawn> | null = null;
|
|
let closed = false;
|
|
const send = (line: string) => {
|
|
try {
|
|
controller.enqueue(enc.encode(`data: ${line}\n\n`));
|
|
} catch {
|
|
/* closed */
|
|
}
|
|
};
|
|
const cleanup = () => {
|
|
if (closed) return;
|
|
closed = true;
|
|
clearInterval(hb);
|
|
try {
|
|
proc?.kill();
|
|
} catch {
|
|
/* gone */
|
|
}
|
|
try {
|
|
controller.close();
|
|
} catch {
|
|
/* closed */
|
|
}
|
|
};
|
|
|
|
try {
|
|
proc = Bun.spawn(['docker', 'logs', '-f', '--tail', String(lines), id], { stdout: 'pipe', stderr: 'pipe' });
|
|
} catch (err) {
|
|
send(`[error spawning docker logs: ${String(err)}]`);
|
|
controller.close();
|
|
return;
|
|
}
|
|
|
|
const pump = async (rs: ReadableStream<Uint8Array> | null) => {
|
|
if (!rs) return;
|
|
const reader = rs.getReader();
|
|
const dec = new TextDecoder();
|
|
let buf = '';
|
|
try {
|
|
while (true) {
|
|
const { done, value } = await reader.read();
|
|
if (done) break;
|
|
buf += dec.decode(value, { stream: true });
|
|
const parts = buf.split('\n');
|
|
buf = parts.pop() ?? '';
|
|
for (const l of parts) send(l);
|
|
}
|
|
} catch {
|
|
/* ended */
|
|
}
|
|
};
|
|
|
|
// Containers log to both stdout and stderr; stream both.
|
|
void Promise.all([
|
|
pump(proc.stdout as ReadableStream<Uint8Array>),
|
|
pump(proc.stderr as ReadableStream<Uint8Array>),
|
|
]).finally(cleanup);
|
|
|
|
const hb = setInterval(() => {
|
|
try {
|
|
controller.enqueue(enc.encode(': hb\n\n'));
|
|
} catch {
|
|
/* closed */
|
|
}
|
|
}, 15_000);
|
|
ctx.req.raw.signal.addEventListener('abort', cleanup);
|
|
setTimeout(cleanup, 60 * 60 * 1000);
|
|
},
|
|
});
|
|
|
|
return new Response(stream, {
|
|
headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', Connection: 'keep-alive' },
|
|
});
|
|
});
|