add three resource-pressure sections, each asked rather than assumed
Swap covers memory pressure. These are its neighbours: 8. Emergency disk ballast the same valve, for disk 9. earlyoom what happens when swap runs out too 10. inotify watch limit the silent one All three follow the rule this script now works to: the role sets which way the recommendation points, never whether the question is asked. A dev machine is still offered the ballast, with the recommendation pointing the other way; a server is still offered the inotify raise, because anything running `bun --watch` or serving a file browser is a watcher too. The ballast is section 22 of the original, moved up beside swap where it belongs and moved out of the user's home. The original wrote the checker into $USER_HOME/.local/bin and ran it from a root cron — a root cron executing a script in a directory its owner can write is a privilege escalation waiting to be noticed. Moot on a box where that user already has passwordless sudo, but wrong. Both the checker and the file are in root-owned system paths now. Two bugs found by running the generated checker rather than reading it: It df'd the ballast's own directory, which does not exist before the ballast is created — and with `set -euo pipefail` that meant cron mailing an error every ten minutes. It now walks up to a directory that exists, and the installer creates the directory itself rather than depending on the create step. The inotify text claimed a default of 8192. This host is at 29461: Ubuntu raised it, and stating a number the reader can see is wrong on their own screen undermines the rest of the explanation. It now describes the failure instead and prints the machine's actual value. earlyoom is a distro package and a systemd unit, so it is checked with `systemctl is-active` and reports honestly when it installs but fails to start. Verified: checker --status and its no-op path both exit 0 with no directory present, and the helpers report correctly against this host. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -173,6 +173,168 @@ swappiness_set() {
|
||||
sysctl -q -w "vm.swappiness=$1"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Emergency disk ballast
|
||||
# -----------------------------------------------------------------------------
|
||||
#
|
||||
# The same idea as swap, one layer down. Swap is the valve for memory pressure;
|
||||
# this is the valve for disk pressure.
|
||||
#
|
||||
# A junk file holding no data, sized at 10% of free disk. Its only job is to be
|
||||
# deleted when the filesystem is about to fill, buying enough headroom to log in
|
||||
# and clean up properly instead of meeting a wedged box — Docker, journald and
|
||||
# postgres all misbehave badly at 100% full, and some of them do not recover on
|
||||
# their own.
|
||||
#
|
||||
# A one-shot valve: once spent, it has to be recreated.
|
||||
#
|
||||
# ── Moved out of the user's home ──
|
||||
#
|
||||
# The original put the checker in $USER_HOME/.local/bin and ran it from a root
|
||||
# cron. A root cron executing a script inside a directory its owner can write is
|
||||
# a privilege escalation waiting to be noticed — moot on a box where that user
|
||||
# already has passwordless sudo, but wrong, and not something to carry forward.
|
||||
# Both the script and the file now live in root-owned system paths.
|
||||
|
||||
BALLAST_FILE=/var/lib/machine-setup/ballast.bin
|
||||
BALLAST_CHECKER=/usr/local/sbin/emergency-disk-check
|
||||
BALLAST_CRON=/etc/cron.d/emergency-disk-check
|
||||
BALLAST_THRESHOLD=10
|
||||
|
||||
ballast_exists() { [[ -f "$BALLAST_FILE" ]]; }
|
||||
|
||||
ballast_size_human() { du -h "$BALLAST_FILE" 2>/dev/null | cut -f1; }
|
||||
|
||||
# 10% of what is free right now, in MiB.
|
||||
ballast_size_mb() {
|
||||
local free_kb
|
||||
free_kb="$(df -Pk /var/lib | awk 'NR == 2 { print $4 }')"
|
||||
echo $((free_kb / 1024 / 10))
|
||||
}
|
||||
|
||||
ballast_create() {
|
||||
local mb="$1"
|
||||
mkdir -p "$(dirname "$BALLAST_FILE")"
|
||||
|
||||
# fallocate reserves real blocks. A sparse file made with truncate would
|
||||
# reserve nothing and free nothing when deleted, which is the entire point.
|
||||
if ! fallocate -l "${mb}M" "$BALLAST_FILE" 2>/dev/null; then
|
||||
info " fallocate is not usable here — writing with dd, which is slower"
|
||||
dd if=/dev/zero of="$BALLAST_FILE" bs=1M count="$mb" status=none
|
||||
fi
|
||||
chmod 600 "$BALLAST_FILE"
|
||||
}
|
||||
|
||||
ballast_install_checker() {
|
||||
mkdir -p "$(dirname "$BALLAST_FILE")"
|
||||
cat >"$BALLAST_CHECKER" <<CHECKER
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Emergency disk ballast checker. Installed by machine-setup.
|
||||
#
|
||||
# Deletes the pre-allocated ballast file when free space falls below the
|
||||
# threshold, buying headroom to log in and clean up. Run with --status to see
|
||||
# where things stand without changing anything.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
BALLAST="${BALLAST_FILE}"
|
||||
THRESHOLD=${BALLAST_THRESHOLD}
|
||||
TAG="emergency-disk"
|
||||
|
||||
# Walk up to a directory that exists. The ballast's own directory is gone if
|
||||
# somebody cleaned up after the valve was spent, and df failing under
|
||||
# \`set -e\` would make cron mail an error every ten minutes.
|
||||
MOUNT_DIR="\$(dirname "\$BALLAST")"
|
||||
while [[ ! -d "\$MOUNT_DIR" && "\$MOUNT_DIR" != "/" ]]; do MOUNT_DIR="\$(dirname "\$MOUNT_DIR")"; done
|
||||
USE_PCT="\$(df -P "\$MOUNT_DIR" | awk 'NR == 2 { gsub(/%/, "", \$5); print \$5 }')"
|
||||
FREE_PCT=\$((100 - USE_PCT))
|
||||
|
||||
if [[ "\${1:-}" == "--status" ]]; then
|
||||
echo "Mount: \$(df -P "\$MOUNT_DIR" | awk 'NR == 2 { print \$6 }')"
|
||||
echo "Free: \${FREE_PCT}% (threshold: \${THRESHOLD}%)"
|
||||
if [[ -f "\$BALLAST" ]]; then
|
||||
echo "Ballast: present, \$(du -h "\$BALLAST" | cut -f1) — \$BALLAST"
|
||||
else
|
||||
echo "Ballast: ABSENT (already spent) — \$BALLAST"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
((FREE_PCT < THRESHOLD)) || exit 0
|
||||
|
||||
if [[ -f "\$BALLAST" ]]; then
|
||||
FREED="\$(du -h "\$BALLAST" | cut -f1)"
|
||||
rm -f "\$BALLAST"
|
||||
logger -t "\$TAG" -p user.crit \\
|
||||
"Free space \${FREE_PCT}% below \${THRESHOLD}% — deleted ballast, reclaimed \${FREED}. CLEAN UP NOW: this valve is spent."
|
||||
else
|
||||
logger -t "\$TAG" -p user.crit \\
|
||||
"Free space \${FREE_PCT}% below \${THRESHOLD}% — ballast already spent, no headroom left to reclaim."
|
||||
fi
|
||||
CHECKER
|
||||
|
||||
chown root:root "$BALLAST_CHECKER"
|
||||
chmod 755 "$BALLAST_CHECKER"
|
||||
|
||||
cat >"$BALLAST_CRON" <<CRON
|
||||
# Emergency disk ballast — deletes the ballast file if free space drops below ${BALLAST_THRESHOLD}%.
|
||||
# Installed by machine-setup. Check status: ${BALLAST_CHECKER} --status
|
||||
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
*/10 * * * * root ${BALLAST_CHECKER}
|
||||
CRON
|
||||
chmod 644 "$BALLAST_CRON"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# earlyoom
|
||||
# -----------------------------------------------------------------------------
|
||||
#
|
||||
# What happens when swap runs out too.
|
||||
#
|
||||
# The kernel's own OOM killer waits until allocation genuinely fails, and by then
|
||||
# the machine has usually spent minutes thrashing — unresponsive, ssh refusing to
|
||||
# connect, nothing to do but reset it. earlyoom watches free memory and kills the
|
||||
# largest consumer while there is still enough left to stay reachable.
|
||||
|
||||
earlyoom_is_active() { systemctl is-active --quiet earlyoom 2>/dev/null; }
|
||||
|
||||
earlyoom_install() {
|
||||
pkg_is_installed earlyoom || pkg_install_now earlyoom
|
||||
systemctl enable --now earlyoom >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Resource limits
|
||||
# -----------------------------------------------------------------------------
|
||||
#
|
||||
# inotify watches: how many files one user can have the kernel watching. The
|
||||
# stock limit is small enough that one file watcher walking
|
||||
# node_modules. Every watcher on the machine draws from the same pool.
|
||||
#
|
||||
# The failure is silent, which is what makes it worth setting in advance: nothing
|
||||
# errors, the watcher simply stops noticing changes. Hot reload goes quiet, a
|
||||
# build stops rebuilding, and the reason is never on screen.
|
||||
#
|
||||
# Mostly a development concern, but not exclusively — anything running `bun
|
||||
# --watch` or serving a file browser is a watcher too.
|
||||
|
||||
INOTIFY_WATCHES=524288
|
||||
INOTIFY_INSTANCES=1024
|
||||
|
||||
inotify_current_watches() { sysctl -n fs.inotify.max_user_watches 2>/dev/null || echo 0; }
|
||||
|
||||
inotify_raise() {
|
||||
cat >/etc/sysctl.d/99-machine-setup-inotify.conf <<EOF
|
||||
# Raised by machine-setup: the 8192 default is exhausted by file watchers, and
|
||||
# the failure is silent — the watcher stops noticing changes without an error.
|
||||
fs.inotify.max_user_watches=${INOTIFY_WATCHES}
|
||||
fs.inotify.max_user_instances=${INOTIFY_INSTANCES}
|
||||
EOF
|
||||
sysctl -q -w "fs.inotify.max_user_watches=${INOTIFY_WATCHES}"
|
||||
sysctl -q -w "fs.inotify.max_user_instances=${INOTIFY_INSTANCES}"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Timezone
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user