pastilhasandClaude Opus 5
6ffd3534bd
Servers only now. On a machine you sit at, a filling disk announces itself — the editor refuses to save, the browser complains — and you are there to deal with it. The reserve is for the box nobody is watching, where the first sign is a service that stopped working hours ago. Skipped rather than asked, but said out loud with the reason and recorded in the summary. A section that silently produces no output is indistinguishable from one that failed. The cron this section installs was already there and is unchanged: /etc/cron.d runs the checker as root every ten minutes, and it deletes the ballast when free space falls under the threshold. What changed is where its message goes. Both alerts now run through one notify() inside the generated checker rather than calling logger directly, so there is a single place to add a second channel. Today it is still syslog only — the message lands in the journal and nowhere else, so nobody learns about it until they go looking, which is precisely the wrong moment. Push, mail or Officer's own notify sidecar hook in there. It also echoes to stderr now, so running the checker by hand shows the message instead of appearing to do nothing. Verified: dev reports not-applicable and asks nothing, vps still asks and records a refusal, and the regenerated checker parses and reports status. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
403 lines
16 KiB
Bash
403 lines
16 KiB
Bash
#!/bin/bash
|
|
# =============================================================================
|
|
# machine-setup — system configuration
|
|
# =============================================================================
|
|
#
|
|
# Definitions only, like the other lib/ files. Locale, and the system-level
|
|
# settings that follow it.
|
|
|
|
[[ -n "${MACHINE_SETUP_SYSTEM_LOADED:-}" ]] && return 0
|
|
MACHINE_SETUP_SYSTEM_LOADED=1
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Locale
|
|
# -----------------------------------------------------------------------------
|
|
#
|
|
# Two separate facts, and the original only handled one of them:
|
|
#
|
|
# what a new login shell is told to use — LANG in /etc/default/locale
|
|
# whether that locale actually exists — whether it has been generated
|
|
#
|
|
# Setting LANG to a locale that was never generated is the state that produces
|
|
# "setlocale: LC_ALL: cannot change locale" on every ssh login and every perl
|
|
# invocation. Both are checked, so the step can say which one is missing.
|
|
|
|
# What a new login shell will be handed, or empty if nothing is configured.
|
|
locale_current() {
|
|
if [[ -r /etc/default/locale ]]; then
|
|
awk -F= '/^LANG=/ { gsub(/"/, "", $2); print $2 }' /etc/default/locale
|
|
elif [[ -r /etc/locale.conf ]]; then
|
|
awk -F= '/^LANG=/ { gsub(/"/, "", $2); print $2 }' /etc/locale.conf
|
|
fi
|
|
}
|
|
|
|
# Has this locale actually been built?
|
|
#
|
|
# `locale -a` prints en_US.utf8 where the configuration spells it en_US.UTF-8,
|
|
# so both sides are folded to lower case with the dashes removed before
|
|
# comparing. A literal match here would report a perfectly good locale missing.
|
|
locale_is_generated() {
|
|
local want="${1,,}"
|
|
want="${want//-/}"
|
|
locale -a 2>/dev/null | tr '[:upper:]' '[:lower:]' | tr -d '-' | grep -qx "$want"
|
|
}
|
|
|
|
locale_set() {
|
|
local want="$1"
|
|
local escaped="${want//./\\.}"
|
|
|
|
case "$PM" in
|
|
apt)
|
|
# locale-gen comes from the `locales` package, which minimal images and
|
|
# most cloud base images do not ship. Without this the step fails with
|
|
# "locale-gen: command not found" halfway through.
|
|
if ! pkg_is_installed locales; then
|
|
info " installing locales, which provides locale-gen"
|
|
pkg_install_now locales
|
|
fi
|
|
|
|
# Uncomment it if it is there commented out, add it if it is absent.
|
|
# Editing the file rather than passing the name to locale-gen is what makes
|
|
# it survive: a locale generated by argument alone is lost the next time
|
|
# anything regenerates from /etc/locale.gen.
|
|
if grep -qE "^#[[:space:]]*${escaped}[[:space:]]" /etc/locale.gen 2>/dev/null; then
|
|
sed -i "s/^#[[:space:]]*\(${escaped}[[:space:]]\)/\1/" /etc/locale.gen
|
|
elif ! grep -qE "^${escaped}[[:space:]]" /etc/locale.gen 2>/dev/null; then
|
|
# The charset is the part after the dot: en_US.UTF-8 -> UTF-8
|
|
echo "${want} ${want##*.}" >>/etc/locale.gen
|
|
fi
|
|
|
|
locale-gen
|
|
update-locale LANG="$want"
|
|
;;
|
|
pacman)
|
|
if grep -qE "^#[[:space:]]*${escaped}[[:space:]]" /etc/locale.gen 2>/dev/null; then
|
|
sed -i "s/^#[[:space:]]*\(${escaped}[[:space:]]\)/\1/" /etc/locale.gen
|
|
fi
|
|
locale-gen
|
|
echo "LANG=${want}" >/etc/locale.conf
|
|
;;
|
|
dnf)
|
|
# No locale.gen here — the locales come prebuilt in langpack packages.
|
|
pkg_install_now "glibc-langpack-${want%%_*}"
|
|
localectl set-locale "LANG=${want}"
|
|
;;
|
|
brew)
|
|
warn "macOS has no system locale to set — it is per-user, from the terminal's settings"
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Swap
|
|
# -----------------------------------------------------------------------------
|
|
|
|
SWAPFILE=/swapfile
|
|
|
|
# Rounded to nearest, not floored: a 4 GiB swapfile is 4194300 kB, which floors
|
|
# to 3 and reads as though a gigabyte went missing. Same for RAM, where 3.7 GiB
|
|
# reporting as "3G" makes the sizing tiers look wrong.
|
|
kb_to_gb_rounded() { echo $((($1 + 524288) / 1048576)); }
|
|
|
|
# Total active swap in GiB, 0 if there is none.
|
|
#
|
|
# From /proc/meminfo rather than by grepping swapon's output for a slash, which
|
|
# is what the original did to spot a swap FILE — that test reports no swap at all
|
|
# on a machine using zram or a swap partition, and the step would then add a
|
|
# swapfile beside perfectly good swap.
|
|
swap_active_gb() { kb_to_gb_rounded "$(awk '/^SwapTotal:/ { print $2 }' /proc/meminfo)"; }
|
|
|
|
ram_gb() { kb_to_gb_rounded "$(awk '/^MemTotal:/ { print $2 }' /proc/meminfo)"; }
|
|
|
|
# Free space on the filesystem that would hold the swapfile, in GiB. Floored
|
|
# rather than rounded, deliberately: this one decides how much to allocate, and
|
|
# rounding up invents space that is not there.
|
|
disk_free_gb() { echo $(($(df -Pk "$(dirname "$SWAPFILE")" | awk 'NR == 2 { print $4 }') / 1024 / 1024)); }
|
|
|
|
# How much swap this machine should have.
|
|
#
|
|
# The tiers are the original's. What is new is that the answer is capped by what
|
|
# is actually on the disk — the original would try to fallocate 8G on a VPS with
|
|
# 4G free, fail, and take the run down with it.
|
|
swap_recommended_gb() {
|
|
local ram size
|
|
ram="$(ram_gb)"
|
|
if ((ram <= 2)); then
|
|
size=2
|
|
elif ((ram <= 8)); then
|
|
size=4
|
|
else
|
|
size=8
|
|
fi
|
|
|
|
# Leave a few gigabytes behind. A swapfile that fills the disk is a worse
|
|
# problem than no swapfile.
|
|
local room=$(($(disk_free_gb) - 5))
|
|
((room < size)) && size="$room"
|
|
((size < 1)) && size=0
|
|
echo "$size"
|
|
}
|
|
|
|
# How eagerly the kernel swaps, by role.
|
|
#
|
|
# 10 on a server: swapping is the emergency valve, not a routine, and the cost of
|
|
# a page fault on a request path is latency somebody is waiting for. A desktop is
|
|
# the opposite case — swapping out an application nobody has touched in an hour
|
|
# is exactly what you want — so dev keeps the kernel default of 60.
|
|
swappiness_for_role() { if is_server; then echo 10; else echo 60; fi; }
|
|
|
|
swap_create() {
|
|
local gb="$1"
|
|
|
|
# fallocate is instant but produces a file some filesystems refuse to swap on
|
|
# (btrfs without the right attributes, zfs at all). dd is slow and always
|
|
# works, so it is the fallback rather than the default.
|
|
if ! fallocate -l "${gb}G" "$SWAPFILE" 2>/dev/null; then
|
|
info " fallocate is not usable here — writing the file with dd, which is slower"
|
|
dd if=/dev/zero of="$SWAPFILE" bs=1M count=$((gb * 1024)) status=none
|
|
fi
|
|
|
|
chmod 600 "$SWAPFILE"
|
|
mkswap "$SWAPFILE" >/dev/null
|
|
swapon "$SWAPFILE"
|
|
|
|
grep -qs "^${SWAPFILE}[[:space:]]" /etc/fstab || echo "${SWAPFILE} none swap sw 0 0" >>/etc/fstab
|
|
}
|
|
|
|
# Written as a drop-in rather than by rewriting /etc/sysctl.conf in place. The
|
|
# original sed'd that file, which means the setting is tangled up with whatever
|
|
# else lives there and is invisible to anyone looking for what this script did.
|
|
swappiness_set() {
|
|
echo "vm.swappiness=$1" >/etc/sysctl.d/99-machine-setup-swappiness.conf
|
|
sysctl -q -w "vm.swappiness=$1"
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Emergency disk ballast
|
|
# -----------------------------------------------------------------------------
|
|
#
|
|
# The same idea as swap, one layer down. Swap is the valve for memory pressure;
|
|
# this is the valve for disk pressure.
|
|
#
|
|
# A junk file holding no data, sized at 10% of free disk. Its only job is to be
|
|
# deleted when the filesystem is about to fill, buying enough headroom to log in
|
|
# and clean up properly instead of meeting a wedged box — Docker, journald and
|
|
# postgres all misbehave badly at 100% full, and some of them do not recover on
|
|
# their own.
|
|
#
|
|
# A one-shot valve: once spent, it has to be recreated.
|
|
#
|
|
# ── Moved out of the user's home ──
|
|
#
|
|
# The original put the checker in $USER_HOME/.local/bin and ran it from a root
|
|
# cron. A root cron executing a script inside a directory its owner can write is
|
|
# a privilege escalation waiting to be noticed — moot on a box where that user
|
|
# already has passwordless sudo, but wrong, and not something to carry forward.
|
|
# Both the script and the file now live in root-owned system paths.
|
|
|
|
# Where it goes is asked rather than decided. A ballast only protects the
|
|
# filesystem it is ON — the checker measures its own directory — so the choice is
|
|
# also a choice of which mount is being protected. Defaults to the user's home,
|
|
# which on most machines is the same filesystem as / and is the easiest place to
|
|
# find it again months later.
|
|
BALLAST_FILE=""
|
|
BALLAST_CHECKER=/usr/local/sbin/emergency-disk-check
|
|
BALLAST_CRON=/etc/cron.d/emergency-disk-check
|
|
BALLAST_THRESHOLD=10
|
|
BALLAST_NAME=emergency-disk-ballast.bin
|
|
|
|
ballast_exists() { [[ -n "$BALLAST_FILE" && -f "$BALLAST_FILE" ]]; }
|
|
|
|
ballast_size_human() { du -h "$BALLAST_FILE" 2>/dev/null | cut -f1; }
|
|
|
|
# The nearest directory that exists, walking up. A path being chosen for the
|
|
# ballast does not mean anything has created it yet, and df cannot measure a
|
|
# directory that is not there.
|
|
existing_ancestor() {
|
|
local dir="$1"
|
|
while [[ ! -d "$dir" && "$dir" != "/" ]]; do dir="$(dirname "$dir")"; done
|
|
echo "$dir"
|
|
}
|
|
|
|
# Free space in KiB on whichever filesystem would hold this path.
|
|
ballast_free_kb() { df -Pk "$(existing_ancestor "$1")" | awk 'NR == 2 { print $4 }'; }
|
|
|
|
ballast_create() {
|
|
local mb="$1"
|
|
mkdir -p "$(dirname "$BALLAST_FILE")"
|
|
|
|
# fallocate reserves real blocks. A sparse file made with truncate would
|
|
# reserve nothing and free nothing when deleted, which is the entire point.
|
|
if ! fallocate -l "${mb}M" "$BALLAST_FILE" 2>/dev/null; then
|
|
info " fallocate is not usable here — writing with dd, which is slower"
|
|
dd if=/dev/zero of="$BALLAST_FILE" bs=1M count="$mb" status=none
|
|
fi
|
|
chmod 600 "$BALLAST_FILE"
|
|
}
|
|
|
|
ballast_install_checker() {
|
|
mkdir -p "$(dirname "$BALLAST_FILE")"
|
|
cat >"$BALLAST_CHECKER" <<CHECKER
|
|
#!/usr/bin/env bash
|
|
#
|
|
# Emergency disk ballast checker. Installed by machine-setup.
|
|
#
|
|
# Deletes the pre-allocated ballast file when free space falls below the
|
|
# threshold, buying headroom to log in and clean up. Run with --status to see
|
|
# where things stand without changing anything.
|
|
|
|
set -euo pipefail
|
|
|
|
BALLAST="${BALLAST_FILE}"
|
|
THRESHOLD=${BALLAST_THRESHOLD}
|
|
TAG="emergency-disk"
|
|
|
|
# Walk up to a directory that exists. The ballast's own directory is gone if
|
|
# somebody cleaned up after the valve was spent, and df failing under
|
|
# \`set -e\` would make cron mail an error every ten minutes.
|
|
MOUNT_DIR="\$(dirname "\$BALLAST")"
|
|
while [[ ! -d "\$MOUNT_DIR" && "\$MOUNT_DIR" != "/" ]]; do MOUNT_DIR="\$(dirname "\$MOUNT_DIR")"; done
|
|
USE_PCT="\$(df -P "\$MOUNT_DIR" | awk 'NR == 2 { gsub(/%/, "", \$5); print \$5 }')"
|
|
FREE_PCT=\$((100 - USE_PCT))
|
|
|
|
if [[ "\${1:-}" == "--status" ]]; then
|
|
echo "Mount: \$(df -P "\$MOUNT_DIR" | awk 'NR == 2 { print \$6 }')"
|
|
echo "Free: \${FREE_PCT}% (threshold: \${THRESHOLD}%)"
|
|
if [[ -f "\$BALLAST" ]]; then
|
|
echo "Ballast: present, \$(du -h "\$BALLAST" | cut -f1) — \$BALLAST"
|
|
else
|
|
echo "Ballast: ABSENT (already spent) — \$BALLAST"
|
|
fi
|
|
exit 0
|
|
fi
|
|
|
|
# Everything urgent goes through here, so there is one place to add a second
|
|
# channel later. Today it is syslog only, which means the message is in the
|
|
# journal and nowhere else — nobody finds out until they go looking, which is
|
|
# exactly the wrong moment. Push, mail or Officer's own notify sidecar hook in
|
|
# here.
|
|
notify() {
|
|
logger -t "\$TAG" -p user.crit "\$1"
|
|
# A copy on stderr as well, so a human running this by hand sees it.
|
|
echo "\$1" >&2
|
|
}
|
|
|
|
((FREE_PCT < THRESHOLD)) || exit 0
|
|
|
|
if [[ -f "\$BALLAST" ]]; then
|
|
FREED="\$(du -h "\$BALLAST" | cut -f1)"
|
|
rm -f "\$BALLAST"
|
|
notify "Free space \${FREE_PCT}% below \${THRESHOLD}% — deleted ballast, reclaimed \${FREED}. CLEAN UP NOW: this valve is spent."
|
|
else
|
|
notify "Free space \${FREE_PCT}% below \${THRESHOLD}% — ballast already spent, no headroom left to reclaim."
|
|
fi
|
|
CHECKER
|
|
|
|
chown root:root "$BALLAST_CHECKER"
|
|
chmod 755 "$BALLAST_CHECKER"
|
|
|
|
cat >"$BALLAST_CRON" <<CRON
|
|
# Emergency disk ballast — deletes the ballast file if free space drops below ${BALLAST_THRESHOLD}%.
|
|
# Installed by machine-setup. Check status: ${BALLAST_CHECKER} --status
|
|
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
|
*/10 * * * * root ${BALLAST_CHECKER}
|
|
CRON
|
|
chmod 644 "$BALLAST_CRON"
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# earlyoom
|
|
# -----------------------------------------------------------------------------
|
|
#
|
|
# What happens when swap runs out too.
|
|
#
|
|
# The kernel's own OOM killer waits until allocation genuinely fails, and by then
|
|
# the machine has usually spent minutes thrashing — unresponsive, ssh refusing to
|
|
# connect, nothing to do but reset it. earlyoom watches free memory and kills the
|
|
# largest consumer while there is still enough left to stay reachable.
|
|
|
|
earlyoom_is_active() { systemctl is-active --quiet earlyoom 2>/dev/null; }
|
|
|
|
earlyoom_install() {
|
|
pkg_is_installed earlyoom || pkg_install_now earlyoom
|
|
systemctl enable --now earlyoom >/dev/null 2>&1
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Resource limits
|
|
# -----------------------------------------------------------------------------
|
|
#
|
|
# inotify watches: how many files one user can have the kernel watching. The
|
|
# stock limit is small enough that one file watcher walking
|
|
# node_modules. Every watcher on the machine draws from the same pool.
|
|
#
|
|
# The failure is silent, which is what makes it worth setting in advance: nothing
|
|
# errors, the watcher simply stops noticing changes. Hot reload goes quiet, a
|
|
# build stops rebuilding, and the reason is never on screen.
|
|
#
|
|
# Mostly a development concern, but not exclusively — anything running `bun
|
|
# --watch` or serving a file browser is a watcher too.
|
|
|
|
INOTIFY_WATCHES=524288
|
|
INOTIFY_INSTANCES=1024
|
|
|
|
inotify_current_watches() { sysctl -n fs.inotify.max_user_watches 2>/dev/null || echo 0; }
|
|
|
|
inotify_raise() {
|
|
cat >/etc/sysctl.d/99-machine-setup-inotify.conf <<EOF
|
|
# Raised by machine-setup: the 8192 default is exhausted by file watchers, and
|
|
# the failure is silent — the watcher stops noticing changes without an error.
|
|
fs.inotify.max_user_watches=${INOTIFY_WATCHES}
|
|
fs.inotify.max_user_instances=${INOTIFY_INSTANCES}
|
|
EOF
|
|
sysctl -q -w "fs.inotify.max_user_watches=${INOTIFY_WATCHES}"
|
|
sysctl -q -w "fs.inotify.max_user_instances=${INOTIFY_INSTANCES}"
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Timezone
|
|
# -----------------------------------------------------------------------------
|
|
|
|
# The shortlist offered at the prompt. Any zone name can be typed instead, so
|
|
# this is a convenience rather than a limit.
|
|
TZ_OPTIONS=(UTC Europe/Lisbon Europe/London Europe/Berlin Europe/Stockholm US/Eastern US/Pacific Asia/Tokyo)
|
|
|
|
# What the machine is set to now.
|
|
#
|
|
# Three sources because they disagree about availability rather than about the
|
|
# answer: timedatectl is absent without systemd (containers, WSL), /etc/timezone
|
|
# is Debian-specific, and the /etc/localtime symlink is the one thing that is
|
|
# always true when any of them are.
|
|
timezone_current() {
|
|
if command -v timedatectl &>/dev/null && timedatectl show -p Timezone --value 2>/dev/null | grep -q .; then
|
|
timedatectl show -p Timezone --value 2>/dev/null
|
|
elif [[ -r /etc/timezone ]]; then
|
|
tr -d '[:space:]' </etc/timezone
|
|
elif [[ -L /etc/localtime ]]; then
|
|
readlink -f /etc/localtime | sed 's|.*/zoneinfo/||'
|
|
fi
|
|
}
|
|
|
|
# Checked against the zoneinfo database before it is used. `timedatectl
|
|
# set-timezone` on a name that does not exist fails, and under `set -e` that
|
|
# takes the whole run down over a typo.
|
|
timezone_is_valid() { [[ -f "/usr/share/zoneinfo/$1" ]]; }
|
|
|
|
timezone_set() {
|
|
local tz="$1"
|
|
case "$PM" in
|
|
brew) systemsetup -settimezone "$tz" >/dev/null ;;
|
|
*)
|
|
# timedatectl where there is a systemd to talk to; the files directly
|
|
# otherwise, which is the same thing it would have written.
|
|
if command -v timedatectl &>/dev/null && [[ "$IS_WSL" != true ]]; then
|
|
timedatectl set-timezone "$tz"
|
|
else
|
|
ln -sf "/usr/share/zoneinfo/${tz}" /etc/localtime
|
|
echo "$tz" >/etc/timezone
|
|
fi
|
|
;;
|
|
esac
|
|
}
|