platform/scripts/setup/machine-setup/lib/system.sh

#!/bin/bash
# =============================================================================
# machine-setup — system configuration
# =============================================================================
#
# Definitions only, like the other lib/ files. Locale, and the system-level
# settings that follow it.

[[ -n "${MACHINE_SETUP_SYSTEM_LOADED:-}" ]] && return 0
MACHINE_SETUP_SYSTEM_LOADED=1

# -----------------------------------------------------------------------------
# Locale
# -----------------------------------------------------------------------------
#
# Two separate facts, and the original only handled one of them:
#
#   what a new login shell is told to use   — LANG in /etc/default/locale
#   whether that locale actually exists     — whether it has been generated
#
# Setting LANG to a locale that was never generated is the state that produces
# "setlocale: LC_ALL: cannot change locale" on every ssh login and every perl
# invocation. Both are checked, so the step can say which one is missing.

# What a new login shell will be handed, or empty if nothing is configured.
locale_current() {
  if [[ -r /etc/default/locale ]]; then
    awk -F= '/^LANG=/ { gsub(/"/, "", $2); print $2 }' /etc/default/locale
  elif [[ -r /etc/locale.conf ]]; then
    awk -F= '/^LANG=/ { gsub(/"/, "", $2); print $2 }' /etc/locale.conf
  fi
}

# Has this locale actually been built?
#
# `locale -a` prints en_US.utf8 where the configuration spells it en_US.UTF-8,
# so both sides are folded to lower case with the dashes removed before
# comparing. A literal match here would report a perfectly good locale missing.
locale_is_generated() {
  local want="${1,,}"
  want="${want//-/}"
  locale -a 2>/dev/null | tr '[:upper:]' '[:lower:]' | tr -d '-' | grep -qx "$want"
}

locale_set() {
  local want="$1"
  local escaped="${want//./\\.}"

  case "$PM" in
    apt)
      # locale-gen comes from the `locales` package, which minimal images and
      # most cloud base images do not ship. Without this the step fails with
      # "locale-gen: command not found" halfway through.
      if ! pkg_is_installed locales; then
        info "     installing locales, which provides locale-gen"
        pkg_install_now locales
      fi

      # Uncomment it if it is there commented out, add it if it is absent.
      # Editing the file rather than passing the name to locale-gen is what makes
      # it survive: a locale generated by argument alone is lost the next time
      # anything regenerates from /etc/locale.gen.
      if grep -qE "^#[[:space:]]*${escaped}[[:space:]]" /etc/locale.gen 2>/dev/null; then
        sed -i "s/^#[[:space:]]*\(${escaped}[[:space:]]\)/\1/" /etc/locale.gen
      elif ! grep -qE "^${escaped}[[:space:]]" /etc/locale.gen 2>/dev/null; then
        # The charset is the part after the dot: en_US.UTF-8 -> UTF-8
        echo "${want} ${want##*.}" >>/etc/locale.gen
      fi

      locale-gen
      update-locale LANG="$want"
      ;;
    pacman)
      if grep -qE "^#[[:space:]]*${escaped}[[:space:]]" /etc/locale.gen 2>/dev/null; then
        sed -i "s/^#[[:space:]]*\(${escaped}[[:space:]]\)/\1/" /etc/locale.gen
      fi
      locale-gen
      echo "LANG=${want}" >/etc/locale.conf
      ;;
    dnf)
      # No locale.gen here — the locales come prebuilt in langpack packages.
      pkg_install_now "glibc-langpack-${want%%_*}"
      localectl set-locale "LANG=${want}"
      ;;
    brew)
      warn "macOS has no system locale to set — it is per-user, from the terminal's settings"
      return 1
      ;;
  esac
}

# -----------------------------------------------------------------------------
# Swap
# -----------------------------------------------------------------------------

SWAPFILE=/swapfile

# Rounded to nearest, not floored: a 4 GiB swapfile is 4194300 kB, which floors
# to 3 and reads as though a gigabyte went missing. Same for RAM, where 3.7 GiB
# reporting as "3G" makes the sizing tiers look wrong.
kb_to_gb_rounded() { echo $((($1 + 524288) / 1048576)); }

# Total active swap in GiB, 0 if there is none.
#
# From /proc/meminfo rather than by grepping swapon's output for a slash, which
# is what the original did to spot a swap FILE — that test reports no swap at all
# on a machine using zram or a swap partition, and the step would then add a
# swapfile beside perfectly good swap.
swap_active_gb() { kb_to_gb_rounded "$(awk '/^SwapTotal:/ { print $2 }' /proc/meminfo)"; }

ram_gb() { kb_to_gb_rounded "$(awk '/^MemTotal:/ { print $2 }' /proc/meminfo)"; }

# Free space on the filesystem that would hold the swapfile, in GiB. Floored
# rather than rounded, deliberately: this one decides how much to allocate, and
# rounding up invents space that is not there.
disk_free_gb() { echo $(($(df -Pk "$(dirname "$SWAPFILE")" | awk 'NR == 2 { print $4 }') / 1024 / 1024)); }

# How much swap this machine should have.
#
# The tiers are the original's. What is new is that the answer is capped by what
# is actually on the disk — the original would try to fallocate 8G on a VPS with
# 4G free, fail, and take the run down with it.
swap_recommended_gb() {
  local ram size
  ram="$(ram_gb)"
  if ((ram <= 2)); then
    size=2
  elif ((ram <= 8)); then
    size=4
  else
    size=8
  fi

  # Leave a few gigabytes behind. A swapfile that fills the disk is a worse
  # problem than no swapfile.
  local room=$(($(disk_free_gb) - 5))
  ((room < size)) && size="$room"
  ((size < 1)) && size=0
  echo "$size"
}

# How eagerly the kernel swaps, by role.
#
# 10 on a server: swapping is the emergency valve, not a routine, and the cost of
# a page fault on a request path is latency somebody is waiting for. A desktop is
# the opposite case — swapping out an application nobody has touched in an hour
# is exactly what you want — so dev keeps the kernel default of 60.
swappiness_for_role() { if is_server; then echo 10; else echo 60; fi; }

swap_create() {
  local gb="$1"

  # fallocate is instant but produces a file some filesystems refuse to swap on
  # (btrfs without the right attributes, zfs at all). dd is slow and always
  # works, so it is the fallback rather than the default.
  if ! fallocate -l "${gb}G" "$SWAPFILE" 2>/dev/null; then
    info "     fallocate is not usable here — writing the file with dd, which is slower"
    dd if=/dev/zero of="$SWAPFILE" bs=1M count=$((gb * 1024)) status=none
  fi

  chmod 600 "$SWAPFILE"
  mkswap "$SWAPFILE" >/dev/null
  swapon "$SWAPFILE"

  grep -qs "^${SWAPFILE}[[:space:]]" /etc/fstab || echo "${SWAPFILE} none swap sw 0 0" >>/etc/fstab
}

# Written as a drop-in rather than by rewriting /etc/sysctl.conf in place. The
# original sed'd that file, which means the setting is tangled up with whatever
# else lives there and is invisible to anyone looking for what this script did.
swappiness_set() {
  echo "vm.swappiness=$1" >/etc/sysctl.d/99-machine-setup-swappiness.conf
  sysctl -q -w "vm.swappiness=$1"
}

# -----------------------------------------------------------------------------
# Emergency disk ballast
# -----------------------------------------------------------------------------
#
# The same idea as swap, one layer down. Swap is the valve for memory pressure;
# this is the valve for disk pressure.
#
# A junk file holding no data, sized at 10% of free disk. Its only job is to be
# deleted when the filesystem is about to fill, buying enough headroom to log in
# and clean up properly instead of meeting a wedged box — Docker, journald and
# postgres all misbehave badly at 100% full, and some of them do not recover on
# their own.
#
# A one-shot valve: once spent, it has to be recreated.
#
# ── Moved out of the user's home ──
#
# The original put the checker in $USER_HOME/.local/bin and ran it from a root
# cron. A root cron executing a script inside a directory its owner can write is
# a privilege escalation waiting to be noticed — moot on a box where that user
# already has passwordless sudo, but wrong, and not something to carry forward.
# Both the script and the file now live in root-owned system paths.

# Where it goes is asked rather than decided. A ballast only protects the
# filesystem it is ON — the checker measures its own directory — so the choice is
# also a choice of which mount is being protected. Defaults to the user's home,
# which on most machines is the same filesystem as / and is the easiest place to
# find it again months later.
BALLAST_FILE=""
BALLAST_CHECKER=/usr/local/sbin/emergency-disk-check
BALLAST_CRON=/etc/cron.d/emergency-disk-check
BALLAST_THRESHOLD=10
BALLAST_NAME=emergency-disk-ballast.bin

ballast_exists() { [[ -n "$BALLAST_FILE" && -f "$BALLAST_FILE" ]]; }

ballast_size_human() { du -h "$BALLAST_FILE" 2>/dev/null | cut -f1; }

# The nearest directory that exists, walking up. A path being chosen for the
# ballast does not mean anything has created it yet, and df cannot measure a
# directory that is not there.
existing_ancestor() {
  local dir="$1"
  while [[ ! -d "$dir" && "$dir" != "/" ]]; do dir="$(dirname "$dir")"; done
  echo "$dir"
}

# Free space in KiB on whichever filesystem would hold this path.
ballast_free_kb() { df -Pk "$(existing_ancestor "$1")" | awk 'NR == 2 { print $4 }'; }

ballast_create() {
  local mb="$1"
  mkdir -p "$(dirname "$BALLAST_FILE")"

  # fallocate reserves real blocks. A sparse file made with truncate would
  # reserve nothing and free nothing when deleted, which is the entire point.
  if ! fallocate -l "${mb}M" "$BALLAST_FILE" 2>/dev/null; then
    info "     fallocate is not usable here — writing with dd, which is slower"
    dd if=/dev/zero of="$BALLAST_FILE" bs=1M count="$mb" status=none
  fi
  chmod 600 "$BALLAST_FILE"
}

ballast_install_checker() {
  mkdir -p "$(dirname "$BALLAST_FILE")"
  cat >"$BALLAST_CHECKER" <<CHECKER
#!/usr/bin/env bash
#
# Emergency disk ballast checker. Installed by machine-setup.
#
# Deletes the pre-allocated ballast file when free space falls below the
# threshold, buying headroom to log in and clean up. Run with --status to see
# where things stand without changing anything.

set -euo pipefail

BALLAST="${BALLAST_FILE}"
THRESHOLD=${BALLAST_THRESHOLD}
TAG="emergency-disk"

# Walk up to a directory that exists. The ballast's own directory is gone if
# somebody cleaned up after the valve was spent, and df failing under
# \`set -e\` would make cron mail an error every ten minutes.
MOUNT_DIR="\$(dirname "\$BALLAST")"
while [[ ! -d "\$MOUNT_DIR" && "\$MOUNT_DIR" != "/" ]]; do MOUNT_DIR="\$(dirname "\$MOUNT_DIR")"; done
USE_PCT="\$(df -P "\$MOUNT_DIR" | awk 'NR == 2 { gsub(/%/, "", \$5); print \$5 }')"
FREE_PCT=\$((100 - USE_PCT))

if [[ "\${1:-}" == "--status" ]]; then
  echo "Mount:     \$(df -P "\$MOUNT_DIR" | awk 'NR == 2 { print \$6 }')"
  echo "Free:      \${FREE_PCT}% (threshold: \${THRESHOLD}%)"
  if [[ -f "\$BALLAST" ]]; then
    echo "Ballast:   present, \$(du -h "\$BALLAST" | cut -f1) — \$BALLAST"
  else
    echo "Ballast:   ABSENT (already spent) — \$BALLAST"
  fi
  exit 0
fi

# Everything urgent goes through here, so there is one place to add a second
# channel later. Today it is syslog only, which means the message is in the
# journal and nowhere else — nobody finds out until they go looking, which is
# exactly the wrong moment. Push, mail or Officer's own notify sidecar hook in
# here.
notify() {
  logger -t "\$TAG" -p user.crit "\$1"
  # A copy on stderr as well, so a human running this by hand sees it.
  echo "\$1" >&2
}

((FREE_PCT < THRESHOLD)) || exit 0

if [[ -f "\$BALLAST" ]]; then
  FREED="\$(du -h "\$BALLAST" | cut -f1)"
  rm -f "\$BALLAST"
  notify "Free space \${FREE_PCT}% below \${THRESHOLD}% — deleted ballast, reclaimed \${FREED}. CLEAN UP NOW: this valve is spent."
else
  notify "Free space \${FREE_PCT}% below \${THRESHOLD}% — ballast already spent, no headroom left to reclaim."
fi
CHECKER

  chown root:root "$BALLAST_CHECKER"
  chmod 755 "$BALLAST_CHECKER"

  cat >"$BALLAST_CRON" <<CRON
# Emergency disk ballast — deletes the ballast file if free space drops below ${BALLAST_THRESHOLD}%.
# Installed by machine-setup. Check status: ${BALLAST_CHECKER} --status
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
*/10 * * * * root ${BALLAST_CHECKER}
CRON
  chmod 644 "$BALLAST_CRON"
}

# -----------------------------------------------------------------------------
# earlyoom
# -----------------------------------------------------------------------------
#
# What happens when swap runs out too.
#
# The kernel's own OOM killer waits until allocation genuinely fails, and by then
# the machine has usually spent minutes thrashing — unresponsive, ssh refusing to
# connect, nothing to do but reset it. earlyoom watches free memory and kills the
# largest consumer while there is still enough left to stay reachable.

earlyoom_is_active() { systemctl is-active --quiet earlyoom 2>/dev/null; }

earlyoom_install() {
  pkg_is_installed earlyoom || pkg_install_now earlyoom
  systemctl enable --now earlyoom >/dev/null 2>&1
}

# -----------------------------------------------------------------------------
# Resource limits
# -----------------------------------------------------------------------------
#
# inotify watches: how many files one user can have the kernel watching. The
# stock limit is small enough that one file watcher walking
# node_modules. Every watcher on the machine draws from the same pool.
#
# The failure is silent, which is what makes it worth setting in advance: nothing
# errors, the watcher simply stops noticing changes. Hot reload goes quiet, a
# build stops rebuilding, and the reason is never on screen.
#
# Mostly a development concern, but not exclusively — anything running `bun
# --watch` or serving a file browser is a watcher too.

INOTIFY_WATCHES=524288
INOTIFY_INSTANCES=1024

inotify_current_watches() { sysctl -n fs.inotify.max_user_watches 2>/dev/null || echo 0; }

inotify_raise() {
  cat >/etc/sysctl.d/99-machine-setup-inotify.conf <<EOF
# Raised by machine-setup: the 8192 default is exhausted by file watchers, and
# the failure is silent — the watcher stops noticing changes without an error.
fs.inotify.max_user_watches=${INOTIFY_WATCHES}
fs.inotify.max_user_instances=${INOTIFY_INSTANCES}
EOF
  sysctl -q -w "fs.inotify.max_user_watches=${INOTIFY_WATCHES}"
  sysctl -q -w "fs.inotify.max_user_instances=${INOTIFY_INSTANCES}"
}

# -----------------------------------------------------------------------------
# Sleep and suspend
# -----------------------------------------------------------------------------
#
# A server that suspends is a server that is off. The machine stops answering,
# and on a box with no keyboard attached there is nothing to wake it — which is
# the whole failure: it looks like a crash, and the only fix is physical.
#
# Two independent mechanisms, and both have to be dealt with:
#
#   the sleep targets    what suspend/hibernate hang off. Masking them means
#                        nothing can trigger a sleep, including a stray
#                        `systemctl suspend`
#   logind's handlers    what closing a lid, pressing the power button or going
#                        idle DO. These are what a desktop image sets, and they
#                        act before anything reaches a target
#
# Written as a drop-in rather than by editing logind.conf in place, so what this
# script set is one file that can be read or deleted on its own.

SLEEP_TARGETS=(sleep.target suspend.target hibernate.target hybrid-sleep.target)
LOGIND_DROPIN=/etc/systemd/logind.conf.d/99-machine-setup.conf

# The value actually in force for a logind setting, or empty for the default.
# Drop-ins override the main file and later ones override earlier, so the last
# match wins — reading only logind.conf would miss a setting made by a drop-in
# and report the machine as unconfigured when it is not.
logind_effective() {
  local key="$1"
  {
    [[ -r /etc/systemd/logind.conf ]] && grep -hE "^${key}=" /etc/systemd/logind.conf
    for f in /etc/systemd/logind.conf.d/*.conf; do
      [[ -r "$f" ]] && grep -hE "^${key}=" "$f"
    done
  } 2>/dev/null | tail -1 | cut -d= -f2-
}

sleep_targets_masked() {
  local t
  for t in "${SLEEP_TARGETS[@]}"; do
    [[ "$(systemctl is-enabled "$t" 2>/dev/null)" == "masked" ]] || return 1
  done
}

# What the machine should be set to. RuntimeDirectorySize is deliberately NOT
# here: the original set it to 10% alongside these, which is both unrelated to
# sleeping — it is the size of /run — and systemd's own default, so the line
# never did anything.
logind_wanted() {
  cat <<'EOF'
HandleLidSwitch=ignore
HandleLidSwitchExternalPower=ignore
HandleLidSwitchDocked=ignore
HandlePowerKey=ignore
IdleAction=none
EOF
}

# Is every wanted setting already in force?
logind_is_configured() {
  local line key value
  while IFS= read -r line; do
    key="${line%%=*}"
    value="${line#*=}"
    [[ "$(logind_effective "$key")" == "$value" ]] || return 1
  done < <(logind_wanted)
}

disable_sleep() {
  systemctl mask "${SLEEP_TARGETS[@]}" >/dev/null 2>&1

  mkdir -p "$(dirname "$LOGIND_DROPIN")"
  {
    echo "# Written by machine-setup: this machine is a server and must not sleep."
    echo "[Login]"
    logind_wanted
  } >"$LOGIND_DROPIN"

  # Only restart when something actually changed — a needless restart of logind
  # disturbs live sessions, and this step runs on every pass.
  systemctl restart systemd-logind
}

# -----------------------------------------------------------------------------
# Timezone
# -----------------------------------------------------------------------------

# The shortlist offered at the prompt. Any zone name can be typed instead, so
# this is a convenience rather than a limit.
TZ_OPTIONS=(UTC Europe/Lisbon Europe/London Europe/Berlin Europe/Stockholm US/Eastern US/Pacific Asia/Tokyo)

# What the machine is set to now.
#
# Three sources because they disagree about availability rather than about the
# answer: timedatectl is absent without systemd (containers, WSL), /etc/timezone
# is Debian-specific, and the /etc/localtime symlink is the one thing that is
# always true when any of them are.
timezone_current() {
  if command -v timedatectl &>/dev/null && timedatectl show -p Timezone --value 2>/dev/null | grep -q .; then
    timedatectl show -p Timezone --value 2>/dev/null
  elif [[ -r /etc/timezone ]]; then
    tr -d '[:space:]' </etc/timezone
  elif [[ -L /etc/localtime ]]; then
    readlink -f /etc/localtime | sed 's|.*/zoneinfo/||'
  fi
}

# Checked against the zoneinfo database before it is used. `timedatectl
# set-timezone` on a name that does not exist fails, and under `set -e` that
# takes the whole run down over a typo.
timezone_is_valid() { [[ -f "/usr/share/zoneinfo/$1" ]]; }

timezone_set() {
  local tz="$1"
  case "$PM" in
    brew) systemsetup -settimezone "$tz" >/dev/null ;;
    *)
      # timedatectl where there is a systemd to talk to; the files directly
      # otherwise, which is the same thing it would have written.
      if command -v timedatectl &>/dev/null && [[ "$IS_WSL" != true ]]; then
        timedatectl set-timezone "$tz"
      else
        ln -sf "/usr/share/zoneinfo/${tz}" /etc/localtime
        echo "$tz" >/etc/timezone
      fi
      ;;
  esac
}