Initial Commit
This commit is contained in:
@@ -0,0 +1,41 @@
|
||||
// =====================================================================
|
||||
// Who may unfreeze a base image.
|
||||
//
|
||||
// The check is subject.active - a live, foreground login session of the
|
||||
// operator. Deliberately NOT a group membership test: a group is
|
||||
// inherited by every process a member starts, including a background
|
||||
// service, and the whole point of this gate is that a human is present.
|
||||
//
|
||||
// `subject.local` was in this rule first and was removed on purpose.
|
||||
// This machine's GNOME desktop is normally reached over RDP rather than
|
||||
// at the console, and logind does not call such a session local - so the
|
||||
// strict form refused the operator at their own desktop while changing
|
||||
// nothing about what it excludes. The agent has no session at all;
|
||||
// vmd has no session at all; a systemd service has no session at all.
|
||||
// `active` already rules out every one of them.
|
||||
//
|
||||
// The agent is not in this picture at all. It runs as owuivm, which
|
||||
// cannot write a base image by file permission, and it has no verb that
|
||||
// asks anything else to. This rule is what a person passes through, not
|
||||
// a check the agent could fail.
|
||||
// =====================================================================
|
||||
polkit.addRule(function(action, subject) {
|
||||
if (action.id == "org.freedesktop.systemd1.manage-units") {
|
||||
var unit = action.lookup("unit") || "";
|
||||
if (unit.indexOf("owuivm-maint@") === 0 ||
|
||||
unit.indexOf("owuivm-detach@") === 0 ||
|
||||
unit.indexOf("owuivm-thaw@") === 0 ||
|
||||
unit.indexOf("owuivm-freeze@") === 0 ||
|
||||
// Putting a disc in the library, and renaming a machine.
|
||||
// Neither touches a base image, but both write somewhere the
|
||||
// operator's own account cannot, so they pass the same gate
|
||||
// as everything else here rather than inventing a second one.
|
||||
unit.indexOf("owuivm-insert@") === 0 ||
|
||||
unit.indexOf("owuivm-rename@") === 0) {
|
||||
if (subject.active && subject.user == "@OPERATOR@") {
|
||||
return polkit.Result.YES;
|
||||
}
|
||||
return polkit.Result.NO;
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
# =====================================================================
|
||||
# Second-layer egress filter for the VM sandbox.
|
||||
#
|
||||
# WHY THIS IS THE SECOND LAYER AND NOT THE FIRST
|
||||
#
|
||||
# The primary control is in owuivm.service: systemd's IPAddressDeny=
|
||||
# attaches a BPF filter to the unit's cgroup, which `nft flush ruleset`
|
||||
# cannot reach and which applies to every socket the process opens.
|
||||
# This table says the same thing a second time, independently.
|
||||
#
|
||||
# It is deliberately NOT trusted on its own. nft uid-matching is
|
||||
# fail-open: a `ufw reload`, an `nft -f` carrying `flush ruleset`, a
|
||||
# package upgrade, or this unit simply failing to start, and the rules
|
||||
# are gone with nothing to notice. netguard/firewall.sh exists in this
|
||||
# suite for exactly that class of failure - "a firewall that silently
|
||||
# failed to apply is worse than none, because everything downstream
|
||||
# carries on as though it were enforced".
|
||||
#
|
||||
# So the test that proves the design is not "these rules are loaded".
|
||||
# It is: delete this table, re-run the whole guest negative suite, and
|
||||
# watch it still pass. Then strip IPAddressDeny from a scratch unit and
|
||||
# run it again. Two proofs, neither trusting the other.
|
||||
#
|
||||
# WHAT IT MATCHES
|
||||
#
|
||||
# Every guest packet is re-originated by the qemu process as an ordinary
|
||||
# host socket (see the -netdev user decision in vmd), so the whole guest
|
||||
# is reachable as "traffic owned by uid owuivm". There is no tap, no
|
||||
# bridge and no forwarding path, which is why nothing here hooks forward.
|
||||
#
|
||||
# Note 10.0.0.0/8 does NOT block the guest's own 10.0.2.x addresses.
|
||||
# Those exist only inside libslirp and never become host packets. The
|
||||
# 127.0.0.0/8 line is what closes SLIRP's host aliases - 10.0.2.2 is the
|
||||
# host's loopback and 10.0.2.3 its DNS forwarder - which is why the
|
||||
# guest images carry a static public resolver instead of asking DHCP.
|
||||
# =====================================================================
|
||||
|
||||
table inet owuivm_guard {
|
||||
set rfc1918_and_local {
|
||||
type ipv4_addr
|
||||
flags interval
|
||||
elements = {
|
||||
0.0.0.0/8,
|
||||
10.0.0.0/8,
|
||||
100.64.0.0/10,
|
||||
127.0.0.0/8,
|
||||
169.254.0.0/16,
|
||||
172.16.0.0/12,
|
||||
192.0.0.0/24,
|
||||
192.168.0.0/16,
|
||||
224.0.0.0/4,
|
||||
240.0.0.0/4
|
||||
}
|
||||
}
|
||||
|
||||
chain out {
|
||||
# priority filter - 10, so this runs ahead of ufw's own output
|
||||
# chain. policy accept: this table only ever drops, exactly like
|
||||
# /etc/nftables.d/50-node_exporter_guard.nft does for input.
|
||||
type filter hook output priority filter - 10; policy accept;
|
||||
|
||||
meta skuid "owuivm" ip daddr @rfc1918_and_local \
|
||||
counter log prefix "owuivm-blocked: " level info drop
|
||||
|
||||
# The guest has no IPv6 at all: the netdev is ipv6=off and the
|
||||
# unit omits AF_INET6 from RestrictAddressFamilies. This is the
|
||||
# third statement of the same thing, and it costs nothing.
|
||||
meta skuid "owuivm" meta nfproto ipv6 counter drop
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
# Installed to /etc/apparmor.d/local/usr.bin.swtpm
|
||||
#
|
||||
# WHY THIS IS NEEDED
|
||||
#
|
||||
# Ubuntu ships an AppArmor profile for swtpm that permits libvirt's paths
|
||||
# and essentially nothing else: /run/libvirt/qemu/swtpm/*.sock,
|
||||
# /var/lib/libvirt/swtpm/**, and a couple of others. This suite does not
|
||||
# use libvirt, so a TPM socket under /run/owuivm was refused with
|
||||
# "Could not open UnixIO socket: Permission denied" - which reads like a
|
||||
# file-mode problem and is not one. Nothing appears in dmesg for it
|
||||
# either, because the profile denies quietly.
|
||||
#
|
||||
# Windows 11 will not install without a TPM, so this is the difference
|
||||
# between having a Windows machine and not.
|
||||
#
|
||||
# <local/...> is the profile's own documented extension point, so this
|
||||
# survives a swtpm package update in a way that editing the shipped
|
||||
# profile would not.
|
||||
#
|
||||
# Narrow on purpose: one directory, whose contents are per-session TPM
|
||||
# state that is thrown away with the session.
|
||||
owner /run/owuivm/** rwk,
|
||||
/run/owuivm/*/swtpm.sock rwk,
|
||||
Executable
+379
@@ -0,0 +1,379 @@
|
||||
#!/bin/bash
|
||||
# =====================================================================
|
||||
# Installs the host side of the VM sandbox. Idempotent: run it again
|
||||
# after editing anything in this directory.
|
||||
#
|
||||
# Everything here is versioned in the repo and copied into place, rather
|
||||
# than edited live, so that this directory is the source of truth and
|
||||
# `install.sh` is the only thing that has ever touched
|
||||
# /etc/systemd/system. The archived sub-agent stack was bitten twice by
|
||||
# the opposite arrangement, where a redeploy regenerated config that had
|
||||
# been hand-edited on disk.
|
||||
# =====================================================================
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$(readlink -f "$0")")"
|
||||
|
||||
[ "$(id -u)" -eq 0 ] || { echo "install.sh must run as root" >&2; exit 1; }
|
||||
|
||||
# --- who operates this machine ----------------------------------------
|
||||
# The operator gets the desktop shortcut and is the person polkit lets
|
||||
# unfreeze a base image. It used to be the literal string `operator` in
|
||||
# three files, which is fine on one host and wrong on every other.
|
||||
#
|
||||
# SUDO_USER is the right source: this script runs as root via sudo, so it
|
||||
# names the human who asked for it. logname is the fallback for a real
|
||||
# root login. configs/host.env overrides both, for the case where the
|
||||
# person installing is not the person who will use it.
|
||||
CONFIG_DIR="$(cd ../../configs 2>/dev/null && pwd || true)"
|
||||
[ -n "$CONFIG_DIR" ] && [ -f "$CONFIG_DIR/host.env" ] && . "$CONFIG_DIR/host.env"
|
||||
|
||||
OPERATOR=${OWUIVM_OPERATOR:-${SUDO_USER:-$(logname 2>/dev/null || true)}}
|
||||
if [ -z "$OPERATOR" ] || [ "$OPERATOR" = "root" ]; then
|
||||
echo "FAILED: cannot tell who the operator is." >&2
|
||||
echo " Run this with sudo from your own account, or set OWUIVM_OPERATOR" >&2
|
||||
echo " in configs/host.env. Whoever it is gets the desktop shortcut and" >&2
|
||||
echo " is the only account polkit will let unfreeze a base image." >&2
|
||||
exit 1
|
||||
fi
|
||||
getent passwd "$OPERATOR" >/dev/null || {
|
||||
echo "FAILED: there is no user '$OPERATOR' on this machine" >&2; exit 1; }
|
||||
OPERATOR_HOME=$(getent passwd "$OPERATOR" | cut -d: -f6)
|
||||
|
||||
POOL_ROOT=$(printf '%s' "${OWUIVM_POOL_ROOT:-/Pool}" | sed 's:/*$::')
|
||||
[ -n "$POOL_ROOT" ] || POOL_ROOT=/Pool
|
||||
|
||||
# --- how much of the machine one VM may take --------------------------
|
||||
# Derived from what is actually here rather than from the machine this
|
||||
# was written on, which had 28 threads and 58 GB. Ollama needs cores for
|
||||
# sampling while a session runs, so taking every core makes both slower.
|
||||
TOTAL_MB=$(awk '/MemTotal/ {print int($2/1024)}' /proc/meminfo)
|
||||
CPUS_TOTAL=$(nproc)
|
||||
VM_MEM_MB=${OWUIVM_VM_MEM_MB:-$(( TOTAL_MB/2 < 8192 ? TOTAL_MB/2 : 8192 ))}
|
||||
VM_CPUS=${OWUIVM_VM_CPUS:-$(( CPUS_TOTAL-4 < 8 ? (CPUS_TOTAL-4 < 2 ? 2 : CPUS_TOTAL-4) : 8 ))}
|
||||
# The cgroup ceilings sit above the guest's own figures, because qemu's
|
||||
# overhead is outside the guest's RAM and would otherwise push the unit
|
||||
# over its own limit and get it killed.
|
||||
VM_MEMORY_MAX=${OWUIVM_MEMORY_MAX:-$(( VM_MEM_MB/1024 + 3 ))G}
|
||||
VM_CPU_QUOTA=${OWUIVM_CPU_QUOTA:-$(( (VM_CPUS-1) * 100 ))%}
|
||||
MAINT_CPU_QUOTA=${OWUIVM_MAINT_CPU_QUOTA:-$(( VM_CPUS * 100 ))%}
|
||||
# Empty means no pinning, which is right on a machine that is not this
|
||||
# one. Pinning matters when something else is using specific cores.
|
||||
VM_ALLOWED_CPUS=${OWUIVM_ALLOWED_CPUS:-}
|
||||
|
||||
# --- what the daemon is allowed to reach ------------------------------
|
||||
# vmd runs with IPAddressDeny=any and an allowlist. Gitea is not on it by
|
||||
# default, because a fresh install has no Gitea: publishing is optional
|
||||
# and the daemon should not be given reach it does not need.
|
||||
#
|
||||
# BOTH the LAN address and the container's own network are needed when
|
||||
# you do add it - systemd's filter is a BPF program on the cgroup's
|
||||
# packets, and Docker's DNAT has already rewritten the destination by the
|
||||
# time it sees them. See docs/setup/gitea.md.
|
||||
if [ -n "${OWUIVM_GITEA_ALLOW:-}" ]; then
|
||||
GITEA_ALLOW=$(printf '%s\n' $OWUIVM_GITEA_ALLOW | sed 's/^/IPAddressAllow=/')
|
||||
else
|
||||
GITEA_ALLOW="# No Gitea configured. See docs/setup/gitea.md to add one."
|
||||
fi
|
||||
TOOLS_SUBNET=${OWUIVM_TOOLS_SUBNET:-172.22.0.0/24}
|
||||
|
||||
echo "==> operator: $OPERATOR pool: $POOL_ROOT"
|
||||
echo " vm: ${VM_MEM_MB}MB / ${VM_CPUS} cpus ceilings: $VM_MEMORY_MAX $VM_CPU_QUOTA ${VM_ALLOWED_CPUS:+cpus $VM_ALLOWED_CPUS}"
|
||||
|
||||
# Renders the @PLACEHOLDER@ tokens in a versioned file and installs the
|
||||
# result. Everything in vm/host that mentions a path, a size or an
|
||||
# address this host chose goes through here, so the repo stays free of
|
||||
# any one machine's numbers.
|
||||
render() {
|
||||
local src="$1" dst="$2" mode="${3:-0644}" tmp allow
|
||||
tmp=$(mktemp); allow=$(mktemp)
|
||||
printf '%s\n' "$GITEA_ALLOW" > "$allow"
|
||||
|
||||
# The multi-line one first, by file, because a sed replacement cannot
|
||||
# carry newlines portably.
|
||||
sed -e "/@GITEA_ALLOW@/{r $allow" -e "d;}" "$src" > "$tmp"
|
||||
|
||||
sed -i -e "s|@POOL_ROOT@|$POOL_ROOT|g" \
|
||||
-e "s|@OPERATOR@|$OPERATOR|g" \
|
||||
-e "s|@VM_MEMORY_MAX@|$VM_MEMORY_MAX|g" \
|
||||
-e "s|@VM_CPU_QUOTA@|$VM_CPU_QUOTA|g" \
|
||||
-e "s|@MAINT_CPU_QUOTA@|$MAINT_CPU_QUOTA|g" \
|
||||
-e "s|@TOOLS_SUBNET@|$TOOLS_SUBNET|g" "$tmp"
|
||||
|
||||
# An empty AllowedCPUs= RESETS the property rather than leaving it
|
||||
# alone, so with no pinning the line is removed rather than blanked.
|
||||
if [ -z "$VM_ALLOWED_CPUS" ]; then
|
||||
sed -i '/^AllowedCPUs=@VM_ALLOWED_CPUS@$/d' "$tmp"
|
||||
else
|
||||
sed -i "s|@VM_ALLOWED_CPUS@|$VM_ALLOWED_CPUS|g" "$tmp"
|
||||
fi
|
||||
|
||||
# A placeholder that survived means a value was added to a template and
|
||||
# not to this function. Failing here is cheap; finding out later,
|
||||
# because a unit refused to start with a syntax error, is not.
|
||||
if grep -q '@[A-Z_]\+@' "$tmp"; then
|
||||
echo "FAILED: $src has an unrendered placeholder:" >&2
|
||||
grep -o '@[A-Z_]\+@' "$tmp" | sort -u | sed 's/^/ /' >&2
|
||||
rm -f "$tmp" "$allow"; exit 1
|
||||
fi
|
||||
install -o root -g root -m "$mode" "$tmp" "$dst"
|
||||
rm -f "$tmp" "$allow"
|
||||
}
|
||||
|
||||
echo "==> preflight"
|
||||
# EVERY REQUIREMENT CHECKED HERE, BEFORE ANYTHING IS CREATED.
|
||||
#
|
||||
# This script makes users, datasets, units and firewall rules. Finding
|
||||
# out at step nine that swtpm is missing leaves a half-installed host and
|
||||
# a confusing error; the point of this block is that it either all works
|
||||
# or nothing has happened yet.
|
||||
missing=""
|
||||
for cmd in qemu-system-x86_64 qemu-img swtpm socat nft setfacl visudo zfs python3; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || missing="$missing $cmd"
|
||||
done
|
||||
# virtiofsd moved between packages and distributions; both locations count.
|
||||
[ -x /usr/libexec/virtiofsd ] || [ -x /usr/lib/qemu/virtiofsd ] \
|
||||
|| command -v virtiofsd >/dev/null 2>&1 || missing="$missing virtiofsd"
|
||||
if [ -n "$missing" ]; then
|
||||
echo "FAILED: missing:$missing" >&2
|
||||
echo " On Ubuntu: sudo apt install qemu-system-x86 qemu-utils virtiofsd ovmf \\" >&2
|
||||
echo " swtpm-tools socat nftables acl zfsutils-linux" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The firmware. Windows needs the Secure Boot pair specifically, and a
|
||||
# missing one of those fails much later, as a machine that will not
|
||||
# install, with an error that looks like hardware.
|
||||
for fw in /usr/share/OVMF/OVMF_CODE_4M.fd /usr/share/OVMF/OVMF_VARS_4M.fd; do
|
||||
[ -f "$fw" ] || { echo "FAILED: $fw is missing - install the 'ovmf' package" >&2; exit 1; }
|
||||
done
|
||||
for fw in /usr/share/OVMF/OVMF_CODE_4M.secboot.fd /usr/share/OVMF/OVMF_VARS_4M.ms.fd; do
|
||||
[ -f "$fw" ] || echo " note: $fw is missing; Windows guests will not install (Ubuntu is fine)"
|
||||
done
|
||||
|
||||
[ -e /dev/kvm ] || { echo "FAILED: /dev/kvm is missing. This host cannot run VMs -" >&2
|
||||
echo " check virtualisation is enabled in firmware, and that kvm modules load." >&2
|
||||
exit 1; }
|
||||
|
||||
# The pool. Its quotas are what stop a runaway guest filling the root
|
||||
# filesystem and taking Docker, Ollama and Open WebUI down with it, so a
|
||||
# plain directory is not a substitute.
|
||||
if ! zfs list -H -o name "${POOL_ROOT#/}" >/dev/null 2>&1; then
|
||||
echo "FAILED: no ZFS pool named '${POOL_ROOT#/}' (looked because OWUIVM_POOL_ROOT=$POOL_ROOT)." >&2
|
||||
echo " Run 'zfs list' to see what you have, then set OWUIVM_POOL_ROOT in" >&2
|
||||
echo " configs/host.env to the mountpoint of the pool you want to use." >&2
|
||||
exit 1
|
||||
fi
|
||||
for ds in vmprojects vmrun; do
|
||||
zfs list -H -o name "${POOL_ROOT#/}/$ds" >/dev/null 2>&1 || {
|
||||
echo " creating dataset ${POOL_ROOT#/}/$ds"
|
||||
zfs create -o exec=off -o setuid=off -o devices=off "${POOL_ROOT#/}/$ds"
|
||||
}
|
||||
done
|
||||
# The overlay dataset carries a quota so a runaway guest hits ENOSPC
|
||||
# inside its own ceiling rather than filling the pool.
|
||||
zfs set quota="${OWUIVM_OVERLAY_QUOTA:-40G}" "${POOL_ROOT#/}/vmrun"
|
||||
|
||||
echo "==> users and groups"
|
||||
getent passwd owuivm >/dev/null || useradd --system --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin --user-group owuivm
|
||||
getent passwd owuivmd >/dev/null || useradd --system --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin --user-group owuivmd
|
||||
getent passwd owuivmadm >/dev/null || useradd --system --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin --user-group owuivmadm
|
||||
# gid 10001 is the vmagent container's uid:gid. No userns remapping is
|
||||
# configured on this host, so container uid 10001 IS host uid 10001, and
|
||||
# that is what makes a mode-0660 socket directory a real boundary.
|
||||
getent group owuitools-agent >/dev/null || groupadd --system --gid 10001 owuitools-agent
|
||||
usermod -aG kvm owuivm
|
||||
usermod -aG owuivm,owuitools-agent owuivmd
|
||||
# The maintenance unit runs as owuivmadm and has to create its own
|
||||
# session directory under /run/owuivm, which is owned by vmd.
|
||||
usermod -aG owuivm owuivmadm
|
||||
|
||||
echo "==> directories"
|
||||
# Base images are readable by everyone and writable by owuivmadm alone.
|
||||
# "Frozen" is literally the owner write bit: the maintenance unit runs
|
||||
# `chmod u+w` to unfreeze and `chmod u-w` to refreeze, so the state is
|
||||
# visible in `ls -l` and the agent's uid can never be the one to change
|
||||
# it - owuivm is not the owner and is not in the group.
|
||||
install -d -o root -g root -m 0755 /var/lib/owuivm
|
||||
# 0775 and operator in the group: the operator installs and edits base
|
||||
# images, so they need to create a disk and drop an install marker
|
||||
# beside it. This does not weaken the freeze - the freeze exists to stop
|
||||
# the AGENT, which runs as owuivm and is in neither the owner nor the
|
||||
# group, and which has no verb that asks anything else to change a base.
|
||||
# The operator already has passwordless sudo; pretending otherwise here
|
||||
# would be theatre.
|
||||
install -d -o owuivmadm -g owuivmadm -m 0775 /var/lib/owuivm/base
|
||||
usermod -aG owuivmadm "$OPERATOR" 2>/dev/null || true
|
||||
install -d -o owuivmadm -g owuivmadm -m 0755 /var/lib/owuivm/iso
|
||||
# Where the maintenance app stages an installation disc before the
|
||||
# privileged unit checks it. OWNED BY THE OPERATOR, group owuivmadm: the
|
||||
# operator writes it as owner - so copying out of their own home needs no
|
||||
# privilege and no group that would only take effect at their next login
|
||||
# - and the unit reads it through the group. See owuivm-insert-media.
|
||||
install -d -o "$OPERATOR" -g owuivmadm -m 0770 /var/lib/owuivm/incoming
|
||||
install -d -o owuivmd -g owuivmd -m 0750 /var/lib/owuivm/state
|
||||
# What each session actually did, kept after the machine is gone.
|
||||
# See vm/vmd/sessionlog.py - the transcript used to die with the VM.
|
||||
install -d -o owuivmd -g owuivmd -m 0750 /var/lib/owuivm/sessions
|
||||
install -d -o root -g root -m 0755 /usr/local/libexec
|
||||
|
||||
# /Pool is 0770 root:docker - neither service user can traverse it, and
|
||||
# adding them to `docker` would hand them the docker socket, which is
|
||||
# root. A traverse-only ACL is the whole of what they actually need:
|
||||
# they reach their own datasets and still cannot list the parent.
|
||||
echo "==> $POOL_ROOT traversal"
|
||||
# owuivmadm is on this list because a FROZEN maintenance boot writes its
|
||||
# throwaway disk to /Pool/vmrun, the same dataset and the same 40 GB
|
||||
# quota a session's overlay uses. Traverse only: the directories below
|
||||
# carry their own permissions and none of them operator it anything here.
|
||||
setfacl -m u:owuivmd:--x,u:owuivm:--x,u:owuivmadm:--x "$POOL_ROOT"
|
||||
for u in owuivmd owuivm owuivmadm; do
|
||||
getfacl -p "$POOL_ROOT" | grep -q "^user:$u:--x" || { echo "FAILED: $POOL_ROOT ACL did not apply for $u" >&2; exit 1; }
|
||||
done
|
||||
|
||||
echo "==> scripts"
|
||||
install -o root -g root -m 0755 owuivm-guest-run /usr/local/bin/owuivm-guest-run
|
||||
render owuivm-launch /usr/local/libexec/owuivm-launch 0755
|
||||
render owuivm-teardown /usr/local/libexec/owuivm-teardown 0755
|
||||
render owuivm-maint-prepare /usr/local/libexec/owuivm-maint-prepare 0755
|
||||
install -o root -g root -m 0755 owuivm-insert-media /usr/local/libexec/owuivm-insert-media
|
||||
install -o root -g root -m 0755 owuivm-set-name /usr/local/libexec/owuivm-set-name
|
||||
install -d -o root -g root -m 0755 /usr/local/sbin
|
||||
render owuivm-ctl /usr/local/sbin/owuivm-ctl 0755
|
||||
|
||||
echo "==> sudoers"
|
||||
# visudo -c first: a malformed sudoers file locks every sudo on the host,
|
||||
# and installing it and finding out afterwards is not recoverable from a
|
||||
# non-root shell.
|
||||
visudo -cf owuivm.sudoers >/dev/null
|
||||
install -o root -g root -m 0440 owuivm.sudoers /etc/sudoers.d/owuivm
|
||||
|
||||
echo "==> apparmor (swtpm)"
|
||||
# Ubuntu's swtpm profile allows libvirt's paths only. Without this, the
|
||||
# software TPM cannot create its socket and Windows 11 refuses to
|
||||
# install - with an error that looks like a permission problem on the
|
||||
# directory rather than a policy denial.
|
||||
if [ -d /etc/apparmor.d/local ]; then
|
||||
install -o root -g root -m 0644 apparmor-local-swtpm /etc/apparmor.d/local/usr.bin.swtpm
|
||||
apparmor_parser -r /etc/apparmor.d/usr.bin.swtpm 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo "==> nftables guard"
|
||||
install -d -o root -g root -m 0755 /etc/nftables.d
|
||||
install -o root -g root -m 0644 60-owuivm.nft /etc/nftables.d/60-owuivm.nft
|
||||
nft -c -f /etc/nftables.d/60-owuivm.nft
|
||||
|
||||
echo "==> vmd"
|
||||
# The daemon's code lives under /usr/local/lib, root-owned: vmd runs as
|
||||
# owuivmd and must not be able to rewrite the program it is running, nor
|
||||
# the one root runs through sudo.
|
||||
install -d -o root -g root -m 0755 /usr/local/lib/owuivm
|
||||
for f in ../vmd/*.py; do install -o root -g root -m 0644 "$f" /usr/local/lib/owuivm/; done
|
||||
install -o root -g root -m 0755 ../vmd/vmctl /usr/local/bin/vmctl
|
||||
install -d -o root -g owuivmd -m 0750 /etc/owuivm
|
||||
# The daemon's settings come from configs/vmd.env, and are installed only
|
||||
# if they are not already there: the live copy holds the Gitea token, and
|
||||
# re-running this script must never wipe it. Edit /etc/owuivm/vmd.env to
|
||||
# change the running daemon.
|
||||
if [ ! -f /etc/owuivm/vmd.env ]; then
|
||||
install -o root -g owuivmd -m 0640 ../../configs/vmd.env /etc/owuivm/vmd.env
|
||||
echo " wrote /etc/owuivm/vmd.env - fill in OWUITOOLS_GITEA_TOKEN and"
|
||||
echo " OWUITOOLS_AUTH_TOKEN, or the agent publishes nothing and has no"
|
||||
echo " lookup tools. See docs/setup/gitea.md."
|
||||
fi
|
||||
# The pool has to reach the daemon too, and unlike the rest of vmd.env it
|
||||
# is not something the operator types - it is whatever this script was
|
||||
# told. Appended rather than written, so re-running never disturbs the
|
||||
# token beside it.
|
||||
if ! grep -q '^OWUIVM_POOL_ROOT=' /etc/owuivm/vmd.env; then
|
||||
printf '\n# Where overlays and project datasets live. Set by install.sh.\nOWUIVM_POOL_ROOT=%s\n' \
|
||||
"$POOL_ROOT" >> /etc/owuivm/vmd.env
|
||||
echo " recorded OWUIVM_POOL_ROOT=$POOL_ROOT in /etc/owuivm/vmd.env"
|
||||
fi
|
||||
|
||||
# The machine registry lives HERE and not in /etc/owuivm, which is 0750
|
||||
# root:owuivmd - only the daemon can even traverse it. Four different
|
||||
# accounts need to read this file (the operator's desktop app, the
|
||||
# maintenance units as owuivmadm, vmd, and the container), and the
|
||||
# maintenance units need to WRITE the renames beside it. Measured the
|
||||
# hard way: with it in /etc/owuivm the app could not read its own
|
||||
# machine list and the rename unit died on `mktemp: permission denied`.
|
||||
#
|
||||
# Owned by owuivmadm so the rename unit can write vm-names.toml here,
|
||||
# world-readable because none of it is a secret.
|
||||
install -d -o owuivmadm -g owuivmadm -m 0755 /var/lib/owuivm/registry
|
||||
# Overwritten on every install, unlike vmd.env: it carries no secret, and
|
||||
# the operator's renames live in vm-names.toml beside it, which is merged
|
||||
# over the top and is NOT touched here.
|
||||
install -o owuivmadm -g owuivmadm -m 0644 ../../configs/vms.toml \
|
||||
/var/lib/owuivm/registry/vms.toml
|
||||
|
||||
# The tool switch, so vmd offers the sub-agent the same lookup tools the
|
||||
# server is actually serving. Same reasoning: a tool switched off in one
|
||||
# place and offered in the other is a 404 that reads as a broken tool.
|
||||
install -o root -g root -m 0644 ../../configs/tools.toml /etc/owuivm/tools.toml
|
||||
|
||||
echo "==> the sandbox's resolver"
|
||||
# Bind-mounted over /etc/resolv.conf inside the VM units, so SLIRP's
|
||||
# built-in forwarder relays to a public resolver instead of the host's -
|
||||
# which the sandbox denies, and which knows the LAN's internal names.
|
||||
install -o root -g root -m 0644 vm-resolv.conf /etc/owuivm/resolv.conf
|
||||
|
||||
echo "==> systemd"
|
||||
render owuivmd.service /etc/systemd/system/owuivmd.service
|
||||
render owuivm-tmpfiles.conf /usr/lib/tmpfiles.d/owuivm.conf
|
||||
install -o root -g root -m 0644 owuivm-nft-guard.service /etc/systemd/system/owuivm-nft-guard.service
|
||||
render owuivm.service /etc/systemd/system/owuivm.service
|
||||
render owuivm-maint@.service /etc/systemd/system/owuivm-maint@.service
|
||||
install -o root -g root -m 0644 owuivm-detach@.service /etc/systemd/system/owuivm-detach@.service
|
||||
install -o root -g root -m 0644 owuivm-thaw@.service /etc/systemd/system/owuivm-thaw@.service
|
||||
install -o root -g root -m 0644 owuivm-freeze@.service /etc/systemd/system/owuivm-freeze@.service
|
||||
install -o root -g root -m 0644 owuivm-insert@.service /etc/systemd/system/owuivm-insert@.service
|
||||
install -o root -g root -m 0644 owuivm-rename@.service /etc/systemd/system/owuivm-rename@.service
|
||||
|
||||
echo "==> maintenance tool (desktop)"
|
||||
# Authorises an ACTIVE LOCAL SESSION of the operator to start the
|
||||
# maintenance unit - not a group, which a background process would
|
||||
# inherit. This is the gate an unfreeze passes through.
|
||||
install -d -o root -g root -m 0755 /etc/polkit-1/rules.d
|
||||
# The operator's name is substituted in rather than committed: the rule
|
||||
# is what decides who may unfreeze a base image, so it has to name a real
|
||||
# account on THIS machine.
|
||||
render 50-owuivm.rules /etc/polkit-1/rules.d/50-owuivm.rules
|
||||
install -o root -g root -m 0755 ../gui/owuivm-maintenance.py /usr/local/bin/owuivm-maintenance
|
||||
install -d -o root -g root -m 0755 /usr/local/share/applications
|
||||
install -o root -g root -m 0644 ../gui/owuivm-maintenance.desktop \
|
||||
/usr/local/share/applications/owuivm-maintenance.desktop
|
||||
# The shortcut the operator actually asked for. Written as the desktop
|
||||
# user, marked trusted, or GNOME shows it as an untrusted text file.
|
||||
if [ -n "$OPERATOR_HOME" ] && [ -d "$OPERATOR_HOME/Desktop" ]; then
|
||||
install -o "$OPERATOR" -g "$OPERATOR" -m 0755 ../gui/owuivm-maintenance.desktop \
|
||||
"$OPERATOR_HOME/Desktop/owuivm-maintenance.desktop"
|
||||
su - "$OPERATOR" -c \
|
||||
"gio set '$OPERATOR_HOME/Desktop/owuivm-maintenance.desktop' metadata::trusted true" \
|
||||
>/dev/null 2>&1 || true
|
||||
fi
|
||||
systemd-tmpfiles --create /usr/lib/tmpfiles.d/owuivm.conf
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now owuivm-nft-guard.service
|
||||
systemctl enable --now owuivmd.service
|
||||
sleep 1
|
||||
systemctl is-active --quiet owuivmd.service || {
|
||||
echo "FAILED: owuivmd did not start" >&2
|
||||
journalctl -u owuivmd.service -n 20 --no-pager >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo "==> verify the guard actually took"
|
||||
# A firewall that silently failed to apply is worse than none, because
|
||||
# everything downstream carries on as though it were enforced. Same
|
||||
# assertion netguard/firewall.sh makes about its own rules.
|
||||
nft list table inet owuivm_guard >/dev/null
|
||||
nft list table inet owuivm_guard | grep -q 'meta skuid' \
|
||||
|| { echo "FAILED: owuivm_guard loaded but has no skuid rule" >&2; exit 1; }
|
||||
|
||||
echo
|
||||
echo "host side installed."
|
||||
echo " base images /var/lib/owuivm/base (0444 = frozen, owner owuivmadm)"
|
||||
echo " overlays $POOL_ROOT/vmrun"
|
||||
echo " projects $POOL_ROOT/vmprojects"
|
||||
echo " guard table inet owuivm_guard (second layer; primary is owuivm.service)"
|
||||
Executable
+113
@@ -0,0 +1,113 @@
|
||||
#!/bin/bash
|
||||
# =====================================================================
|
||||
# The only thing vmd is allowed to run as root.
|
||||
#
|
||||
# vmd runs as owuivmd and needs four privileged things: start and stop
|
||||
# the VM unit, create a project dataset with a quota, and start and stop
|
||||
# virtiofsd. Granting it `systemctl` or `zfs` outright would be granting
|
||||
# it every unit and every dataset on this host, so it gets this instead:
|
||||
# a fixed verb list, one optional argument, and a pattern that argument
|
||||
# must match before it is used for anything.
|
||||
#
|
||||
# The sudoers entry names this path and nothing else. Everything here is
|
||||
# an array-form command - there is no `eval`, no unquoted expansion and
|
||||
# no shell metacharacter anywhere a project name can reach.
|
||||
#
|
||||
# THE PATTERN IS DUPLICATED ON PURPOSE
|
||||
#
|
||||
# It also lives in vm/vmd/project.py, and tests/test_vm_tasks.py asserts
|
||||
# the two are identical. Importing it would mean root code reading a
|
||||
# module out of a tree that vmd's own user can write to, which is exactly
|
||||
# backwards. Two copies and a test that they agree is the cheaper risk.
|
||||
# =====================================================================
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_RE='^[a-z0-9][a-z0-9-]{0,29}[a-z0-9]$'
|
||||
PROJECTS_DATASET="Pool/vmprojects"
|
||||
# The pool root is filled in by install.sh. This runs under sudo, where
|
||||
# the environment is not preserved, so it cannot read a setting the way
|
||||
# vmd does - the value has to be baked in when it is installed.
|
||||
PROJECTS_ROOT="@POOL_ROOT@/vmprojects"
|
||||
QUOTA="10G"
|
||||
RESERVED=" base iso state run git gitea owuivm "
|
||||
|
||||
die() { printf 'owuivm-ctl: %s\n' "$*" >&2; exit 64; }
|
||||
|
||||
check_name() {
|
||||
local n="${1:-}"
|
||||
[[ "$n" =~ $PROJECT_RE ]] || die "'$n' is not a valid project name"
|
||||
[[ "$RESERVED" == *" $n "* ]] && die "'$n' is reserved"
|
||||
return 0
|
||||
}
|
||||
|
||||
verb="${1:-}"; shift || true
|
||||
|
||||
case "$verb" in
|
||||
vm-start) exec /usr/bin/systemctl start owuivm.service ;;
|
||||
vm-stop) exec /usr/bin/systemctl stop owuivm.service ;;
|
||||
vm-status) exec /usr/bin/systemctl is-active owuivm.service ;;
|
||||
# `reset-failed` on its own, so a unit that died once cannot wedge
|
||||
# every later session behind "start request repeated too quickly".
|
||||
vm-reset) exec /usr/bin/systemctl reset-failed owuivm.service ;;
|
||||
|
||||
project-ensure)
|
||||
name="${1:-}"; check_name "$name"
|
||||
if ! /usr/sbin/zfs list -H -o name -- "$PROJECTS_DATASET/$name" >/dev/null 2>&1; then
|
||||
# Via systemd-run, not directly. `zfs create` MOUNTS the new
|
||||
# dataset, and mount() is not in @system-service - which is the
|
||||
# syscall filter vmd runs under and which every child of vmd
|
||||
# inherits, sudo included. Run directly it dies with SIGSYS, and
|
||||
# "Bad system call" from zfs is a genuinely confusing way to learn
|
||||
# that. systemd-run spawns from PID 1 instead, so this lands in its
|
||||
# own unit outside vmd's cgroup, and vmd's own filter stays tight
|
||||
# rather than being widened to @mount for one call.
|
||||
/usr/bin/systemd-run --quiet --wait --collect \
|
||||
--unit="owuivm-zfs-create" --property=Type=oneshot \
|
||||
-- /usr/sbin/zfs create -o "quota=$QUOTA" -- "$PROJECTS_DATASET/$name"
|
||||
fi
|
||||
# setgid so everything the guest writes stays in a group vmd and
|
||||
# virtiofsd can both still read afterwards.
|
||||
chown owuivmd:owuivm -- "$PROJECTS_ROOT/$name"
|
||||
chmod 2770 -- "$PROJECTS_ROOT/$name"
|
||||
printf '%s\n' "$PROJECTS_ROOT/$name"
|
||||
;;
|
||||
|
||||
project-usage)
|
||||
name="${1:-}"; check_name "$name"
|
||||
exec /usr/sbin/zfs get -Hp -o value used,quota -- "$PROJECTS_DATASET/$name"
|
||||
;;
|
||||
|
||||
vfs-start)
|
||||
name="${1:-}"; check_name "$name"
|
||||
session="${2:-}"
|
||||
[[ "$session" =~ ^[0-9a-f]{32}$ ]] || die "session id is not 32 hex characters"
|
||||
# Type=exec, NOT notify: virtiofsd never sends sd_notify(READY=1), so
|
||||
# systemd waits the full start timeout and then kills a daemon that
|
||||
# was working perfectly - the log says "Waiting for vhost-user socket
|
||||
# connection..." right up until it is terminated. Readiness here is
|
||||
# the socket file appearing, which vmd waits for itself.
|
||||
exec /usr/bin/systemd-run \
|
||||
--unit="owuivm-virtiofsd" --collect --property=Type=exec \
|
||||
--property=Restart=no --property=NoNewPrivileges=yes \
|
||||
--property=ProtectSystem=strict \
|
||||
--property="ReadWritePaths=/run/owuivm @POOL_ROOT@/vmprojects/$name" \
|
||||
--property=PrivateNetwork=yes \
|
||||
-- /usr/libexec/virtiofsd \
|
||||
--socket-path="/run/owuivm/$session/vfs.sock" \
|
||||
--socket-group=owuivm \
|
||||
--shared-dir="$PROJECTS_ROOT/$name" \
|
||||
--sandbox=namespace \
|
||||
--modcaps=-mknod:-setfcap
|
||||
# No --xattr: it is a bare flag in virtiofsd 1.10 (there is no
|
||||
# --xattr=false), and extended attributes are off unless it is given.
|
||||
# Leaving them off is what is wanted - an xattr is another thing the
|
||||
# guest can set on a file that then lives on the host's array.
|
||||
;;
|
||||
|
||||
vfs-stop)
|
||||
/usr/bin/systemctl stop owuivm-virtiofsd.service >/dev/null 2>&1 || true
|
||||
/usr/bin/systemctl reset-failed owuivm-virtiofsd.service >/dev/null 2>&1 || true
|
||||
;;
|
||||
|
||||
*) die "unknown verb '${verb}'" ;;
|
||||
esac
|
||||
@@ -0,0 +1,36 @@
|
||||
# =====================================================================
|
||||
# Detach the installation media from a base image.
|
||||
#
|
||||
# systemctl start owuivm-detach@ubuntu.service
|
||||
#
|
||||
# WHY THIS IS A UNIT AND NOT A LINE IN THE DESKTOP TOOL
|
||||
#
|
||||
# The marker lives in /var/lib/owuivm/base, which the operator's account
|
||||
# does not own. It could be made to - they were added to the owuivmadm
|
||||
# group - but a unix group takes effect at the NEXT login, so a tool that
|
||||
# depends on it fails confusingly for exactly as long as the session that
|
||||
# installed it lasts. That trap has already been walked into once here.
|
||||
#
|
||||
# Going through a unit uses the mechanism that is already known to work
|
||||
# for this operator: polkit, the same check that starts a maintenance
|
||||
# boot. 50-owuivm.rules authorises both.
|
||||
#
|
||||
# Removing the marker is all it does. What it means is "this image is
|
||||
# installed now": the launcher stops attaching the disc, and the machine
|
||||
# boots itself.
|
||||
# =====================================================================
|
||||
[Unit]
|
||||
Description=Detach installation media from the %i base image
|
||||
ConditionPathExists=/var/lib/owuivm/base/%i.install
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=owuivmadm
|
||||
Group=owuivmadm
|
||||
RemainAfterExit=no
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=/var/lib/owuivm/base
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
NoNewPrivileges=yes
|
||||
ExecStart=/bin/rm -f /var/lib/owuivm/base/%i.install
|
||||
@@ -0,0 +1,34 @@
|
||||
# =====================================================================
|
||||
# The operator's own verb: freeze a base image.
|
||||
#
|
||||
# systemctl start owuivm-freeze@ubuntu.service
|
||||
#
|
||||
# One chmod, in a directory the operator's account does not own. It goes
|
||||
# through a unit for the same reason owuivm-detach@ does: polkit is the
|
||||
# check that already works for this operator, and a unix group would
|
||||
# only take effect at their next login.
|
||||
#
|
||||
# Whether an image is frozen is the owner write bit and nothing else, so
|
||||
# these two verbs are the whole mechanism and `ls -l` is the whole
|
||||
# report. Neither will run while the machine is up - qemu already holds
|
||||
# the file open, so a chmod then would change what the mode SAYS without
|
||||
# changing anything about what is happening.
|
||||
# =====================================================================
|
||||
[Unit]
|
||||
Description=Freeze the %i base image
|
||||
ConditionPathExists=/var/lib/owuivm/base/%i.qcow2
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=owuivmadm
|
||||
Group=owuivmadm
|
||||
RemainAfterExit=no
|
||||
# The same umask the maintenance unit uses, so the checksum manifest this
|
||||
# writes keeps the mode it has always had rather than a default 0644.
|
||||
UMask=0007
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=/var/lib/owuivm/base
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
NoNewPrivileges=yes
|
||||
ExecStart=/usr/local/libexec/owuivm-maint-prepare %i freeze
|
||||
Executable
+67
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Push a local script into the running maintenance guest and run it.
|
||||
|
||||
sudo owuivm-guest-run vm/images/ubuntu/by-hand.sh
|
||||
|
||||
WHY THIS EXISTS
|
||||
|
||||
Preparing a base image by hand means typing into a VNC console, and the
|
||||
list of things to do to it is a hundred lines long. Everything on that
|
||||
list is ordinary shell, so the only real problem is getting the text in
|
||||
there - and the machine already has a channel for exactly that, the same
|
||||
one every session uses.
|
||||
|
||||
It needs qemu-guest-agent inside the guest, which is also the one thing
|
||||
on that list that genuinely has to be typed by hand: it is what this
|
||||
speaks to. So the sequence is always "install the guest agent by hand,
|
||||
then push the rest".
|
||||
|
||||
Deliberately the maintenance socket only. A session's guest belongs to
|
||||
the agent, and a way to inject a script into it from the host is not
|
||||
something this host should own.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, "/usr/local/lib/owuivm")
|
||||
|
||||
from guest import Guest, GuestError # noqa: E402
|
||||
|
||||
SOCKET = "/run/owuivm/00000000000000000000000000000000/ga.sock"
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) != 2:
|
||||
print(f"usage: {argv[0]} <script>", file=sys.stderr)
|
||||
return 64
|
||||
with open(argv[1], "rb") as handle:
|
||||
body = handle.read()
|
||||
remote = "/root/" + argv[1].rsplit("/", 1)[-1]
|
||||
|
||||
try:
|
||||
with Guest(SOCKET, timeout=30.0) as agent:
|
||||
if not agent.ping(timeout=10.0):
|
||||
raise GuestError("the guest agent did not answer")
|
||||
agent.write_file(remote, body)
|
||||
print(f"pushed {len(body)} bytes to {remote}; running it\n", flush=True)
|
||||
# No timeout worth arguing about: this installs packages over
|
||||
# a link that is as fast as the host's, and a run that is
|
||||
# still going is not a run that has failed.
|
||||
result = agent.shell(f"bash {remote}", timeout_s=1800.0)
|
||||
except (GuestError, OSError) as exc:
|
||||
print(f"could not reach the machine: {exc}", file=sys.stderr)
|
||||
print("\nIs it running, and is qemu-guest-agent installed inside it?"
|
||||
"\n sudo apt install -y qemu-guest-agent"
|
||||
"\n sudo systemctl enable --now qemu-guest-agent", file=sys.stderr)
|
||||
return 69
|
||||
|
||||
sys.stdout.write(result["stdout"])
|
||||
if result["stderr"].strip():
|
||||
sys.stderr.write("\n-- stderr --\n" + result["stderr"])
|
||||
print(f"\nexit code: {result['exit_code']} ({result['seconds']}s)")
|
||||
return 0 if result["exit_code"] == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv))
|
||||
Executable
+82
@@ -0,0 +1,82 @@
|
||||
#!/bin/bash
|
||||
# =====================================================================
|
||||
# Take an installation disc the operator staged, and put it in the
|
||||
# library where the launcher will find it.
|
||||
#
|
||||
# owuivm-insert-media <machine-id>
|
||||
#
|
||||
# WHY THE OPERATOR STAGES IT AND THIS ONLY MOVES IT
|
||||
#
|
||||
# The obvious design is "pass the chosen path to a privileged unit and
|
||||
# let it copy". It does not work and should not:
|
||||
#
|
||||
# * the maintenance units run with ProtectHome=yes, so the unit cannot
|
||||
# see ~/Downloads at all; and
|
||||
# * owuivmadm is not in the operator's group, so even without that it
|
||||
# could not read a file under a 0750 home; and
|
||||
# * fixing either by running the copy as root would hand the operator a
|
||||
# "copy any file root can read into a world-readable directory"
|
||||
# primitive, in exchange for saving one move.
|
||||
#
|
||||
# So the desktop tool copies the disc - as the operator, out of their own
|
||||
# home, needing no privilege at all - into /var/lib/owuivm/incoming,
|
||||
# which they own. This runs afterwards and does the part that does need
|
||||
# privilege: check it, and move it into /var/lib/owuivm/iso.
|
||||
#
|
||||
# The move is a rename within one filesystem, so it is instant and
|
||||
# atomic. The copy, which is the slow part, already happened.
|
||||
# =====================================================================
|
||||
set -euo pipefail
|
||||
|
||||
ISOS=${OWUIVM_ISO_DIR:-/var/lib/owuivm/iso}
|
||||
INCOMING=${OWUIVM_INCOMING_DIR:-/var/lib/owuivm/incoming}
|
||||
REGISTRY=${OWUIVM_REGISTRY:-/var/lib/owuivm/registry/vms.toml}
|
||||
|
||||
name="${1:-}"
|
||||
[[ "$name" =~ ^[a-z0-9][a-z0-9-]{0,30}$ ]] || { echo "bad machine id '$name'" >&2; exit 64; }
|
||||
|
||||
# The machine has to be one the registry knows. Reading it with grep
|
||||
# rather than a TOML parser keeps this script dependency-free; the id is
|
||||
# already pattern-checked above, so it cannot be a regex of its own.
|
||||
if [ -r "$REGISTRY" ]; then
|
||||
grep -qE "^[[:space:]]*id[[:space:]]*=[[:space:]]*\"$name\"" "$REGISTRY" \
|
||||
|| { echo "no machine '$name' in $REGISTRY" >&2; exit 64; }
|
||||
fi
|
||||
|
||||
shopt -s nullglob
|
||||
staged=("$INCOMING/$name".*.iso)
|
||||
if [ ${#staged[@]} -eq 0 ]; then
|
||||
echo "nothing staged for $name in $INCOMING" >&2
|
||||
exit 65
|
||||
fi
|
||||
# Newest wins, and everything else for this machine is discarded below:
|
||||
# a half-copied disc from an earlier attempt must not be installable.
|
||||
disc=""
|
||||
for f in "${staged[@]}"; do
|
||||
[ -z "$disc" ] || [ "$f" -nt "$disc" ] && disc="$f"
|
||||
done
|
||||
|
||||
# A regular file, not a symlink into somewhere this account should not be
|
||||
# reading, and not a device. -h before -f because -f follows links.
|
||||
[ ! -h "$disc" ] || { echo "$disc is a symlink; refusing" >&2; exit 65; }
|
||||
[ -f "$disc" ] || { echo "$disc is not a regular file" >&2; exit 65; }
|
||||
|
||||
# An ISO 9660 image carries the string CD001 at offset 0x8001. Costs one
|
||||
# read and catches the case this is most likely to hit in practice: a
|
||||
# download that was actually an HTML error page, or a .zip renamed by
|
||||
# hand, which would otherwise fail much later as a machine that will not
|
||||
# boot for no visible reason.
|
||||
magic=$(dd if="$disc" bs=1 skip=32769 count=5 2>/dev/null || true)
|
||||
[ "$magic" = "CD001" ] || {
|
||||
echo "$(basename "$disc") is not an ISO 9660 image (no CD001 signature)" >&2
|
||||
rm -f -- "$disc"
|
||||
exit 65
|
||||
}
|
||||
|
||||
install -d -m 0755 "$ISOS"
|
||||
target="$ISOS/$(basename "$disc")"
|
||||
mv -f -- "$disc" "$target"
|
||||
chmod 0644 "$target"
|
||||
# Whatever else was staged for this machine is stale by definition.
|
||||
rm -f -- "$INCOMING/$name".*.iso
|
||||
echo "inserted $(basename "$target")"
|
||||
@@ -0,0 +1,33 @@
|
||||
# =====================================================================
|
||||
# Put a staged installation disc into the library.
|
||||
#
|
||||
# systemctl start owuivm-insert@ubuntu.service
|
||||
#
|
||||
# The instance is the MACHINE ID, not a path. The desktop tool has
|
||||
# already copied the disc into /var/lib/owuivm/incoming, as the operator,
|
||||
# out of their own home - see the header of owuivm-insert-media for why
|
||||
# that is the only arrangement that works. All this unit does is check
|
||||
# the staged file and move it, which is a rename within one filesystem.
|
||||
#
|
||||
# ProtectHome stays ON. This never reads the operator's home; that is the
|
||||
# entire point of the staging step.
|
||||
# =====================================================================
|
||||
[Unit]
|
||||
Description=Insert the staged installation disc for %i
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=owuivmadm
|
||||
Group=owuivmadm
|
||||
RemainAfterExit=no
|
||||
UMask=0022
|
||||
ProtectSystem=strict
|
||||
# The staging directory, and the library it moves the disc into.
|
||||
ReadWritePaths=/var/lib/owuivm/iso /var/lib/owuivm/incoming
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
NoNewPrivileges=yes
|
||||
# A 6-8 GB disc has already been copied by this point; the move itself is
|
||||
# instant. This is only a backstop against a wedged filesystem.
|
||||
TimeoutStartSec=120
|
||||
ExecStart=/usr/local/libexec/owuivm-insert-media %i
|
||||
Executable
+418
@@ -0,0 +1,418 @@
|
||||
#!/bin/bash
|
||||
# =====================================================================
|
||||
# The one place a qemu command line is built.
|
||||
#
|
||||
# WHY A SCRIPT AND NOT ExecStart=
|
||||
#
|
||||
# The isolation argument in TUNING.md Change 59 rests on the qemu argv
|
||||
# being a CONSTANT - specifically that -netdev never grows a guestfwd=,
|
||||
# smb=, tftp= or bootfile=. Those four are handled inside the qemu
|
||||
# process and never become host sockets, so neither the cgroup filter in
|
||||
# owuivm.service nor the nft table in 60-owuivm.nft can see them. No
|
||||
# filter can catch what no filter can observe; the only defence is that
|
||||
# the string is fixed and a test asserts it.
|
||||
#
|
||||
# So everything security-relevant is literal text in this file. What
|
||||
# varies between sessions is a session id and two paths, and those are
|
||||
# re-validated HERE as well as in vmd - this is the point where a name
|
||||
# becomes a path, and one check at the boundary is worth more than any
|
||||
# number upstream (the same reasoning as matlab.py's helper filenames).
|
||||
#
|
||||
# WHAT IS DELIBERATELY ABSENT
|
||||
#
|
||||
# -sandbox QEMU's own seccomp layer. MEASURED TO BE MUTUALLY
|
||||
# EXCLUSIVE with systemd's SystemCallFilter= on this
|
||||
# build: under the unit's filter, qemu reports "There
|
||||
# is no option group 'sandbox'" and exits 1, while the
|
||||
# identical command line outside it starts fine.
|
||||
# libseccomp IS linked - the two layers simply cannot
|
||||
# both install a filter.
|
||||
# systemd's is the one kept: it is a broader allowlist
|
||||
# (@system-service excludes @mount, @swap, @module,
|
||||
# @reboot and @obsolete outright), it covers the whole
|
||||
# process tree rather than just qemu, and it is
|
||||
# visible to `systemd-analyze security`. What is given
|
||||
# up is spawn=deny, and the vector that mattered there
|
||||
# - `migrate exec:` over QMP - is already closed by
|
||||
# vmd exposing typed verbs and never a raw command.
|
||||
# -runas `-sandbox elevateprivileges=deny` would have blocked
|
||||
# setuid anyway; systemd's User= drops the uid before
|
||||
# exec, which is stronger and needs no cooperation
|
||||
# from qemu.
|
||||
# -daemonize systemd supervises the process; a double fork would
|
||||
# hide the exit status and break ExecStopPost=.
|
||||
# usb passthrough, no host device reaches the guest except /dev/kvm,
|
||||
# vfio, -virtfs which DevicePolicy=closed enforces.
|
||||
# romfile the virtio-net PXE option ROM is dropped: a boot
|
||||
# path we never use is a parser we never need.
|
||||
# =====================================================================
|
||||
set -euo pipefail
|
||||
|
||||
die() { printf 'owuivm-launch: %s\n' "$*" >&2; exit 78; } # EX_CONFIG
|
||||
|
||||
MODE="${OWUIVM_MODE:-session}"
|
||||
SESSION="${OWUIVM_SESSION:-}"
|
||||
DISK="${OWUIVM_DISK:-}"
|
||||
VARS="${OWUIVM_VARS:-}"
|
||||
SHARE="${OWUIVM_SHARE:-}"
|
||||
# A saved machine to resume instead of booting one. See the warm-start
|
||||
# comment further down; empty for every ordinary boot.
|
||||
INCOMING="${OWUIVM_INCOMING:-}"
|
||||
MEM_MB="${OWUIVM_MEM_MB:-8192}"
|
||||
CPUS="${OWUIVM_CPUS:-8}"
|
||||
RUN="/run/owuivm/$SESSION"
|
||||
|
||||
[[ "$SESSION" =~ ^[0-9a-f]{32}$ ]] || die "session id is not 32 hex characters"
|
||||
[[ "$MEM_MB" =~ ^[0-9]{3,6}$ ]] || die "memory is not a plain number of MB"
|
||||
[[ "$CPUS" =~ ^[0-9]{1,2}$ ]] || die "cpu count is not a plain number"
|
||||
case "$MODE" in session|maint) ;; *) die "mode must be session or maint" ;; esac
|
||||
|
||||
# A session boots a throwaway overlay; maintenance boots a base disk that
|
||||
# somebody has deliberately unfrozen. The read-only case is enforced by
|
||||
# the file mode (base images are 0444 owuivmadm) rather than by this
|
||||
# flag, so a bug here cannot make a base writable.
|
||||
case "$MODE:$DISK" in
|
||||
session:@POOL_ROOT@/vmrun/*.qcow2) ;;
|
||||
maint:/var/lib/owuivm/base/*.qcow2) ;;
|
||||
*) die "disk path is not permitted for mode $MODE: $DISK" ;;
|
||||
esac
|
||||
[[ "$DISK" != *..* ]] || die "disk path contains .."
|
||||
[[ -f "$DISK" ]] || die "disk does not exist: $DISK"
|
||||
|
||||
[[ "$VARS" =~ ^/run/owuivm/[0-9a-f]{32}/OVMF_VARS\.fd$ ]] || die "vars path is not a session file"
|
||||
[[ -f "$VARS" ]] || die "vars file does not exist: $VARS"
|
||||
|
||||
# INSTALL MEDIA, MAINTENANCE MODE ONLY.
|
||||
#
|
||||
# The operator installs a base image by hand: they need the machine
|
||||
# booted from an ISO with a console attached, which is the same thing
|
||||
# maintenance already does plus one CD-ROM. A session must never get one
|
||||
# - it runs from a frozen disk that is already installed, and a bootable
|
||||
# CD in front of it is just another thing that could change what boots.
|
||||
#
|
||||
# The request is a file beside the base image saying which ISO, rather
|
||||
# than an environment variable, because the desktop tool that writes it
|
||||
# runs as the operator and cannot write into /run/owuivm. The path is
|
||||
# re-validated here anyway: this is where a name becomes a device.
|
||||
# Which image this is, from the disk's own name. A session's overlay is
|
||||
# named after its session id, so the name comes from the backing file.
|
||||
if [[ "$MODE" == "maint" ]]; then
|
||||
name=$(basename "$DISK" .qcow2)
|
||||
else
|
||||
name=$(basename "$(qemu-img info --output=json "$DISK" 2>/dev/null \
|
||||
| sed -n 's/.*"backing-filename": "\([^"]*\)".*/\1/p' | head -1)" .qcow2)
|
||||
fi
|
||||
[[ "$name" =~ ^(ubuntu|windows)$ ]] || die "cannot tell which image this is: '$name'"
|
||||
|
||||
# A FROZEN MAINTENANCE BOOT IS A SESSION'S BOOT WITH A SCREEN ATTACHED.
|
||||
#
|
||||
# The operator's main reason to start a base image is to check that it
|
||||
# comes up the way the agent will find it - and doing that must not be
|
||||
# able to change it. So the disk's own mode decides what is booted:
|
||||
# frozen means a throwaway overlay on top of the untouched base, exactly
|
||||
# what session.py builds for an agent, and unfrozen means the base
|
||||
# itself because somebody deliberately made it writable.
|
||||
#
|
||||
# The overlay path is DERIVED from a name this script has already
|
||||
# validated, never taken from the caller - which is why the path rule
|
||||
# above still only permits a base image here.
|
||||
# `-w` is deliberately NOT the test. It asks "can THIS process write
|
||||
# it", and root can write a 0444 file - so a maintenance boot started by
|
||||
# root would silently take the other branch and open the base directly.
|
||||
# Frozen is the owner write bit and nothing else, which is the same
|
||||
# definition the desktop tool, vmd and the prepare script all use.
|
||||
frozen_disk() { (( 8#$(stat -c %a "$1" 2>/dev/null || echo 600) & 8#200 )) && return 1 || return 0; }
|
||||
|
||||
if [[ "$MODE" == "maint" ]] && frozen_disk "$DISK"; then
|
||||
overlay="@POOL_ROOT@/vmrun/maint-$name.qcow2"
|
||||
rm -f "$overlay"
|
||||
qemu-img create -f qcow2 -b "$DISK" -F qcow2 "$overlay" >/dev/null \
|
||||
|| die "could not create the throwaway disk for a frozen boot: $overlay"
|
||||
# qemu-img asks for 0644 and the unit's umask clears the group write
|
||||
# bit it never requested, leaving a file the qemu user can read and
|
||||
# not write. Here qemu IS this user, but the mode is set explicitly so
|
||||
# the two callers of this line agree.
|
||||
chmod 0660 "$overlay" 2>/dev/null || true
|
||||
DISK="$overlay"
|
||||
fi
|
||||
|
||||
ISO=""
|
||||
if [[ "$MODE" == "maint" ]]; then
|
||||
marker="/var/lib/owuivm/base/$name.install"
|
||||
if [[ -f "$marker" ]]; then
|
||||
ISO=$(head -c 400 "$marker" | tr -d '\r\n')
|
||||
[[ "$ISO" =~ ^/var/lib/owuivm/iso/[A-Za-z0-9._-]+\.iso$ ]] \
|
||||
|| die "install media must be an .iso in /var/lib/owuivm/iso: $ISO"
|
||||
[[ "$ISO" != *..* ]] || die "install media path contains .."
|
||||
[[ -f "$ISO" ]] || die "install media does not exist: $ISO"
|
||||
fi
|
||||
fi
|
||||
|
||||
# WARM START: RESUME A SAVED MACHINE INSTEAD OF BOOTING ONE.
|
||||
#
|
||||
# A Windows boot costs ninety seconds and the agent cannot hide all of it
|
||||
# behind its planning turn. Resuming a saved machine is reading a file:
|
||||
# measured, 3.92 GB of state written in 14.6s, and the guest comes back
|
||||
# with its desktop drawn and its PowerShell already warm - which is the
|
||||
# ninety seconds AND the first-command cold start, both gone.
|
||||
#
|
||||
# The path is checked hard because a migration stream is executable input
|
||||
# in every sense that matters: it sets the machine's registers, its
|
||||
# device state and its memory. It may only ever be a file this host
|
||||
# wrote, in the directory only the image owner can write.
|
||||
if [[ -n "$INCOMING" ]]; then
|
||||
[[ "$INCOMING" =~ ^/var/lib/owuivm/warm/(ubuntu|windows)/state$ ]] \
|
||||
|| die "resume state is not one of this host's own: $INCOMING"
|
||||
[[ "$INCOMING" != *..* ]] || die "resume path contains .."
|
||||
[[ -f "$INCOMING" ]] || die "resume state does not exist: $INCOMING"
|
||||
fi
|
||||
|
||||
if [[ -n "$SHARE" ]]; then
|
||||
[[ "$SHARE" =~ ^@POOL_ROOT@/vmprojects/[a-z0-9][a-z0-9-]{1,30}$ ]] || die "share path is not a project dataset"
|
||||
[[ "$SHARE" != *..* ]] || die "share path contains .."
|
||||
[[ -d "$SHARE" ]] || die "share does not exist: $SHARE"
|
||||
fi
|
||||
|
||||
# WINDOWS 11 WILL NOT INSTALL WITHOUT SECURE BOOT AND A TPM.
|
||||
#
|
||||
# Its setup checks for both and stops with "This PC can't run Windows 11"
|
||||
# - which reads like a hardware problem rather than a missing device
|
||||
# model. Ubuntu needs neither and is given neither, so the two machines
|
||||
# differ only where they have to.
|
||||
#
|
||||
# `smm=on` is already on the machine type above and is a prerequisite:
|
||||
# without it the secure-boot firmware loads but cannot protect its own
|
||||
# variable store, and the enrolled keys are decorative.
|
||||
OVMF_CODE=/usr/share/OVMF/OVMF_CODE_4M.fd
|
||||
TPM_ARGS=()
|
||||
CPU=host
|
||||
RTC_BASE=utc
|
||||
|
||||
if [[ "$name" == "windows" ]]; then
|
||||
# HYPER-V ENLIGHTENMENTS: THE DIFFERENCE BETWEEN "SLOW" AND "FINE".
|
||||
#
|
||||
# Windows assumes it may be a Hyper-V guest and looks for the
|
||||
# paravirtual interfaces that say so. Finding none, it behaves like a
|
||||
# kernel on bare metal that has been lied to about its hardware: it
|
||||
# spins on locks a hypervisor could have yielded, takes a VM exit for
|
||||
# every timer tick and every inter-processor interrupt, and drives its
|
||||
# APIC through a trap on each access. Measured on this host, an idle
|
||||
# Windows 11 installer held qemu at 347% of one 700% quota while the
|
||||
# cgroup was neither short of memory (memory.events max 0) nor
|
||||
# meaningfully throttled (0.35s of throttling in three hours) - the
|
||||
# work was real, it was simply wasted.
|
||||
#
|
||||
# Each of these tells Windows that somebody is listening:
|
||||
# relaxed stop watchdogging a clock that a hypervisor may pause
|
||||
# vapic/avic APIC through memory instead of a VM exit per access
|
||||
# time a paravirtual reference clock
|
||||
# spinlocks give up the CPU after N spins instead of burning them
|
||||
# synic/stimer synthetic interrupt controller and timers, with
|
||||
# stimer-direct delivering them without a round trip
|
||||
# tlbflush/ipi one hypercall instead of one IPI per target CPU
|
||||
# vpindex/runtime/frequencies/reset the identifiers and the reset
|
||||
# path the above are defined in terms of
|
||||
#
|
||||
# Listed explicitly rather than `hv-passthrough`, which hands over
|
||||
# whatever the host kernel happens to support: these images are frozen
|
||||
# and are meant to behave the same after a host upgrade as before it.
|
||||
# All of them verified accepted by this host's KVM before being added.
|
||||
CPU="host,hv-relaxed,hv-vapic,hv-avic,hv-time,hv-spinlocks=0x1fff"
|
||||
CPU="$CPU,hv-vpindex,hv-runtime,hv-synic,hv-stimer,hv-stimer-direct"
|
||||
CPU="$CPU,hv-reset,hv-frequencies,hv-tlbflush,hv-ipi"
|
||||
|
||||
# Windows reads the RTC as local time and has done since DOS. With the
|
||||
# host at UTC that is not a performance problem, it is a clock several
|
||||
# hours wrong - which breaks TLS verification, and therefore breaks
|
||||
# `pip`, `winget` and every script that fetches anything, in a way that
|
||||
# reads as a network failure.
|
||||
RTC_BASE=localtime
|
||||
|
||||
OVMF_CODE=/usr/share/OVMF/OVMF_CODE_4M.secboot.fd
|
||||
[[ -f "$OVMF_CODE" ]] || die "secure-boot firmware is missing: $OVMF_CODE"
|
||||
|
||||
# swtpm is a separate process holding the TPM's state. It is started
|
||||
# here rather than as its own unit because it must be listening before
|
||||
# qemu opens the chardev, and it exits on its own when qemu goes away
|
||||
# (--terminate), so there is nothing left to reap.
|
||||
mkdir -p "$RUN/tpm"
|
||||
/usr/bin/swtpm socket --tpm2 --terminate \
|
||||
--tpmstate "dir=$RUN/tpm" \
|
||||
--ctrl "type=unixio,path=$RUN/swtpm.sock" \
|
||||
--log level=0 &
|
||||
for _ in $(seq 1 50); do [[ -S "$RUN/swtpm.sock" ]] && break; sleep 0.1; done
|
||||
[[ -S "$RUN/swtpm.sock" ]] || die "the software TPM did not start"
|
||||
TPM_ARGS=(
|
||||
-chardev "socket,id=chrtpm,path=$RUN/swtpm.sock"
|
||||
-tpmdev emulator,id=tpm0,chardev=chrtpm
|
||||
-device tpm-tis,tpmdev=tpm0
|
||||
)
|
||||
fi
|
||||
|
||||
args=(
|
||||
-name "owuivm-$MODE"
|
||||
-no-user-config
|
||||
-nodefaults
|
||||
-machine q35,accel=kvm,smm=on,vmport=off
|
||||
-cpu "$CPU"
|
||||
-smp "$CPUS"
|
||||
-m "$MEM_MB"
|
||||
-rtc "base=$RTC_BASE,driftfix=slew"
|
||||
-global kvm-pit.lost_tick_policy=delay
|
||||
|
||||
# Shared memory is a hard requirement of vhost-user-fs; it has to be
|
||||
# the machine's only memory backend, so it is set unconditionally
|
||||
# rather than only when a share is attached - a guest that boots with a
|
||||
# different memory topology depending on whether a project was passed
|
||||
# is a guest that behaves differently in testing than in use.
|
||||
-object "memory-backend-memfd,id=mem0,size=${MEM_MB}M,share=on"
|
||||
-numa node,memdev=mem0
|
||||
|
||||
-drive "if=pflash,format=raw,unit=0,readonly=on,file=$OVMF_CODE"
|
||||
-drive "if=pflash,format=raw,unit=1,file=$VARS"
|
||||
|
||||
# cache=none is O_DIRECT: this VM's disk I/O does not go through the
|
||||
# host's page cache at all. That is not a throughput tweak, it is what
|
||||
# keeps the cgroup accounting honest - see MemoryMax in the unit. The
|
||||
# guest's RAM is unreclaimable shmem, so any page cache charged to the
|
||||
# same cgroup is in a fight it cannot win, and the loser was the
|
||||
# installer media being read from disk over and over.
|
||||
# Verified O_DIRECT works on both filesystems in play here: ext4 for
|
||||
# the base images, and ZFS 2.4.1 for the session overlays.
|
||||
-drive "file=$DISK,if=none,id=disk0,format=qcow2,discard=unmap,cache=none,aio=native"
|
||||
-device virtio-blk-pci,drive=disk0,bootindex=0
|
||||
|
||||
# THE LINE THE WHOLE ISOLATION ARGUMENT RESTS ON.
|
||||
# user-mode networking: no tap, no bridge, no CAP_NET_ADMIN, and no
|
||||
# layer-2 presence anywhere on this host. Every guest packet leaves as
|
||||
# an ordinary socket owned by this process's uid, which is what makes
|
||||
# both IPAddressDeny= and `meta skuid owuivm` able to see it at all.
|
||||
# ipv6=off because the host has no global v6 and enumerating v6 ranges
|
||||
# nobody has yet is how a filter silently stops matching.
|
||||
-netdev user,id=net0,ipv6=off
|
||||
-device virtio-net-pci,netdev=net0,romfile=
|
||||
|
||||
# virtio-vga, not the plain VGA device: a desktop on the emulated VGA
|
||||
# MMIO framebuffer repaints through a slow path for every update, and
|
||||
# with llvmpipe doing the rendering anyway that is the difference
|
||||
# between usable and painful.
|
||||
#
|
||||
# Still 2D. NOT virtio-gpu-gl/virgl: QMP screendump reads the console
|
||||
# surface, and a device rendering through a GL context does not give it
|
||||
# one - the agent's whole GUI mode depends on that surface existing.
|
||||
# virtio-vga also keeps a VGA compatibility mode, which is what the
|
||||
# Windows installer uses before its drivers are loaded.
|
||||
-device virtio-vga,max_outputs=1
|
||||
-display none
|
||||
|
||||
# Absolute pointing, so a click at (x,y) lands at (x,y) whatever the
|
||||
# guest's pointer acceleration is doing.
|
||||
-device qemu-xhci,id=xhci
|
||||
-device usb-tablet,bus=xhci.0
|
||||
|
||||
# A ROOT PORT THAT IS USUALLY EMPTY, AND WHY IT IS ALWAYS THERE.
|
||||
#
|
||||
# q35's root complex is not hot-pluggable, so a machine resumed from a
|
||||
# saved state has nowhere to put the project share - and the share
|
||||
# cannot be in the saved state, because virtiofsd refuses migration
|
||||
# ("vhost-user backend lacks VHOST_USER_PROTOCOL_F_LOG_SHMFD").
|
||||
#
|
||||
# So the share lives behind this port on EVERY path: plugged in at
|
||||
# startup on a cold boot, hot-plugged into the same address on a warm
|
||||
# one. Same bus, same address, one device as far as the guest is
|
||||
# concerned - a drive letter that moves between sessions would be worse
|
||||
# than a slow boot.
|
||||
#
|
||||
# It is present even when no share is attached, because a saved state
|
||||
# and the machine that loads it must have exactly the same devices.
|
||||
-device pcie-root-port,id=vfsport,bus=pcie.0,addr=0x10,chassis=1,slot=1
|
||||
|
||||
# Two virtio-serial ports, no network involved in either: the standard
|
||||
# guest-agent channel, and a second one for the Windows broker that
|
||||
# runs in the interactive admin session (qemu-ga there is LocalSystem,
|
||||
# which is not the "normal elevated PowerShell" this is meant to give).
|
||||
-device virtio-serial-pci,id=vser0
|
||||
-chardev "socket,id=chga,path=$RUN/ga.sock,server=on,wait=off"
|
||||
-device virtserialport,chardev=chga,bus=vser0.0,name=org.qemu.guest_agent.0
|
||||
-chardev "socket,id=chbr,path=$RUN/broker.sock,server=on,wait=off"
|
||||
-device virtserialport,chardev=chbr,bus=vser0.0,name=com.owuitools.broker.0
|
||||
|
||||
# Control plane. 0600 inside a 0700 directory, because QMP is a
|
||||
# capability and not a debug port: it carries human-monitor-command,
|
||||
# migrate exec: and a screendump that writes a caller-chosen filename.
|
||||
# vmd never forwards a command to it, only typed verbs built from
|
||||
# constants.
|
||||
-qmp "unix:$RUN/qmp.sock,server=on,wait=off"
|
||||
)
|
||||
|
||||
# Secure boot is not just the firmware file: without this the variable
|
||||
# store is writable from inside the guest and the whole thing is theatre.
|
||||
if [[ "$name" == "windows" ]]; then
|
||||
args+=( -global driver=cfi.pflash01,property=secure,value=on )
|
||||
args+=( "${TPM_ARGS[@]}" )
|
||||
fi
|
||||
|
||||
if [[ -n "$SHARE" ]]; then
|
||||
args+=(
|
||||
-chardev "socket,id=chfs,path=$RUN/vfs.sock"
|
||||
-device vhost-user-fs-pci,chardev=chfs,tag=project,bus=vfsport,addr=0x0
|
||||
)
|
||||
fi
|
||||
|
||||
if [[ -n "$INCOMING" ]]; then
|
||||
# `file:` rather than `exec:`. Measured: `exec:` fails with "Unable to
|
||||
# write to command: Broken pipe", because this unit's syscall filter
|
||||
# and NoNewPrivileges stop qemu spawning the helper - which is the
|
||||
# sandbox doing its job. `file:` opens a file and needs nobody.
|
||||
args+=( -incoming "file:$INCOMING" )
|
||||
fi
|
||||
|
||||
if [[ "$MODE" == "maint" ]]; then
|
||||
# THE CONSOLE IS A UNIX SOCKET, NOT A TCP PORT, AND THAT IS NOT STYLE.
|
||||
#
|
||||
# systemd's IPAddressDeny= filters INGRESS as well as egress, checking
|
||||
# the SOURCE address on the way in. This unit denies 127.0.0.0/8 -
|
||||
# which is what stops the guest reaching the host's own loopback
|
||||
# through SLIRP's 10.0.2.2 alias - and the source of a connection to a
|
||||
# loopback VNC port is, of course, 127.0.0.1. So the same rule that
|
||||
# protects the host silently dropped the operator's own console: the
|
||||
# port listened, `query-vnc` reported it enabled, and every client sat
|
||||
# there until it timed out. Measured both ways: with the deny list
|
||||
# cleared, the server greets with RFB 003.008 immediately.
|
||||
#
|
||||
# A unix socket is not IP, so the filter does not apply to it, and the
|
||||
# guest's egress policy stays exactly as strict as a session's. The
|
||||
# desktop tool relays it to a loopback port for the viewer, from a
|
||||
# process outside this cgroup.
|
||||
args+=( -vnc "unix:/run/owuivm-maint/vnc.sock" )
|
||||
if [[ -n "$ISO" ]]; then
|
||||
# The DISK is tried first and the media second, which is the opposite
|
||||
# of the obvious arrangement and is deliberate.
|
||||
#
|
||||
# UEFI walks its boot entries in order, so a blank or half-installed
|
||||
# disk has nothing to boot and falls through to the installer by
|
||||
# itself, while a finished install boots itself and ignores the media
|
||||
# still sitting in the drive. That makes an interrupted install
|
||||
# resumable without anyone having to decide whether it finished -
|
||||
# which is exactly the judgement that went wrong before: a partial
|
||||
# install had written 3.45 GB, was declared complete on size alone,
|
||||
# lost its media, and came up in an EFI shell with no way back.
|
||||
args+=(
|
||||
-drive "file=$ISO,if=none,id=cd0,media=cdrom,readonly=on,cache=none,aio=native"
|
||||
-device ide-cd,drive=cd0,bus=ide.0,bootindex=1
|
||||
)
|
||||
# A SECOND disc, for Windows only: its setup cannot see a virtio-blk
|
||||
# disk and will report that there are no drives to install to, which
|
||||
# looks like a broken VM rather than a missing driver. Point setup's
|
||||
# "Load driver" at this disc, amd64\w11.
|
||||
virtio=/var/lib/owuivm/iso/virtio-win.iso
|
||||
if [[ "$name" == "windows" && -f "$virtio" ]]; then
|
||||
args+=(
|
||||
-drive "file=$virtio,if=none,id=cd1,media=cdrom,readonly=on,cache=none,aio=native"
|
||||
-device ide-cd,drive=cd1,bus=ide.1
|
||||
)
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
exec /usr/bin/qemu-system-x86_64 "${args[@]}"
|
||||
Executable
+276
@@ -0,0 +1,276 @@
|
||||
#!/bin/bash
|
||||
# =====================================================================
|
||||
# Everything a maintenance boot needs, done from inside the unit.
|
||||
#
|
||||
# ExecStartPre= owuivm-maint-prepare <name> open
|
||||
# ExecStopPost= owuivm-maint-prepare <name> close
|
||||
# the toggle: owuivm-maint-prepare <name> thaw | freeze
|
||||
#
|
||||
# STARTING A MACHINE AND UNFREEZING ITS DISK ARE TWO DIFFERENT DECISIONS
|
||||
#
|
||||
# They used to be one: ExecStartPre unfroze, ExecStopPost re-froze, and
|
||||
# the only way to look at a base image was to make it writable first.
|
||||
# That is backwards for the thing the operator most wants to do - boot
|
||||
# the image exactly as the agent will get it, to see that it comes up,
|
||||
# WITHOUT the possibility of changing it by being there.
|
||||
#
|
||||
# So the write bit is now set and cleared by `thaw` and `freeze`, which
|
||||
# are the operator's own verbs, and `open` boots whatever mode the disk
|
||||
# is already in:
|
||||
#
|
||||
# frozen (0444) -> a throwaway overlay, created by owuivm-launch,
|
||||
# discarded by `close`. Identical to what a session
|
||||
# boots. Nothing the operator does can reach the base.
|
||||
# unfrozen (0664) -> the base disk itself, and changes are kept.
|
||||
#
|
||||
# The safety net that ExecStopPost used to provide is not gone, it has
|
||||
# moved to where it cannot be forgotten: vmd refuses to start a session
|
||||
# at all while a base image is writable, so an image left unfrozen stops
|
||||
# the agent rather than quietly exposing it.
|
||||
#
|
||||
# WHY THE UNIT DOES THIS AND NOT THE DESKTOP TOOL
|
||||
#
|
||||
# The first version had the GUI create the disk and write the ISO
|
||||
# marker, which meant the operator's own account needed write access to
|
||||
# /var/lib/owuivm/base - so it added them to a group. That works and it
|
||||
# is also a trap: a unix group takes effect at the NEXT login, so the
|
||||
# tool fails in a confusing way for exactly as long as the session that
|
||||
# installed it lasts. Doing it here needs no group at all. The desktop
|
||||
# tool starts and stops a unit and touches nothing.
|
||||
#
|
||||
# `close` runs even when qemu is SIGKILLed, which is the point: the
|
||||
# overlay of a frozen boot is removed however that boot ended.
|
||||
#
|
||||
# The checksum is re-recorded on freeze rather than checked. The operator
|
||||
# is MEANT to have changed the disk - that is what maintenance is - so
|
||||
# the manifest records what it became, and the previous value is kept
|
||||
# beside it so a change can be seen to have happened.
|
||||
# =====================================================================
|
||||
set -euo pipefail
|
||||
|
||||
# Overridable so tests/test_vm_freeze.sh can exercise the freeze logic
|
||||
# against a scratch directory. Defaults are the real thing; nothing but
|
||||
# the test ever sets them.
|
||||
BASE=${OWUIVM_BASE_DIR:-/var/lib/owuivm/base}
|
||||
ISOS=${OWUIVM_ISO_DIR:-/var/lib/owuivm/iso}
|
||||
# OVERRIDABLE FOR THE SAME REASON THE OTHERS ARE, AND IT COST AN INSTALL.
|
||||
#
|
||||
# A maintenance boot's session id is a constant, so this path is a
|
||||
# constant too - and a test running as root that used the real one left
|
||||
# a root:root directory behind. The next `open` then failed on
|
||||
# `cp: cannot remove .../OVMF_VARS.fd: Permission denied`, which reads
|
||||
# like a bug in the tool and is really a test's litter.
|
||||
RUN=${OWUIVM_RUN_DIR:-/run/owuivm/00000000000000000000000000000000}
|
||||
OVMF_VARS=/usr/share/OVMF/OVMF_VARS_4M.fd
|
||||
# Windows 11 plus whatever a session installs on top of it. qcow2 is
|
||||
# sparse, so this is a ceiling rather than an allocation - the 24.04
|
||||
# Ubuntu image settled at 7.5 GB of a 60 GB disk.
|
||||
DISK_SIZE=60G
|
||||
|
||||
name="${1:-}"; action="${2:-}"
|
||||
[[ "$name" =~ ^(ubuntu|windows)$ ]] || { echo "unknown image '$name'" >&2; exit 64; }
|
||||
|
||||
disk="$BASE/$name.qcow2"
|
||||
vars="$BASE/$name-OVMF_VARS.fd"
|
||||
marker="$BASE/$name.install"
|
||||
# The throwaway disk of a frozen maintenance boot. Same dataset as a
|
||||
# session's overlay, and for the same reason: a 40 GB ZFS quota, so a
|
||||
# runaway guest hits ENOSPC inside its own quota instead of filling the
|
||||
# root filesystem and taking Docker, Ollama and Open WebUI with it.
|
||||
overlay=${OWUIVM_OVERLAY_DIR:-@POOL_ROOT@/vmrun}/maint-$name.qcow2
|
||||
|
||||
# The owner write bit, not `-w`: this script runs as owuivmadm in the
|
||||
# unit and as root in the tests, and root can write a read-only file -
|
||||
# so `-w` would give the two a different idea of the same disk.
|
||||
frozen() {
|
||||
[ -e "$disk" ] || return 1
|
||||
[ $(( 8#$(stat -c %a "$disk" 2>/dev/null || echo 600) & 8#200 )) -eq 0 ]
|
||||
}
|
||||
|
||||
# chmod under a running qemu is theatre: it already holds the file open,
|
||||
# so the mode says one thing and the machine does another. Both toggle
|
||||
# verbs refuse rather than lie about it.
|
||||
running() {
|
||||
systemctl is-active --quiet "owuivm-maint@$name.service" 2>/dev/null
|
||||
}
|
||||
|
||||
find_iso() {
|
||||
# One ISO per operating system is the normal case: fetch.sh puts it
|
||||
# there and there is nothing else to confuse it with. If somebody has
|
||||
# kept two, the newest wins and the name is printed, so what booted is
|
||||
# never a mystery.
|
||||
local pattern
|
||||
case "$name" in
|
||||
ubuntu) pattern='ubuntu*.iso' ;;
|
||||
# NOT '*.iso'. That matched the newest ISO in the directory, which -
|
||||
# with an Ubuntu image sitting next to it - is how a Windows
|
||||
# maintenance boot would have quietly booted the Ubuntu installer.
|
||||
# fetch.sh refuses Windows media that does not match this shape, so
|
||||
# the two ends agree.
|
||||
windows) pattern='[Ww]in*.iso' ;;
|
||||
esac
|
||||
# shellcheck disable=SC2086
|
||||
ls -t "$ISOS"/$pattern 2>/dev/null | head -1
|
||||
}
|
||||
|
||||
case "$action" in
|
||||
open)
|
||||
# One maintenance machine at a time. `Conflicts=owuivm.service` on
|
||||
# the unit stops a session and a maintenance boot overlapping, but a
|
||||
# template unit cannot conflict with its own other instances - so
|
||||
# ubuntu and windows could both come up, fight over the same console
|
||||
# socket, and share out the host's memory between them.
|
||||
for other in ubuntu windows; do
|
||||
[ "$other" = "$name" ] && continue
|
||||
if systemctl is-active --quiet "owuivm-maint@$other.service"; then
|
||||
echo "the $other image is already open for maintenance; close that first" >&2
|
||||
exit 69
|
||||
fi
|
||||
done
|
||||
mkdir -p "$RUN"
|
||||
# Say whose it is rather than letting cp report it one line later.
|
||||
if [ ! -w "$RUN" ]; then
|
||||
echo "cannot write $RUN (owned by $(stat -c %U "$RUN" 2>/dev/null)) - stale state from something else" >&2
|
||||
exit 73
|
||||
fi
|
||||
if [ ! -f "$disk" ]; then
|
||||
iso=$(find_iso)
|
||||
[ -n "$iso" ] || {
|
||||
echo "no installation media in $ISOS - run vm/images/fetch.sh first" >&2
|
||||
exit 66
|
||||
}
|
||||
echo "no $name image yet; creating a blank $DISK_SIZE disk and booting $(basename "$iso")"
|
||||
qemu-img create -f qcow2 "$disk" "$DISK_SIZE" >/dev/null
|
||||
# Windows needs Microsoft's keys already enrolled in the variable
|
||||
# store, or Secure Boot is on with nothing it trusts and its
|
||||
# installer refuses the machine. Ubuntu takes the plain store.
|
||||
case "$name" in
|
||||
windows) cp -f /usr/share/OVMF/OVMF_VARS_4M.ms.fd "$vars" ;;
|
||||
*) cp -f "$OVMF_VARS" "$vars" ;;
|
||||
esac
|
||||
printf '%s\n' "$iso" > "$marker"
|
||||
fi
|
||||
# A disk that has just been created is writable, which is what an
|
||||
# install needs - there is nothing to protect yet. Every other boot
|
||||
# takes the disk as it finds it.
|
||||
cp -f "$vars" "$RUN/OVMF_VARS.fd"
|
||||
chmod 0660 "$RUN/OVMF_VARS.fd"
|
||||
# Never inherit the last boot's throwaway disk. It would still have a
|
||||
# valid backing file, so nothing would fail - the operator would
|
||||
# simply be shown a machine carrying changes they thought they had
|
||||
# discarded.
|
||||
rm -f "$overlay"
|
||||
if frozen; then
|
||||
echo "frozen boot: $disk stays read-only; changes go to $overlay and are discarded"
|
||||
else
|
||||
echo "unfrozen boot: $disk is writable and changes are kept"
|
||||
fi
|
||||
;;
|
||||
close)
|
||||
# NOTHING HERE MAY BE ABLE TO ABORT WHAT COMES AFTER IT.
|
||||
#
|
||||
# That is not a general principle, it is this file's scar. The marker
|
||||
# logic here once used `du -B1 --apparent-size=0`, and --apparent-size
|
||||
# takes no argument - so du exited non-zero, `set -euo pipefail`
|
||||
# killed the script at that line, and the chmod that used to live
|
||||
# below it never ran. Every clean shutdown left the base image
|
||||
# WRITABLE, silently, with only "Control process exited,
|
||||
# status=1/FAILURE" in the log.
|
||||
set +e
|
||||
|
||||
# The throwaway disk goes whatever happened, including a SIGKILL -
|
||||
# this line is why `close` is ExecStopPost and not ExecStop.
|
||||
rm -f "$overlay"
|
||||
|
||||
# A FROZEN BOOT LEAVES NOTHING BEHIND, AND THAT INCLUDES FIRMWARE.
|
||||
#
|
||||
# OVMF rewrites its boot order on every boot, so a look-at-it boot
|
||||
# would otherwise still change the image it was supposed not to
|
||||
# touch. `frozen` is the whole test: if the operator never took the
|
||||
# write bit off, nothing they did is kept.
|
||||
if frozen; then
|
||||
rm -f "$RUN/OVMF_VARS.fd"
|
||||
printf 'frozen boot closed: %s unchanged (mode %s)\n' \
|
||||
"$disk" "$(stat -c %a "$disk" 2>/dev/null)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Copy the edited variables back, or a UEFI boot entry the installer
|
||||
# added is silently lost and the disk will not boot next time. Only
|
||||
# when the disk has something on it: carrying back the variables of a
|
||||
# boot that installed nothing is how a base image accumulates entries
|
||||
# pointing at device paths that no longer exist.
|
||||
allocated=$(du -B1 "$disk" 2>/dev/null | cut -f1)
|
||||
[ -n "$allocated" ] || allocated=0
|
||||
if [ -f "$RUN/OVMF_VARS.fd" ] && [ "$allocated" -gt 536870912 ]; then
|
||||
chmod u+w "$vars" 2>/dev/null
|
||||
cp -f "$RUN/OVMF_VARS.fd" "$vars"
|
||||
fi
|
||||
rm -f "$RUN/OVMF_VARS.fd"
|
||||
|
||||
# THE MEDIA STAYS ATTACHED, AND NOTHING HERE GUESSES.
|
||||
#
|
||||
# This used to remove the install marker when the disk had grown past
|
||||
# half a gigabyte, on the reasoning that a disk with contents has an
|
||||
# operating system on it. A partial install had written 3.45 GB, was
|
||||
# declared finished, lost its media, and came back in an EFI shell
|
||||
# with no way to resume - the disk was neither blank enough to start
|
||||
# over nor complete enough to boot.
|
||||
#
|
||||
# Size cannot distinguish "installed" from "interrupted halfway", and
|
||||
# neither can anything else available from out here. So it no longer
|
||||
# tries: the marker persists, and the boot order in owuivm-launch
|
||||
# makes that harmless - a finished install boots itself and ignores
|
||||
# the media, an unfinished one falls through to it. Detaching the
|
||||
# media is an explicit act by the operator, in the maintenance tool.
|
||||
|
||||
# DELIBERATELY NO chmod. The image is left exactly as the operator
|
||||
# set it: they unfroze it on purpose and may well want to start it
|
||||
# again. What stops a forgotten thaw from reaching the agent is vmd
|
||||
# refusing to open a session on a writable base - a refusal with a
|
||||
# message, rather than a chmod nobody saw.
|
||||
printf 'unfrozen boot closed: %s is still WRITABLE (mode %s) - freeze it when done\n' \
|
||||
"$disk" "$(stat -c %a "$disk" 2>/dev/null)"
|
||||
exit 0
|
||||
;;
|
||||
freeze|thaw)
|
||||
# The operator's own verbs. Each is one chmod, which is why the state
|
||||
# is legible in `ls -l` and why nothing has to be trusted about it.
|
||||
if running; then
|
||||
echo "$name is running; shut it down before changing the disk's mode" >&2
|
||||
exit 69
|
||||
fi
|
||||
[ -e "$disk" ] || { echo "there is no $name image yet" >&2; exit 66; }
|
||||
set +e
|
||||
if [ "$action" = thaw ]; then
|
||||
if systemctl is-active --quiet owuivm.service 2>/dev/null; then
|
||||
echo "an agent session is running; it must finish before an image can be unfrozen" >&2
|
||||
exit 69
|
||||
fi
|
||||
chmod 0660 "$disk" 2>/dev/null
|
||||
chmod 0660 "$vars" 2>/dev/null
|
||||
printf 'thawed: %s (mode now %s) - changes made from now on are KEPT\n' \
|
||||
"$disk" "$(stat -c %a "$disk" 2>/dev/null)"
|
||||
exit 0
|
||||
fi
|
||||
chmod 0444 "$disk" "$vars" 2>/dev/null
|
||||
# Re-recorded rather than checked: the operator is MEANT to have
|
||||
# changed the disk, so the manifest records what it became and the
|
||||
# previous value is kept beside it, where a change can be seen to
|
||||
# have happened.
|
||||
if [ -f "$BASE/$name.sha256" ]; then
|
||||
mv -f "$BASE/$name.sha256" "$BASE/$name.sha256.previous"
|
||||
fi
|
||||
sha256sum "$disk" "$vars" > "$BASE/$name.sha256" 2>/dev/null
|
||||
# World-readable on purpose: vmd compares a saved machine against
|
||||
# this before resuming it, and vmd is not in the image owner's group.
|
||||
# It says what a public file hashes to; there is nothing in it to keep.
|
||||
chmod 0644 "$BASE/$name.sha256" 2>/dev/null
|
||||
# Say what it actually is, not what was intended: a freeze that did
|
||||
# not take should be visible in the log rather than asserted there.
|
||||
printf 'frozen: %s (mode now %s)\n' "$disk" "$(stat -c %a "$disk" 2>/dev/null)"
|
||||
exit 0
|
||||
;;
|
||||
*) echo "usage: owuivm-maint-prepare {ubuntu|windows} {open|close|thaw|freeze}" >&2
|
||||
exit 64 ;;
|
||||
esac
|
||||
@@ -0,0 +1,159 @@
|
||||
# =====================================================================
|
||||
# Maintenance mode: the operator unfreezing a base image to change it.
|
||||
#
|
||||
# THIS IS WHY THE AGENT CAN NEVER UNFREEZE ANYTHING
|
||||
#
|
||||
# The sandbox runs as owuivm. Base images are mode 0444 owned by
|
||||
# owuivmadm, in a directory owuivm does not own and is not in the group
|
||||
# of. So the uid that runs every session physically cannot clear a write
|
||||
# bit, and there is no verb in vmd that asks anyone else to.
|
||||
#
|
||||
# This unit is the only path that can, and it runs as owuivmadm. Starting
|
||||
# it is gated by polkit (50-owuivm.rules) on an ACTIVE LOCAL SESSION of
|
||||
# the operator - not on group membership, which a background process
|
||||
# could inherit. An agent asking for an unfreeze produces a sentence in a
|
||||
# report; a person at the machine is what produces a writable disk.
|
||||
#
|
||||
# FREEZING IS A chmod, WHICH MEANS IT IS VISIBLE
|
||||
#
|
||||
# `ls -l` says whether an image is frozen. This unit does not change it
|
||||
# in either direction: starting a machine and unfreezing its disk are
|
||||
# separate decisions, made by separate buttons, because the most useful
|
||||
# thing the operator can do is boot the image exactly as the agent will
|
||||
# get it - which must not be able to change it.
|
||||
#
|
||||
# A frozen disk therefore boots a throwaway overlay and ExecStopPost
|
||||
# throws it away; an unfrozen one boots itself and keeps what happens.
|
||||
# An image left unfrozen is caught where it matters rather than here:
|
||||
# vmd refuses to open a session on a writable base.
|
||||
# =====================================================================
|
||||
[Unit]
|
||||
Description=owuitools VM maintenance (%i base image)
|
||||
After=owuivm-nft-guard.service
|
||||
Requires=owuivm-nft-guard.service
|
||||
# Refuses to run while a session is live: one VM at a time, and two
|
||||
# processes with the same disk open is how a qcow2 gets corrupted.
|
||||
Conflicts=owuivm.service
|
||||
# Deliberately NO ConditionPathExists on the disk: starting this unit
|
||||
# with no image is how one gets installed. ExecStartPre creates a blank
|
||||
# disk and points the machine at the installation media.
|
||||
ConditionPathExists=/dev/kvm
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=owuivmadm
|
||||
Group=owuivmadm
|
||||
SupplementaryGroups=kvm
|
||||
# 0007 so the console socket qemu creates is group-writable, and the
|
||||
# directory's setgid bit makes that group the OPERATOR's - the person who has
|
||||
# to be able to connect to it.
|
||||
UMask=0007
|
||||
|
||||
Environment=OWUIVM_MODE=maint
|
||||
Environment=OWUIVM_SESSION=00000000000000000000000000000000
|
||||
Environment=OWUIVM_MEM_MB=8192
|
||||
Environment=OWUIVM_CPUS=8
|
||||
Environment=OWUIVM_DISK=/var/lib/owuivm/base/%i.qcow2
|
||||
Environment=OWUIVM_VARS=/run/owuivm/00000000000000000000000000000000/OVMF_VARS.fd
|
||||
|
||||
ExecStartPre=/usr/local/libexec/owuivm-maint-prepare %i open
|
||||
ExecStart=/usr/local/libexec/owuivm-launch
|
||||
# The power button first, then the tidying. Without this, stopping the
|
||||
# unit yanked power on the disk the operator was editing.
|
||||
ExecStop=/usr/local/libexec/owuivm-powerdown ${OWUIVM_SESSION}
|
||||
ExecStopPost=/usr/local/libexec/owuivm-maint-prepare %i close
|
||||
|
||||
# The maintenance VM gets exactly the sandbox's network policy. It is the
|
||||
# operator driving it rather than an agent, but it is the same disk that
|
||||
# every session will later run from: if something is installed here that
|
||||
# only works because this VM could reach the LAN, that breaks later, in a
|
||||
# session, where it is much harder to see.
|
||||
IPAddressDeny=127.0.0.0/8 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 100.64.0.0/10 192.0.0.0/24 224.0.0.0/4 240.0.0.0/4
|
||||
RestrictAddressFamilies=AF_UNIX AF_INET
|
||||
# No exception needed: the console is a unix socket, so this unit opens
|
||||
# no listening TCP port at all - the same as a session.
|
||||
SocketBindDeny=any
|
||||
|
||||
DevicePolicy=closed
|
||||
DeviceAllow=/dev/kvm rw
|
||||
NoNewPrivileges=yes
|
||||
# THE SANDBOX GETS ITS OWN RESOLVER, AND THIS IS NOT COSMETIC.
|
||||
#
|
||||
# SLIRP hands every guest 10.0.2.3 as its nameserver over DHCP. That is a
|
||||
# forwarder inside the qemu process, and it relays to whatever the HOST's
|
||||
# /etc/resolv.conf says - here 127.0.0.53, systemd-resolved, which
|
||||
# forwards to the LAN's own DNS and knows every internal name in the
|
||||
# house. This unit denies 127.0.0.0/8, correctly, so that relay is dead.
|
||||
#
|
||||
# The result is a guest with working TCP to the internet and no name
|
||||
# resolution whatsoever, which presents as "no internet" and sends
|
||||
# whoever is looking at it to entirely the wrong place. It did exactly
|
||||
# that to a hand-installed image.
|
||||
#
|
||||
# Bind-mounting a resolv.conf of our own fixes it for every guest without
|
||||
# the guest being configured at all, and without the host's resolver ever
|
||||
# becoming reachable from the sandbox.
|
||||
BindReadOnlyPaths=/etc/owuivm/resolv.conf:/etc/resolv.conf
|
||||
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
# The overlay dataset holds the throwaway disk of a frozen boot. It is the same
|
||||
# dataset a session's overlay lives on, and its 40 GB quota is what
|
||||
# stops a guest that writes forever from filling the root filesystem.
|
||||
ReadWritePaths=/run/owuivm /run/owuivm-maint /var/lib/owuivm/base @POOL_ROOT@/vmrun
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelLogs=yes
|
||||
ProtectControlGroups=yes
|
||||
RestrictNamespaces=yes
|
||||
RestrictRealtime=yes
|
||||
LockPersonality=yes
|
||||
SystemCallArchitectures=native
|
||||
SystemCallFilter=@system-service
|
||||
# NOT optional, and its absence is not a smaller version of having it.
|
||||
# Without this, a filtered syscall raises SIGSYS and kills the process;
|
||||
# with it, the call returns EPERM and the caller decides what to do.
|
||||
# qemu calls seccomp(2) at startup to probe for its own sandbox - which
|
||||
# @system-service does not permit - so a maintenance boot died instantly
|
||||
# with status=31/SYS until this line was added, while a session boot,
|
||||
# which has always had it, simply got EPERM and carried on.
|
||||
SystemCallErrorNumber=EPERM
|
||||
|
||||
# MEASURED: 10G made the machine crawl, and it was not obvious why.
|
||||
#
|
||||
# The guest's RAM is a memfd with share=on (virtiofs requires it), so the
|
||||
# kernel accounts all 8 GiB of it as *shmem* in this cgroup - and with no
|
||||
# swap on this host, shmem cannot be reclaimed at all. That left about
|
||||
# 1.5 GiB for every page of ISO and disk I/O the VM does, so the cgroup
|
||||
# hit its hard limit 6,782 times in one boot (memory.events max), evicting
|
||||
# and re-reading the same installer media over and over.
|
||||
#
|
||||
# The first fix was to raise this to 14G so the page cache had room.
|
||||
# That worked - reclaim went to zero - and it was the wrong fix: it gave
|
||||
# one VM twelve gigabytes of a 58 GB host that is also running Ollama,
|
||||
# and the host started running short instead. Raising a limit to make
|
||||
# room for something that should not have been in there.
|
||||
#
|
||||
# The right fix is one line in owuivm-launch: cache=none on every drive,
|
||||
# so this VM's I/O never enters the page cache at all. What is left in
|
||||
# the cgroup is the guest's RAM and qemu itself, which is what this limit
|
||||
# was always meant to bound. 11G is 8 GiB of guest, about 1 GiB of qemu,
|
||||
# and slack.
|
||||
MemoryMax=@VM_MEMORY_MAX@
|
||||
# 800%, NOT 700%, AND THE DIFFERENCE IS NOT THEORETICAL.
|
||||
#
|
||||
# The machine has 8 vCPUs, so 800% is exactly what it can consume and
|
||||
# 700% means one of its eight can never run. Measured during a Windows
|
||||
# install: qemu sat at 697% of the 700% ceiling, with 1,792 throttle
|
||||
# events and 103 seconds of CPU actually withheld - while the operator
|
||||
# watched a progress bar and wondered whether nested virtualisation was
|
||||
# to blame. It was not; it was this line.
|
||||
#
|
||||
# The session unit keeps 700% on purpose: it runs while the main model
|
||||
# may be answering someone, and it is pinned to eight threads that
|
||||
# Ollama can also use. A maintenance boot is one person doing one thing,
|
||||
# and it should have the whole machine it was given.
|
||||
CPUQuota=@MAINT_CPU_QUOTA@
|
||||
TasksMax=512
|
||||
TimeoutStopSec=120
|
||||
KillMode=mixed
|
||||
@@ -0,0 +1,21 @@
|
||||
[Unit]
|
||||
# Deliberately the same shape as /etc/systemd/system/node_exporter_nft_guard.service,
|
||||
# which is this host's established pattern for a composable guard table
|
||||
# that lives alongside ufw's own ip/ip6 filter tables rather than fighting
|
||||
# them. Ordered Before= the VM unit so a guest can never run with the
|
||||
# second layer absent - though note the FIRST layer is IPAddressDeny= in
|
||||
# owuivm.service, which does not depend on this unit at all.
|
||||
Description=Second-layer egress filter for the owuitools VM sandbox
|
||||
Documentation=https://documentation.ubuntu.com/security/security-features/network/firewall/nftables/
|
||||
Before=owuivm.service owuivm-maint.service
|
||||
After=local-fs.target
|
||||
ConditionPathExists=/etc/nftables.d/60-owuivm.nft
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/bin/sh -c '/usr/sbin/nft delete table inet owuivm_guard >/dev/null 2>&1 || true; exec /usr/sbin/nft -f /etc/nftables.d/60-owuivm.nft'
|
||||
ExecStop=/bin/sh -c '/usr/sbin/nft delete table inet owuivm_guard >/dev/null 2>&1 || true'
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/bin/bash
|
||||
# =====================================================================
|
||||
# Presses the virtual power button, from the unit's ExecStop=.
|
||||
#
|
||||
# WHY THIS EXISTS
|
||||
#
|
||||
# Stopping the unit used to SIGTERM qemu, which from inside the machine
|
||||
# is indistinguishable from pulling the plug out of the wall. For a
|
||||
# session that is fine - the overlay is thrown away either side of it -
|
||||
# but a MAINTENANCE boot is the operator editing the base image that
|
||||
# every future session runs from, and yanking power on that is how a
|
||||
# frozen image acquires a dirty filesystem, or a half-written registry.
|
||||
#
|
||||
# ExecStop= runs while the main process is still alive, so this sends
|
||||
# `system_powerdown` over QMP and returns. systemd then waits for qemu
|
||||
# to exit by itself, up to TimeoutStopSec - and if the guest ignores the
|
||||
# button, or has already hung, the SIGTERM still happens afterwards. So
|
||||
# this is a courtesy with a deadline, not a new way to get stuck.
|
||||
#
|
||||
# Deliberately never fails: a stop path that can error is a stop path
|
||||
# that leaves the unit in `failed` and the next start blocked.
|
||||
# =====================================================================
|
||||
set -u
|
||||
|
||||
SESSION="${1:-${OWUIVM_SESSION:-}}"
|
||||
[[ "$SESSION" =~ ^[0-9a-f]{32}$ ]] || exit 0
|
||||
SOCK="/run/owuivm/$SESSION/qmp.sock"
|
||||
[ -S "$SOCK" ] || exit 0
|
||||
|
||||
/usr/bin/python3 - "$SOCK" <<'PY' 2>/dev/null || true
|
||||
import json, socket, sys, time
|
||||
path = sys.argv[1]
|
||||
try:
|
||||
s = socket.socket(socket.AF_UNIX); s.settimeout(5); s.connect(path)
|
||||
f = s.makefile("rwb")
|
||||
f.readline() # greeting
|
||||
f.write(b'{"execute":"qmp_capabilities"}\n'); f.flush(); f.readline()
|
||||
f.write(b'{"execute":"system_powerdown"}\n'); f.flush(); f.readline()
|
||||
# Give the guest a moment to notice the button before systemd starts
|
||||
# counting; an OS that has begun shutting down will not be helped by
|
||||
# being asked twice.
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
f.write(b'{"execute":"query-status"}\n'); f.flush()
|
||||
line = f.readline()
|
||||
if not line:
|
||||
break # qemu has gone: done
|
||||
try:
|
||||
if not (json.loads(line).get("return") or {}).get("running", True):
|
||||
break
|
||||
except ValueError:
|
||||
pass
|
||||
time.sleep(1)
|
||||
except Exception:
|
||||
pass
|
||||
PY
|
||||
exit 0
|
||||
@@ -0,0 +1,31 @@
|
||||
# =====================================================================
|
||||
# Record a new display name for a machine.
|
||||
#
|
||||
# systemctl start owuivm-rename@ubuntu.service
|
||||
#
|
||||
# The wanted name is read from /var/lib/owuivm/incoming/<id>.name, which
|
||||
# the desktop tool wrote as the operator. A unit instance carries one
|
||||
# string and that string has to be the machine id, so the name travels
|
||||
# through the same staging directory the installation media uses.
|
||||
#
|
||||
# This writes ONLY /var/lib/owuivm/registry/vm-names.toml, an override file that
|
||||
# readers merge over the shipped registry. configs/vms.toml is never
|
||||
# edited by a program: it is mostly comments, and no TOML writer keeps
|
||||
# them.
|
||||
# =====================================================================
|
||||
[Unit]
|
||||
Description=Rename the %i machine
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=owuivmadm
|
||||
Group=owuivmadm
|
||||
RemainAfterExit=no
|
||||
UMask=0022
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=/var/lib/owuivm/registry /var/lib/owuivm/incoming
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
NoNewPrivileges=yes
|
||||
TimeoutStartSec=30
|
||||
ExecStart=/usr/local/libexec/owuivm-set-name %i
|
||||
Executable
+69
@@ -0,0 +1,69 @@
|
||||
#!/bin/bash
|
||||
# =====================================================================
|
||||
# Rename a machine, as far as anything is allowed to.
|
||||
#
|
||||
# owuivm-set-name <machine-id>
|
||||
#
|
||||
# Reads the wanted name from /var/lib/owuivm/incoming/<id>.name, which
|
||||
# the desktop tool wrote as the operator, and records it in the override
|
||||
# file. Same staging trick as owuivm-insert-media, and for the same
|
||||
# reason: a systemd unit instance carries one string, and that string is
|
||||
# the machine id.
|
||||
#
|
||||
# WHY AN OVERRIDE FILE AND NOT AN EDIT TO configs/vms.toml
|
||||
#
|
||||
# Because that file is mostly comments, and they are the useful part of
|
||||
# it - what an id is, why the ISO glob is not `*.iso`, what the prompt
|
||||
# fields do. No TOML writer preserves comments, so a program that edits
|
||||
# the registry in place is a program that deletes its own documentation
|
||||
# the first time someone presses Rename.
|
||||
#
|
||||
# So the registry stays exactly as shipped and this writes a separate,
|
||||
# tiny file that readers merge over the top. Only `name` is overridable,
|
||||
# which is also the only field that is safe to change: `id` is identity
|
||||
# and the rest is prompt text.
|
||||
# =====================================================================
|
||||
set -euo pipefail
|
||||
|
||||
INCOMING=${OWUIVM_INCOMING_DIR:-/var/lib/owuivm/incoming}
|
||||
OVERRIDES=${OWUIVM_NAME_OVERRIDES:-/var/lib/owuivm/registry/vm-names.toml}
|
||||
REGISTRY=${OWUIVM_REGISTRY:-/var/lib/owuivm/registry/vms.toml}
|
||||
|
||||
id="${1:-}"
|
||||
[[ "$id" =~ ^[a-z0-9][a-z0-9-]{0,30}$ ]] || { echo "bad machine id '$id'" >&2; exit 64; }
|
||||
|
||||
if [ -r "$REGISTRY" ]; then
|
||||
grep -qE "^[[:space:]]*id[[:space:]]*=[[:space:]]*\"$id\"" "$REGISTRY" \
|
||||
|| { echo "no machine '$id' in $REGISTRY" >&2; exit 64; }
|
||||
fi
|
||||
|
||||
src="$INCOMING/$id.name"
|
||||
[ -f "$src" ] || { echo "no name staged for $id" >&2; exit 65; }
|
||||
[ ! -h "$src" ] || { echo "$src is a symlink; refusing" >&2; exit 65; }
|
||||
|
||||
# First line only, trimmed. A name is a label in a window title and in a
|
||||
# report; a newline in it would be a second line of a TOML file.
|
||||
wanted=$(head -c 200 "$src" | head -1 | tr -d '\r' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//')
|
||||
rm -f -- "$src"
|
||||
[ -n "$wanted" ] || { echo "the name was empty" >&2; exit 64; }
|
||||
# Printable, and nothing that would need escaping in a TOML basic string.
|
||||
printf '%s' "$wanted" | grep -qP '^[^\x00-\x1f"\\]{1,40}$' \
|
||||
|| { echo "a name must be 1-40 printable characters and no quotes or backslashes" >&2; exit 64; }
|
||||
|
||||
tmp=$(mktemp "${OVERRIDES}.XXXXXX")
|
||||
trap 'rm -f "$tmp"' EXIT
|
||||
{
|
||||
echo '# Machine display names, set from the Tool Server VM Maintenance app.'
|
||||
echo '# Merged over the `name` field in vms.toml. Delete a line to go back'
|
||||
echo '# to the shipped name; delete the file to reset all of them.'
|
||||
echo '[names]'
|
||||
# Every other machine's current override, kept.
|
||||
if [ -r "$OVERRIDES" ]; then
|
||||
grep -E '^[a-z0-9-]+[[:space:]]*=' "$OVERRIDES" | grep -vE "^$id[[:space:]]*=" || true
|
||||
fi
|
||||
printf '%s = "%s"\n' "$id" "$wanted"
|
||||
} > "$tmp"
|
||||
chmod 0644 "$tmp"
|
||||
mv -f "$tmp" "$OVERRIDES"
|
||||
trap - EXIT
|
||||
echo "$id is now called $wanted"
|
||||
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/bin/bash
|
||||
# =====================================================================
|
||||
# Runs from ExecStopPost=, so it happens however qemu died - clean exit,
|
||||
# SIGTERM, SIGKILL, OOM, or a crash. That is the whole reason it exists
|
||||
# as a unit hook rather than as code in vmd: the archived sub-agent stack
|
||||
# recorded a cancellation and killed nothing, and the run carried on.
|
||||
#
|
||||
# It runs as owuivm, which owns the overlay and the session directory's
|
||||
# contents, so it needs no privilege at all to undo its own mess. vmd
|
||||
# reaps again when it starts; neither is trusted to be the one that ran.
|
||||
#
|
||||
# Deliberately never fails: a teardown that exits non-zero puts the unit
|
||||
# into `failed` and the next `systemctl start` needs a reset-failed first,
|
||||
# which turns a transient into a wedge.
|
||||
# =====================================================================
|
||||
set -u
|
||||
|
||||
SESSION="${OWUIVM_SESSION:-}"
|
||||
[[ "$SESSION" =~ ^[0-9a-f]{32}$ ]] || exit 0
|
||||
|
||||
RUN="/run/owuivm/$SESSION"
|
||||
|
||||
# The overlay is disposable by definition - every session starts from the
|
||||
# frozen base - so it goes unconditionally rather than being kept "just
|
||||
# in case". Anything worth keeping was written to the project share.
|
||||
DISK="${OWUIVM_DISK:-}"
|
||||
case "$DISK" in
|
||||
@POOL_ROOT@/vmrun/*.qcow2) rm -f -- "$DISK" 2>/dev/null ;;
|
||||
esac
|
||||
|
||||
rm -f -- "$RUN"/qmp.sock "$RUN"/ga.sock "$RUN"/broker.sock \
|
||||
"$RUN"/vnc.sock "$RUN"/OVMF_VARS.fd 2>/dev/null
|
||||
|
||||
# Tells vmd the machine is gone without it having to race systemd's own
|
||||
# state, and gives the orphan reaper something to find if vmd was not
|
||||
# running at the time.
|
||||
: > "$RUN/STOPPED" 2>/dev/null
|
||||
|
||||
exit 0
|
||||
@@ -0,0 +1,34 @@
|
||||
# =====================================================================
|
||||
# The operator's own verb: thaw a base image.
|
||||
#
|
||||
# systemctl start owuivm-thaw@ubuntu.service
|
||||
#
|
||||
# One chmod, in a directory the operator's account does not own. It goes
|
||||
# through a unit for the same reason owuivm-detach@ does: polkit is the
|
||||
# check that already works for this operator, and a unix group would
|
||||
# only take effect at their next login.
|
||||
#
|
||||
# Whether an image is frozen is the owner write bit and nothing else, so
|
||||
# these two verbs are the whole mechanism and `ls -l` is the whole
|
||||
# report. Neither will run while the machine is up - qemu already holds
|
||||
# the file open, so a chmod then would change what the mode SAYS without
|
||||
# changing anything about what is happening.
|
||||
# =====================================================================
|
||||
[Unit]
|
||||
Description=Thaw the %i base image
|
||||
ConditionPathExists=/var/lib/owuivm/base/%i.qcow2
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=owuivmadm
|
||||
Group=owuivmadm
|
||||
RemainAfterExit=no
|
||||
# The same umask the maintenance unit uses, so the checksum manifest this
|
||||
# writes keeps the mode it has always had rather than a default 0644.
|
||||
UMask=0007
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=/var/lib/owuivm/base
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
NoNewPrivileges=yes
|
||||
ExecStart=/usr/local/libexec/owuivm-maint-prepare %i thaw
|
||||
@@ -0,0 +1,14 @@
|
||||
# /run/owuivm is where every session's sockets and its OVMF vars live,
|
||||
# and where a maintenance boot puts its own. Group-writable because three
|
||||
# users legitimately create things here: vmd (sessions), qemu (sockets),
|
||||
# and owuivmadm (maintenance). 0770: nothing else on this host has any
|
||||
# business reading a QMP socket path, and QMP is a capability.
|
||||
d /run/owuivm 0770 owuivmd owuivm -
|
||||
|
||||
# The maintenance console's socket. Separate from /run/owuivm because
|
||||
# that one is owned by vmd and deliberately unreadable to anyone else,
|
||||
# while this one has to be reachable by the operator's desktop session.
|
||||
# setgid, with the OPERATOR as the group, so the socket qemu creates
|
||||
# inside it is group-owned by the person who needs to connect to it.
|
||||
# install.sh substitutes @OPERATOR@ for whoever ran it.
|
||||
d /run/owuivm-maint 2770 owuivmadm @OPERATOR@ -
|
||||
@@ -0,0 +1,169 @@
|
||||
# =====================================================================
|
||||
# The VM sandbox. One unit, deliberately NOT a template.
|
||||
#
|
||||
# "Only one VM at a time" is a requirement, and systemd enforces it for
|
||||
# free here: a second `systemctl start owuivm.service` on an already
|
||||
# running unit is a no-op. Making this a template (owuivm@.service) would
|
||||
# move that guarantee into vmd's bookkeeping, where a crash between
|
||||
# "check" and "start" breaks it.
|
||||
#
|
||||
# THIS FILE IS THE PRIMARY ISOLATION CONTROL, NOT THE FIREWALL.
|
||||
#
|
||||
# IPAddressDeny= attaches a BPF filter to this unit's cgroup. It applies
|
||||
# to every socket the process opens, it covers loopback, and - the point -
|
||||
# `nft flush ruleset`, a ufw reload and an nftables package upgrade all
|
||||
# leave it completely untouched. 60-owuivm.nft says the same thing again
|
||||
# independently, and the negative suite is run twice: once with the nft
|
||||
# table deleted, once with these lines stripped from a scratch unit.
|
||||
# Neither layer is allowed to be the reason the suite passes.
|
||||
#
|
||||
# The guest reaches the internet and nothing else. Note that the deny on
|
||||
# 10.0.0.0/8 does not affect the guest's own 10.0.2.15 - those addresses
|
||||
# live inside libslirp and never become host packets - while the deny on
|
||||
# 127.0.0.0/8 is what closes SLIRP's host aliases 10.0.2.2 (this host's
|
||||
# loopback) and 10.0.2.3 (this host's DNS forwarder).
|
||||
# =====================================================================
|
||||
[Unit]
|
||||
Description=owuitools VM sandbox (one session at a time)
|
||||
After=network-online.target owuivm-nft-guard.service
|
||||
Wants=owuivm-nft-guard.service
|
||||
ConditionPathExists=/run/owuivm/session.env
|
||||
ConditionPathExists=/dev/kvm
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
EnvironmentFile=/run/owuivm/session.env
|
||||
ExecStart=/usr/local/libexec/owuivm-launch
|
||||
|
||||
# Best-effort teardown that survives SIGKILL of the main process, because
|
||||
# systemd runs it whatever killed qemu. vmd reaps too, on its own start;
|
||||
# neither is trusted to be the only one that ran.
|
||||
#
|
||||
# A SESSION MACHINE IS KILLED, NOT ASKED TO LEAVE.
|
||||
#
|
||||
# The maintenance unit presses the power button and waits, because the
|
||||
# disk it is holding is the base image and pulling the plug on it is how
|
||||
# a qcow2 gets corrupted. A session has neither reason: its disk is a
|
||||
# throwaway overlay that is deleted moments later, and the project share
|
||||
# has already been committed and pushed by the time anything here runs -
|
||||
# `close` publishes BEFORE it destroys.
|
||||
#
|
||||
# So the only thing an ACPI power-down would buy is a tidier-looking
|
||||
# guest shutdown, at a cost of up to ninety seconds of CPU per session
|
||||
# spent watching a machine stop caring about itself. The operator's
|
||||
# call, and the right one.
|
||||
KillSignal=SIGKILL
|
||||
ExecStopPost=/usr/local/libexec/owuivm-teardown
|
||||
|
||||
User=owuivm
|
||||
Group=owuivm
|
||||
SupplementaryGroups=kvm
|
||||
# 0007 so the sockets qemu creates in the session directory are group
|
||||
# writable and vmd - a member of group owuivm - can connect to them.
|
||||
UMask=0007
|
||||
|
||||
# --- the network boundary -------------------------------------------
|
||||
# DENY-ONLY, AND THERE MUST NEVER BE AN IPAddressAllow= LINE HERE.
|
||||
#
|
||||
# systemd.resource-control(5) applies the two lists in this order:
|
||||
# 1. granted if the address matches IPAddressAllow=
|
||||
# 2. otherwise denied if it matches IPAddressDeny=
|
||||
# 3. otherwise granted
|
||||
# There is no longest-prefix arbitration between them. An
|
||||
# `IPAddressAllow=0.0.0.0/0` intended to mean "the internet is fine" is
|
||||
# therefore matched by rule 1 for EVERY address and makes this entire
|
||||
# deny list dead. That was the first thing written here, and it was
|
||||
# measured doing nothing: with the nft table deleted, a process in this
|
||||
# cgroup reached 192.168.1.10:7200, 172.17.0.1:11434 and 127.0.0.1:3000.
|
||||
# Rule 3 is what grants the internet; it needs no help.
|
||||
IPAddressDeny=127.0.0.0/8 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 100.64.0.0/10 192.0.0.0/24 224.0.0.0/4 240.0.0.0/4
|
||||
# No AF_INET6 at all. Cheaper and more durable than listing v6 ranges
|
||||
# that do not exist on this host yet; a range nobody added is a rule that
|
||||
# silently stops matching.
|
||||
RestrictAddressFamilies=AF_UNIX AF_INET
|
||||
# qemu opens no listening socket of its own here - no hostfwd, no VNC on
|
||||
# TCP - and this asserts it rather than trusting the argv.
|
||||
SocketBindDeny=any
|
||||
|
||||
# --- devices ---------------------------------------------------------
|
||||
# PrivateDevices=yes would hide /dev/kvm, so the allowlist is explicit
|
||||
# instead. DevicePolicy=closed still permits null/zero/full/random/
|
||||
# urandom/tty, which is everything else qemu needs.
|
||||
DevicePolicy=closed
|
||||
DeviceAllow=/dev/kvm rw
|
||||
|
||||
# --- filesystem ------------------------------------------------------
|
||||
# THE SANDBOX GETS ITS OWN RESOLVER, AND THIS IS NOT COSMETIC.
|
||||
#
|
||||
# SLIRP hands every guest 10.0.2.3 as its nameserver over DHCP. That is a
|
||||
# forwarder inside the qemu process, and it relays to whatever the HOST's
|
||||
# /etc/resolv.conf says - here 127.0.0.53, systemd-resolved, which
|
||||
# forwards to the LAN's own DNS and knows every internal name in the
|
||||
# house. This unit denies 127.0.0.0/8, correctly, so that relay is dead.
|
||||
#
|
||||
# The result is a guest with working TCP to the internet and no name
|
||||
# resolution whatsoever, which presents as "no internet" and sends
|
||||
# whoever is looking at it to entirely the wrong place. It did exactly
|
||||
# that to a hand-installed image.
|
||||
#
|
||||
# Bind-mounting a resolv.conf of our own fixes it for every guest without
|
||||
# the guest being configured at all, and without the host's resolver ever
|
||||
# becoming reachable from the sandbox.
|
||||
BindReadOnlyPaths=/etc/owuivm/resolv.conf:/etc/resolv.conf
|
||||
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
ReadWritePaths=/run/owuivm @POOL_ROOT@/vmrun
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelLogs=yes
|
||||
ProtectControlGroups=yes
|
||||
ProtectClock=yes
|
||||
ProtectHostname=yes
|
||||
ProtectProc=invisible
|
||||
RestrictNamespaces=yes
|
||||
RestrictRealtime=yes
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
MemoryDenyWriteExecute=no
|
||||
NoNewPrivileges=yes
|
||||
SystemCallArchitectures=native
|
||||
SystemCallFilter=@system-service
|
||||
SystemCallErrorNumber=EPERM
|
||||
|
||||
# --- resources -------------------------------------------------------
|
||||
# 8 GB of guest RAM plus qemu's own overhead. CPUQuota is 700% and not
|
||||
# 800% on purpose: Ollama needs cores for sampling while a session runs,
|
||||
# and this host has one GPU shared with the model the agent is using.
|
||||
# MEASURED: 10G made the machine crawl, and it was not obvious why.
|
||||
#
|
||||
# The guest's RAM is a memfd with share=on (virtiofs requires it), so the
|
||||
# kernel accounts all 8 GiB of it as *shmem* in this cgroup - and with no
|
||||
# swap on this host, shmem cannot be reclaimed at all. That left about
|
||||
# 1.5 GiB for every page of ISO and disk I/O the VM does, so the cgroup
|
||||
# hit its hard limit 6,782 times in one boot (memory.events max), evicting
|
||||
# and re-reading the same installer media over and over.
|
||||
#
|
||||
# The first fix was to raise this to 14G so the page cache had room.
|
||||
# That worked - reclaim went to zero - and it was the wrong fix: it gave
|
||||
# one VM twelve gigabytes of a 58 GB host that is also running Ollama,
|
||||
# and the host started running short instead. Raising a limit to make
|
||||
# room for something that should not have been in there.
|
||||
#
|
||||
# The right fix is one line in owuivm-launch: cache=none on every drive,
|
||||
# so this VM's I/O never enters the page cache at all. What is left in
|
||||
# the cgroup is the guest's RAM and qemu itself, which is what this limit
|
||||
# was always meant to bound. 11G is 8 GiB of guest, about 1 GiB of qemu,
|
||||
# and slack.
|
||||
MemoryMax=@VM_MEMORY_MAX@
|
||||
CPUQuota=@VM_CPU_QUOTA@
|
||||
AllowedCPUs=@VM_ALLOWED_CPUS@
|
||||
TasksMax=512
|
||||
|
||||
# A guest that will not die is a guest that wedges the next caller.
|
||||
# Short, because nothing here waits for the guest any more: SIGKILL goes
|
||||
# to qemu the moment the unit is stopped and this is only the backstop
|
||||
# for a process that ignores even that.
|
||||
TimeoutStopSec=10
|
||||
KillMode=mixed
|
||||
Restart=no
|
||||
@@ -0,0 +1,9 @@
|
||||
# vmd's entire root surface. One path, a fixed verb list inside it, and
|
||||
# no wildcard that could become a different command.
|
||||
#
|
||||
# Deliberately NOT `systemctl` or `zfs` directly: either would be every
|
||||
# unit or every dataset on this host. `owuivm-ctl` is 80 lines, it is
|
||||
# array-form throughout, and the one caller-supplied value it accepts is
|
||||
# pattern-checked before it is used.
|
||||
owuivmd ALL=(root) NOPASSWD: /usr/local/sbin/owuivm-ctl
|
||||
Defaults!/usr/local/sbin/owuivm-ctl !requiretty
|
||||
@@ -0,0 +1,113 @@
|
||||
# =====================================================================
|
||||
# vmd: the privileged side of the VM sandbox.
|
||||
#
|
||||
# Runs as owuivmd, NOT root. Its four privileged needs - start and stop
|
||||
# the VM unit, create a project dataset with a quota, start and stop
|
||||
# virtiofsd - go through /usr/local/sbin/owuivm-ctl, which is the single
|
||||
# entry in its sudoers file and accepts a fixed verb list.
|
||||
#
|
||||
# ITS NETWORK IS AN ALLOW-LIST, WHICH IS THE OPPOSITE OF THE VM'S.
|
||||
#
|
||||
# The sandbox unit uses deny-only, because the guest should reach the
|
||||
# whole internet except a few ranges. vmd is the other way round: it
|
||||
# needs exactly one destination, Gitea on the LAN, and nothing else -
|
||||
# not the internet, not Ollama, not Open WebUI, not the other hosts.
|
||||
#
|
||||
# systemd.resource-control(5) checks IPAddressAllow= FIRST and grants on
|
||||
# a match, so an allow-list is written as `deny any` plus the addresses
|
||||
# that are wanted. That is the documented recipe, and it is the exact
|
||||
# inverse of the mistake that was made in owuivm.service - where an
|
||||
# `IPAddressAllow=0.0.0.0/0` silently disabled the whole deny list.
|
||||
# =====================================================================
|
||||
[Unit]
|
||||
Description=owuitools VM sandbox daemon
|
||||
After=network-online.target zfs-mount.service
|
||||
Wants=network-online.target
|
||||
# A daemon that can start a VM while the guard table is absent would be
|
||||
# a daemon that can start an unfiltered VM.
|
||||
Requires=owuivm-nft-guard.service
|
||||
After=owuivm-nft-guard.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=owuivmd
|
||||
Group=owuivmd
|
||||
SupplementaryGroups=owuivm owuitools-agent
|
||||
UMask=0007
|
||||
Environment=PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1
|
||||
EnvironmentFile=-/etc/owuivm/vmd.env
|
||||
ExecStart=/usr/bin/python3 /usr/local/lib/owuivm/vmd.py
|
||||
|
||||
# --- network: one destination -----------------------------------------
|
||||
IPAddressDeny=any
|
||||
# BOTH of these, and the second is not redundant.
|
||||
#
|
||||
# Gitea is published on 192.168.1.10:7200, and allowing only that
|
||||
# address does not work: systemd's filter is a BPF program on the
|
||||
# cgroup's packets, and by the time it sees them Docker's DNAT has
|
||||
# already rewritten the destination to the container's own address.
|
||||
# Measured - with only the .10 rule, a connect to 192.168.1.10:7200
|
||||
# times out; adding gitea's docker network makes it succeed immediately.
|
||||
#
|
||||
# The second entry is gitea_default, whose only member is the gitea
|
||||
# container. Naming the container's current address instead would be
|
||||
# narrower and would break the next time it is recreated.
|
||||
@GITEA_ALLOW@
|
||||
# The tool gateway, for the agent's web_search and read_page. The whole
|
||||
# owuitools_core subnet rather than the gw's address, for the reason
|
||||
# above: it is pinned in docker-compose.yml, and naming one container's
|
||||
# address is a rule that breaks the next time it is recreated.
|
||||
#
|
||||
# Measured the hard way: this was missing, and `web_search` inside a
|
||||
# session came back "could not reach the search service: timed out"
|
||||
# while the same call from a shell as the same user worked perfectly -
|
||||
# because a shell is not in the unit's cgroup, and the filter is on the
|
||||
# cgroup. The same shape as every other time this has bitten.
|
||||
IPAddressAllow=@TOOLS_SUBNET@
|
||||
RestrictAddressFamilies=AF_UNIX AF_INET
|
||||
# It answers on a unix socket and listens on nothing else.
|
||||
SocketBindDeny=any
|
||||
|
||||
# --- filesystem --------------------------------------------------------
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
# /var/lib/owuivm/warm holds the saved machines vmd writes and reads
|
||||
# back. Not the image directory next to it: vmd must never be able to
|
||||
# touch a base image, and this keeps that true while giving it
|
||||
# somewhere of its own.
|
||||
ReadWritePaths=/run/owuivm @POOL_ROOT@/vmprojects @POOL_ROOT@/vmrun /var/lib/owuivm/warm /var/lib/owuivm/sessions
|
||||
# /etc/sudoers.d/60-cis-logfile makes every sudo call write here. That is
|
||||
# an audit control and is not being disabled for this daemon's
|
||||
# convenience; ProtectSystem=strict just has to make room for it, or vmd
|
||||
# cannot invoke its own privileged helper at all.
|
||||
ReadWritePaths=/var/log/sudo.log
|
||||
# Explicitly NOT writable: /var/lib/owuivm/base. vmd reads the frozen
|
||||
# images to clone from them and must never be able to change one.
|
||||
ReadOnlyPaths=/var/lib/owuivm/base
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectKernelLogs=yes
|
||||
ProtectControlGroups=yes
|
||||
ProtectClock=yes
|
||||
ProtectHostname=yes
|
||||
ProtectProc=invisible
|
||||
RestrictNamespaces=yes
|
||||
RestrictRealtime=yes
|
||||
RestrictSUIDSGID=no
|
||||
LockPersonality=yes
|
||||
# NoNewPrivileges cannot be set: sudo to owuivm-ctl is how vmd starts the
|
||||
# VM at all, and sudo is setuid. The compensating control is that the
|
||||
# sudoers entry names exactly one program, and that program takes a fixed
|
||||
# verb list with a pattern-checked argument.
|
||||
NoNewPrivileges=no
|
||||
SystemCallArchitectures=native
|
||||
SystemCallFilter=@system-service
|
||||
|
||||
MemoryMax=512M
|
||||
TasksMax=128
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,19 @@
|
||||
# The resolver the SANDBOX's network sees. Not the host's.
|
||||
#
|
||||
# SLIRP gives every guest 10.0.2.3 as its nameserver via DHCP. That is a
|
||||
# forwarder inside the qemu process, and it relays to whatever the HOST's
|
||||
# /etc/resolv.conf says - which here is 127.0.0.53, systemd-resolved,
|
||||
# which forwards to the LAN's own DNS servers and knows every internal
|
||||
# name in the house.
|
||||
#
|
||||
# The sandbox denies 127.0.0.0/8, correctly, so that relay is dead: a
|
||||
# guest that takes DHCP gets working TCP to the internet and no name
|
||||
# resolution at all, which presents as "no internet" and sends people
|
||||
# looking in the wrong place.
|
||||
#
|
||||
# systemd bind-mounts this file over /etc/resolv.conf inside the VM
|
||||
# unit's mount namespace, so the forwarder relays HERE instead. The guest
|
||||
# needs no configuration, the host's resolver is still never reachable
|
||||
# from the sandbox, and nothing internal is ever asked about.
|
||||
nameserver 1.1.1.1
|
||||
nameserver 9.9.9.9
|
||||
Reference in New Issue
Block a user