331 lines
16 KiB
Bash
Executable File
331 lines
16 KiB
Bash
Executable File
#!/bin/bash
|
|
# =====================================================================
|
|
# Structural assertions about the VM sandbox's host side.
|
|
#
|
|
# THE POINT OF THIS FILE
|
|
#
|
|
# The guest cannot reach this host or the LAN, and that claim rests on
|
|
# two controls which are deliberately independent of each other:
|
|
#
|
|
# 1. IPAddressDeny= in owuivm.service - a BPF filter on the unit's
|
|
# cgroup. Survives `nft flush ruleset`, a ufw reload and an
|
|
# nftables package upgrade.
|
|
# 2. the `inet owuivm_guard` nft table - the host's usual convention,
|
|
# and the thing an operator will actually think to look at.
|
|
#
|
|
# Checking that both are loaded proves nothing useful, because in normal
|
|
# operation either one alone would hide a broken other. So this script
|
|
# turns each OFF in turn and re-runs the same probe. Both runs must pass.
|
|
#
|
|
# This is not theoretical. The first version of owuivm.service carried
|
|
# `IPAddressAllow=0.0.0.0/0` alongside the deny list, on the assumption
|
|
# that longest-prefix would let the denies win. It does not - the allow
|
|
# list is checked first and grants unconditionally - so layer 1 was
|
|
# doing nothing at all, and every check passed anyway because layer 2
|
|
# was covering for it. This script is what found that.
|
|
#
|
|
# sudo ./tests/vm_host_asserts.sh
|
|
# =====================================================================
|
|
set -uo pipefail
|
|
|
|
UNIT=/etc/systemd/system/owuivm.service
|
|
PROBE=/usr/local/libexec/owuivm-probe.py
|
|
DENY='127.0.0.0/8 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 100.64.0.0/10 192.0.0.0/24 224.0.0.0/4 240.0.0.0/4'
|
|
|
|
# Must be unreachable from the sandbox uid. These are this host's own LAN
|
|
# address, the LAN gateway, Ollama on the docker bridge, a host service on
|
|
# loopback, and the owuitools gateway.
|
|
MUST_FAIL="192.168.1.10:7200 192.168.1.1:80 172.17.0.1:11434 127.0.0.1:3000 172.22.0.10:8000"
|
|
# Must be reachable, or the suite proves only that networking is broken.
|
|
MUST_PASS="1.1.1.1:443 8.8.8.8:53"
|
|
|
|
fails=0
|
|
ok() { printf ' ok %s\n' "$*"; }
|
|
bad() { printf ' FAIL %s\n' "$*"; fails=$((fails+1)); }
|
|
|
|
[ "$(id -u)" -eq 0 ] || { echo "run as root" >&2; exit 1; }
|
|
|
|
# However this script exits, the guard goes back - and so does any base
|
|
# image this suite unfroze. An image left writable is not dangerous (vmd
|
|
# refuses to open a session on one) but it stops the agent working, and a
|
|
# test suite must not be able to do that to the machine it ran on.
|
|
restore() {
|
|
systemctl restart owuivm-nft-guard.service >/dev/null 2>&1
|
|
for img in /var/lib/owuivm/base/*.qcow2; do
|
|
[ -e "$img" ] || continue
|
|
systemctl is-active --quiet "owuivm-maint@$(basename "$img" .qcow2).service" && continue
|
|
chmod 0444 "$img" 2>/dev/null
|
|
done
|
|
}
|
|
trap restore EXIT INT TERM
|
|
|
|
install -o root -g root -m 0755 "$(dirname "$(readlink -f "$0")")/vm_probe.py" "$PROBE"
|
|
|
|
probe() { # probe <systemd-run properties...> ; prints "target verdict"
|
|
systemd-run --quiet --wait --pipe --collect --uid=owuivm "$@" \
|
|
/usr/bin/python3 "$PROBE" $MUST_FAIL $MUST_PASS 2>/dev/null
|
|
}
|
|
|
|
check_probe() { # check_probe <label> <properties...>
|
|
local label="$1"; shift
|
|
local out; out="$(probe "$@")"
|
|
local t v
|
|
for t in $MUST_FAIL; do
|
|
v=$(awk -v t="$t" '$1==t{print $2}' <<<"$out")
|
|
[ "$v" = "REACHED" ] && bad "$label: $t was REACHABLE" || ok "$label: $t blocked"
|
|
done
|
|
for t in $MUST_PASS; do
|
|
v=$(awk -v t="$t" '$1==t{print $2}' <<<"$out")
|
|
[ "$v" = "REACHED" ] && ok "$label: $t reachable (positive control)" \
|
|
|| bad "$label: $t unreachable - the probe proves nothing"
|
|
done
|
|
}
|
|
|
|
echo "== 1. the unit says what it must, and does not say what it must not"
|
|
grep -q '^IPAddressDeny=' "$UNIT" && ok "IPAddressDeny is present" || bad "IPAddressDeny missing"
|
|
grep -q '^IPAddressAllow=' "$UNIT" \
|
|
&& bad "IPAddressAllow is present - it is checked FIRST and grants unconditionally, which kills the deny list" \
|
|
|| ok "no IPAddressAllow (deny-only, as required)"
|
|
for k in 'RestrictAddressFamilies=AF_UNIX AF_INET' 'NoNewPrivileges=yes' \
|
|
'ProtectSystem=strict' 'DevicePolicy=closed' 'SocketBindDeny=any' 'User=owuivm'; do
|
|
grep -qF "$k" "$UNIT" && ok "$k" || bad "missing: $k"
|
|
done
|
|
grep -q '^PrivateDevices=' "$UNIT" && bad "PrivateDevices would hide /dev/kvm" || ok "no PrivateDevices"
|
|
|
|
echo
|
|
echo "== 2. LAYER 1 ALONE: nft guard deleted, cgroup filter only"
|
|
nft delete table inet owuivm_guard >/dev/null 2>&1
|
|
nft list table inet owuivm_guard >/dev/null 2>&1 \
|
|
&& { bad "could not delete the guard table - layer 1 is not being tested alone"; } \
|
|
|| ok "guard table removed"
|
|
check_probe "L1" --property="IPAddressDeny=$DENY"
|
|
|
|
echo
|
|
echo "== 3. LAYER 2 ALONE: guard table restored, no cgroup filter"
|
|
systemctl restart owuivm-nft-guard.service
|
|
nft list table inet owuivm_guard >/dev/null 2>&1 && ok "guard table restored" || bad "guard table did not come back"
|
|
check_probe "L2"
|
|
|
|
echo
|
|
echo "== 4. the guard table is the one we shipped"
|
|
nft list table inet owuivm_guard | grep -q 'meta skuid' && ok "skuid rule present" || bad "no skuid rule"
|
|
nft list table inet owuivm_guard | grep -q 'nfproto ipv6' && ok "ipv6 drop present" || bad "no ipv6 drop"
|
|
|
|
echo
|
|
echo "== 5. the agent cannot unfreeze a base image, by two independent means"
|
|
# The operator asked for this specifically: the agent must have to ASK,
|
|
# and asking must terminate at a person. Both halves are checked, because
|
|
# either one alone could be quietly removed and everything would still
|
|
# appear to work.
|
|
# A base image being built is legitimately writable and owned by owuivm -
|
|
# that is what building one looks like - so say so rather than reporting
|
|
# a security failure that is actually a build in progress.
|
|
# An image is legitimately writable while it is being built OR while the
|
|
# operator has it open for maintenance - that is what maintenance IS.
|
|
# Reporting either as a security failure trains people to ignore this
|
|
# suite, which is worse than not having it.
|
|
building=""
|
|
pgrep -f 'images/ubuntu/build.sh|images/windows/build.sh' >/dev/null 2>&1 && building="image build"
|
|
for img in ubuntu windows; do
|
|
systemctl is-active --quiet "owuivm-maint@$img.service" 2>/dev/null && building="maintenance boot"
|
|
done
|
|
for img in /var/lib/owuivm/base/*.qcow2; do
|
|
[ -e "$img" ] || continue
|
|
perms=$(stat -c '%a %U' "$img")
|
|
case "$perms" in
|
|
"444 owuivmadm") ok "$(basename "$img") is frozen (0444 owuivmadm)" ;;
|
|
*) if [ -n "$building" ]; then
|
|
printf ' skip %s is %s - a %s is in progress; re-run when it finishes\n' \
|
|
"$(basename "$img")" "$perms" "$building"
|
|
else
|
|
bad "$(basename "$img") is $perms - a base image must be 0444 owned by owuivmadm"
|
|
fi ;;
|
|
esac
|
|
done
|
|
# owuivm is the uid every session runs as. It must not be able to make a
|
|
# base writable even if it tried.
|
|
if sudo -u owuivm test -w /var/lib/owuivm/base 2>/dev/null; then
|
|
bad "the sandbox uid can write the base image directory"
|
|
else
|
|
ok "the sandbox uid cannot write the base image directory"
|
|
fi
|
|
# vmd is the most privileged thing the agent can reach. Its wrapper must
|
|
# have no verb for this at all - not a verb that checks something.
|
|
#
|
|
# Output is captured into a variable and matched afterwards, NOT piped
|
|
# into grep. This script runs under `set -o pipefail`, and both commands
|
|
# below are MEANT to exit non-zero - so `cmd | grep -q` fails the whole
|
|
# pipeline even when grep matched perfectly, and the test reports the
|
|
# opposite of what it found. It did exactly that on the first run.
|
|
out=$(sudo -u owuivmd sudo -n /usr/local/sbin/owuivm-ctl maint-start ubuntu 2>&1 || true)
|
|
case "$out" in
|
|
*"unknown verb"*) ok "vmd's privileged helper has no unfreeze verb" ;;
|
|
*) bad "vmd's privileged helper answered an unfreeze request: $out" ;;
|
|
esac
|
|
|
|
# polkit gates the only unit that can, on a session of the operator.
|
|
#
|
|
# THE STATE IS SAMPLED BEFORE AS WELL AS AFTER, and that is not
|
|
# defensive style - the first version of this check was not, and it
|
|
# STOPPED AN INSTALL THAT WAS ALREADY RUNNING. It read `is-active` after
|
|
# its own `start`, saw "active" because a maintenance boot had been up
|
|
# for half an hour, concluded a non-local session had started it, and
|
|
# "cleaned up" by stopping it. A test that cannot tell "I caused this"
|
|
# from "this was already true" does not just report wrongly; it acts.
|
|
before=$(systemctl is-active owuivm-maint@ubuntu.service 2>/dev/null || true)
|
|
if [ "$before" = "active" ]; then
|
|
printf ' skip a maintenance boot is already running - not touching it\n'
|
|
else
|
|
out=$(sudo -u owuivmd systemctl start owuivm-maint@ubuntu.service 2>&1 || true)
|
|
after=$(systemctl is-active owuivm-maint@ubuntu.service 2>/dev/null || true)
|
|
if [ "$after" = "active" ]; then
|
|
bad "a non-local session STARTED the maintenance unit"
|
|
systemctl stop owuivm-maint@ubuntu.service >/dev/null 2>&1
|
|
elif printf '%s' "$out" | grep -qi "access denied"; then
|
|
ok "polkit refuses the maintenance unit to a non-local session"
|
|
else
|
|
bad "the maintenance unit did not start, but not because polkit refused: $out"
|
|
fi
|
|
fi
|
|
|
|
echo
|
|
echo "== 5a. an unfrozen image stops a session, rather than being handed to one"
|
|
# WHY THIS EXISTS: the freeze used to be put back automatically when a
|
|
# maintenance boot ended, which meant two things nobody wanted - looking
|
|
# at an image required making it writable first, and the guarantee rested
|
|
# on a cleanup step that has already been seen to fail silently.
|
|
#
|
|
# Now the operator freezes and unfreezes deliberately, and this is the
|
|
# check that makes forgetting harmless: vmd will not open a session on a
|
|
# writable base. It is the only thing standing where the auto-freeze
|
|
# used to, so it is asserted here rather than assumed.
|
|
if systemctl is-active --quiet owuivm.service || \
|
|
systemctl is-active --quiet 'owuivm-maint@ubuntu.service'; then
|
|
printf ' skip a machine is running; not touching the ubuntu image\n'
|
|
elif [ ! -f /var/lib/owuivm/base/ubuntu.qcow2 ]; then
|
|
printf ' skip no ubuntu image yet\n'
|
|
else
|
|
chmod 0660 /var/lib/owuivm/base/ubuntu.qcow2
|
|
refusal=$(sudo -u owuivmd /usr/local/bin/vmctl open --os ubuntu --project smoke 2>&1 | head -2)
|
|
case "$refusal" in
|
|
*"unfrozen, so no session can start"*)
|
|
ok "a session refuses to start on a writable base, and says why" ;;
|
|
*) bad "a writable base did not produce the refusal - got: $refusal" ;;
|
|
esac
|
|
# And it must refuse BEFORE anything exists to clean up.
|
|
[ -z "$(ls /Pool/vmrun/*.qcow2 2>/dev/null)" ] \
|
|
&& ok "and it refused before creating anything" \
|
|
|| bad "an overlay was created for a session that could not run"
|
|
pgrep -u owuivm qemu-system-x86_64 >/dev/null 2>&1 \
|
|
&& bad "a qemu process was started for a refused session" \
|
|
|| ok "and no machine was started"
|
|
chmod 0444 /var/lib/owuivm/base/ubuntu.qcow2
|
|
ok "the image is frozen again ($(stat -c %a /var/lib/owuivm/base/ubuntu.qcow2))"
|
|
fi
|
|
|
|
echo
|
|
echo "== 5b. the maintenance console is not a TCP port"
|
|
# It was one, and it did not work: this unit denies 127.0.0.0/8, and
|
|
# systemd's filter checks the SOURCE address on ingress as well as the
|
|
# destination on egress - so the rule that stops the guest reaching the
|
|
# host also silently dropped the operator's own console. The fix was a
|
|
# unix socket, which is not IP, and this asserts it stays that way.
|
|
if grep -q 'vnc "unix:' /usr/local/libexec/owuivm-launch; then
|
|
ok "the console is served on a unix socket"
|
|
else
|
|
bad "the console is not on a unix socket - a loopback TCP port is dropped by this unit's own IP filter"
|
|
fi
|
|
if grep -q '^SocketBindAllow=' /etc/systemd/system/owuivm-maint@.service; then
|
|
bad "the maintenance unit still permits a listening TCP port"
|
|
else
|
|
ok "the maintenance unit binds no TCP port at all"
|
|
fi
|
|
|
|
echo
|
|
echo "== 5c. the VM keeps its I/O out of the page cache"
|
|
# MEASURED: 10G made a machine crawl. The guest's RAM is a memfd with
|
|
# share=on, so all 8 GiB of it is accounted as shmem in the unit's
|
|
# cgroup - and with no swap on this host, shmem cannot be reclaimed at
|
|
# all. That left ~1.5 GiB for every page of ISO and disk I/O, and the
|
|
# cgroup hit its hard limit 6,782 times in a single boot, evicting and
|
|
# re-reading the same installer media over and over.
|
|
#
|
|
# The fix is cache=none on every drive, NOT a bigger limit - raising it
|
|
# to 14G fixed the VM and left the host short instead. So the drives are
|
|
# what this checks first; the limit is the backstop behind them.
|
|
for f in /usr/local/libexec/owuivm-launch; do
|
|
n=$(grep -c 'cache=none' "$f")
|
|
[ "$n" -ge 3 ] && ok "every drive is O_DIRECT ($n of them)" \
|
|
|| bad "only $n drive(s) use cache=none - VM I/O will be charged to the cgroup as page cache"
|
|
done
|
|
for unit in /etc/systemd/system/owuivm.service /etc/systemd/system/owuivm-maint@.service; do
|
|
want=$(grep -oP '^MemoryMax=\K.*' "$unit" | tail -1)
|
|
mem=$(grep -oP '^Environment=OWUIVM_MEM_MB=\K[0-9]+' "$unit" | tail -1)
|
|
mem=${mem:-8192}
|
|
case "$want" in
|
|
1[1-9]G|[2-9][0-9]G) ok "$(basename "$unit"): MemoryMax=$want leaves room above ${mem}M of guest RAM" ;;
|
|
*) bad "$(basename "$unit"): MemoryMax=$want is too close to ${mem}M of unreclaimable shmem" ;;
|
|
esac
|
|
done
|
|
# THE TWO MACHINES END DIFFERENTLY, AND EACH WAY IS DELIBERATE.
|
|
#
|
|
# Maintenance holds the base image open, so pulling the plug on it is how
|
|
# a qcow2 gets corrupted - it presses the power button and waits. A
|
|
# session holds a throwaway overlay that is deleted moments later, and
|
|
# its project share has already been committed and pushed by the time the
|
|
# unit stops, so an ACPI power-down would buy a tidy-looking exit for up
|
|
# to ninety seconds of CPU per session. It is killed instead.
|
|
if systemctl show owuivm-maint@ubuntu.service -p ExecStop --value 2>/dev/null \
|
|
| grep -q owuivm-powerdown; then
|
|
ok "maintenance presses the power button and waits for the guest"
|
|
else
|
|
bad "the maintenance unit yanks power on a base image it has open for writing"
|
|
fi
|
|
if systemctl show owuivm.service -p ExecStop --value 2>/dev/null | grep -q owuivm-powerdown; then
|
|
bad "a session waits for an ACPI shutdown it has no reason to wait for"
|
|
else
|
|
ok "a session machine is killed rather than asked to leave"
|
|
fi
|
|
case "$(systemctl show owuivm.service -p KillSignal --value 2>/dev/null)" in
|
|
9) ok "and the kill is SIGKILL, not a request qemu could take its time over" ;;
|
|
*) bad "the session unit's KillSignal is not SIGKILL" ;;
|
|
esac
|
|
case "$(systemctl show owuivm.service -p TimeoutStopUSec --value 2>/dev/null)" in
|
|
[1-9]s|1[0-5]s) ok "with a short backstop ($(systemctl show owuivm.service -p TimeoutStopUSec --value))" ;;
|
|
*) bad "the session stop timeout is $(systemctl show owuivm.service -p TimeoutStopUSec --value), which is a wait nobody needs" ;;
|
|
esac
|
|
|
|
echo
|
|
echo "== 5d. the sandbox resolves names without touching the host's resolver"
|
|
# SLIRP's built-in forwarder relays to the HOST's /etc/resolv.conf, which
|
|
# is systemd-resolved on 127.0.0.53 - an address these units deny, and a
|
|
# service that knows the LAN's internal names. Without a resolver of its
|
|
# own, a guest that takes DHCP gets working TCP and no name resolution,
|
|
# which presents as "no internet" and is looked for in the wrong place.
|
|
[ -f /etc/owuivm/resolv.conf ] && ok "the sandbox has its own resolv.conf" \
|
|
|| bad "/etc/owuivm/resolv.conf is missing"
|
|
if grep -qE '^nameserver 127\.' /etc/owuivm/resolv.conf 2>/dev/null; then
|
|
bad "the sandbox's resolver points at loopback - which this unit denies"
|
|
else
|
|
ok "it points somewhere the sandbox can actually reach"
|
|
fi
|
|
for unit in /etc/systemd/system/owuivm.service /etc/systemd/system/owuivm-maint@.service; do
|
|
if grep -q 'BindReadOnlyPaths=/etc/owuivm/resolv.conf:/etc/resolv.conf' "$unit"; then
|
|
ok "$(basename "$unit"): uses it instead of the host's"
|
|
else
|
|
bad "$(basename "$unit"): SLIRP will forward DNS to the host's resolver, which is denied"
|
|
fi
|
|
done
|
|
|
|
echo
|
|
echo "== 6. no session left running"
|
|
if pgrep -u owuivm -f qemu-system-x86_64 >/dev/null; then
|
|
if [ -n "$building" ]; then printf ' skip a qemu is running - that is the %s\n' "$building"
|
|
else bad "a qemu process is still running as owuivm"; fi
|
|
else ok "no stray qemu"; fi
|
|
ls /Pool/vmrun/*.qcow2 >/dev/null 2>&1 && bad "an overlay was left behind in /Pool/vmrun" || ok "no stray overlay"
|
|
|
|
echo
|
|
if [ "$fails" -gt 0 ]; then echo "FAILED: $fails"; exit 1; fi
|
|
echo "all host assertions pass"
|