419 lines
20 KiB
Bash
Executable File
419 lines
20 KiB
Bash
Executable File
#!/bin/bash
|
|
# =====================================================================
|
|
# The one place a qemu command line is built.
|
|
#
|
|
# WHY A SCRIPT AND NOT ExecStart=
|
|
#
|
|
# The isolation argument in TUNING.md Change 59 rests on the qemu argv
|
|
# being a CONSTANT - specifically that -netdev never grows a guestfwd=,
|
|
# smb=, tftp= or bootfile=. Those four are handled inside the qemu
|
|
# process and never become host sockets, so neither the cgroup filter in
|
|
# owuivm.service nor the nft table in 60-owuivm.nft can see them. No
|
|
# filter can catch what no filter can observe; the only defence is that
|
|
# the string is fixed and a test asserts it.
|
|
#
|
|
# So everything security-relevant is literal text in this file. What
|
|
# varies between sessions is a session id and two paths, and those are
|
|
# re-validated HERE as well as in vmd - this is the point where a name
|
|
# becomes a path, and one check at the boundary is worth more than any
|
|
# number upstream (the same reasoning as matlab.py's helper filenames).
|
|
#
|
|
# WHAT IS DELIBERATELY ABSENT
|
|
#
|
|
# -sandbox QEMU's own seccomp layer. MEASURED TO BE MUTUALLY
|
|
# EXCLUSIVE with systemd's SystemCallFilter= on this
|
|
# build: under the unit's filter, qemu reports "There
|
|
# is no option group 'sandbox'" and exits 1, while the
|
|
# identical command line outside it starts fine.
|
|
# libseccomp IS linked - the two layers simply cannot
|
|
# both install a filter.
|
|
# systemd's is the one kept: it is a broader allowlist
|
|
# (@system-service excludes @mount, @swap, @module,
|
|
# @reboot and @obsolete outright), it covers the whole
|
|
# process tree rather than just qemu, and it is
|
|
# visible to `systemd-analyze security`. What is given
|
|
# up is spawn=deny, and the vector that mattered there
|
|
# - `migrate exec:` over QMP - is already closed by
|
|
# vmd exposing typed verbs and never a raw command.
|
|
# -runas `-sandbox elevateprivileges=deny` would have blocked
|
|
# setuid anyway; systemd's User= drops the uid before
|
|
# exec, which is stronger and needs no cooperation
|
|
# from qemu.
|
|
# -daemonize systemd supervises the process; a double fork would
|
|
# hide the exit status and break ExecStopPost=.
|
|
# usb passthrough, no host device reaches the guest except /dev/kvm,
|
|
# vfio, -virtfs which DevicePolicy=closed enforces.
|
|
# romfile the virtio-net PXE option ROM is dropped: a boot
|
|
# path we never use is a parser we never need.
|
|
# =====================================================================
|
|
set -euo pipefail
|
|
|
|
die() { printf 'owuivm-launch: %s\n' "$*" >&2; exit 78; } # EX_CONFIG
|
|
|
|
MODE="${OWUIVM_MODE:-session}"
|
|
SESSION="${OWUIVM_SESSION:-}"
|
|
DISK="${OWUIVM_DISK:-}"
|
|
VARS="${OWUIVM_VARS:-}"
|
|
SHARE="${OWUIVM_SHARE:-}"
|
|
# A saved machine to resume instead of booting one. See the warm-start
|
|
# comment further down; empty for every ordinary boot.
|
|
INCOMING="${OWUIVM_INCOMING:-}"
|
|
MEM_MB="${OWUIVM_MEM_MB:-8192}"
|
|
CPUS="${OWUIVM_CPUS:-8}"
|
|
RUN="/run/owuivm/$SESSION"
|
|
|
|
[[ "$SESSION" =~ ^[0-9a-f]{32}$ ]] || die "session id is not 32 hex characters"
|
|
[[ "$MEM_MB" =~ ^[0-9]{3,6}$ ]] || die "memory is not a plain number of MB"
|
|
[[ "$CPUS" =~ ^[0-9]{1,2}$ ]] || die "cpu count is not a plain number"
|
|
case "$MODE" in session|maint) ;; *) die "mode must be session or maint" ;; esac
|
|
|
|
# A session boots a throwaway overlay; maintenance boots a base disk that
|
|
# somebody has deliberately unfrozen. The read-only case is enforced by
|
|
# the file mode (base images are 0444 owuivmadm) rather than by this
|
|
# flag, so a bug here cannot make a base writable.
|
|
case "$MODE:$DISK" in
|
|
session:@POOL_ROOT@/vmrun/*.qcow2) ;;
|
|
maint:/var/lib/owuivm/base/*.qcow2) ;;
|
|
*) die "disk path is not permitted for mode $MODE: $DISK" ;;
|
|
esac
|
|
[[ "$DISK" != *..* ]] || die "disk path contains .."
|
|
[[ -f "$DISK" ]] || die "disk does not exist: $DISK"
|
|
|
|
[[ "$VARS" =~ ^/run/owuivm/[0-9a-f]{32}/OVMF_VARS\.fd$ ]] || die "vars path is not a session file"
|
|
[[ -f "$VARS" ]] || die "vars file does not exist: $VARS"
|
|
|
|
# INSTALL MEDIA, MAINTENANCE MODE ONLY.
|
|
#
|
|
# The operator installs a base image by hand: they need the machine
|
|
# booted from an ISO with a console attached, which is the same thing
|
|
# maintenance already does plus one CD-ROM. A session must never get one
|
|
# - it runs from a frozen disk that is already installed, and a bootable
|
|
# CD in front of it is just another thing that could change what boots.
|
|
#
|
|
# The request is a file beside the base image saying which ISO, rather
|
|
# than an environment variable, because the desktop tool that writes it
|
|
# runs as the operator and cannot write into /run/owuivm. The path is
|
|
# re-validated here anyway: this is where a name becomes a device.
|
|
# Which image this is, from the disk's own name. A session's overlay is
|
|
# named after its session id, so the name comes from the backing file.
|
|
if [[ "$MODE" == "maint" ]]; then
|
|
name=$(basename "$DISK" .qcow2)
|
|
else
|
|
name=$(basename "$(qemu-img info --output=json "$DISK" 2>/dev/null \
|
|
| sed -n 's/.*"backing-filename": "\([^"]*\)".*/\1/p' | head -1)" .qcow2)
|
|
fi
|
|
[[ "$name" =~ ^(ubuntu|windows)$ ]] || die "cannot tell which image this is: '$name'"
|
|
|
|
# A FROZEN MAINTENANCE BOOT IS A SESSION'S BOOT WITH A SCREEN ATTACHED.
|
|
#
|
|
# The operator's main reason to start a base image is to check that it
|
|
# comes up the way the agent will find it - and doing that must not be
|
|
# able to change it. So the disk's own mode decides what is booted:
|
|
# frozen means a throwaway overlay on top of the untouched base, exactly
|
|
# what session.py builds for an agent, and unfrozen means the base
|
|
# itself because somebody deliberately made it writable.
|
|
#
|
|
# The overlay path is DERIVED from a name this script has already
|
|
# validated, never taken from the caller - which is why the path rule
|
|
# above still only permits a base image here.
|
|
# `-w` is deliberately NOT the test. It asks "can THIS process write
|
|
# it", and root can write a 0444 file - so a maintenance boot started by
|
|
# root would silently take the other branch and open the base directly.
|
|
# Frozen is the owner write bit and nothing else, which is the same
|
|
# definition the desktop tool, vmd and the prepare script all use.
|
|
frozen_disk() { (( 8#$(stat -c %a "$1" 2>/dev/null || echo 600) & 8#200 )) && return 1 || return 0; }
|
|
|
|
if [[ "$MODE" == "maint" ]] && frozen_disk "$DISK"; then
|
|
overlay="@POOL_ROOT@/vmrun/maint-$name.qcow2"
|
|
rm -f "$overlay"
|
|
qemu-img create -f qcow2 -b "$DISK" -F qcow2 "$overlay" >/dev/null \
|
|
|| die "could not create the throwaway disk for a frozen boot: $overlay"
|
|
# qemu-img asks for 0644 and the unit's umask clears the group write
|
|
# bit it never requested, leaving a file the qemu user can read and
|
|
# not write. Here qemu IS this user, but the mode is set explicitly so
|
|
# the two callers of this line agree.
|
|
chmod 0660 "$overlay" 2>/dev/null || true
|
|
DISK="$overlay"
|
|
fi
|
|
|
|
ISO=""
|
|
if [[ "$MODE" == "maint" ]]; then
|
|
marker="/var/lib/owuivm/base/$name.install"
|
|
if [[ -f "$marker" ]]; then
|
|
ISO=$(head -c 400 "$marker" | tr -d '\r\n')
|
|
[[ "$ISO" =~ ^/var/lib/owuivm/iso/[A-Za-z0-9._-]+\.iso$ ]] \
|
|
|| die "install media must be an .iso in /var/lib/owuivm/iso: $ISO"
|
|
[[ "$ISO" != *..* ]] || die "install media path contains .."
|
|
[[ -f "$ISO" ]] || die "install media does not exist: $ISO"
|
|
fi
|
|
fi
|
|
|
|
# WARM START: RESUME A SAVED MACHINE INSTEAD OF BOOTING ONE.
|
|
#
|
|
# A Windows boot costs ninety seconds and the agent cannot hide all of it
|
|
# behind its planning turn. Resuming a saved machine is reading a file:
|
|
# measured, 3.92 GB of state written in 14.6s, and the guest comes back
|
|
# with its desktop drawn and its PowerShell already warm - which is the
|
|
# ninety seconds AND the first-command cold start, both gone.
|
|
#
|
|
# The path is checked hard because a migration stream is executable input
|
|
# in every sense that matters: it sets the machine's registers, its
|
|
# device state and its memory. It may only ever be a file this host
|
|
# wrote, in the directory only the image owner can write.
|
|
if [[ -n "$INCOMING" ]]; then
|
|
[[ "$INCOMING" =~ ^/var/lib/owuivm/warm/(ubuntu|windows)/state$ ]] \
|
|
|| die "resume state is not one of this host's own: $INCOMING"
|
|
[[ "$INCOMING" != *..* ]] || die "resume path contains .."
|
|
[[ -f "$INCOMING" ]] || die "resume state does not exist: $INCOMING"
|
|
fi
|
|
|
|
if [[ -n "$SHARE" ]]; then
|
|
[[ "$SHARE" =~ ^@POOL_ROOT@/vmprojects/[a-z0-9][a-z0-9-]{1,30}$ ]] || die "share path is not a project dataset"
|
|
[[ "$SHARE" != *..* ]] || die "share path contains .."
|
|
[[ -d "$SHARE" ]] || die "share does not exist: $SHARE"
|
|
fi
|
|
|
|
# WINDOWS 11 WILL NOT INSTALL WITHOUT SECURE BOOT AND A TPM.
|
|
#
|
|
# Its setup checks for both and stops with "This PC can't run Windows 11"
|
|
# - which reads like a hardware problem rather than a missing device
|
|
# model. Ubuntu needs neither and is given neither, so the two machines
|
|
# differ only where they have to.
|
|
#
|
|
# `smm=on` is already on the machine type above and is a prerequisite:
|
|
# without it the secure-boot firmware loads but cannot protect its own
|
|
# variable store, and the enrolled keys are decorative.
|
|
OVMF_CODE=/usr/share/OVMF/OVMF_CODE_4M.fd
|
|
TPM_ARGS=()
|
|
CPU=host
|
|
RTC_BASE=utc
|
|
|
|
if [[ "$name" == "windows" ]]; then
|
|
# HYPER-V ENLIGHTENMENTS: THE DIFFERENCE BETWEEN "SLOW" AND "FINE".
|
|
#
|
|
# Windows assumes it may be a Hyper-V guest and looks for the
|
|
# paravirtual interfaces that say so. Finding none, it behaves like a
|
|
# kernel on bare metal that has been lied to about its hardware: it
|
|
# spins on locks a hypervisor could have yielded, takes a VM exit for
|
|
# every timer tick and every inter-processor interrupt, and drives its
|
|
# APIC through a trap on each access. Measured on this host, an idle
|
|
# Windows 11 installer held qemu at 347% of one 700% quota while the
|
|
# cgroup was neither short of memory (memory.events max 0) nor
|
|
# meaningfully throttled (0.35s of throttling in three hours) - the
|
|
# work was real, it was simply wasted.
|
|
#
|
|
# Each of these tells Windows that somebody is listening:
|
|
# relaxed stop watchdogging a clock that a hypervisor may pause
|
|
# vapic/avic APIC through memory instead of a VM exit per access
|
|
# time a paravirtual reference clock
|
|
# spinlocks give up the CPU after N spins instead of burning them
|
|
# synic/stimer synthetic interrupt controller and timers, with
|
|
# stimer-direct delivering them without a round trip
|
|
# tlbflush/ipi one hypercall instead of one IPI per target CPU
|
|
# vpindex/runtime/frequencies/reset the identifiers and the reset
|
|
# path the above are defined in terms of
|
|
#
|
|
# Listed explicitly rather than `hv-passthrough`, which hands over
|
|
# whatever the host kernel happens to support: these images are frozen
|
|
# and are meant to behave the same after a host upgrade as before it.
|
|
# All of them verified accepted by this host's KVM before being added.
|
|
CPU="host,hv-relaxed,hv-vapic,hv-avic,hv-time,hv-spinlocks=0x1fff"
|
|
CPU="$CPU,hv-vpindex,hv-runtime,hv-synic,hv-stimer,hv-stimer-direct"
|
|
CPU="$CPU,hv-reset,hv-frequencies,hv-tlbflush,hv-ipi"
|
|
|
|
# Windows reads the RTC as local time and has done since DOS. With the
|
|
# host at UTC that is not a performance problem, it is a clock several
|
|
# hours wrong - which breaks TLS verification, and therefore breaks
|
|
# `pip`, `winget` and every script that fetches anything, in a way that
|
|
# reads as a network failure.
|
|
RTC_BASE=localtime
|
|
|
|
OVMF_CODE=/usr/share/OVMF/OVMF_CODE_4M.secboot.fd
|
|
[[ -f "$OVMF_CODE" ]] || die "secure-boot firmware is missing: $OVMF_CODE"
|
|
|
|
# swtpm is a separate process holding the TPM's state. It is started
|
|
# here rather than as its own unit because it must be listening before
|
|
# qemu opens the chardev, and it exits on its own when qemu goes away
|
|
# (--terminate), so there is nothing left to reap.
|
|
mkdir -p "$RUN/tpm"
|
|
/usr/bin/swtpm socket --tpm2 --terminate \
|
|
--tpmstate "dir=$RUN/tpm" \
|
|
--ctrl "type=unixio,path=$RUN/swtpm.sock" \
|
|
--log level=0 &
|
|
for _ in $(seq 1 50); do [[ -S "$RUN/swtpm.sock" ]] && break; sleep 0.1; done
|
|
[[ -S "$RUN/swtpm.sock" ]] || die "the software TPM did not start"
|
|
TPM_ARGS=(
|
|
-chardev "socket,id=chrtpm,path=$RUN/swtpm.sock"
|
|
-tpmdev emulator,id=tpm0,chardev=chrtpm
|
|
-device tpm-tis,tpmdev=tpm0
|
|
)
|
|
fi
|
|
|
|
args=(
|
|
-name "owuivm-$MODE"
|
|
-no-user-config
|
|
-nodefaults
|
|
-machine q35,accel=kvm,smm=on,vmport=off
|
|
-cpu "$CPU"
|
|
-smp "$CPUS"
|
|
-m "$MEM_MB"
|
|
-rtc "base=$RTC_BASE,driftfix=slew"
|
|
-global kvm-pit.lost_tick_policy=delay
|
|
|
|
# Shared memory is a hard requirement of vhost-user-fs; it has to be
|
|
# the machine's only memory backend, so it is set unconditionally
|
|
# rather than only when a share is attached - a guest that boots with a
|
|
# different memory topology depending on whether a project was passed
|
|
# is a guest that behaves differently in testing than in use.
|
|
-object "memory-backend-memfd,id=mem0,size=${MEM_MB}M,share=on"
|
|
-numa node,memdev=mem0
|
|
|
|
-drive "if=pflash,format=raw,unit=0,readonly=on,file=$OVMF_CODE"
|
|
-drive "if=pflash,format=raw,unit=1,file=$VARS"
|
|
|
|
# cache=none is O_DIRECT: this VM's disk I/O does not go through the
|
|
# host's page cache at all. That is not a throughput tweak, it is what
|
|
# keeps the cgroup accounting honest - see MemoryMax in the unit. The
|
|
# guest's RAM is unreclaimable shmem, so any page cache charged to the
|
|
# same cgroup is in a fight it cannot win, and the loser was the
|
|
# installer media being read from disk over and over.
|
|
# Verified O_DIRECT works on both filesystems in play here: ext4 for
|
|
# the base images, and ZFS 2.4.1 for the session overlays.
|
|
-drive "file=$DISK,if=none,id=disk0,format=qcow2,discard=unmap,cache=none,aio=native"
|
|
-device virtio-blk-pci,drive=disk0,bootindex=0
|
|
|
|
# THE LINE THE WHOLE ISOLATION ARGUMENT RESTS ON.
|
|
# user-mode networking: no tap, no bridge, no CAP_NET_ADMIN, and no
|
|
# layer-2 presence anywhere on this host. Every guest packet leaves as
|
|
# an ordinary socket owned by this process's uid, which is what makes
|
|
# both IPAddressDeny= and `meta skuid owuivm` able to see it at all.
|
|
# ipv6=off because the host has no global v6 and enumerating v6 ranges
|
|
# nobody has yet is how a filter silently stops matching.
|
|
-netdev user,id=net0,ipv6=off
|
|
-device virtio-net-pci,netdev=net0,romfile=
|
|
|
|
# virtio-vga, not the plain VGA device: a desktop on the emulated VGA
|
|
# MMIO framebuffer repaints through a slow path for every update, and
|
|
# with llvmpipe doing the rendering anyway that is the difference
|
|
# between usable and painful.
|
|
#
|
|
# Still 2D. NOT virtio-gpu-gl/virgl: QMP screendump reads the console
|
|
# surface, and a device rendering through a GL context does not give it
|
|
# one - the agent's whole GUI mode depends on that surface existing.
|
|
# virtio-vga also keeps a VGA compatibility mode, which is what the
|
|
# Windows installer uses before its drivers are loaded.
|
|
-device virtio-vga,max_outputs=1
|
|
-display none
|
|
|
|
# Absolute pointing, so a click at (x,y) lands at (x,y) whatever the
|
|
# guest's pointer acceleration is doing.
|
|
-device qemu-xhci,id=xhci
|
|
-device usb-tablet,bus=xhci.0
|
|
|
|
# A ROOT PORT THAT IS USUALLY EMPTY, AND WHY IT IS ALWAYS THERE.
|
|
#
|
|
# q35's root complex is not hot-pluggable, so a machine resumed from a
|
|
# saved state has nowhere to put the project share - and the share
|
|
# cannot be in the saved state, because virtiofsd refuses migration
|
|
# ("vhost-user backend lacks VHOST_USER_PROTOCOL_F_LOG_SHMFD").
|
|
#
|
|
# So the share lives behind this port on EVERY path: plugged in at
|
|
# startup on a cold boot, hot-plugged into the same address on a warm
|
|
# one. Same bus, same address, one device as far as the guest is
|
|
# concerned - a drive letter that moves between sessions would be worse
|
|
# than a slow boot.
|
|
#
|
|
# It is present even when no share is attached, because a saved state
|
|
# and the machine that loads it must have exactly the same devices.
|
|
-device pcie-root-port,id=vfsport,bus=pcie.0,addr=0x10,chassis=1,slot=1
|
|
|
|
# Two virtio-serial ports, no network involved in either: the standard
|
|
# guest-agent channel, and a second one for the Windows broker that
|
|
# runs in the interactive admin session (qemu-ga there is LocalSystem,
|
|
# which is not the "normal elevated PowerShell" this is meant to give).
|
|
-device virtio-serial-pci,id=vser0
|
|
-chardev "socket,id=chga,path=$RUN/ga.sock,server=on,wait=off"
|
|
-device virtserialport,chardev=chga,bus=vser0.0,name=org.qemu.guest_agent.0
|
|
-chardev "socket,id=chbr,path=$RUN/broker.sock,server=on,wait=off"
|
|
-device virtserialport,chardev=chbr,bus=vser0.0,name=com.owuitools.broker.0
|
|
|
|
# Control plane. 0600 inside a 0700 directory, because QMP is a
|
|
# capability and not a debug port: it carries human-monitor-command,
|
|
# migrate exec: and a screendump that writes a caller-chosen filename.
|
|
# vmd never forwards a command to it, only typed verbs built from
|
|
# constants.
|
|
-qmp "unix:$RUN/qmp.sock,server=on,wait=off"
|
|
)
|
|
|
|
# Secure boot is not just the firmware file: without this the variable
|
|
# store is writable from inside the guest and the whole thing is theatre.
|
|
if [[ "$name" == "windows" ]]; then
|
|
args+=( -global driver=cfi.pflash01,property=secure,value=on )
|
|
args+=( "${TPM_ARGS[@]}" )
|
|
fi
|
|
|
|
if [[ -n "$SHARE" ]]; then
|
|
args+=(
|
|
-chardev "socket,id=chfs,path=$RUN/vfs.sock"
|
|
-device vhost-user-fs-pci,chardev=chfs,tag=project,bus=vfsport,addr=0x0
|
|
)
|
|
fi
|
|
|
|
if [[ -n "$INCOMING" ]]; then
|
|
# `file:` rather than `exec:`. Measured: `exec:` fails with "Unable to
|
|
# write to command: Broken pipe", because this unit's syscall filter
|
|
# and NoNewPrivileges stop qemu spawning the helper - which is the
|
|
# sandbox doing its job. `file:` opens a file and needs nobody.
|
|
args+=( -incoming "file:$INCOMING" )
|
|
fi
|
|
|
|
if [[ "$MODE" == "maint" ]]; then
|
|
# THE CONSOLE IS A UNIX SOCKET, NOT A TCP PORT, AND THAT IS NOT STYLE.
|
|
#
|
|
# systemd's IPAddressDeny= filters INGRESS as well as egress, checking
|
|
# the SOURCE address on the way in. This unit denies 127.0.0.0/8 -
|
|
# which is what stops the guest reaching the host's own loopback
|
|
# through SLIRP's 10.0.2.2 alias - and the source of a connection to a
|
|
# loopback VNC port is, of course, 127.0.0.1. So the same rule that
|
|
# protects the host silently dropped the operator's own console: the
|
|
# port listened, `query-vnc` reported it enabled, and every client sat
|
|
# there until it timed out. Measured both ways: with the deny list
|
|
# cleared, the server greets with RFB 003.008 immediately.
|
|
#
|
|
# A unix socket is not IP, so the filter does not apply to it, and the
|
|
# guest's egress policy stays exactly as strict as a session's. The
|
|
# desktop tool relays it to a loopback port for the viewer, from a
|
|
# process outside this cgroup.
|
|
args+=( -vnc "unix:/run/owuivm-maint/vnc.sock" )
|
|
if [[ -n "$ISO" ]]; then
|
|
# The DISK is tried first and the media second, which is the opposite
|
|
# of the obvious arrangement and is deliberate.
|
|
#
|
|
# UEFI walks its boot entries in order, so a blank or half-installed
|
|
# disk has nothing to boot and falls through to the installer by
|
|
# itself, while a finished install boots itself and ignores the media
|
|
# still sitting in the drive. That makes an interrupted install
|
|
# resumable without anyone having to decide whether it finished -
|
|
# which is exactly the judgement that went wrong before: a partial
|
|
# install had written 3.45 GB, was declared complete on size alone,
|
|
# lost its media, and came up in an EFI shell with no way back.
|
|
args+=(
|
|
-drive "file=$ISO,if=none,id=cd0,media=cdrom,readonly=on,cache=none,aio=native"
|
|
-device ide-cd,drive=cd0,bus=ide.0,bootindex=1
|
|
)
|
|
# A SECOND disc, for Windows only: its setup cannot see a virtio-blk
|
|
# disk and will report that there are no drives to install to, which
|
|
# looks like a broken VM rather than a missing driver. Point setup's
|
|
# "Load driver" at this disc, amd64\w11.
|
|
virtio=/var/lib/owuivm/iso/virtio-win.iso
|
|
if [[ "$name" == "windows" && -f "$virtio" ]]; then
|
|
args+=(
|
|
-drive "file=$virtio,if=none,id=cd1,media=cdrom,readonly=on,cache=none,aio=native"
|
|
-device ide-cd,drive=cd1,bus=ide.1
|
|
)
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
exec /usr/bin/qemu-system-x86_64 "${args[@]}"
|