#!/bin/bash # ===================================================================== # The one place a qemu command line is built. # # WHY A SCRIPT AND NOT ExecStart= # # The isolation argument in TUNING.md Change 59 rests on the qemu argv # being a CONSTANT - specifically that -netdev never grows a guestfwd=, # smb=, tftp= or bootfile=. Those four are handled inside the qemu # process and never become host sockets, so neither the cgroup filter in # owuivm.service nor the nft table in 60-owuivm.nft can see them. No # filter can catch what no filter can observe; the only defence is that # the string is fixed and a test asserts it. # # So everything security-relevant is literal text in this file. What # varies between sessions is a session id and two paths, and those are # re-validated HERE as well as in vmd - this is the point where a name # becomes a path, and one check at the boundary is worth more than any # number upstream (the same reasoning as matlab.py's helper filenames). # # WHAT IS DELIBERATELY ABSENT # # -sandbox QEMU's own seccomp layer. MEASURED TO BE MUTUALLY # EXCLUSIVE with systemd's SystemCallFilter= on this # build: under the unit's filter, qemu reports "There # is no option group 'sandbox'" and exits 1, while the # identical command line outside it starts fine. # libseccomp IS linked - the two layers simply cannot # both install a filter. # systemd's is the one kept: it is a broader allowlist # (@system-service excludes @mount, @swap, @module, # @reboot and @obsolete outright), it covers the whole # process tree rather than just qemu, and it is # visible to `systemd-analyze security`. What is given # up is spawn=deny, and the vector that mattered there # - `migrate exec:` over QMP - is already closed by # vmd exposing typed verbs and never a raw command. # -runas `-sandbox elevateprivileges=deny` would have blocked # setuid anyway; systemd's User= drops the uid before # exec, which is stronger and needs no cooperation # from qemu. # -daemonize systemd supervises the process; a double fork would # hide the exit status and break ExecStopPost=. # usb passthrough, no host device reaches the guest except /dev/kvm, # vfio, -virtfs which DevicePolicy=closed enforces. # romfile the virtio-net PXE option ROM is dropped: a boot # path we never use is a parser we never need. # ===================================================================== set -euo pipefail die() { printf 'owuivm-launch: %s\n' "$*" >&2; exit 78; } # EX_CONFIG MODE="${OWUIVM_MODE:-session}" SESSION="${OWUIVM_SESSION:-}" DISK="${OWUIVM_DISK:-}" VARS="${OWUIVM_VARS:-}" SHARE="${OWUIVM_SHARE:-}" # A saved machine to resume instead of booting one. See the warm-start # comment further down; empty for every ordinary boot. INCOMING="${OWUIVM_INCOMING:-}" MEM_MB="${OWUIVM_MEM_MB:-8192}" CPUS="${OWUIVM_CPUS:-8}" RUN="/run/owuivm/$SESSION" [[ "$SESSION" =~ ^[0-9a-f]{32}$ ]] || die "session id is not 32 hex characters" [[ "$MEM_MB" =~ ^[0-9]{3,6}$ ]] || die "memory is not a plain number of MB" [[ "$CPUS" =~ ^[0-9]{1,2}$ ]] || die "cpu count is not a plain number" case "$MODE" in session|maint) ;; *) die "mode must be session or maint" ;; esac # A session boots a throwaway overlay; maintenance boots a base disk that # somebody has deliberately unfrozen. The read-only case is enforced by # the file mode (base images are 0444 owuivmadm) rather than by this # flag, so a bug here cannot make a base writable. case "$MODE:$DISK" in session:@POOL_ROOT@/vmrun/*.qcow2) ;; maint:/var/lib/owuivm/base/*.qcow2) ;; *) die "disk path is not permitted for mode $MODE: $DISK" ;; esac [[ "$DISK" != *..* ]] || die "disk path contains .." [[ -f "$DISK" ]] || die "disk does not exist: $DISK" [[ "$VARS" =~ ^/run/owuivm/[0-9a-f]{32}/OVMF_VARS\.fd$ ]] || die "vars path is not a session file" [[ -f "$VARS" ]] || die "vars file does not exist: $VARS" # INSTALL MEDIA, MAINTENANCE MODE ONLY. # # The operator installs a base image by hand: they need the machine # booted from an ISO with a console attached, which is the same thing # maintenance already does plus one CD-ROM. A session must never get one # - it runs from a frozen disk that is already installed, and a bootable # CD in front of it is just another thing that could change what boots. # # The request is a file beside the base image saying which ISO, rather # than an environment variable, because the desktop tool that writes it # runs as the operator and cannot write into /run/owuivm. The path is # re-validated here anyway: this is where a name becomes a device. # Which image this is, from the disk's own name. A session's overlay is # named after its session id, so the name comes from the backing file. if [[ "$MODE" == "maint" ]]; then name=$(basename "$DISK" .qcow2) else name=$(basename "$(qemu-img info --output=json "$DISK" 2>/dev/null \ | sed -n 's/.*"backing-filename": "\([^"]*\)".*/\1/p' | head -1)" .qcow2) fi [[ "$name" =~ ^(ubuntu|windows)$ ]] || die "cannot tell which image this is: '$name'" # A FROZEN MAINTENANCE BOOT IS A SESSION'S BOOT WITH A SCREEN ATTACHED. # # The operator's main reason to start a base image is to check that it # comes up the way the agent will find it - and doing that must not be # able to change it. So the disk's own mode decides what is booted: # frozen means a throwaway overlay on top of the untouched base, exactly # what session.py builds for an agent, and unfrozen means the base # itself because somebody deliberately made it writable. # # The overlay path is DERIVED from a name this script has already # validated, never taken from the caller - which is why the path rule # above still only permits a base image here. # `-w` is deliberately NOT the test. It asks "can THIS process write # it", and root can write a 0444 file - so a maintenance boot started by # root would silently take the other branch and open the base directly. # Frozen is the owner write bit and nothing else, which is the same # definition the desktop tool, vmd and the prepare script all use. frozen_disk() { (( 8#$(stat -c %a "$1" 2>/dev/null || echo 600) & 8#200 )) && return 1 || return 0; } if [[ "$MODE" == "maint" ]] && frozen_disk "$DISK"; then overlay="@POOL_ROOT@/vmrun/maint-$name.qcow2" rm -f "$overlay" qemu-img create -f qcow2 -b "$DISK" -F qcow2 "$overlay" >/dev/null \ || die "could not create the throwaway disk for a frozen boot: $overlay" # qemu-img asks for 0644 and the unit's umask clears the group write # bit it never requested, leaving a file the qemu user can read and # not write. Here qemu IS this user, but the mode is set explicitly so # the two callers of this line agree. chmod 0660 "$overlay" 2>/dev/null || true DISK="$overlay" fi ISO="" if [[ "$MODE" == "maint" ]]; then marker="/var/lib/owuivm/base/$name.install" if [[ -f "$marker" ]]; then ISO=$(head -c 400 "$marker" | tr -d '\r\n') [[ "$ISO" =~ ^/var/lib/owuivm/iso/[A-Za-z0-9._-]+\.iso$ ]] \ || die "install media must be an .iso in /var/lib/owuivm/iso: $ISO" [[ "$ISO" != *..* ]] || die "install media path contains .." [[ -f "$ISO" ]] || die "install media does not exist: $ISO" fi fi # WARM START: RESUME A SAVED MACHINE INSTEAD OF BOOTING ONE. # # A Windows boot costs ninety seconds and the agent cannot hide all of it # behind its planning turn. Resuming a saved machine is reading a file: # measured, 3.92 GB of state written in 14.6s, and the guest comes back # with its desktop drawn and its PowerShell already warm - which is the # ninety seconds AND the first-command cold start, both gone. # # The path is checked hard because a migration stream is executable input # in every sense that matters: it sets the machine's registers, its # device state and its memory. It may only ever be a file this host # wrote, in the directory only the image owner can write. if [[ -n "$INCOMING" ]]; then [[ "$INCOMING" =~ ^/var/lib/owuivm/warm/(ubuntu|windows)/state$ ]] \ || die "resume state is not one of this host's own: $INCOMING" [[ "$INCOMING" != *..* ]] || die "resume path contains .." [[ -f "$INCOMING" ]] || die "resume state does not exist: $INCOMING" fi if [[ -n "$SHARE" ]]; then [[ "$SHARE" =~ ^@POOL_ROOT@/vmprojects/[a-z0-9][a-z0-9-]{1,30}$ ]] || die "share path is not a project dataset" [[ "$SHARE" != *..* ]] || die "share path contains .." [[ -d "$SHARE" ]] || die "share does not exist: $SHARE" fi # WINDOWS 11 WILL NOT INSTALL WITHOUT SECURE BOOT AND A TPM. # # Its setup checks for both and stops with "This PC can't run Windows 11" # - which reads like a hardware problem rather than a missing device # model. Ubuntu needs neither and is given neither, so the two machines # differ only where they have to. # # `smm=on` is already on the machine type above and is a prerequisite: # without it the secure-boot firmware loads but cannot protect its own # variable store, and the enrolled keys are decorative. OVMF_CODE=/usr/share/OVMF/OVMF_CODE_4M.fd TPM_ARGS=() CPU=host RTC_BASE=utc if [[ "$name" == "windows" ]]; then # HYPER-V ENLIGHTENMENTS: THE DIFFERENCE BETWEEN "SLOW" AND "FINE". # # Windows assumes it may be a Hyper-V guest and looks for the # paravirtual interfaces that say so. Finding none, it behaves like a # kernel on bare metal that has been lied to about its hardware: it # spins on locks a hypervisor could have yielded, takes a VM exit for # every timer tick and every inter-processor interrupt, and drives its # APIC through a trap on each access. Measured on this host, an idle # Windows 11 installer held qemu at 347% of one 700% quota while the # cgroup was neither short of memory (memory.events max 0) nor # meaningfully throttled (0.35s of throttling in three hours) - the # work was real, it was simply wasted. # # Each of these tells Windows that somebody is listening: # relaxed stop watchdogging a clock that a hypervisor may pause # vapic/avic APIC through memory instead of a VM exit per access # time a paravirtual reference clock # spinlocks give up the CPU after N spins instead of burning them # synic/stimer synthetic interrupt controller and timers, with # stimer-direct delivering them without a round trip # tlbflush/ipi one hypercall instead of one IPI per target CPU # vpindex/runtime/frequencies/reset the identifiers and the reset # path the above are defined in terms of # # Listed explicitly rather than `hv-passthrough`, which hands over # whatever the host kernel happens to support: these images are frozen # and are meant to behave the same after a host upgrade as before it. # All of them verified accepted by this host's KVM before being added. CPU="host,hv-relaxed,hv-vapic,hv-avic,hv-time,hv-spinlocks=0x1fff" CPU="$CPU,hv-vpindex,hv-runtime,hv-synic,hv-stimer,hv-stimer-direct" CPU="$CPU,hv-reset,hv-frequencies,hv-tlbflush,hv-ipi" # Windows reads the RTC as local time and has done since DOS. With the # host at UTC that is not a performance problem, it is a clock several # hours wrong - which breaks TLS verification, and therefore breaks # `pip`, `winget` and every script that fetches anything, in a way that # reads as a network failure. RTC_BASE=localtime OVMF_CODE=/usr/share/OVMF/OVMF_CODE_4M.secboot.fd [[ -f "$OVMF_CODE" ]] || die "secure-boot firmware is missing: $OVMF_CODE" # swtpm is a separate process holding the TPM's state. It is started # here rather than as its own unit because it must be listening before # qemu opens the chardev, and it exits on its own when qemu goes away # (--terminate), so there is nothing left to reap. mkdir -p "$RUN/tpm" /usr/bin/swtpm socket --tpm2 --terminate \ --tpmstate "dir=$RUN/tpm" \ --ctrl "type=unixio,path=$RUN/swtpm.sock" \ --log level=0 & for _ in $(seq 1 50); do [[ -S "$RUN/swtpm.sock" ]] && break; sleep 0.1; done [[ -S "$RUN/swtpm.sock" ]] || die "the software TPM did not start" TPM_ARGS=( -chardev "socket,id=chrtpm,path=$RUN/swtpm.sock" -tpmdev emulator,id=tpm0,chardev=chrtpm -device tpm-tis,tpmdev=tpm0 ) fi args=( -name "owuivm-$MODE" -no-user-config -nodefaults -machine q35,accel=kvm,smm=on,vmport=off -cpu "$CPU" -smp "$CPUS" -m "$MEM_MB" -rtc "base=$RTC_BASE,driftfix=slew" -global kvm-pit.lost_tick_policy=delay # Shared memory is a hard requirement of vhost-user-fs; it has to be # the machine's only memory backend, so it is set unconditionally # rather than only when a share is attached - a guest that boots with a # different memory topology depending on whether a project was passed # is a guest that behaves differently in testing than in use. -object "memory-backend-memfd,id=mem0,size=${MEM_MB}M,share=on" -numa node,memdev=mem0 -drive "if=pflash,format=raw,unit=0,readonly=on,file=$OVMF_CODE" -drive "if=pflash,format=raw,unit=1,file=$VARS" # cache=none is O_DIRECT: this VM's disk I/O does not go through the # host's page cache at all. That is not a throughput tweak, it is what # keeps the cgroup accounting honest - see MemoryMax in the unit. The # guest's RAM is unreclaimable shmem, so any page cache charged to the # same cgroup is in a fight it cannot win, and the loser was the # installer media being read from disk over and over. # Verified O_DIRECT works on both filesystems in play here: ext4 for # the base images, and ZFS 2.4.1 for the session overlays. -drive "file=$DISK,if=none,id=disk0,format=qcow2,discard=unmap,cache=none,aio=native" -device virtio-blk-pci,drive=disk0,bootindex=0 # THE LINE THE WHOLE ISOLATION ARGUMENT RESTS ON. # user-mode networking: no tap, no bridge, no CAP_NET_ADMIN, and no # layer-2 presence anywhere on this host. Every guest packet leaves as # an ordinary socket owned by this process's uid, which is what makes # both IPAddressDeny= and `meta skuid owuivm` able to see it at all. # ipv6=off because the host has no global v6 and enumerating v6 ranges # nobody has yet is how a filter silently stops matching. -netdev user,id=net0,ipv6=off -device virtio-net-pci,netdev=net0,romfile= # virtio-vga, not the plain VGA device: a desktop on the emulated VGA # MMIO framebuffer repaints through a slow path for every update, and # with llvmpipe doing the rendering anyway that is the difference # between usable and painful. # # Still 2D. NOT virtio-gpu-gl/virgl: QMP screendump reads the console # surface, and a device rendering through a GL context does not give it # one - the agent's whole GUI mode depends on that surface existing. # virtio-vga also keeps a VGA compatibility mode, which is what the # Windows installer uses before its drivers are loaded. -device virtio-vga,max_outputs=1 -display none # Absolute pointing, so a click at (x,y) lands at (x,y) whatever the # guest's pointer acceleration is doing. -device qemu-xhci,id=xhci -device usb-tablet,bus=xhci.0 # A ROOT PORT THAT IS USUALLY EMPTY, AND WHY IT IS ALWAYS THERE. # # q35's root complex is not hot-pluggable, so a machine resumed from a # saved state has nowhere to put the project share - and the share # cannot be in the saved state, because virtiofsd refuses migration # ("vhost-user backend lacks VHOST_USER_PROTOCOL_F_LOG_SHMFD"). # # So the share lives behind this port on EVERY path: plugged in at # startup on a cold boot, hot-plugged into the same address on a warm # one. Same bus, same address, one device as far as the guest is # concerned - a drive letter that moves between sessions would be worse # than a slow boot. # # It is present even when no share is attached, because a saved state # and the machine that loads it must have exactly the same devices. -device pcie-root-port,id=vfsport,bus=pcie.0,addr=0x10,chassis=1,slot=1 # Two virtio-serial ports, no network involved in either: the standard # guest-agent channel, and a second one for the Windows broker that # runs in the interactive admin session (qemu-ga there is LocalSystem, # which is not the "normal elevated PowerShell" this is meant to give). -device virtio-serial-pci,id=vser0 -chardev "socket,id=chga,path=$RUN/ga.sock,server=on,wait=off" -device virtserialport,chardev=chga,bus=vser0.0,name=org.qemu.guest_agent.0 -chardev "socket,id=chbr,path=$RUN/broker.sock,server=on,wait=off" -device virtserialport,chardev=chbr,bus=vser0.0,name=com.owuitools.broker.0 # Control plane. 0600 inside a 0700 directory, because QMP is a # capability and not a debug port: it carries human-monitor-command, # migrate exec: and a screendump that writes a caller-chosen filename. # vmd never forwards a command to it, only typed verbs built from # constants. -qmp "unix:$RUN/qmp.sock,server=on,wait=off" ) # Secure boot is not just the firmware file: without this the variable # store is writable from inside the guest and the whole thing is theatre. if [[ "$name" == "windows" ]]; then args+=( -global driver=cfi.pflash01,property=secure,value=on ) args+=( "${TPM_ARGS[@]}" ) fi if [[ -n "$SHARE" ]]; then args+=( -chardev "socket,id=chfs,path=$RUN/vfs.sock" -device vhost-user-fs-pci,chardev=chfs,tag=project,bus=vfsport,addr=0x0 ) fi if [[ -n "$INCOMING" ]]; then # `file:` rather than `exec:`. Measured: `exec:` fails with "Unable to # write to command: Broken pipe", because this unit's syscall filter # and NoNewPrivileges stop qemu spawning the helper - which is the # sandbox doing its job. `file:` opens a file and needs nobody. args+=( -incoming "file:$INCOMING" ) fi if [[ "$MODE" == "maint" ]]; then # THE CONSOLE IS A UNIX SOCKET, NOT A TCP PORT, AND THAT IS NOT STYLE. # # systemd's IPAddressDeny= filters INGRESS as well as egress, checking # the SOURCE address on the way in. This unit denies 127.0.0.0/8 - # which is what stops the guest reaching the host's own loopback # through SLIRP's 10.0.2.2 alias - and the source of a connection to a # loopback VNC port is, of course, 127.0.0.1. So the same rule that # protects the host silently dropped the operator's own console: the # port listened, `query-vnc` reported it enabled, and every client sat # there until it timed out. Measured both ways: with the deny list # cleared, the server greets with RFB 003.008 immediately. # # A unix socket is not IP, so the filter does not apply to it, and the # guest's egress policy stays exactly as strict as a session's. The # desktop tool relays it to a loopback port for the viewer, from a # process outside this cgroup. args+=( -vnc "unix:/run/owuivm-maint/vnc.sock" ) if [[ -n "$ISO" ]]; then # The DISK is tried first and the media second, which is the opposite # of the obvious arrangement and is deliberate. # # UEFI walks its boot entries in order, so a blank or half-installed # disk has nothing to boot and falls through to the installer by # itself, while a finished install boots itself and ignores the media # still sitting in the drive. That makes an interrupted install # resumable without anyone having to decide whether it finished - # which is exactly the judgement that went wrong before: a partial # install had written 3.45 GB, was declared complete on size alone, # lost its media, and came up in an EFI shell with no way back. args+=( -drive "file=$ISO,if=none,id=cd0,media=cdrom,readonly=on,cache=none,aio=native" -device ide-cd,drive=cd0,bus=ide.0,bootindex=1 ) # A SECOND disc, for Windows only: its setup cannot see a virtio-blk # disk and will report that there are no drives to install to, which # looks like a broken VM rather than a missing driver. Point setup's # "Load driver" at this disc, amd64\w11. virtio=/var/lib/owuivm/iso/virtio-win.iso if [[ "$name" == "windows" && -f "$virtio" ]]; then args+=( -drive "file=$virtio,if=none,id=cd1,media=cdrom,readonly=on,cache=none,aio=native" -device ide-cd,drive=cd1,bus=ide.1 ) fi fi fi exec /usr/bin/qemu-system-x86_64 "${args[@]}"