170 lines
7.6 KiB
Desktop File
170 lines
7.6 KiB
Desktop File
# =====================================================================
|
|
# The VM sandbox. One unit, deliberately NOT a template.
|
|
#
|
|
# "Only one VM at a time" is a requirement, and systemd enforces it for
|
|
# free here: a second `systemctl start owuivm.service` on an already
|
|
# running unit is a no-op. Making this a template (owuivm@.service) would
|
|
# move that guarantee into vmd's bookkeeping, where a crash between
|
|
# "check" and "start" breaks it.
|
|
#
|
|
# THIS FILE IS THE PRIMARY ISOLATION CONTROL, NOT THE FIREWALL.
|
|
#
|
|
# IPAddressDeny= attaches a BPF filter to this unit's cgroup. It applies
|
|
# to every socket the process opens, it covers loopback, and - the point -
|
|
# `nft flush ruleset`, a ufw reload and an nftables package upgrade all
|
|
# leave it completely untouched. 60-owuivm.nft says the same thing again
|
|
# independently, and the negative suite is run twice: once with the nft
|
|
# table deleted, once with these lines stripped from a scratch unit.
|
|
# Neither layer is allowed to be the reason the suite passes.
|
|
#
|
|
# The guest reaches the internet and nothing else. Note that the deny on
|
|
# 10.0.0.0/8 does not affect the guest's own 10.0.2.15 - those addresses
|
|
# live inside libslirp and never become host packets - while the deny on
|
|
# 127.0.0.0/8 is what closes SLIRP's host aliases 10.0.2.2 (this host's
|
|
# loopback) and 10.0.2.3 (this host's DNS forwarder).
|
|
# =====================================================================
|
|
[Unit]
|
|
Description=owuitools VM sandbox (one session at a time)
|
|
After=network-online.target owuivm-nft-guard.service
|
|
Wants=owuivm-nft-guard.service
|
|
ConditionPathExists=/run/owuivm/session.env
|
|
ConditionPathExists=/dev/kvm
|
|
|
|
[Service]
|
|
Type=simple
|
|
EnvironmentFile=/run/owuivm/session.env
|
|
ExecStart=/usr/local/libexec/owuivm-launch
|
|
|
|
# Best-effort teardown that survives SIGKILL of the main process, because
|
|
# systemd runs it whatever killed qemu. vmd reaps too, on its own start;
|
|
# neither is trusted to be the only one that ran.
|
|
#
|
|
# A SESSION MACHINE IS KILLED, NOT ASKED TO LEAVE.
|
|
#
|
|
# The maintenance unit presses the power button and waits, because the
|
|
# disk it is holding is the base image and pulling the plug on it is how
|
|
# a qcow2 gets corrupted. A session has neither reason: its disk is a
|
|
# throwaway overlay that is deleted moments later, and the project share
|
|
# has already been committed and pushed by the time anything here runs -
|
|
# `close` publishes BEFORE it destroys.
|
|
#
|
|
# So the only thing an ACPI power-down would buy is a tidier-looking
|
|
# guest shutdown, at a cost of up to ninety seconds of CPU per session
|
|
# spent watching a machine stop caring about itself. The operator's
|
|
# call, and the right one.
|
|
KillSignal=SIGKILL
|
|
ExecStopPost=/usr/local/libexec/owuivm-teardown
|
|
|
|
User=owuivm
|
|
Group=owuivm
|
|
SupplementaryGroups=kvm
|
|
# 0007 so the sockets qemu creates in the session directory are group
|
|
# writable and vmd - a member of group owuivm - can connect to them.
|
|
UMask=0007
|
|
|
|
# --- the network boundary -------------------------------------------
|
|
# DENY-ONLY, AND THERE MUST NEVER BE AN IPAddressAllow= LINE HERE.
|
|
#
|
|
# systemd.resource-control(5) applies the two lists in this order:
|
|
# 1. granted if the address matches IPAddressAllow=
|
|
# 2. otherwise denied if it matches IPAddressDeny=
|
|
# 3. otherwise granted
|
|
# There is no longest-prefix arbitration between them. An
|
|
# `IPAddressAllow=0.0.0.0/0` intended to mean "the internet is fine" is
|
|
# therefore matched by rule 1 for EVERY address and makes this entire
|
|
# deny list dead. That was the first thing written here, and it was
|
|
# measured doing nothing: with the nft table deleted, a process in this
|
|
# cgroup reached 192.168.1.10:7200, 172.17.0.1:11434 and 127.0.0.1:3000.
|
|
# Rule 3 is what grants the internet; it needs no help.
|
|
IPAddressDeny=127.0.0.0/8 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 100.64.0.0/10 192.0.0.0/24 224.0.0.0/4 240.0.0.0/4
|
|
# No AF_INET6 at all. Cheaper and more durable than listing v6 ranges
|
|
# that do not exist on this host yet; a range nobody added is a rule that
|
|
# silently stops matching.
|
|
RestrictAddressFamilies=AF_UNIX AF_INET
|
|
# qemu opens no listening socket of its own here - no hostfwd, no VNC on
|
|
# TCP - and this asserts it rather than trusting the argv.
|
|
SocketBindDeny=any
|
|
|
|
# --- devices ---------------------------------------------------------
|
|
# PrivateDevices=yes would hide /dev/kvm, so the allowlist is explicit
|
|
# instead. DevicePolicy=closed still permits null/zero/full/random/
|
|
# urandom/tty, which is everything else qemu needs.
|
|
DevicePolicy=closed
|
|
DeviceAllow=/dev/kvm rw
|
|
|
|
# --- filesystem ------------------------------------------------------
|
|
# THE SANDBOX GETS ITS OWN RESOLVER, AND THIS IS NOT COSMETIC.
|
|
#
|
|
# SLIRP hands every guest 10.0.2.3 as its nameserver over DHCP. That is a
|
|
# forwarder inside the qemu process, and it relays to whatever the HOST's
|
|
# /etc/resolv.conf says - here 127.0.0.53, systemd-resolved, which
|
|
# forwards to the LAN's own DNS and knows every internal name in the
|
|
# house. This unit denies 127.0.0.0/8, correctly, so that relay is dead.
|
|
#
|
|
# The result is a guest with working TCP to the internet and no name
|
|
# resolution whatsoever, which presents as "no internet" and sends
|
|
# whoever is looking at it to entirely the wrong place. It did exactly
|
|
# that to a hand-installed image.
|
|
#
|
|
# Bind-mounting a resolv.conf of our own fixes it for every guest without
|
|
# the guest being configured at all, and without the host's resolver ever
|
|
# becoming reachable from the sandbox.
|
|
BindReadOnlyPaths=/etc/owuivm/resolv.conf:/etc/resolv.conf
|
|
|
|
ProtectSystem=strict
|
|
ProtectHome=yes
|
|
PrivateTmp=yes
|
|
ReadWritePaths=/run/owuivm @POOL_ROOT@/vmrun
|
|
ProtectKernelTunables=yes
|
|
ProtectKernelLogs=yes
|
|
ProtectControlGroups=yes
|
|
ProtectClock=yes
|
|
ProtectHostname=yes
|
|
ProtectProc=invisible
|
|
RestrictNamespaces=yes
|
|
RestrictRealtime=yes
|
|
RestrictSUIDSGID=yes
|
|
LockPersonality=yes
|
|
MemoryDenyWriteExecute=no
|
|
NoNewPrivileges=yes
|
|
SystemCallArchitectures=native
|
|
SystemCallFilter=@system-service
|
|
SystemCallErrorNumber=EPERM
|
|
|
|
# --- resources -------------------------------------------------------
|
|
# 8 GB of guest RAM plus qemu's own overhead. CPUQuota is 700% and not
|
|
# 800% on purpose: Ollama needs cores for sampling while a session runs,
|
|
# and this host has one GPU shared with the model the agent is using.
|
|
# MEASURED: 10G made the machine crawl, and it was not obvious why.
|
|
#
|
|
# The guest's RAM is a memfd with share=on (virtiofs requires it), so the
|
|
# kernel accounts all 8 GiB of it as *shmem* in this cgroup - and with no
|
|
# swap on this host, shmem cannot be reclaimed at all. That left about
|
|
# 1.5 GiB for every page of ISO and disk I/O the VM does, so the cgroup
|
|
# hit its hard limit 6,782 times in one boot (memory.events max), evicting
|
|
# and re-reading the same installer media over and over.
|
|
#
|
|
# The first fix was to raise this to 14G so the page cache had room.
|
|
# That worked - reclaim went to zero - and it was the wrong fix: it gave
|
|
# one VM twelve gigabytes of a 58 GB host that is also running Ollama,
|
|
# and the host started running short instead. Raising a limit to make
|
|
# room for something that should not have been in there.
|
|
#
|
|
# The right fix is one line in owuivm-launch: cache=none on every drive,
|
|
# so this VM's I/O never enters the page cache at all. What is left in
|
|
# the cgroup is the guest's RAM and qemu itself, which is what this limit
|
|
# was always meant to bound. 11G is 8 GiB of guest, about 1 GiB of qemu,
|
|
# and slack.
|
|
MemoryMax=@VM_MEMORY_MAX@
|
|
CPUQuota=@VM_CPU_QUOTA@
|
|
AllowedCPUs=@VM_ALLOWED_CPUS@
|
|
TasksMax=512
|
|
|
|
# A guest that will not die is a guest that wedges the next caller.
|
|
# Short, because nothing here waits for the guest any more: SIGKILL goes
|
|
# to qemu the moment the unit is stopped and this is only the backstop
|
|
# for a process that ignores even that.
|
|
TimeoutStopSec=10
|
|
KillMode=mixed
|
|
Restart=no
|