Grant/owui-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial Commit

Browse Source
This commit is contained in:
Grant
2026-09-15 15:24:03 -04:00
commit 4dc7086e7e
605 changed files with 101380 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vm/vmd/session.py
+932
View File
@@ -0,0 +1,932 @@
"""One VM session: create it, use it, destroy it.
WHAT A SESSION IS
The span of one vm_agent tool call. It starts from the frozen base image
every single time and leaves nothing behind except what was written to
the project share. There is no "resume a machine" path and there is not
meant to be: a machine that carries state between sessions is a machine
where last week's half-finished experiment silently changes this week's
answer.
WHY systemd OWNS THE PROCESS AND THIS MODULE DOES NOT
"Only one VM at a time" is enforced by owuivm.service being a single unit
rather than a template - starting an already-running unit is a no-op, so
the guarantee holds even if this process crashes between checking and
starting. Everything else follows from the same decision: the hardening
is declarative and auditable in one file, ExecStopPost= tears down even
when qemu is SIGKILLed, and cancellation is `systemctl stop`, which does
not care what any Python loop is doing.
That last point is the specific mistake being avoided. The archived
sub-agent stack's cancel verb recorded an intention and killed nothing,
so a cancelled run carried on to completion.
"""
from __future__ import annotations
import json
import os
import secrets
import shutil
import subprocess
import time
from dataclasses import dataclass
import project
from guest import Guest, GuestError, GuestTimeout
from qmp import Qmp, QmpError
RUN_ROOT = "/run/owuivm"
# See project.py for why the pool is a setting. Same value, same source.
POOL_ROOT = (os.environ.get("OWUIVM_POOL_ROOT") or "/Pool").rstrip("/") or "/Pool"
OVERLAY_ROOT = f"{POOL_ROOT}/vmrun"
BASE_ROOT = "/var/lib/owuivm/base"
CTL = "/usr/local/sbin/owuivm-ctl"
# How long to wait for the guest agent after the unit starts. A cold boot
# of the Ubuntu image is a few seconds; this is generous enough to cover a
# host under load and short enough that a machine which is never coming up
# is reported as such rather than eating the caller's whole budget.
BOOT_TIMEOUT_S = 180.0
SUPPORTED_OS = ("ubuntu", "windows")
class SessionError(RuntimeError):
"""Phrased for the model to read."""
@dataclass
class Session:
id: str
os_name: str
project_name: str | None
run_dir: str
overlay: str
vars_file: str
share: str | None
started_at: float
# Resumed from a saved machine rather than booted. What follows from
# it: the guest starts paused and has to be told to run, and the
# project share has to be plugged in afterwards because a saved state
# cannot contain one.
warm: bool = False
# -- paths ---------------------------------------------------------
@property
def qmp_path(self) -> str:
return os.path.join(self.run_dir, "qmp.sock")
@property
def ga_path(self) -> str:
return os.path.join(self.run_dir, "ga.sock")
@property
def cancel_path(self) -> str:
return os.path.join(self.run_dir, "CANCEL")
# -- cancellation --------------------------------------------------
def cancelled(self) -> bool:
"""A file, not a flag in memory.
Anything that can write the run directory can cancel a session,
including a process that is not this one - which is the point.
The agent loop checks it between steps, and `systemctl stop`
happens regardless of whether anybody checked.
"""
return os.path.exists(self.cancel_path)
def cancel(self) -> None:
try:
with open(self.cancel_path, "w") as handle:
handle.write(str(time.time()))
except OSError:
pass
_ctl("vm-stop")
# -- the two channels ----------------------------------------------
def qmp(self) -> Qmp:
return Qmp(self.qmp_path)
def guest(self, timeout: float = 15.0) -> Guest:
return Guest(self.ga_path, timeout=timeout, os_name=self.os_name)
def alive(self) -> bool:
return _ctl("vm-status", check=False).stdout.strip() == "active"
def _ctl(*args: str, check: bool = True) -> subprocess.CompletedProcess:
result = subprocess.run(
["/usr/bin/sudo", "-n", CTL, *args],
capture_output=True, text=True, timeout=120, check=False,
)
if check and result.returncode != 0:
raise SessionError(
f"privileged helper refused '{args[0]}': "
f"{(result.stderr or result.stdout).strip()}"
)
return result
# NOT in the image directory. That belongs to owuivmadm and vmd cannot
# write there, which is the right arrangement - vmd must never be able to
# touch a base image. A saved machine is vmd's own artefact: it creates
# it, it validates it, and it is the only thing that reads it back.
WARM_ROOT = "/var/lib/owuivm/warm"
BASE_ROOT = "/var/lib/owuivm/base"
def warm_set(os_name: str) -> dict | None:
"""The saved machine for this image, if there is a valid one.
VALID MEANS "TAKEN FROM THE DISK THAT IS THERE NOW".
A saved state describes memory that matches a particular disk, down
to the page cache. Resume it against a base image that has been
changed since and nothing reports an error - the guest simply has a
filesystem in memory that disagrees with the one on disk, which is
how a disk gets corrupted rather than how it fails.
The freeze already records a checksum of the image, so the manifest
records which checksum the state was taken from and this compares
them. Any mismatch, missing file, or unreadable manifest and the
answer is None, which means a cold boot: slower, and always correct.
"""
if os_name not in SUPPORTED_OS:
return None
warm = os.path.join(WARM_ROOT, os_name)
disk = os.path.join(warm, f"{os_name}.qcow2")
state = os.path.join(warm, "state")
varsf = os.path.join(warm, "OVMF_VARS.fd")
manifest = os.path.join(warm, "manifest.json")
if not all(os.path.isfile(p) for p in (disk, state, varsf, manifest)):
return None
try:
with open(manifest, encoding="utf-8") as handle:
meta = json.load(handle)
except (OSError, ValueError):
return None
recorded = str(meta.get("base_sha256") or "")
if not recorded or recorded != _base_checksum(os_name):
return None
return {"disk": disk, "state": state, "vars": varsf, "meta": meta}
def _base_checksum(os_name: str) -> str:
"""What the freeze recorded for this image, not a checksum taken now.
Hashing forty gigabytes at the start of every session would cost more
than the boot this is meant to save. The manifest file written when
the image was frozen is the record, and it is written by the only
thing that can change the image.
"""
path = os.path.join(BASE_ROOT, f"{os_name}.sha256")
try:
with open(path, encoding="utf-8") as handle:
for line in handle:
digest, _, name = line.partition(" ")
if name.strip().endswith(f"{os_name}.qcow2"):
return digest.strip()
except OSError:
pass
return ""
def _base_paths(os_name: str) -> tuple[str, str]:
if os_name not in SUPPORTED_OS:
raise SessionError(
f"'{os_name}' is not a machine this runs. Choose 'ubuntu' or 'windows'."
)
disk = os.path.join(BASE_ROOT, f"{os_name}.qcow2")
vars_file = os.path.join(BASE_ROOT, f"{os_name}-OVMF_VARS.fd")
if not os.path.exists(disk):
raise SessionError(
f"the {os_name} base image has not been built yet, so there is no "
f"{os_name} machine to start. Nothing is wrong with the request."
)
return disk, vars_file
def reap(verbose: bool = False) -> list[str]:
"""Clear anything a previous run left behind.
Same reasoning as octave/run.sh's `rm -rf "${JOBS:?}/run"/*` at start:
a crash, an OOM kill or a reboot leaves a stale unit, a mounted
virtiofsd and an orphan overlay, and every one of those wedges the
next caller until it times out - which looks exactly like the new
request being broken.
"""
done: list[str] = []
if _ctl("vm-status", check=False).stdout.strip() == "active":
_ctl("vm-stop", check=False)
done.append("stopped a VM that was still running")
_ctl("vm-reset", check=False)
_ctl("vfs-stop", check=False)
for name in os.listdir(RUN_ROOT) if os.path.isdir(RUN_ROOT) else []:
path = os.path.join(RUN_ROOT, name)
if os.path.isdir(path) and len(name) == 32:
shutil.rmtree(path, ignore_errors=True)
done.append(f"removed stale session directory {name}")
if os.path.isdir(OVERLAY_ROOT):
for name in os.listdir(OVERLAY_ROOT):
# `.state` as well as `.qcow2`: a machine state is four
# gigabytes, and an interrupted save used to leave one behind
# with nothing to notice it. Three of them were found holding
# 11.6 GB after an afternoon of building this.
if name.endswith(".qcow2") or name.endswith(".state"):
try:
os.unlink(os.path.join(OVERLAY_ROOT, name))
done.append(f"removed orphan {name}")
except OSError:
pass
# A saved machine that was half-written when something interrupted it.
# It can never be used - the swap into place is the last step - so it
# is only taking up room.
if os.path.isdir(WARM_ROOT):
for name in os.listdir(WARM_ROOT):
if name.endswith(".new"):
shutil.rmtree(os.path.join(WARM_ROOT, name), ignore_errors=True)
done.append(f"removed a half-written saved machine ({name})")
env = os.path.join(RUN_ROOT, "session.env")
if os.path.exists(env):
try:
os.unlink(env)
except OSError:
pass
if verbose:
for line in done:
print(f" reaped: {line}")
return done
def create(os_name: str, project_name: str | None = None,
mem_mb: int = 8192, cpus: int = 8, wait: bool = True,
allow_warm: bool = True) -> Session:
"""Start a machine; optionally wait until it can be talked to.
`wait=False` exists so the caller can do something useful with the
twenty-odd seconds a machine takes to boot. The agent spends them
writing its task list, which also has the effect of getting the model
loaded and its prompt prefilled before there is anything to do - so
neither side is idle waiting for the other.
"""
base_disk, base_vars = _base_paths(os_name)
# THE BASE MUST BE FROZEN, AND THIS IS WHERE THAT IS ENFORCED.
#
# It used to be enforced by the maintenance unit re-freezing on the
# way out, which is the wrong place twice over: it made looking at an
# image mean unfreezing it, and it put the guarantee in the hands of
# a step that has already been seen to fail silently (a broken `du`
# aborted it for a while, and every clean shutdown left the image
# writable with only a status=1 in the log).
#
# So the freeze is now the operator's own button, and the guarantee
# is checked here, at the moment a session would otherwise be handed
# a writable base. A refusal with a sentence in it, rather than a
# chmod nobody saw.
if os.stat(base_disk).st_mode & 0o222:
raise SessionError(
f"the {os_name} base image is unfrozen, so no session can start. "
"It is open for maintenance, or was left that way: freeze it in "
"Tool Server VM Maintenance and try again.")
# Nothing from a previous session is allowed to still be around. This
# is not defensive tidying - a leftover unit means `vm-start` is a
# silent no-op and the caller talks to the wrong machine.
reap()
session_id = secrets.token_hex(16)
run_dir = os.path.join(RUN_ROOT, session_id)
overlay = os.path.join(OVERLAY_ROOT, f"{session_id}.qcow2")
vars_file = os.path.join(run_dir, "OVMF_VARS.fd")
# 0007 so qemu, which shares the group, can write the sockets and the
# overlay that this process is about to create for it.
os.umask(0o007)
os.makedirs(run_dir, mode=0o2770, exist_ok=True)
try:
os.chown(run_dir, -1, _gid("owuivm"))
os.chmod(run_dir, 0o2770)
except (OSError, KeyError):
pass
share = None
if project_name:
project.validate(project_name)
share = _ctl("project-ensure", project_name).stdout.strip()
if not share:
raise SessionError(f"could not prepare storage for project '{project_name}'.")
# WARM OR COLD, AND WHAT CHANGES.
#
# A warm start branches the overlay from the SAVED disk rather than
# the base - the saved memory describes that disk - and boots into
# the saved state instead of the firmware. The base is still under
# both of them, still frozen, still the thing nothing can write.
warm = warm_set(os_name) if allow_warm else None
backing = warm["disk"] if warm else base_disk
vars_source = warm["vars"] if warm else base_vars
# The overlay is the immutability guarantee, and it is structural: the
# base is opened as a read-only backing file, and nothing this session
# writes can reach it. The base is also mode 0444 owned by a user this
# process is not, so there are two independent reasons it cannot change.
result = subprocess.run(
["/usr/bin/qemu-img", "create", "-f", "qcow2",
"-b", backing, "-F", "qcow2", overlay],
capture_output=True, text=True, timeout=60, check=False,
)
if result.returncode != 0:
raise SessionError(f"could not create the session disk: {result.stderr.strip()}")
# Explicitly 0660, and umask cannot do this for us. qemu-img asks the
# kernel for 0644, so a umask of 0007 clears a group-write bit that
# was never requested in the first place and leaves 0640 - readable
# by the qemu user, writable by nobody but vmd. qemu then fails with
# a bare "Could not open ...: Permission denied" on a file it can
# plainly see, which is a memorable half hour.
os.chmod(overlay, 0o660)
shutil.copyfile(vars_source, vars_file)
os.chmod(vars_file, 0o660)
with open(os.path.join(RUN_ROOT, "session.env"), "w") as handle:
handle.write(
f"OWUIVM_MODE=session\n"
f"OWUIVM_SESSION={session_id}\n"
f"OWUIVM_DISK={overlay}\n"
f"OWUIVM_VARS={vars_file}\n"
# Deliberately empty on a warm start even when there IS a
# share: the device is hot-plugged after the state has
# loaded, into the port the argv always carries. virtiofsd is
# still started below, so its socket is waiting.
f"OWUIVM_SHARE={'' if warm else (share or '')}\n"
f"OWUIVM_MEM_MB={int(mem_mb)}\n"
f"OWUIVM_CPUS={int(cpus)}\n"
+ (f"OWUIVM_INCOMING={warm['state']}\n" if warm else "")
)
session = Session(
id=session_id, os_name=os_name, project_name=project_name,
run_dir=run_dir, overlay=overlay, vars_file=vars_file,
share=share, started_at=time.monotonic(), warm=bool(warm),
)
if share:
# virtiofsd first: qemu's vhost-user chardev connects to its
# socket at startup and fails outright if it is not there yet.
_ctl("vfs-start", project_name or "", session_id)
_wait_for(lambda: os.path.exists(os.path.join(run_dir, "vfs.sock")), 20.0,
"the project share did not come up")
_ctl("vm-start")
if wait:
try:
await_ready(session)
except SessionError:
destroy(session)
raise
return session
def await_ready(session: Session, timeout: float = BOOT_TIMEOUT_S,
desktop: bool = False) -> dict:
"""Block until the machine can be talked to, and say what it waited for."""
started = time.monotonic()
if session.warm:
_resume(session)
_wait_for_guest(session, timeout)
clock = _sync_clock(session)
if session.warm:
_attach_share(session)
else:
_warm_the_shell(session)
_mount_share(session)
result = {"seconds": round(time.monotonic() - started, 1), "desktop": False,
"clock": clock, "facts": _machine_facts(session)}
if desktop:
result["desktop"] = _wait_for_desktop(session, timeout - result["seconds"])
result["seconds"] = round(time.monotonic() - started, 1)
return result
def _machine_facts(session: Session) -> str:
"""One command, so the agent does not spend three turns asking.
THIS IS TOOL OUTPUT, NOT PROMPT TEXT, AND THAT IS THE POINT.
Three times now this stack has watched a model treat a system prompt
as advice and tool output as fact. So what is actually installed in
this machine is discovered here and handed over with the machine,
rather than written into a prompt that will drift from the image the
first time somebody adds a package.
Best-effort by design: a machine that answers `run` but not this is
still a machine worth handing over, so a failure here costs a line of
text and nothing else.
"""
if session.os_name == "windows":
return _windows_facts(session)
probe = (
"printf 'os: '; (. /etc/os-release 2>/dev/null && echo \"$PRETTY_NAME\") "
"|| uname -sr\n"
"printf 'python: '; (python3 -V 2>&1 | head -1) || echo none\n"
"printf 'browsers: '; for b in chromium chromium-browser google-chrome "
"brave-browser firefox; do command -v $b >/dev/null 2>&1 && printf '%s ' $b; done; echo\n"
"printf 'desktop user: '; id -nu 1000 2>/dev/null || echo none\n"
"printf 'project folder: '; if mountpoint -q /mnt/project 2>/dev/null; then "
"df -h /mnt/project | awk 'NR==2{print \"/mnt/project, \" $4 \" free of \" $2}'; "
"else echo 'not mounted - nothing you write will be kept'; fi\n"
)
try:
with session.guest(timeout=20.0) as agent:
result = agent.shell(probe, timeout_s=25.0)
except (GuestError, OSError):
return ""
return (result.get("stdout") or "").strip()[:1200]
def _resume(session: Session) -> None:
"""Let a machine that loaded from a saved state start running.
qemu holds an incoming migration paused until it is told otherwise,
which is the right default - it is how a real migration hands over
without two copies running at once - and here it just means one
command. Measured on this host: the state loads and the machine is
running 7.2 seconds after the unit starts, against 90 for a boot.
"""
deadline = time.monotonic() + 120.0
last = ""
while time.monotonic() < deadline:
try:
with Qmp(session.qmp_path, timeout=20.0) as qmp:
state = qmp.resume()
if state == "running":
return
last = state
except (QmpError, OSError) as exc:
last = str(exc)
time.sleep(0.5)
raise SessionError(
f"the saved machine did not start running (last state: {last or 'unknown'})."
)
def _attach_share(session: Session) -> None:
"""Plug the project share into a machine that resumed without one.
A saved state cannot contain the share - virtiofsd refuses migration
- so it is added afterwards, into the root port the argv always
carries, at the same address a cold boot uses. Then the guest is told
to look again: Linux mounts it the same way it always does, and
Windows has a service that has to be restarted to notice a device
that appeared after it started.
"""
if not session.share:
return
socket_path = os.path.join(session.run_dir, "vfs.sock")
try:
with Qmp(session.qmp_path, timeout=30.0) as qmp:
qmp.attach_share(socket_path)
except (QmpError, OSError) as exc:
raise SessionError(
f"the machine resumed but its project folder could not be attached: {exc}"
) from exc
if session.os_name == "windows":
# The VirtIO-FS service enumerates at start; a device that turns
# up later is not noticed until it does that again.
try:
with session.guest(timeout=45.0) as agent:
agent.shell(
"Restart-Service VirtioFsSvc -Force; "
# Wait for the drive rather than for a number of
# seconds: it is usually there in one and the guess
# was costing three on every warm session.
"for ($i=0; $i -lt 30; $i++) { "
" if (Get-PSDrive -Name Z -EA SilentlyContinue) { exit 0 }; "
" Start-Sleep -Milliseconds 500 } ; exit 1",
timeout_s=120)
except (GuestError, GuestTimeout, OSError) as exc:
raise SessionError(
f"the share was attached but the machine did not pick it up: {exc}"
) from exc
def _sync_clock(session: Session) -> str:
"""Set the guest's clock from the host's, and CHECK that it took.
A resumed machine believes it is whenever its state was saved.
Measured on the first warm Windows session: twenty-six seconds
behind, and a snapshot left over a weekend would be days out. Windows
minds most - a wrong clock fails TLS, so every download breaks in a
way that reads as a network fault.
The first version set the time and swallowed any error, which is how
it came to be twenty-six seconds behind while reporting nothing: the
share service had just been restarted, the guest agent was busy for a
moment, the call failed and nobody was told. So this one sets the
time, READS IT BACK, and tries again if it did not take - and says
what happened rather than returning silence either way.
"""
tolerance = 2.0
last = "not attempted"
for attempt in range(4):
try:
with session.guest(timeout=30.0) as agent:
agent.set_time(int(time.time() * 1_000_000_000))
# guest-get-time answers in nanoseconds, UTC, from the
# guest's own clock - which is the thing being checked.
reported = int(agent.info_time())
off = abs(time.time() - reported / 1_000_000_000)
if off <= tolerance:
return f"set, within {off:.1f}s of the host"
last = f"still {off:.1f}s out"
except (GuestError, GuestTimeout, OSError) as exc:
last = str(exc)
time.sleep(2.0 * (attempt + 1))
return f"could not be set ({last})"
def _warm_the_shell(session: Session) -> None:
"""Pay for PowerShell's cold start while nobody is waiting on it.
MEASURED on this Windows image: the FIRST PowerShell in a freshly
booted guest took over ninety seconds - Defender inspecting it and
.NET warming up - and every one after that took five to nine. The
machine-facts probe, which is the first thing to run, timed out and
handed the agent nothing at all about the machine it was given.
The agent is writing its task list through all of this, so the cost
is already being paid by somebody else's clock. Deliberately
best-effort and deliberately silent: a machine that cannot warm up is
a machine the next call will report on properly.
"""
if session.os_name != "windows":
return
try:
with session.guest(timeout=30.0) as agent:
agent.exec(["powershell", "-NoProfile", "-NonInteractive", "-Command", "exit"],
timeout_s=240.0)
except (GuestError, GuestTimeout, OSError):
pass
def _windows_facts(session: Session) -> str:
"""The same question asked of Windows, including the one that matters.
Whether the project folder is there. virtiofs on Windows needs WinFsp
and the VirtIO-FS service, neither of which the guest agent implies -
so a Windows session can come up perfectly and have nowhere to put
anything that survives it. The agent is told, because the alternative
is that it works for an hour and the report links to an empty repo.
"""
# Written as a script rather than one long -Command line, and with
# single quotes throughout: the first version nested double quotes
# inside a Python string inside PowerShell and produced nothing at
# all, silently, which is the one outcome a fact-gathering probe
# must not have.
probe = r"""
$ErrorActionPreference = 'SilentlyContinue'
'os: ' + (Get-CimInstance Win32_OperatingSystem).Caption
'powershell: ' + $PSVersionTable.PSVersion.ToString()
$py = (Get-Command python.exe).Source
'python: ' + $(if ($py) { (& $py -V 2>&1) } else { 'not on PATH for this account' })
# By path, not by PATH: these are installed per-machine and the guest
# agent runs as SYSTEM, whose PATH is not the logged-in user's.
$known = @(
"$env:ProgramFiles\BraveSoftware\Brave-Browser\Application\brave.exe",
"${env:ProgramFiles(x86)}\BraveSoftware\Brave-Browser\Application\brave.exe",
"$env:ProgramFiles\Google\Chrome\Application\chrome.exe",
"$env:ProgramFiles\Mozilla Firefox\firefox.exe"
) | Where-Object { Test-Path $_ } | ForEach-Object { Split-Path $_ -Leaf }
'browsers: ' + $(if ($known) { $known -join ' ' } else { 'none found' })
'running as: ' + [Environment]::UserName + ' (the guest agent is LocalSystem, not the desktop user)'
# Measured on the first real Windows session: the agent wrapped its
# commands in `powershell.exe -Command "..."` although it was already in
# PowerShell, and used `&&`, which 5.1 does not have. Two wasted turns
# each time, and both are facts about this machine rather than advice.
'shell: your commands already run in PowerShell - do not wrap them in powershell.exe'
'note: this is PowerShell 5.1, which has no && or ||. Use ; or separate calls.'
$z = Get-PSDrive -Name Z
'project folder: ' + $(if ($z) { 'Z:\, ' + [math]::Round($z.Free/1GB,1) + ' GB free' } else { 'NOT MOUNTED - nothing you write will be kept. Say so rather than working around it.' })
"""
# MEASURED: PowerShell's first start under LocalSystem on a freshly
# booted Windows guest took 17 seconds, and the whole probe more than
# the 40 it was given - so the facts came back empty, silently, on
# exactly the machine that most needed them. Windows gets a minute
# and a half; it is once per session and it is the difference between
# the agent knowing there is no project folder and finding out.
try:
with session.guest(timeout=45.0) as agent:
result = agent.shell(probe, timeout_s=90.0)
except (GuestError, OSError) as exc:
return f"(could not read this machine's details: {exc})"
return (result.get("stdout") or "").strip()[:1200]
def _mount_share(session: Session) -> None:
"""Put the project folder at /mnt/project, whatever the image does.
THE SESSION MOUNTS IT, NOT THE IMAGE.
The share is one virtiofs tag, `project`, and the base image built by
the autoinstall carries an fstab line for it. An image installed by
hand does not, and the failure that produces is a bad one: everything
works, the agent writes its files, `git` on the host commits an empty
directory, and the only sign is that the Gitea repository has nothing
in it. Nothing in the guest and nothing on the host reports an error.
So the mount is made here, every session, from the side that knows
whether there is a share at all. It is idempotent - an image that
does carry the fstab line is already mounted and this is one
`mountpoint` check - and it means a hand-built image needs nothing
but qemu-guest-agent to be a working base.
"""
if not session.share:
return
if session.os_name != "ubuntu":
# Windows mounts virtiofs through WinFsp as a drive letter, set
# up inside the image; there is no equivalent command to run
# here. See vm/images/windows/build.md.
return
script = (
"set -e\n"
"mkdir -p /mnt/project\n"
"mountpoint -q /mnt/project || mount -t virtiofs project /mnt/project\n"
"mountpoint -q /mnt/project\n"
)
try:
with session.guest(timeout=30.0) as agent:
result = agent.shell(script, timeout_s=30.0)
except (GuestError, OSError) as exc:
raise SessionError(
f"the machine is up but its project folder could not be mounted: {exc}"
) from exc
if result["exit_code"] != 0:
detail = (result["stderr"] or result["stdout"]).strip()[:300]
raise SessionError(
"the machine is up but the project folder did not mount, so nothing "
"written in this session would be kept. "
f"`mount -t virtiofs project /mnt/project` said: {detail or 'nothing'}"
)
def _wait_for_guest(session: Session, timeout: float = BOOT_TIMEOUT_S) -> None:
"""Boot supervision: report honestly rather than pretend."""
deadline = time.monotonic() + timeout
_wait_for(lambda: os.path.exists(session.ga_path), 30.0,
"qemu did not create its guest-agent socket")
last: str = ""
while time.monotonic() < deadline:
if not session.alive():
raise SessionError(
"the machine stopped while it was booting. "
f"Check: journalctl -u owuivm.service -n 50"
)
try:
with session.guest(timeout=5.0) as agent:
if agent.ping(timeout=5.0):
return
except (GuestError, OSError) as exc:
last = str(exc)
time.sleep(1.0)
raise SessionError(
f"the {session.os_name} machine booted but its guest agent never "
f"answered within {timeout:.0f}s, so there is no way to run "
f"anything on it. Last error: {last or 'none'}"
)
def await_desktop(session: Session, timeout: float = 300.0) -> bool:
"""Block until the graphical session is usable. Raises if it never is.
The public form of the wait below, called lazily by vmd the first time
a session reaches for the screen - see _require_desktop. A job that
only runs commands never calls it and never pays for it.
"""
if not _wait_for_desktop(session, timeout):
raise SessionError(
"the desktop did not finish coming up within "
f"{timeout:.0f}s, so there is nothing on screen to look at"
)
return True
def _wait_for_desktop(session: Session, timeout: float) -> bool:
"""Wait for a graphical session that is actually ON THE SCREEN.
THE PROBE USED TO ASK THE WRONG QUESTION, TWICE.
First it asked whether any seat session existed - which gdm's own
greeter satisfies, long before the user is logged in. Then, with that
fixed, a machine still handed over showing a boot splash: measured,
`gnome-shell` was running, plymouth was gone, graphical.target was
active, and the framebuffer sat on the splash for over three minutes.
The reason is that the display shows the FOREGROUND VIRTUAL TERMINAL,
and the session was not on it: `fgconsole` said 1, the session's TTY
was tty2, and `loginctl` agreed the session was `Active=no`. Plymouth
had had tty1 and nothing switched away from it. `chvt 2` put the
desktop on screen in under a second.
That is also why it looked intermittent - sometimes gdm switches and
sometimes it does not. So this does not wait for the switch to happen
on its own; it makes it happen, and then checks.
"""
if timeout <= 0:
return False
if session.os_name == "windows":
return _wait_for_windows_desktop(session, timeout)
probe = (
# The operator's session, not gdm's greeter.
"sid=$(loginctl list-sessions --no-legend | "
"awk '$3!=\"gdm\" && $4==\"seat0\"{print $1}' | head -1); "
"[ -n \"$sid\" ] || exit 1; "
# Put it on the screen if something else still has the console.
"vt=$(loginctl show-session \"$sid\" -p TTY --value 2>/dev/null); vt=${vt#tty}; "
"case \"$vt\" in [0-9]*) "
" [ \"$(fgconsole 2>/dev/null)\" = \"$vt\" ] || chvt \"$vt\" 2>/dev/null ;; "
"esac; "
"[ \"$(loginctl show-session \"$sid\" -p Active --value 2>/dev/null)\" = yes ] || exit 1; "
"pgrep -x gnome-shell >/dev/null || exit 1; "
# Plymouth still running means the splash still owns the display.
"! pgrep -x plymouthd >/dev/null"
)
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
with session.guest(timeout=10.0) as agent:
if agent.shell(probe, timeout_s=20)["exit_code"] == 0:
# MEASURED: mutter takes two to three seconds to
# repaint the whole screen after a VT switch. With a
# one-second settle the handover screenshot came back
# 3.4 KB - a near-black frame - and three seconds
# later it was the full 400 KB desktop. The first
# picture the agent sees is this one, so it waits.
time.sleep(3.0)
return True
except (GuestError, OSError):
pass
time.sleep(2.0)
return False
def _wait_for_windows_desktop(session: Session, timeout: float) -> bool:
"""Windows has no VT to switch and no compositor to wait for.
What it has is explorer.exe, which exists once somebody is logged in
and the shell has drawn itself. Auto-logon is part of the image
contract, so this is a wait rather than a question.
"""
probe = ("if (Get-Process explorer -ErrorAction SilentlyContinue) { exit 0 } else { exit 1 }")
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
with session.guest(timeout=15.0) as agent:
if agent.shell(probe, timeout_s=25)["exit_code"] == 0:
time.sleep(2.0)
return True
except (GuestError, OSError):
pass
time.sleep(3.0)
return False
def _wait_for(predicate, timeout: float, message: str) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if predicate():
return
time.sleep(0.1)
raise SessionError(message)
def snapshot(os_name: str, settle: float = 45.0) -> dict:
"""Save a booted machine so later sessions can resume it.
WHAT IS SAVED, AND WHY IT IS FOUR THINGS RATHER THAN ONE.
A saved machine is memory plus the exact disk that memory describes,
plus the firmware variables it booted with. Separate them and nothing
complains: the guest simply runs with a filesystem in memory that
disagrees with the one on disk. So all of it is written together, and
a manifest records the checksum of the base image underneath - which
is what `warm_set` checks before trusting any of it.
Taken WITHOUT a project share, because virtiofsd refuses migration
("vhost-user backend lacks VHOST_USER_PROTOCOL_F_LOG_SHMFD"). The
share is plugged in after the resume instead.
"""
if os_name not in SUPPORTED_OS:
raise SessionError(f"'{os_name}' is not a machine this runs.")
base_sha = _base_checksum(os_name)
if not base_sha:
raise SessionError(
f"no checksum is recorded for the {os_name} image, so a saved machine "
"could never be shown to match it. Freeze the image first.")
session = create(os_name, None, wait=False, allow_warm=False)
try:
await_ready(session, desktop=(os_name != "windows"))
# Let the machine finish what a fresh boot does after it answers -
# service startup, the first-run work Windows does - so the saved
# machine is one somebody would want to be handed.
time.sleep(settle)
warm = os.path.join(WARM_ROOT, os_name)
staging = warm + ".new"
shutil.rmtree(staging, ignore_errors=True)
os.makedirs(staging, mode=0o751, exist_ok=True)
# Written straight into the staging directory through a descriptor
# vmd owns - see Qmp.save_state_to_fd. Four gigabytes that never
# get copied, and a file qemu could not have named.
state_path = os.path.join(staging, "state")
fd = os.open(state_path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o644)
try:
with Qmp(session.qmp_path, timeout=30.0) as qmp:
qmp.save_state_to_fd(fd)
finally:
os.close(fd)
# The disk first: it is the thing the memory describes.
subprocess.run(["/usr/bin/qemu-img", "convert", "-O", "qcow2",
"-B", _base_paths(os_name)[0], "-F", "qcow2",
session.overlay, os.path.join(staging, f"{os_name}.qcow2")],
capture_output=True, text=True, timeout=900, check=True)
shutil.copyfile(session.vars_file, os.path.join(staging, "OVMF_VARS.fd"))
with open(os.path.join(staging, "manifest.json"), "w", encoding="utf-8") as handle:
json.dump({"os": os_name, "base_sha256": base_sha,
"created": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
"state_bytes": os.path.getsize(state_path)}, handle, indent=2)
# WHO MAY READ WHAT, AND WHY THEY DIFFER.
#
# `state` is a four-gigabyte image of the machine's memory. It is
# readable by the qemu user, which has to load it, and by nobody
# else - a memory dump is not a thing to leave world-readable
# whatever is in it.
#
# `manifest.json` says which image the saved machine belongs to
# and when it was taken, and the desktop tool - which runs as the
# operator, not as either service user - shows that on the row.
# So the directory is traversable and the manifest is readable,
# and neither tells anyone anything that `ls -l` would not.
for name in os.listdir(staging):
os.chmod(os.path.join(staging, name),
0o644 if name == "manifest.json" else 0o640)
# In place only once every piece is written: a half-written warm
# set that passed the checksum test would be worse than none.
shutil.rmtree(warm, ignore_errors=True)
os.rename(staging, warm)
return {"os": os_name, "path": warm,
"state_gb": round(os.path.getsize(os.path.join(warm, "state")) / 1e9, 2),
"disk_mb": round(os.path.getsize(os.path.join(warm, f"{os_name}.qcow2")) / 1e6, 1),
"base_sha256": base_sha}
finally:
destroy(session)
def destroy(session: Session, graceful: bool = False) -> None:
"""Stop the machine and remove everything it used.
Not graceful by default, deliberately. The overlay is deleted a few
lines further down and the project share was committed and pushed
before this was called, so asking the guest to shut down politely
buys a tidier-looking exit and costs up to ninety seconds of CPU per
session. `systemctl stop` on a unit that now kills with SIGKILL is
the whole teardown.
Deliberately never raises. A teardown that fails loudly leaves the
unit in `failed` and the overlay on disk, and the NEXT caller pays
for it - which is the worst possible place to surface the problem.
"""
if graceful:
try:
with session.guest(timeout=5.0) as agent:
agent.shutdown("powerdown")
_wait_for(lambda: not session.alive(), 20.0, "")
except Exception: # noqa: BLE001
pass
_ctl("vm-stop", check=False)
_ctl("vm-reset", check=False)
_ctl("vfs-stop", check=False)
try:
os.unlink(session.overlay)
except OSError:
pass
shutil.rmtree(session.run_dir, ignore_errors=True)
try:
os.unlink(os.path.join(RUN_ROOT, "session.env"))
except OSError:
pass
def _gid(name: str) -> int:
import grp
return grp.getgrnam(name).gr_gid