933 lines
39 KiB
Python
933 lines
39 KiB
Python
"""One VM session: create it, use it, destroy it.
|
|
|
|
WHAT A SESSION IS
|
|
|
|
The span of one vm_agent tool call. It starts from the frozen base image
|
|
every single time and leaves nothing behind except what was written to
|
|
the project share. There is no "resume a machine" path and there is not
|
|
meant to be: a machine that carries state between sessions is a machine
|
|
where last week's half-finished experiment silently changes this week's
|
|
answer.
|
|
|
|
WHY systemd OWNS THE PROCESS AND THIS MODULE DOES NOT
|
|
|
|
"Only one VM at a time" is enforced by owuivm.service being a single unit
|
|
rather than a template - starting an already-running unit is a no-op, so
|
|
the guarantee holds even if this process crashes between checking and
|
|
starting. Everything else follows from the same decision: the hardening
|
|
is declarative and auditable in one file, ExecStopPost= tears down even
|
|
when qemu is SIGKILLed, and cancellation is `systemctl stop`, which does
|
|
not care what any Python loop is doing.
|
|
|
|
That last point is the specific mistake being avoided. The archived
|
|
sub-agent stack's cancel verb recorded an intention and killed nothing,
|
|
so a cancelled run carried on to completion.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import secrets
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
from dataclasses import dataclass
|
|
|
|
import project
|
|
from guest import Guest, GuestError, GuestTimeout
|
|
from qmp import Qmp, QmpError
|
|
|
|
RUN_ROOT = "/run/owuivm"
|
|
# See project.py for why the pool is a setting. Same value, same source.
|
|
POOL_ROOT = (os.environ.get("OWUIVM_POOL_ROOT") or "/Pool").rstrip("/") or "/Pool"
|
|
OVERLAY_ROOT = f"{POOL_ROOT}/vmrun"
|
|
BASE_ROOT = "/var/lib/owuivm/base"
|
|
CTL = "/usr/local/sbin/owuivm-ctl"
|
|
|
|
# How long to wait for the guest agent after the unit starts. A cold boot
|
|
# of the Ubuntu image is a few seconds; this is generous enough to cover a
|
|
# host under load and short enough that a machine which is never coming up
|
|
# is reported as such rather than eating the caller's whole budget.
|
|
BOOT_TIMEOUT_S = 180.0
|
|
|
|
SUPPORTED_OS = ("ubuntu", "windows")
|
|
|
|
|
|
class SessionError(RuntimeError):
|
|
"""Phrased for the model to read."""
|
|
|
|
|
|
@dataclass
|
|
class Session:
|
|
id: str
|
|
os_name: str
|
|
project_name: str | None
|
|
run_dir: str
|
|
overlay: str
|
|
vars_file: str
|
|
share: str | None
|
|
started_at: float
|
|
# Resumed from a saved machine rather than booted. What follows from
|
|
# it: the guest starts paused and has to be told to run, and the
|
|
# project share has to be plugged in afterwards because a saved state
|
|
# cannot contain one.
|
|
warm: bool = False
|
|
|
|
# -- paths ---------------------------------------------------------
|
|
@property
|
|
def qmp_path(self) -> str:
|
|
return os.path.join(self.run_dir, "qmp.sock")
|
|
|
|
@property
|
|
def ga_path(self) -> str:
|
|
return os.path.join(self.run_dir, "ga.sock")
|
|
|
|
@property
|
|
def cancel_path(self) -> str:
|
|
return os.path.join(self.run_dir, "CANCEL")
|
|
|
|
# -- cancellation --------------------------------------------------
|
|
def cancelled(self) -> bool:
|
|
"""A file, not a flag in memory.
|
|
|
|
Anything that can write the run directory can cancel a session,
|
|
including a process that is not this one - which is the point.
|
|
The agent loop checks it between steps, and `systemctl stop`
|
|
happens regardless of whether anybody checked.
|
|
"""
|
|
return os.path.exists(self.cancel_path)
|
|
|
|
def cancel(self) -> None:
|
|
try:
|
|
with open(self.cancel_path, "w") as handle:
|
|
handle.write(str(time.time()))
|
|
except OSError:
|
|
pass
|
|
_ctl("vm-stop")
|
|
|
|
# -- the two channels ----------------------------------------------
|
|
def qmp(self) -> Qmp:
|
|
return Qmp(self.qmp_path)
|
|
|
|
def guest(self, timeout: float = 15.0) -> Guest:
|
|
return Guest(self.ga_path, timeout=timeout, os_name=self.os_name)
|
|
|
|
def alive(self) -> bool:
|
|
return _ctl("vm-status", check=False).stdout.strip() == "active"
|
|
|
|
|
|
def _ctl(*args: str, check: bool = True) -> subprocess.CompletedProcess:
|
|
result = subprocess.run(
|
|
["/usr/bin/sudo", "-n", CTL, *args],
|
|
capture_output=True, text=True, timeout=120, check=False,
|
|
)
|
|
if check and result.returncode != 0:
|
|
raise SessionError(
|
|
f"privileged helper refused '{args[0]}': "
|
|
f"{(result.stderr or result.stdout).strip()}"
|
|
)
|
|
return result
|
|
|
|
|
|
# NOT in the image directory. That belongs to owuivmadm and vmd cannot
|
|
# write there, which is the right arrangement - vmd must never be able to
|
|
# touch a base image. A saved machine is vmd's own artefact: it creates
|
|
# it, it validates it, and it is the only thing that reads it back.
|
|
WARM_ROOT = "/var/lib/owuivm/warm"
|
|
BASE_ROOT = "/var/lib/owuivm/base"
|
|
|
|
|
|
def warm_set(os_name: str) -> dict | None:
|
|
"""The saved machine for this image, if there is a valid one.
|
|
|
|
VALID MEANS "TAKEN FROM THE DISK THAT IS THERE NOW".
|
|
|
|
A saved state describes memory that matches a particular disk, down
|
|
to the page cache. Resume it against a base image that has been
|
|
changed since and nothing reports an error - the guest simply has a
|
|
filesystem in memory that disagrees with the one on disk, which is
|
|
how a disk gets corrupted rather than how it fails.
|
|
|
|
The freeze already records a checksum of the image, so the manifest
|
|
records which checksum the state was taken from and this compares
|
|
them. Any mismatch, missing file, or unreadable manifest and the
|
|
answer is None, which means a cold boot: slower, and always correct.
|
|
"""
|
|
if os_name not in SUPPORTED_OS:
|
|
return None
|
|
warm = os.path.join(WARM_ROOT, os_name)
|
|
disk = os.path.join(warm, f"{os_name}.qcow2")
|
|
state = os.path.join(warm, "state")
|
|
varsf = os.path.join(warm, "OVMF_VARS.fd")
|
|
manifest = os.path.join(warm, "manifest.json")
|
|
if not all(os.path.isfile(p) for p in (disk, state, varsf, manifest)):
|
|
return None
|
|
try:
|
|
with open(manifest, encoding="utf-8") as handle:
|
|
meta = json.load(handle)
|
|
except (OSError, ValueError):
|
|
return None
|
|
recorded = str(meta.get("base_sha256") or "")
|
|
if not recorded or recorded != _base_checksum(os_name):
|
|
return None
|
|
return {"disk": disk, "state": state, "vars": varsf, "meta": meta}
|
|
|
|
|
|
def _base_checksum(os_name: str) -> str:
|
|
"""What the freeze recorded for this image, not a checksum taken now.
|
|
|
|
Hashing forty gigabytes at the start of every session would cost more
|
|
than the boot this is meant to save. The manifest file written when
|
|
the image was frozen is the record, and it is written by the only
|
|
thing that can change the image.
|
|
"""
|
|
path = os.path.join(BASE_ROOT, f"{os_name}.sha256")
|
|
try:
|
|
with open(path, encoding="utf-8") as handle:
|
|
for line in handle:
|
|
digest, _, name = line.partition(" ")
|
|
if name.strip().endswith(f"{os_name}.qcow2"):
|
|
return digest.strip()
|
|
except OSError:
|
|
pass
|
|
return ""
|
|
|
|
|
|
def _base_paths(os_name: str) -> tuple[str, str]:
|
|
if os_name not in SUPPORTED_OS:
|
|
raise SessionError(
|
|
f"'{os_name}' is not a machine this runs. Choose 'ubuntu' or 'windows'."
|
|
)
|
|
disk = os.path.join(BASE_ROOT, f"{os_name}.qcow2")
|
|
vars_file = os.path.join(BASE_ROOT, f"{os_name}-OVMF_VARS.fd")
|
|
if not os.path.exists(disk):
|
|
raise SessionError(
|
|
f"the {os_name} base image has not been built yet, so there is no "
|
|
f"{os_name} machine to start. Nothing is wrong with the request."
|
|
)
|
|
return disk, vars_file
|
|
|
|
|
|
def reap(verbose: bool = False) -> list[str]:
|
|
"""Clear anything a previous run left behind.
|
|
|
|
Same reasoning as octave/run.sh's `rm -rf "${JOBS:?}/run"/*` at start:
|
|
a crash, an OOM kill or a reboot leaves a stale unit, a mounted
|
|
virtiofsd and an orphan overlay, and every one of those wedges the
|
|
next caller until it times out - which looks exactly like the new
|
|
request being broken.
|
|
"""
|
|
done: list[str] = []
|
|
if _ctl("vm-status", check=False).stdout.strip() == "active":
|
|
_ctl("vm-stop", check=False)
|
|
done.append("stopped a VM that was still running")
|
|
_ctl("vm-reset", check=False)
|
|
_ctl("vfs-stop", check=False)
|
|
|
|
for name in os.listdir(RUN_ROOT) if os.path.isdir(RUN_ROOT) else []:
|
|
path = os.path.join(RUN_ROOT, name)
|
|
if os.path.isdir(path) and len(name) == 32:
|
|
shutil.rmtree(path, ignore_errors=True)
|
|
done.append(f"removed stale session directory {name}")
|
|
|
|
if os.path.isdir(OVERLAY_ROOT):
|
|
for name in os.listdir(OVERLAY_ROOT):
|
|
# `.state` as well as `.qcow2`: a machine state is four
|
|
# gigabytes, and an interrupted save used to leave one behind
|
|
# with nothing to notice it. Three of them were found holding
|
|
# 11.6 GB after an afternoon of building this.
|
|
if name.endswith(".qcow2") or name.endswith(".state"):
|
|
try:
|
|
os.unlink(os.path.join(OVERLAY_ROOT, name))
|
|
done.append(f"removed orphan {name}")
|
|
except OSError:
|
|
pass
|
|
|
|
# A saved machine that was half-written when something interrupted it.
|
|
# It can never be used - the swap into place is the last step - so it
|
|
# is only taking up room.
|
|
if os.path.isdir(WARM_ROOT):
|
|
for name in os.listdir(WARM_ROOT):
|
|
if name.endswith(".new"):
|
|
shutil.rmtree(os.path.join(WARM_ROOT, name), ignore_errors=True)
|
|
done.append(f"removed a half-written saved machine ({name})")
|
|
|
|
env = os.path.join(RUN_ROOT, "session.env")
|
|
if os.path.exists(env):
|
|
try:
|
|
os.unlink(env)
|
|
except OSError:
|
|
pass
|
|
if verbose:
|
|
for line in done:
|
|
print(f" reaped: {line}")
|
|
return done
|
|
|
|
|
|
def create(os_name: str, project_name: str | None = None,
|
|
mem_mb: int = 8192, cpus: int = 8, wait: bool = True,
|
|
allow_warm: bool = True) -> Session:
|
|
"""Start a machine; optionally wait until it can be talked to.
|
|
|
|
`wait=False` exists so the caller can do something useful with the
|
|
twenty-odd seconds a machine takes to boot. The agent spends them
|
|
writing its task list, which also has the effect of getting the model
|
|
loaded and its prompt prefilled before there is anything to do - so
|
|
neither side is idle waiting for the other.
|
|
"""
|
|
base_disk, base_vars = _base_paths(os_name)
|
|
|
|
# THE BASE MUST BE FROZEN, AND THIS IS WHERE THAT IS ENFORCED.
|
|
#
|
|
# It used to be enforced by the maintenance unit re-freezing on the
|
|
# way out, which is the wrong place twice over: it made looking at an
|
|
# image mean unfreezing it, and it put the guarantee in the hands of
|
|
# a step that has already been seen to fail silently (a broken `du`
|
|
# aborted it for a while, and every clean shutdown left the image
|
|
# writable with only a status=1 in the log).
|
|
#
|
|
# So the freeze is now the operator's own button, and the guarantee
|
|
# is checked here, at the moment a session would otherwise be handed
|
|
# a writable base. A refusal with a sentence in it, rather than a
|
|
# chmod nobody saw.
|
|
if os.stat(base_disk).st_mode & 0o222:
|
|
raise SessionError(
|
|
f"the {os_name} base image is unfrozen, so no session can start. "
|
|
"It is open for maintenance, or was left that way: freeze it in "
|
|
"Tool Server VM Maintenance and try again.")
|
|
|
|
# Nothing from a previous session is allowed to still be around. This
|
|
# is not defensive tidying - a leftover unit means `vm-start` is a
|
|
# silent no-op and the caller talks to the wrong machine.
|
|
reap()
|
|
|
|
session_id = secrets.token_hex(16)
|
|
run_dir = os.path.join(RUN_ROOT, session_id)
|
|
overlay = os.path.join(OVERLAY_ROOT, f"{session_id}.qcow2")
|
|
vars_file = os.path.join(run_dir, "OVMF_VARS.fd")
|
|
|
|
# 0007 so qemu, which shares the group, can write the sockets and the
|
|
# overlay that this process is about to create for it.
|
|
os.umask(0o007)
|
|
os.makedirs(run_dir, mode=0o2770, exist_ok=True)
|
|
try:
|
|
os.chown(run_dir, -1, _gid("owuivm"))
|
|
os.chmod(run_dir, 0o2770)
|
|
except (OSError, KeyError):
|
|
pass
|
|
|
|
share = None
|
|
if project_name:
|
|
project.validate(project_name)
|
|
share = _ctl("project-ensure", project_name).stdout.strip()
|
|
if not share:
|
|
raise SessionError(f"could not prepare storage for project '{project_name}'.")
|
|
|
|
# WARM OR COLD, AND WHAT CHANGES.
|
|
#
|
|
# A warm start branches the overlay from the SAVED disk rather than
|
|
# the base - the saved memory describes that disk - and boots into
|
|
# the saved state instead of the firmware. The base is still under
|
|
# both of them, still frozen, still the thing nothing can write.
|
|
warm = warm_set(os_name) if allow_warm else None
|
|
backing = warm["disk"] if warm else base_disk
|
|
vars_source = warm["vars"] if warm else base_vars
|
|
|
|
# The overlay is the immutability guarantee, and it is structural: the
|
|
# base is opened as a read-only backing file, and nothing this session
|
|
# writes can reach it. The base is also mode 0444 owned by a user this
|
|
# process is not, so there are two independent reasons it cannot change.
|
|
result = subprocess.run(
|
|
["/usr/bin/qemu-img", "create", "-f", "qcow2",
|
|
"-b", backing, "-F", "qcow2", overlay],
|
|
capture_output=True, text=True, timeout=60, check=False,
|
|
)
|
|
if result.returncode != 0:
|
|
raise SessionError(f"could not create the session disk: {result.stderr.strip()}")
|
|
|
|
# Explicitly 0660, and umask cannot do this for us. qemu-img asks the
|
|
# kernel for 0644, so a umask of 0007 clears a group-write bit that
|
|
# was never requested in the first place and leaves 0640 - readable
|
|
# by the qemu user, writable by nobody but vmd. qemu then fails with
|
|
# a bare "Could not open ...: Permission denied" on a file it can
|
|
# plainly see, which is a memorable half hour.
|
|
os.chmod(overlay, 0o660)
|
|
|
|
shutil.copyfile(vars_source, vars_file)
|
|
os.chmod(vars_file, 0o660)
|
|
|
|
with open(os.path.join(RUN_ROOT, "session.env"), "w") as handle:
|
|
handle.write(
|
|
f"OWUIVM_MODE=session\n"
|
|
f"OWUIVM_SESSION={session_id}\n"
|
|
f"OWUIVM_DISK={overlay}\n"
|
|
f"OWUIVM_VARS={vars_file}\n"
|
|
# Deliberately empty on a warm start even when there IS a
|
|
# share: the device is hot-plugged after the state has
|
|
# loaded, into the port the argv always carries. virtiofsd is
|
|
# still started below, so its socket is waiting.
|
|
f"OWUIVM_SHARE={'' if warm else (share or '')}\n"
|
|
f"OWUIVM_MEM_MB={int(mem_mb)}\n"
|
|
f"OWUIVM_CPUS={int(cpus)}\n"
|
|
+ (f"OWUIVM_INCOMING={warm['state']}\n" if warm else "")
|
|
)
|
|
|
|
session = Session(
|
|
id=session_id, os_name=os_name, project_name=project_name,
|
|
run_dir=run_dir, overlay=overlay, vars_file=vars_file,
|
|
share=share, started_at=time.monotonic(), warm=bool(warm),
|
|
)
|
|
|
|
if share:
|
|
# virtiofsd first: qemu's vhost-user chardev connects to its
|
|
# socket at startup and fails outright if it is not there yet.
|
|
_ctl("vfs-start", project_name or "", session_id)
|
|
_wait_for(lambda: os.path.exists(os.path.join(run_dir, "vfs.sock")), 20.0,
|
|
"the project share did not come up")
|
|
|
|
_ctl("vm-start")
|
|
if wait:
|
|
try:
|
|
await_ready(session)
|
|
except SessionError:
|
|
destroy(session)
|
|
raise
|
|
return session
|
|
|
|
|
|
def await_ready(session: Session, timeout: float = BOOT_TIMEOUT_S,
|
|
desktop: bool = False) -> dict:
|
|
"""Block until the machine can be talked to, and say what it waited for."""
|
|
started = time.monotonic()
|
|
if session.warm:
|
|
_resume(session)
|
|
_wait_for_guest(session, timeout)
|
|
clock = _sync_clock(session)
|
|
if session.warm:
|
|
_attach_share(session)
|
|
else:
|
|
_warm_the_shell(session)
|
|
_mount_share(session)
|
|
result = {"seconds": round(time.monotonic() - started, 1), "desktop": False,
|
|
"clock": clock, "facts": _machine_facts(session)}
|
|
if desktop:
|
|
result["desktop"] = _wait_for_desktop(session, timeout - result["seconds"])
|
|
result["seconds"] = round(time.monotonic() - started, 1)
|
|
return result
|
|
|
|
|
|
def _machine_facts(session: Session) -> str:
|
|
"""One command, so the agent does not spend three turns asking.
|
|
|
|
THIS IS TOOL OUTPUT, NOT PROMPT TEXT, AND THAT IS THE POINT.
|
|
|
|
Three times now this stack has watched a model treat a system prompt
|
|
as advice and tool output as fact. So what is actually installed in
|
|
this machine is discovered here and handed over with the machine,
|
|
rather than written into a prompt that will drift from the image the
|
|
first time somebody adds a package.
|
|
|
|
Best-effort by design: a machine that answers `run` but not this is
|
|
still a machine worth handing over, so a failure here costs a line of
|
|
text and nothing else.
|
|
"""
|
|
if session.os_name == "windows":
|
|
return _windows_facts(session)
|
|
probe = (
|
|
"printf 'os: '; (. /etc/os-release 2>/dev/null && echo \"$PRETTY_NAME\") "
|
|
"|| uname -sr\n"
|
|
"printf 'python: '; (python3 -V 2>&1 | head -1) || echo none\n"
|
|
"printf 'browsers: '; for b in chromium chromium-browser google-chrome "
|
|
"brave-browser firefox; do command -v $b >/dev/null 2>&1 && printf '%s ' $b; done; echo\n"
|
|
"printf 'desktop user: '; id -nu 1000 2>/dev/null || echo none\n"
|
|
"printf 'project folder: '; if mountpoint -q /mnt/project 2>/dev/null; then "
|
|
"df -h /mnt/project | awk 'NR==2{print \"/mnt/project, \" $4 \" free of \" $2}'; "
|
|
"else echo 'not mounted - nothing you write will be kept'; fi\n"
|
|
)
|
|
try:
|
|
with session.guest(timeout=20.0) as agent:
|
|
result = agent.shell(probe, timeout_s=25.0)
|
|
except (GuestError, OSError):
|
|
return ""
|
|
return (result.get("stdout") or "").strip()[:1200]
|
|
|
|
|
|
def _resume(session: Session) -> None:
|
|
"""Let a machine that loaded from a saved state start running.
|
|
|
|
qemu holds an incoming migration paused until it is told otherwise,
|
|
which is the right default - it is how a real migration hands over
|
|
without two copies running at once - and here it just means one
|
|
command. Measured on this host: the state loads and the machine is
|
|
running 7.2 seconds after the unit starts, against 90 for a boot.
|
|
"""
|
|
deadline = time.monotonic() + 120.0
|
|
last = ""
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with Qmp(session.qmp_path, timeout=20.0) as qmp:
|
|
state = qmp.resume()
|
|
if state == "running":
|
|
return
|
|
last = state
|
|
except (QmpError, OSError) as exc:
|
|
last = str(exc)
|
|
time.sleep(0.5)
|
|
raise SessionError(
|
|
f"the saved machine did not start running (last state: {last or 'unknown'})."
|
|
)
|
|
|
|
|
|
def _attach_share(session: Session) -> None:
|
|
"""Plug the project share into a machine that resumed without one.
|
|
|
|
A saved state cannot contain the share - virtiofsd refuses migration
|
|
- so it is added afterwards, into the root port the argv always
|
|
carries, at the same address a cold boot uses. Then the guest is told
|
|
to look again: Linux mounts it the same way it always does, and
|
|
Windows has a service that has to be restarted to notice a device
|
|
that appeared after it started.
|
|
"""
|
|
if not session.share:
|
|
return
|
|
socket_path = os.path.join(session.run_dir, "vfs.sock")
|
|
try:
|
|
with Qmp(session.qmp_path, timeout=30.0) as qmp:
|
|
qmp.attach_share(socket_path)
|
|
except (QmpError, OSError) as exc:
|
|
raise SessionError(
|
|
f"the machine resumed but its project folder could not be attached: {exc}"
|
|
) from exc
|
|
|
|
if session.os_name == "windows":
|
|
# The VirtIO-FS service enumerates at start; a device that turns
|
|
# up later is not noticed until it does that again.
|
|
try:
|
|
with session.guest(timeout=45.0) as agent:
|
|
agent.shell(
|
|
"Restart-Service VirtioFsSvc -Force; "
|
|
# Wait for the drive rather than for a number of
|
|
# seconds: it is usually there in one and the guess
|
|
# was costing three on every warm session.
|
|
"for ($i=0; $i -lt 30; $i++) { "
|
|
" if (Get-PSDrive -Name Z -EA SilentlyContinue) { exit 0 }; "
|
|
" Start-Sleep -Milliseconds 500 } ; exit 1",
|
|
timeout_s=120)
|
|
except (GuestError, GuestTimeout, OSError) as exc:
|
|
raise SessionError(
|
|
f"the share was attached but the machine did not pick it up: {exc}"
|
|
) from exc
|
|
|
|
|
|
def _sync_clock(session: Session) -> str:
|
|
"""Set the guest's clock from the host's, and CHECK that it took.
|
|
|
|
A resumed machine believes it is whenever its state was saved.
|
|
Measured on the first warm Windows session: twenty-six seconds
|
|
behind, and a snapshot left over a weekend would be days out. Windows
|
|
minds most - a wrong clock fails TLS, so every download breaks in a
|
|
way that reads as a network fault.
|
|
|
|
The first version set the time and swallowed any error, which is how
|
|
it came to be twenty-six seconds behind while reporting nothing: the
|
|
share service had just been restarted, the guest agent was busy for a
|
|
moment, the call failed and nobody was told. So this one sets the
|
|
time, READS IT BACK, and tries again if it did not take - and says
|
|
what happened rather than returning silence either way.
|
|
"""
|
|
tolerance = 2.0
|
|
last = "not attempted"
|
|
for attempt in range(4):
|
|
try:
|
|
with session.guest(timeout=30.0) as agent:
|
|
agent.set_time(int(time.time() * 1_000_000_000))
|
|
# guest-get-time answers in nanoseconds, UTC, from the
|
|
# guest's own clock - which is the thing being checked.
|
|
reported = int(agent.info_time())
|
|
off = abs(time.time() - reported / 1_000_000_000)
|
|
if off <= tolerance:
|
|
return f"set, within {off:.1f}s of the host"
|
|
last = f"still {off:.1f}s out"
|
|
except (GuestError, GuestTimeout, OSError) as exc:
|
|
last = str(exc)
|
|
time.sleep(2.0 * (attempt + 1))
|
|
return f"could not be set ({last})"
|
|
|
|
|
|
def _warm_the_shell(session: Session) -> None:
|
|
"""Pay for PowerShell's cold start while nobody is waiting on it.
|
|
|
|
MEASURED on this Windows image: the FIRST PowerShell in a freshly
|
|
booted guest took over ninety seconds - Defender inspecting it and
|
|
.NET warming up - and every one after that took five to nine. The
|
|
machine-facts probe, which is the first thing to run, timed out and
|
|
handed the agent nothing at all about the machine it was given.
|
|
|
|
The agent is writing its task list through all of this, so the cost
|
|
is already being paid by somebody else's clock. Deliberately
|
|
best-effort and deliberately silent: a machine that cannot warm up is
|
|
a machine the next call will report on properly.
|
|
"""
|
|
if session.os_name != "windows":
|
|
return
|
|
try:
|
|
with session.guest(timeout=30.0) as agent:
|
|
agent.exec(["powershell", "-NoProfile", "-NonInteractive", "-Command", "exit"],
|
|
timeout_s=240.0)
|
|
except (GuestError, GuestTimeout, OSError):
|
|
pass
|
|
|
|
|
|
def _windows_facts(session: Session) -> str:
|
|
"""The same question asked of Windows, including the one that matters.
|
|
|
|
Whether the project folder is there. virtiofs on Windows needs WinFsp
|
|
and the VirtIO-FS service, neither of which the guest agent implies -
|
|
so a Windows session can come up perfectly and have nowhere to put
|
|
anything that survives it. The agent is told, because the alternative
|
|
is that it works for an hour and the report links to an empty repo.
|
|
"""
|
|
# Written as a script rather than one long -Command line, and with
|
|
# single quotes throughout: the first version nested double quotes
|
|
# inside a Python string inside PowerShell and produced nothing at
|
|
# all, silently, which is the one outcome a fact-gathering probe
|
|
# must not have.
|
|
probe = r"""
|
|
$ErrorActionPreference = 'SilentlyContinue'
|
|
'os: ' + (Get-CimInstance Win32_OperatingSystem).Caption
|
|
'powershell: ' + $PSVersionTable.PSVersion.ToString()
|
|
$py = (Get-Command python.exe).Source
|
|
'python: ' + $(if ($py) { (& $py -V 2>&1) } else { 'not on PATH for this account' })
|
|
# By path, not by PATH: these are installed per-machine and the guest
|
|
# agent runs as SYSTEM, whose PATH is not the logged-in user's.
|
|
$known = @(
|
|
"$env:ProgramFiles\BraveSoftware\Brave-Browser\Application\brave.exe",
|
|
"${env:ProgramFiles(x86)}\BraveSoftware\Brave-Browser\Application\brave.exe",
|
|
"$env:ProgramFiles\Google\Chrome\Application\chrome.exe",
|
|
"$env:ProgramFiles\Mozilla Firefox\firefox.exe"
|
|
) | Where-Object { Test-Path $_ } | ForEach-Object { Split-Path $_ -Leaf }
|
|
'browsers: ' + $(if ($known) { $known -join ' ' } else { 'none found' })
|
|
'running as: ' + [Environment]::UserName + ' (the guest agent is LocalSystem, not the desktop user)'
|
|
# Measured on the first real Windows session: the agent wrapped its
|
|
# commands in `powershell.exe -Command "..."` although it was already in
|
|
# PowerShell, and used `&&`, which 5.1 does not have. Two wasted turns
|
|
# each time, and both are facts about this machine rather than advice.
|
|
'shell: your commands already run in PowerShell - do not wrap them in powershell.exe'
|
|
'note: this is PowerShell 5.1, which has no && or ||. Use ; or separate calls.'
|
|
$z = Get-PSDrive -Name Z
|
|
'project folder: ' + $(if ($z) { 'Z:\, ' + [math]::Round($z.Free/1GB,1) + ' GB free' } else { 'NOT MOUNTED - nothing you write will be kept. Say so rather than working around it.' })
|
|
"""
|
|
# MEASURED: PowerShell's first start under LocalSystem on a freshly
|
|
# booted Windows guest took 17 seconds, and the whole probe more than
|
|
# the 40 it was given - so the facts came back empty, silently, on
|
|
# exactly the machine that most needed them. Windows gets a minute
|
|
# and a half; it is once per session and it is the difference between
|
|
# the agent knowing there is no project folder and finding out.
|
|
try:
|
|
with session.guest(timeout=45.0) as agent:
|
|
result = agent.shell(probe, timeout_s=90.0)
|
|
except (GuestError, OSError) as exc:
|
|
return f"(could not read this machine's details: {exc})"
|
|
return (result.get("stdout") or "").strip()[:1200]
|
|
|
|
|
|
def _mount_share(session: Session) -> None:
|
|
"""Put the project folder at /mnt/project, whatever the image does.
|
|
|
|
THE SESSION MOUNTS IT, NOT THE IMAGE.
|
|
|
|
The share is one virtiofs tag, `project`, and the base image built by
|
|
the autoinstall carries an fstab line for it. An image installed by
|
|
hand does not, and the failure that produces is a bad one: everything
|
|
works, the agent writes its files, `git` on the host commits an empty
|
|
directory, and the only sign is that the Gitea repository has nothing
|
|
in it. Nothing in the guest and nothing on the host reports an error.
|
|
|
|
So the mount is made here, every session, from the side that knows
|
|
whether there is a share at all. It is idempotent - an image that
|
|
does carry the fstab line is already mounted and this is one
|
|
`mountpoint` check - and it means a hand-built image needs nothing
|
|
but qemu-guest-agent to be a working base.
|
|
"""
|
|
if not session.share:
|
|
return
|
|
if session.os_name != "ubuntu":
|
|
# Windows mounts virtiofs through WinFsp as a drive letter, set
|
|
# up inside the image; there is no equivalent command to run
|
|
# here. See vm/images/windows/build.md.
|
|
return
|
|
script = (
|
|
"set -e\n"
|
|
"mkdir -p /mnt/project\n"
|
|
"mountpoint -q /mnt/project || mount -t virtiofs project /mnt/project\n"
|
|
"mountpoint -q /mnt/project\n"
|
|
)
|
|
try:
|
|
with session.guest(timeout=30.0) as agent:
|
|
result = agent.shell(script, timeout_s=30.0)
|
|
except (GuestError, OSError) as exc:
|
|
raise SessionError(
|
|
f"the machine is up but its project folder could not be mounted: {exc}"
|
|
) from exc
|
|
if result["exit_code"] != 0:
|
|
detail = (result["stderr"] or result["stdout"]).strip()[:300]
|
|
raise SessionError(
|
|
"the machine is up but the project folder did not mount, so nothing "
|
|
"written in this session would be kept. "
|
|
f"`mount -t virtiofs project /mnt/project` said: {detail or 'nothing'}"
|
|
)
|
|
|
|
|
|
def _wait_for_guest(session: Session, timeout: float = BOOT_TIMEOUT_S) -> None:
|
|
"""Boot supervision: report honestly rather than pretend."""
|
|
deadline = time.monotonic() + timeout
|
|
_wait_for(lambda: os.path.exists(session.ga_path), 30.0,
|
|
"qemu did not create its guest-agent socket")
|
|
last: str = ""
|
|
while time.monotonic() < deadline:
|
|
if not session.alive():
|
|
raise SessionError(
|
|
"the machine stopped while it was booting. "
|
|
f"Check: journalctl -u owuivm.service -n 50"
|
|
)
|
|
try:
|
|
with session.guest(timeout=5.0) as agent:
|
|
if agent.ping(timeout=5.0):
|
|
return
|
|
except (GuestError, OSError) as exc:
|
|
last = str(exc)
|
|
time.sleep(1.0)
|
|
raise SessionError(
|
|
f"the {session.os_name} machine booted but its guest agent never "
|
|
f"answered within {timeout:.0f}s, so there is no way to run "
|
|
f"anything on it. Last error: {last or 'none'}"
|
|
)
|
|
|
|
|
|
def await_desktop(session: Session, timeout: float = 300.0) -> bool:
|
|
"""Block until the graphical session is usable. Raises if it never is.
|
|
|
|
The public form of the wait below, called lazily by vmd the first time
|
|
a session reaches for the screen - see _require_desktop. A job that
|
|
only runs commands never calls it and never pays for it.
|
|
"""
|
|
if not _wait_for_desktop(session, timeout):
|
|
raise SessionError(
|
|
"the desktop did not finish coming up within "
|
|
f"{timeout:.0f}s, so there is nothing on screen to look at"
|
|
)
|
|
return True
|
|
|
|
|
|
def _wait_for_desktop(session: Session, timeout: float) -> bool:
|
|
"""Wait for a graphical session that is actually ON THE SCREEN.
|
|
|
|
THE PROBE USED TO ASK THE WRONG QUESTION, TWICE.
|
|
|
|
First it asked whether any seat session existed - which gdm's own
|
|
greeter satisfies, long before the user is logged in. Then, with that
|
|
fixed, a machine still handed over showing a boot splash: measured,
|
|
`gnome-shell` was running, plymouth was gone, graphical.target was
|
|
active, and the framebuffer sat on the splash for over three minutes.
|
|
|
|
The reason is that the display shows the FOREGROUND VIRTUAL TERMINAL,
|
|
and the session was not on it: `fgconsole` said 1, the session's TTY
|
|
was tty2, and `loginctl` agreed the session was `Active=no`. Plymouth
|
|
had had tty1 and nothing switched away from it. `chvt 2` put the
|
|
desktop on screen in under a second.
|
|
|
|
That is also why it looked intermittent - sometimes gdm switches and
|
|
sometimes it does not. So this does not wait for the switch to happen
|
|
on its own; it makes it happen, and then checks.
|
|
"""
|
|
if timeout <= 0:
|
|
return False
|
|
if session.os_name == "windows":
|
|
return _wait_for_windows_desktop(session, timeout)
|
|
probe = (
|
|
# The operator's session, not gdm's greeter.
|
|
"sid=$(loginctl list-sessions --no-legend | "
|
|
"awk '$3!=\"gdm\" && $4==\"seat0\"{print $1}' | head -1); "
|
|
"[ -n \"$sid\" ] || exit 1; "
|
|
# Put it on the screen if something else still has the console.
|
|
"vt=$(loginctl show-session \"$sid\" -p TTY --value 2>/dev/null); vt=${vt#tty}; "
|
|
"case \"$vt\" in [0-9]*) "
|
|
" [ \"$(fgconsole 2>/dev/null)\" = \"$vt\" ] || chvt \"$vt\" 2>/dev/null ;; "
|
|
"esac; "
|
|
"[ \"$(loginctl show-session \"$sid\" -p Active --value 2>/dev/null)\" = yes ] || exit 1; "
|
|
"pgrep -x gnome-shell >/dev/null || exit 1; "
|
|
# Plymouth still running means the splash still owns the display.
|
|
"! pgrep -x plymouthd >/dev/null"
|
|
)
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with session.guest(timeout=10.0) as agent:
|
|
if agent.shell(probe, timeout_s=20)["exit_code"] == 0:
|
|
# MEASURED: mutter takes two to three seconds to
|
|
# repaint the whole screen after a VT switch. With a
|
|
# one-second settle the handover screenshot came back
|
|
# 3.4 KB - a near-black frame - and three seconds
|
|
# later it was the full 400 KB desktop. The first
|
|
# picture the agent sees is this one, so it waits.
|
|
time.sleep(3.0)
|
|
return True
|
|
except (GuestError, OSError):
|
|
pass
|
|
time.sleep(2.0)
|
|
return False
|
|
|
|
|
|
def _wait_for_windows_desktop(session: Session, timeout: float) -> bool:
|
|
"""Windows has no VT to switch and no compositor to wait for.
|
|
|
|
What it has is explorer.exe, which exists once somebody is logged in
|
|
and the shell has drawn itself. Auto-logon is part of the image
|
|
contract, so this is a wait rather than a question.
|
|
"""
|
|
probe = ("if (Get-Process explorer -ErrorAction SilentlyContinue) { exit 0 } else { exit 1 }")
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with session.guest(timeout=15.0) as agent:
|
|
if agent.shell(probe, timeout_s=25)["exit_code"] == 0:
|
|
time.sleep(2.0)
|
|
return True
|
|
except (GuestError, OSError):
|
|
pass
|
|
time.sleep(3.0)
|
|
return False
|
|
|
|
|
|
def _wait_for(predicate, timeout: float, message: str) -> None:
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
if predicate():
|
|
return
|
|
time.sleep(0.1)
|
|
raise SessionError(message)
|
|
|
|
|
|
def snapshot(os_name: str, settle: float = 45.0) -> dict:
|
|
"""Save a booted machine so later sessions can resume it.
|
|
|
|
WHAT IS SAVED, AND WHY IT IS FOUR THINGS RATHER THAN ONE.
|
|
|
|
A saved machine is memory plus the exact disk that memory describes,
|
|
plus the firmware variables it booted with. Separate them and nothing
|
|
complains: the guest simply runs with a filesystem in memory that
|
|
disagrees with the one on disk. So all of it is written together, and
|
|
a manifest records the checksum of the base image underneath - which
|
|
is what `warm_set` checks before trusting any of it.
|
|
|
|
Taken WITHOUT a project share, because virtiofsd refuses migration
|
|
("vhost-user backend lacks VHOST_USER_PROTOCOL_F_LOG_SHMFD"). The
|
|
share is plugged in after the resume instead.
|
|
"""
|
|
if os_name not in SUPPORTED_OS:
|
|
raise SessionError(f"'{os_name}' is not a machine this runs.")
|
|
base_sha = _base_checksum(os_name)
|
|
if not base_sha:
|
|
raise SessionError(
|
|
f"no checksum is recorded for the {os_name} image, so a saved machine "
|
|
"could never be shown to match it. Freeze the image first.")
|
|
|
|
session = create(os_name, None, wait=False, allow_warm=False)
|
|
try:
|
|
await_ready(session, desktop=(os_name != "windows"))
|
|
# Let the machine finish what a fresh boot does after it answers -
|
|
# service startup, the first-run work Windows does - so the saved
|
|
# machine is one somebody would want to be handed.
|
|
time.sleep(settle)
|
|
warm = os.path.join(WARM_ROOT, os_name)
|
|
staging = warm + ".new"
|
|
shutil.rmtree(staging, ignore_errors=True)
|
|
os.makedirs(staging, mode=0o751, exist_ok=True)
|
|
|
|
# Written straight into the staging directory through a descriptor
|
|
# vmd owns - see Qmp.save_state_to_fd. Four gigabytes that never
|
|
# get copied, and a file qemu could not have named.
|
|
state_path = os.path.join(staging, "state")
|
|
fd = os.open(state_path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o644)
|
|
try:
|
|
with Qmp(session.qmp_path, timeout=30.0) as qmp:
|
|
qmp.save_state_to_fd(fd)
|
|
finally:
|
|
os.close(fd)
|
|
# The disk first: it is the thing the memory describes.
|
|
subprocess.run(["/usr/bin/qemu-img", "convert", "-O", "qcow2",
|
|
"-B", _base_paths(os_name)[0], "-F", "qcow2",
|
|
session.overlay, os.path.join(staging, f"{os_name}.qcow2")],
|
|
capture_output=True, text=True, timeout=900, check=True)
|
|
shutil.copyfile(session.vars_file, os.path.join(staging, "OVMF_VARS.fd"))
|
|
with open(os.path.join(staging, "manifest.json"), "w", encoding="utf-8") as handle:
|
|
json.dump({"os": os_name, "base_sha256": base_sha,
|
|
"created": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
|
"state_bytes": os.path.getsize(state_path)}, handle, indent=2)
|
|
# WHO MAY READ WHAT, AND WHY THEY DIFFER.
|
|
#
|
|
# `state` is a four-gigabyte image of the machine's memory. It is
|
|
# readable by the qemu user, which has to load it, and by nobody
|
|
# else - a memory dump is not a thing to leave world-readable
|
|
# whatever is in it.
|
|
#
|
|
# `manifest.json` says which image the saved machine belongs to
|
|
# and when it was taken, and the desktop tool - which runs as the
|
|
# operator, not as either service user - shows that on the row.
|
|
# So the directory is traversable and the manifest is readable,
|
|
# and neither tells anyone anything that `ls -l` would not.
|
|
for name in os.listdir(staging):
|
|
os.chmod(os.path.join(staging, name),
|
|
0o644 if name == "manifest.json" else 0o640)
|
|
# In place only once every piece is written: a half-written warm
|
|
# set that passed the checksum test would be worse than none.
|
|
shutil.rmtree(warm, ignore_errors=True)
|
|
os.rename(staging, warm)
|
|
return {"os": os_name, "path": warm,
|
|
"state_gb": round(os.path.getsize(os.path.join(warm, "state")) / 1e9, 2),
|
|
"disk_mb": round(os.path.getsize(os.path.join(warm, f"{os_name}.qcow2")) / 1e6, 1),
|
|
"base_sha256": base_sha}
|
|
finally:
|
|
destroy(session)
|
|
|
|
|
|
def destroy(session: Session, graceful: bool = False) -> None:
|
|
"""Stop the machine and remove everything it used.
|
|
|
|
Not graceful by default, deliberately. The overlay is deleted a few
|
|
lines further down and the project share was committed and pushed
|
|
before this was called, so asking the guest to shut down politely
|
|
buys a tidier-looking exit and costs up to ninety seconds of CPU per
|
|
session. `systemctl stop` on a unit that now kills with SIGKILL is
|
|
the whole teardown.
|
|
|
|
Deliberately never raises. A teardown that fails loudly leaves the
|
|
unit in `failed` and the overlay on disk, and the NEXT caller pays
|
|
for it - which is the worst possible place to surface the problem.
|
|
"""
|
|
if graceful:
|
|
try:
|
|
with session.guest(timeout=5.0) as agent:
|
|
agent.shutdown("powerdown")
|
|
_wait_for(lambda: not session.alive(), 20.0, "")
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
_ctl("vm-stop", check=False)
|
|
_ctl("vm-reset", check=False)
|
|
_ctl("vfs-stop", check=False)
|
|
try:
|
|
os.unlink(session.overlay)
|
|
except OSError:
|
|
pass
|
|
shutil.rmtree(session.run_dir, ignore_errors=True)
|
|
try:
|
|
os.unlink(os.path.join(RUN_ROOT, "session.env"))
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _gid(name: str) -> int:
|
|
import grp
|
|
return grp.getgrnam(name).gr_gid
|