Grant/owui-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial Commit

Browse Source
This commit is contained in:
Grant
2026-09-15 15:24:03 -04:00
commit 4dc7086e7e
605 changed files with 101380 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/golden/agent_tool_schemas.json
+1624
View File
File diff suppressed because it is too large Load Diff
tests/matlab_availability.m
+38
View File
@@ -0,0 +1,38 @@
names = { ...
'fitlm','fitglm','fitcsvm','fitctree','fitrtree','ridge','lasso','anova1','anovan', ...
'readtable','writetable','table','timetable','categorical','string','strlength','contains', ...
'startsWith','endsWith','join','split','dictionary','containers.Map','arguments', ...
'bandpass','bandstop','lowpass','highpass','pspectrum','designfilt','butter','cheby1', ...
'fir1','filtfilt','freqz','spectrogram','findpeaks','resample','hilbert','xcorr', ...
'imread','imwrite','imshow','imresize','rgb2gray','bwlabel','regionprops','edge','imfilter', ...
'syms','vpa','solve','simplify','diff','int','laplace','ilaplace', ...
'trainNetwork','dlarray','predict','tf','ss','bode','step','impulse','margin','rlocus', ...
'lsim','pid','feedback','c2d','place','lqr','kalman', ...
'fit','cftool','polyfit','interp1','spline','fminsearch','fminunc','fmincon','linprog', ...
'quadprog','lsqnonlin','lsqcurvefit','ode45','ode15s','integral','quad', ...
'sim','gpuArray','parfeval','histfit','boxplot','normpdf','normcdf','ttest','ttest2', ...
'mvnrnd','pca','kmeans','cvpartition','crossval','confusionmat','randsample', ...
'strjoin','strsplit','validateattributes','inputParser','matches','pad','erase','insertAfter', ...
'websave','webread','jsondecode','jsonencode','datetime','duration','calendarDuration', ...
'movmean','movsum','movmedian','normalize','rmoutliers','fillmissing','groupsummary', ...
'histcounts','discretize','bounds','vecnorm','pagemtimes','tensorprod','deal','memoize' ...
};
missing = {}; present = {};
for k = 1:numel(names)
n = names{k};
e = 0;
try
e = exist(n);
catch
e = 0;
end
if e == 0
missing{end+1} = n;
else
present{end+1} = n;
end
end
printf('__MISSING__\n');
for k=1:numel(missing); printf('%s\n', missing{k}); end
printf('__PRESENT__\n');
for k=1:numel(present); printf('%s\n', present{k}); end
tests/matlab_availability.sh
Executable
+19
View File
@@ -0,0 +1,19 @@
#!/usr/bin/env bash
# Re-measure which MATLAB functions this sandbox actually has.
#
# _MATLAB_ONLY in routers/matlab.py must match reality, and reality moves
# when the image is rebuilt on a newer Debian or a forge package changes.
# Guessing this list was wrong in BOTH directions the first time: fitlm
# and imread are present, contains and datetime are not.
set -euo pipefail
cd "$(dirname "$0")/.."
exec docker run --rm --network none -u 10001:10001 \
-v "$(pwd)/tests/matlab_availability.m":/tmp/avail.m:ro \
--tmpfs /tmp/octhome:uid=10001 -e HOME=/tmp/octhome \
--entrypoint bash local/owuitools-octave:latest -c '
mkdir -p /tmp/octhome
octave --no-window-system --norc --quiet --eval "
warning(\"off\",\"all\");
try, pkg load signal; pkg load control; pkg load statistics; pkg load optim; catch, end
run(\"/tmp/avail.m\");
" 2>/dev/null'
tests/smoke.py
+569
View File
@@ -0,0 +1,569 @@
"""Exercise every owuitools endpoint the way Open WebUI would.
Runs on openwebui_default and talks to the gateway by name, so it takes the
same path a real tool call does - through nginx, into the guarded
namespace - rather than testing the app in isolation where the firewall,
the proxy and the auth header would all be bypassed.
"""
import time
import json
import os
import sys
import textwrap
import httpx
BASE = os.environ.get("OWUITOOLS_BASE", "http://owuitools-gw:8000")
TOKEN = os.environ["OWUITOOLS_AUTH_TOKEN"]
MAX_CHARS = int(os.environ.get("OWUITOOLS_MAX_RESPONSE_CHARS", "12000"))
VERBOSE = "-v" in sys.argv
# (tool, payload, must_appear_in_response_or_None[, must_NOT_appear])
# The fourth element is optional and may be one string or a tuple of them.
# Several behaviours here are only testable as an absence - an NXDOMAIN that
# must not be reported as a policy denial, a corrected hint that must not
# still carry the wrong claim - and a missing negative reads as a pass.
CASES = [
# --- offline / deterministic: these must always pass -----------------
("subnet_calc", {"network": "192.168.1.0/24"}, "192.168.1.255"),
("subnet_calc", {"network": "10.0.0.0/8", "split_into": 4}, "10.64.0.0/10"),
("subnet_calc", {"network": "2001:db8::/32"}, "2001:db8::"),
("network_reference", {"lookup_type": "port", "value": "443"}, "https"),
("network_reference", {"lookup_type": "http_status", "value": "418"}, "teapot"),
("network_reference", {"lookup_type": "mac_vendor", "value": "00:1A:2B:3C:4D:5E"}, None),
("ohms_law_power", {"voltage": "12", "resistance": "220"}, "54.55 mA"),
("ohms_law_power", {"power": "0.5", "resistance": "100"}, None),
("component_calc", {"calculation": "resistor_color_code", "text": "brown black red gold"}, "1 kΩ"),
("component_calc", {"calculation": "resistor_to_color", "value": "4k7"}, "yellow"),
("component_calc", {"calculation": "nearest_e_series", "value": "3300", "series": "E24"}, None),
("component_calc", {"calculation": "rc_filter", "resistance": "10k", "capacitance": "100n"}, "159.2 Hz"),
("component_calc", {"calculation": "lc_resonance", "inductance": "10u", "capacitance": "100n"}, None),
("component_calc", {"calculation": "voltage_divider", "voltage": "12", "resistance": "10k", "resistance2": "2k2"}, None),
("component_calc", {"calculation": "series_parallel", "text": "10k, 10k", "combination": "parallel"}, "5 kΩ"),
("pcb_calc", {"calculation": "trace_width", "current": "2"}, "mm"),
("pcb_calc", {"calculation": "microstrip_impedance", "trace_width": "0.3", "dielectric_height": "0.2"}, "Ω"),
("pcb_calc", {"calculation": "via_current", "via_diameter": "0.3", "current": "1"}, None),
("wiring_calc", {"calculation": "awg_properties", "awg": 14}, "mm²"),
("wiring_calc", {"calculation": "voltage_drop", "awg": 12, "length": "30", "current": "10", "voltage": "12"}, "%"),
("wiring_calc", {"calculation": "led_resistor", "voltage": "5", "led_forward_voltage": "2.1", "led_current": "0.02"}, "145 Ω"),
("wiring_calc", {"calculation": "decibels", "value": "30", "db_from": "dBm"}, "1 W"),
("wiring_calc", {"calculation": "battery_life", "capacity_mah": 2000, "load_ma": 150}, "hours"),
# --- MATLAB sandbox -------------------------------------------------
# These assert the three verdicts the tool exists to distinguish:
# it worked, it is not valid MATLAB, and it failed for a reason that
# is not the model's fault.
("matlab_run", {"code": "x = linspace(0,1,5);\ndisp(sum(x))"}, "MATLAB run OK"),
("matlab_run", {"code": "a=[1 2 3];\nb=[1 2];\nc=a*b;"}, "nonconformant"),
("matlab_run", {"code": "a=[1 2 3];\nb=[1 2];\nc=a*b;"}, "main.m line 3"),
("matlab_run", {"code": "for i=1:2\n printf('%d',i);\nendfor"}, "NOT VALID MATLAB"),
# Was: "contains is real MATLAB but not available here". Change 58 made
# it available, so what is pinned now is that it runs and that nothing
# tells the model to rewrite it.
("matlab_run", {"code": "s = contains('abc','b');\ndisp(s)"}, "1", "not available"),
("matlab_run", {"code": "while true; end", "timeout_seconds": 3}, "Timed out"),
("matlab_run", {"code": "function r=f(a)\n r=a;\nend"}, "function definition, not a script"),
("matlab_run", {"code": "disp(addup(2,3))",
"helper_files": [{"name": "addup.m", "code": "function r=addup(a,b)\n r=a+b;\nend"}]}, "5"),
("matlab_run", {"code": "x = webread('http://example.com');"}, "no network"),
("matlab_run", {"code": ""}, "CANNOT DO THAT"),
# Regressions found by adversarial review; each of these was broken.
("matlab_run", {"code": "x=1;\n# double it\nx=x*2;\ndisp(x)"}, "NOT VALID MATLAB"),
("matlab_run", {"code": "plot_style = 'x';\ns = sprintf('a', ...\n '--r');\ndisp(1)"}, "MATLAB run OK"),
("matlab_run", {"code": "A=[1 1;1 1];\nx=A\\[1;2];\ndisp('done')"}, "Warnings"),
("matlab_run", {"code": "a = zeros(60000,60000);", "timeout_seconds": 30}, "out of memory"),
("matlab_run", {"code": "disp(addup(2,3))",
"helper_files": [{"name": "2bad.m", "code": "function r=addup(a,b)\n r=a+b;\nend"}]},
"Helper files ignored"),
# --- network-dependent ----------------------------------------------
("web_search", {"query": "RFC 9293 TCP specification", "max_results": 4}, None),
("web_search", {"query": "systemd-resolved DNSSEC failure", "category": "it", "max_results": 3}, None),
("fetch_page", {"url": "https://example.com", "max_chars": 1500}, "Example Domain"),
("wikipedia_lookup", {"topic": "OSI model"}, "layer"),
("search_stackexchange", {"query": "bash trap EXIT", "site": "unix", "max_results": 2}, None),
("lookup_rfc", {"number": 9293, "offset": 0}, "RFC 9293"),
("lookup_rfc", {"query": "border gateway protocol"}, "Proposed Standard"),
("lookup_cve", {"cve_id": "CVE-2021-44228"}, "Log4j"),
("read_man_page", {"name": "ip", "section": 8}, "NAME"),
("lookup_package", {"name": "requests", "ecosystem": "pypi"}, "Latest version"),
("lookup_package", {"name": "torvalds/linux", "ecosystem": "github"}, "tags"),
("dns_lookup", {"name": "cloudflare.com", "record_type": "A"}, "TTL"),
("dns_lookup", {"name": "google.com", "record_type": "MX"}, None),
("inspect_tls", {"hostname": "example.com"}, "Issuer"),
("http_headers", {"url": "https://example.com"}, "Security headers"),
("lookup_ip_or_asn", {"target": "8.8.8.8"}, None),
("find_datasheet", {"part_number": "LM358"}, None),
("search_papers", {"query": "GaN transistor switching losses", "max_results": 2}, None),
# --- guardrails: these must be REFUSED, not fulfilled ----------------
("fetch_page", {"url": "http://192.168.1.10:9443/"}, "CANNOT DO THAT"),
("fetch_page", {"url": "http://172.19.0.2:8080/api/config"}, "CANNOT DO THAT"),
("fetch_page", {"url": "file:///etc/passwd"}, "CANNOT DO THAT"),
("fetch_page", {"url": "http://localhost:8000/health"}, "CANNOT DO THAT"),
("dns_lookup", {"name": "router.local", "record_type": "A"}, "CANNOT DO THAT"),
("lookup_ip_or_asn", {"target": "192.168.1.1"}, "CANNOT DO THAT"),
("ohms_law_power", {"voltage": "12"}, "CANNOT DO THAT"),
("component_calc", {"calculation": "rc_filter", "resistance": "10k"}, "CANNOT DO THAT"),
# Models pass "" for optionals they chose not to use; must not 422.
("web_search", {"query": "test", "site": "", "language": "", "max_results": 3}, None),
("lookup_rfc", {"number": 1918, "query": ""}, "1918"),
# --- citation metadata: a page has to say what it is -----------------
# The URLs are stable documents chosen for their metadata, not their
# ranking: a paper of record on PLOS and the arXiv abstract of
# "Attention Is All You Need". Two cases per page where two separate
# fields are being asserted, the same way the MATLAB cases above do.
("fetch_page", {"url": "https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0173152",
"max_chars": 1200}, "CITE AS: Ravenscroft"),
("fetch_page", {"url": "https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0173152",
"max_chars": 1200}, "Journal: PLOS ONE"),
("fetch_page", {"url": "https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0173152",
"max_chars": 1200}, "DOI: 10.1371"),
# Comma-formatted authors are "Vaswani, Ashish"; the surname is the cite.
("fetch_page", {"url": "https://arxiv.org/abs/1706.03762", "max_chars": 900},
"CITE AS: Vaswani", "CITE AS: Ashish"),
# What arrived is not always what citing it would imply.
("fetch_page", {"url": "https://arxiv.org/abs/1706.03762", "max_chars": 700}, "abstract page"),
("fetch_page", {"url": "https://pubmed.ncbi.nlm.nih.gov/32939066/", "max_chars": 900},
"not the paper"),
("fetch_page", {"url": "https://example.com", "max_chars": 800},
"No author, journal or DOI metadata"),
# --- reading inside a document ---------------------------------------
("fetch_page", {"url": "https://www.rfc-editor.org/rfc/rfc9293.txt",
"search": "Retransmission Timeout", "max_chars": 1200}, "Jumped to"),
("fetch_page", {"url": "https://example.com", "search": "zzzqqqnotpresent"},
"does not occur in this document"),
# The continuation hint pushes on; it must not read as permission to stop.
("fetch_page", {"url": "https://www.rfc-editor.org/rfc/rfc9293.txt", "max_chars": 1000},
"Keep reading if the answer may be further in", "Only do so if this section did not answer"),
("lookup_rfc", {"number": 1035, "search": "2.3.4"}, "skipped"),
# --- stub and failure honesty: a non-answer must not look like one ---
("fetch_page", {"url": "https://www.cisa.gov/known-exploited-vulnerabilities-catalog"},
"client-rendered or gated"),
# A name that does not exist is a DNS fact, not a firewall refusal; saying
# "private network" there sends the model looking for a way around it.
("fetch_page", {"url": "https://api.no-such-host-owuitools.invalid/x"},
"does not resolve", "private network"),
("inspect_tls", {"hostname": "no-such-host-owuitools.invalid"},
"does not resolve", "private network"),
# --- search relevance and category honesty ---------------------------
# An invented title returns something from every engine; the results are
# not about the query, and saying so is the whole point.
("web_search", {"query": '"Ferrous lattice resonance in dodecahedral copper" Whitlock Ambersley',
"max_results": 5}, "LOW RELEVANCE"),
("web_search", {"query": "RFC 9293 TCP retransmission timeout computation", "max_results": 4},
"Snippets are search-engine excerpts", "LOW RELEVANCE"),
("web_search", {"query": "amoxicillin adult dose sinusitis mg how much should i take",
"max_results": 4}, "dailymed"),
("web_search", {"query": "ibuprofen maximum daily dose mg adult", "category": "science",
"max_results": 4}, "scholarly indexes only"),
# Structural, not about which papers rank: some scholarly result carries
# enough metadata to cite. Only a dead scholarly index breaks this.
("web_search", {"query": "metformin lactic acidosis cohort study", "category": "science",
"max_results": 6}, "CITE AS:"),
# --- wikipedia resolution --------------------------------------------
("wikipedia_lookup", {"topic": "Mercury"}, "disambiguation page"),
("wikipedia_lookup", {"topic": "Marguerite Delacroix-Whitmore"}, "NO ARTICLE"),
# An exact title must win over whatever full-text search would have found.
("wikipedia_lookup", {"topic": "Canberra"}, "# Canberra"),
("wikipedia_lookup", {"topic": "Canberra", "full_article": True,
"search": "Federal Capital Territory"}, "Jumped to"),
("wikipedia_lookup", {"topic": "Eiffel Tower"}, "Introduction only"),
# --- v3: a citation the answer can lift ------------------------------
# Two graded rounds: 1 of 25 tool-assisted answers carried a URL, and the
# one that did lifted a ready-made CITE AS line. Every tool that returns a
# single identifiable document now ends with the line the answer wants.
("fetch_page", {"url": "https://example.com", "max_chars": 800},
"\n---\nSources: https://example.com"),
("wikipedia_lookup", {"topic": "Canberra"},
"\n---\nSources: https://en.wikipedia.org/wiki/Canberra"),
("lookup_rfc", {"number": 1035, "offset": 0}, "\n---\nSources: RFC 1035"),
("read_man_page", {"name": "ls"}, "\n---\nSources: man ls"),
("lookup_package", {"name": "requests", "ecosystem": "pypi"},
"\n---\nSources: https://"),
# A paper carries its citation as well as its URL.
("fetch_page", {"url": "https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0173152",
"max_chars": 1200}, "Sources: Ravenscroft"),
# Still there at the largest window the schema allows, where the page
# budget is tightest. (That `cap()` re-attaches it after trimming is a
# pure-function property; this asserts the room reservation upstream.)
("fetch_page", {"url": "https://www.rfc-editor.org/rfc/rfc9293.txt", "max_chars": 10000},
"\n---\nSources: https://www.rfc-editor.org/rfc/rfc9293.txt"),
# --- v3: a jump says which section it landed in -----------------------
# "Constitution" inside the Canberra article matched "1 Constitution
# Avenue" - a street - and the model read the window as having checked the
# constitutional basis for the capital. A character offset cannot say you
# are in the wrong subject; the enclosing heading can.
("wikipedia_lookup", {"topic": "Canberra", "full_article": True, "search": "Constitution"},
"inside the section"),
("lookup_rfc", {"number": 1035, "search": "63 octets"}, "inside the section"),
("read_man_page", {"name": "ls", "search": "--almost-all"}, "inside the section"),
# --- v3: a search miss does not throw the document away ---------------
# One search string missing is not the same finding as the document being
# useless; returning only "not here" made the two indistinguishable, and
# the model abandoned 54,000 characters of the right article.
("wikipedia_lookup", {"topic": "Sense", "full_article": True, "search": "zzzqqqnotpresent"},
"Showing the opening of the document instead"),
("lookup_rfc", {"number": 1035, "search": "zzzqqqnotpresent"},
"Showing the opening of the document instead"),
("read_man_page", {"name": "ls", "search": "zzzqqqnotpresent"},
"Showing the opening of the document instead"),
# --- v3: which of your terms is absent, as a count --------------------
# A model hunting a half-remembered paper puts its own guesses in the
# query; topically plausible results then read as partial confirmation.
("web_search", {"query": "Korhonen coffee randomized trial type 2 diabetes Circulation 2018",
"max_results": 6}, "Appears in none of these results"),
("web_search", {"query": "linux kernel scheduler documentation", "category": "it",
"max_results": 3}, None, "Appears in none of these results"),
# An index listing is not the document, and cannot be quoted as one.
("lookup_rfc", {"query": "border gateway protocol"}, "These are index entries"),
# --- per-tool additions ----------------------------------------------
("http_headers", {"url": "https://github.com"}, "max-age="),
# Splitting past /64 breaks SLAAC, which the split is silent about otherwise.
("subnet_calc", {"network": "2001:db8:1::/64", "split_into": 4}, "RFC 7421"),
# --- Change 55: the v3 round's tool defects ---------------------------
# A documentation address is named as such, with its RFC, and a public
# address carries its origin AS from the routing view, not just its
# registration.
("lookup_ip_or_asn", {"target": "203.0.113.10"}, "RFC 5737"),
("lookup_ip_or_asn", {"target": "8.8.8.8"}, "Origin AS"),
("subnet_calc", {"network": "203.0.113.0/24"}, "reserved for documentation"),
# Board dimensions with units, and no negative impedance ever printed.
("pcb_calc", {"calculation": "microstrip_impedance", "trace_width": "12mil", "dielectric_height": "8 mil"}, "0.305 mm"),
("pcb_calc", {"calculation": "microstrip_impedance", "trace_width": "0.3mm", "dielectric_height": "1.6mm"}, "Ω", "≈ -"),
("pcb_calc", {"calculation": "microstrip_impedance", "trace_width": "10", "dielectric_height": "0.1"}, "CANNOT DO THAT", "Ω**"),
# A search miss on an unsectioned name checks the sibling sections.
("read_man_page", {"name": "crontab", "search": "PATH"}, "section 5"),
# PMC and PubMed pages are served from the Europe PMC open-access API.
("fetch_page", {"url": "https://pubmed.ncbi.nlm.nih.gov/24459154/", "max_chars": 900}, "Retrieved via"),
("fetch_page", {"url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC3898757/", "max_chars": 900}, "Diabetes Care"),
# A PDF whose text layer is glyph codes is refused, not served as text.
("fetch_page", {"url": "https://www.infineon.com/assets/row/public/documents/24/49/infineon-irlz44n-datasheet-en.pdf",
"max_chars": 900}, "not decodable"),
# An oversized window is clamped; a bad parameter is prose, not a 422.
("fetch_page", {"url": "https://example.com", "max_chars": 10716}, "Example Domain", "422"),
("fetch_page", {"url": "https://example.com", "offset": -5}, "CANNOT DO THAT"),
# A page-numberless contents entry (RFC v3 text format) is stepped over.
("lookup_rfc", {"number": 9110, "search": "15.5.19"}, "skipped 1 table-of-contents"),
("lookup_rfc", {"number": 9110, "search": "15.5.19"}, "15.5. Client Error 4xx"),
# Parse-error and ordering hints.
("matlab_run", {"code": "x = 2;\nfprintf('today\\'s value: %d\\n', x);", "purpose": "scratch"}, "two quotes"),
# Was: the hint explaining that Octave cannot see a function defined
# below its call. Change 58 lifts those out instead, so the script the
# hint was written for now simply runs; the hint survives for the files
# the lift refuses, and is checked by tests/test_local_functions.py.
("matlab_run", {"code": "disp(addup(2,3))\nfunction r = addup(a,b)\n r = a+b;\nend", "purpose": "scratch"},
"5", "below the line that calls it"),
("subnet_calc", {"network": "192.168.1.0/24", "split_into": 4}, "192.168.1.192", "RFC 7421"),
("pcb_calc", {"calculation": "microstrip_width_for_impedance", "target_impedance": "50",
"dielectric_height": "0.2"}, "Width for"),
# IPC-2152 is not a licence to narrow an external trace, which is what the
# old wording invited.
("pcb_calc", {"calculation": "trace_width", "current": "3"},
"not a way to shrink an external trace", "usually less conservative standard if you need to squeeze"),
("read_man_page", {"name": "ip route"}, "ip-route", "No manual page found"),
# Both labels come off the vendor-pattern URL, not off search ranking:
# LM358DR strips to LM358, so the official PDF is a variant, not the part.
("find_datasheet", {"part_number": "LM358"}, "[EXACT LM358]"),
("find_datasheet", {"part_number": "LM358DR"}, "VARIANT: names 'LM358'"),
("lookup_cve", {"cve_id": "CVE-2024-6387"}, "openssh: before"),
("lookup_cve", {"cve_id": "CVE-2021-44228"}, "CISA KEV"),
("matlab_run", {"code": "n = numel(regexp('a b', '\\bb', 'match'));\ndisp(n)"}, "BACKSPACE"),
# purpose='scratch' is the model working for itself: give it the Octave
# equivalent and drop the MATLAB-validity verdict, which is not the job.
# `contains` is a shim now (Change 58) and runs, so the equivalent-advice
# path is tested with a name that is still missing.
("matlab_run", {"code": "s = split('a b');\ndisp(s)", "purpose": "scratch"},
"strsplit", "Do NOT rewrite"),
("matlab_run", {"code": "x=1;\n# double it\nx=x*2;\ndisp(x)", "purpose": "scratch"},
None, "NOT VALID MATLAB"),
# --- Change 56: the v4 round's easy wins ---------------------------------
# A cable length in feet is feet, not femtometres.
("wiring_calc", {"calculation": "voltage_drop", "awg": 12, "length": "100ft", "current": "20",
"voltage": "120"}, "30.48 m (100 ft)", ("fV", "1e-13")),
("wiring_calc", {"calculation": "voltage_drop", "awg": 12, "length": "100 ft", "current": "20",
"voltage": "120"}, "5.29%"),
# A bare "30m" is thirty metres, not thirty milli-anything.
("wiring_calc", {"calculation": "voltage_drop", "awg": 6, "length": "30m", "current": "20",
"voltage": "12"}, "over 30 m of"),
("wiring_calc", {"calculation": "voltage_drop", "awg": 6, "length": "100k", "current": "20",
"voltage": "12"}, "Could not read"),
("wiring_calc", {"calculation": "three_phase_drop", "awg": 4, "length": "250 ft", "current": "40",
"voltage": "480", "power_factor": 0.85}, "76.2 m (250 ft)"),
# The width at the other copper weight, so "2 oz saves 40%" is not worked by hand.
("pcb_calc", {"calculation": "trace_width", "current": "3", "temperature_rise": 10}, "At 2 oz copper"),
("pcb_calc", {"calculation": "trace_width", "current": "3", "temperature_rise": 10}, "26.9 mil"),
# One lint entry per rule, with the line numbers, not one per line.
("matlab_run", {"code": 'x = "a";\ny = "b";\nz = "c";\ndisp(1)', "purpose": "deliverable"},
"(3 places)"),
# The std/var weight flag, stated the right way round.
("matlab_run", {"code": "s = std([1 2 3], 0, 1);\ndisp(s)", "purpose": "scratch"}, "N-1"),
("matlab_run", {"code": "s = std([1 2 3]);\ndisp(s)", "purpose": "scratch"}, None, "weight flag"),
# A topic-shaped query falls back to its phrases instead of NO RESULTS.
("lookup_rfc", {"query": "DNS domain name maximum length 63 octets label"}, "RFC 1035"),
("lookup_rfc", {"query": "DNS domain name maximum length 63 octets label"}, "these titles contain"),
# Icon-only anchors keep their href, so an index page still leads somewhere.
("fetch_page", {"url": "https://investors.raspberrypi.com/reports", "max_chars": 6000},
"investors.raspberrypi.com/reports/1"),
# A byline's credential is not a co-author.
("fetch_page", {"url": "https://www.poison.org/articles/acetaminophen-take-it-safely",
"max_chars": 1200}, "Johnson-Arbor,", "Johnson-Arbor et al."),
# A site's own name is not a Dr com.
("fetch_page", {"url": "https://www.electricallicenserenewal.com/Electrical-Continuing-Education-Courses/NEC-Content.php?sectionID=808",
"max_chars": 1200}, None, "CITE AS: com,"),
# --- Change 58: closer to MATLAB ----------------------------------------
# A script that calls its local functions before defining them is
# ordinary MATLAB and used to fail here with "'double_it' undefined".
("matlab_run", {"code": "x = 3;\ndisp(double_it(x));\n\n"
"function y = double_it(v)\n y = 2 * v;\nend\n"},
"6", "undefined"),
# An error inside one of those functions still reports the line number
# of the model's own script, not of the file we made from it.
("matlab_run", {"code": "v = [1 2 3];\ndisp(pick(v));\n\n"
"function y = pick(x)\n y = x(9);\nend\n"},
"main.m line 5 (in pick)"),
# The shims: each is a MATLAB function this Octave does not have.
("matlab_run", {"code": "disp(contains('hello world', 'o w'));\n"
"disp(strlength({'a','abc'}));\n"
"disp(maxk([3 1 4 1 5], 2));\n"
"disp(histcounts([1 2 2 3], [1 2 3 4]));\n"
"disp(rmmissing([1 NaN 3]));",
"purpose": "scratch"}, "1 2 1", "undefined"),
# histcounts without edges refuses rather than inventing a binning rule.
("matlab_run", {"code": "disp(histcounts([1 2 3]));", "purpose": "scratch"},
"automatic bin selection is not reproduced"),
# MATLAB call forms this Octave does not accept, named as such.
("matlab_run", {"code": "disp(round(pi, 3));", "purpose": "scratch"}, "round(x * 10^n)"),
("matlab_run", {"code": "disp(sum(magic(3), 'all'));", "purpose": "scratch"}, "sum(A(:))"),
# Octave's own parser, reporting what no regex rule covers.
("matlab_run", {"code": "x = 5;\nx .*= 2;\ndisp(x)\n"}, "Octave's parser read"),
# A function named after a MATLAB built-in shadows it in the user's MATLAB.
("matlab_run", {"code": "disp(normalize([1 2]));\n\n"
"function y = normalize(x)\n y = x / max(x);\nend\n"},
"Shadows a MATLAB function"),
# Display and file format, two preferences set in the driver.
("matlab_run", {"code": "x = []", "purpose": "scratch"}, "x = []", "[](0x0)"),
("matlab_run", {"code": "x = 1;\nsave('d.mat', 'x');\n"
"f = fopen('d.mat','r'); h = fread(f, 24, 'char=>char')'; fclose(f);\n"
"disp(strtrim(h));", "purpose": "scratch"},
"MATLAB 5.0 MAT-file", "Created by Octave"),
# --- Change 57: plots that are not written on top of themselves ----------
# A figure comes back as a data URI whose first bytes ARE the assertion:
# PNG puts width and height at a fixed offset, so the base64 prefix below
# is exactly "1100 x 825" and nothing else. It fails if the shim is off
# the path, if print goes back to scaling type with -S, or if the canvas
# changes size - the three ways this rendering has broken.
("matlab_run", {"code": "t = linspace(0, 1, 200);\n"
"plot(t, sin(2*pi*t), 'LineWidth', 1.4);\n"
"title('Smoke test');\n"
"xlabel('Time (s)'); ylabel('Amplitude');\n"
"legend('one cycle');\n"
"print('plot.png', '-dpng');",
"return_figure": "plot.png", "purpose": "scratch"},
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAABEwAAAM5"),
# The shape follows the figure the script asked for: 1200x400 on screen
# is 1100x440 on paper, not a 4:3 picture with the plot squashed into it.
("matlab_run", {"code": "figure('position', [0 0 1200 400]);\n"
"plot(1:10, (1:10).^2);\n"
"print('plot.png', '-dpng');",
"return_figure": "plot.png", "purpose": "scratch"},
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAABEwAAAG4"),
# --- the VM sub-agent -------------------------------------------------
# Only the refusal paths belong in this suite. A real session starts a
# machine, runs a model with thinking on, and takes minutes to hours -
# far past run_case's 90 s - so the end-to-end case lives in its own
# opt-in script. What is checked here is that every way of asking for
# something impossible comes back as prose the model can act on rather
# than as a traceback, a 500, or a silent success.
("vm_agent", {"task": "run the build and report what happens",
"os": "ubuntu", "interface": "cli", "project": "../gitea"},
"CANNOT DO THAT"),
("vm_agent", {"task": "run the build and report what happens",
"os": "ubuntu", "interface": "cli", "project": "Bad_Name"},
"CANNOT DO THAT"),
("vm_agent", {"task": "run the build and report what happens",
"os": "plan9", "interface": "cli", "project": "smoke"},
"CANNOT DO THAT"),
# `interface` is VESTIGIAL and is ignored - the cli/gui split was removed
# when every session got the whole toolbox, and vm.py hardcodes "gui".
# This case used to pass a nonsense interface with a VALID project and
# expect a refusal, which stopped being true when the parameter stopped
# being read: the request became a perfectly good one and STARTED A REAL
# SESSION inside the smoke suite, which then died on run_case's 90 s
# timeout with a transport error. Measured 2026-09-15; the session was
# cancelled correctly by the disconnect path, so it left nothing behind
# and the only visible symptom was one confusing failure.
#
# The case is still worth having - a stray parameter must be tolerated,
# not 500 - so it keeps the nonsense interface and pairs it with a
# project name that is refused outright. Now it asserts both things and
# cannot start anything.
("vm_agent", {"task": "run the build and report what happens",
"os": "ubuntu", "interface": "telepathy", "project": "Bad_Name"},
"CANNOT DO THAT"),
# A task too short to act on: the agent cannot see the conversation, so
# "do it" is not a task, it is a guess waiting to happen.
("vm_agent", {"task": "fix it", "os": "ubuntu", "interface": "cli"},
"CANNOT DO THAT"),
# The pre-flight that exists because guessing the machine wrong costs a
# whole session: a Python task with no OS named in it must come back as a
# question for the user, not as a started VM.
("vm_agent", {"task": "write a python script that prints the first 20 primes and run it",
"os": "ubuntu", "interface": "cli", "project": "smoke"},
"CANNOT DO THAT YET"),
# The mirror of that - a task that DOES name its machine and must be let
# through - is deliberately not here: letting it through starts a real
# VM session, which is minutes long by design and would hang this suite.
# It is covered in tests/test_vm_preflight.py instead, against the
# function rather than over HTTP.
("vm_agent", {"cancel": True, "session_id": "not-a-session",
"task": "x" * 20, "os": "ubuntu", "interface": "cli"},
"CANNOT DO THAT"),
# A tool result that IS a data: URI is handed to the model as a vision
# input by Open WebUI, and budget.cap() passes it through untouched. A
# guest could otherwise choose a picture for the main model to look at.
("vm_agent", {"task": "run the build and report what happens",
"os": "ubuntu", "interface": "cli", "project": "../gitea"},
None, "data:image"),
# Nothing in a refusal should ever carry a credential or a host path.
("vm_agent", {"task": "run the build and report what happens",
"os": "plan9", "interface": "cli", "project": "smoke"},
None, ("GITEA_TOKEN", "Bearer", "/var/lib/owuivm", "Traceback")),
]
# The suite above sends no chat id, on purpose: note_fetch no-ops without one,
# so the per-conversation page budget never touches those cases and the order
# they run in cannot matter. The budget itself is only observable across an
# ordered sequence sharing one chat id, so these four run last, with that
# header attached per request rather than set on the client - it cannot leak
# back into anything above, and a re-run gets a fresh conversation.
FETCH_BUDGET_CASES = [
("fetch_page", {"url": "https://www.rfc-editor.org/rfc/rfc8446.txt", "max_chars": 800},
None, "STOP FETCHING"),
# Reading further into a page already open is free: same URL, new window.
("fetch_page", {"url": "https://www.rfc-editor.org/rfc/rfc8446.txt", "offset": 5000,
"max_chars": 800}, "characters 5,000-", "STOP FETCHING"),
("fetch_page", {"url": "https://api.no-such-host-owuitools.invalid/y"}, "does not resolve"),
# The same dead URL a second time is declined from memory, not re-attempted.
("fetch_page", {"url": "https://api.no-such-host-owuitools.invalid/y"}, "not retried"),
]
def run_case(client, case, failures, oversized, headers=None) -> bool:
"""Post one case, print its one-line result, return whether it passed."""
tool, payload, expect = case[:3]
forbid = case[3] if len(case) > 3 else None
forbidden = [forbid] if isinstance(forbid, str) else list(forbid or ())
label = f"{tool}({', '.join(f'{k}={v!r}' for k, v in list(payload.items())[:2])})"
try:
response = client.post(f"/tools/{tool}", json=payload, headers=headers)
except Exception as exc:
failures.append(f"{label}: transport error {type(exc).__name__}: {exc}")
print(f"FAIL {label}\n transport {type(exc).__name__}")
return False
if response.status_code != 200:
failures.append(f"{label}: HTTP {response.status_code} {response.text[:200]}")
print(f"FAIL {label}\n HTTP {response.status_code}: {response.text[:160]}")
return False
body = response.json()
if not isinstance(body, str):
failures.append(f"{label}: returned {type(body).__name__}, expected a string")
print(f"FAIL {label}\n returned {type(body).__name__}, not str")
return False
if not body.strip():
failures.append(f"{label}: empty body")
print(f"FAIL {label}\n empty response")
return False
# A figure is a data: URI, and a picture is allowed to be bigger than a
# page of prose: Open WebUI routes it to the image slot, not the text the
# cap exists to protect.
if len(body) > MAX_CHARS + 400 and not body.startswith("data:"):
oversized.append(f"{label}: {len(body)} chars")
if expect is not None and expect.lower() not in body.lower():
failures.append(f"{label}: expected {expect!r} in response; got: {body[:220]}")
print(f"FAIL {label}\n expected {expect!r}\n got: {body[:200]!r}")
return False
if present := [f for f in forbidden if f.lower() in body.lower()]:
failures.append(f"{label}: must not say {present!r}; got: {body[:220]}")
print(f"FAIL {label}\n must not contain {present!r}\n got: {body[:200]!r}")
return False
print(f"ok {label} [{len(body):,} chars]")
if VERBOSE:
print(textwrap.indent(body[:600], " | "))
return True
def main() -> int:
client = httpx.Client(
base_url=BASE,
headers={"Authorization": f"Bearer {TOKEN}",
# Unique per run: the probe limiter is per user per hour, and
# running this suite four times in an hour otherwise exhausts
# the budget and reports three real tools as broken.
"X-OpenWebUI-User-Id": f"smoketest-{os.getpid()}-{int(time.time())}",
"X-OpenWebUI-User-Email": "smoke@test",
"X-OpenWebUI-User-Role": "user"},
timeout=90.0,
)
passed = failed = 0
failures = []
oversized = []
for case in CASES:
ok = run_case(client, case, failures, oversized)
passed, failed = (passed + 1, failed) if ok else (passed, failed + 1)
# One chat id, this run only, for the ordered budget sequence. It goes on
# the requests rather than the client so nothing above can pick it up.
chat_headers = {"X-OpenWebUI-Chat-Id": f"smoketest-{os.getpid()}-{int(time.time())}"}
print("\n--- fetch budget: ordered, sharing one chat id ---")
for case in FETCH_BUDGET_CASES:
ok = run_case(client, case, failures, oversized, headers=chat_headers)
passed, failed = (passed + 1, failed) if ok else (passed, failed + 1)
print()
print("=" * 68)
print(f"passed {passed} / {passed + failed}")
if oversized:
print(f"\nOVER SIZE CAP ({MAX_CHARS} chars) — these would eat the context window:")
for item in oversized:
print(" ", item)
if failures:
print("\nFAILURES:")
for item in failures:
print(" -", item)
return 1
print("\nAll tools responded correctly and stayed within the size cap.")
return 0
if __name__ == "__main__":
sys.exit(main())
tests/smoke.sh
Executable
+18
View File
@@ -0,0 +1,18 @@
#!/usr/bin/env bash
# Run the full tool smoke test the way Open WebUI reaches the suite:
# from openwebui_default, through the gateway, with a real bearer token.
#
# ./tests/smoke.sh normal
# ./tests/smoke.sh -v also print the first lines of each response
set -euo pipefail
cd "$(dirname "$0")/.."
set -a; . ./.env; set +a
exec docker run --rm \
--network openwebui_default \
-e OWUITOOLS_AUTH_TOKEN="${OWUITOOLS_AUTH_TOKEN}" \
-e OWUITOOLS_MAX_RESPONSE_CHARS="${OWUITOOLS_MAX_RESPONSE_CHARS:-12000}" \
-v "$(pwd)/tests/smoke.py:/smoke.py:ro" \
--entrypoint python \
local/owuitools-api:latest /smoke.py "$@"
tests/test_local_functions.py
+217
View File
@@ -0,0 +1,217 @@
#!/usr/bin/env python3
"""Unit tests for the script-local function lift (matlab_lint.split_local_functions).
Pure Python, no container and no network:
python3 tests/test_local_functions.py
This one is a unit test rather than a smoke case because it is a source
transform on the user's own code. The smoke suite can only tell us that a
script ran; these cases are about the files it must NOT touch, and about
the line numbers staying where the model put them, which is what an error
message points at.
"""
import os
import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "app"))
from owuitools.matlab_lint import split_local_functions # noqa: E402
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(f"{label}: {detail}")
print(f"FAIL {label}\n {detail}")
def lifted(label: str, code: str, names: list[str], script_must_not: str | None = "function") -> None:
r = split_local_functions(code)
check(f"{label}: lifts {names}",
[f.name for f in r.functions] == names,
f"got {[f.name for f in r.functions]}, bailed={r.bailed!r}")
check(f"{label}: line count preserved",
len(r.script.split("\n")) == len(code.split("\n")),
f"{len(r.script.split(chr(10)))} vs {len(code.split(chr(10)))}")
if names and script_must_not:
check(f"{label}: definitions gone from the script",
script_must_not not in r.script,
r.script)
def bails(label: str, code: str) -> None:
r = split_local_functions(code)
check(f"{label}: left alone",
r.functions == () and r.script == code,
f"lifted {[f.name for f in r.functions]}, bailed={r.bailed!r}")
# --- the case this exists for -------------------------------------------
lifted("plain script", """x = 3;
disp(double_it(x));
function y = double_it(v)
y = 2 * v;
end
""", ["double_it"])
lifted("two functions", """disp(a(1) + b(2));
function y = a(v)
y = v;
end
function y = b(v)
y = v * 10;
end
""", ["a", "b"])
lifted("output list and no args", """disp(pair());
function [p, q] = pair()
p = 1; q = 2;
end
""", ["pair"])
lifted("no parentheses in signature", """disp(answer);
function answer = answer
answer = 42;
end
""", ["answer"])
# --- things that must not confuse the block scan ------------------------
lifted("one-line if/end above", """v = [1 2 3];
if numel(v) > 2, disp(tail(v)); end
function y = tail(x)
y = x(end);
end
""", ["tail"])
lifted("end used as an index", """v = [1 2 3];
disp(v(end));
disp(last(v));
function y = last(x)
s = struct('f', {1, 2});
y = x(end) + numel(s);
end
""", ["last"])
lifted("switch and try inside the function", """disp(pick(2));
function y = pick(n)
switch n
case 1
y = 'one';
otherwise
try
y = 'other';
catch
y = '';
end
end
end
""", ["pick"])
# The script keeps its own text verbatim, so the word survives inside the
# string and the comment - what must not survive is a real definition.
lifted("the word function inside a string", """s = 'function y = fake(x)';
disp(real_one(s));
function y = real_one(x)
y = numel(x);
end
""", ["real_one"], script_must_not=None)
lifted("a comment that looks like a definition", """% function y = commented(x)
disp(genuine(1));
function y = genuine(x)
y = x;
end
""", ["genuine"], script_must_not=None)
# --- files it must refuse to touch --------------------------------------
bails("function file", """function y = only(x)
y = x;
end
""")
bails("code after the definitions", """disp(f(1));
function y = f(x)
y = x;
end
disp('trailing statement');
""")
bails("classdef", """classdef Thing
properties
v = 1;
end
end
""")
bails("nested function", """disp(outer(1));
function y = outer(x)
function z = inner(w)
z = w + 1;
end
y = inner(x);
end
""")
bails("duplicate names", """disp(f(1));
function y = f(x)
y = x;
end
function y = f(x)
y = x + 1;
end
""")
bails("unbalanced end", """disp(1);
end
""")
bails("no functions at all", """x = 1;
disp(x);
""")
# A signature split across lines still names its function on the first
# line, and the whole block - continuation and all - is lifted verbatim.
lifted("continued signature", """disp(f(1));
function y = f(x, ...
z)
y = x;
end
""", ["f"])
# One that continues BEFORE the name has nothing to read, and is refused.
bails("name on the continued line", """disp(f(1));
function y = ...
f(x)
y = x;
end
""")
# --- the blanking must be exact -----------------------------------------
_code = "a = 1;\ndisp(f(a));\nfunction y = f(x)\n y = x;\nend\n"
_r = split_local_functions(_code)
check("script keeps its first lines verbatim",
_r.script.split("\n")[:2] == ["a = 1;", "disp(f(a));"],
_r.script.split("\n")[:2])
check("lifted text is the original source",
_r.functions[0].text == "function y = f(x)\n y = x;\nend\n",
repr(_r.functions[0].text))
check("lifted function remembers the line it came from",
_r.functions[0].line == 3, _r.functions[0].line)
check("tail is blank, not deleted",
_r.script.split("\n")[2:] == ["", "", "", ""],
repr(_r.script.split("\n")[2:]))
print()
print("=" * 60)
if FAILS:
print(f"FAILED {len(FAILS)}")
for f in FAILS:
print(" -", f)
sys.exit(1)
print("all local-function cases pass")
tests/test_tool_toggle.py
+134
View File
@@ -0,0 +1,134 @@
#!/usr/bin/env python3
"""The tool switch: configs/tools.toml and how main.py applies it.
python3 tests/test_tool_toggle.py
Pure Python, no container and no network. Two different things are being
checked and only one of them is about the loader.
The first is loader semantics - what a missing file, a typo, or an
unmentioned tool does. Those are all "fail open" by design, because this
is config that REMOVES capability and losing it should give you the
system it replaced rather than a server with no tools.
The second is that the file and the code agree. A tool added to a router
without a line here would be on by default, which is correct, but it
would also be invisible in the one file that is supposed to answer "what
does this deployment offer" - so the agreement is asserted rather than
hoped for.
The route-level behaviour - that a disabled tool leaves the OpenAPI spec
AND stops answering - needs fastapi and redis, so it lives in
tests/test_tool_toggle.sh, which runs it inside the api image.
"""
import os
import re
import sys
import tempfile
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "app"))
from owuitools import toolconfig # noqa: E402
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
CONFIG = os.path.join(ROOT, "configs", "tools.toml")
ROUTERS = os.path.join(ROOT, "app", "owuitools", "routers")
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(f"{label}: {detail}")
print(f"FAIL {label}\n {detail}")
def implemented() -> set[str]:
"""Every operation_id the routers define, read from the source.
Deliberately a regex over the files rather than an import: the app
needs fastapi and redis, and this test is meant to run anywhere.
"""
found = set()
for name in sorted(os.listdir(ROUTERS)):
if not name.endswith(".py"):
continue
with open(os.path.join(ROUTERS, name), encoding="utf-8") as handle:
found |= set(re.findall(r'operation_id="([a-z0-9_]+)"', handle.read()))
return found
# --- the file itself --------------------------------------------------
config = toolconfig.load(CONFIG)
tools = implemented()
check("configs/tools.toml parses and is not empty", bool(config), repr(config)[:120])
check("every tool the routers define has a line in the config",
tools <= set(config), f"missing: {sorted(tools - set(config))}")
check("the config names no tool that does not exist",
set(config) <= tools, f"unknown: {sorted(set(config) - tools)}")
check("every tool ships enabled by default",
all(config.values()), f"off: {sorted(k for k, v in config.items() if not v)}")
# Every tool that needs setup beyond `docker compose up` must say where
# to read about it. A switch you cannot act on is worse than no switch.
with open(CONFIG, encoding="utf-8") as handle:
body = handle.read()
for needs_setup in ("web_search", "wikipedia_lookup", "search_stackexchange",
"lookup_cve", "lookup_package", "matlab_run", "vm_agent",
"find_datasheet", "search_papers"):
block = body.rsplit(f"\n{needs_setup} = ", 1)[0]
comment = block.rsplit("\n\n", 1)[-1]
check(f"{needs_setup} points at a setup doc",
"docs/setup/" in comment or "Setup:" in comment,
f"no docs/setup pointer in the comment above {needs_setup}")
# --- loader semantics -------------------------------------------------
check("a missing config file leaves every tool on",
toolconfig.load("/nonexistent/tools.toml") == {}, "should be empty, i.e. no opinion")
check("a missing config disables nothing",
toolconfig.disabled(tools, "/nonexistent/tools.toml") == set(), "should disable nothing")
with tempfile.NamedTemporaryFile("w", suffix=".toml", delete=False) as handle:
handle.write("[tools]\nweb_search = false\nnot_a_tool = false\nfetch_page = true\n")
partial = handle.name
off = toolconfig.disabled(tools, partial)
check("a tool set to false is reported disabled", off == {"web_search"}, f"got {off}")
check("a name the server does not implement is ignored, not disabled",
"not_a_tool" not in off, f"got {off}")
check("a tool the config does not mention stays on",
"subnet_calc" not in off, f"got {off}")
with tempfile.NamedTemporaryFile("w", suffix=".toml", delete=False) as handle:
handle.write("[tools]\nthis is not valid toml = = =\n")
broken = handle.name
check("an unparseable config leaves every tool on",
toolconfig.disabled(tools, broken) == set(), "a broken file must not remove tools")
os.unlink(partial)
os.unlink(broken)
# --- the seam -------------------------------------------------------
# FastAPI 0.141 stopped flattening included routers: include_router()
# appends one opaque _IncludedRouter to app.routes, so filtering
# app.routes removes nothing while looking exactly like it worked.
# Measured - the first version of this feature shipped that bug.
with open(os.path.join(ROOT, "app", "owuitools", "main.py"), encoding="utf-8") as handle:
main_src = handle.read()
check("the filter runs on the routers, before include_router",
"for router in _routers for route in router.routes" in main_src,
"main.py must filter router.routes, not app.routes")
check("main.py does not filter app.routes",
"app.routes[:]" not in main_src,
"app.routes holds no APIRoute on fastapi >= 0.141; that filter is a no-op")
print()
if FAILS:
print(f"{len(FAILS)} FAILED")
for line in FAILS:
print(f" - {line}")
raise SystemExit(1)
print("all tool-toggle checks passed")
tests/test_tool_toggle.sh
Executable
+18
View File
@@ -0,0 +1,18 @@
#!/usr/bin/env bash
# The half of the tool switch that needs a real app: the OpenAPI spec and
# the routes. Runs inside the api image because it needs fastapi and redis,
# with the source mounted over the baked copy so it tests the tree you are
# editing rather than the last image you built.
#
# ./tests/test_tool_toggle.sh
#
# No network: --network none, and nothing here reaches out.
set -euo pipefail
cd "$(dirname "$0")/.."
exec docker run --rm --network none \
-v "$(pwd)/app/owuitools:/app/owuitools:ro" \
-v "$(pwd)/configs:/configs:ro" \
-v "$(pwd)/tests/tool_toggle_routes.py:/tool_toggle_routes.py:ro" \
--entrypoint python \
local/owuitools-api:latest /tool_toggle_routes.py "$@"
tests/test_vm_agent_stream.py
+133
View File
@@ -0,0 +1,133 @@
#!/usr/bin/env python3
"""Streaming exists for the person watching, and may not change the turn.
python3 tests/test_vm_agent_stream.py
WHY
The agent's calls were non-streaming on purpose: it acts on a finished
message, so nothing in the loop needed tokens as they arrived. That was
true of the agent and false of the operator, who asked to see what the
model is saying while it says it - a turn on this model runs tens of
seconds, and a motionless box makes "still thinking" look like "wedged".
So the reply is now read as it arrives. The rule that makes that safe is
that it is a DISPLAY change and nothing else: the assembled message handed
back must be identical in shape and content to the one the non-streaming
call returned, because the agent is not allowed to care which path
produced it. These cases are all about that equivalence, plus the one
about the callback - which belongs to the watcher, so it must never be
able to fail a turn.
"""
import json
import os
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.join(HERE, "..", "vmagent", "app"))
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(label)
print(f"FAIL {label}\n {detail}")
import ollama # noqa: E402
CALL = {"function": {"name": "run", "arguments": {"command": "uname -a"}}}
def chunks(*entries) -> list:
return [json.dumps(e).encode() for e in entries]
# --- the assembled message ----------------------------------------------
seen: list[str] = []
body = ollama._read_stream(chunks(
{"message": {"content": "Check"}},
{"message": {"content": "ing the"}},
{"message": {"content": " kernel."}},
{"message": {"tool_calls": [CALL]}},
{"done": True, "done_reason": "stop", "prompt_eval_count": 120, "eval_count": 44,
"message": {}},
), seen.append)
check("the text is joined in order", body["message"]["content"] == "Checking the kernel.",
body["message"]["content"])
check("tool calls survive the stream", body["message"]["tool_calls"] == [CALL])
check("the counts come off the final chunk",
body.get("prompt_eval_count") == 120 and body.get("eval_count") == 44)
check("and so does why it stopped", body.get("done_reason") == "stop")
check("the role is set, as the non-streaming reply sets it",
body["message"]["role"] == "assistant")
# The watcher is handed the text SO FAR, not the delta - it paints a line,
# it does not append to one.
check("the callback gets the running text, not the fragment",
seen == ["Check", "Checking the", "Checking the kernel."], seen)
# --- thinking ------------------------------------------------------------
thought: list[str] = []
body = ollama._read_stream(chunks(
{"message": {"thinking": "weighing it up"}},
{"message": {"content": "Right, "}},
{"message": {"thinking": " and more"}},
{"message": {"content": "doing it."}},
{"done": True, "message": {}},
), thought.append)
check("thinking is kept on the message", body["message"]["thinking"] == "weighing it up and more")
check("it is shown while there is nothing else to show", thought[0] == "weighing it up",
"a model that is thinking has not said anything yet, and a blank minute reads as hung")
check("but once it speaks, its words are what is shown",
thought[-1] == "Right, doing it." and " and more" not in thought[-1],
thought)
# --- nothing in a frame may cost the turn --------------------------------
body = ollama._read_stream(
[b'{"message": {"content": "a"}}', b"not json at all", b"",
b'{"message": {"content": "b"}, "done": true}'],
lambda _t: None)
check("a malformed frame is skipped, not raised on",
body["message"]["content"] == "ab",
"losing a whole turn over one bad line would be a poor trade")
def explode(_text):
raise RuntimeError("the watcher's callback failed")
try:
body = ollama._read_stream(chunks(
{"message": {"content": "still works"}}, {"done": True, "message": {}}), explode)
survived = body["message"]["content"] == "still works"
except Exception as exc: # noqa: BLE001
survived = False
why = f"{type(exc).__name__}: {exc}"
check("a callback that throws cannot fail the turn", survived, locals().get("why", ""))
# --- an empty reply is still a reply --------------------------------------
body = ollama._read_stream(chunks({"done": True, "message": {}}), seen.append)
check("a reply with nothing in it assembles rather than crashing",
body["message"]["content"] == "" and body["message"]["tool_calls"] == [])
# --- and the old path is untouched ----------------------------------------
src = open(os.path.join(HERE, "..", "vmagent", "app", "ollama.py"), encoding="utf-8").read()
check("streaming is off unless somebody is watching",
'"stream": on_text is not None' in src,
"no caller without a callback may be made to pay for this")
check("only the agent's own turn streams; the summariser does not",
open(os.path.join(HERE, "..", "vmagent", "app", "loop.py"),
encoding="utf-8").read().count("on_text=") == 1,
"nobody watches a compaction summary")
print()
if FAILS:
print(f"FAILED {len(FAILS)}")
raise SystemExit(1)
print("the watcher sees the model talking, and the turn cannot tell the difference")
tests/test_vm_agent_tool_config.py
+168
View File
@@ -0,0 +1,168 @@
#!/usr/bin/env python3
"""The sub-agent's tool definitions, now that they live in a config file.
python3 tests/test_vm_agent_tool_config.py
Pure Python, no container and no network.
WHY THE GOLDEN FILE EXISTS
configs/agent/tools.toml replaced ~380 lines of hand-written Python
schemas. Every description in it is prompt text that took measurement to
arrive at, so the move had to be provably lossless rather than
approximately right - a dropped sentence in a tool description is not a
crash, it is a session that goes wrong later for no visible reason.
tests/golden/agent_tool_schemas.json is what the hand-written code
produced, captured before it was touched. If this test passes, the
config-driven builder is byte-identical to it for both machines and for
the booting case.
Regenerating the golden is therefore a deliberate act: do it only when
you MEANT to change what the agent is told, and read the diff.
"""
import importlib
import json
import os
import sys
import tempfile
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
sys.path.insert(0, os.path.join(ROOT, "vmagent", "app"))
import tools # noqa: E402
GOLDEN = os.path.join(ROOT, "tests", "golden", "agent_tool_schemas.json")
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(f"{label}: {detail}")
print(f"FAIL {label}\n {detail}")
def reload_with(**env):
"""Re-import tools.py against a different config, then put it back."""
saved = {k: os.environ.get(k) for k in env}
os.environ.update({k: v for k, v in env.items() if v is not None})
try:
return importlib.reload(tools)
finally:
for key, value in saved.items():
if value is None:
os.environ.pop(key, None)
else:
os.environ[key] = value
# --- the golden ------------------------------------------------------
with open(GOLDEN, encoding="utf-8") as handle:
golden = json.load(handle)
for key, want in golden.items():
got = tools.schemas("ubuntu", booting=True) if key == "booting" else tools.schemas(key)
check(f"{key}: identical to the hand-written schemas", got == want,
f"{len(got)} tools vs {len(want)}; "
f"first difference at "
f"{next((w['function']['name'] for g, w in zip(got, want) if g != w), 'length')}")
# --- the registry ----------------------------------------------------
registry = tools.machines()
check("configs/vms.toml defines both machines",
set(registry) == {"ubuntu", "windows"}, str(sorted(registry)))
check("every machine has the three prompt fields",
all(all(m.get(k) for k in ("shell", "project", "example")) for m in registry.values()),
str(registry))
# A placeholder that survives into a rendered schema is prompt text the
# model reads as literal "{project}", which is worse than either machine's
# real value. Cheap to assert, and invisible if it ever happens.
for os_name in ("ubuntu", "windows"):
blob = json.dumps(tools.schemas(os_name))
leftover = [p for p in ("{shell}", "{project}", "{example}") if p in blob]
check(f"{os_name}: no unsubstituted placeholders", not leftover, str(leftover))
check("the two machines differ in exactly the tools that mention the shell",
[t["function"]["name"]
for t, w in zip(tools.schemas("ubuntu"), tools.schemas("windows")) if t != w]
== ["run", "write_file"],
"only run and write_file should vary by machine")
# --- what the config may and may not do -------------------------------
with tempfile.TemporaryDirectory() as tmp:
# A name with no dispatch behind it must be refused when it is read,
# naming the file - not offered to the model and then failing.
with open(os.path.join(tmp, "tools.toml"), "w", encoding="utf-8") as handle:
handle.write('[[tool]]\nname = "unfreeze_base_image"\ngroup = "cli"\n'
'enabled = true\nrequired = []\ndescription = "no"\n')
refused = False
try:
reload_with(OWUITOOLS_AGENT_CONFIG_DIR=tmp,
OWUITOOLS_VMS_CONFIG=os.path.join(ROOT, "configs", "vms.toml"))
tools.schemas("ubuntu")
except tools.ToolConfigError as exc:
refused = "unfreeze_base_image" in str(exc) and "DISPATCHABLE" in str(exc)
except Exception: # noqa: BLE001
refused = False
check("config cannot operator the agent a verb that is not in DISPATCHABLE", refused,
"a name with no dispatch must raise ToolConfigError naming the file")
# enabled = false removes a tool.
with open(os.path.join(tmp, "tools.toml"), "w", encoding="utf-8") as handle:
handle.write('[[tool]]\nname = "run"\ngroup = "cli"\nenabled = false\n'
'required = []\ndescription = "x"\n'
'[[tool]]\nname = "screenshot"\ngroup = "gui"\nenabled = true\n'
'required = []\ndescription = "y"\n')
reload_with(OWUITOOLS_AGENT_CONFIG_DIR=tmp,
OWUITOOLS_VMS_CONFIG=os.path.join(ROOT, "configs", "vms.toml"))
names = [t["function"]["name"] for t in tools.schemas("ubuntu")]
check("enabled = false removes the tool", names == ["screenshot"], str(names))
# A lookup the SERVER has switched off must not be offered here: it
# would 404, which reads as a broken tool rather than an absent one.
with open(os.path.join(tmp, "tools.toml"), "w", encoding="utf-8") as handle:
handle.write('[[tool]]\nname = "search_papers"\ngroup = "research"\nenabled = true\n'
'required = []\ndescription = "x"\n'
'[[tool]]\nname = "read_page"\ngroup = "cli"\nenabled = true\n'
'required = []\ndescription = "y"\n'
'[[tool]]\nname = "check_internet"\ngroup = "cli"\nenabled = true\n'
'required = []\ndescription = "z"\n')
server = os.path.join(tmp, "server-tools.toml")
with open(server, "w", encoding="utf-8") as handle:
# fetch_page is what the agent's read_page is a call to.
handle.write('[tools]\nsearch_papers = false\nfetch_page = false\n')
reload_with(OWUITOOLS_AGENT_CONFIG_DIR=tmp,
OWUITOOLS_VMS_CONFIG=os.path.join(ROOT, "configs", "vms.toml"),
OWUITOOLS_TOOLS_CONFIG=server)
names = [t["function"]["name"] for t in tools.schemas("ubuntu")]
check("a lookup the server switched off is not offered to the agent",
names == ["check_internet"], str(names))
check("and read_page follows the server's fetch_page, not its own name",
"read_page" not in names, str(names))
reload_with(OWUITOOLS_AGENT_CONFIG_DIR=None)
# --- dispatch still lines up -----------------------------------------
check("every offered tool is dispatchable",
{t["function"]["name"] for t in tools.schemas("ubuntu")} <= tools.DISPATCHABLE,
"schemas() offered something call() cannot route")
# vmd's allowlist is the real gate - the agent asking for a lookup vmd
# does not list gets a refusal, not a result. The two must be one set.
sys.path.insert(0, os.path.join(ROOT, "vm", "vmd"))
import websearch # noqa: E402
check("the research group matches vmd's own allowlist",
tools.RESEARCH_NAMES == frozenset(websearch.RESEARCH_TOOLS),
f"agent={sorted(tools.RESEARCH_NAMES)} vmd={sorted(websearch.RESEARCH_TOOLS)}")
print()
if FAILS:
print(f"{len(FAILS)} FAILED")
for line in FAILS:
print(f" - {line}")
raise SystemExit(1)
print("the externalised tool definitions are faithful")
tests/test_vm_boot_overlap.py
Executable
+103
View File
@@ -0,0 +1,103 @@
#!/usr/bin/env python3
"""The planning turn happens while the machine boots, not after it.
python3 tests/test_vm_boot_overlap.py
WHY
A machine takes twenty-odd seconds to boot and the agent's first turn -
prefill, thinking, one tool call - takes about as long. Done in sequence
that is a minute of one side waiting for the other, every session. Done
at once it is free, and it has a second effect worth as much: the model
is loaded and its prompt prefilled by the time there is anything to do.
The risk in overlapping them is an agent that tries to touch a machine
that is not there yet. Two things prevent it, and this file checks both:
the loop does not OFFER the guest tools while booting, and vmd refuses
them regardless of what was offered.
"""
import os
import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "vmagent", "app"))
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "vm", "vmd"))
import tools # noqa: E402
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(f"{label}: {detail}")
print(f"FAIL {label}\n {detail}")
# --- what is offered while the machine is coming up ---------------------
for interface in ("cli", "gui"):
booting = [t["function"]["name"] for t in tools.schemas("ubuntu", booting=True)]
ready = [t["function"]["name"] for t in tools.schemas("ubuntu", booting=False)]
check(f"{interface}: planning tools are offered while booting",
"plan_tasks" in booting, booting)
guest = [n for n in booting
if n in ("run", "write_file", "read_file", "list_dir", "screenshot",
"click", "type", "key", "wait", "check_internet", "git_commit",
"reboot_vm", "finish")]
check(f"{interface}: no guest tool is offered while booting",
not guest, f"offered {guest}")
check(f"{interface}: the full set arrives once ready",
"run" in ready and "finish" in ready, ready)
check("gui gets the screen tools only once ready",
"screenshot" in [t["function"]["name"] for t in tools.schemas("ubuntu", booting=False)]
and "screenshot" not in [t["function"]["name"] for t in tools.schemas("ubuntu", booting=True)],
"screenshot offered at the wrong time")
# --- the wait tool ------------------------------------------------------
cli = {t["function"]["name"]: t for t in tools.schemas("ubuntu", booting=False)}
check("wait is available at the command line, not only in a desktop session",
"wait" in cli, sorted(cli))
wait_text = (cli["wait"]["function"]["description"]
+ str(cli["wait"]["function"]["parameters"])).lower()
# The cap is one more sentence paid for on every turn, and it only
# matters in the case where it is exceeded - where the tool says so.
check("the wait description does not spend tokens on the cap",
not any(w in wait_text for w in ("20 minutes", "1200", "maximum", "capped", "limit")),
wait_text[:160])
import vmd # noqa: E402
check("the cap exists in software", vmd.MAX_WAIT_SECONDS == 1200.0, vmd.MAX_WAIT_SECONDS)
# --- the gate lives in vmd too, not only in what the loop offers --------
src = open(os.path.join(os.path.dirname(__file__), "..", "vm", "vmd", "vmd.py")).read()
check("every guest verb is gated on readiness",
src.count("_require_ready()") >= 12, src.count("_require_ready()"))
check("the gate tells 'no machine' apart from 'still booting'",
"there is no machine running" in src and "still booting" in src, "")
check("session_ready is not a tool the agent can call",
"session_ready" not in str(tools.schemas("ubuntu", booting=False)),
"the agent can ask whether the machine is up, and it will, repeatedly")
# --- the opening message sets the expectation ---------------------------
import loop # noqa: E402
job = loop.Job(task="do a thing", os_name="ubuntu", interface="gui", project=None)
opening = loop._opening_message(job, {})
for phrase in ("booting", "plan_tasks", "told"):
check(f"the opening message mentions {phrase!r}", phrase in opening, opening[:200])
check("it does not claim the machine is usable yet",
"is booting" in opening or "not ready" in opening, opening[:200])
print()
print("=" * 60)
if FAILS:
print(f"FAILED {len(FAILS)}")
for f in FAILS:
print(" -", f)
sys.exit(1)
print("all boot-overlap cases pass")
tests/test_vm_finish_shapes.py
+89
View File
@@ -0,0 +1,89 @@
#!/usr/bin/env python3
"""`finish` must accept what the model actually sends, not only the schema.
python3 tests/test_vm_finish_shapes.py
WHY
A session wrote the script, syntax-checked it, built a fourteen-file test
tree with awkward filenames, cross-checked its output against an
independent listing, demonstrated the error paths, committed, and passed
its own verification command. Then it could not close.
`finish` declares `verification` as an object - `{command, expected}` -
and the model sends the command as a bare string often enough to matter.
`v_finish` called `.get` on it, which raised
AttributeError: 'str' object has no attribute 'get'
out of the verb. The agent saw only "the supervisor refuses the call",
tried seven times with varying arguments, then marked the task blocked
and gave up. All the work was done and committed; the session simply
could not be ended.
The identical mistake had already been found and fixed in the live view's
renderer. That one cost a line of output. This one cost the end of a
session - so the rule is the same wherever a model-supplied value is
read: take what arrives, not what was asked for.
"""
import os
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
VMD = os.path.join(HERE, "..", "vm", "vmd")
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(label)
print(f"FAIL {label}\n {detail}")
SRC = open(os.path.join(VMD, "vmd.py"), encoding="utf-8").read()
body = SRC[SRC.index("def v_finish"):SRC.index("def v_report")]
def normalise(value):
"""The shape-tolerant read, lifted out of v_finish so it can be tested."""
check_ = value
if isinstance(check_, str):
check_ = {"command": check_}
elif not isinstance(check_, dict):
check_ = {}
return str(check_.get("command", "")).strip(), str(check_.get("expected", "")).strip()
# --- what the model sends -------------------------------------------------
check("the documented shape works",
normalise({"command": "pytest -q", "expected": "0 failed"})
== ("pytest -q", "0 failed"))
check("a bare string is taken as the command",
normalise("bash -n topfiles.sh") == ("bash -n topfiles.sh", ""),
"this is the one that raised AttributeError and ended the session")
check("a missing verification is empty, not an exception",
normalise(None) == ("", ""))
for junk in (12, [], ["a"], True):
got = normalise(junk)
check(f"{type(junk).__name__} does not raise", got == ("", ""), str(got))
# --- and the real verb does the same --------------------------------------
check("v_finish normalises before reading",
"isinstance(check, str)" in body and 'check = {"command": check}' in body,
"the verb itself has to do this, not just this test")
check("it no longer calls .get on whatever arrived",
'(args.get("verification") or {}).get(' not in body,
"that is the exact expression that raised")
check("a verification with no command is still refused clearly",
"finish needs a verification" in body,
"tolerating the shape must not mean accepting nothing")
print()
if FAILS:
print(f"FAILED {len(FAILS)}")
raise SystemExit(1)
print("finish takes what the model sends, and a finished session can end")
tests/test_vm_freeze.sh
Executable
+283
View File
@@ -0,0 +1,283 @@
#!/bin/bash
# =====================================================================
# The freeze must happen, and must happen first.
#
# ./tests/test_vm_freeze.sh
#
# WHY THIS FILE EXISTS
#
# Freezing a base image is one `chmod 0444`, which makes it look like
# the kind of thing that cannot go wrong. It went wrong: a later line in
# the same function used `du --apparent-size=0`, which is not valid -
# --apparent-size takes no argument - and under `set -euo pipefail` that
# killed the script before the chmod ran. Every clean shutdown left the
# base image WRITABLE, and the only sign was
# "Control process exited, status=1/FAILURE" in the journal.
#
# So this runs the real script against a scratch directory and checks
# the mode afterwards, rather than checking that the line is present.
# =====================================================================
set -uo pipefail
PREPARE=/usr/local/libexec/owuivm-maint-prepare
fails=0
ok() { printf ' ok %s\n' "$*"; }
bad() { printf ' FAIL %s\n' "$*"; fails=$((fails+1)); }
work=$(mktemp -d); trap 'rm -rf "$work"' EXIT
base="$work/base"; isos="$work/iso"
mkdir -p "$base" "$isos"
# EVERY DIRECTORY THIS TOUCHES IS ITS OWN, INCLUDING THE RUN DIRECTORY.
#
# It used to use the real /run/owuivm/<constant session id>. Run as root,
# that left a root:root directory behind, and the operator's next
# maintenance boot died in ExecStartPre on "cp: cannot remove
# .../OVMF_VARS.fd: Permission denied" - a tool that looked broken
# because a test had littered.
run_dir="$work/run"
mkdir -p "$run_dir"
run_verb() {
OWUIVM_BASE_DIR="$base" OWUIVM_ISO_DIR="$isos" OWUIVM_OVERLAY_DIR="$work" \
OWUIVM_RUN_DIR="$run_dir" \
"$PREPARE" ubuntu "$1" 2>&1
}
# `close` is what the unit runs on the way out; `freeze` and `thaw` are
# the operator's own buttons. They used to be one verb, which is why
# looking at an image meant unfreezing it.
close() { run_verb close; }
freeze() { run_verb freeze; }
thaw() { run_verb thaw; }
echo "== a disk with an OS on it"
# 600 MB: over the "something was installed" threshold.
dd if=/dev/zero of="$base/ubuntu.qcow2" bs=1M count=600 status=none
cp /usr/share/OVMF/OVMF_VARS_4M.fd "$base/ubuntu-OVMF_VARS.fd"
chmod 0664 "$base/ubuntu.qcow2" "$base/ubuntu-OVMF_VARS.fd"
echo "$isos/ubuntu-x.iso" > "$base/ubuntu.install"
out=$(freeze)
mode=$(stat -c %a "$base/ubuntu.qcow2")
[ "$mode" = "444" ] && ok "the disk is frozen (0444)" || bad "the disk is $mode, not 444 — $out"
out=$(close)
# The marker is SUPPOSED to survive now. It used to be removed once the
# disk passed half a gigabyte, which declared a partial install finished,
# took its media away, and left a disk that was neither blank enough to
# restart nor complete enough to boot. Size cannot tell "installed" from
# "interrupted halfway", so nothing guesses: the boot order in
# owuivm-launch tries the disk first and falls through to the media, and
# detaching it is an explicit act by the operator.
[ -f "$base/ubuntu.install" ] && ok "the install marker survives a freeze (nothing guesses)" \
|| bad "the install marker was removed - a partial install cannot be resumed"
[ -f "$base/ubuntu.sha256" ] && ok "a checksum was recorded" || bad "no checksum recorded"
echo
echo "== a disk nothing was installed on yet"
rm -rf "$base"/*; mkdir -p "$base"
dd if=/dev/zero of="$base/ubuntu.qcow2" bs=1M count=1 status=none
cp /usr/share/OVMF/OVMF_VARS_4M.fd "$base/ubuntu-OVMF_VARS.fd"
chmod 0664 "$base/ubuntu.qcow2" "$base/ubuntu-OVMF_VARS.fd"
echo "$isos/ubuntu-x.iso" > "$base/ubuntu.install"
out=$(freeze)
mode=$(stat -c %a "$base/ubuntu.qcow2")
[ "$mode" = "444" ] && ok "the disk is frozen even when nothing was installed" \
|| bad "the disk is $mode, not 444 — $out"
out=$(close)
# An interrupted install must be resumable; removing the marker here is
# what left a blank disk with no installer attached and no way back.
[ -f "$base/ubuntu.install" ] && ok "the install marker survived an interrupted install" \
|| bad "the install marker was removed after an interrupted install"
echo
echo "== firmware variables are only kept when there is an OS to keep them for"
rm -rf "$base"/*; mkdir -p "$base"
dd if=/dev/zero of="$base/ubuntu.qcow2" bs=1M count=1 status=none
printf 'ORIGINAL' > "$base/ubuntu-OVMF_VARS.fd"
chmod 0664 "$base/ubuntu.qcow2" "$base/ubuntu-OVMF_VARS.fd"
if true; then
printf 'REWRITTEN-BY-A-BOOT-THAT-INSTALLED-NOTHING' > "$run_dir/OVMF_VARS.fd"
close >/dev/null
if grep -q ORIGINAL "$base/ubuntu-OVMF_VARS.fd" 2>/dev/null; then
ok "a boot that installed nothing did not rewrite the frozen firmware variables"
else
bad "firmware variables from an empty boot were carried into the base image"
fi
rm -f "$run_dir/OVMF_VARS.fd"
fi
echo
echo "== the freeze survives a broken step after it"
rm -rf "$base"/*; mkdir -p "$base"
dd if=/dev/zero of="$base/ubuntu.qcow2" bs=1M count=600 status=none
cp /usr/share/OVMF/OVMF_VARS_4M.fd "$base/ubuntu-OVMF_VARS.fd"
chmod 0664 "$base/ubuntu.qcow2" "$base/ubuntu-OVMF_VARS.fd"
# A directory where a file is expected: the checksum step will fail.
mkdir -p "$base/ubuntu.sha256.previous"
out=$(freeze)
mode=$(stat -c %a "$base/ubuntu.qcow2")
[ "$mode" = "444" ] && ok "still frozen despite a later step failing" \
|| bad "a failure after the chmod undid the freeze — $out"
echo
echo "== a frozen boot leaves nothing behind"
# The point of the whole split: the operator can start a base image to
# see that it still comes up, exactly as the agent will get it, and
# being there cannot change it. OVMF rewrites its boot order on every
# boot, so "unchanged" has to include the firmware variables.
rm -rf "$base"/*; mkdir -p "$base"
dd if=/dev/zero of="$base/ubuntu.qcow2" bs=1M count=600 status=none
printf 'ORIGINAL-FIRMWARE' > "$base/ubuntu-OVMF_VARS.fd"
chmod 0444 "$base/ubuntu.qcow2" "$base/ubuntu-OVMF_VARS.fd"
disk_sum=$(sha256sum "$base/ubuntu.qcow2" | cut -d' ' -f1)
touch "$work/maint-ubuntu.qcow2"
if [ -d "$run_dir" ] || mkdir -p "$run_dir" 2>/dev/null; then
printf 'REWRITTEN-BY-A-LOOK-AT-IT-BOOT' > "$run_dir/OVMF_VARS.fd"
out=$(close)
grep -q ORIGINAL-FIRMWARE "$base/ubuntu-OVMF_VARS.fd" \
&& ok "a frozen boot did not rewrite the firmware variables" \
|| bad "a frozen boot changed the base image's firmware — $out"
[ "$(sha256sum "$base/ubuntu.qcow2" | cut -d' ' -f1)" = "$disk_sum" ] \
&& ok "the base image is byte-identical after a frozen boot" \
|| bad "the base image changed during a frozen boot"
[ -f "$work/maint-ubuntu.qcow2" ] \
&& bad "the throwaway disk survived the boot - its changes would come back" \
|| ok "the throwaway disk was removed"
rm -f "$run_dir/OVMF_VARS.fd"
fi
echo
echo "== the operator's own verbs, which nothing else may do for them"
rm -rf "$base"/*; mkdir -p "$base"
dd if=/dev/zero of="$base/ubuntu.qcow2" bs=1M count=600 status=none
cp /usr/share/OVMF/OVMF_VARS_4M.fd "$base/ubuntu-OVMF_VARS.fd"
chmod 0444 "$base/ubuntu.qcow2" "$base/ubuntu-OVMF_VARS.fd"
out=$(thaw)
[ "$(stat -c %a "$base/ubuntu.qcow2")" = "660" ] \
&& ok "thaw makes the image writable" || bad "thaw left it $(stat -c %a "$base/ubuntu.qcow2") — $out"
# And `close` must NOT put it back. That auto-freeze is what made
# looking at an image mean unfreezing it, and it hid a forgotten thaw;
# the refusal now lives in vmd, where a session cannot start without it.
out=$(close)
[ "$(stat -c %a "$base/ubuntu.qcow2")" = "660" ] \
&& ok "closing an unfrozen boot leaves it unfrozen, and says so" \
|| bad "close re-froze the image behind the operator — $out"
case "$out" in
*WRITABLE*) ok "and the log says the image is still writable" ;;
*) bad "close said nothing about leaving a writable image: $out" ;;
esac
out=$(freeze)
[ "$(stat -c %a "$base/ubuntu.qcow2")" = "444" ] \
&& ok "freeze puts it back" || bad "freeze left it $(stat -c %a "$base/ubuntu.qcow2") — $out"
[ -f "$base/ubuntu.sha256" ] && ok "and re-records the checksum" || bad "no checksum after freeze"
echo
echo "== the desktop tool starts units and touches nothing"
# WHY: the tool created the blank disk, the firmware variables and the
# install marker itself, as the operator. Pressing "Install from ISO" for
# Windows failed with a bare permission error, because `operator` is a
# member of owuivmadm in /etc/group and the desktop SESSION predates the
# usermod - a unix group only reaches a process at its next login, so the
# group exists, `id` agrees, and nothing running has it.
#
# It was wrong a second way that would have surfaced much later: it
# copied the plain OVMF variables for both images, where Windows needs
# the store with Microsoft's keys enrolled or Secure Boot has nothing it
# trusts and setup refuses the machine.
gui=/usr/local/bin/owuivm-maintenance
for forbidden in "qemu-img" "shutil.copyfile" "OVMF_VARS ="; do
grep -q "$forbidden" "$gui" 2>/dev/null \
&& bad "the desktop tool still does '$forbidden' itself - it cannot, and it gets it wrong" \
|| ok "the desktop tool does not do '$forbidden' itself"
done
echo
echo "== what the unit makes for a brand new image"
rm -rf "$base"/*; mkdir -p "$base"
for name in ubuntu windows; do
out=$(OWUIVM_BASE_DIR="$base" OWUIVM_ISO_DIR=/var/lib/owuivm/iso \
OWUIVM_OVERLAY_DIR="$work" OWUIVM_RUN_DIR="$run_dir" \
"$PREPARE" "$name" open 2>&1)
if [ ! -f "$base/$name.qcow2" ]; then
printf ' skip %s: no media in /var/lib/owuivm/iso (%s)\n' "$name" "${out##*$'\n'}"
continue
fi
ok "$name: a blank disk was created"
[ $(( 8#$(stat -c %a "$base/$name.qcow2") & 8#200 )) -ne 0 ] \
&& ok "$name: and it is writable, because an install has to write to it" \
|| bad "$name: the new disk is read-only, so the installer cannot use it"
grep -q '^/var/lib/owuivm/iso/' "$base/$name.install" 2>/dev/null \
&& ok "$name: the media it will boot is recorded ($(basename "$(cat "$base/$name.install")"))" \
|| bad "$name: no install marker, so the installer is never attached"
# The one difference between the two images that is easy to get wrong
# and impossible to see until Windows setup refuses the machine.
case "$name" in
windows) want=/usr/share/OVMF/OVMF_VARS_4M.ms.fd
label="the store with Microsoft's keys enrolled" ;;
*) want=/usr/share/OVMF/OVMF_VARS_4M.fd
label="the plain store" ;;
esac
cmp -s "$base/$name-OVMF_VARS.fd" "$want" \
&& ok "$name: firmware variables are $label" \
|| bad "$name: firmware variables are NOT $label - Secure Boot will not behave"
done
rm -f "$base"/*.install
echo
echo "== and the operator can actually do that explicit act"
# The line above says detaching the media is the operator's decision.
# It was a sentence in a comment for a while and nothing more: the
# operator's only way to make it was to delete a file in a directory
# their own account cannot write. These four check that the control
# exists, that it is reachable, and that it does one thing.
unit=/etc/systemd/system/owuivm-detach@.service
if [ -f "$unit" ]; then
ok "there is a unit for it"
exec_line=$(sed -n 's/^ExecStart=//p' "$unit")
case "$exec_line" in
*"rm -f /var/lib/owuivm/base/%i.install") ok "it removes the marker and nothing else" ;;
*) bad "the detach unit does more than remove the marker: $exec_line" ;;
esac
grep -q '^ConditionPathExists=/var/lib/owuivm/base/%i.install' "$unit" \
&& ok "pressing it twice is not an error" \
|| bad "no ConditionPathExists - a second press reports a failure"
else
bad "no owuivm-detach@.service - detaching the media is still a shell command"
fi
for verb in thaw freeze; do
u="/etc/systemd/system/owuivm-$verb@.service"
[ -f "$u" ] && ok "there is a unit for $verb" \
|| bad "no owuivm-$verb@.service - the operator cannot $verb an image"
grep -q "owuivm-$verb@" /etc/polkit-1/rules.d/50-owuivm.rules \
&& ok "polkit authorises $verb" \
|| bad "owuivm-$verb@ is not in the polkit rule, so the desktop tool cannot start it"
done
# One button, two verbs, and the label has to be the state - a toggle
# that does not say which way it points is worse than two buttons.
grep -q 'owuivm-{verb}@{name}.service' /usr/local/bin/owuivm-maintenance 2>/dev/null \
&& ok "the maintenance tool has the freeze/unfreeze control" \
|| bad "the desktop tool cannot freeze or unfreeze anything"
grep -q '"Unfreeze" if state\["frozen"\] else "Freeze"' /usr/local/bin/owuivm-maintenance 2>/dev/null \
&& ok "and its label is the image's state" \
|| bad "the toggle does not say which way it points"
grep -q 'owuivm-detach@' /etc/polkit-1/rules.d/50-owuivm.rules \
&& ok "polkit authorises it for an active session of the operator" \
|| bad "the detach unit is not in the polkit rule, so the desktop tool cannot start it"
# The service without a button is the same sentence in a different file.
gui=/usr/local/bin/owuivm-maintenance
grep -q 'owuivm-detach@{name}.service' "$gui" 2>/dev/null \
&& ok "the maintenance tool has a control wired to it" \
|| bad "the desktop tool still has no way to press it"
# The one thing pressing it early would cost: a machine shut down
# mid-install with no disc to go back to.
grep -q 'state\["installing"\] and not state\["running"\]' "$gui" 2>/dev/null \
&& ok "the control is dead while the machine is running" \
|| bad "the control can be pressed mid-install"
echo
if [ "$fails" -gt 0 ]; then echo "FAILED $fails"; exit 1; fi
echo "the freeze holds"
tests/test_vm_git_hardening.py
Executable
+218
View File
@@ -0,0 +1,218 @@
#!/usr/bin/env python3
"""Proves the host's git calls cannot be turned into code execution.
python3 tests/test_vm_git_hardening.py
WHY THIS TEST EXISTS
The project share is mounted inside the VM, so anything the guest runs
can write `.git/` - and git is designed to execute things it finds there.
vmd commits that tree on the host. Without care, "the agent finished and
vmd committed the result" means "the guest chose a program and vmd ran
it".
Every case here plants a real attack and then asserts it did not fire.
The first case is a CONTROL: it uses plain git and the attack MUST
succeed. Without that, this file would pass just as happily if git had
stopped running hooks altogether, or if the planted hook were never
executable, and it would be proving nothing at all.
"""
import os
import subprocess
import sys
import tempfile
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "vm", "vmd"))
import gitea # noqa: E402
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(f"{label}: {detail}")
print(f"FAIL {label}\n {detail}")
def make_repo(tmp: str, marker: str) -> tuple[str, str]:
"""A repo booby-trapped the way a hostile guest would leave it."""
repo = os.path.join(tmp, "project")
os.makedirs(os.path.join(repo, "evil"), exist_ok=True)
subprocess.run(["git", "init", "-q", "-b", "main", repo], check=True)
with open(os.path.join(repo, "file.txt"), "w") as handle:
handle.write("content\n")
hooks = os.path.join(repo, ".git", "hooks")
os.makedirs(hooks, exist_ok=True)
for path in (os.path.join(hooks, "pre-commit"), os.path.join(repo, "evil", "pre-commit")):
with open(path, "w") as handle:
handle.write(f"#!/bin/sh\necho fired >> {marker}\n")
os.chmod(path, 0o755)
# The second half of the attack: even with .git/hooks neutralised,
# repository config can point hooksPath somewhere else entirely.
subprocess.run(["git", "-C", repo, "config", "core.hooksPath",
os.path.join(repo, "evil")], check=True)
return repo, marker
with tempfile.TemporaryDirectory() as tmp:
marker = os.path.join(tmp, "fired")
# --- CONTROL: the attack must work against plain git ---------------
repo, _ = make_repo(tmp, marker)
subprocess.run(["git", "-C", repo, "add", "-A"], check=True,
capture_output=True)
subprocess.run(["git", "-C", repo, "-c", "user.name=t", "-c", "user.email=t@t",
"commit", "-qm", "control"], capture_output=True)
check("CONTROL: plain git runs the planted hook",
os.path.exists(marker),
"the hook did not fire, so this file proves nothing - fix the control first")
# --- the real case -------------------------------------------------
if os.path.exists(marker):
os.unlink(marker)
with open(os.path.join(repo, "file.txt"), "a") as handle:
handle.write("more\n")
sha = gitea.commit(repo, "hardened commit")
check("gitea.commit still commits", bool(sha), f"returned {sha!r}")
check("no hook fired under gitea.commit",
not os.path.exists(marker),
"a planted hook executed on the host during a commit")
# --- and the flags that make that true are actually present --------
flags = " ".join(gitea._SAFE_FLAGS)
for needed in ("core.hooksPath=/var/empty", "core.fsmonitor=false",
"protocol.ext.allow=never", "protocol.file.allow=never"):
check(f"flag present: {needed}", needed in flags, flags)
for needed in ("GIT_CONFIG_GLOBAL", "GIT_CONFIG_SYSTEM"):
check(f"env neutralised: {needed}", gitea._SAFE_ENV.get(needed) == "/dev/null",
str(gitea._SAFE_ENV))
# --- a token must never reach the tree -----------------------------
config_path = os.path.join(repo, ".git", "config")
config = open(config_path).read() if os.path.exists(config_path) else ""
check("no credential was written into .git/config",
"@" not in config.split("url =")[-1][:200] if "url =" in config else True,
config[:200])
# --- what the person is handed at the end --------------------------------
#
# Two complaints from a real session, both about the last thing the user
# sees. They were handed the repository URL when they had asked for a link
# to download the script - a page to navigate rather than a file to save.
# And the repository was labelled "Created by the owuitools VM agent for
# project 'systeminfo-ps'", which says nothing about what the project is.
work = tempfile.mkdtemp()
def run_git(*args):
subprocess.run(["git", "-C", work, *args], check=True, capture_output=True,
env={**os.environ, "GIT_CONFIG_GLOBAL": "/dev/null",
"GIT_CONFIG_SYSTEM": "/dev/null"})
run_git("init", "-q", "-b", "main")
for name in ("SystemInfo.ps1", "systeminfo.json", ".gitignore"):
open(os.path.join(work, name), "w").write("x")
os.makedirs(os.path.join(work, "sub"), exist_ok=True)
open(os.path.join(work, "sub", "deep.ps1"), "w").write("x")
open(os.path.join(work, "left-behind.txt"), "w").write("x")
run_git("add", "SystemInfo.ps1", "systeminfo.json", ".gitignore", "sub/deep.ps1")
run_git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "x")
tracked = gitea.tracked_files(work)
check("a download link is only offered for a file that was committed",
"SystemInfo.ps1" in tracked and "left-behind.txt" not in tracked,
"a link to an uncommitted file is a 404 with the operator's name on it")
check("and not for the repository's own dotfiles", ".gitignore" not in tracked)
check("the file asked for comes before anything nested",
tracked.index("SystemInfo.ps1") < tracked.index("sub/deep.ps1"))
# The link handed to the user is the releases page, because a link into
# the repository does NOT download: measured, Gitea serves the raw file as
# `text/plain` with `Content-Disposition: inline`, so the browser shows the
# source. Release assets are attachments.
check("a release is built from what was committed, and is capped",
gitea.RELEASE_ASSETS_MAX > 0 and gitea.RELEASE_ASSET_MAX > 0)
check("the asset list is the committed files",
"tracked_files(repo_path" in open(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
"..", "vm", "vmd", "gitea.py"), encoding="utf-8").read(),
"an asset for an uncommitted file is a download of something that was thrown away")
gitea_src = open(os.path.join(os.path.dirname(os.path.abspath(__file__)),
"..", "vm", "vmd", "gitea.py"), encoding="utf-8").read()
check("a nested file keeps its path in the asset name",
"name.replace('/', '_')" in gitea_src,
"a release has no folders, so two files called the same thing would collide")
vmd_src = open(os.path.join(os.path.dirname(os.path.abspath(__file__)),
"..", "vm", "vmd", "vmd.py"), encoding="utf-8").read()
publish = vmd_src[vmd_src.index("def _publish"):vmd_src.index("def v_session_status")]
check("the release is published after the push, not instead of it",
publish.index("gitea.push(") < publish.index("gitea.publish_release("),
"the tag has to point at a commit the server already has")
check("a release that fails does not fail the session",
"release_error" in publish and "publish_error" in publish,
"the work is committed and pushed by then; only the download page is missing")
check("uploading an asset is multipart, as Gitea requires",
"multipart/form-data; boundary=" in gitea_src)
check("the upload boundary is not guessable from the file",
"os.urandom" in gitea_src,
"a fixed boundary appearing inside a file would corrupt the upload")
# The description is generated from the job, so it cannot become a pitch
# and cannot describe work that was never asked for.
desc = gitea.short_description(
"Goal: Write a PowerShell script that collects system information and prints it. "
"Then test both modes and report the results verbatim.", "systeminfo-ps")
check("the repository says what the project is", "PowerShell script" in desc, desc)
check("it is one line and short",
"\n" not in desc and len(desc) <= gitea.SUMMARY_CAP + 3, f"{len(desc)}: {desc}")
check("it stops at the first sentence rather than reciting the job",
"test both modes" not in desc, desc)
check("a label like 'Goal:' is not part of the description",
not desc.lower().startswith("goal"), desc)
check("a job with nothing usable still gets a description",
gitea.short_description("", "demo") != "")
# A job opens by addressing the machine and telling it to build something.
# A repository description labels the thing that was built, so neither the
# machine nor the instruction belongs in it. Measured on a real session,
# the first version read "On the Ubuntu machine, write a small Python
# script called wordcount.py that takes a filename and prints the..." -
# the instruction, verbatim and cut off.
built = gitea.short_description(
"On the Ubuntu machine, write a small Python script called wordcount.py that takes "
"a filename and prints the ten most common words with their counts. Save it in the "
"project folder and run it.", "wordcount-demo")
check("the machine is not part of the description",
"ubuntu" not in built.lower() and not built.lower().startswith("on the"), built)
check("nor is the instruction that asked for it",
not built.lower().startswith("write"), built)
check("what is left describes the thing that was built",
built.startswith("Small Python script called wordcount.py"), built)
# Only verbs that CREATE are stripped. A project whose job was to test
# something is accurately described by the word "test".
checked = gitea.short_description(
"Test the bash command find -mtime +7 -delete and report which files it removes.",
"find-test")
check("a job that was a test still says so", checked.lower().startswith("test"), checked)
print()
print("=" * 60)
if FAILS:
print(f"FAILED {len(FAILS)}")
for f in FAILS:
print(" -", f)
sys.exit(1)
print("all git hardening cases pass")
tests/test_vm_host_render.sh
Executable
+111
View File
@@ -0,0 +1,111 @@
#!/usr/bin/env bash
# The host templates, rendered the way install.sh renders them, checked
# without installing anything.
#
# ./tests/test_vm_host_render.sh
#
# WHY THIS EXISTS
#
# vm/host is full of @PLACEHOLDER@ tokens now, so that no machine's pool
# path, memory ceiling or LAN address is baked into the repo. That moves
# a whole class of mistake to install time: a token added to a template
# and not to install.sh's render() produces a systemd unit with a literal
# @VM_MEMORY_MAX@ in it, which fails when the unit is STARTED - long
# after the install said it succeeded.
#
# So this renders every template with sample values that are nothing like
# this machine's, and asserts the results are complete and valid. It
# touches /etc nowhere and needs no root.
set -uo pipefail
cd "$(dirname "$0")/.."
HOST=vm/host
FAILS=0
check() { if [ "$2" -eq 0 ]; then echo "ok $1"; else echo "FAIL $1"; FAILS=$((FAILS+1)); fi; }
WORK=$(mktemp -d); trap 'rm -rf "$WORK"' EXIT
# Deliberately unlike this host: a different pool, different sizing, a
# different operator, so a value that leaked through as a default shows up.
POOL_ROOT=/tank
OPERATOR=someone
VM_MEMORY_MAX=6G
VM_CPU_QUOTA=300%
MAINT_CPU_QUOTA=400%
TOOLS_SUBNET=10.9.0.0/24
GITEA_ALLOW="IPAddressAllow=10.9.9.9/32"
VM_ALLOWED_CPUS=""
render() { # the same transformation install.sh applies
local src="$1" dst="$2" allow="$WORK/allow"
printf '%s\n' "$GITEA_ALLOW" > "$allow"
sed -e "/@GITEA_ALLOW@/{r $allow" -e "d;}" "$src" > "$dst"
sed -i -e "s|@POOL_ROOT@|$POOL_ROOT|g" -e "s|@OPERATOR@|$OPERATOR|g" \
-e "s|@VM_MEMORY_MAX@|$VM_MEMORY_MAX|g" -e "s|@VM_CPU_QUOTA@|$VM_CPU_QUOTA|g" \
-e "s|@MAINT_CPU_QUOTA@|$MAINT_CPU_QUOTA|g" -e "s|@TOOLS_SUBNET@|$TOOLS_SUBNET|g" "$dst"
if [ -z "$VM_ALLOWED_CPUS" ]; then
sed -i '/^AllowedCPUs=@VM_ALLOWED_CPUS@$/d' "$dst"
else
sed -i "s|@VM_ALLOWED_CPUS@|$VM_ALLOWED_CPUS|g" "$dst"
fi
}
TEMPLATES="owuivm.service owuivmd.service owuivm-maint@.service owuivm-ctl owuivm-launch
owuivm-teardown owuivm-maint-prepare 50-owuivm.rules owuivm-tmpfiles.conf
owuivm-insert@.service owuivm-rename@.service owuivm-thaw@.service
owuivm-freeze@.service owuivm-detach@.service owuivm-nft-guard.service"
# --- every template renders completely --------------------------------
for t in $TEMPLATES; do
[ -f "$HOST/$t" ] || { echo "FAIL $t is missing"; FAILS=$((FAILS+1)); continue; }
render "$HOST/$t" "$WORK/$t"
left=$(grep -o '@[A-Z_]\+@' "$WORK/$t" | sort -u | tr '\n' ' ')
[ -z "$left" ]; check "$t renders with nothing left over${left:+ (found: $left)}" $?
done
# --- nothing of THIS machine survives, in the parts that DO something --
# Rendered with someone else's values, no line that systemd or bash acts
# on may still carry ours.
#
# Comments are excluded deliberately. They cite the measurements the
# defaults came from - "qemu sat at 697% of the 700% ceiling, with 1,792
# throttle events" - and that evidence is the reason anyone could change
# a number safely. Stripping it to satisfy a grep would make the files
# portable and useless.
strip_comments() { grep -rhvE '^[[:space:]]*(#|//)' "$WORK" ; }
for pattern in '/Pool' '20-27' '11G' '700%' '800%' '172\.22\.0\.0' '(^|[^a-zA-Z])operator([^a-zA-Z]|$)'; do
! strip_comments | grep -qE "$pattern"
check "no '$pattern' in any line that is acted on" $?
done
# --- and the sample values did arrive ---------------------------------
grep -q '/tank/vmrun' "$WORK/owuivm.service"; check "the pool reached owuivm.service" $?
grep -q '/tank/vmprojects' "$WORK/owuivmd.service"; check "the pool reached owuivmd.service" $?
grep -q '/tank/vmprojects' "$WORK/owuivm-ctl"; check "the pool reached owuivm-ctl" $?
grep -q 'MemoryMax=6G' "$WORK/owuivm.service"; check "the memory ceiling was substituted" $?
grep -q 'CPUQuota=300%' "$WORK/owuivm.service"; check "the cpu quota was substituted" $?
grep -q 'IPAddressAllow=10.9.9.9/32' "$WORK/owuivmd.service"; check "the gitea allowlist was inserted" $?
grep -q 'subject.user == "someone"' "$WORK/50-owuivm.rules"; check "the operator reached polkit" $?
! grep -q '^AllowedCPUs=' "$WORK/owuivm.service"
check "with no pinning, AllowedCPUs is REMOVED rather than left empty" $?
# --- the results are valid ---------------------------------------------
# An empty AllowedCPUs= would RESET the property rather than leave it
# alone, which is why the line is deleted; systemd-analyze is what says
# the rest of the file is still a unit.
if command -v systemd-analyze >/dev/null 2>&1; then
for u in owuivm.service owuivmd.service "owuivm-maint@.service" owuivm-insert@.service owuivm-rename@.service; do
out=$(systemd-analyze verify "$WORK/$u" 2>&1 | grep -viE 'Unknown key|command .* is not executable|Failed to (resolve|open)|/usr/local|not found' || true)
[ -z "$out" ]; check "systemd accepts $u${out:+ -- $out}" $?
done
else
echo "skip systemd-analyze is not here; unit validity not checked"
fi
for sh in owuivm-ctl owuivm-launch owuivm-teardown owuivm-maint-prepare; do
bash -n "$WORK/$sh"; check "$sh is still valid bash after rendering" $?
done
echo
if [ "$FAILS" -gt 0 ]; then echo "$FAILS FAILED"; exit 1; fi
echo "every host template renders into something valid, with no trace of this machine"
tests/test_vm_incremental_build.py
+155
View File
@@ -0,0 +1,155 @@
#!/usr/bin/env python3
"""Building a file in pieces, because designing it whole is what fails.
python3 tests/test_vm_incremental_build.py
WHY
The agent had `write_file` and nothing else. Every change to a file meant
reproducing the whole file, and a file you must reproduce whole is a file
you must design whole before you can write any of it.
Measured: a session spent twenty-two minutes and roughly fifty thousand
tokens inside ONE turn, designing a WPF application in its head, still on
task one of twelve, having made no tool call for twenty minutes. It was
not stuck and it was not looping. It was doing the only thing the tools
allowed.
The fix is the shape of the tools, not a rule in the prompt - this stack
has now three times proved a prompt rule loses to what the tools make
easy. So: `append_file` to build forward, `edit_file` to revise, and a cap
on `write_file` that makes those the path of least resistance.
`edit_file` is modelled on the one the author of this file uses all day:
exact-text replacement that FAILS LOUDLY when the text is missing or
ambiguous. That failure is the feature. Text that is not there means a
mistaken belief about the file, and a refusal now is far cheaper than a
silent half-edit found three sections later.
"""
import os
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.join(HERE, "..", "vm", "vmd"))
sys.path.insert(0, os.path.join(HERE, "..", "vmagent", "app"))
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(label)
print(f"FAIL {label}\n {detail}")
import tools # noqa: E402
VMD_SRC = open(os.path.join(HERE, "..", "vm", "vmd", "vmd.py"), encoding="utf-8").read()
# --- the tools exist and say what they are for ---------------------------
for os_name in ("ubuntu", "windows"):
names = [t["function"]["name"] for t in tools.schemas(os_name)]
check(f"{os_name}: append_file is offered", "append_file" in names)
check(f"{os_name}: edit_file is offered", "edit_file" in names)
schema = {t["function"]["name"]: t["function"] for t in tools.schemas("ubuntu")}
check("edit_file asks for exact text", "EXACTLY once" in schema["edit_file"]["description"])
check("and says a refusal is information, not an obstacle",
"wrong" in schema["edit_file"]["description"])
check("write_file says it is for starting, not for everything",
"skeleton" in schema["write_file"]["description"]
and "16 KB" in schema["write_file"]["description"])
check("append_file says what it is for",
"section" in schema["append_file"]["description"])
check("edit_file can replace every occurrence when that is meant",
"replace_all" in schema["edit_file"]["parameters"]["properties"])
check("but replace_all is not required, so the default is the safe one",
"replace_all" not in schema["edit_file"]["parameters"]["required"])
# --- the cap is a forcing function, and says so --------------------------
import re # noqa: E402
cap = int(re.search(r"MAX_WRITE_BYTES = ([\d_]+)", VMD_SRC).group(1).replace("_", ""))
check("one write is capped well below the file limit", 4000 < cap < 64000, f"{cap}")
check("the refusal explains why rather than just refusing",
"design a whole program" in schema["write_file"]["description"]
or "designing a whole program" in schema["write_file"]["description"])
check("and the refusal names the way past it",
"append_file" in VMD_SRC.split("MAX_WRITE_BYTES:")[1][:900]
and "edit_file" in VMD_SRC.split("MAX_WRITE_BYTES:")[1][:900])
check("generated data still has an escape hatch",
"command in the machine" in VMD_SRC.split("MAX_WRITE_BYTES:")[1][:900],
"a big data file is not the same mistake as a big program")
# --- edit_file's refusals ------------------------------------------------
body = VMD_SRC[VMD_SRC.index("def v_edit_file"):VMD_SRC.index("def v_list_dir")]
check("an edit whose text is not in the file is refused",
"not in the file" in body)
check("an ambiguous edit is refused rather than guessed at",
"ambiguous" in body and "found > 1" in body)
check("a no-op edit is refused", "identical" in body)
check("the file is read before it is written",
body.index("agent.read_file") < body.index("agent.write_file"),
"an edit that writes without reading is a guess")
check("and the agent is told to check it still parses",
"still parses" in body)
# --- append_file ---------------------------------------------------------
ap = VMD_SRC[VMD_SRC.index("def v_append_file"):VMD_SRC.index("def v_edit_file")]
check("appending to a file that is not there creates it",
"existing = b\"\"" in ap)
check("an append is capped like a write", "MAX_WRITE_BYTES" in ap)
# --- the directory bug this uncovered ------------------------------------
#
# write_file ran `mkdir -p "$(dirname ...)"` on every guest. On Windows
# that reaches PowerShell, where mkdir takes no -p and dirname is not a
# command - and the result was discarded, so it failed silently and any
# nested path failed to write. A top-level file worked, because Z:\
# already exists, which is the kind of half-working that survives testing.
parent = VMD_SRC[VMD_SRC.index("def _make_parent"):VMD_SRC.index("def v_write_file")]
check("the parent directory is made with the right shell for the machine",
"windows" in parent and "New-Item" in parent and "mkdir -p" in parent,
"this is the third bash-to-PowerShell bug in this subsystem")
# Two goes at this failed the same way. The second used `Split-Path -Parent
# -LiteralPath`, which PowerShell 5.1 rejects as an ambiguous parameter
# set: $p came back null, Test-Path and New-Item both bound null and did
# nothing, and because those are NON-TERMINATING errors PowerShell still
# exited 0. Measured: PARENT=[], EXISTS_AFTER=, exit 0, no directory.
parent_code = parent.split('"""')[2] # past the docstring; the code only
check("the path is split by vmd, not parsed by the guest",
"Split-Path" not in parent_code and "dirname" not in parent_code
and "rsplit(" in parent_code,
"vmd built this path, so it can take the parent without asking a shell")
check("and success is confirmed rather than read off an exit code",
"MADE" in parent_code and "stdout" in parent_code,
"PowerShell exits 0 on a non-terminating error, so zero proves nothing")
check("a file at the share root needs no folder made",
"len(parent) <= 2" in parent_code)
check("both writers use it",
VMD_SRC.count("_make_parent(sess, agent, path)") == 2,
"append_file would have reintroduced it")
# --- one turn cannot run away --------------------------------------------
ollama_src = open(os.path.join(HERE, "..", "vmagent", "app", "ollama.py"),
encoding="utf-8").read()
check("a single turn has an output ceiling", "num_predict" in ollama_src,
"there was none: one turn ran 22 minutes and ~50,000 tokens")
# --- the feed shows the new verbs ----------------------------------------
import loop # noqa: E402
for verb, args in (("append_file", {"path": "a.ps1", "content": "x" * 40}),
("edit_file", {"path": "a.ps1", "old": "foo", "new": "bar"})):
check(f"{verb} appears in the command feed", loop._terminal_for(verb, args) != [])
check("an append reads as an append, not a write",
">>" in loop._terminal_for("append_file", {"path": "a", "content": "x"})[0])
print()
if FAILS:
print(f"FAILED {len(FAILS)}")
raise SystemExit(1)
print("a file can be built a section at a time, and a wrong belief about it is refused")
tests/test_vm_insert_media.sh
Executable
+94
View File
@@ -0,0 +1,94 @@
#!/usr/bin/env bash
# The insert-a-disc helper, against scratch directories.
#
# ./tests/test_vm_insert_media.sh
#
# NEVER against the real library. Tests in this suite have damaged live
# machines twice by sharing state with production, and this one is a
# particularly easy way to do it: a staged file called `ubuntu.anything.iso`
# lands in /var/lib/owuivm/iso matching `ubuntu*.iso`, and because the
# rule is "newest wins" it would silently become the disc the next install
# boots from. So every path is overridden and nothing real is touched.
set -uo pipefail
cd "$(dirname "$0")/.."
HELPER=$(pwd)/vm/host/owuivm-insert-media
FAILS=0
check() { # check <label> <condition-exit-code>
if [ "$2" -eq 0 ]; then echo "ok $1"; else echo "FAIL $1"; FAILS=$((FAILS+1)); fi
}
WORK=$(mktemp -d)
trap 'rm -rf "$WORK"' EXIT
export OWUIVM_ISO_DIR="$WORK/iso"
export OWUIVM_INCOMING_DIR="$WORK/incoming"
export OWUIVM_REGISTRY="$WORK/vms.toml"
mkdir -p "$OWUIVM_ISO_DIR" "$OWUIVM_INCOMING_DIR"
cat > "$OWUIVM_REGISTRY" <<'REG'
[[machine]]
id = "ubuntu"
name = "Ubuntu"
iso = "ubuntu*.iso"
REG
# A file that carries the ISO 9660 signature: CD001 at offset 0x8001.
make_iso() {
dd if=/dev/zero of="$1" bs=1024 count=40 status=none
printf 'CD001' | dd of="$1" bs=1 seek=32769 conv=notrunc status=none
}
# --- the happy path ---------------------------------------------------
make_iso "$OWUIVM_INCOMING_DIR/ubuntu.ubuntu-26.04-desktop-amd64.iso"
out=$("$HELPER" ubuntu 2>&1); rc=$?
check "a real ISO is accepted" $rc
[ -f "$OWUIVM_ISO_DIR/ubuntu.ubuntu-26.04-desktop-amd64.iso" ]
check "and lands in the library" $?
[ -z "$(ls -A "$OWUIVM_INCOMING_DIR")" ]
check "and the staging directory is left empty" $?
[ "$(stat -c %a "$OWUIVM_ISO_DIR/ubuntu.ubuntu-26.04-desktop-amd64.iso")" = "644" ]
check "and is readable by the launcher" $?
# --- the case this exists to catch ------------------------------------
# A download that was really an HTML error page, or a renamed zip. It
# would otherwise fail much later, as a machine that will not boot, for
# no visible reason.
echo "<html>404 Not Found</html>" > "$OWUIVM_INCOMING_DIR/ubuntu.not-really.iso"
out=$("$HELPER" ubuntu 2>&1); rc=$?
[ $rc -ne 0 ]; check "a file that is not an ISO is refused" $?
grep -q "CD001" <<<"$out"; check "and the refusal says why" $?
[ ! -f "$OWUIVM_ISO_DIR/ubuntu.not-really.iso" ]
check "and it does not reach the library" $?
[ ! -f "$OWUIVM_INCOMING_DIR/ubuntu.not-really.iso" ]
check "and the rejected file is removed rather than left to be retried" $?
# --- input validation -------------------------------------------------
out=$("$HELPER" "../../etc" 2>&1); rc=$?
[ $rc -eq 64 ]; check "a path-shaped machine id is refused" $?
out=$("$HELPER" "windows" 2>&1); rc=$?
[ $rc -eq 64 ]; check "a machine not in the registry is refused" $?
out=$("$HELPER" ubuntu 2>&1); rc=$?
[ $rc -eq 65 ]; check "nothing staged is reported, not treated as success" $?
# A symlink is how this would be turned into "copy any file the unit can
# read into a world-readable directory".
ln -s /etc/shadow "$OWUIVM_INCOMING_DIR/ubuntu.sneaky.iso"
out=$("$HELPER" ubuntu 2>&1); rc=$?
[ $rc -ne 0 ]; check "a symlink is refused" $?
[ ! -e "$OWUIVM_ISO_DIR/ubuntu.sneaky.iso" ]
check "and nothing is copied out of it" $?
rm -f "$OWUIVM_INCOMING_DIR/ubuntu.sneaky.iso"
# --- newest wins, and the rest is cleared -----------------------------
make_iso "$OWUIVM_INCOMING_DIR/ubuntu.old.iso"
sleep 1.1
make_iso "$OWUIVM_INCOMING_DIR/ubuntu.new.iso"
out=$("$HELPER" ubuntu 2>&1); rc=$?
check "with two staged discs it still succeeds" $rc
[ -f "$OWUIVM_ISO_DIR/ubuntu.new.iso" ]; check "the newest one is taken" $?
[ ! -f "$OWUIVM_ISO_DIR/ubuntu.old.iso" ]
check "and a half-finished earlier attempt is not installable" $?
[ -z "$(ls -A "$OWUIVM_INCOMING_DIR")" ]; check "staging is cleared" $?
echo
if [ "$FAILS" -gt 0 ]; then echo "$FAILS FAILED"; exit 1; fi
echo "the disc inserter checks what it is given"
tests/test_vm_launch_argv.sh
Executable
+247
View File
@@ -0,0 +1,247 @@
#!/bin/bash
# =====================================================================
# Asserts the qemu command line, without starting a machine.
#
# sudo ./tests/test_vm_launch_argv.sh
#
# WHY THE ARGV IS WORTH A TEST OF ITS OWN
#
# Some of the isolation cannot be observed from outside the process.
# `-netdev user` accepts guestfwd=, smb=, tftp= and bootfile=, and every
# one of those is handled INSIDE qemu - the traffic never becomes a host
# socket, so neither the cgroup filter nor the nft table can see it. No
# filter can catch what no filter can observe; the only defence is that
# the string is fixed, and the only way to know it is still fixed is to
# look at it.
#
# The script is run with its final `exec qemu-system-x86_64` replaced by
# a printf, so every branch runs for real - the path checks, the
# firmware choice, the TPM setup - and nothing boots.
# =====================================================================
set -uo pipefail
LAUNCH=/usr/local/libexec/owuivm-launch
BASE="" # set to the scratch tree once $work exists
fails=0
ok() { printf ' ok %s\n' "$*"; }
bad() { printf ' FAIL %s\n' "$*"; fails=$((fails+1)); }
[ "$(id -u)" -eq 0 ] || { echo "run as root" >&2; exit 1; }
work=$(mktemp -d); trap 'rm -rf "$work"' EXIT
mkdir -p "$work/base" "$work/run" "$work/vmrun"
BASE="$work/base"
# THIS TEST IS HERMETIC, AND IT LEARNED TO BE THE HARD WAY.
#
# It used to drive the real launcher against the REAL directories: the
# real base images, and the real /run/owuivm/<session id> - which is a
# CONSTANT for maintenance boots. While an operator was installing
# Windows, a run of this file overwrote that machine's OVMF variables,
# deleted its swtpm state and pkill'd its swtpm (matching on a session id
# it shared), which killed qemu with "tpm-emulator: Could not stop TPM";
# and its mode-flipping loop left the real base image frozen, so the
# operator's next boot silently ran on a throwaway overlay and put a
# 16 GB Windows install somewhere it was going to be deleted from.
#
# So the copy of the launcher gets its paths rewritten as well as its
# exec line. The path RULES are still exercised - they are rewritten
# consistently - but every path they resolve to is inside $work.
sed -e 's|^exec /usr/bin/qemu-system-x86_64 .*|printf "%s\\n" "${args[@]}"|' \
-e "s|/var/lib/owuivm/base|$work/base|g" \
-e "s|/Pool/vmrun|$work/vmrun|g" \
-e "s|/run/owuivm|$work/run|g" \
"$LAUNCH" > "$work/launch"
chmod +x "$work/launch"
# And it refuses outright while a machine is up. Hermetic or not, this
# starts a real swtpm and nothing good comes of doing that beside a
# running VM.
for unit in owuivm.service owuivm-maint@ubuntu.service owuivm-maint@windows.service; do
if systemctl is-active --quiet "$unit" 2>/dev/null; then
echo "REFUSING: $unit is running. This test builds command lines and starts" >&2
echo "a TPM emulator; run it when no machine is up." >&2
exit 2
fi
done
argv_for() { # argv_for <ubuntu|windows>
local name="$1" session="ffffffffffffffffffffffffffffffff"
# Inside $work, like everything else: the copy of the launcher has had
# /run/owuivm rewritten, so a real path here fails its own vars check
# and the whole file reports "produced no command line at all".
local run="$work/run/$session"
mkdir -p "$run"
cp -f /usr/share/OVMF/OVMF_VARS_4M.fd "$run/OVMF_VARS.fd"
# A disk only has to exist and be in the right place for the checks.
local made=""
if [ ! -f "$BASE/$name.qcow2" ]; then
qemu-img create -f qcow2 "$BASE/$name.qcow2" 1G >/dev/null 2>&1
made=yes
fi
OWUIVM_MODE=maint OWUIVM_SESSION="$session" \
OWUIVM_DISK="$BASE/$name.qcow2" OWUIVM_VARS="$run/OVMF_VARS.fd" \
OWUIVM_SHARE="" OWUIVM_MEM_MB=8192 OWUIVM_CPUS=8 \
"$work/launch" 2>/dev/null
[ -n "$made" ] && rm -f "$BASE/$name.qcow2"
# Matched on the scratch directory, not on a session id that a real
# maintenance boot also uses.
pkill -f "swtpm socket .*$work" 2>/dev/null
rm -rf "$run/tpm" "$run/swtpm.sock"
return 0
}
for name in ubuntu windows; do
echo "== $name"
argv=$(argv_for "$name")
[ -n "$argv" ] || { bad "$name: produced no command line at all"; continue; }
netdev=$(printf '%s\n' "$argv" | grep -A0 '^user,' || true)
[ -n "$netdev" ] || netdev=$(printf '%s\n' "$argv" | grep '^user,id=net0' || true)
if [ -n "$netdev" ]; then
ok "$name: netdev is '$netdev'"
for forbidden in guestfwd smb= tftp= bootfile restrict=off; do
case "$netdev" in
*"$forbidden"*) bad "$name: netdev contains $forbidden - qemu handles that internally, where no filter can see it" ;;
*) ok "$name: netdev has no $forbidden" ;;
esac
done
case "$netdev" in
*ipv6=off*) ok "$name: ipv6 is off on the netdev" ;;
*) bad "$name: ipv6 is not disabled on the netdev" ;;
esac
else
bad "$name: no user-mode netdev found - a tap or bridge would bypass the whole model"
fi
# No inbound path into the guest, ever.
case "$argv" in
*hostfwd*) bad "$name: a hostfwd is present - that is a listening socket on this host" ;;
*) ok "$name: no hostfwd" ;;
esac
# Nothing of the host's should be reachable as a device.
for forbidden in "-virtfs" "vfio" "usb-host" "hostdev"; do
case "$argv" in
*"$forbidden"*) bad "$name: $forbidden is present" ;;
*) ok "$name: no $forbidden" ;;
esac
done
done
echo
echo "== what only Windows gets"
win=$(argv_for windows); ubu=$(argv_for ubuntu)
case "$win" in
*secboot*) ok "windows uses the secure-boot firmware" ;;
*) bad "windows is not using secure-boot firmware - its installer will refuse the machine" ;;
esac
case "$win" in
*tpm-tis*) ok "windows has a TPM" ;;
*) bad "windows has no TPM - its installer will refuse the machine" ;;
esac
case "$win" in
*"property=secure,value=on"*) ok "windows has a protected variable store" ;;
*) bad "windows secure boot is present but its variable store is writable from the guest" ;;
esac
case "$ubu" in
*tpm-tis*) bad "ubuntu has a TPM it does not need" ;;
*) ok "ubuntu has no TPM" ;;
esac
case "$ubu" in
*secboot*) bad "ubuntu is using secure-boot firmware it does not need" ;;
*) ok "ubuntu uses the plain firmware" ;;
esac
echo
echo "== the two machines are told what they are"
# Windows without the Hyper-V interfaces spins: it takes a VM exit per
# timer tick and per IPI, and burns CPU on locks a hypervisor could have
# yielded. Measured on this host as 347% CPU on an idle installer, in a
# cgroup that was neither short of memory nor throttled. Linux needs none
# of it - it has virtio - so this is the one place the two machines
# deliberately differ in their CPU model, and it is asserted both ways so
# neither drifts into the other's shape.
for name in ubuntu windows; do
argv=$(argv_for "$name")
cpu=$(printf '%s\n' "$argv" | grep '^host' | head -1)
case "$name:$cpu" in
windows:*hv-stimer-direct*)
ok "windows: Hyper-V enlightenments are on"
for flag in hv-relaxed hv-vapic hv-time hv-spinlocks hv-synic hv-tlbflush hv-ipi; do
case "$cpu" in
*"$flag"*) : ;;
*) bad "windows: $flag is missing from the cpu model" ;;
esac
done
case "$cpu" in
*hv-passthrough*) bad "windows: hv-passthrough hands over whatever the host has today - a frozen image must not change under a host upgrade" ;;
*) ok "windows: the enlightenments are listed, not passed through" ;;
esac ;;
windows:*) bad "windows: no Hyper-V enlightenments - it will spin. cpu='$cpu'" ;;
ubuntu:host) ok "ubuntu: plain -cpu host, which is all Linux wants" ;;
ubuntu:*) bad "ubuntu: cpu model is '$cpu', not plain host" ;;
esac
# Windows reads the RTC as local time. On a host at UTC that leaves the
# guest hours wrong, which breaks TLS and reads as a network fault.
rtc=$(printf '%s\n' "$argv" | grep '^base=' | head -1)
case "$name:$rtc" in
windows:base=localtime*) ok "windows: the clock is one it understands" ;;
windows:*) bad "windows: rtc is '$rtc' - its clock will be hours out" ;;
ubuntu:base=utc*) ok "ubuntu: the clock is UTC" ;;
ubuntu:*) bad "ubuntu: rtc is '$rtc'" ;;
esac
done
echo
echo "== which disk a maintenance boot opens"
# THE RULE: the disk's own mode decides. Frozen means the operator is
# looking at the image as the agent will get it, so qemu is given a
# throwaway overlay and the base is opened only as a read-only backing
# file. Unfrozen means they deliberately made it writable, so qemu gets
# the image itself.
#
# Asserted against whatever is really on this host rather than against a
# fixed expectation, plus a scratch image in the other slot so both
# branches run wherever this is executed.
disk0_of() { printf '%s\n' "$1" | grep '^file=.*id=disk0' | head -1; }
for name in ubuntu windows; do
disk="$BASE/$name.qcow2"
created=""
if [ ! -f "$disk" ]; then
qemu-img create -f qcow2 "$disk" 1G >/dev/null 2>&1
created=yes
fi
for mode in 0444 0664; do
chmod "$mode" "$disk"
line=$(disk0_of "$(argv_for "$name")")
case "$mode:$line" in
0444:file=$work/vmrun/maint-$name.qcow2,*)
ok "$name frozen: boots a throwaway disk, not the base" ;;
0444:*)
bad "$name frozen: a read-only base was opened directly - $line" ;;
0664:file=$disk,*)
ok "$name unfrozen: boots the base image itself" ;;
0664:*)
bad "$name unfrozen: did not open the base - $line" ;;
esac
# The overlay must be a fresh copy of the base every time, or a
# "discarded" change comes back on the next boot.
if [ "$mode" = 0444 ]; then
backing=$(qemu-img info --output=json "$work/vmrun/maint-$name.qcow2" 2>/dev/null \
| sed -n 's/.*"backing-filename": "\([^"]*\)".*/\1/p' | head -1)
[ "$backing" = "$disk" ] \
&& ok "$name frozen: the throwaway disk is backed by the base" \
|| bad "$name frozen: backing file is '$backing', not $disk"
fi
rm -f "$work/vmrun/maint-$name.qcow2"
done
rm -f "$disk"
done
echo
if [ "$fails" -gt 0 ]; then echo "FAILED $fails"; exit 1; fi
echo "the qemu command line is what it should be"
tests/test_vm_live_view.py
+335
View File
@@ -0,0 +1,335 @@
#!/usr/bin/env python3
"""The live view in the chat: two boxes, the right height, and one of each.
python3 tests/test_vm_live_view.py
WHY
This file exists because of two bugs that both looked fine from the
outside and neither of which any other test could see.
THE FIRST was a second copy of `_embed` left behind by a bad edit. Python
keeps the last definition of a method, so the class parsed, imported and
ran - and silently used the old single-box version while the new two-box
one sat above it, unreachable. Nothing failed. The only symptom was the
wrong thing on the screen.
THE SECOND was the height. Open WebUI sizes one of these frames two ways:
it reads `contentDocument.body.scrollHeight` on load, and it listens for
`postMessage({type:"iframe:height"})`. The first cannot work here - the
frame is sandboxed without `allow-same-origin`, so the read throws and is
swallowed by a bare `catch {}` - and the frame then keeps its stylesheet
default of 150px. That was the scrollbar. Measured in the browser: a
seven-line task list needs 138px and a twenty-line terminal 364px, and
both frames sat at 150 until the frame posted its own height.
There is a third trap inside the fix, and it has its own case below:
measuring `documentElement.scrollHeight` does not work either, because
`html{overflow:hidden}` makes it report the frame's CURRENT height
whenever the content is shorter. A box measured that way grows and never
shrinks. The wrapper's own bounding box has no such floor.
"""
import ast
import asyncio
import os
import re
import sys
import types
HERE = os.path.dirname(os.path.abspath(__file__))
FILTER = os.path.join(HERE, "..", "owui-functions", "vm_agent_progress.py")
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(label)
print(f"FAIL {label}\n {detail}")
src = open(FILTER, encoding="utf-8").read()
# --- no method is defined twice -----------------------------------------
tree = ast.parse(src)
cls = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == "Filter")
names = [n.name for n in cls.body
if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))]
dupes = sorted({n for n in names if names.count(n) > 1})
check("no method in the filter is defined twice", not dupes,
f"{dupes} - the later copy wins silently and the earlier one is dead code")
# --- load it with the two runtime imports stubbed out --------------------
aiohttp = types.ModuleType("aiohttp")
aiohttp.ClientError = Exception
aiohttp.ClientSession = object
aiohttp.ClientTimeout = object
sys.modules["aiohttp"] = aiohttp
pydantic = types.ModuleType("pydantic")
class _BaseModel:
def __init__(self, **kw):
for key, value in kw.items():
setattr(self, key, value)
pydantic.BaseModel = _BaseModel
pydantic.Field = lambda default=None, **kw: default
sys.modules["pydantic"] = pydantic
namespace: dict = {}
exec(compile(src, FILTER, "exec"), namespace)
Filter = namespace["Filter"]
class Valves:
show_terminal = True
show_screenshots = True
def embeds_for(data: dict) -> list:
out: list = []
async def emitter(event):
out.append(event)
asyncio.run(Filter._embed(emitter, data, Valves()))
return out[-1]["data"]["embeds"]
TASKS = "\n".join(f"[ ] {i + 1}. a task" for i in range(7))
CLI = {"tasks": TASKS, "terminal": ["$ one", "$ two"]}
GUI = {"tasks": TASKS, "shot": "data:image/jpeg;base64,AAAA",
"caption": "the desktop", "terminal": ["$ one", "$ two"]}
# --- the boxes, and what is in each ---------------------------------------
#
# The layout the operator asked for. The screen box is the only one that
# comes and goes: it appears when the agent takes a screenshot and not
# before, because most jobs never look at the screen and a picture of an
# idle desktop sitting in the chat from the handover onwards told the
# reader nothing.
#
# before any screenshot after one
# ------------------- -------------------
# live line live line
# Task List: Task List:
# tasks tasks
# ------------------- -------------------
# -------------------
# the machine's screen
# -------------------
# ------------------- -------------------
# command feed (20) command feed (20)
# ------------------- -------------------
plain = embeds_for(CLI)
check("a session with no screenshot yet gets two boxes", len(plain) == 2,
f"got {len(plain)} - the screen box has nothing to show")
check("the task list is the first of them", "a task" in plain[0])
check("and the command feed is the second", "$ one" in plain[1])
check("neither box contains the other's contents",
"$ one" not in plain[0] and "a task" not in plain[1],
"this is what a single joined box looked like")
check("and no picture is invented to fill the space", "<img" not in "".join(plain))
shot = embeds_for(GUI)
check("once a screenshot exists there are three boxes", len(shot) == 3,
f"got {len(shot)}")
check("the screen goes between the list and the feed",
"<img" in shot[1] and "the desktop" in shot[1] and "$ one" in shot[2],
"the feed is context for what is on the screen above it")
# One height, always. It used to be halved when a screen was shown, which
# meant the feed shrank for something that might never arrive.
def rows_in(box):
return box.count('<div style="white-space:pre;overflow:hidden;')
check("the command feed is the same height with or without a screen",
rows_in(shot[2]) == rows_in(plain[1]) == namespace["TERMINAL_LINES"],
f"{rows_in(shot[2])} vs {rows_in(plain[1])}")
# --- a viewer who joins late must not sit in front of an empty message ----
#
# These updates are pushed. Someone who reloads the page, or opens the chat
# on a second device, joins partway through and receives only what comes
# NEXT - and a session can hold one step for minutes ("Waiting for the
# machine to finish booting" held across four polls, measured). Emitting
# only on change leaves that viewer looking at nothing for the length of the
# step. The view is therefore re-sent on a slow heartbeat as well.
check("the whole view is re-sent even when nothing changed",
"or resend" in src and "RESEND_SECONDS" in src,
"a reader who reloads mid-session would see nothing until the next change")
check("the heartbeat is slower than the poll",
namespace["RESEND_SECONDS"] > 2.0,
"re-sending on every poll would be a message every two seconds for an hour")
check("re-sending replaces the view rather than appending to it",
'"replace": True' in src,
"otherwise a heartbeat leaves one frame per tick on the message")
# The end of the watcher is not the end of the session. This block runs on
# cancellation and on the watcher's own deadline too, and in those cases the
# machine is still running - saying it has been destroyed is simply false.
check("the machine is only called destroyed when the session said it was done",
"ended_cleanly" in src and "No longer watching this session." in src,
"the watcher stopping is not the same event as the session ending")
# --- a session that has said nothing must still look alive ---------------
#
# "Starting a machine" is the first thing the watcher emits, before there
# is any progress to read. If the progress file never appears - the job
# queued behind another session, the runner restarting, a boot that is
# simply slow - that sentence is the last word the reader gets. Reported
# as exactly that: a toast stuck on "Starting a machine" with no boxes,
# and a frozen line is indistinguishable from a broken one.
check("a silent start begins counting rather than repeating itself",
"QUIET_SECONDS" in src and "so far, nothing back" in src,
"a number that moves is the difference between waiting and wondering")
check("it waits a sensible moment before doing so",
10.0 <= namespace["QUIET_SECONDS"] <= 60.0,
f"{namespace['QUIET_SECONDS']}s - a normal boot must not trip it")
check("and the elapsed time comes from when the watch began",
"began = asyncio.get_running_loop().time()" in src)
# --- the confirmation dialog shows a summary, not the job -----------------
#
# The job the model writes is an instruction sheet for another machine, and
# a real one measured 2,600 characters. Nobody reads that to decide whether
# to press OK, and Open WebUI does not scroll the dialog: measured, a job
# that long pushed OK and Cancel 150px below the bottom of an 800px window,
# so the question could not be answered at all.
#
# The summary is built from the job rather than written by the model. This
# is the one screen where a summary that drifts from the text is a real
# problem - the person approves the summary and the machine is sent the job.
JOB = """Set up and test a log-file cleanup one-liner on this Ubuntu machine.
STEP 1 - Set up test files. Create the directory /tmp/logtest, then create:
- /tmp/logtest/recent.log -> mtime = now
- /tmp/logtest/ten_days.log -> mtime = 10 days ago
STEP 2 - BEFORE snapshot. Print the current time and list every file.
STEP 3 - DRY RUN. Run the command without -delete and capture the output.
STEP 4 - REAL RUN. Run the one-liner and capture the exit code.
STEP 5 - AFTER snapshot. Re-list everything the same way as STEP 2.
STEP 6 - Compare. Say which files went.
STEP 7 - FINAL REPORT. State for each file whether it was DELETED or KEPT."""
summary = Filter._summarise(JOB)
check("the dialog is given a summary, not the job", len(summary) < len(JOB) / 3,
f"{len(summary)} of {len(JOB)} characters - this dialog does not scroll")
check("it is short enough to leave the buttons on screen",
len(summary.splitlines()) <= Filter.SUMMARY_STEPS + 2,
"a job tall enough to outgrow the window makes the question unanswerable")
# EVERY BULLET IS ONE LINE, AND THE WHOLE DIALOG FITS A SMALL WINDOW.
#
# Measured in a 631px-tall browser - not an unusual window, just one that
# is not maximised - a 21-line message put OK and Cancel 65px below the
# bottom edge, and Open WebUI does not scroll that dialog.
check("no bullet is long enough to wrap on a narrow dialog",
all(len(line) <= Filter.SUMMARY_STEP + 4 for line in summary.splitlines()),
"a wrapped bullet costs two lines and there is no room for six of those")
check("the summary itself is at most seven lines",
len(summary.splitlines()) <= 7, f"{len(summary.splitlines())}")
check("it opens with what the job is for", summary.splitlines()[0].startswith("Set up and test"))
check("and lists what will actually be done",
"Set up test files" in summary and "DRY RUN" in summary)
# The whole message, not just the summary: the framing and the closing
# line are paid for out of the same budget.
def dialog_lines(job_text):
"""The message as _maybe_ask builds it, without needing a session."""
body = Filter._summarise(job_text)
framing = 2 # the one-line preamble plus its blank line
heading = 1 # WHAT IT WILL DO
closing = 2 # blank line plus the one-line instructions
return framing + heading + len(body.splitlines()) + closing
check("a long job still leaves a dialog that fits a small window",
dialog_lines(JOB) <= 12,
f"{dialog_lines(JOB)} lines - measured, 21 put the buttons off-screen at 631px")
check("steps it could not fit are counted rather than dropped silently",
"more step" in summary, "a summary that hides work is worse than no summary")
# Every line of the summary has to come from the job. This is the property
# that makes it safe to approve: it cannot describe work that is not there.
# "and N more steps" is the summariser's own phrasing; everything else has
# to be the job's own words.
ADDED = {"more", "step", "steps", "and"}
words = set(re.findall(r"[a-z0-9]+", JOB.lower()))
made_up = [w for w in re.findall(r"[a-z0-9]+", summary.lower())
if len(w) > 3 and w not in words and w not in ADDED]
check("nothing in the summary was invented", not made_up, f"{made_up}")
# The data a step operates on is not itself a step.
check("a bulleted list under a step is not mistaken for the steps",
"recent.log" not in summary,
"measured: counting those bullets pushed the real steps out entirely")
# A job with no headings still gets summarised rather than dumped.
prose = ("Check the disk setup. Report free space on every filesystem. "
"Report anything over 90 percent full. List the block devices.")
check("a job written as prose is summarised too",
"\n- " in Filter._summarise(prose))
# Open WebUI renders this dialog as markdown, where a plain newline is a
# SOFT break and consecutive lines are joined into one paragraph. Measured,
# that ran two steps together on one wrapped line. A "- " list item at
# column 0 is the form the renderer keeps on its own line; an indented one
# becomes a nested or literal block, and a bullet character of my own is
# not markdown at all and gets joined like any other prose line.
check("each step is a markdown list item at column 0",
all(line.startswith("- ") for line in summary.splitlines()[1:]),
"anything else is joined into a paragraph by the renderer")
check("and no step is indented into a nested block",
"\n - " not in summary and "\n\u2022" not in summary)
# A step that is only a label says nothing by itself.
labelled = Filter._summarise(
"Do the thing.\n1. Set up.\n2. Report:\n - which files went and which stayed")
check("a step that is only a label takes the line under it",
"which files went" in labelled,
"a bullet reading just 'Report:' spends a line and says nothing")
check("a one-line job is left alone",
"\n" not in Filter._summarise("Run uname -a and report the output."))
check("an empty job says so rather than showing nothing",
"empty" in Filter._summarise(" "))
# The colon case: splitting sentences on ":" turned a real step into "Run:".
# "e.g." and "(i.e." are not the ends of sentences, and the punctuation
# around them is part of the word as the splitter sees it.
for abbreviated, survives in (
("Create a folder, e.g. /tmp/logtest, and fill it", "logtest"),
("Report which files went (i.e. deleted) and which stayed", "deleted"),
("Check approx. 20 files and report", "files")):
check(f"an abbreviation does not end a step: {abbreviated[:28]}...",
survives in Filter._headline(abbreviated, 90),
"the bullet was cut off mid-abbreviation")
check("a step is not cut off at a colon",
"ps -p 1" in Filter._summarise("STEP 1 - Run: ps -p 1 -o comm= and check it says systemd"),
"the useful half of a step is always after the colon")
# --- no em dashes in anything the operator reads -------------------------
prose = [line for line in src.splitlines()
if "—" in line and not line.lstrip().startswith("#")]
check("no em dash reaches the chat", not prose, f"{prose[:2]}")
print()
if FAILS:
print(f"FAILED {len(FAILS)}")
raise SystemExit(1)
print("two boxes, each the size of what is in it, and a dialog that can be answered")
tests/test_vm_live_view_collapse.py
+206
View File
@@ -0,0 +1,206 @@
#!/usr/bin/env python3
"""Each box can be clicked shut, and the live line is revealed, not dumped.
python3 tests/test_vm_live_view_collapse.py
WHAT WAS ASKED FOR
* the live line above the task list should arrive a few characters at a
time, so that its speed is the token rate rather than a poll artefact;
* clicking the task list should leave two lines - that live line, and
the step the session is on;
* clicking the screen should halve it, on black;
* clicking the feed should halve its height;
* clicking a second time should put any of them back.
THREE THINGS HERE WERE MEASURED IN A BROWSER, NOT REASONED ABOUT
1. `window.name` SURVIVES THE srcdoc SWAP AND NOTHING ELSE DOES. The view
is re-sent every couple of seconds and each send replaces the frame's
srcdoc, destroying the document. The frame is sandboxed without
`allow-same-origin`, so it has an opaque origin: localStorage and
cookies throw and there is no listener in Open WebUI to postMessage
to. `window.name` belongs to the browsing context, which survives.
Measured in that exact sandbox: the third load read back the name its
two predecessors wrote. Without it a box springs open a second after
being clicked shut.
2. THE STATE IS KEYED BY WHICH BOX IT IS. The screen box appears
mid-session and pushes the feed down a slot, so the frame that was the
feed becomes the screen - and would inherit its state.
3. `flex:none` ON THE FEED ROWS IS LOAD-BEARING. The rows sit in a flex
column so a collapsed feed keeps its newest lines against the bottom.
Flex items shrink by default, so without it the rows compressed into
the smaller box and the text overlapped itself: measured, twenty lines
crushed into ten lines of space instead of ten lines shown and ten
lost off the top.
Browser-verified, in a real sandboxed frame, before this file was
written: task box 196px -> 51px and back; screen 500px -> 269px with the
image exactly half its width on black; feed 364px -> 190px showing ten
rows of an unchanged 17.4px. All three survived two srcdoc swaps.
"""
import asyncio
import json
import os
import re
import sys
import types
HERE = os.path.dirname(os.path.abspath(__file__))
FILTER = os.path.join(HERE, "..", "owui-functions", "vm_agent_progress.py")
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(f"{label}: {detail}")
print(f"FAIL {label}\n {detail}")
src = open(FILTER, encoding="utf-8").read()
aiohttp = types.ModuleType("aiohttp")
aiohttp.ClientError = Exception
aiohttp.ClientSession = object
aiohttp.ClientTimeout = object
sys.modules["aiohttp"] = aiohttp
pydantic = types.ModuleType("pydantic")
class _BaseModel:
def __init__(self, **kw):
for key, value in kw.items():
setattr(self, key, value)
pydantic.BaseModel = _BaseModel
pydantic.Field = lambda default=None, **kw: default
sys.modules["pydantic"] = pydantic
namespace: dict = {}
exec(compile(src, FILTER, "exec"), namespace)
Filter = namespace["Filter"]
class Valves:
show_terminal = True
show_screenshots = True
DATA = {
"tasks": "\n".join(f"[ ] {i + 1}. step {i + 1}" for i in range(6)),
"status": "3/6 · the step it is on",
"speech": "the model's own voice, mid-sentence",
"shot": "data:image/jpeg;base64,AAAA",
"caption": "the desktop",
"terminal": ["$ one", "$ two"],
}
out: list = []
async def _run():
async def emitter(event):
out.append(event)
await Filter._embed(emitter, DATA, Valves())
asyncio.run(_run())
boxes = out[-1]["data"]["embeds"]
check("three boxes once there is a screenshot", len(boxes) == 3, str(len(boxes)))
TASKS, SCREEN, FEED = boxes
def config(box: str) -> dict:
found = re.search(r'<script type="application/json" id="cfg">(.*?)</script>', box, re.S)
return json.loads(found.group(1).replace("<\\/", "</")) if found else {}
# --- every box says which one it is ---------------------------------------
check("each box carries its own config", all(config(b) for b in boxes))
check("and they are the three kinds, in order",
[config(b).get("k") for b in boxes] == ["tasks", "screen", "feed"],
str([config(b).get("k") for b in boxes]))
check("the config is JSON, not generated JavaScript",
'type="application/json"' in TASKS,
"the live line is the model's own words - a quote in it would end the script")
raw_cfg = re.search(r'id="cfg">(.*?)</script>', TASKS, re.S).group(1)
check("and a closing tag inside it cannot end the script early",
"</" not in raw_cfg,
"the HTML parser ends the block at the first </script>, wherever it appears, "
"so the JSON escapes it as <\\/ - which JSON reads back as a plain slash")
# --- the task box collapses to two lines ----------------------------------
check("the task box has a live line, the list, and the current step",
all(x in TASKS for x in ('id="live"', 'id="full"', 'id="cur"')))
check("the current step is the one the collapsed box will show",
"the step it is on" in TASKS,
"collapsed, the second line is the status - so it has to be in the box")
check("the step line is hidden until the box is clicked",
'id="cur" style="display:none' in TASKS)
check("the live line is right-anchored, because the server sends the tail",
"justify-content:flex-end" in TASKS,
"the end of the sentence is the part that is moving")
check("and it holds its line even before the model has spoken",
"min-height" in TASKS,
"otherwise the collapsed box is one line until the first token")
# --- the screen box halves onto black -------------------------------------
check("the screen box has a wrapper to blacken and an image to shrink",
'id="wrap"' in SCREEN and 'id="shot"' in SCREEN)
# --- the feed box halves its height ---------------------------------------
rows = FEED.count('<div style="white-space:pre;overflow:hidden;')
check("the feed is exactly as many rows as it claims",
rows == namespace["TERMINAL_LINES"], str(rows))
check("the rows are bottom-anchored, so a half feed keeps the newest lines",
"justify-content:flex-end" in FEED and 'id="rows"' in FEED)
check("AND THE ROWS CANNOT SHRINK",
FEED.count("flex:none") == namespace["TERMINAL_LINES"],
"flex items shrink by default: without this the twenty rows compress "
"into the smaller box and the text overlaps itself")
check("the feed knows how many rows it has, for the half",
config(FEED).get("n") == namespace["TERMINAL_LINES"])
# --- the script: state that outlives the document -------------------------
script = namespace["_VIEW_SCRIPT"]
check("state is read from and written to window.name",
"window.name" in script and "JSON.stringify(st)" in script,
"the only store that survives a srcdoc swap in an opaque origin")
check("state is discarded when the box is a different one",
"st.k!==kind" in script.replace(" ", ""),
"the screen box appears mid-session and pushes the feed down a slot")
check("a click toggles it, saves it, applies it and re-reports the height",
all(x in script.replace(" ", "") for x in
("addEventListener('click'", "st.c=st.c?0:1", "save();apply();post();")),
"the frame sizes itself, so a box that changes height must say so")
check("the height is still reported the only way that works here",
"iframe:height" in script and "postMessage" in script,
"contentDocument is cross-origin in this sandbox and fails silently")
check("no tooltip was added", "title=" not in TASKS and "title=" not in FEED)
# --- the script: the live line is revealed, not dumped --------------------
check("the line is appended a few characters at a time",
"setInterval" in script and "pending.slice" in script)
check("it continues the previous line rather than retyping it",
"st.d" in script,
"the text so far rides in window.name across the swap")
check("it works out what is new from the overlap with what is shown",
"target.slice(0,k)" in script.replace(" ", ""),
"the server sends a sliding window of the tail, not a growing string")
check("and the buffer it keeps is bounded",
"CAP" in script and "slice(-CAP)" in script.replace(" ", ""))
print()
if FAILS:
print(f"{len(FAILS)} failed")
for line in FAILS:
print(" -", line)
sys.exit(1)
print("every box can be clicked shut, and the live line arrives at the rate it was written")
tests/test_vm_live_view_safety.py
+117
View File
@@ -0,0 +1,117 @@
#!/usr/bin/env python3
"""The picture of the session may never be able to destroy the session.
python3 tests/test_vm_live_view_safety.py
WHY
A session ran all six of its tasks, verified them, and returned nothing.
The agent loop died at the last step with:
File "/app/loop.py", line 464, in _terminal_for
return [f"$ {str(arguments.get('verification', {}).get('command', ''))[:300]}"]
AttributeError: 'str' object has no attribute 'get'
`finish` is declared as taking `verification` as an object with a
`command` in it. The model sent a string. The renderer for the live view -
a cosmetic feature, added so somebody watching the chat can see what is
happening - called `.get` on that string, and the exception came back out
through the agent loop and threw the finished work away.
The rule this file enforces is the one that was missing: the live view has
no bearing on whether the work succeeds, so it does not get a vote. Every
renderer must survive whatever the model sends, and the call itself is
wrapped so that even an unanticipated shape costs a line of output rather
than a session.
"""
import os
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
# The container is built from two places - vmagent/app plus the socket
# client, which is shared with the host's own vmctl (vmagent/Dockerfile:39).
sys.path.insert(0, os.path.join(HERE, "..", "vmagent", "app"))
sys.path.insert(0, os.path.join(HERE, "..", "vm", "vmd"))
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(label)
print(f"FAIL {label}\n {detail}")
import loop # noqa: E402
# --- the exact call that killed a session -------------------------------
try:
loop._terminal_for("finish", {"summary": "done", "verification": "ls -l /tmp"})
survived = True
except Exception as exc: # noqa: BLE001
survived = False
detail = f"{type(exc).__name__}: {exc}"
check("finish with verification as a string does not raise", survived,
locals().get("detail", ""))
shown = loop._terminal_for("finish", {"verification": "ls -l /tmp"})
check("and the command it sent is still shown",
any("ls -l /tmp" in line for line in shown), f"{shown}")
check("the documented shape still works",
any("pytest" in line for line in
loop._terminal_for("finish", {"verification": {"command": "pytest"}})))
check("and a finish with no verification shows nothing rather than an empty prompt",
loop._terminal_for("finish", {}) == [])
# --- every renderer, against everything a model might send --------------
JUNK = ["a string", 12, None, [], ["a", "list"], True, {"unexpected": "keys"}, {}]
NAMES = ["run", "write_file", "read_file", "list_dir", "git_commit", "check_internet",
"reboot_vm", "web_search", "read_page", "open_page", "wait", "finish",
"plan_tasks", "update_task", "ask_operator", "not_a_tool"]
broke = []
for tool_name in NAMES:
for junk in JUNK:
try:
out = loop._terminal_for(tool_name, junk)
if not isinstance(out, list):
broke.append(f"{tool_name}({junk!r}) returned {type(out).__name__}")
except Exception as exc: # noqa: BLE001
broke.append(f"{tool_name}({junk!r}) raised {type(exc).__name__}: {exc}")
check("no renderer raises on any argument shape a model can send", not broke,
"\n ".join(broke[:6]))
# --- and the call site swallows even what the renderers do not ----------
class Exploding:
def terminal(self, *_lines):
raise RuntimeError("the emitter itself failed")
def detonate(_name, _payload):
raise ValueError("the renderer itself failed")
try:
loop._show(Exploding(), detonate, "run", {"command": "true"})
loop._show(Exploding(), loop._terminal_for, "run", {"command": "true"})
contained = True
except Exception as exc: # noqa: BLE001
contained = False
why = f"{type(exc).__name__}: {exc}"
check("a failure anywhere in the view is contained at the call site", contained,
locals().get("why", ""))
# --- the loop must not call a renderer any other way --------------------
src = open(os.path.join(HERE, "..", "vmagent", "app", "loop.py"), encoding="utf-8").read()
body = src[src.index("def _converse"):src.index("def _show")]
check("the loop only renders through the guarded path",
"ui.terminal(" not in body,
"an unguarded ui.terminal(*_render(...)) is how the first one got in")
print()
if FAILS:
print(f"FAILED {len(FAILS)}")
raise SystemExit(1)
print("the live view cannot take a session down with it")
tests/test_vm_preflight.py
Executable
+94
View File
@@ -0,0 +1,94 @@
#!/usr/bin/env python3
"""Unit tests for vm_agent's one pre-flight refusal.
python3 tests/test_vm_preflight.py
WHY THIS IS NOT A SMOKE CASE
The interesting half - a task that names its machine and is therefore let
through - starts a real VM session when exercised over HTTP, and a
session is minutes long by design. The smoke suite has a 90 s per-case
timeout, so only the refusals can live there; the "does not fire" side
has to be tested against the function.
WHAT IT IS FOR
A Python script runs on both machines, a .sh only on Ubuntu, a .ps1 only
on Windows. Guessing wrong destroys a whole session, because the machine
does not survive it. The system prompt asks the model to check with the
user first, but a prompt rule loses to an explicit user demand - the
finding restated four times in TUNING.md - so the tool refuses in its own
output, which is what this model treats as fact.
The refusal has to be narrow. Firing when the caller has already said
which machine would be worse than not firing at all: it would train the
model to add a magic word rather than to ask.
"""
import os
import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "app"))
from owuitools.vm_preflight import CROSS_PLATFORM, names_its_machine, should_ask # noqa: E402
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(f"{label}: {detail}")
print(f"FAIL {label}\n {detail}")
fires = should_ask
# --- must stop and ask: cross-platform, no machine named -----------------
for task in (
"write a python script that prints the first 20 primes and run it",
"check whether this pip package installs cleanly",
"run the node build and tell me if it passes",
"does this Java program compile",
"build the rust crate and report the warnings",
):
check(f"asks: {task[:46]!r}", fires(task, "ubuntu"), "did not fire")
# --- must NOT fire: the machine is already named -------------------------
for task, os_name in (
("write a bash script for ubuntu that prints the first 20 primes", "ubuntu"),
("run this python script on windows with powershell", "windows"),
("check the python version in the ubuntu container image", "ubuntu"),
("test this .ps1 file", "windows"),
("apt install python3-venv and check it works", "ubuntu"),
("run the python tests under linux", "ubuntu"),
("run install.bat and see if python is on PATH afterwards", "windows"),
):
check(f"lets through: {task[:42]!r}", not fires(task, os_name), "fired anyway")
# --- must NOT fire: nothing cross-platform in the task -------------------
for task, os_name in (
("write a script that renames every file in the folder", "ubuntu"),
("check how much disk space is free", "windows"),
("compile the C program and run it", "ubuntu"),
):
check(f"no trigger: {task[:42]!r}", not fires(task, os_name), "fired on a task with no trigger")
# --- the check must not be fooled by the word appearing in another word --
check("does not fire on 'pygmy' or 'pyramid'",
not CROSS_PLATFORM.search("draw a pyramid of pygmy goats"),
"matched a substring inside an unrelated word")
check("does not fire on 'gopher' for 'go'",
not CROSS_PLATFORM.search("write a gopher client"),
"'go' matched inside 'gopher'")
print()
print("=" * 60)
if FAILS:
print(f"FAILED {len(FAILS)}")
for f in FAILS:
print(" -", f)
sys.exit(1)
print("all vm_agent pre-flight cases pass")
tests/test_vm_prompt_consistency.py
+130
View File
@@ -0,0 +1,130 @@
#!/usr/bin/env python3
"""The text the two models read must agree with the code they drive.
python3 tests/test_vm_prompt_consistency.py
WHY THIS EXISTS
Prompt text rots quietly. When the CLI/GUI split was removed, the code
changed in one afternoon and the prose kept saying "in either mode" for
a week. When the handover screenshot was removed, the agent's prompt went
on promising "you will be handed a picture of it". Neither was caught,
because nothing reads the prompts the way the model does - as fact.
So this reads them. Every claim checked here is one that was found wrong
in a real prompt, tool description or report, and each check is scoped
to the text the model actually sees, not to comments about it.
"""
import os
import re
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.join(HERE, "..")
sys.path.insert(0, os.path.join(ROOT, "vmagent", "app"))
import tools # noqa: E402
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(f"{label}: {detail}")
print(f"FAIL {label}\n {detail}")
def read(*parts: str) -> str:
return open(os.path.join(ROOT, *parts), encoding="utf-8").read()
prompt = read("configs", "agent", "prompt.md")
screen = read("configs", "agent", "prompt.screen.md")
ubuntu = read("configs", "agent", "prompt.ubuntu.md")
windows = read("configs", "agent", "prompt.windows.md")
neutral = prompt + screen
agent_facing = neutral + ubuntu + windows
# What the sub-agent's model is actually sent: descriptions, not comments.
schemas = tools.schemas("ubuntu") + tools.schemas("windows")
descriptions = "\n".join(
s["function"]["description"] + " " +
" ".join(str(p.get("description", "")) for p in s["function"]["parameters"]["properties"].values())
for s in schemas)
tool_names = {s["function"]["name"] for s in tools.schemas("ubuntu")}
# The main model's side: its system prompt section and the tool docstring.
system_prompt = read("configs", "assistant", "system-prompt.md")
vm_src = read("app", "owuitools", "routers", "vm.py")
docstring = re.search(r'async def vm_agent\(.*?"""(.*?)"""', vm_src, re.S)
main_facing = system_prompt + (docstring.group(1) if docstring else "")
report_src = read("vm", "vmd", "report.py")
filter_src = read("owui-functions", "vm_agent_progress.py")
# --- there is one kind of session, and no text may say otherwise ---------
STALE = ("either mode", "CLI mode", "GUI mode", "cli mode", "gui mode",
"CLI session", "GUI session", "in a desktop session")
for where, text in (("agent prompts", agent_facing), ("tool descriptions", descriptions),
("main model text", main_facing), ("report", report_src)):
hits = [s for s in STALE if s in text]
check(f"no CLI/GUI mode language in the {where}", not hits, str(hits))
valves = re.findall(r'description="([^"]*)"', filter_src[:8000])
check("the filter's valve text does not describe modes that no longer exist",
not any(("CLI" in v or "GUI" in v) for v in valves), str(valves))
# --- promises that were once true ----------------------------------------
check("the prompt no longer promises a handover screenshot",
"handed a picture" not in prompt and "picture of it along" not in prompt,
"the handover screenshot was removed; the model must not wait for one")
# --- every tool the prompt names exists -----------------------------------
named = set(re.findall(r"`([a-z]+(?:_[a-z]+)+)`", agent_facing))
missing = sorted(n for n in named if n not in tool_names and n not in {
"py_compile", # python3 -m py_compile: a command, not a tool
"in_progress", # a task status, not a tool
})
check("every tool the prompts name is a tool the agent has", not missing, str(missing))
# --- machine paths stay in the machine's own section ----------------------
for path in ("/tmp", "/mnt/project", "Z:\\", "C:\\Windows", "apt", "winget"):
# Word-bounded, or "apt" matches "captured".
check(f"the machine-neutral prompt does not say {path!r}",
not re.search(r"(?<![A-Za-z])" + re.escape(path) + r"(?![A-Za-z])", neutral),
"a Windows session planned to 'check /mnt/project' because the text in front "
"of it mentioned the Linux path")
check("Ubuntu is told where scratch goes", "/tmp" in ubuntu)
check("Windows is told where scratch goes", "C:\\Windows\\Temp" in windows)
# --- the same thing is not said four times --------------------------------
graphical = sum(1 for t in (prompt, screen, ubuntu, windows)
if "nobody is looking" in t or "nobody can see" in t)
check("'a program started with run cannot be seen' is said once, not four times",
graphical <= 1, f"said in {graphical} files")
# --- the release and the link ----------------------------------------------
check("the agent is told scratch is filtered on commit",
"held back" in prompt or "held back" in descriptions,
"a filter nobody mentions is indistinguishable from a bug")
check("git_commit's description says what it will not commit",
"never reach the release" in descriptions)
check("the main model is told the release excludes scratch",
"scratch are never published" in system_prompt)
check("the main model is forbidden from inventing a link",
"Never assemble a URL" in system_prompt and "Never build a URL" in main_facing,
"one reply sent the user to a different project's release")
check("the report states a missing link as a fact",
"There is no link to give" in report_src,
"tool output is the channel the model treats as fact")
print()
if FAILS:
print(f"{len(FAILS)} failed")
for line in FAILS:
print(" -", line)
sys.exit(1)
print("all prompt-consistency checks passed")
tests/test_vm_release_contents.py
+181
View File
@@ -0,0 +1,181 @@
#!/usr/bin/env python3
"""A release carries the work, not the debris left beside it.
python3 tests/test_vm_release_contents.py
WHAT WENT WRONG, MEASURED
Release `v3` of the project `windows-troubleshooting-toolkit` held ten
files:
_build.ps1 _launch.bat _structcheck.ps1 _w_diagnostics.ps1
_w_network.ps1 _w_performance.ps1 _w_system.ps1 check_disp.ps1
check_disp2.ps1 check_disp3.ps1
Every one of them scratch, and not one of them the 34 KB
`TroubleshootingToolkit.ps1` the session had been asked to build. Two
causes, compounding: `git add -A` commits whatever the agent left lying
in the project folder, and the asset list sorts `_build` ahead of
`TroubleshootingToolkit` before stopping at ten.
WHY THE FIX IS AT THE COMMIT AND NOT AT THE RELEASE
Gitea builds a release's "Source code (zip)" from the tree its tag points
at, and that archive is the only thing on the page that downloads -
measured, `Content-Disposition: attachment`, where every uploaded asset
including a `.zip` comes back `inline`. So curating the asset list alone
would leave the actual download full of scratch. Filtering at the commit
cleans the repository, the assets and the archive together.
THE RISK THIS FILTER CARRIES, AND THE TWO GUARDS ON IT
Excluding by name can exclude the wrong thing. So: the patterns stay
narrow enough that `find_duplicates.py` and `Check-AutoStartServices.ps1`
- both real deliverables from real sessions - survive; a project that
would otherwise publish nothing at all publishes everything instead,
because then the tests were the job; and whatever is held back is named
back to the agent in tool output.
"""
import os
import subprocess
import sys
import tempfile
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.join(HERE, "..", "vm", "vmd"))
import gitea # noqa: E402
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(f"{label}: {detail}")
print(f"FAIL {label}\n {detail}")
def make_tree(root: str, names: list[str]) -> None:
for name in names:
full = os.path.join(root, name)
os.makedirs(os.path.dirname(full), exist_ok=True)
with open(full, "w") as handle:
handle.write("x\n")
def committed(repo: str) -> set[str]:
result = gitea._git(repo, "ls-tree", "-r", "--name-only", "HEAD")
return {n for n in result.stdout.split("\n") if n}
# --- the session that went wrong, rebuilt ---------------------------------
#
# The real file list from that project, verbatim.
TOOLKIT = "TroubleshootingToolkit.ps1"
REAL_SESSION = [
TOOLKIT,
"_build.ps1", "_launch.bat", "_structcheck.ps1", "_w_diagnostics.ps1",
"_w_network.ps1", "_w_performance.ps1", "_w_system.ps1",
"check_disp.ps1", "check_disp2.ps1", "check_disp3.ps1",
"csharp_out.txt", "csharp_thread.txt", "smoke.ps1", "t_out.txt",
"test_concurrency.ps1", "test_cs.cs", "test_cs.exe", "test_job.ps1",
"test_net.ps1", "test_thread.ps1", "test_pump.ps1",
]
with tempfile.TemporaryDirectory() as tmp:
repo = os.path.join(tmp, "windows-troubleshooting-toolkit")
os.makedirs(repo)
make_tree(repo, REAL_SESSION)
gitea.init(repo, "windows-troubleshooting-toolkit")
skipped = gitea.held_back(repo)
check("the toolkit is not held back", TOOLKIT not in skipped, str(skipped[:5]))
for name in ("_build.ps1", "_w_system.ps1", "test_cs.exe", "test_job.ps1",
"smoke.ps1", "t_out.txt"):
check(f"held back: {name}", name in skipped, f"held back: {skipped}")
sha = gitea.commit(repo, "the session's work")
check("the commit happened", bool(sha), f"returned {sha!r}")
tree = committed(repo)
check("THE DELIVERABLE IS IN THE RELEASE", TOOLKIT in tree,
"this is the exact failure: the release did not contain the toolkit")
check("and the scratch is not",
not ({"_build.ps1", "test_cs.exe", "smoke.ps1"} & tree),
f"committed: {sorted(tree)}")
assets = gitea.tracked_files(repo, limit=gitea.RELEASE_ASSETS_MAX)
# `check_disp*.ps1` and `csharp_thread.txt` survive, and that is the
# deliberate half of the trade: the patterns stay narrow enough that
# `check_disk.py` from a "check disk health" job would survive too.
# What matters is that the deliverable can no longer be crowded out.
check("THE DELIVERABLE IS AN ASSET", TOOLKIT in assets, str(assets))
check("and the cap can no longer push it out",
len(assets) < gitea.RELEASE_ASSETS_MAX, f"{len(assets)} assets")
check("scratch alone is not a reason to commit again",
not gitea.has_changes(repo),
"a tree holding only untracked scratch has nothing to publish")
# --- the guard against excluding a real deliverable ------------------------
with tempfile.TemporaryDirectory() as tmp:
repo = os.path.join(tmp, "dup-finder")
os.makedirs(repo)
# Both of these are real deliverables from real sessions, and both
# read as scratch to a filter that is one pattern too greedy.
make_tree(repo, ["find_duplicates.py", "Check-AutoStartServices.ps1",
"README.md", "latest/notes.md", "src/main.py"])
gitea.init(repo, "dup-finder")
gitea.commit(repo, "work")
tree = committed(repo)
for name in ("find_duplicates.py", "Check-AutoStartServices.ps1",
"README.md", "latest/notes.md", "src/main.py"):
check(f"kept, because it is the work: {name}", name in tree, str(sorted(tree)))
# --- when the tests WERE the job ------------------------------------------
with tempfile.TemporaryDirectory() as tmp:
repo = os.path.join(tmp, "write-some-tests")
os.makedirs(repo)
make_tree(repo, ["test_parser.py", "test_lexer.py"])
gitea.init(repo, "write-some-tests")
gitea.commit(repo, "the tests are the deliverable")
tree = committed(repo)
check("a project with nothing but tests publishes them",
{"test_parser.py", "test_lexer.py"} <= tree,
f"published nothing at all: {sorted(tree)}")
# --- a project resumed after collecting scratch under the old rules --------
with tempfile.TemporaryDirectory() as tmp:
repo = os.path.join(tmp, "resumed")
os.makedirs(repo)
make_tree(repo, ["Toolkit.ps1", "test_old.ps1"])
gitea.init(repo, "resumed")
# Committed the way the previous version of this code would have.
gitea._git(repo, "add", "-A", "--", ".")
subprocess.run(["/usr/bin/git", "-C", repo, *gitea._SAFE_FLAGS,
"commit", "--no-verify", "-q", "-m", "legacy"],
capture_output=True,
env={"PATH": "/usr/bin:/bin", "HOME": "/nonexistent", **gitea._SAFE_ENV},
check=False)
check("the legacy commit really does carry the scratch",
"test_old.ps1" in committed(repo), "the premise of this case is wrong")
check("and the next session notices", gitea.has_changes(repo),
"already-published scratch has to count as something to change")
make_tree(repo, ["Toolkit.ps1"])
gitea.commit(repo, "this session")
tree = committed(repo)
check("a resumed project stops publishing its old scratch",
"test_old.ps1" not in tree and "Toolkit.ps1" in tree, str(sorted(tree)))
check("but the file is still on the array to work with",
os.path.exists(os.path.join(repo, "test_old.ps1")),
"--cached: it stops being published, it is not deleted")
print()
if FAILS:
print(f"{len(FAILS)} failed")
for line in FAILS:
print(" -", line)
sys.exit(1)
print("all release-contents checks passed")
tests/test_vm_second_run.py
+143
View File
@@ -0,0 +1,143 @@
#!/usr/bin/env python3
"""A second run in the same chat is watched, and git stays on the host.
python3 tests/test_vm_second_run.py
THE BUG THIS EXISTS FOR
A conversation asked for an app, got it, then asked for a fix. The second
vm_agent call ran for 45 minutes and finished successfully - and the
person watching saw an empty message and a toast saying the machine had
been destroyed, and reasonably concluded the tool had given up.
The live-progress file is named after the CHAT, not the run. The first
run left it with `done: true`. The second run's watcher started, polled
once, read the previous run's ending, announced it and exited FOUR
SECONDS IN. Its whole status history was:
Calling vm_agent
VM · Starting a machine...
VM · Done
The machine has been destroyed.
while the run it was supposed to be watching carried on to sequence 1035.
Two guards, because a race with one guard is a race: the api deletes the
file before the job is queued, and the watcher ignores anything stamped
earlier than itself.
AND THE OTHER THING THAT RUN DID
It spent its last dozen turns fighting `git` inside the machine - exit
128, "dubious ownership", `safe.directory`, `HOME` - having said "let me
use the git_commit tool" at turn 99 and then gone back to raw git. A
commit made in the guest also skips everything gitea.py does on the host:
the hook and filter hardening, the right author, and leaving scratch out
of what gets published. So it is refused, in tool output.
"""
import ast
import os
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.join(HERE, "..")
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(f"{label}: {detail}")
print(f"FAIL {label}\n {detail}")
def read(*parts: str) -> str:
return open(os.path.join(ROOT, *parts), encoding="utf-8").read()
# --- guard one: the api clears the last run's state before queueing -------
api = read("app", "owuitools", "routers", "vm.py")
submit = api[api.index("async def _submit"):]
check("the api deletes the previous run's progress file",
'os.path.join(JOBS_DIR, "progress"' in submit and "os.unlink(stale)" in submit,
"otherwise the new watcher reads the old run's `done` and stops")
check("and the answer to the previous run's confirmation",
"answer.json" in submit,
"a stale answer would decide a question the person was never asked")
check("it happens before the job is queued, not after",
submit.index("os.unlink(stale)") < submit.index('f"{job_id}.ready"'),
"after the job is queued there is a window for the watcher to read it")
# --- guard two: the watcher ignores anything older than itself ------------
flt = read("owui-functions", "vm_agent_progress.py")
check("the watcher stamps when it started, on the wall clock",
"began_at = time.time()" in flt,
"the loop clock is monotonic and is not comparable with the file's stamp")
check("and discards progress written before then",
'float(data.get("updated") or 0) < began_at' in flt,
"this is what a previous run's file looks like")
check("the file it compares against is stamped on every flush",
'"updated": round(time.time(), 2)' in read("vmagent", "app", "events.py"))
# --- git belongs to the host ---------------------------------------------
vmd = read("vm", "vmd", "vmd.py")
# Lifted out of vmd.py and run, rather than matched as text: the point
# is what it decides, not what it looks like.
tree = ast.parse(vmd)
wanted = ("_GIT_WRITES", "_SEPARATORS", "_PREFIXES", "_git_write_in")
body = []
for node in tree.body:
if isinstance(node, ast.FunctionDef) and node.name in wanted:
body.append(node)
elif isinstance(node, ast.Assign) and any(
isinstance(t, ast.Name) and t.id in wanted for t in node.targets):
body.append(node)
check("the detector and everything it needs was found in vmd.py",
len(body) == len(wanted), f"found {len(body)} of {len(wanted)}")
namespace: dict = {"os": os}
exec(compile(ast.Module(body=body, type_ignores=[]), "vmd", "exec"), namespace)
detect = namespace["_git_write_in"]
# Every one of these is a real command line from the session that failed.
for command, want in [
("cd /mnt/project && git add -A && git status --short", "add"),
("cd /mnt/project && git config --global --add safe.directory /mnt/project "
"&& git add -A && git status --short", "config"),
("cd /mnt/project && export HOME=/root && git -c safe.directory=/mnt/project add -A "
"&& git -c safe.directory=/mnt/project commit -m 'x'", "add"),
("git -C /mnt/project commit -m x", "commit"),
("/usr/bin/git push origin main", "push"),
]:
check(f"refused: {command[:52]}...", detect(command) == want, f"got {detect(command)!r}")
# And the things that must NOT be caught, or the refusal becomes the problem.
for command in [
"git log --oneline -1",
"git status --short",
"python3 -c 'print(\"git add is a string\")'",
"grep -n 'git commit' notes.txt",
"ls -la /mnt/project",
"echo git commit > /tmp/note.txt",
]:
check(f"allowed: {command[:52]}", detect(command) is None, f"got {detect(command)!r}")
run_body = vmd[vmd.index("def v_run("):vmd.index("def v_write_file(")]
check("v_run refuses before it reaches the machine",
"_git_write_in(script)" in run_body and "raise VerbError" in run_body)
check("and the refusal names the tool that does work",
"git_commit" in run_body,
"a refusal that does not say what to do instead is a wall")
check("it explains the exit 128 rather than leaving it to be rediscovered",
"dubious ownership" in run_body)
print()
if FAILS:
print(f"{len(FAILS)} failed")
for line in FAILS:
print(" -", line)
sys.exit(1)
print("a second run is watched, and git stays where it works")
tests/test_vm_session_log.py
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env python3
"""What the agent did is written down, and can be read back.
python3 tests/test_vm_session_log.py
WHY
Asked "did it ever open the app and look at it", the honest answer was
that the record could not say. Everything needed was ALREADY being
recorded - `_record()` has covered screenshot, click, key and type since
they were written - and all of it was discarded when the session ended.
What was left to read were the two records that cannot answer it:
* the tool result counted `run` and nothing else, so a session that
marked "clicked through the GUI" done without ever calling `click`
read exactly like one that did;
* Open WebUI's status history is SAMPLED every two seconds, so a verb
that returns fast is usually overwritten before it is ever seen.
Measured on a real session: 456 samples caught `launch_app` nine
times because it blocks for seconds, and caught not one of the
screenshots that demonstrably happened.
So the transcript is now written to disk as it happens, one file per
session, with the model's own words between the calls - and the tool
result carries the counts, so a claim that disagrees with them is
visible without opening anything.
"""
import ast
import json
import os
import sys
import tempfile
HERE = os.path.dirname(os.path.abspath(__file__))
VMD = os.path.join(HERE, "..", "vm", "vmd")
sys.path.insert(0, VMD)
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(f"{label}: {detail}")
print(f"FAIL {label}\n {detail}")
with tempfile.TemporaryDirectory() as tmp:
os.environ["OWUITOOLS_VM_LOG_DIR"] = tmp
import sessionlog
sessionlog.LOG_DIR = tmp
log = sessionlog.open_log("abc12345deadbeef", chat="chat-1", os="windows",
project="toolkit", owner="someone", task="build a GUI")
check("a log is opened for the session", log is not None)
log.write("plan", count=3)
log.write("note", turn=1, text="I'll write the skeleton first.")
log.write("write_file", path="Z:\\Toolkit.ps1", bytes=4821)
log.write("run", command="powershell -File Z:\\Toolkit.ps1", exit_code=0, seconds=2.1)
log.write("launch_app", path="Toolkit.ps1")
log.write("screenshot", width=1280, height=800, bytes=86000)
log.write("task", id=1, status="done")
log.close(outcome="finished", verified=True, commits=1)
files = [n for n in os.listdir(tmp) if n.endswith(".jsonl")]
check("it is one file per session", len(files) == 1, str(files))
check("named so the newest sorts last", files[0][:2] == "20" and "abc12345" in files[0],
files[0])
records = sessionlog.load(os.path.join(tmp, files[0]))
kinds = [r["kind"] for r in records]
check("it opens with the session and closes with the outcome",
kinds[0] == "start" and kinds[-1] == "end", str(kinds))
check("the chat is recorded, so a chat maps to a session",
records[0].get("chat") == "chat-1",
"without it the only way to tell two runs apart is guessing at timestamps")
check("every call is in order", kinds[1:-1] ==
["plan", "note", "write_file", "run", "launch_app", "screenshot", "task"],
str(kinds))
check("each one is stamped with how far into the session it happened",
all("at" in r for r in records))
# --- the census is the thing that answers the question ---------------
counts = sessionlog.census(records)
check("the census counts the calls, not the talk",
counts.get("screenshot") == 1 and counts.get("launch_app") == 1
and "note" not in counts, str(counts))
check("and a verb never called simply is not there", "click" not in counts, str(counts))
text = sessionlog.render(os.path.join(tmp, files[0]))
check("the rendering names the session and the chat",
"abc12345deadbeef" in text and "chat-1" in text)
check("it shows what the model said beside what it did",
"I'll write the skeleton first." in text and "Toolkit.ps1" in text)
check("it states plainly when a verb was never used",
"it never called click" in text, text[-300:])
check("and the screen verbs it DID use are not reported as missing",
"it never called screenshot" not in text)
# --- a log may never take a session down -----------------------------
log2 = sessionlog.open_log("x", os="ubuntu")
log2._handle.close() # the worst thing that can happen
log2.write("run", command="still fine") # must not raise
log2.close(outcome="finished") # nor this
check("a broken log is silent, not fatal", True)
sessionlog.LOG_DIR = "/var/lib/owuivm/sessions"
# --- the wiring, in the code that has to do it ---------------------------
vmd_src = open(os.path.join(VMD, "vmd.py"), encoding="utf-8").read()
check("every record goes to the log as well as to memory",
"STATE.log.write(kind, **fields)" in vmd_src,
"one hook, so a verb added later is logged without anyone remembering to")
check("the log opens with the session and carries the chat id",
"sessionlog.open_log(" in vmd_src and 'chat=str(args.get("chat_id"' in vmd_src)
check("and closes with what became of it",
"STATE.log.close(" in vmd_src and "outcome=" in vmd_src)
tree = ast.parse(vmd_src)
bodies = {n.name: ast.unparse(n) for n in ast.walk(tree)
if isinstance(n, ast.FunctionDef)}
verbs = {name for name in bodies if name.startswith("v_")}
# Directly, or through the one helper it delegates to - v_click and
# v_move are both `_pointer`, which is where the record lives.
records_directly = {name for name, body in bodies.items() if "_record(" in body}
def records(name: str) -> bool:
if name in records_directly:
return True
return any(f"{helper}(" in bodies[name]
for helper in records_directly if helper != name)
recorded = {name for name in verbs if records(name)}
# Verbs that legitimately record nothing: they change no state in the
# machine, or they ARE the record.
EXEMPT = {"v_session_status", "v_session_close", "v_session_open", "v_answer",
"v_report", "v_note", "v_reap", "v_snapshot_status", "v_session_cancel"}
missing = sorted(verbs - recorded - EXEMPT)
check("every verb that touches the machine is recorded", not missing,
f"{missing} would happen without leaving a trace")
check("the harness logs what the model said each turn",
'vmd.call("note"' in open(os.path.join(HERE, "..", "vmagent", "app", "loop.py"),
encoding="utf-8").read())
sys.path.insert(0, os.path.join(HERE, "..", "vmagent", "app"))
import tools as agent_tools # noqa: E402
offered = {s["function"]["name"] for s in agent_tools.schemas("ubuntu")} | \
{s["function"]["name"] for s in agent_tools.schemas("windows")}
check("and the model is never told that verb exists", "note" not in offered,
"it is the harness's record of the model, not a tool for it")
report_src = open(os.path.join(VMD, "report.py"), encoding="utf-8").read()
check("the tool result carries the counts too",
"Calls it made:" in report_src,
"so the reader sees the numbers without going to the log")
check("and says so when the screen was never looked at",
"It never looked at the screen" in report_src)
print()
if FAILS:
print(f"{len(FAILS)} failed")
for line in FAILS:
print(" -", line)
sys.exit(1)
print("what the agent did is written down, in order, and can be read back")
tests/test_vm_share_mount.py
+107
View File
@@ -0,0 +1,107 @@
#!/usr/bin/env python3
"""The project folder is mounted by the session, not by the image.
python3 tests/test_vm_share_mount.py
WHY
The share is one virtiofs tag. The image built by the autoinstall carries
an fstab line for it; an image installed by hand does not, and the
failure that produces is the worst shape there is - the machine boots,
the agent runs its commands, files are written, the host commits an empty
directory, and the Gitea repository is simply empty. No error anywhere.
So the session mounts it over the guest channel every time, and these
cases cover the three things that has to get right: do nothing when there
is no share, do nothing on Windows (where the mount is WinFsp's job and
there is no command to run from here), and fail LOUDLY rather than carry
on when the mount does not take.
"""
import os
import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "vm", "vmd"))
import session as S # noqa: E402
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(f"{label}: {detail}")
print(f"FAIL {label}\n {detail}")
class FakeGuest:
def __init__(self, result):
self.result = result
self.scripts: list[str] = []
def __enter__(self):
return self
def __exit__(self, *_exc):
return False
def shell(self, script, timeout_s=0):
self.scripts.append(script)
return self.result
class FakeSession:
def __init__(self, os_name="ubuntu", share="/Pool/vmprojects/p", result=None):
self.os_name = os_name
self.share = share
self._guest = FakeGuest(result or {"exit_code": 0, "stdout": "", "stderr": ""})
def guest(self, timeout=0):
return self._guest
# --- nothing to mount ---------------------------------------------------
s = FakeSession(share=None)
S._mount_share(s)
check("no share: nothing is run in the guest", s._guest.scripts == [],
f"ran {s._guest.scripts}")
# --- Windows ------------------------------------------------------------
s = FakeSession(os_name="windows")
S._mount_share(s)
check("windows: no linux mount command is sent", s._guest.scripts == [],
f"ran {s._guest.scripts}")
# --- the normal case ----------------------------------------------------
s = FakeSession()
S._mount_share(s)
script = s._guest.scripts[0] if s._guest.scripts else ""
check("ubuntu: the share is mounted", "mount -t virtiofs project /mnt/project" in script,
f"script was: {script!r}")
check("and the mountpoint is created first", "mkdir -p /mnt/project" in script, script)
# Idempotence is not a nicety here: an image that DOES carry the fstab
# line is already mounted, and a second mount over the top would hide it.
check("and an existing mount is left alone", "mountpoint -q /mnt/project ||" in script, script)
check("and the result is checked rather than assumed",
script.strip().endswith("mountpoint -q /mnt/project"), script)
# --- the failure that must not be silent --------------------------------
s = FakeSession(result={"exit_code": 32, "stdout": "",
"stderr": "unknown filesystem type 'virtiofs'"})
try:
S._mount_share(s)
check("a failed mount stops the session", False, "it carried on regardless")
except S.SessionError as exc:
text = str(exc)
check("a failed mount stops the session", True)
check("and says what would have been lost",
"would be kept" in text or "nothing written" in text, text)
check("and quotes what the guest actually said", "virtiofs" in text, text)
print()
if FAILS:
print(f"FAILED {len(FAILS)}")
raise SystemExit(1)
print("the project folder is mounted by the session, and loudly if not")
tests/test_vm_tasks.py
Executable
+161
View File
@@ -0,0 +1,161 @@
#!/usr/bin/env python3
"""Unit tests for the session task record and project-name validation.
python3 tests/test_vm_tasks.py
Pure Python: no VM, no container, no network. These two modules are the
ones where a bug is silent rather than loud. A project name that slips
through becomes a path, and a task gate that can be talked around gives
the model back the freedom to declare itself finished - which is the
specific failure the archived sub-agent stack was retired for.
"""
import os
import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "vm", "vmd"))
import project # noqa: E402
import tasks # noqa: E402
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(f"{label}: {detail}")
print(f"FAIL {label}\n {detail}")
def raises(label: str, fn, exc) -> None:
try:
fn()
except exc:
print(f"ok {label}")
return
except Exception as other: # noqa: BLE001
FAILS.append(f"{label}: raised {type(other).__name__}, wanted {exc.__name__}")
print(f"FAIL {label}\n raised {type(other).__name__}")
return
FAILS.append(f"{label}: did not raise")
print(f"FAIL {label}\n did not raise")
# --- project names ------------------------------------------------------
for good in ("ok-name", "a1", "x9", "a" * 31, "script-tests"):
check(f"accepts {good!r}", project.validate(good) == good, "rejected")
for bad in ("../gitea", "UPPER", "has space", "-lead", "x", "", "a" * 32,
"dots.here", "under_score", "sub/dir", "name\n", "base", "trailing-"):
raises(f"rejects {bad!r}", lambda b=bad: project.validate(b), project.ProjectError)
# A rejection the model has to act on must say what to do instead.
try:
project.validate("../gitea")
except project.ProjectError as exc:
text = str(exc)
check("rejection names the constraint",
"lowercase" in text and "repository" in text, text)
# --- the task gate ------------------------------------------------------
def fresh() -> tasks.TaskList:
t = tasks.TaskList()
t.plan(["write the script", "run it", "fix what it reports"])
return t
raises("nothing works before plan_tasks",
lambda: tasks.TaskList().update(1, "done"), tasks.TaskError)
check("finish is refused before plan_tasks",
tasks.TaskList().why_not_finished() is not None, "allowed")
raises("a one-item plan is refused",
lambda: tasks.TaskList().plan(["do the thing"]), tasks.TaskError)
raises("a 21-item plan is refused",
lambda: tasks.TaskList().plan([f"t{i}" for i in range(21)]), tasks.TaskError)
raises("planning twice is refused",
lambda: fresh().plan(["a", "b"]), tasks.TaskError)
t = fresh()
check("finish refused while tasks are open", t.why_not_finished() is not None, "allowed")
check("the refusal lists what is open", "3 of 3 tasks are still open" in (t.why_not_finished() or ""),
t.why_not_finished() or "")
t.update(1, "done")
t.update(2, "done")
check("still refused with one open", t.why_not_finished() is not None, "allowed")
raises("blocked without a reason is refused",
lambda: t.update(3, "blocked"), tasks.TaskError)
raises("dropped without a reason is refused",
lambda: t.update(3, "dropped"), tasks.TaskError)
raises("an unknown status is refused",
lambda: t.update(3, "finished-ish", "x"), tasks.TaskError)
raises("an unknown task id is refused",
lambda: t.update(99, "done"), tasks.TaskError)
t.update(3, "blocked", "the package is not in noble")
check("finish allowed once everything is closed", t.why_not_finished() is None,
t.why_not_finished() or "")
check("the reason survives into the table",
"the package is not in noble" in t.render(), t.render())
# --- added tasks stay visible ------------------------------------------
t2 = fresh()
t2.add("check it on a clean boot", after_id=2)
check("an added task is marked as added", "(added mid-session)" in t2.render(), t2.render())
check("an added task lands where asked",
[x.text for x in t2.tasks][2] == "check it on a clean boot",
[x.text for x in t2.tasks])
check("an added task is open, so it blocks finish",
t2.why_not_finished() is not None, "allowed")
# --- the status feed ----------------------------------------------------
t3 = fresh()
check("status line before any progress", t3.status_line() == "0/3 · write the script", t3.status_line())
t3.update(1, "done")
t3.update(2, "in_progress")
check("status line follows the work", t3.status_line() == "1/3 · run it", t3.status_line())
# --- round trip ---------------------------------------------------------
t4 = fresh()
t4.update(1, "done")
t4.add("extra", None)
again = tasks.TaskList.from_json(t4.to_json())
check("survives a round trip", again.render() == t4.render(), again.render())
check("planned survives a round trip", again.planned is True, again.planned)
# --- the list is the only thing the watcher can read --------------------
#
# An agent that plans twelve tasks and then does the work in one long
# stretch without touching the list leaves the person watching on "1/12"
# for twenty minutes - measured - while the file grows and the status line
# says nothing. The work was fine; the report of it was a flat line.
#
# Nudged in TOOL OUTPUT, because the prompt already asks and asking has not
# worked. A nudge and not a refusal: being mid-task is legitimate, and a
# tool that blocked on it would be wrong far more often than right.
LOOP_SRC = open(os.path.join(os.path.dirname(os.path.abspath(__file__)),
"..", "vmagent", "app", "loop.py"), encoding="utf-8").read()
check("a run of steps with no task movement is noticed",
"TASK_DRIFT_LIMIT" in LOOP_SRC and "since_task" in LOOP_SRC)
check("the three task verbs reset it, and nothing else does",
'TASK_VERBS = frozenset({"plan_tasks", "update_task", "add_task"})' in LOOP_SRC)
after = LOOP_SRC.split("elif (since_task := since_task + 1) >= TASK_DRIFT_LIMIT:")[1]
check("it nudges rather than refusing",
"text +=" in after[:500] and "raise" not in after[:500],
"blocking on this would be wrong more often than right")
check("and the counter resets, so it does not fire on every step after",
after.lstrip().startswith("since_task = 0"))
print()
print("=" * 60)
if FAILS:
print(f"FAILED {len(FAILS)}")
for f in FAILS:
print(" -", f)
sys.exit(1)
print("all vm task and project-name cases pass")
tests/test_vm_two_machines.py
+177
View File
@@ -0,0 +1,177 @@
#!/usr/bin/env python3
"""A session runs on one of two machines, and is told about that one only.
python3 tests/test_vm_two_machines.py
WHY
Ubuntu or Windows. The differences are not decoration: the project folder
is `/mnt/project` or `Z:\\`, and the shell is bash or Windows PowerShell
5.1, which has no `&&`.
There used to be four kinds of session - those two machines by cli/gui -
and the second axis is gone. The calling model chose it and chose wrong:
asked to build a GUI application it picked the command line twice, with
the rule in the tool docstring and in the system prompt both times. A
session now gets the command line, the lookup suite AND the screen, and
decides for itself what it needs.
Both machines used to get the same prompt and the same tool descriptions,
which named both and left the agent to work out which half was about it.
It does not work out: this stack has repeatedly proved the model treats
tool text as fact rather than as something to reason around.
Measured, and the reason this file exists: a WINDOWS session wrote itself
the task *"check /mnt/project accessibility"* - the Linux path - because
the text in front of it said that was where work survives and nothing said
otherwise. It cost a step, and it would have cost the work if that path
had been writable.
"""
import os
import sys
import types
HERE = os.path.dirname(os.path.abspath(__file__))
APP = os.path.join(HERE, "..", "vmagent", "app")
sys.path.insert(0, APP)
sys.path.insert(0, os.path.join(HERE, "..", "vm", "vmd"))
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(label)
print(f"FAIL {label}\n {detail}")
import tools # noqa: E402
import loop # noqa: E402
MACHINES = ("ubuntu", "windows")
OTHER = {"ubuntu": "Z:\\", "windows": "/mnt/project"}
OWN = {"ubuntu": "/mnt/project", "windows": "Z:\\"}
# --- the prompt ----------------------------------------------------------
for os_name in MACHINES:
prompt = loop._load_prompt(os_name)
check(f"{os_name}: the prompt names its own project folder",
OWN[os_name] in prompt, os_name)
check(f"{os_name}: and never the other machine's",
OTHER[os_name] not in prompt,
"this is the exact text that put /mnt/project in a Windows task list")
check(f"{os_name}: every session is told it has a screen",
"There is a screen" in prompt)
check(f"{os_name}: and told that most work does not need it",
"Most work does not need any of that" in prompt,
"the screen is the exception; a command is faster and more reliable")
windows = loop._load_prompt("windows")
check("a Windows session is warned that 5.1 has no && or ||",
"&&" in windows and "5.1" in windows,
"this cost a session: the supervisor's own wrapper hit it too")
check("an Ubuntu session is not given PowerShell trivia",
"PowerShell" not in loop._load_prompt("ubuntu"))
# A fragment that is not there must not stop a session starting.
missing = loop._load_prompt("plan9")
check("an unknown machine still produces a usable prompt",
len(missing) > 1000 and "There is a screen" in missing,
"refusing to start over a missing file would be the worse failure")
# --- the tool descriptions ----------------------------------------------
for os_name in MACHINES:
schemas = tools.schemas(os_name)
names = [t["function"]["name"] for t in schemas]
text = " ".join(t["function"]["description"] for t in schemas)
tag = os_name
check(f"{tag}: the tools name its own project folder", OWN[os_name] in text)
check(f"{tag}: and not the other machine's", OTHER[os_name] not in text, tag)
check(f"{tag}: the screen tools are always there",
{"screenshot", "click", "launch_app"} <= set(names))
check(f"{tag}: so is the lookup suite",
"lookup_cve" in names and "web_search" in names,
"a session that cannot cheaply look something up reasons about it instead")
run_win = next(t for t in tools.schemas("windows")
if t["function"]["name"] == "run")["function"]["description"]
run_ubu = next(t for t in tools.schemas("ubuntu")
if t["function"]["name"] == "run")["function"]["description"]
check("the Windows run tool says PowerShell, and says it has no &&",
"PowerShell" in run_win and "&&" in run_win)
check("the Ubuntu run tool says bash, and mentions neither",
"bash" in run_ubu and "PowerShell" not in run_ubu and "&&" not in run_ubu)
check("neither describes the machine the session is not in",
"bash" not in run_win and "Z:" not in run_ubu)
# --- the command feed in a GUI session ----------------------------------
#
# Every tool a GUI session uses is a desktop verb. None of them rendered,
# so the feed under the screen would have been ten blank rows for ever.
for verb, args in (("screenshot", {}), ("click", {"x": 4, "y": 5}),
("key", {"name": "Return"}), ("type", {"text": "x"}),
("scroll", {"x": 1, "y": 2, "amount": -3}), ("move", {"x": 1, "y": 2})):
check(f"a GUI session's {verb} shows in the command feed",
loop._terminal_for(verb, args) != [],
"the third box would be blank rows for the whole session")
check("a click says where it went",
"4" in loop._terminal_for("click", {"x": 4, "y": 5})[0]
and "5" in loop._terminal_for("click", {"x": 4, "y": 5})[0])
# --- putting something on the real desktop -------------------------------
#
# The mechanism for this lived inside open_page and nowhere else, welded
# to browsers. A session that built its own GUI application had to
# rediscover it, and one did: it wrote a working 259-line WinForms toolkit
# in nine steps and then spent the remaining twenty of its sixty trying to
# get a window on screen, and ran out.
for os_name in MACHINES:
names = [t["function"]["name"] for t in tools.schemas(os_name)]
check(f"{os_name}: a session can launch a program onto the desktop",
"launch_app" in names)
launch = next(t for t in tools.schemas("windows")
if t["function"]["name"] == "launch_app")["function"]["description"]
check("it says why run is the wrong way to start a window",
"nobody is looking" in launch,
"starting a GUI from run does not fail, it just never appears")
check("and that a screenshot is still what proves it appeared",
"screenshot" in launch)
browser_src = open(os.path.join(HERE, "..", "vm", "vmd", "browser.py"),
encoding="utf-8").read()
check("no launcher passes env, because that is what removed PATH",
"env=[" not in browser_src and "OWUIVM_" not in browser_src,
"passing env REPLACES the child environment: schtasks then cannot be found")
check("and the check excludes the launcher's own process",
"$PID" in browser_src,
"otherwise it matches itself and reports a program that never started")
# Passing `env` to guest-exec REPLACES the environment, so PATH is gone
# and a bare "powershell" cannot be resolved. Measured: every launch died
# with "Failed to execute child process (No such file or directory)",
# including notepad.exe. Latent in open_page on Windows since it was
# written, because `run` passes no env and so never showed it.
check("the Windows interpreter is named in full wherever env is passed",
'"powershell", "-NoProfile"' not in browser_src
and "WINDOWS_POWERSHELL" in browser_src,
"a bare name needs a PATH, and passing env takes PATH away")
check("and the launch is confirmed by the process still being alive",
"exited" in browser_src and "93" in browser_src,
"a GUI program that dies on a startup error also returns cleanly")
# A desktop session needs more turns: look, wait, click, look again.
loop_src = open(os.path.join(HERE, "..", "vmagent", "app", "loop.py"), encoding="utf-8").read()
check("there is one step budget and it allows for looking at a screen",
"GUI_MAX_STEPS" not in loop_src and 'MAX_STEPS", "110"' in loop_src,
"every session can reach for the screen, so every session gets the room")
print()
if FAILS:
print(f"FAILED {len(FAILS)}")
raise SystemExit(1)
print("each machine is described to itself, and every session has the whole toolbox")
tests/test_vm_warm_start.py
+150
View File
@@ -0,0 +1,150 @@
#!/usr/bin/env python3
"""A saved machine must be fast, must match its image, and must never be load-bearing.
python3 tests/test_vm_warm_start.py
WHY
Resuming a saved machine turns a 90-second Windows boot into about 30
seconds of session setup, and the cold start of PowerShell - measured at
over ninety seconds on a fresh boot - is inside the saved memory rather
than in front of the agent.
The risk is not that it is slow, it is that it is WRONG. A saved state
describes memory that matches one particular disk, down to the page
cache. Resume it against an image that has changed since and nothing
reports an error: the guest simply runs with a filesystem in memory that
disagrees with the one on disk. So the interlock - the checksum recorded
when the image was frozen - is the part that matters, and these cases are
mostly about it.
The second rule is that this is an optimisation. An optimisation that can
take the feature down is not one, so a saved machine that will not load
must become a cold boot rather than a failed session.
"""
import json
import os
import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "vm", "vmd"))
import session as S # noqa: E402
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(f"{label}: {detail}")
print(f"FAIL {label}\n {detail}")
# --- the interlock ------------------------------------------------------
import tempfile # noqa: E402
work = tempfile.mkdtemp()
S.WARM_ROOT = os.path.join(work, "warm")
S.BASE_ROOT = os.path.join(work, "base")
os.makedirs(os.path.join(S.WARM_ROOT, "windows"))
os.makedirs(S.BASE_ROOT)
warm = os.path.join(S.WARM_ROOT, "windows")
for name in ("windows.qcow2", "state", "OVMF_VARS.fd"):
open(os.path.join(warm, name), "wb").write(b"x")
with open(os.path.join(S.BASE_ROOT, "windows.sha256"), "w") as fh:
fh.write(f"{'a' * 64} {S.BASE_ROOT}/windows.qcow2\n"
f"{'b' * 64} {S.BASE_ROOT}/windows-OVMF_VARS.fd\n")
def manifest(sha: str) -> None:
with open(os.path.join(warm, "manifest.json"), "w") as fh:
json.dump({"os": "windows", "base_sha256": sha}, fh)
manifest("a" * 64)
check("a saved machine matching the image is usable", S.warm_set("windows") is not None)
manifest("c" * 64)
check("one taken from a different image is refused", S.warm_set("windows") is None,
"a mismatched checksum was accepted - this is how a disk gets corrupted")
manifest("a" * 64)
os.rename(os.path.join(warm, "state"), os.path.join(warm, "state.away"))
check("one with no state file is refused", S.warm_set("windows") is None)
os.rename(os.path.join(warm, "state.away"), os.path.join(warm, "state"))
os.rename(os.path.join(warm, "manifest.json"), os.path.join(warm, "m.away"))
check("one with no manifest is refused", S.warm_set("windows") is None,
"nothing could vouch for it and it was used anyway")
os.rename(os.path.join(warm, "m.away"), os.path.join(warm, "manifest.json"))
with open(os.path.join(warm, "manifest.json"), "w") as fh:
fh.write("{not json")
check("an unreadable manifest is refused", S.warm_set("windows") is None)
manifest("a" * 64)
# An image with no recorded checksum cannot be vouched for either. That is
# the state an image is in before it has ever been frozen.
os.rename(os.path.join(S.BASE_ROOT, "windows.sha256"),
os.path.join(S.BASE_ROOT, "away"))
check("an image with no recorded checksum is refused", S.warm_set("windows") is None,
"there was nothing to compare against and it was used anyway")
os.rename(os.path.join(S.BASE_ROOT, "away"),
os.path.join(S.BASE_ROOT, "windows.sha256"))
check("and it is usable again once everything lines up", S.warm_set("windows") is not None)
check("an image with no saved machine at all is simply not warm",
S.warm_set("ubuntu") is None)
# --- the source, read as source -----------------------------------------
src = open(os.path.join(os.path.dirname(__file__), "..", "vm", "vmd", "session.py")).read()
vmd_src = open(os.path.join(os.path.dirname(__file__), "..", "vm", "vmd", "vmd.py")).read()
qmp_src = open(os.path.join(os.path.dirname(__file__), "..", "vm", "vmd", "qmp.py")).read()
launch = open(os.path.join(os.path.dirname(__file__), "..", "vm", "host", "owuivm-launch")).read()
# The share cannot be in a saved state - virtiofsd refuses migration with
# "vhost-user backend lacks VHOST_USER_PROTOCOL_F_LOG_SHMFD" - so a warm
# session must not be given one at boot and must plug it in afterwards.
check("a warm start boots without the share attached",
"'' if warm else (share or '')" in src,
"a saved machine cannot carry one; qemu refuses to migrate with it")
check("and the share is plugged in after the resume",
"_attach_share" in src and "attach_share" in qmp_src)
check("into the port the argv always carries",
"pcie-root-port,id=vfsport" in launch and 'bus="vfsport"' in qmp_src,
"q35's root complex is not hot-pluggable; without the port there is nowhere to put it")
check("at the same address a cold boot uses",
"bus=vfsport,addr=0x0" in launch and 'addr="0x0"' in qmp_src,
"a drive letter that moves between sessions is worse than a slow boot")
# The fallback is the whole safety argument.
check("a warm start that fails becomes a cold boot",
"allow_warm=False" in vmd_src and "warm_start_failed" in vmd_src,
"an optimisation that can take the feature down is not one")
check("and the caller is told it happened",
"booted from cold" in vmd_src)
# The clock: a resumed machine believes it is whenever it was saved.
check("the clock is set on every session, not only resumed ones",
"_sync_clock(session)" in src and "clock = _sync_clock" in src)
check("and the set is verified rather than assumed",
"info_time" in src and "within" in src,
"the first version swallowed the error and ran 26s behind")
# State is written through a descriptor vmd owns.
check("machine state is written through a descriptor, not a path",
"save_state_to_fd" in qmp_src and "send_fds" in qmp_src,
"qemu writes 0600 as its own user; the alternatives are a 4 GB copy "
"or letting the sandbox write the file a later session resumes from")
check("and the resume path only accepts this host's own saved state",
"/var/lib/owuivm/warm/(ubuntu|windows)/state$" in launch,
"a migration stream sets the machine's registers, memory and devices")
print()
if FAILS:
print(f"FAILED {len(FAILS)}")
raise SystemExit(1)
print("a saved machine is fast, is checked against its image, and cannot break a session")
tests/test_vm_web_tools.py
+266
View File
@@ -0,0 +1,266 @@
#!/usr/bin/env python3
"""Searching and opening a page must cost two calls, and stay outside the VM.
python3 tests/test_vm_web_tools.py
WHY
Reading one web page in GUI mode used to cost six round trips through a
27B model with thinking on: screenshot to find the browser, click to
focus it, click the address bar, type, Return, screenshot. Five of those
six are operating a text field.
So there are three tools instead: `web_search` and `read_page`, which run
on the HOST against the same SearXNG and the same fetcher the main model
uses, and `open_page`, which puts a URL on the screen in one call.
What these cases protect:
* the search must never be reachable FROM the machine. SearXNG sits in
another container's network namespace behind a default-deny filter, and
the whole sandbox argument rests on the guest not being able to reach
the house's own services.
* `open_page` is GUI-only. In CLI mode there is no desktop, and a tool
that cannot work is a tool the model will spend a turn discovering that
about.
* the URL is passed to the guest as an environment variable, never
interpolated into the shell script it runs.
"""
import os
import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "vmagent", "app"))
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "vm", "vmd"))
import browser
BROWSER_SRC = open(browser.__file__, encoding="utf-8").read() # noqa: E402
import tools # noqa: E402
import websearch # noqa: E402
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(f"{label}: {detail}")
print(f"FAIL {label}\n {detail}")
# --- what is offered, and where --------------------------------------
# ONE KIND OF SESSION. It used to be cli or gui, chosen by the calling
# model, with the screen tools in one and the lookup suite in the other.
# The caller chose wrong, a screenless session could build a window and
# never see it, and a screen-only session reasoned about syntax instead of
# looking it up. Every session now has all three.
offered = [t["function"]["name"] for t in tools.schemas("ubuntu")]
cli = gui = offered
planning = [t["function"]["name"] for t in tools.schemas("ubuntu", booting=True)]
for name in ("web_search", "read_page"):
check(f"{name} is offered in CLI mode", name in cli, str(cli))
check(f"{name} is offered in GUI mode", name in gui, str(gui))
check("open_page is there, because every session has a screen", "open_page" in offered)
check("none of them are offered while the machine boots",
not {"web_search", "read_page", "open_page"} & set(planning), str(planning))
# --- the URL never becomes shell ---------------------------------------
for bad in ("file:///etc/passwd", "javascript:alert(1)", "ftp://example.com",
"http://user:pw@example.com/", "https://example.com/ ; rm -rf /",
"https://example.com/`id`",
"not a url", "", "https://" + "a" * 3000):
try:
browser.validate(bad)
check(f"refuses {bad[:40]!r}", False, "it was accepted")
except browser.BrowserError:
check(f"refuses {bad[:40]!r}", True)
# `$`, `(` and `)` are RFC 3986 sub-delims and real URLs are full of them -
# every Wikipedia disambiguation page, for one. Rejecting them would break
# ordinary use to defend against an injection that cannot happen: the URL
# never enters a shell string. It is asserted inert below instead.
for good in ("https://example.com", "http://example.com/a/b?c=d&e=f#g",
"https://docs.python.org/3/library/os.html",
"https://en.wikipedia.org/wiki/Python_(programming_language)",
"https://example.com/$(id)"):
try:
browser.validate(good)
check(f"accepts {good}", True)
except browser.BrowserError as exc:
check(f"accepts {good}", False, str(exc))
# THE URL IS SUBSTITUTED, AND IT USED TO ARRIVE IN THE ENVIRONMENT.
#
# Passing `env` to guest-exec REPLACES the child environment rather than
# adding to it, so PATH disappears - a bare "powershell" could not be
# resolved, and inside the script neither could `schtasks`. Measured: every
# launch died with "Failed to execute child process (No such file or
# directory)", including notepad.exe. `run` never showed it because
# `shell()` passes no env at all, so this was latent in open_page on
# Windows from the day it was written.
#
# So the value is substituted into the script instead, and the safety
# question moves with it: a URL must not be able to become code.
for script in (browser.LINUX_SCRIPT, browser.WINDOWS_SCRIPT):
check("the script has exactly one slot for the URL and nothing else formatted",
script.count("__URL__") == 1 and "{url}" not in script and "%s" not in script,
script[:200])
check("no launcher passes an environment any more",
"env=[" not in BROWSER_SRC, "that is what removed PATH")
check("the value is quoted into a single-quoted literal",
browser._ps_quote(chr(39)) == chr(39) * 2,
"doubling the quote is what stops it closing the literal early")
check("so a quote in a URL cannot end the string early",
chr(39) not in browser._ps_quote("x" + chr(39) + "y").replace(chr(39) * 2, ""))
check("the only other substitution is a constant",
browser.CHROMIUM_FLAGS.startswith("--") and "__FLAGS__" in browser.LINUX_SCRIPT)
check("and starts the browser as the desktop user, not as root",
'runuser -u "$user"' in browser.LINUX_SCRIPT)
# systemd-run --user, not a hand-built environment: gnome-shell is the
# compositor and never had WAYLAND_DISPLAY to hand down, so reading its
# /proc/environ produced an X11 fallback that died. The user manager is
# where a desktop exports what a launcher needs.
check("through the user's own systemd instance, which has the display environment",
"systemd-run --user" in browser.LINUX_SCRIPT)
# Assert against the CODE, not the prose. The comments in that script
# name both `pgrep` and the mistakes that are no longer in it, and a
# naive `in` test reads a warning as the thing it warns about - the same
# shape of error as `pgrep -f chromium` matching its own command line.
code = "\n".join(line for line in browser.LINUX_SCRIPT.splitlines()
if not line.lstrip().startswith("#"))
check("and detaches by construction, so the call does not wait for a browser to close",
"--unit=" in code and not any(l.rstrip().endswith("&") for l in code.splitlines()),
"something is still backgrounded with &")
check("and the launch is verified by that unit's state, not by a process name",
"is-active" in code and "pgrep" not in code,
"a pgrep survived in the code")
# LocalSystem cannot draw on the logged-on user's desktop; a GUI process
# started from session 0 succeeds and is invisible, which is worse than
# failing. The scheduled task is what crosses into the session.
check("windows crosses into the interactive session",
"/it" in browser.WINDOWS_SCRIPT and "schtasks" in browser.WINDOWS_SCRIPT)
check("and cleans the task up again",
"schtasks /delete" in browser.WINDOWS_SCRIPT)
# --- the search goes to the host's gateway, with a bearer token --------
sent: dict = {}
def fake_post(path, payload):
sent["path"], sent["payload"] = path, payload
return "results"
websearch._post = fake_post # noqa: SLF001
websearch.search("qemu guest agent", max_results=99, site="qemu.org")
check("search posts to /web_search", sent["path"] == "/web_search", str(sent))
check("and clamps max_results to what the endpoint accepts",
sent["payload"]["max_results"] == 15, str(sent["payload"]))
check("and passes a site filter through", sent["payload"].get("site") == "qemu.org",
str(sent["payload"]))
websearch.read("https://example.com", max_chars=999999)
check("read_page posts to /fetch_page", sent["path"] == "/fetch_page", str(sent))
check("and clamps the window", sent["payload"]["max_chars"] == 20000, str(sent["payload"]))
for empty in ("", " "):
try:
websearch.search(empty)
check("an empty search is refused", False, "it was accepted")
except websearch.SearchError:
check("an empty search is refused", True)
break
# --- and it is the HOST that searches, never the guest -----------------
# The verb list is the contract: if a future change moved the search into
# the machine, it would have to pass through a guest command, and this is
# where that would show up.
vmd_source = open(os.path.join(os.path.dirname(__file__), "..", "vm", "vmd", "vmd.py")).read()
start = vmd_source.index("def v_web_search")
end = vmd_source.index("def v_open_page")
check("web_search and read_page never touch the guest",
"guest(" not in vmd_source[start:end] and "shell(" not in vmd_source[start:end],
"a guest call appeared in a host-side verb")
check("open_page does require a ready machine",
"_require_ready()" in vmd_source[vmd_source.index("def v_open_page"):
vmd_source.index("def v_reboot_vm")])
# --- the lookup suite, and whether it still matches the endpoints ------
# These schemas are hand-written so they stay terse - the agent pays for
# every one of them on every turn - which means they can drift from the
# endpoints they call. Measured the first time this ran: read_man_page
# takes `name`, and the schema said `page`, so the agent's call came back
# "Field required" from a tool that was working perfectly.
# EVERY SESSION GETS THE LOOKUP SUITE, GUI INCLUDED.
#
# It used to be CLI-only, on the reasoning that a desktop session could
# read anything it needed by opening it in the browser. True, and far too
# expensive: open_page costs a screenshot per scroll where read_page costs
# one call. Worse, a session that cannot cheaply look something up reasons
# about it instead - measured, one spent eleven minutes and eight context
# compactions deriving how to run background work in PowerShell, which the
# documentation states outright, while a parse-clean 19 KB application it
# had already written sat untouched.
cli_names, gui_names = set(cli), set(gui)
# Read back off the RENDERED schemas rather than a module-level list: the
# definitions live in configs/agent/tools.toml now, so what matters is
# what comes out of schemas(), not what went into it.
research_schemas = [t for t in tools.schemas("ubuntu")
if t["function"]["name"] in tools.RESEARCH_NAMES]
for t in research_schemas:
name = t["function"]["name"]
check(f"{name} is offered in both modes",
name in cli_names and name in gui_names,
"being stuck costs more than a few extra tool schemas")
check("and the screen tools are there alongside them",
{"open_page", "screenshot", "click", "move", "key", "type", "scroll",
"launch_app"} <= set(offered), str(sorted(offered)))
try:
import json as _json
import urllib.request as _req
spec = _json.load(_req.urlopen("http://172.22.0.10:8000/openapi.json", timeout=10))
except Exception as exc: # noqa: BLE001
print(f"skip the gateway is not reachable from here ({type(exc).__name__}); "
"schema drift not checked")
else:
comps = spec["components"]["schemas"]
for t in research_schemas:
name = t["function"]["name"]
try:
body = spec["paths"][f"/tools/{name}"]["post"]["requestBody"]
ref = body["content"]["application/json"]["schema"]["$ref"].rsplit("/", 1)[-1]
real = comps[ref]
except KeyError:
check(f"{name} still exists on the gateway", False, "the endpoint is gone")
continue
ours = set(t["function"]["parameters"]["properties"])
theirs = set(real.get("properties", {}))
invented = ours - theirs
missing = set(real.get("required", [])) - set(t["function"]["parameters"].get("required", []))
check(f"{name}'s schema matches the endpoint",
not invented and not missing,
f"invented={sorted(invented)} missing-required={sorted(missing)}")
# And the allowlist on vmd's side is the real gate, so it must name
# tools that exist - and must not name the ones deliberately left out.
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "vm", "vmd"))
import websearch as ws # noqa: E402
for name in ws.RESEARCH_TOOLS:
check(f"the allowlist entry {name} is a real endpoint",
f"/tools/{name}" in spec["paths"])
for forbidden in ("vm_agent", "matlab_run", "dns_lookup", "http_headers",
"inspect_tls", "lookup_ip_or_asn"):
check(f"{forbidden} is not reachable from a session",
forbidden not in ws.RESEARCH_TOOLS,
"the machine can answer that itself, or should never call it")
print()
if FAILS:
print(f"FAILED {len(FAILS)}")
raise SystemExit(1)
print("the web tools cost two calls and stay outside the machine")
tests/test_vm_working_directory.py
+103
View File
@@ -0,0 +1,103 @@
#!/usr/bin/env python3
"""The agent and its supervisor must mean the same thing by "here".
python3 tests/test_vm_working_directory.py
WHY
A Windows session was asked to write a PowerShell script, test it, and
keep it. It did all three, perfectly, and the session was reported as a
failure.
The agent's `run` inherited whatever directory the guest agent happened to
start in - on Windows, `C:\\Windows\\system32`. So `Set-Content
Get-SystemInfo.ps1`, `Get-Content .\\Get-SystemInfo.ps1` and
`powershell .\\Get-SystemInfo.ps1` all agreed with each other and all
passed. The supervisor then re-ran the verification from the PROJECT
folder, because that is the only place work survives, and found nothing
there. The model read the failure, worked out what had happened, and spent
a second whole session redoing it - roughly four minutes of machine time
to move a file.
Two rules come out of that, and this file holds both:
1. There is ONE definition of where a command runs, and both sides call
it. Two copies of this logic is exactly how they came to disagree.
2. It is the project folder, because that is the only thing that
outlives the machine. A file written anywhere else is deleted with
the disk.
"""
import ast
import os
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
VMD = os.path.join(HERE, "..", "vm", "vmd")
sys.path.insert(0, VMD)
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(label)
print(f"FAIL {label}\n {detail}")
import verify # noqa: E402
class FakeSession:
def __init__(self, os_name, share):
self.os_name = os_name
self.share = share
# --- the prelude itself --------------------------------------------------
linux = verify.working_directory_prelude(FakeSession("ubuntu", "/Pool/vmprojects/x"))
windows = verify.working_directory_prelude(FakeSession("windows", "/Pool/vmprojects/x"))
check("a Linux command starts in the project folder", "/mnt/project" in linux, linux)
check("a Windows command starts in the project folder", "Z:\\" in windows, windows)
check("the Windows prelude is PowerShell, not shell",
"Set-Location" in windows and "||" not in windows,
"PowerShell has no ||; this exact mistake reported correct work as a claim")
check("the Linux prelude is shell", "cd " in linux)
check("neither prelude can fail the command it prefixes",
"SilentlyContinue" in windows and "2>/dev/null" in linux,
"a missing share must not turn every command into an error")
# With no project folder there is nowhere to keep anything, but a command
# still has to run somewhere predictable.
check("a session with no project folder still lands somewhere definite",
"C:\\" in verify.working_directory_prelude(FakeSession("windows", None))
and "/root" in verify.working_directory_prelude(FakeSession("ubuntu", None)))
# --- and both sides use it -----------------------------------------------
vmd_src = open(os.path.join(VMD, "vmd.py"), encoding="utf-8").read()
verify_src = open(os.path.join(VMD, "verify.py"), encoding="utf-8").read()
tree = ast.parse(vmd_src)
run_verb = next(n for n in ast.walk(tree)
if isinstance(n, ast.FunctionDef) and n.name == "v_run")
run_src = ast.get_source_segment(vmd_src, run_verb) or ""
check("the agent's own run is given the prelude",
"working_directory_prelude" in run_src,
"this is the half that was missing, and it cost a whole extra session")
check("the supervisor's re-run is given the same prelude",
"working_directory_prelude" in verify_src)
# One definition, not two. The duplicate is how they drifted apart.
check("the directory is decided in exactly one place",
verify_src.count('Set-Location -LiteralPath') == 1
and vmd_src.count("Set-Location") == 0,
"a second copy is how the agent and the supervisor came to disagree")
print()
if FAILS:
print(f"FAILED {len(FAILS)}")
raise SystemExit(1)
print("the agent and its supervisor start in the same place, and it is the one that survives")
tests/tool_toggle_routes.py
+84
View File
@@ -0,0 +1,84 @@
#!/usr/bin/env python3
"""Run INSIDE the api image by tests/test_tool_toggle.sh - see that file.
Asserts the half of the switch that needs a real app: a disabled tool is
absent from the OpenAPI spec the model reads, AND its endpoint 404s. Both
halves matter. Absent-from-the-spec is what stops the context being spent
describing it; gone-from-the-routes is what stops a stale cached spec
somewhere else from calling it anyway.
"""
import importlib
import os
import sys
import tempfile
os.environ["OWUITOOLS_AUTH_TOKEN"] = "toggle-probe"
# Makes vm_agent appear: main.py registers that router only when the job
# volume is mounted, so without this the tool is absent for a reason that
# has nothing to do with the switch being tested.
os.environ["OWUITOOLS_VM_JOBS_DIR"] = tempfile.mkdtemp()
sys.path.insert(0, "/app")
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(f"{label}: {detail}")
print(f"FAIL {label}\n {detail}")
def offered(config_path: str):
"""Reimport the app against a given config and return (app, tool names)."""
os.environ["OWUITOOLS_TOOLS_CONFIG"] = config_path
for module in [name for name in list(sys.modules) if name.startswith("owuitools")]:
del sys.modules[module]
main = importlib.import_module("owuitools.main")
spec = main.app.openapi()
names = {operation.get("operationId")
for path in spec["paths"].values() for operation in path.values()}
return main.app, {name for name in names if name}
_, everything = offered("/configs/tools.toml")
check("the shipped config offers every tool", len(everything) == 22,
f"{len(everything)} tools: {sorted(everything)}")
with tempfile.NamedTemporaryFile("w", suffix=".toml", delete=False) as handle:
handle.write("[tools]\nsearch_papers = false\npcb_calc = false\n"
"vm_agent = false\nnot_a_tool = false\n")
switched = handle.name
app, trimmed = offered(switched)
check("exactly the three disabled tools left the spec",
everything - trimmed == {"search_papers", "pcb_calc", "vm_agent"},
f"removed {sorted(everything - trimmed)}")
check("nothing else was removed", len(trimmed) == len(everything) - 3,
f"{len(trimmed)} remain of {len(everything)}")
from starlette.testclient import TestClient # noqa: E402
client = TestClient(app)
auth = {"Authorization": "Bearer toggle-probe"}
disabled_call = client.post("/tools/pcb_calc", json={"calculation": "via_current"}, headers=auth)
check("a disabled tool's endpoint is gone, not merely undescribed",
disabled_call.status_code == 404, f"got HTTP {disabled_call.status_code}")
enabled_call = client.post("/tools/subnet_calc", json={"network": "192.168.1.0/24"}, headers=auth)
check("an enabled tool still answers", enabled_call.status_code == 200,
f"got HTTP {enabled_call.status_code}")
_, fallback = offered("/nonexistent/tools.toml")
check("a missing config leaves every tool on", fallback == everything,
f"{len(fallback)} vs {len(everything)}")
os.unlink(switched)
print()
if FAILS:
print(f"{len(FAILS)} FAILED")
for line in FAILS:
print(f" - {line}")
raise SystemExit(1)
print("all route-level toggle checks passed")
tests/try-tool.sh
Executable
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env bash
# Call one tool by hand, the way Open WebUI would.
#
# ./tests/try-tool.sh is the server up?
# ./tests/try-tool.sh web_search '{"query":"systemd socket activation"}'
# ./tests/try-tool.sh subnet_calc '{"network":"192.168.1.0/24"}'
#
# WHY THIS IS NOT `curl http://localhost:8000/...`
#
# Because that does not work, and worse, it can appear to. NOTHING in
# this stack publishes a port to the host: the app container has no
# default route and the gateway is only on Open WebUI's docker network.
# A curl to localhost:8000 reaches whatever else happens to be listening
# there - on the machine this was written on, something answered "OK" and
# looked for all the world like a healthy tool server.
#
# So the call is made from inside the api container, which is where the
# token already is. Same code path a real tool call takes, minus nginx.
set -euo pipefail
cd "$(dirname "$0")/.."
TOOL="${1:-}"
BODY="${2:-{\}}"
if [ -z "$TOOL" ]; then
docker compose exec -T api python -c \
"import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:8000/health').read().decode())"
exit 0
fi
docker compose exec -T -e TOOL="$TOOL" -e BODY="$BODY" api python - <<'PY'
import json, os, sys, urllib.error, urllib.request
tool, body = os.environ["TOOL"], os.environ["BODY"]
try:
json.loads(body)
except ValueError as exc:
sys.exit(f"the second argument is not valid JSON: {exc}")
request = urllib.request.Request(
f"http://127.0.0.1:8000/tools/{tool}",
data=body.encode(),
headers={"Authorization": "Bearer " + os.environ["OWUITOOLS_AUTH_TOKEN"],
"Content-Type": "application/json"},
)
try:
answer = urllib.request.urlopen(request, timeout=120).read().decode()
except urllib.error.HTTPError as exc:
if exc.code == 404:
sys.exit(f"there is no tool called {tool!r}. It may be switched off in "
"configs/tools.toml - see docs/tools.md.")
sys.exit(f"HTTP {exc.code}: {exc.read().decode()[:400]}")
# Tools return a JSON string; print what the model would actually read.
try:
answer = json.loads(answer)
except ValueError:
pass
print(answer if isinstance(answer, str) else json.dumps(answer, indent=2))
PY
tests/vm_agent_asserts.sh
Executable
+139
View File
@@ -0,0 +1,139 @@
#!/bin/bash
# =====================================================================
# What the sub-agent container can and cannot reach.
#
# ./tests/vm_agent_asserts.sh
#
# WHY THIS IS A SEPARATE BOUNDARY FROM THE GUEST'S
#
# The VM's isolation is about code somebody else wrote. This one is about
# OUR code, and it exists because of two specific things on the other
# side of it:
#
# Ollama has no authentication at all, and serves DELETE /api/delete,
# POST /api/pull and POST /api/create. Anything that can reach
# 172.17.0.1:11434 can delete the model the whole assistant runs on.
#
# The Open WebUI key has to be an ADMIN key, because anyone may invoke
# vm_agent and the chat belongs to whoever is talking. An admin key in
# the agent container would be an agent that can read every
# conversation on this instance.
#
# So the agent reaches exactly one address, and the thing at that address
# exposes exactly three URLs. This checks both halves.
# =====================================================================
set -uo pipefail
AGENT=owuitools-vmagent
GW=172.22.2.10:8000
fails=0
ok() { printf ' ok %s\n' "$*"; }
bad() { printf ' FAIL %s\n' "$*"; fails=$((fails+1)); }
docker inspect "$AGENT" >/dev/null 2>&1 || { echo "$AGENT is not running" >&2; exit 1; }
echo "== 1. the agent reaches vmgw, and nothing else"
probe=$(docker exec -i "$AGENT" python3 - <<'PY'
import socket
def p(h, port):
try:
socket.create_connection((h, port), 3).close(); return "REACHED"
except Exception: return "blocked"
for h, port, name in [("172.22.2.10", 8000, "vmgw"), ("172.17.0.1", 11434, "ollama-direct"),
("172.19.0.2", 8080, "openwebui-direct"), ("192.168.1.10", 7200, "gitea-lan"),
("192.168.1.10", 22, "host-sshd"), ("192.168.1.1", 80, "lan-gateway"),
("1.1.1.1", 443, "internet")]:
print(name, p(h, port))
PY
)
while read -r name verdict; do
[ -z "$name" ] && continue
if [ "$name" = "vmgw" ]; then
[ "$verdict" = "REACHED" ] && ok "vmgw is reachable (positive control)" \
|| bad "vmgw is NOT reachable - this suite proves nothing"
else
[ "$verdict" = "blocked" ] && ok "$name is blocked" || bad "$name was REACHABLE"
fi
done <<< "$probe"
echo
echo "== 2. vmgw exposes two URLs and refuses everything else"
paths=$(docker exec -i "$AGENT" python3 - <<PY
import json, urllib.request, urllib.error
BASE = "http://$GW"
def code(path, method="GET"):
try:
r = urllib.request.Request(BASE + path, method=method,
data=b"{}" if method == "POST" else None)
if method == "POST": r.add_header("Content-Type", "application/json")
with urllib.request.build_opener(urllib.request.ProxyHandler({})).open(r, timeout=8) as resp:
return resp.status
except urllib.error.HTTPError as e:
return e.code
except Exception:
return 0
print("health", code("/health"))
print("ollama-chat-path-exists", 0 if code("/ollama/api/ps") != 200 else 200)
for p, m in [("/ollama/api/delete", "DELETE"), ("/ollama/api/pull", "POST"),
("/ollama/api/create", "POST"), ("/ollama/api/tags", "GET"),
("/ollama/../api/delete", "DELETE"),
# These two were a real location once, with an Open WebUI
# ADMIN key injected here as a header. That design is gone -
# live progress is collected by a filter inside Open WebUI -
# and these assert that it stays gone.
("/ui/00000000-0000-0000-0000-000000000000/00000000-0000-0000-0000-000000000000/event", "POST"),
("/api/v1/chats/a/messages/b/event", "POST")]:
print("refuse:" + p, code(p, m))
PY
)
while read -r name value; do
[ -z "$name" ] && continue
case "$name" in
health) [ "$value" = "200" ] && ok "/health answers" || bad "/health returned $value" ;;
ollama-chat-path-exists)
[ "$value" = "200" ] && ok "the permitted Ollama path works (positive control)" \
|| bad "the permitted Ollama path returned $value" ;;
refuse:*)
p=${name#refuse:}
[ "$value" = "404" ] && ok "refused $p" || bad "$p returned $value, expected 404" ;;
esac
done <<< "$paths"
echo
echo "== 3. no Open WebUI credential exists anywhere in this stack"
# Not just absent from the agent: absent from the gateway too, which is
# where it used to live. The whole point of moving live progress into a
# filter was that no such key has to exist.
gw_env=$(docker exec owuitools-vmgw env 2>/dev/null || true)
if printf '%s' "$gw_env" | grep -q '^OWUITOOLS_OWUI_API_KEY=.'; then
bad "owuitools-vmgw still holds an Open WebUI API key"
else
ok "owuitools-vmgw holds no Open WebUI API key"
fi
# A proxy_pass, not a mention: the config's own header explains what used
# to be here, and matching that comment reported the opposite of the truth.
if docker exec owuitools-vmgw grep -rqE '^[^#]*proxy_pass.*chats' /etc/nginx/conf.d/ 2>/dev/null; then
bad "owuitools-vmgw still proxies the Open WebUI event endpoint"
else
ok "owuitools-vmgw does not proxy the Open WebUI event endpoint"
fi
env_dump=$(docker exec "$AGENT" env 2>/dev/null || true)
for secret in OWUITOOLS_OWUI_API_KEY GITEA_TOKEN OWUITOOLS_AUTH_TOKEN; do
if printf '%s' "$env_dump" | grep -q "^${secret}=."; then
bad "$secret is in the agent's environment"
else
ok "$secret is not in the agent's environment"
fi
done
# vmd's config holds the Gitea token and is 0640 root:owuivmd.
if docker exec "$AGENT" test -r /etc/owuivm/vmd.env 2>/dev/null; then
bad "the agent can read vmd's credential file"
else
ok "the agent cannot read vmd's credential file"
fi
echo
if [ "$fails" -gt 0 ]; then echo "FAILED $fails"; exit 1; fi
echo "the agent container is confined as designed"
tests/vm_host_asserts.sh
Executable
+330
View File
@@ -0,0 +1,330 @@
#!/bin/bash
# =====================================================================
# Structural assertions about the VM sandbox's host side.
#
# THE POINT OF THIS FILE
#
# The guest cannot reach this host or the LAN, and that claim rests on
# two controls which are deliberately independent of each other:
#
# 1. IPAddressDeny= in owuivm.service - a BPF filter on the unit's
# cgroup. Survives `nft flush ruleset`, a ufw reload and an
# nftables package upgrade.
# 2. the `inet owuivm_guard` nft table - the host's usual convention,
# and the thing an operator will actually think to look at.
#
# Checking that both are loaded proves nothing useful, because in normal
# operation either one alone would hide a broken other. So this script
# turns each OFF in turn and re-runs the same probe. Both runs must pass.
#
# This is not theoretical. The first version of owuivm.service carried
# `IPAddressAllow=0.0.0.0/0` alongside the deny list, on the assumption
# that longest-prefix would let the denies win. It does not - the allow
# list is checked first and grants unconditionally - so layer 1 was
# doing nothing at all, and every check passed anyway because layer 2
# was covering for it. This script is what found that.
#
# sudo ./tests/vm_host_asserts.sh
# =====================================================================
set -uo pipefail
UNIT=/etc/systemd/system/owuivm.service
PROBE=/usr/local/libexec/owuivm-probe.py
DENY='127.0.0.0/8 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 100.64.0.0/10 192.0.0.0/24 224.0.0.0/4 240.0.0.0/4'
# Must be unreachable from the sandbox uid. These are this host's own LAN
# address, the LAN gateway, Ollama on the docker bridge, a host service on
# loopback, and the owuitools gateway.
MUST_FAIL="192.168.1.10:7200 192.168.1.1:80 172.17.0.1:11434 127.0.0.1:3000 172.22.0.10:8000"
# Must be reachable, or the suite proves only that networking is broken.
MUST_PASS="1.1.1.1:443 8.8.8.8:53"
fails=0
ok() { printf ' ok %s\n' "$*"; }
bad() { printf ' FAIL %s\n' "$*"; fails=$((fails+1)); }
[ "$(id -u)" -eq 0 ] || { echo "run as root" >&2; exit 1; }
# However this script exits, the guard goes back - and so does any base
# image this suite unfroze. An image left writable is not dangerous (vmd
# refuses to open a session on one) but it stops the agent working, and a
# test suite must not be able to do that to the machine it ran on.
restore() {
systemctl restart owuivm-nft-guard.service >/dev/null 2>&1
for img in /var/lib/owuivm/base/*.qcow2; do
[ -e "$img" ] || continue
systemctl is-active --quiet "owuivm-maint@$(basename "$img" .qcow2).service" && continue
chmod 0444 "$img" 2>/dev/null
done
}
trap restore EXIT INT TERM
install -o root -g root -m 0755 "$(dirname "$(readlink -f "$0")")/vm_probe.py" "$PROBE"
probe() { # probe <systemd-run properties...> ; prints "target verdict"
systemd-run --quiet --wait --pipe --collect --uid=owuivm "$@" \
/usr/bin/python3 "$PROBE" $MUST_FAIL $MUST_PASS 2>/dev/null
}
check_probe() { # check_probe <label> <properties...>
local label="$1"; shift
local out; out="$(probe "$@")"
local t v
for t in $MUST_FAIL; do
v=$(awk -v t="$t" '$1==t{print $2}' <<<"$out")
[ "$v" = "REACHED" ] && bad "$label: $t was REACHABLE" || ok "$label: $t blocked"
done
for t in $MUST_PASS; do
v=$(awk -v t="$t" '$1==t{print $2}' <<<"$out")
[ "$v" = "REACHED" ] && ok "$label: $t reachable (positive control)" \
|| bad "$label: $t unreachable - the probe proves nothing"
done
}
echo "== 1. the unit says what it must, and does not say what it must not"
grep -q '^IPAddressDeny=' "$UNIT" && ok "IPAddressDeny is present" || bad "IPAddressDeny missing"
grep -q '^IPAddressAllow=' "$UNIT" \
&& bad "IPAddressAllow is present - it is checked FIRST and grants unconditionally, which kills the deny list" \
|| ok "no IPAddressAllow (deny-only, as required)"
for k in 'RestrictAddressFamilies=AF_UNIX AF_INET' 'NoNewPrivileges=yes' \
'ProtectSystem=strict' 'DevicePolicy=closed' 'SocketBindDeny=any' 'User=owuivm'; do
grep -qF "$k" "$UNIT" && ok "$k" || bad "missing: $k"
done
grep -q '^PrivateDevices=' "$UNIT" && bad "PrivateDevices would hide /dev/kvm" || ok "no PrivateDevices"
echo
echo "== 2. LAYER 1 ALONE: nft guard deleted, cgroup filter only"
nft delete table inet owuivm_guard >/dev/null 2>&1
nft list table inet owuivm_guard >/dev/null 2>&1 \
&& { bad "could not delete the guard table - layer 1 is not being tested alone"; } \
|| ok "guard table removed"
check_probe "L1" --property="IPAddressDeny=$DENY"
echo
echo "== 3. LAYER 2 ALONE: guard table restored, no cgroup filter"
systemctl restart owuivm-nft-guard.service
nft list table inet owuivm_guard >/dev/null 2>&1 && ok "guard table restored" || bad "guard table did not come back"
check_probe "L2"
echo
echo "== 4. the guard table is the one we shipped"
nft list table inet owuivm_guard | grep -q 'meta skuid' && ok "skuid rule present" || bad "no skuid rule"
nft list table inet owuivm_guard | grep -q 'nfproto ipv6' && ok "ipv6 drop present" || bad "no ipv6 drop"
echo
echo "== 5. the agent cannot unfreeze a base image, by two independent means"
# The operator asked for this specifically: the agent must have to ASK,
# and asking must terminate at a person. Both halves are checked, because
# either one alone could be quietly removed and everything would still
# appear to work.
# A base image being built is legitimately writable and owned by owuivm -
# that is what building one looks like - so say so rather than reporting
# a security failure that is actually a build in progress.
# An image is legitimately writable while it is being built OR while the
# operator has it open for maintenance - that is what maintenance IS.
# Reporting either as a security failure trains people to ignore this
# suite, which is worse than not having it.
building=""
pgrep -f 'images/ubuntu/build.sh|images/windows/build.sh' >/dev/null 2>&1 && building="image build"
for img in ubuntu windows; do
systemctl is-active --quiet "owuivm-maint@$img.service" 2>/dev/null && building="maintenance boot"
done
for img in /var/lib/owuivm/base/*.qcow2; do
[ -e "$img" ] || continue
perms=$(stat -c '%a %U' "$img")
case "$perms" in
"444 owuivmadm") ok "$(basename "$img") is frozen (0444 owuivmadm)" ;;
*) if [ -n "$building" ]; then
printf ' skip %s is %s - a %s is in progress; re-run when it finishes\n' \
"$(basename "$img")" "$perms" "$building"
else
bad "$(basename "$img") is $perms - a base image must be 0444 owned by owuivmadm"
fi ;;
esac
done
# owuivm is the uid every session runs as. It must not be able to make a
# base writable even if it tried.
if sudo -u owuivm test -w /var/lib/owuivm/base 2>/dev/null; then
bad "the sandbox uid can write the base image directory"
else
ok "the sandbox uid cannot write the base image directory"
fi
# vmd is the most privileged thing the agent can reach. Its wrapper must
# have no verb for this at all - not a verb that checks something.
#
# Output is captured into a variable and matched afterwards, NOT piped
# into grep. This script runs under `set -o pipefail`, and both commands
# below are MEANT to exit non-zero - so `cmd | grep -q` fails the whole
# pipeline even when grep matched perfectly, and the test reports the
# opposite of what it found. It did exactly that on the first run.
out=$(sudo -u owuivmd sudo -n /usr/local/sbin/owuivm-ctl maint-start ubuntu 2>&1 || true)
case "$out" in
*"unknown verb"*) ok "vmd's privileged helper has no unfreeze verb" ;;
*) bad "vmd's privileged helper answered an unfreeze request: $out" ;;
esac
# polkit gates the only unit that can, on a session of the operator.
#
# THE STATE IS SAMPLED BEFORE AS WELL AS AFTER, and that is not
# defensive style - the first version of this check was not, and it
# STOPPED AN INSTALL THAT WAS ALREADY RUNNING. It read `is-active` after
# its own `start`, saw "active" because a maintenance boot had been up
# for half an hour, concluded a non-local session had started it, and
# "cleaned up" by stopping it. A test that cannot tell "I caused this"
# from "this was already true" does not just report wrongly; it acts.
before=$(systemctl is-active owuivm-maint@ubuntu.service 2>/dev/null || true)
if [ "$before" = "active" ]; then
printf ' skip a maintenance boot is already running - not touching it\n'
else
out=$(sudo -u owuivmd systemctl start owuivm-maint@ubuntu.service 2>&1 || true)
after=$(systemctl is-active owuivm-maint@ubuntu.service 2>/dev/null || true)
if [ "$after" = "active" ]; then
bad "a non-local session STARTED the maintenance unit"
systemctl stop owuivm-maint@ubuntu.service >/dev/null 2>&1
elif printf '%s' "$out" | grep -qi "access denied"; then
ok "polkit refuses the maintenance unit to a non-local session"
else
bad "the maintenance unit did not start, but not because polkit refused: $out"
fi
fi
echo
echo "== 5a. an unfrozen image stops a session, rather than being handed to one"
# WHY THIS EXISTS: the freeze used to be put back automatically when a
# maintenance boot ended, which meant two things nobody wanted - looking
# at an image required making it writable first, and the guarantee rested
# on a cleanup step that has already been seen to fail silently.
#
# Now the operator freezes and unfreezes deliberately, and this is the
# check that makes forgetting harmless: vmd will not open a session on a
# writable base. It is the only thing standing where the auto-freeze
# used to, so it is asserted here rather than assumed.
if systemctl is-active --quiet owuivm.service || \
systemctl is-active --quiet 'owuivm-maint@ubuntu.service'; then
printf ' skip a machine is running; not touching the ubuntu image\n'
elif [ ! -f /var/lib/owuivm/base/ubuntu.qcow2 ]; then
printf ' skip no ubuntu image yet\n'
else
chmod 0660 /var/lib/owuivm/base/ubuntu.qcow2
refusal=$(sudo -u owuivmd /usr/local/bin/vmctl open --os ubuntu --project smoke 2>&1 | head -2)
case "$refusal" in
*"unfrozen, so no session can start"*)
ok "a session refuses to start on a writable base, and says why" ;;
*) bad "a writable base did not produce the refusal - got: $refusal" ;;
esac
# And it must refuse BEFORE anything exists to clean up.
[ -z "$(ls /Pool/vmrun/*.qcow2 2>/dev/null)" ] \
&& ok "and it refused before creating anything" \
|| bad "an overlay was created for a session that could not run"
pgrep -u owuivm qemu-system-x86_64 >/dev/null 2>&1 \
&& bad "a qemu process was started for a refused session" \
|| ok "and no machine was started"
chmod 0444 /var/lib/owuivm/base/ubuntu.qcow2
ok "the image is frozen again ($(stat -c %a /var/lib/owuivm/base/ubuntu.qcow2))"
fi
echo
echo "== 5b. the maintenance console is not a TCP port"
# It was one, and it did not work: this unit denies 127.0.0.0/8, and
# systemd's filter checks the SOURCE address on ingress as well as the
# destination on egress - so the rule that stops the guest reaching the
# host also silently dropped the operator's own console. The fix was a
# unix socket, which is not IP, and this asserts it stays that way.
if grep -q 'vnc "unix:' /usr/local/libexec/owuivm-launch; then
ok "the console is served on a unix socket"
else
bad "the console is not on a unix socket - a loopback TCP port is dropped by this unit's own IP filter"
fi
if grep -q '^SocketBindAllow=' /etc/systemd/system/owuivm-maint@.service; then
bad "the maintenance unit still permits a listening TCP port"
else
ok "the maintenance unit binds no TCP port at all"
fi
echo
echo "== 5c. the VM keeps its I/O out of the page cache"
# MEASURED: 10G made a machine crawl. The guest's RAM is a memfd with
# share=on, so all 8 GiB of it is accounted as shmem in the unit's
# cgroup - and with no swap on this host, shmem cannot be reclaimed at
# all. That left ~1.5 GiB for every page of ISO and disk I/O, and the
# cgroup hit its hard limit 6,782 times in a single boot, evicting and
# re-reading the same installer media over and over.
#
# The fix is cache=none on every drive, NOT a bigger limit - raising it
# to 14G fixed the VM and left the host short instead. So the drives are
# what this checks first; the limit is the backstop behind them.
for f in /usr/local/libexec/owuivm-launch; do
n=$(grep -c 'cache=none' "$f")
[ "$n" -ge 3 ] && ok "every drive is O_DIRECT ($n of them)" \
|| bad "only $n drive(s) use cache=none - VM I/O will be charged to the cgroup as page cache"
done
for unit in /etc/systemd/system/owuivm.service /etc/systemd/system/owuivm-maint@.service; do
want=$(grep -oP '^MemoryMax=\K.*' "$unit" | tail -1)
mem=$(grep -oP '^Environment=OWUIVM_MEM_MB=\K[0-9]+' "$unit" | tail -1)
mem=${mem:-8192}
case "$want" in
1[1-9]G|[2-9][0-9]G) ok "$(basename "$unit"): MemoryMax=$want leaves room above ${mem}M of guest RAM" ;;
*) bad "$(basename "$unit"): MemoryMax=$want is too close to ${mem}M of unreclaimable shmem" ;;
esac
done
# THE TWO MACHINES END DIFFERENTLY, AND EACH WAY IS DELIBERATE.
#
# Maintenance holds the base image open, so pulling the plug on it is how
# a qcow2 gets corrupted - it presses the power button and waits. A
# session holds a throwaway overlay that is deleted moments later, and
# its project share has already been committed and pushed by the time the
# unit stops, so an ACPI power-down would buy a tidy-looking exit for up
# to ninety seconds of CPU per session. It is killed instead.
if systemctl show owuivm-maint@ubuntu.service -p ExecStop --value 2>/dev/null \
| grep -q owuivm-powerdown; then
ok "maintenance presses the power button and waits for the guest"
else
bad "the maintenance unit yanks power on a base image it has open for writing"
fi
if systemctl show owuivm.service -p ExecStop --value 2>/dev/null | grep -q owuivm-powerdown; then
bad "a session waits for an ACPI shutdown it has no reason to wait for"
else
ok "a session machine is killed rather than asked to leave"
fi
case "$(systemctl show owuivm.service -p KillSignal --value 2>/dev/null)" in
9) ok "and the kill is SIGKILL, not a request qemu could take its time over" ;;
*) bad "the session unit's KillSignal is not SIGKILL" ;;
esac
case "$(systemctl show owuivm.service -p TimeoutStopUSec --value 2>/dev/null)" in
[1-9]s|1[0-5]s) ok "with a short backstop ($(systemctl show owuivm.service -p TimeoutStopUSec --value))" ;;
*) bad "the session stop timeout is $(systemctl show owuivm.service -p TimeoutStopUSec --value), which is a wait nobody needs" ;;
esac
echo
echo "== 5d. the sandbox resolves names without touching the host's resolver"
# SLIRP's built-in forwarder relays to the HOST's /etc/resolv.conf, which
# is systemd-resolved on 127.0.0.53 - an address these units deny, and a
# service that knows the LAN's internal names. Without a resolver of its
# own, a guest that takes DHCP gets working TCP and no name resolution,
# which presents as "no internet" and is looked for in the wrong place.
[ -f /etc/owuivm/resolv.conf ] && ok "the sandbox has its own resolv.conf" \
|| bad "/etc/owuivm/resolv.conf is missing"
if grep -qE '^nameserver 127\.' /etc/owuivm/resolv.conf 2>/dev/null; then
bad "the sandbox's resolver points at loopback - which this unit denies"
else
ok "it points somewhere the sandbox can actually reach"
fi
for unit in /etc/systemd/system/owuivm.service /etc/systemd/system/owuivm-maint@.service; do
if grep -q 'BindReadOnlyPaths=/etc/owuivm/resolv.conf:/etc/resolv.conf' "$unit"; then
ok "$(basename "$unit"): uses it instead of the host's"
else
bad "$(basename "$unit"): SLIRP will forward DNS to the host's resolver, which is denied"
fi
done
echo
echo "== 6. no session left running"
if pgrep -u owuivm -f qemu-system-x86_64 >/dev/null; then
if [ -n "$building" ]; then printf ' skip a qemu is running - that is the %s\n' "$building"
else bad "a qemu process is still running as owuivm"; fi
else ok "no stray qemu"; fi
ls /Pool/vmrun/*.qcow2 >/dev/null 2>&1 && bad "an overlay was left behind in /Pool/vmrun" || ok "no stray overlay"
echo
if [ "$fails" -gt 0 ]; then echo "FAILED: $fails"; exit 1; fi
echo "all host assertions pass"
tests/vm_negative_guest.sh
Executable
+81
View File
@@ -0,0 +1,81 @@
#!/bin/bash
# =====================================================================
# Runs INSIDE the sandbox VM. Proves the guest cannot reach this host or
# the LAN, and - just as important - that it CAN reach the internet.
#
# WHY THE POSITIVE CONTROLS ARE NOT OPTIONAL
#
# A suite made only of "this must fail" passes perfectly on a machine
# with no network at all, on a machine that is switched off, and on a
# machine where the test harness is broken. Every must-fail target here
# is paired with a must-succeed one, so a pass means "the filter works"
# rather than "nothing works".
#
# ping IS EXPECTED TO FAIL, AND THAT IS NOT THE FILTER
#
# net.ipv4.ping_group_range on the host is "1 0", so no group may open an
# unprivileged ICMP socket and libslirp cannot forward one. ping fails
# here permanently, on a machine whose HTTPS works perfectly. It is
# asserted as a known-fail rather than left to be rediscovered as a
# mystery by everyone who ever debugs this.
# =====================================================================
fails=0
pass() { printf ' ok %s\n' "$*"; }
fail() { printf ' FAIL %s\n' "$*"; fails=$((fails+1)); }
# A TCP connect with a short timeout, using bash's own /dev/tcp so the
# result does not depend on nc, curl or python being installed.
reach() {
timeout 4 bash -c "exec 3<>/dev/tcp/$1/$2" 2>/dev/null
}
echo "== must be UNREACHABLE: the host, the LAN, and the stack =="
# 10.0.2.2 and 10.0.2.3 are SLIRP's aliases for the host's loopback and
# the host's DNS forwarder. They are the closest thing to a back door
# this design has, and the deny on 127.0.0.0/8 is what closes them.
for target in \
"10.0.2.2:22|SLIRP alias for the host's loopback (sshd)" \
"10.0.2.2:3000|SLIRP alias for the host's loopback (Open WebUI)" \
"192.168.1.10:22|this host on the LAN (sshd)" \
"192.168.1.10:3000|this host on the LAN (Open WebUI)" \
"192.168.1.10:7200|this host on the LAN (Gitea)" \
"192.168.1.10:9443|this host on the LAN (Portainer)" \
"192.168.1.1:80|the LAN gateway" \
"172.17.0.1:11434|Ollama on the docker bridge" \
"172.22.0.10:8000|the owuitools gateway" \
"169.254.169.254:80|cloud metadata" ; do
addr="${target%%|*}"; what="${target#*|}"
host="${addr%:*}"; port="${addr##*:}"
if reach "$host" "$port"; then fail "REACHED $addr - $what"; else pass "blocked $addr ($what)"; fi
done
echo
echo "== must be REACHABLE: the internet (or this suite proves nothing) =="
for target in "1.1.1.1:443|Cloudflare DNS over TLS" "8.8.8.8:53|Google DNS" "1.1.1.1:80|a plain HTTP port"; do
addr="${target%%|*}"; what="${target#*|}"
host="${addr%:*}"; port="${addr##*:}"
if reach "$host" "$port"; then pass "reachable $addr ($what)"; else fail "UNREACHABLE $addr - $what: the positive control failed, so the negatives above mean nothing"; fi
done
echo
echo "== name resolution and a real fetch =="
if getent hosts archive.ubuntu.com >/dev/null 2>&1; then pass "DNS resolves (static 1.1.1.1, not SLIRP's forwarder)"; else fail "DNS does not resolve"; fi
if timeout 20 curl -fsS -o /dev/null https://example.com/ 2>/dev/null; then pass "HTTPS to a public host works"; else fail "HTTPS to a public host failed"; fi
echo
echo "== known-fail, documented: ICMP =="
if timeout 5 ping -c1 -W2 8.8.8.8 >/dev/null 2>&1; then
fail "ping WORKED - ping_group_range must have changed; update the docs and check_internet's note"
else
pass "ping fails as expected (host ping_group_range is '1 0'; use curl instead)"
fi
echo
echo "== no IPv6, and no neighbours on the LAN =="
if ip -6 addr show scope global 2>/dev/null | grep -q inet6; then fail "the guest has a global IPv6 address"; else pass "no global IPv6"; fi
neighbours=$(ip neigh show 2>/dev/null | grep -cvE '^\s*$|10\.0\.2\.' || true)
if [ "${neighbours:-0}" -gt 0 ]; then fail "the guest can see $neighbours neighbour(s) outside its own SLIRP subnet"; else pass "no neighbours outside 10.0.2.0/24"; fi
echo
if [ "$fails" -gt 0 ]; then echo "FAILED $fails"; exit 1; fi
echo "the guest is isolated as designed"
tests/vm_probe.py
Executable
+24
View File
@@ -0,0 +1,24 @@
#!/usr/bin/env python3
"""Connectivity probe, run as the sandbox uid by tests/vm_host_asserts.sh.
Prints one `target VERDICT` line per argument. Deliberately dumb: it makes
a TCP connection and reports whether it completed. A blocked address looks
the same as an unreachable one from in here, which is why the caller pairs
every must-fail target with a must-pass positive control - without those,
a suite like this passes just as happily when networking is simply broken.
"""
import socket
import sys
def probe(host: str, port: int, timeout: float = 4.0) -> str:
try:
socket.create_connection((host, port), timeout).close()
return "REACHED"
except Exception as exc:
return f"blocked({type(exc).__name__})"
for spec in sys.argv[1:]:
host, _, port = spec.rpartition(":")
print(f"{spec} {probe(host, int(port))}", flush=True)