Grant/owui-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial Commit

Browse Source
This commit is contained in:
Grant
2026-09-15 15:24:03 -04:00
commit 4dc7086e7e
605 changed files with 101380 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/smoke.py
+569
View File
@@ -0,0 +1,569 @@
"""Exercise every owuitools endpoint the way Open WebUI would.
Runs on openwebui_default and talks to the gateway by name, so it takes the
same path a real tool call does - through nginx, into the guarded
namespace - rather than testing the app in isolation where the firewall,
the proxy and the auth header would all be bypassed.
"""
import time
import json
import os
import sys
import textwrap
import httpx
BASE = os.environ.get("OWUITOOLS_BASE", "http://owuitools-gw:8000")
TOKEN = os.environ["OWUITOOLS_AUTH_TOKEN"]
MAX_CHARS = int(os.environ.get("OWUITOOLS_MAX_RESPONSE_CHARS", "12000"))
VERBOSE = "-v" in sys.argv
# (tool, payload, must_appear_in_response_or_None[, must_NOT_appear])
# The fourth element is optional and may be one string or a tuple of them.
# Several behaviours here are only testable as an absence - an NXDOMAIN that
# must not be reported as a policy denial, a corrected hint that must not
# still carry the wrong claim - and a missing negative reads as a pass.
CASES = [
# --- offline / deterministic: these must always pass -----------------
("subnet_calc", {"network": "192.168.1.0/24"}, "192.168.1.255"),
("subnet_calc", {"network": "10.0.0.0/8", "split_into": 4}, "10.64.0.0/10"),
("subnet_calc", {"network": "2001:db8::/32"}, "2001:db8::"),
("network_reference", {"lookup_type": "port", "value": "443"}, "https"),
("network_reference", {"lookup_type": "http_status", "value": "418"}, "teapot"),
("network_reference", {"lookup_type": "mac_vendor", "value": "00:1A:2B:3C:4D:5E"}, None),
("ohms_law_power", {"voltage": "12", "resistance": "220"}, "54.55 mA"),
("ohms_law_power", {"power": "0.5", "resistance": "100"}, None),
("component_calc", {"calculation": "resistor_color_code", "text": "brown black red gold"}, "1 kΩ"),
("component_calc", {"calculation": "resistor_to_color", "value": "4k7"}, "yellow"),
("component_calc", {"calculation": "nearest_e_series", "value": "3300", "series": "E24"}, None),
("component_calc", {"calculation": "rc_filter", "resistance": "10k", "capacitance": "100n"}, "159.2 Hz"),
("component_calc", {"calculation": "lc_resonance", "inductance": "10u", "capacitance": "100n"}, None),
("component_calc", {"calculation": "voltage_divider", "voltage": "12", "resistance": "10k", "resistance2": "2k2"}, None),
("component_calc", {"calculation": "series_parallel", "text": "10k, 10k", "combination": "parallel"}, "5 kΩ"),
("pcb_calc", {"calculation": "trace_width", "current": "2"}, "mm"),
("pcb_calc", {"calculation": "microstrip_impedance", "trace_width": "0.3", "dielectric_height": "0.2"}, "Ω"),
("pcb_calc", {"calculation": "via_current", "via_diameter": "0.3", "current": "1"}, None),
("wiring_calc", {"calculation": "awg_properties", "awg": 14}, "mm²"),
("wiring_calc", {"calculation": "voltage_drop", "awg": 12, "length": "30", "current": "10", "voltage": "12"}, "%"),
("wiring_calc", {"calculation": "led_resistor", "voltage": "5", "led_forward_voltage": "2.1", "led_current": "0.02"}, "145 Ω"),
("wiring_calc", {"calculation": "decibels", "value": "30", "db_from": "dBm"}, "1 W"),
("wiring_calc", {"calculation": "battery_life", "capacity_mah": 2000, "load_ma": 150}, "hours"),
# --- MATLAB sandbox -------------------------------------------------
# These assert the three verdicts the tool exists to distinguish:
# it worked, it is not valid MATLAB, and it failed for a reason that
# is not the model's fault.
("matlab_run", {"code": "x = linspace(0,1,5);\ndisp(sum(x))"}, "MATLAB run OK"),
("matlab_run", {"code": "a=[1 2 3];\nb=[1 2];\nc=a*b;"}, "nonconformant"),
("matlab_run", {"code": "a=[1 2 3];\nb=[1 2];\nc=a*b;"}, "main.m line 3"),
("matlab_run", {"code": "for i=1:2\n printf('%d',i);\nendfor"}, "NOT VALID MATLAB"),
# Was: "contains is real MATLAB but not available here". Change 58 made
# it available, so what is pinned now is that it runs and that nothing
# tells the model to rewrite it.
("matlab_run", {"code": "s = contains('abc','b');\ndisp(s)"}, "1", "not available"),
("matlab_run", {"code": "while true; end", "timeout_seconds": 3}, "Timed out"),
("matlab_run", {"code": "function r=f(a)\n r=a;\nend"}, "function definition, not a script"),
("matlab_run", {"code": "disp(addup(2,3))",
"helper_files": [{"name": "addup.m", "code": "function r=addup(a,b)\n r=a+b;\nend"}]}, "5"),
("matlab_run", {"code": "x = webread('http://example.com');"}, "no network"),
("matlab_run", {"code": ""}, "CANNOT DO THAT"),
# Regressions found by adversarial review; each of these was broken.
("matlab_run", {"code": "x=1;\n# double it\nx=x*2;\ndisp(x)"}, "NOT VALID MATLAB"),
("matlab_run", {"code": "plot_style = 'x';\ns = sprintf('a', ...\n '--r');\ndisp(1)"}, "MATLAB run OK"),
("matlab_run", {"code": "A=[1 1;1 1];\nx=A\\[1;2];\ndisp('done')"}, "Warnings"),
("matlab_run", {"code": "a = zeros(60000,60000);", "timeout_seconds": 30}, "out of memory"),
("matlab_run", {"code": "disp(addup(2,3))",
"helper_files": [{"name": "2bad.m", "code": "function r=addup(a,b)\n r=a+b;\nend"}]},
"Helper files ignored"),
# --- network-dependent ----------------------------------------------
("web_search", {"query": "RFC 9293 TCP specification", "max_results": 4}, None),
("web_search", {"query": "systemd-resolved DNSSEC failure", "category": "it", "max_results": 3}, None),
("fetch_page", {"url": "https://example.com", "max_chars": 1500}, "Example Domain"),
("wikipedia_lookup", {"topic": "OSI model"}, "layer"),
("search_stackexchange", {"query": "bash trap EXIT", "site": "unix", "max_results": 2}, None),
("lookup_rfc", {"number": 9293, "offset": 0}, "RFC 9293"),
("lookup_rfc", {"query": "border gateway protocol"}, "Proposed Standard"),
("lookup_cve", {"cve_id": "CVE-2021-44228"}, "Log4j"),
("read_man_page", {"name": "ip", "section": 8}, "NAME"),
("lookup_package", {"name": "requests", "ecosystem": "pypi"}, "Latest version"),
("lookup_package", {"name": "torvalds/linux", "ecosystem": "github"}, "tags"),
("dns_lookup", {"name": "cloudflare.com", "record_type": "A"}, "TTL"),
("dns_lookup", {"name": "google.com", "record_type": "MX"}, None),
("inspect_tls", {"hostname": "example.com"}, "Issuer"),
("http_headers", {"url": "https://example.com"}, "Security headers"),
("lookup_ip_or_asn", {"target": "8.8.8.8"}, None),
("find_datasheet", {"part_number": "LM358"}, None),
("search_papers", {"query": "GaN transistor switching losses", "max_results": 2}, None),
# --- guardrails: these must be REFUSED, not fulfilled ----------------
("fetch_page", {"url": "http://192.168.1.10:9443/"}, "CANNOT DO THAT"),
("fetch_page", {"url": "http://172.19.0.2:8080/api/config"}, "CANNOT DO THAT"),
("fetch_page", {"url": "file:///etc/passwd"}, "CANNOT DO THAT"),
("fetch_page", {"url": "http://localhost:8000/health"}, "CANNOT DO THAT"),
("dns_lookup", {"name": "router.local", "record_type": "A"}, "CANNOT DO THAT"),
("lookup_ip_or_asn", {"target": "192.168.1.1"}, "CANNOT DO THAT"),
("ohms_law_power", {"voltage": "12"}, "CANNOT DO THAT"),
("component_calc", {"calculation": "rc_filter", "resistance": "10k"}, "CANNOT DO THAT"),
# Models pass "" for optionals they chose not to use; must not 422.
("web_search", {"query": "test", "site": "", "language": "", "max_results": 3}, None),
("lookup_rfc", {"number": 1918, "query": ""}, "1918"),
# --- citation metadata: a page has to say what it is -----------------
# The URLs are stable documents chosen for their metadata, not their
# ranking: a paper of record on PLOS and the arXiv abstract of
# "Attention Is All You Need". Two cases per page where two separate
# fields are being asserted, the same way the MATLAB cases above do.
("fetch_page", {"url": "https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0173152",
"max_chars": 1200}, "CITE AS: Ravenscroft"),
("fetch_page", {"url": "https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0173152",
"max_chars": 1200}, "Journal: PLOS ONE"),
("fetch_page", {"url": "https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0173152",
"max_chars": 1200}, "DOI: 10.1371"),
# Comma-formatted authors are "Vaswani, Ashish"; the surname is the cite.
("fetch_page", {"url": "https://arxiv.org/abs/1706.03762", "max_chars": 900},
"CITE AS: Vaswani", "CITE AS: Ashish"),
# What arrived is not always what citing it would imply.
("fetch_page", {"url": "https://arxiv.org/abs/1706.03762", "max_chars": 700}, "abstract page"),
("fetch_page", {"url": "https://pubmed.ncbi.nlm.nih.gov/32939066/", "max_chars": 900},
"not the paper"),
("fetch_page", {"url": "https://example.com", "max_chars": 800},
"No author, journal or DOI metadata"),
# --- reading inside a document ---------------------------------------
("fetch_page", {"url": "https://www.rfc-editor.org/rfc/rfc9293.txt",
"search": "Retransmission Timeout", "max_chars": 1200}, "Jumped to"),
("fetch_page", {"url": "https://example.com", "search": "zzzqqqnotpresent"},
"does not occur in this document"),
# The continuation hint pushes on; it must not read as permission to stop.
("fetch_page", {"url": "https://www.rfc-editor.org/rfc/rfc9293.txt", "max_chars": 1000},
"Keep reading if the answer may be further in", "Only do so if this section did not answer"),
("lookup_rfc", {"number": 1035, "search": "2.3.4"}, "skipped"),
# --- stub and failure honesty: a non-answer must not look like one ---
("fetch_page", {"url": "https://www.cisa.gov/known-exploited-vulnerabilities-catalog"},
"client-rendered or gated"),
# A name that does not exist is a DNS fact, not a firewall refusal; saying
# "private network" there sends the model looking for a way around it.
("fetch_page", {"url": "https://api.no-such-host-owuitools.invalid/x"},
"does not resolve", "private network"),
("inspect_tls", {"hostname": "no-such-host-owuitools.invalid"},
"does not resolve", "private network"),
# --- search relevance and category honesty ---------------------------
# An invented title returns something from every engine; the results are
# not about the query, and saying so is the whole point.
("web_search", {"query": '"Ferrous lattice resonance in dodecahedral copper" Whitlock Ambersley',
"max_results": 5}, "LOW RELEVANCE"),
("web_search", {"query": "RFC 9293 TCP retransmission timeout computation", "max_results": 4},
"Snippets are search-engine excerpts", "LOW RELEVANCE"),
("web_search", {"query": "amoxicillin adult dose sinusitis mg how much should i take",
"max_results": 4}, "dailymed"),
("web_search", {"query": "ibuprofen maximum daily dose mg adult", "category": "science",
"max_results": 4}, "scholarly indexes only"),
# Structural, not about which papers rank: some scholarly result carries
# enough metadata to cite. Only a dead scholarly index breaks this.
("web_search", {"query": "metformin lactic acidosis cohort study", "category": "science",
"max_results": 6}, "CITE AS:"),
# --- wikipedia resolution --------------------------------------------
("wikipedia_lookup", {"topic": "Mercury"}, "disambiguation page"),
("wikipedia_lookup", {"topic": "Marguerite Delacroix-Whitmore"}, "NO ARTICLE"),
# An exact title must win over whatever full-text search would have found.
("wikipedia_lookup", {"topic": "Canberra"}, "# Canberra"),
("wikipedia_lookup", {"topic": "Canberra", "full_article": True,
"search": "Federal Capital Territory"}, "Jumped to"),
("wikipedia_lookup", {"topic": "Eiffel Tower"}, "Introduction only"),
# --- v3: a citation the answer can lift ------------------------------
# Two graded rounds: 1 of 25 tool-assisted answers carried a URL, and the
# one that did lifted a ready-made CITE AS line. Every tool that returns a
# single identifiable document now ends with the line the answer wants.
("fetch_page", {"url": "https://example.com", "max_chars": 800},
"\n---\nSources: https://example.com"),
("wikipedia_lookup", {"topic": "Canberra"},
"\n---\nSources: https://en.wikipedia.org/wiki/Canberra"),
("lookup_rfc", {"number": 1035, "offset": 0}, "\n---\nSources: RFC 1035"),
("read_man_page", {"name": "ls"}, "\n---\nSources: man ls"),
("lookup_package", {"name": "requests", "ecosystem": "pypi"},
"\n---\nSources: https://"),
# A paper carries its citation as well as its URL.
("fetch_page", {"url": "https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0173152",
"max_chars": 1200}, "Sources: Ravenscroft"),
# Still there at the largest window the schema allows, where the page
# budget is tightest. (That `cap()` re-attaches it after trimming is a
# pure-function property; this asserts the room reservation upstream.)
("fetch_page", {"url": "https://www.rfc-editor.org/rfc/rfc9293.txt", "max_chars": 10000},
"\n---\nSources: https://www.rfc-editor.org/rfc/rfc9293.txt"),
# --- v3: a jump says which section it landed in -----------------------
# "Constitution" inside the Canberra article matched "1 Constitution
# Avenue" - a street - and the model read the window as having checked the
# constitutional basis for the capital. A character offset cannot say you
# are in the wrong subject; the enclosing heading can.
("wikipedia_lookup", {"topic": "Canberra", "full_article": True, "search": "Constitution"},
"inside the section"),
("lookup_rfc", {"number": 1035, "search": "63 octets"}, "inside the section"),
("read_man_page", {"name": "ls", "search": "--almost-all"}, "inside the section"),
# --- v3: a search miss does not throw the document away ---------------
# One search string missing is not the same finding as the document being
# useless; returning only "not here" made the two indistinguishable, and
# the model abandoned 54,000 characters of the right article.
("wikipedia_lookup", {"topic": "Sense", "full_article": True, "search": "zzzqqqnotpresent"},
"Showing the opening of the document instead"),
("lookup_rfc", {"number": 1035, "search": "zzzqqqnotpresent"},
"Showing the opening of the document instead"),
("read_man_page", {"name": "ls", "search": "zzzqqqnotpresent"},
"Showing the opening of the document instead"),
# --- v3: which of your terms is absent, as a count --------------------
# A model hunting a half-remembered paper puts its own guesses in the
# query; topically plausible results then read as partial confirmation.
("web_search", {"query": "Korhonen coffee randomized trial type 2 diabetes Circulation 2018",
"max_results": 6}, "Appears in none of these results"),
("web_search", {"query": "linux kernel scheduler documentation", "category": "it",
"max_results": 3}, None, "Appears in none of these results"),
# An index listing is not the document, and cannot be quoted as one.
("lookup_rfc", {"query": "border gateway protocol"}, "These are index entries"),
# --- per-tool additions ----------------------------------------------
("http_headers", {"url": "https://github.com"}, "max-age="),
# Splitting past /64 breaks SLAAC, which the split is silent about otherwise.
("subnet_calc", {"network": "2001:db8:1::/64", "split_into": 4}, "RFC 7421"),
# --- Change 55: the v3 round's tool defects ---------------------------
# A documentation address is named as such, with its RFC, and a public
# address carries its origin AS from the routing view, not just its
# registration.
("lookup_ip_or_asn", {"target": "203.0.113.10"}, "RFC 5737"),
("lookup_ip_or_asn", {"target": "8.8.8.8"}, "Origin AS"),
("subnet_calc", {"network": "203.0.113.0/24"}, "reserved for documentation"),
# Board dimensions with units, and no negative impedance ever printed.
("pcb_calc", {"calculation": "microstrip_impedance", "trace_width": "12mil", "dielectric_height": "8 mil"}, "0.305 mm"),
("pcb_calc", {"calculation": "microstrip_impedance", "trace_width": "0.3mm", "dielectric_height": "1.6mm"}, "Ω", "≈ -"),
("pcb_calc", {"calculation": "microstrip_impedance", "trace_width": "10", "dielectric_height": "0.1"}, "CANNOT DO THAT", "Ω**"),
# A search miss on an unsectioned name checks the sibling sections.
("read_man_page", {"name": "crontab", "search": "PATH"}, "section 5"),
# PMC and PubMed pages are served from the Europe PMC open-access API.
("fetch_page", {"url": "https://pubmed.ncbi.nlm.nih.gov/24459154/", "max_chars": 900}, "Retrieved via"),
("fetch_page", {"url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC3898757/", "max_chars": 900}, "Diabetes Care"),
# A PDF whose text layer is glyph codes is refused, not served as text.
("fetch_page", {"url": "https://www.infineon.com/assets/row/public/documents/24/49/infineon-irlz44n-datasheet-en.pdf",
"max_chars": 900}, "not decodable"),
# An oversized window is clamped; a bad parameter is prose, not a 422.
("fetch_page", {"url": "https://example.com", "max_chars": 10716}, "Example Domain", "422"),
("fetch_page", {"url": "https://example.com", "offset": -5}, "CANNOT DO THAT"),
# A page-numberless contents entry (RFC v3 text format) is stepped over.
("lookup_rfc", {"number": 9110, "search": "15.5.19"}, "skipped 1 table-of-contents"),
("lookup_rfc", {"number": 9110, "search": "15.5.19"}, "15.5. Client Error 4xx"),
# Parse-error and ordering hints.
("matlab_run", {"code": "x = 2;\nfprintf('today\\'s value: %d\\n', x);", "purpose": "scratch"}, "two quotes"),
# Was: the hint explaining that Octave cannot see a function defined
# below its call. Change 58 lifts those out instead, so the script the
# hint was written for now simply runs; the hint survives for the files
# the lift refuses, and is checked by tests/test_local_functions.py.
("matlab_run", {"code": "disp(addup(2,3))\nfunction r = addup(a,b)\n r = a+b;\nend", "purpose": "scratch"},
"5", "below the line that calls it"),
("subnet_calc", {"network": "192.168.1.0/24", "split_into": 4}, "192.168.1.192", "RFC 7421"),
("pcb_calc", {"calculation": "microstrip_width_for_impedance", "target_impedance": "50",
"dielectric_height": "0.2"}, "Width for"),
# IPC-2152 is not a licence to narrow an external trace, which is what the
# old wording invited.
("pcb_calc", {"calculation": "trace_width", "current": "3"},
"not a way to shrink an external trace", "usually less conservative standard if you need to squeeze"),
("read_man_page", {"name": "ip route"}, "ip-route", "No manual page found"),
# Both labels come off the vendor-pattern URL, not off search ranking:
# LM358DR strips to LM358, so the official PDF is a variant, not the part.
("find_datasheet", {"part_number": "LM358"}, "[EXACT LM358]"),
("find_datasheet", {"part_number": "LM358DR"}, "VARIANT: names 'LM358'"),
("lookup_cve", {"cve_id": "CVE-2024-6387"}, "openssh: before"),
("lookup_cve", {"cve_id": "CVE-2021-44228"}, "CISA KEV"),
("matlab_run", {"code": "n = numel(regexp('a b', '\\bb', 'match'));\ndisp(n)"}, "BACKSPACE"),
# purpose='scratch' is the model working for itself: give it the Octave
# equivalent and drop the MATLAB-validity verdict, which is not the job.
# `contains` is a shim now (Change 58) and runs, so the equivalent-advice
# path is tested with a name that is still missing.
("matlab_run", {"code": "s = split('a b');\ndisp(s)", "purpose": "scratch"},
"strsplit", "Do NOT rewrite"),
("matlab_run", {"code": "x=1;\n# double it\nx=x*2;\ndisp(x)", "purpose": "scratch"},
None, "NOT VALID MATLAB"),
# --- Change 56: the v4 round's easy wins ---------------------------------
# A cable length in feet is feet, not femtometres.
("wiring_calc", {"calculation": "voltage_drop", "awg": 12, "length": "100ft", "current": "20",
"voltage": "120"}, "30.48 m (100 ft)", ("fV", "1e-13")),
("wiring_calc", {"calculation": "voltage_drop", "awg": 12, "length": "100 ft", "current": "20",
"voltage": "120"}, "5.29%"),
# A bare "30m" is thirty metres, not thirty milli-anything.
("wiring_calc", {"calculation": "voltage_drop", "awg": 6, "length": "30m", "current": "20",
"voltage": "12"}, "over 30 m of"),
("wiring_calc", {"calculation": "voltage_drop", "awg": 6, "length": "100k", "current": "20",
"voltage": "12"}, "Could not read"),
("wiring_calc", {"calculation": "three_phase_drop", "awg": 4, "length": "250 ft", "current": "40",
"voltage": "480", "power_factor": 0.85}, "76.2 m (250 ft)"),
# The width at the other copper weight, so "2 oz saves 40%" is not worked by hand.
("pcb_calc", {"calculation": "trace_width", "current": "3", "temperature_rise": 10}, "At 2 oz copper"),
("pcb_calc", {"calculation": "trace_width", "current": "3", "temperature_rise": 10}, "26.9 mil"),
# One lint entry per rule, with the line numbers, not one per line.
("matlab_run", {"code": 'x = "a";\ny = "b";\nz = "c";\ndisp(1)', "purpose": "deliverable"},
"(3 places)"),
# The std/var weight flag, stated the right way round.
("matlab_run", {"code": "s = std([1 2 3], 0, 1);\ndisp(s)", "purpose": "scratch"}, "N-1"),
("matlab_run", {"code": "s = std([1 2 3]);\ndisp(s)", "purpose": "scratch"}, None, "weight flag"),
# A topic-shaped query falls back to its phrases instead of NO RESULTS.
("lookup_rfc", {"query": "DNS domain name maximum length 63 octets label"}, "RFC 1035"),
("lookup_rfc", {"query": "DNS domain name maximum length 63 octets label"}, "these titles contain"),
# Icon-only anchors keep their href, so an index page still leads somewhere.
("fetch_page", {"url": "https://investors.raspberrypi.com/reports", "max_chars": 6000},
"investors.raspberrypi.com/reports/1"),
# A byline's credential is not a co-author.
("fetch_page", {"url": "https://www.poison.org/articles/acetaminophen-take-it-safely",
"max_chars": 1200}, "Johnson-Arbor,", "Johnson-Arbor et al."),
# A site's own name is not a Dr com.
("fetch_page", {"url": "https://www.electricallicenserenewal.com/Electrical-Continuing-Education-Courses/NEC-Content.php?sectionID=808",
"max_chars": 1200}, None, "CITE AS: com,"),
# --- Change 58: closer to MATLAB ----------------------------------------
# A script that calls its local functions before defining them is
# ordinary MATLAB and used to fail here with "'double_it' undefined".
("matlab_run", {"code": "x = 3;\ndisp(double_it(x));\n\n"
"function y = double_it(v)\n y = 2 * v;\nend\n"},
"6", "undefined"),
# An error inside one of those functions still reports the line number
# of the model's own script, not of the file we made from it.
("matlab_run", {"code": "v = [1 2 3];\ndisp(pick(v));\n\n"
"function y = pick(x)\n y = x(9);\nend\n"},
"main.m line 5 (in pick)"),
# The shims: each is a MATLAB function this Octave does not have.
("matlab_run", {"code": "disp(contains('hello world', 'o w'));\n"
"disp(strlength({'a','abc'}));\n"
"disp(maxk([3 1 4 1 5], 2));\n"
"disp(histcounts([1 2 2 3], [1 2 3 4]));\n"
"disp(rmmissing([1 NaN 3]));",
"purpose": "scratch"}, "1 2 1", "undefined"),
# histcounts without edges refuses rather than inventing a binning rule.
("matlab_run", {"code": "disp(histcounts([1 2 3]));", "purpose": "scratch"},
"automatic bin selection is not reproduced"),
# MATLAB call forms this Octave does not accept, named as such.
("matlab_run", {"code": "disp(round(pi, 3));", "purpose": "scratch"}, "round(x * 10^n)"),
("matlab_run", {"code": "disp(sum(magic(3), 'all'));", "purpose": "scratch"}, "sum(A(:))"),
# Octave's own parser, reporting what no regex rule covers.
("matlab_run", {"code": "x = 5;\nx .*= 2;\ndisp(x)\n"}, "Octave's parser read"),
# A function named after a MATLAB built-in shadows it in the user's MATLAB.
("matlab_run", {"code": "disp(normalize([1 2]));\n\n"
"function y = normalize(x)\n y = x / max(x);\nend\n"},
"Shadows a MATLAB function"),
# Display and file format, two preferences set in the driver.
("matlab_run", {"code": "x = []", "purpose": "scratch"}, "x = []", "[](0x0)"),
("matlab_run", {"code": "x = 1;\nsave('d.mat', 'x');\n"
"f = fopen('d.mat','r'); h = fread(f, 24, 'char=>char')'; fclose(f);\n"
"disp(strtrim(h));", "purpose": "scratch"},
"MATLAB 5.0 MAT-file", "Created by Octave"),
# --- Change 57: plots that are not written on top of themselves ----------
# A figure comes back as a data URI whose first bytes ARE the assertion:
# PNG puts width and height at a fixed offset, so the base64 prefix below
# is exactly "1100 x 825" and nothing else. It fails if the shim is off
# the path, if print goes back to scaling type with -S, or if the canvas
# changes size - the three ways this rendering has broken.
("matlab_run", {"code": "t = linspace(0, 1, 200);\n"
"plot(t, sin(2*pi*t), 'LineWidth', 1.4);\n"
"title('Smoke test');\n"
"xlabel('Time (s)'); ylabel('Amplitude');\n"
"legend('one cycle');\n"
"print('plot.png', '-dpng');",
"return_figure": "plot.png", "purpose": "scratch"},
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAABEwAAAM5"),
# The shape follows the figure the script asked for: 1200x400 on screen
# is 1100x440 on paper, not a 4:3 picture with the plot squashed into it.
("matlab_run", {"code": "figure('position', [0 0 1200 400]);\n"
"plot(1:10, (1:10).^2);\n"
"print('plot.png', '-dpng');",
"return_figure": "plot.png", "purpose": "scratch"},
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAABEwAAAG4"),
# --- the VM sub-agent -------------------------------------------------
# Only the refusal paths belong in this suite. A real session starts a
# machine, runs a model with thinking on, and takes minutes to hours -
# far past run_case's 90 s - so the end-to-end case lives in its own
# opt-in script. What is checked here is that every way of asking for
# something impossible comes back as prose the model can act on rather
# than as a traceback, a 500, or a silent success.
("vm_agent", {"task": "run the build and report what happens",
"os": "ubuntu", "interface": "cli", "project": "../gitea"},
"CANNOT DO THAT"),
("vm_agent", {"task": "run the build and report what happens",
"os": "ubuntu", "interface": "cli", "project": "Bad_Name"},
"CANNOT DO THAT"),
("vm_agent", {"task": "run the build and report what happens",
"os": "plan9", "interface": "cli", "project": "smoke"},
"CANNOT DO THAT"),
# `interface` is VESTIGIAL and is ignored - the cli/gui split was removed
# when every session got the whole toolbox, and vm.py hardcodes "gui".
# This case used to pass a nonsense interface with a VALID project and
# expect a refusal, which stopped being true when the parameter stopped
# being read: the request became a perfectly good one and STARTED A REAL
# SESSION inside the smoke suite, which then died on run_case's 90 s
# timeout with a transport error. Measured 2026-09-15; the session was
# cancelled correctly by the disconnect path, so it left nothing behind
# and the only visible symptom was one confusing failure.
#
# The case is still worth having - a stray parameter must be tolerated,
# not 500 - so it keeps the nonsense interface and pairs it with a
# project name that is refused outright. Now it asserts both things and
# cannot start anything.
("vm_agent", {"task": "run the build and report what happens",
"os": "ubuntu", "interface": "telepathy", "project": "Bad_Name"},
"CANNOT DO THAT"),
# A task too short to act on: the agent cannot see the conversation, so
# "do it" is not a task, it is a guess waiting to happen.
("vm_agent", {"task": "fix it", "os": "ubuntu", "interface": "cli"},
"CANNOT DO THAT"),
# The pre-flight that exists because guessing the machine wrong costs a
# whole session: a Python task with no OS named in it must come back as a
# question for the user, not as a started VM.
("vm_agent", {"task": "write a python script that prints the first 20 primes and run it",
"os": "ubuntu", "interface": "cli", "project": "smoke"},
"CANNOT DO THAT YET"),
# The mirror of that - a task that DOES name its machine and must be let
# through - is deliberately not here: letting it through starts a real
# VM session, which is minutes long by design and would hang this suite.
# It is covered in tests/test_vm_preflight.py instead, against the
# function rather than over HTTP.
("vm_agent", {"cancel": True, "session_id": "not-a-session",
"task": "x" * 20, "os": "ubuntu", "interface": "cli"},
"CANNOT DO THAT"),
# A tool result that IS a data: URI is handed to the model as a vision
# input by Open WebUI, and budget.cap() passes it through untouched. A
# guest could otherwise choose a picture for the main model to look at.
("vm_agent", {"task": "run the build and report what happens",
"os": "ubuntu", "interface": "cli", "project": "../gitea"},
None, "data:image"),
# Nothing in a refusal should ever carry a credential or a host path.
("vm_agent", {"task": "run the build and report what happens",
"os": "plan9", "interface": "cli", "project": "smoke"},
None, ("GITEA_TOKEN", "Bearer", "/var/lib/owuivm", "Traceback")),
]
# The suite above sends no chat id, on purpose: note_fetch no-ops without one,
# so the per-conversation page budget never touches those cases and the order
# they run in cannot matter. The budget itself is only observable across an
# ordered sequence sharing one chat id, so these four run last, with that
# header attached per request rather than set on the client - it cannot leak
# back into anything above, and a re-run gets a fresh conversation.
FETCH_BUDGET_CASES = [
("fetch_page", {"url": "https://www.rfc-editor.org/rfc/rfc8446.txt", "max_chars": 800},
None, "STOP FETCHING"),
# Reading further into a page already open is free: same URL, new window.
("fetch_page", {"url": "https://www.rfc-editor.org/rfc/rfc8446.txt", "offset": 5000,
"max_chars": 800}, "characters 5,000-", "STOP FETCHING"),
("fetch_page", {"url": "https://api.no-such-host-owuitools.invalid/y"}, "does not resolve"),
# The same dead URL a second time is declined from memory, not re-attempted.
("fetch_page", {"url": "https://api.no-such-host-owuitools.invalid/y"}, "not retried"),
]
def run_case(client, case, failures, oversized, headers=None) -> bool:
"""Post one case, print its one-line result, return whether it passed."""
tool, payload, expect = case[:3]
forbid = case[3] if len(case) > 3 else None
forbidden = [forbid] if isinstance(forbid, str) else list(forbid or ())
label = f"{tool}({', '.join(f'{k}={v!r}' for k, v in list(payload.items())[:2])})"
try:
response = client.post(f"/tools/{tool}", json=payload, headers=headers)
except Exception as exc:
failures.append(f"{label}: transport error {type(exc).__name__}: {exc}")
print(f"FAIL {label}\n transport {type(exc).__name__}")
return False
if response.status_code != 200:
failures.append(f"{label}: HTTP {response.status_code} {response.text[:200]}")
print(f"FAIL {label}\n HTTP {response.status_code}: {response.text[:160]}")
return False
body = response.json()
if not isinstance(body, str):
failures.append(f"{label}: returned {type(body).__name__}, expected a string")
print(f"FAIL {label}\n returned {type(body).__name__}, not str")
return False
if not body.strip():
failures.append(f"{label}: empty body")
print(f"FAIL {label}\n empty response")
return False
# A figure is a data: URI, and a picture is allowed to be bigger than a
# page of prose: Open WebUI routes it to the image slot, not the text the
# cap exists to protect.
if len(body) > MAX_CHARS + 400 and not body.startswith("data:"):
oversized.append(f"{label}: {len(body)} chars")
if expect is not None and expect.lower() not in body.lower():
failures.append(f"{label}: expected {expect!r} in response; got: {body[:220]}")
print(f"FAIL {label}\n expected {expect!r}\n got: {body[:200]!r}")
return False
if present := [f for f in forbidden if f.lower() in body.lower()]:
failures.append(f"{label}: must not say {present!r}; got: {body[:220]}")
print(f"FAIL {label}\n must not contain {present!r}\n got: {body[:200]!r}")
return False
print(f"ok {label} [{len(body):,} chars]")
if VERBOSE:
print(textwrap.indent(body[:600], " | "))
return True
def main() -> int:
client = httpx.Client(
base_url=BASE,
headers={"Authorization": f"Bearer {TOKEN}",
# Unique per run: the probe limiter is per user per hour, and
# running this suite four times in an hour otherwise exhausts
# the budget and reports three real tools as broken.
"X-OpenWebUI-User-Id": f"smoketest-{os.getpid()}-{int(time.time())}",
"X-OpenWebUI-User-Email": "smoke@test",
"X-OpenWebUI-User-Role": "user"},
timeout=90.0,
)
passed = failed = 0
failures = []
oversized = []
for case in CASES:
ok = run_case(client, case, failures, oversized)
passed, failed = (passed + 1, failed) if ok else (passed, failed + 1)
# One chat id, this run only, for the ordered budget sequence. It goes on
# the requests rather than the client so nothing above can pick it up.
chat_headers = {"X-OpenWebUI-Chat-Id": f"smoketest-{os.getpid()}-{int(time.time())}"}
print("\n--- fetch budget: ordered, sharing one chat id ---")
for case in FETCH_BUDGET_CASES:
ok = run_case(client, case, failures, oversized, headers=chat_headers)
passed, failed = (passed + 1, failed) if ok else (passed, failed + 1)
print()
print("=" * 68)
print(f"passed {passed} / {passed + failed}")
if oversized:
print(f"\nOVER SIZE CAP ({MAX_CHARS} chars) — these would eat the context window:")
for item in oversized:
print(" ", item)
if failures:
print("\nFAILURES:")
for item in failures:
print(" -", item)
return 1
print("\nAll tools responded correctly and stayed within the size cap.")
return 0
if __name__ == "__main__":
sys.exit(main())