570 lines
35 KiB
Python
570 lines
35 KiB
Python
"""Exercise every owuitools endpoint the way Open WebUI would.
|
|
|
|
Runs on openwebui_default and talks to the gateway by name, so it takes the
|
|
same path a real tool call does - through nginx, into the guarded
|
|
namespace - rather than testing the app in isolation where the firewall,
|
|
the proxy and the auth header would all be bypassed.
|
|
"""
|
|
|
|
import time
|
|
import json
|
|
import os
|
|
import sys
|
|
import textwrap
|
|
|
|
import httpx
|
|
|
|
BASE = os.environ.get("OWUITOOLS_BASE", "http://owuitools-gw:8000")
|
|
TOKEN = os.environ["OWUITOOLS_AUTH_TOKEN"]
|
|
MAX_CHARS = int(os.environ.get("OWUITOOLS_MAX_RESPONSE_CHARS", "12000"))
|
|
VERBOSE = "-v" in sys.argv
|
|
|
|
# (tool, payload, must_appear_in_response_or_None[, must_NOT_appear])
|
|
# The fourth element is optional and may be one string or a tuple of them.
|
|
# Several behaviours here are only testable as an absence - an NXDOMAIN that
|
|
# must not be reported as a policy denial, a corrected hint that must not
|
|
# still carry the wrong claim - and a missing negative reads as a pass.
|
|
CASES = [
|
|
# --- offline / deterministic: these must always pass -----------------
|
|
("subnet_calc", {"network": "192.168.1.0/24"}, "192.168.1.255"),
|
|
("subnet_calc", {"network": "10.0.0.0/8", "split_into": 4}, "10.64.0.0/10"),
|
|
("subnet_calc", {"network": "2001:db8::/32"}, "2001:db8::"),
|
|
("network_reference", {"lookup_type": "port", "value": "443"}, "https"),
|
|
("network_reference", {"lookup_type": "http_status", "value": "418"}, "teapot"),
|
|
("network_reference", {"lookup_type": "mac_vendor", "value": "00:1A:2B:3C:4D:5E"}, None),
|
|
("ohms_law_power", {"voltage": "12", "resistance": "220"}, "54.55 mA"),
|
|
("ohms_law_power", {"power": "0.5", "resistance": "100"}, None),
|
|
("component_calc", {"calculation": "resistor_color_code", "text": "brown black red gold"}, "1 kΩ"),
|
|
("component_calc", {"calculation": "resistor_to_color", "value": "4k7"}, "yellow"),
|
|
("component_calc", {"calculation": "nearest_e_series", "value": "3300", "series": "E24"}, None),
|
|
("component_calc", {"calculation": "rc_filter", "resistance": "10k", "capacitance": "100n"}, "159.2 Hz"),
|
|
("component_calc", {"calculation": "lc_resonance", "inductance": "10u", "capacitance": "100n"}, None),
|
|
("component_calc", {"calculation": "voltage_divider", "voltage": "12", "resistance": "10k", "resistance2": "2k2"}, None),
|
|
("component_calc", {"calculation": "series_parallel", "text": "10k, 10k", "combination": "parallel"}, "5 kΩ"),
|
|
("pcb_calc", {"calculation": "trace_width", "current": "2"}, "mm"),
|
|
("pcb_calc", {"calculation": "microstrip_impedance", "trace_width": "0.3", "dielectric_height": "0.2"}, "Ω"),
|
|
("pcb_calc", {"calculation": "via_current", "via_diameter": "0.3", "current": "1"}, None),
|
|
("wiring_calc", {"calculation": "awg_properties", "awg": 14}, "mm²"),
|
|
("wiring_calc", {"calculation": "voltage_drop", "awg": 12, "length": "30", "current": "10", "voltage": "12"}, "%"),
|
|
("wiring_calc", {"calculation": "led_resistor", "voltage": "5", "led_forward_voltage": "2.1", "led_current": "0.02"}, "145 Ω"),
|
|
("wiring_calc", {"calculation": "decibels", "value": "30", "db_from": "dBm"}, "1 W"),
|
|
("wiring_calc", {"calculation": "battery_life", "capacity_mah": 2000, "load_ma": 150}, "hours"),
|
|
|
|
# --- MATLAB sandbox -------------------------------------------------
|
|
# These assert the three verdicts the tool exists to distinguish:
|
|
# it worked, it is not valid MATLAB, and it failed for a reason that
|
|
# is not the model's fault.
|
|
("matlab_run", {"code": "x = linspace(0,1,5);\ndisp(sum(x))"}, "MATLAB run OK"),
|
|
("matlab_run", {"code": "a=[1 2 3];\nb=[1 2];\nc=a*b;"}, "nonconformant"),
|
|
("matlab_run", {"code": "a=[1 2 3];\nb=[1 2];\nc=a*b;"}, "main.m line 3"),
|
|
("matlab_run", {"code": "for i=1:2\n printf('%d',i);\nendfor"}, "NOT VALID MATLAB"),
|
|
# Was: "contains is real MATLAB but not available here". Change 58 made
|
|
# it available, so what is pinned now is that it runs and that nothing
|
|
# tells the model to rewrite it.
|
|
("matlab_run", {"code": "s = contains('abc','b');\ndisp(s)"}, "1", "not available"),
|
|
("matlab_run", {"code": "while true; end", "timeout_seconds": 3}, "Timed out"),
|
|
("matlab_run", {"code": "function r=f(a)\n r=a;\nend"}, "function definition, not a script"),
|
|
("matlab_run", {"code": "disp(addup(2,3))",
|
|
"helper_files": [{"name": "addup.m", "code": "function r=addup(a,b)\n r=a+b;\nend"}]}, "5"),
|
|
("matlab_run", {"code": "x = webread('http://example.com');"}, "no network"),
|
|
("matlab_run", {"code": ""}, "CANNOT DO THAT"),
|
|
# Regressions found by adversarial review; each of these was broken.
|
|
("matlab_run", {"code": "x=1;\n# double it\nx=x*2;\ndisp(x)"}, "NOT VALID MATLAB"),
|
|
("matlab_run", {"code": "plot_style = 'x';\ns = sprintf('a', ...\n '--r');\ndisp(1)"}, "MATLAB run OK"),
|
|
("matlab_run", {"code": "A=[1 1;1 1];\nx=A\\[1;2];\ndisp('done')"}, "Warnings"),
|
|
("matlab_run", {"code": "a = zeros(60000,60000);", "timeout_seconds": 30}, "out of memory"),
|
|
("matlab_run", {"code": "disp(addup(2,3))",
|
|
"helper_files": [{"name": "2bad.m", "code": "function r=addup(a,b)\n r=a+b;\nend"}]},
|
|
"Helper files ignored"),
|
|
|
|
# --- network-dependent ----------------------------------------------
|
|
("web_search", {"query": "RFC 9293 TCP specification", "max_results": 4}, None),
|
|
("web_search", {"query": "systemd-resolved DNSSEC failure", "category": "it", "max_results": 3}, None),
|
|
("fetch_page", {"url": "https://example.com", "max_chars": 1500}, "Example Domain"),
|
|
("wikipedia_lookup", {"topic": "OSI model"}, "layer"),
|
|
("search_stackexchange", {"query": "bash trap EXIT", "site": "unix", "max_results": 2}, None),
|
|
("lookup_rfc", {"number": 9293, "offset": 0}, "RFC 9293"),
|
|
("lookup_rfc", {"query": "border gateway protocol"}, "Proposed Standard"),
|
|
("lookup_cve", {"cve_id": "CVE-2021-44228"}, "Log4j"),
|
|
("read_man_page", {"name": "ip", "section": 8}, "NAME"),
|
|
("lookup_package", {"name": "requests", "ecosystem": "pypi"}, "Latest version"),
|
|
("lookup_package", {"name": "torvalds/linux", "ecosystem": "github"}, "tags"),
|
|
("dns_lookup", {"name": "cloudflare.com", "record_type": "A"}, "TTL"),
|
|
("dns_lookup", {"name": "google.com", "record_type": "MX"}, None),
|
|
("inspect_tls", {"hostname": "example.com"}, "Issuer"),
|
|
("http_headers", {"url": "https://example.com"}, "Security headers"),
|
|
("lookup_ip_or_asn", {"target": "8.8.8.8"}, None),
|
|
("find_datasheet", {"part_number": "LM358"}, None),
|
|
("search_papers", {"query": "GaN transistor switching losses", "max_results": 2}, None),
|
|
|
|
# --- guardrails: these must be REFUSED, not fulfilled ----------------
|
|
("fetch_page", {"url": "http://192.168.1.10:9443/"}, "CANNOT DO THAT"),
|
|
("fetch_page", {"url": "http://172.19.0.2:8080/api/config"}, "CANNOT DO THAT"),
|
|
("fetch_page", {"url": "file:///etc/passwd"}, "CANNOT DO THAT"),
|
|
("fetch_page", {"url": "http://localhost:8000/health"}, "CANNOT DO THAT"),
|
|
("dns_lookup", {"name": "router.local", "record_type": "A"}, "CANNOT DO THAT"),
|
|
("lookup_ip_or_asn", {"target": "192.168.1.1"}, "CANNOT DO THAT"),
|
|
("ohms_law_power", {"voltage": "12"}, "CANNOT DO THAT"),
|
|
("component_calc", {"calculation": "rc_filter", "resistance": "10k"}, "CANNOT DO THAT"),
|
|
# Models pass "" for optionals they chose not to use; must not 422.
|
|
("web_search", {"query": "test", "site": "", "language": "", "max_results": 3}, None),
|
|
("lookup_rfc", {"number": 1918, "query": ""}, "1918"),
|
|
|
|
# --- citation metadata: a page has to say what it is -----------------
|
|
# The URLs are stable documents chosen for their metadata, not their
|
|
# ranking: a paper of record on PLOS and the arXiv abstract of
|
|
# "Attention Is All You Need". Two cases per page where two separate
|
|
# fields are being asserted, the same way the MATLAB cases above do.
|
|
("fetch_page", {"url": "https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0173152",
|
|
"max_chars": 1200}, "CITE AS: Ravenscroft"),
|
|
("fetch_page", {"url": "https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0173152",
|
|
"max_chars": 1200}, "Journal: PLOS ONE"),
|
|
("fetch_page", {"url": "https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0173152",
|
|
"max_chars": 1200}, "DOI: 10.1371"),
|
|
# Comma-formatted authors are "Vaswani, Ashish"; the surname is the cite.
|
|
("fetch_page", {"url": "https://arxiv.org/abs/1706.03762", "max_chars": 900},
|
|
"CITE AS: Vaswani", "CITE AS: Ashish"),
|
|
# What arrived is not always what citing it would imply.
|
|
("fetch_page", {"url": "https://arxiv.org/abs/1706.03762", "max_chars": 700}, "abstract page"),
|
|
("fetch_page", {"url": "https://pubmed.ncbi.nlm.nih.gov/32939066/", "max_chars": 900},
|
|
"not the paper"),
|
|
("fetch_page", {"url": "https://example.com", "max_chars": 800},
|
|
"No author, journal or DOI metadata"),
|
|
|
|
# --- reading inside a document ---------------------------------------
|
|
("fetch_page", {"url": "https://www.rfc-editor.org/rfc/rfc9293.txt",
|
|
"search": "Retransmission Timeout", "max_chars": 1200}, "Jumped to"),
|
|
("fetch_page", {"url": "https://example.com", "search": "zzzqqqnotpresent"},
|
|
"does not occur in this document"),
|
|
# The continuation hint pushes on; it must not read as permission to stop.
|
|
("fetch_page", {"url": "https://www.rfc-editor.org/rfc/rfc9293.txt", "max_chars": 1000},
|
|
"Keep reading if the answer may be further in", "Only do so if this section did not answer"),
|
|
("lookup_rfc", {"number": 1035, "search": "2.3.4"}, "skipped"),
|
|
|
|
# --- stub and failure honesty: a non-answer must not look like one ---
|
|
("fetch_page", {"url": "https://www.cisa.gov/known-exploited-vulnerabilities-catalog"},
|
|
"client-rendered or gated"),
|
|
# A name that does not exist is a DNS fact, not a firewall refusal; saying
|
|
# "private network" there sends the model looking for a way around it.
|
|
("fetch_page", {"url": "https://api.no-such-host-owuitools.invalid/x"},
|
|
"does not resolve", "private network"),
|
|
("inspect_tls", {"hostname": "no-such-host-owuitools.invalid"},
|
|
"does not resolve", "private network"),
|
|
|
|
# --- search relevance and category honesty ---------------------------
|
|
# An invented title returns something from every engine; the results are
|
|
# not about the query, and saying so is the whole point.
|
|
("web_search", {"query": '"Ferrous lattice resonance in dodecahedral copper" Whitlock Ambersley',
|
|
"max_results": 5}, "LOW RELEVANCE"),
|
|
("web_search", {"query": "RFC 9293 TCP retransmission timeout computation", "max_results": 4},
|
|
"Snippets are search-engine excerpts", "LOW RELEVANCE"),
|
|
("web_search", {"query": "amoxicillin adult dose sinusitis mg how much should i take",
|
|
"max_results": 4}, "dailymed"),
|
|
("web_search", {"query": "ibuprofen maximum daily dose mg adult", "category": "science",
|
|
"max_results": 4}, "scholarly indexes only"),
|
|
# Structural, not about which papers rank: some scholarly result carries
|
|
# enough metadata to cite. Only a dead scholarly index breaks this.
|
|
("web_search", {"query": "metformin lactic acidosis cohort study", "category": "science",
|
|
"max_results": 6}, "CITE AS:"),
|
|
|
|
# --- wikipedia resolution --------------------------------------------
|
|
("wikipedia_lookup", {"topic": "Mercury"}, "disambiguation page"),
|
|
("wikipedia_lookup", {"topic": "Marguerite Delacroix-Whitmore"}, "NO ARTICLE"),
|
|
# An exact title must win over whatever full-text search would have found.
|
|
("wikipedia_lookup", {"topic": "Canberra"}, "# Canberra"),
|
|
("wikipedia_lookup", {"topic": "Canberra", "full_article": True,
|
|
"search": "Federal Capital Territory"}, "Jumped to"),
|
|
("wikipedia_lookup", {"topic": "Eiffel Tower"}, "Introduction only"),
|
|
|
|
# --- v3: a citation the answer can lift ------------------------------
|
|
# Two graded rounds: 1 of 25 tool-assisted answers carried a URL, and the
|
|
# one that did lifted a ready-made CITE AS line. Every tool that returns a
|
|
# single identifiable document now ends with the line the answer wants.
|
|
("fetch_page", {"url": "https://example.com", "max_chars": 800},
|
|
"\n---\nSources: https://example.com"),
|
|
("wikipedia_lookup", {"topic": "Canberra"},
|
|
"\n---\nSources: https://en.wikipedia.org/wiki/Canberra"),
|
|
("lookup_rfc", {"number": 1035, "offset": 0}, "\n---\nSources: RFC 1035"),
|
|
("read_man_page", {"name": "ls"}, "\n---\nSources: man ls"),
|
|
("lookup_package", {"name": "requests", "ecosystem": "pypi"},
|
|
"\n---\nSources: https://"),
|
|
# A paper carries its citation as well as its URL.
|
|
("fetch_page", {"url": "https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0173152",
|
|
"max_chars": 1200}, "Sources: Ravenscroft"),
|
|
# Still there at the largest window the schema allows, where the page
|
|
# budget is tightest. (That `cap()` re-attaches it after trimming is a
|
|
# pure-function property; this asserts the room reservation upstream.)
|
|
("fetch_page", {"url": "https://www.rfc-editor.org/rfc/rfc9293.txt", "max_chars": 10000},
|
|
"\n---\nSources: https://www.rfc-editor.org/rfc/rfc9293.txt"),
|
|
|
|
# --- v3: a jump says which section it landed in -----------------------
|
|
# "Constitution" inside the Canberra article matched "1 Constitution
|
|
# Avenue" - a street - and the model read the window as having checked the
|
|
# constitutional basis for the capital. A character offset cannot say you
|
|
# are in the wrong subject; the enclosing heading can.
|
|
("wikipedia_lookup", {"topic": "Canberra", "full_article": True, "search": "Constitution"},
|
|
"inside the section"),
|
|
("lookup_rfc", {"number": 1035, "search": "63 octets"}, "inside the section"),
|
|
("read_man_page", {"name": "ls", "search": "--almost-all"}, "inside the section"),
|
|
|
|
# --- v3: a search miss does not throw the document away ---------------
|
|
# One search string missing is not the same finding as the document being
|
|
# useless; returning only "not here" made the two indistinguishable, and
|
|
# the model abandoned 54,000 characters of the right article.
|
|
("wikipedia_lookup", {"topic": "Sense", "full_article": True, "search": "zzzqqqnotpresent"},
|
|
"Showing the opening of the document instead"),
|
|
("lookup_rfc", {"number": 1035, "search": "zzzqqqnotpresent"},
|
|
"Showing the opening of the document instead"),
|
|
("read_man_page", {"name": "ls", "search": "zzzqqqnotpresent"},
|
|
"Showing the opening of the document instead"),
|
|
|
|
# --- v3: which of your terms is absent, as a count --------------------
|
|
# A model hunting a half-remembered paper puts its own guesses in the
|
|
# query; topically plausible results then read as partial confirmation.
|
|
("web_search", {"query": "Korhonen coffee randomized trial type 2 diabetes Circulation 2018",
|
|
"max_results": 6}, "Appears in none of these results"),
|
|
("web_search", {"query": "linux kernel scheduler documentation", "category": "it",
|
|
"max_results": 3}, None, "Appears in none of these results"),
|
|
# An index listing is not the document, and cannot be quoted as one.
|
|
("lookup_rfc", {"query": "border gateway protocol"}, "These are index entries"),
|
|
|
|
# --- per-tool additions ----------------------------------------------
|
|
("http_headers", {"url": "https://github.com"}, "max-age="),
|
|
# Splitting past /64 breaks SLAAC, which the split is silent about otherwise.
|
|
("subnet_calc", {"network": "2001:db8:1::/64", "split_into": 4}, "RFC 7421"),
|
|
|
|
# --- Change 55: the v3 round's tool defects ---------------------------
|
|
# A documentation address is named as such, with its RFC, and a public
|
|
# address carries its origin AS from the routing view, not just its
|
|
# registration.
|
|
("lookup_ip_or_asn", {"target": "203.0.113.10"}, "RFC 5737"),
|
|
("lookup_ip_or_asn", {"target": "8.8.8.8"}, "Origin AS"),
|
|
("subnet_calc", {"network": "203.0.113.0/24"}, "reserved for documentation"),
|
|
# Board dimensions with units, and no negative impedance ever printed.
|
|
("pcb_calc", {"calculation": "microstrip_impedance", "trace_width": "12mil", "dielectric_height": "8 mil"}, "0.305 mm"),
|
|
("pcb_calc", {"calculation": "microstrip_impedance", "trace_width": "0.3mm", "dielectric_height": "1.6mm"}, "Ω", "≈ -"),
|
|
("pcb_calc", {"calculation": "microstrip_impedance", "trace_width": "10", "dielectric_height": "0.1"}, "CANNOT DO THAT", "Ω**"),
|
|
# A search miss on an unsectioned name checks the sibling sections.
|
|
("read_man_page", {"name": "crontab", "search": "PATH"}, "section 5"),
|
|
# PMC and PubMed pages are served from the Europe PMC open-access API.
|
|
("fetch_page", {"url": "https://pubmed.ncbi.nlm.nih.gov/24459154/", "max_chars": 900}, "Retrieved via"),
|
|
("fetch_page", {"url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC3898757/", "max_chars": 900}, "Diabetes Care"),
|
|
# A PDF whose text layer is glyph codes is refused, not served as text.
|
|
("fetch_page", {"url": "https://www.infineon.com/assets/row/public/documents/24/49/infineon-irlz44n-datasheet-en.pdf",
|
|
"max_chars": 900}, "not decodable"),
|
|
# An oversized window is clamped; a bad parameter is prose, not a 422.
|
|
("fetch_page", {"url": "https://example.com", "max_chars": 10716}, "Example Domain", "422"),
|
|
("fetch_page", {"url": "https://example.com", "offset": -5}, "CANNOT DO THAT"),
|
|
# A page-numberless contents entry (RFC v3 text format) is stepped over.
|
|
("lookup_rfc", {"number": 9110, "search": "15.5.19"}, "skipped 1 table-of-contents"),
|
|
("lookup_rfc", {"number": 9110, "search": "15.5.19"}, "15.5. Client Error 4xx"),
|
|
# Parse-error and ordering hints.
|
|
("matlab_run", {"code": "x = 2;\nfprintf('today\\'s value: %d\\n', x);", "purpose": "scratch"}, "two quotes"),
|
|
# Was: the hint explaining that Octave cannot see a function defined
|
|
# below its call. Change 58 lifts those out instead, so the script the
|
|
# hint was written for now simply runs; the hint survives for the files
|
|
# the lift refuses, and is checked by tests/test_local_functions.py.
|
|
("matlab_run", {"code": "disp(addup(2,3))\nfunction r = addup(a,b)\n r = a+b;\nend", "purpose": "scratch"},
|
|
"5", "below the line that calls it"),
|
|
("subnet_calc", {"network": "192.168.1.0/24", "split_into": 4}, "192.168.1.192", "RFC 7421"),
|
|
("pcb_calc", {"calculation": "microstrip_width_for_impedance", "target_impedance": "50",
|
|
"dielectric_height": "0.2"}, "Width for"),
|
|
# IPC-2152 is not a licence to narrow an external trace, which is what the
|
|
# old wording invited.
|
|
("pcb_calc", {"calculation": "trace_width", "current": "3"},
|
|
"not a way to shrink an external trace", "usually less conservative standard if you need to squeeze"),
|
|
("read_man_page", {"name": "ip route"}, "ip-route", "No manual page found"),
|
|
# Both labels come off the vendor-pattern URL, not off search ranking:
|
|
# LM358DR strips to LM358, so the official PDF is a variant, not the part.
|
|
("find_datasheet", {"part_number": "LM358"}, "[EXACT LM358]"),
|
|
("find_datasheet", {"part_number": "LM358DR"}, "VARIANT: names 'LM358'"),
|
|
("lookup_cve", {"cve_id": "CVE-2024-6387"}, "openssh: before"),
|
|
("lookup_cve", {"cve_id": "CVE-2021-44228"}, "CISA KEV"),
|
|
("matlab_run", {"code": "n = numel(regexp('a b', '\\bb', 'match'));\ndisp(n)"}, "BACKSPACE"),
|
|
# purpose='scratch' is the model working for itself: give it the Octave
|
|
# equivalent and drop the MATLAB-validity verdict, which is not the job.
|
|
# `contains` is a shim now (Change 58) and runs, so the equivalent-advice
|
|
# path is tested with a name that is still missing.
|
|
("matlab_run", {"code": "s = split('a b');\ndisp(s)", "purpose": "scratch"},
|
|
"strsplit", "Do NOT rewrite"),
|
|
("matlab_run", {"code": "x=1;\n# double it\nx=x*2;\ndisp(x)", "purpose": "scratch"},
|
|
None, "NOT VALID MATLAB"),
|
|
|
|
# --- Change 56: the v4 round's easy wins ---------------------------------
|
|
# A cable length in feet is feet, not femtometres.
|
|
("wiring_calc", {"calculation": "voltage_drop", "awg": 12, "length": "100ft", "current": "20",
|
|
"voltage": "120"}, "30.48 m (100 ft)", ("fV", "1e-13")),
|
|
("wiring_calc", {"calculation": "voltage_drop", "awg": 12, "length": "100 ft", "current": "20",
|
|
"voltage": "120"}, "5.29%"),
|
|
# A bare "30m" is thirty metres, not thirty milli-anything.
|
|
("wiring_calc", {"calculation": "voltage_drop", "awg": 6, "length": "30m", "current": "20",
|
|
"voltage": "12"}, "over 30 m of"),
|
|
("wiring_calc", {"calculation": "voltage_drop", "awg": 6, "length": "100k", "current": "20",
|
|
"voltage": "12"}, "Could not read"),
|
|
("wiring_calc", {"calculation": "three_phase_drop", "awg": 4, "length": "250 ft", "current": "40",
|
|
"voltage": "480", "power_factor": 0.85}, "76.2 m (250 ft)"),
|
|
# The width at the other copper weight, so "2 oz saves 40%" is not worked by hand.
|
|
("pcb_calc", {"calculation": "trace_width", "current": "3", "temperature_rise": 10}, "At 2 oz copper"),
|
|
("pcb_calc", {"calculation": "trace_width", "current": "3", "temperature_rise": 10}, "26.9 mil"),
|
|
# One lint entry per rule, with the line numbers, not one per line.
|
|
("matlab_run", {"code": 'x = "a";\ny = "b";\nz = "c";\ndisp(1)', "purpose": "deliverable"},
|
|
"(3 places)"),
|
|
# The std/var weight flag, stated the right way round.
|
|
("matlab_run", {"code": "s = std([1 2 3], 0, 1);\ndisp(s)", "purpose": "scratch"}, "N-1"),
|
|
("matlab_run", {"code": "s = std([1 2 3]);\ndisp(s)", "purpose": "scratch"}, None, "weight flag"),
|
|
# A topic-shaped query falls back to its phrases instead of NO RESULTS.
|
|
("lookup_rfc", {"query": "DNS domain name maximum length 63 octets label"}, "RFC 1035"),
|
|
("lookup_rfc", {"query": "DNS domain name maximum length 63 octets label"}, "these titles contain"),
|
|
# Icon-only anchors keep their href, so an index page still leads somewhere.
|
|
("fetch_page", {"url": "https://investors.raspberrypi.com/reports", "max_chars": 6000},
|
|
"investors.raspberrypi.com/reports/1"),
|
|
# A byline's credential is not a co-author.
|
|
("fetch_page", {"url": "https://www.poison.org/articles/acetaminophen-take-it-safely",
|
|
"max_chars": 1200}, "Johnson-Arbor,", "Johnson-Arbor et al."),
|
|
# A site's own name is not a Dr com.
|
|
("fetch_page", {"url": "https://www.electricallicenserenewal.com/Electrical-Continuing-Education-Courses/NEC-Content.php?sectionID=808",
|
|
"max_chars": 1200}, None, "CITE AS: com,"),
|
|
|
|
# --- Change 58: closer to MATLAB ----------------------------------------
|
|
# A script that calls its local functions before defining them is
|
|
# ordinary MATLAB and used to fail here with "'double_it' undefined".
|
|
("matlab_run", {"code": "x = 3;\ndisp(double_it(x));\n\n"
|
|
"function y = double_it(v)\n y = 2 * v;\nend\n"},
|
|
"6", "undefined"),
|
|
# An error inside one of those functions still reports the line number
|
|
# of the model's own script, not of the file we made from it.
|
|
("matlab_run", {"code": "v = [1 2 3];\ndisp(pick(v));\n\n"
|
|
"function y = pick(x)\n y = x(9);\nend\n"},
|
|
"main.m line 5 (in pick)"),
|
|
# The shims: each is a MATLAB function this Octave does not have.
|
|
("matlab_run", {"code": "disp(contains('hello world', 'o w'));\n"
|
|
"disp(strlength({'a','abc'}));\n"
|
|
"disp(maxk([3 1 4 1 5], 2));\n"
|
|
"disp(histcounts([1 2 2 3], [1 2 3 4]));\n"
|
|
"disp(rmmissing([1 NaN 3]));",
|
|
"purpose": "scratch"}, "1 2 1", "undefined"),
|
|
# histcounts without edges refuses rather than inventing a binning rule.
|
|
("matlab_run", {"code": "disp(histcounts([1 2 3]));", "purpose": "scratch"},
|
|
"automatic bin selection is not reproduced"),
|
|
# MATLAB call forms this Octave does not accept, named as such.
|
|
("matlab_run", {"code": "disp(round(pi, 3));", "purpose": "scratch"}, "round(x * 10^n)"),
|
|
("matlab_run", {"code": "disp(sum(magic(3), 'all'));", "purpose": "scratch"}, "sum(A(:))"),
|
|
# Octave's own parser, reporting what no regex rule covers.
|
|
("matlab_run", {"code": "x = 5;\nx .*= 2;\ndisp(x)\n"}, "Octave's parser read"),
|
|
# A function named after a MATLAB built-in shadows it in the user's MATLAB.
|
|
("matlab_run", {"code": "disp(normalize([1 2]));\n\n"
|
|
"function y = normalize(x)\n y = x / max(x);\nend\n"},
|
|
"Shadows a MATLAB function"),
|
|
# Display and file format, two preferences set in the driver.
|
|
("matlab_run", {"code": "x = []", "purpose": "scratch"}, "x = []", "[](0x0)"),
|
|
("matlab_run", {"code": "x = 1;\nsave('d.mat', 'x');\n"
|
|
"f = fopen('d.mat','r'); h = fread(f, 24, 'char=>char')'; fclose(f);\n"
|
|
"disp(strtrim(h));", "purpose": "scratch"},
|
|
"MATLAB 5.0 MAT-file", "Created by Octave"),
|
|
|
|
# --- Change 57: plots that are not written on top of themselves ----------
|
|
# A figure comes back as a data URI whose first bytes ARE the assertion:
|
|
# PNG puts width and height at a fixed offset, so the base64 prefix below
|
|
# is exactly "1100 x 825" and nothing else. It fails if the shim is off
|
|
# the path, if print goes back to scaling type with -S, or if the canvas
|
|
# changes size - the three ways this rendering has broken.
|
|
("matlab_run", {"code": "t = linspace(0, 1, 200);\n"
|
|
"plot(t, sin(2*pi*t), 'LineWidth', 1.4);\n"
|
|
"title('Smoke test');\n"
|
|
"xlabel('Time (s)'); ylabel('Amplitude');\n"
|
|
"legend('one cycle');\n"
|
|
"print('plot.png', '-dpng');",
|
|
"return_figure": "plot.png", "purpose": "scratch"},
|
|
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAABEwAAAM5"),
|
|
# The shape follows the figure the script asked for: 1200x400 on screen
|
|
# is 1100x440 on paper, not a 4:3 picture with the plot squashed into it.
|
|
("matlab_run", {"code": "figure('position', [0 0 1200 400]);\n"
|
|
"plot(1:10, (1:10).^2);\n"
|
|
"print('plot.png', '-dpng');",
|
|
"return_figure": "plot.png", "purpose": "scratch"},
|
|
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAABEwAAAG4"),
|
|
|
|
# --- the VM sub-agent -------------------------------------------------
|
|
# Only the refusal paths belong in this suite. A real session starts a
|
|
# machine, runs a model with thinking on, and takes minutes to hours -
|
|
# far past run_case's 90 s - so the end-to-end case lives in its own
|
|
# opt-in script. What is checked here is that every way of asking for
|
|
# something impossible comes back as prose the model can act on rather
|
|
# than as a traceback, a 500, or a silent success.
|
|
("vm_agent", {"task": "run the build and report what happens",
|
|
"os": "ubuntu", "interface": "cli", "project": "../gitea"},
|
|
"CANNOT DO THAT"),
|
|
("vm_agent", {"task": "run the build and report what happens",
|
|
"os": "ubuntu", "interface": "cli", "project": "Bad_Name"},
|
|
"CANNOT DO THAT"),
|
|
("vm_agent", {"task": "run the build and report what happens",
|
|
"os": "plan9", "interface": "cli", "project": "smoke"},
|
|
"CANNOT DO THAT"),
|
|
# `interface` is VESTIGIAL and is ignored - the cli/gui split was removed
|
|
# when every session got the whole toolbox, and vm.py hardcodes "gui".
|
|
# This case used to pass a nonsense interface with a VALID project and
|
|
# expect a refusal, which stopped being true when the parameter stopped
|
|
# being read: the request became a perfectly good one and STARTED A REAL
|
|
# SESSION inside the smoke suite, which then died on run_case's 90 s
|
|
# timeout with a transport error. Measured 2026-09-15; the session was
|
|
# cancelled correctly by the disconnect path, so it left nothing behind
|
|
# and the only visible symptom was one confusing failure.
|
|
#
|
|
# The case is still worth having - a stray parameter must be tolerated,
|
|
# not 500 - so it keeps the nonsense interface and pairs it with a
|
|
# project name that is refused outright. Now it asserts both things and
|
|
# cannot start anything.
|
|
("vm_agent", {"task": "run the build and report what happens",
|
|
"os": "ubuntu", "interface": "telepathy", "project": "Bad_Name"},
|
|
"CANNOT DO THAT"),
|
|
# A task too short to act on: the agent cannot see the conversation, so
|
|
# "do it" is not a task, it is a guess waiting to happen.
|
|
("vm_agent", {"task": "fix it", "os": "ubuntu", "interface": "cli"},
|
|
"CANNOT DO THAT"),
|
|
# The pre-flight that exists because guessing the machine wrong costs a
|
|
# whole session: a Python task with no OS named in it must come back as a
|
|
# question for the user, not as a started VM.
|
|
("vm_agent", {"task": "write a python script that prints the first 20 primes and run it",
|
|
"os": "ubuntu", "interface": "cli", "project": "smoke"},
|
|
"CANNOT DO THAT YET"),
|
|
# The mirror of that - a task that DOES name its machine and must be let
|
|
# through - is deliberately not here: letting it through starts a real
|
|
# VM session, which is minutes long by design and would hang this suite.
|
|
# It is covered in tests/test_vm_preflight.py instead, against the
|
|
# function rather than over HTTP.
|
|
("vm_agent", {"cancel": True, "session_id": "not-a-session",
|
|
"task": "x" * 20, "os": "ubuntu", "interface": "cli"},
|
|
"CANNOT DO THAT"),
|
|
# A tool result that IS a data: URI is handed to the model as a vision
|
|
# input by Open WebUI, and budget.cap() passes it through untouched. A
|
|
# guest could otherwise choose a picture for the main model to look at.
|
|
("vm_agent", {"task": "run the build and report what happens",
|
|
"os": "ubuntu", "interface": "cli", "project": "../gitea"},
|
|
None, "data:image"),
|
|
# Nothing in a refusal should ever carry a credential or a host path.
|
|
("vm_agent", {"task": "run the build and report what happens",
|
|
"os": "plan9", "interface": "cli", "project": "smoke"},
|
|
None, ("GITEA_TOKEN", "Bearer", "/var/lib/owuivm", "Traceback")),
|
|
]
|
|
|
|
# The suite above sends no chat id, on purpose: note_fetch no-ops without one,
|
|
# so the per-conversation page budget never touches those cases and the order
|
|
# they run in cannot matter. The budget itself is only observable across an
|
|
# ordered sequence sharing one chat id, so these four run last, with that
|
|
# header attached per request rather than set on the client - it cannot leak
|
|
# back into anything above, and a re-run gets a fresh conversation.
|
|
FETCH_BUDGET_CASES = [
|
|
("fetch_page", {"url": "https://www.rfc-editor.org/rfc/rfc8446.txt", "max_chars": 800},
|
|
None, "STOP FETCHING"),
|
|
# Reading further into a page already open is free: same URL, new window.
|
|
("fetch_page", {"url": "https://www.rfc-editor.org/rfc/rfc8446.txt", "offset": 5000,
|
|
"max_chars": 800}, "characters 5,000-", "STOP FETCHING"),
|
|
("fetch_page", {"url": "https://api.no-such-host-owuitools.invalid/y"}, "does not resolve"),
|
|
# The same dead URL a second time is declined from memory, not re-attempted.
|
|
("fetch_page", {"url": "https://api.no-such-host-owuitools.invalid/y"}, "not retried"),
|
|
]
|
|
|
|
|
|
def run_case(client, case, failures, oversized, headers=None) -> bool:
|
|
"""Post one case, print its one-line result, return whether it passed."""
|
|
tool, payload, expect = case[:3]
|
|
forbid = case[3] if len(case) > 3 else None
|
|
forbidden = [forbid] if isinstance(forbid, str) else list(forbid or ())
|
|
|
|
label = f"{tool}({', '.join(f'{k}={v!r}' for k, v in list(payload.items())[:2])})"
|
|
try:
|
|
response = client.post(f"/tools/{tool}", json=payload, headers=headers)
|
|
except Exception as exc:
|
|
failures.append(f"{label}: transport error {type(exc).__name__}: {exc}")
|
|
print(f"FAIL {label}\n transport {type(exc).__name__}")
|
|
return False
|
|
|
|
if response.status_code != 200:
|
|
failures.append(f"{label}: HTTP {response.status_code} {response.text[:200]}")
|
|
print(f"FAIL {label}\n HTTP {response.status_code}: {response.text[:160]}")
|
|
return False
|
|
|
|
body = response.json()
|
|
if not isinstance(body, str):
|
|
failures.append(f"{label}: returned {type(body).__name__}, expected a string")
|
|
print(f"FAIL {label}\n returned {type(body).__name__}, not str")
|
|
return False
|
|
|
|
if not body.strip():
|
|
failures.append(f"{label}: empty body")
|
|
print(f"FAIL {label}\n empty response")
|
|
return False
|
|
|
|
# A figure is a data: URI, and a picture is allowed to be bigger than a
|
|
# page of prose: Open WebUI routes it to the image slot, not the text the
|
|
# cap exists to protect.
|
|
if len(body) > MAX_CHARS + 400 and not body.startswith("data:"):
|
|
oversized.append(f"{label}: {len(body)} chars")
|
|
|
|
if expect is not None and expect.lower() not in body.lower():
|
|
failures.append(f"{label}: expected {expect!r} in response; got: {body[:220]}")
|
|
print(f"FAIL {label}\n expected {expect!r}\n got: {body[:200]!r}")
|
|
return False
|
|
|
|
if present := [f for f in forbidden if f.lower() in body.lower()]:
|
|
failures.append(f"{label}: must not say {present!r}; got: {body[:220]}")
|
|
print(f"FAIL {label}\n must not contain {present!r}\n got: {body[:200]!r}")
|
|
return False
|
|
|
|
print(f"ok {label} [{len(body):,} chars]")
|
|
if VERBOSE:
|
|
print(textwrap.indent(body[:600], " | "))
|
|
return True
|
|
|
|
|
|
def main() -> int:
|
|
client = httpx.Client(
|
|
base_url=BASE,
|
|
headers={"Authorization": f"Bearer {TOKEN}",
|
|
# Unique per run: the probe limiter is per user per hour, and
|
|
# running this suite four times in an hour otherwise exhausts
|
|
# the budget and reports three real tools as broken.
|
|
"X-OpenWebUI-User-Id": f"smoketest-{os.getpid()}-{int(time.time())}",
|
|
"X-OpenWebUI-User-Email": "smoke@test",
|
|
"X-OpenWebUI-User-Role": "user"},
|
|
timeout=90.0,
|
|
)
|
|
|
|
passed = failed = 0
|
|
failures = []
|
|
oversized = []
|
|
|
|
for case in CASES:
|
|
ok = run_case(client, case, failures, oversized)
|
|
passed, failed = (passed + 1, failed) if ok else (passed, failed + 1)
|
|
|
|
# One chat id, this run only, for the ordered budget sequence. It goes on
|
|
# the requests rather than the client so nothing above can pick it up.
|
|
chat_headers = {"X-OpenWebUI-Chat-Id": f"smoketest-{os.getpid()}-{int(time.time())}"}
|
|
print("\n--- fetch budget: ordered, sharing one chat id ---")
|
|
for case in FETCH_BUDGET_CASES:
|
|
ok = run_case(client, case, failures, oversized, headers=chat_headers)
|
|
passed, failed = (passed + 1, failed) if ok else (passed, failed + 1)
|
|
|
|
print()
|
|
print("=" * 68)
|
|
print(f"passed {passed} / {passed + failed}")
|
|
|
|
if oversized:
|
|
print(f"\nOVER SIZE CAP ({MAX_CHARS} chars) — these would eat the context window:")
|
|
for item in oversized:
|
|
print(" ", item)
|
|
|
|
if failures:
|
|
print("\nFAILURES:")
|
|
for item in failures:
|
|
print(" -", item)
|
|
return 1
|
|
|
|
print("\nAll tools responded correctly and stayed within the size cap.")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|