Initial Commit
This commit is contained in:
@@ -0,0 +1,335 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The live view in the chat: two boxes, the right height, and one of each.
|
||||
|
||||
python3 tests/test_vm_live_view.py
|
||||
|
||||
WHY
|
||||
|
||||
This file exists because of two bugs that both looked fine from the
|
||||
outside and neither of which any other test could see.
|
||||
|
||||
THE FIRST was a second copy of `_embed` left behind by a bad edit. Python
|
||||
keeps the last definition of a method, so the class parsed, imported and
|
||||
ran - and silently used the old single-box version while the new two-box
|
||||
one sat above it, unreachable. Nothing failed. The only symptom was the
|
||||
wrong thing on the screen.
|
||||
|
||||
THE SECOND was the height. Open WebUI sizes one of these frames two ways:
|
||||
it reads `contentDocument.body.scrollHeight` on load, and it listens for
|
||||
`postMessage({type:"iframe:height"})`. The first cannot work here - the
|
||||
frame is sandboxed without `allow-same-origin`, so the read throws and is
|
||||
swallowed by a bare `catch {}` - and the frame then keeps its stylesheet
|
||||
default of 150px. That was the scrollbar. Measured in the browser: a
|
||||
seven-line task list needs 138px and a twenty-line terminal 364px, and
|
||||
both frames sat at 150 until the frame posted its own height.
|
||||
|
||||
There is a third trap inside the fix, and it has its own case below:
|
||||
measuring `documentElement.scrollHeight` does not work either, because
|
||||
`html{overflow:hidden}` makes it report the frame's CURRENT height
|
||||
whenever the content is shorter. A box measured that way grows and never
|
||||
shrinks. The wrapper's own bounding box has no such floor.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import types
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
FILTER = os.path.join(HERE, "..", "owui-functions", "vm_agent_progress.py")
|
||||
|
||||
FAILS: list[str] = []
|
||||
|
||||
|
||||
def check(label: str, condition: bool, detail: str = "") -> None:
|
||||
if condition:
|
||||
print(f"ok {label}")
|
||||
else:
|
||||
FAILS.append(label)
|
||||
print(f"FAIL {label}\n {detail}")
|
||||
|
||||
|
||||
src = open(FILTER, encoding="utf-8").read()
|
||||
|
||||
# --- no method is defined twice -----------------------------------------
|
||||
tree = ast.parse(src)
|
||||
cls = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == "Filter")
|
||||
names = [n.name for n in cls.body
|
||||
if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))]
|
||||
dupes = sorted({n for n in names if names.count(n) > 1})
|
||||
check("no method in the filter is defined twice", not dupes,
|
||||
f"{dupes} - the later copy wins silently and the earlier one is dead code")
|
||||
|
||||
# --- load it with the two runtime imports stubbed out --------------------
|
||||
aiohttp = types.ModuleType("aiohttp")
|
||||
aiohttp.ClientError = Exception
|
||||
aiohttp.ClientSession = object
|
||||
aiohttp.ClientTimeout = object
|
||||
sys.modules["aiohttp"] = aiohttp
|
||||
pydantic = types.ModuleType("pydantic")
|
||||
|
||||
|
||||
class _BaseModel:
|
||||
def __init__(self, **kw):
|
||||
for key, value in kw.items():
|
||||
setattr(self, key, value)
|
||||
|
||||
|
||||
pydantic.BaseModel = _BaseModel
|
||||
pydantic.Field = lambda default=None, **kw: default
|
||||
sys.modules["pydantic"] = pydantic
|
||||
|
||||
namespace: dict = {}
|
||||
exec(compile(src, FILTER, "exec"), namespace)
|
||||
Filter = namespace["Filter"]
|
||||
|
||||
|
||||
class Valves:
|
||||
show_terminal = True
|
||||
show_screenshots = True
|
||||
|
||||
|
||||
def embeds_for(data: dict) -> list:
|
||||
out: list = []
|
||||
|
||||
async def emitter(event):
|
||||
out.append(event)
|
||||
|
||||
asyncio.run(Filter._embed(emitter, data, Valves()))
|
||||
return out[-1]["data"]["embeds"]
|
||||
|
||||
|
||||
TASKS = "\n".join(f"[ ] {i + 1}. a task" for i in range(7))
|
||||
CLI = {"tasks": TASKS, "terminal": ["$ one", "$ two"]}
|
||||
GUI = {"tasks": TASKS, "shot": "data:image/jpeg;base64,AAAA",
|
||||
"caption": "the desktop", "terminal": ["$ one", "$ two"]}
|
||||
|
||||
# --- the boxes, and what is in each ---------------------------------------
|
||||
#
|
||||
# The layout the operator asked for. The screen box is the only one that
|
||||
# comes and goes: it appears when the agent takes a screenshot and not
|
||||
# before, because most jobs never look at the screen and a picture of an
|
||||
# idle desktop sitting in the chat from the handover onwards told the
|
||||
# reader nothing.
|
||||
#
|
||||
# before any screenshot after one
|
||||
# ------------------- -------------------
|
||||
# live line live line
|
||||
# Task List: Task List:
|
||||
# tasks tasks
|
||||
# ------------------- -------------------
|
||||
# -------------------
|
||||
# the machine's screen
|
||||
# -------------------
|
||||
# ------------------- -------------------
|
||||
# command feed (20) command feed (20)
|
||||
# ------------------- -------------------
|
||||
plain = embeds_for(CLI)
|
||||
check("a session with no screenshot yet gets two boxes", len(plain) == 2,
|
||||
f"got {len(plain)} - the screen box has nothing to show")
|
||||
check("the task list is the first of them", "a task" in plain[0])
|
||||
check("and the command feed is the second", "$ one" in plain[1])
|
||||
check("neither box contains the other's contents",
|
||||
"$ one" not in plain[0] and "a task" not in plain[1],
|
||||
"this is what a single joined box looked like")
|
||||
check("and no picture is invented to fill the space", "<img" not in "".join(plain))
|
||||
|
||||
shot = embeds_for(GUI)
|
||||
check("once a screenshot exists there are three boxes", len(shot) == 3,
|
||||
f"got {len(shot)}")
|
||||
check("the screen goes between the list and the feed",
|
||||
"<img" in shot[1] and "the desktop" in shot[1] and "$ one" in shot[2],
|
||||
"the feed is context for what is on the screen above it")
|
||||
|
||||
# One height, always. It used to be halved when a screen was shown, which
|
||||
# meant the feed shrank for something that might never arrive.
|
||||
def rows_in(box):
|
||||
return box.count('<div style="white-space:pre;overflow:hidden;')
|
||||
|
||||
|
||||
check("the command feed is the same height with or without a screen",
|
||||
rows_in(shot[2]) == rows_in(plain[1]) == namespace["TERMINAL_LINES"],
|
||||
f"{rows_in(shot[2])} vs {rows_in(plain[1])}")
|
||||
|
||||
# --- a viewer who joins late must not sit in front of an empty message ----
|
||||
#
|
||||
# These updates are pushed. Someone who reloads the page, or opens the chat
|
||||
# on a second device, joins partway through and receives only what comes
|
||||
# NEXT - and a session can hold one step for minutes ("Waiting for the
|
||||
# machine to finish booting" held across four polls, measured). Emitting
|
||||
# only on change leaves that viewer looking at nothing for the length of the
|
||||
# step. The view is therefore re-sent on a slow heartbeat as well.
|
||||
check("the whole view is re-sent even when nothing changed",
|
||||
"or resend" in src and "RESEND_SECONDS" in src,
|
||||
"a reader who reloads mid-session would see nothing until the next change")
|
||||
check("the heartbeat is slower than the poll",
|
||||
namespace["RESEND_SECONDS"] > 2.0,
|
||||
"re-sending on every poll would be a message every two seconds for an hour")
|
||||
check("re-sending replaces the view rather than appending to it",
|
||||
'"replace": True' in src,
|
||||
"otherwise a heartbeat leaves one frame per tick on the message")
|
||||
|
||||
# The end of the watcher is not the end of the session. This block runs on
|
||||
# cancellation and on the watcher's own deadline too, and in those cases the
|
||||
# machine is still running - saying it has been destroyed is simply false.
|
||||
check("the machine is only called destroyed when the session said it was done",
|
||||
"ended_cleanly" in src and "No longer watching this session." in src,
|
||||
"the watcher stopping is not the same event as the session ending")
|
||||
|
||||
# --- a session that has said nothing must still look alive ---------------
|
||||
#
|
||||
# "Starting a machine" is the first thing the watcher emits, before there
|
||||
# is any progress to read. If the progress file never appears - the job
|
||||
# queued behind another session, the runner restarting, a boot that is
|
||||
# simply slow - that sentence is the last word the reader gets. Reported
|
||||
# as exactly that: a toast stuck on "Starting a machine" with no boxes,
|
||||
# and a frozen line is indistinguishable from a broken one.
|
||||
check("a silent start begins counting rather than repeating itself",
|
||||
"QUIET_SECONDS" in src and "so far, nothing back" in src,
|
||||
"a number that moves is the difference between waiting and wondering")
|
||||
check("it waits a sensible moment before doing so",
|
||||
10.0 <= namespace["QUIET_SECONDS"] <= 60.0,
|
||||
f"{namespace['QUIET_SECONDS']}s - a normal boot must not trip it")
|
||||
check("and the elapsed time comes from when the watch began",
|
||||
"began = asyncio.get_running_loop().time()" in src)
|
||||
|
||||
# --- the confirmation dialog shows a summary, not the job -----------------
|
||||
#
|
||||
# The job the model writes is an instruction sheet for another machine, and
|
||||
# a real one measured 2,600 characters. Nobody reads that to decide whether
|
||||
# to press OK, and Open WebUI does not scroll the dialog: measured, a job
|
||||
# that long pushed OK and Cancel 150px below the bottom of an 800px window,
|
||||
# so the question could not be answered at all.
|
||||
#
|
||||
# The summary is built from the job rather than written by the model. This
|
||||
# is the one screen where a summary that drifts from the text is a real
|
||||
# problem - the person approves the summary and the machine is sent the job.
|
||||
|
||||
JOB = """Set up and test a log-file cleanup one-liner on this Ubuntu machine.
|
||||
|
||||
STEP 1 - Set up test files. Create the directory /tmp/logtest, then create:
|
||||
|
||||
- /tmp/logtest/recent.log -> mtime = now
|
||||
- /tmp/logtest/ten_days.log -> mtime = 10 days ago
|
||||
|
||||
STEP 2 - BEFORE snapshot. Print the current time and list every file.
|
||||
|
||||
STEP 3 - DRY RUN. Run the command without -delete and capture the output.
|
||||
|
||||
STEP 4 - REAL RUN. Run the one-liner and capture the exit code.
|
||||
|
||||
STEP 5 - AFTER snapshot. Re-list everything the same way as STEP 2.
|
||||
|
||||
STEP 6 - Compare. Say which files went.
|
||||
|
||||
STEP 7 - FINAL REPORT. State for each file whether it was DELETED or KEPT."""
|
||||
|
||||
summary = Filter._summarise(JOB)
|
||||
check("the dialog is given a summary, not the job", len(summary) < len(JOB) / 3,
|
||||
f"{len(summary)} of {len(JOB)} characters - this dialog does not scroll")
|
||||
check("it is short enough to leave the buttons on screen",
|
||||
len(summary.splitlines()) <= Filter.SUMMARY_STEPS + 2,
|
||||
"a job tall enough to outgrow the window makes the question unanswerable")
|
||||
|
||||
# EVERY BULLET IS ONE LINE, AND THE WHOLE DIALOG FITS A SMALL WINDOW.
|
||||
#
|
||||
# Measured in a 631px-tall browser - not an unusual window, just one that
|
||||
# is not maximised - a 21-line message put OK and Cancel 65px below the
|
||||
# bottom edge, and Open WebUI does not scroll that dialog.
|
||||
check("no bullet is long enough to wrap on a narrow dialog",
|
||||
all(len(line) <= Filter.SUMMARY_STEP + 4 for line in summary.splitlines()),
|
||||
"a wrapped bullet costs two lines and there is no room for six of those")
|
||||
check("the summary itself is at most seven lines",
|
||||
len(summary.splitlines()) <= 7, f"{len(summary.splitlines())}")
|
||||
check("it opens with what the job is for", summary.splitlines()[0].startswith("Set up and test"))
|
||||
check("and lists what will actually be done",
|
||||
"Set up test files" in summary and "DRY RUN" in summary)
|
||||
# The whole message, not just the summary: the framing and the closing
|
||||
# line are paid for out of the same budget.
|
||||
def dialog_lines(job_text):
|
||||
"""The message as _maybe_ask builds it, without needing a session."""
|
||||
body = Filter._summarise(job_text)
|
||||
framing = 2 # the one-line preamble plus its blank line
|
||||
heading = 1 # WHAT IT WILL DO
|
||||
closing = 2 # blank line plus the one-line instructions
|
||||
return framing + heading + len(body.splitlines()) + closing
|
||||
|
||||
|
||||
check("a long job still leaves a dialog that fits a small window",
|
||||
dialog_lines(JOB) <= 12,
|
||||
f"{dialog_lines(JOB)} lines - measured, 21 put the buttons off-screen at 631px")
|
||||
|
||||
check("steps it could not fit are counted rather than dropped silently",
|
||||
"more step" in summary, "a summary that hides work is worse than no summary")
|
||||
|
||||
# Every line of the summary has to come from the job. This is the property
|
||||
# that makes it safe to approve: it cannot describe work that is not there.
|
||||
# "and N more steps" is the summariser's own phrasing; everything else has
|
||||
# to be the job's own words.
|
||||
ADDED = {"more", "step", "steps", "and"}
|
||||
words = set(re.findall(r"[a-z0-9]+", JOB.lower()))
|
||||
made_up = [w for w in re.findall(r"[a-z0-9]+", summary.lower())
|
||||
if len(w) > 3 and w not in words and w not in ADDED]
|
||||
check("nothing in the summary was invented", not made_up, f"{made_up}")
|
||||
|
||||
# The data a step operates on is not itself a step.
|
||||
check("a bulleted list under a step is not mistaken for the steps",
|
||||
"recent.log" not in summary,
|
||||
"measured: counting those bullets pushed the real steps out entirely")
|
||||
|
||||
# A job with no headings still gets summarised rather than dumped.
|
||||
prose = ("Check the disk setup. Report free space on every filesystem. "
|
||||
"Report anything over 90 percent full. List the block devices.")
|
||||
check("a job written as prose is summarised too",
|
||||
"\n- " in Filter._summarise(prose))
|
||||
|
||||
# Open WebUI renders this dialog as markdown, where a plain newline is a
|
||||
# SOFT break and consecutive lines are joined into one paragraph. Measured,
|
||||
# that ran two steps together on one wrapped line. A "- " list item at
|
||||
# column 0 is the form the renderer keeps on its own line; an indented one
|
||||
# becomes a nested or literal block, and a bullet character of my own is
|
||||
# not markdown at all and gets joined like any other prose line.
|
||||
check("each step is a markdown list item at column 0",
|
||||
all(line.startswith("- ") for line in summary.splitlines()[1:]),
|
||||
"anything else is joined into a paragraph by the renderer")
|
||||
check("and no step is indented into a nested block",
|
||||
"\n - " not in summary and "\n\u2022" not in summary)
|
||||
|
||||
# A step that is only a label says nothing by itself.
|
||||
labelled = Filter._summarise(
|
||||
"Do the thing.\n1. Set up.\n2. Report:\n - which files went and which stayed")
|
||||
check("a step that is only a label takes the line under it",
|
||||
"which files went" in labelled,
|
||||
"a bullet reading just 'Report:' spends a line and says nothing")
|
||||
check("a one-line job is left alone",
|
||||
"\n" not in Filter._summarise("Run uname -a and report the output."))
|
||||
check("an empty job says so rather than showing nothing",
|
||||
"empty" in Filter._summarise(" "))
|
||||
|
||||
# The colon case: splitting sentences on ":" turned a real step into "Run:".
|
||||
# "e.g." and "(i.e." are not the ends of sentences, and the punctuation
|
||||
# around them is part of the word as the splitter sees it.
|
||||
for abbreviated, survives in (
|
||||
("Create a folder, e.g. /tmp/logtest, and fill it", "logtest"),
|
||||
("Report which files went (i.e. deleted) and which stayed", "deleted"),
|
||||
("Check approx. 20 files and report", "files")):
|
||||
check(f"an abbreviation does not end a step: {abbreviated[:28]}...",
|
||||
survives in Filter._headline(abbreviated, 90),
|
||||
"the bullet was cut off mid-abbreviation")
|
||||
|
||||
check("a step is not cut off at a colon",
|
||||
"ps -p 1" in Filter._summarise("STEP 1 - Run: ps -p 1 -o comm= and check it says systemd"),
|
||||
"the useful half of a step is always after the colon")
|
||||
|
||||
# --- no em dashes in anything the operator reads -------------------------
|
||||
prose = [line for line in src.splitlines()
|
||||
if "—" in line and not line.lstrip().startswith("#")]
|
||||
check("no em dash reaches the chat", not prose, f"{prose[:2]}")
|
||||
|
||||
print()
|
||||
if FAILS:
|
||||
print(f"FAILED {len(FAILS)}")
|
||||
raise SystemExit(1)
|
||||
print("two boxes, each the size of what is in it, and a dialog that can be answered")
|
||||
Reference in New Issue
Block a user