118 lines
4.6 KiB
Python
118 lines
4.6 KiB
Python
#!/usr/bin/env python3
|
|
"""The picture of the session may never be able to destroy the session.
|
|
|
|
python3 tests/test_vm_live_view_safety.py
|
|
|
|
WHY
|
|
|
|
A session ran all six of its tasks, verified them, and returned nothing.
|
|
The agent loop died at the last step with:
|
|
|
|
File "/app/loop.py", line 464, in _terminal_for
|
|
return [f"$ {str(arguments.get('verification', {}).get('command', ''))[:300]}"]
|
|
AttributeError: 'str' object has no attribute 'get'
|
|
|
|
`finish` is declared as taking `verification` as an object with a
|
|
`command` in it. The model sent a string. The renderer for the live view -
|
|
a cosmetic feature, added so somebody watching the chat can see what is
|
|
happening - called `.get` on that string, and the exception came back out
|
|
through the agent loop and threw the finished work away.
|
|
|
|
The rule this file enforces is the one that was missing: the live view has
|
|
no bearing on whether the work succeeds, so it does not get a vote. Every
|
|
renderer must survive whatever the model sends, and the call itself is
|
|
wrapped so that even an unanticipated shape costs a line of output rather
|
|
than a session.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
# The container is built from two places - vmagent/app plus the socket
|
|
# client, which is shared with the host's own vmctl (vmagent/Dockerfile:39).
|
|
sys.path.insert(0, os.path.join(HERE, "..", "vmagent", "app"))
|
|
sys.path.insert(0, os.path.join(HERE, "..", "vm", "vmd"))
|
|
|
|
FAILS: list[str] = []
|
|
|
|
|
|
def check(label: str, condition: bool, detail: str = "") -> None:
|
|
if condition:
|
|
print(f"ok {label}")
|
|
else:
|
|
FAILS.append(label)
|
|
print(f"FAIL {label}\n {detail}")
|
|
|
|
|
|
import loop # noqa: E402
|
|
|
|
# --- the exact call that killed a session -------------------------------
|
|
try:
|
|
loop._terminal_for("finish", {"summary": "done", "verification": "ls -l /tmp"})
|
|
survived = True
|
|
except Exception as exc: # noqa: BLE001
|
|
survived = False
|
|
detail = f"{type(exc).__name__}: {exc}"
|
|
check("finish with verification as a string does not raise", survived,
|
|
locals().get("detail", ""))
|
|
|
|
shown = loop._terminal_for("finish", {"verification": "ls -l /tmp"})
|
|
check("and the command it sent is still shown",
|
|
any("ls -l /tmp" in line for line in shown), f"{shown}")
|
|
check("the documented shape still works",
|
|
any("pytest" in line for line in
|
|
loop._terminal_for("finish", {"verification": {"command": "pytest"}})))
|
|
check("and a finish with no verification shows nothing rather than an empty prompt",
|
|
loop._terminal_for("finish", {}) == [])
|
|
|
|
# --- every renderer, against everything a model might send --------------
|
|
JUNK = ["a string", 12, None, [], ["a", "list"], True, {"unexpected": "keys"}, {}]
|
|
NAMES = ["run", "write_file", "read_file", "list_dir", "git_commit", "check_internet",
|
|
"reboot_vm", "web_search", "read_page", "open_page", "wait", "finish",
|
|
"plan_tasks", "update_task", "ask_operator", "not_a_tool"]
|
|
broke = []
|
|
for tool_name in NAMES:
|
|
for junk in JUNK:
|
|
try:
|
|
out = loop._terminal_for(tool_name, junk)
|
|
if not isinstance(out, list):
|
|
broke.append(f"{tool_name}({junk!r}) returned {type(out).__name__}")
|
|
except Exception as exc: # noqa: BLE001
|
|
broke.append(f"{tool_name}({junk!r}) raised {type(exc).__name__}: {exc}")
|
|
check("no renderer raises on any argument shape a model can send", not broke,
|
|
"\n ".join(broke[:6]))
|
|
|
|
# --- and the call site swallows even what the renderers do not ----------
|
|
class Exploding:
|
|
def terminal(self, *_lines):
|
|
raise RuntimeError("the emitter itself failed")
|
|
|
|
|
|
def detonate(_name, _payload):
|
|
raise ValueError("the renderer itself failed")
|
|
|
|
|
|
try:
|
|
loop._show(Exploding(), detonate, "run", {"command": "true"})
|
|
loop._show(Exploding(), loop._terminal_for, "run", {"command": "true"})
|
|
contained = True
|
|
except Exception as exc: # noqa: BLE001
|
|
contained = False
|
|
why = f"{type(exc).__name__}: {exc}"
|
|
check("a failure anywhere in the view is contained at the call site", contained,
|
|
locals().get("why", ""))
|
|
|
|
# --- the loop must not call a renderer any other way --------------------
|
|
src = open(os.path.join(HERE, "..", "vmagent", "app", "loop.py"), encoding="utf-8").read()
|
|
body = src[src.index("def _converse"):src.index("def _show")]
|
|
check("the loop only renders through the guarded path",
|
|
"ui.terminal(" not in body,
|
|
"an unguarded ui.terminal(*_render(...)) is how the first one got in")
|
|
|
|
print()
|
|
if FAILS:
|
|
print(f"FAILED {len(FAILS)}")
|
|
raise SystemExit(1)
|
|
print("the live view cannot take a session down with it")
|