Initial Commit

tests/test_vm_live_view_safety.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""The picture of the session may never be able to destroy the session.
+
+    python3 tests/test_vm_live_view_safety.py
+
+WHY
+
+A session ran all six of its tasks, verified them, and returned nothing.
+The agent loop died at the last step with:
+
+    File "/app/loop.py", line 464, in _terminal_for
+      return [f"$ {str(arguments.get('verification', {}).get('command', ''))[:300]}"]
+    AttributeError: 'str' object has no attribute 'get'
+
+`finish` is declared as taking `verification` as an object with a
+`command` in it. The model sent a string. The renderer for the live view -
+a cosmetic feature, added so somebody watching the chat can see what is
+happening - called `.get` on that string, and the exception came back out
+through the agent loop and threw the finished work away.
+
+The rule this file enforces is the one that was missing: the live view has
+no bearing on whether the work succeeds, so it does not get a vote. Every
+renderer must survive whatever the model sends, and the call itself is
+wrapped so that even an unanticipated shape costs a line of output rather
+than a session.
+"""
+
+import os
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+# The container is built from two places - vmagent/app plus the socket
+# client, which is shared with the host's own vmctl (vmagent/Dockerfile:39).
+sys.path.insert(0, os.path.join(HERE, "..", "vmagent", "app"))
+sys.path.insert(0, os.path.join(HERE, "..", "vm", "vmd"))
+
+FAILS: list[str] = []
+
+
+def check(label: str, condition: bool, detail: str = "") -> None:
+    if condition:
+        print(f"ok    {label}")
+    else:
+        FAILS.append(label)
+        print(f"FAIL  {label}\n      {detail}")
+
+
+import loop  # noqa: E402
+
+# --- the exact call that killed a session -------------------------------
+try:
+    loop._terminal_for("finish", {"summary": "done", "verification": "ls -l /tmp"})
+    survived = True
+except Exception as exc:  # noqa: BLE001
+    survived = False
+    detail = f"{type(exc).__name__}: {exc}"
+check("finish with verification as a string does not raise", survived,
+      locals().get("detail", ""))
+
+shown = loop._terminal_for("finish", {"verification": "ls -l /tmp"})
+check("and the command it sent is still shown",
+      any("ls -l /tmp" in line for line in shown), f"{shown}")
+check("the documented shape still works",
+      any("pytest" in line for line in
+          loop._terminal_for("finish", {"verification": {"command": "pytest"}})))
+check("and a finish with no verification shows nothing rather than an empty prompt",
+      loop._terminal_for("finish", {}) == [])
+
+# --- every renderer, against everything a model might send --------------
+JUNK = ["a string", 12, None, [], ["a", "list"], True, {"unexpected": "keys"}, {}]
+NAMES = ["run", "write_file", "read_file", "list_dir", "git_commit", "check_internet",
+         "reboot_vm", "web_search", "read_page", "open_page", "wait", "finish",
+         "plan_tasks", "update_task", "ask_operator", "not_a_tool"]
+broke = []
+for tool_name in NAMES:
+    for junk in JUNK:
+        try:
+            out = loop._terminal_for(tool_name, junk)
+            if not isinstance(out, list):
+                broke.append(f"{tool_name}({junk!r}) returned {type(out).__name__}")
+        except Exception as exc:  # noqa: BLE001
+            broke.append(f"{tool_name}({junk!r}) raised {type(exc).__name__}: {exc}")
+check("no renderer raises on any argument shape a model can send", not broke,
+      "\n      ".join(broke[:6]))
+
+# --- and the call site swallows even what the renderers do not ----------
+class Exploding:
+    def terminal(self, *_lines):
+        raise RuntimeError("the emitter itself failed")
+
+
+def detonate(_name, _payload):
+    raise ValueError("the renderer itself failed")
+
+
+try:
+    loop._show(Exploding(), detonate, "run", {"command": "true"})
+    loop._show(Exploding(), loop._terminal_for, "run", {"command": "true"})
+    contained = True
+except Exception as exc:  # noqa: BLE001
+    contained = False
+    why = f"{type(exc).__name__}: {exc}"
+check("a failure anywhere in the view is contained at the call site", contained,
+      locals().get("why", ""))
+
+# --- the loop must not call a renderer any other way --------------------
+src = open(os.path.join(HERE, "..", "vmagent", "app", "loop.py"), encoding="utf-8").read()
+body = src[src.index("def _converse"):src.index("def _show")]
+check("the loop only renders through the guarded path",
+      "ui.terminal(" not in body,
+      "an unguarded ui.terminal(*_render(...)) is how the first one got in")
+
+print()
+if FAILS:
+    print(f"FAILED {len(FAILS)}")
+    raise SystemExit(1)
+print("the live view cannot take a session down with it")