104 lines
4.4 KiB
Python
Executable File
104 lines
4.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""The planning turn happens while the machine boots, not after it.
|
|
|
|
python3 tests/test_vm_boot_overlap.py
|
|
|
|
WHY
|
|
|
|
A machine takes twenty-odd seconds to boot and the agent's first turn -
|
|
prefill, thinking, one tool call - takes about as long. Done in sequence
|
|
that is a minute of one side waiting for the other, every session. Done
|
|
at once it is free, and it has a second effect worth as much: the model
|
|
is loaded and its prompt prefilled by the time there is anything to do.
|
|
|
|
The risk in overlapping them is an agent that tries to touch a machine
|
|
that is not there yet. Two things prevent it, and this file checks both:
|
|
the loop does not OFFER the guest tools while booting, and vmd refuses
|
|
them regardless of what was offered.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "vmagent", "app"))
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "vm", "vmd"))
|
|
|
|
import tools # noqa: E402
|
|
|
|
FAILS: list[str] = []
|
|
|
|
|
|
def check(label: str, condition: bool, detail: str = "") -> None:
|
|
if condition:
|
|
print(f"ok {label}")
|
|
else:
|
|
FAILS.append(f"{label}: {detail}")
|
|
print(f"FAIL {label}\n {detail}")
|
|
|
|
|
|
# --- what is offered while the machine is coming up ---------------------
|
|
for interface in ("cli", "gui"):
|
|
booting = [t["function"]["name"] for t in tools.schemas("ubuntu", booting=True)]
|
|
ready = [t["function"]["name"] for t in tools.schemas("ubuntu", booting=False)]
|
|
|
|
check(f"{interface}: planning tools are offered while booting",
|
|
"plan_tasks" in booting, booting)
|
|
guest = [n for n in booting
|
|
if n in ("run", "write_file", "read_file", "list_dir", "screenshot",
|
|
"click", "type", "key", "wait", "check_internet", "git_commit",
|
|
"reboot_vm", "finish")]
|
|
check(f"{interface}: no guest tool is offered while booting",
|
|
not guest, f"offered {guest}")
|
|
check(f"{interface}: the full set arrives once ready",
|
|
"run" in ready and "finish" in ready, ready)
|
|
|
|
check("gui gets the screen tools only once ready",
|
|
"screenshot" in [t["function"]["name"] for t in tools.schemas("ubuntu", booting=False)]
|
|
and "screenshot" not in [t["function"]["name"] for t in tools.schemas("ubuntu", booting=True)],
|
|
"screenshot offered at the wrong time")
|
|
|
|
# --- the wait tool ------------------------------------------------------
|
|
cli = {t["function"]["name"]: t for t in tools.schemas("ubuntu", booting=False)}
|
|
check("wait is available at the command line, not only in a desktop session",
|
|
"wait" in cli, sorted(cli))
|
|
|
|
wait_text = (cli["wait"]["function"]["description"]
|
|
+ str(cli["wait"]["function"]["parameters"])).lower()
|
|
# The cap is one more sentence paid for on every turn, and it only
|
|
# matters in the case where it is exceeded - where the tool says so.
|
|
check("the wait description does not spend tokens on the cap",
|
|
not any(w in wait_text for w in ("20 minutes", "1200", "maximum", "capped", "limit")),
|
|
wait_text[:160])
|
|
|
|
import vmd # noqa: E402
|
|
check("the cap exists in software", vmd.MAX_WAIT_SECONDS == 1200.0, vmd.MAX_WAIT_SECONDS)
|
|
|
|
# --- the gate lives in vmd too, not only in what the loop offers --------
|
|
src = open(os.path.join(os.path.dirname(__file__), "..", "vm", "vmd", "vmd.py")).read()
|
|
check("every guest verb is gated on readiness",
|
|
src.count("_require_ready()") >= 12, src.count("_require_ready()"))
|
|
check("the gate tells 'no machine' apart from 'still booting'",
|
|
"there is no machine running" in src and "still booting" in src, "")
|
|
check("session_ready is not a tool the agent can call",
|
|
"session_ready" not in str(tools.schemas("ubuntu", booting=False)),
|
|
"the agent can ask whether the machine is up, and it will, repeatedly")
|
|
|
|
# --- the opening message sets the expectation ---------------------------
|
|
import loop # noqa: E402
|
|
|
|
job = loop.Job(task="do a thing", os_name="ubuntu", interface="gui", project=None)
|
|
opening = loop._opening_message(job, {})
|
|
for phrase in ("booting", "plan_tasks", "told"):
|
|
check(f"the opening message mentions {phrase!r}", phrase in opening, opening[:200])
|
|
check("it does not claim the machine is usable yet",
|
|
"is booting" in opening or "not ready" in opening, opening[:200])
|
|
|
|
print()
|
|
print("=" * 60)
|
|
if FAILS:
|
|
print(f"FAILED {len(FAILS)}")
|
|
for f in FAILS:
|
|
print(" -", f)
|
|
sys.exit(1)
|
|
print("all boot-overlap cases pass")
|