172 lines
7.5 KiB
Python
172 lines
7.5 KiB
Python
#!/usr/bin/env python3
|
|
"""What the agent did is written down, and can be read back.
|
|
|
|
python3 tests/test_vm_session_log.py
|
|
|
|
WHY
|
|
|
|
Asked "did it ever open the app and look at it", the honest answer was
|
|
that the record could not say. Everything needed was ALREADY being
|
|
recorded - `_record()` has covered screenshot, click, key and type since
|
|
they were written - and all of it was discarded when the session ended.
|
|
What was left to read were the two records that cannot answer it:
|
|
|
|
* the tool result counted `run` and nothing else, so a session that
|
|
marked "clicked through the GUI" done without ever calling `click`
|
|
read exactly like one that did;
|
|
* Open WebUI's status history is SAMPLED every two seconds, so a verb
|
|
that returns fast is usually overwritten before it is ever seen.
|
|
Measured on a real session: 456 samples caught `launch_app` nine
|
|
times because it blocks for seconds, and caught not one of the
|
|
screenshots that demonstrably happened.
|
|
|
|
So the transcript is now written to disk as it happens, one file per
|
|
session, with the model's own words between the calls - and the tool
|
|
result carries the counts, so a claim that disagrees with them is
|
|
visible without opening anything.
|
|
"""
|
|
|
|
import ast
|
|
import json
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
VMD = os.path.join(HERE, "..", "vm", "vmd")
|
|
sys.path.insert(0, VMD)
|
|
|
|
FAILS: list[str] = []
|
|
|
|
|
|
def check(label: str, condition: bool, detail: str = "") -> None:
|
|
if condition:
|
|
print(f"ok {label}")
|
|
else:
|
|
FAILS.append(f"{label}: {detail}")
|
|
print(f"FAIL {label}\n {detail}")
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
os.environ["OWUITOOLS_VM_LOG_DIR"] = tmp
|
|
import sessionlog
|
|
sessionlog.LOG_DIR = tmp
|
|
|
|
log = sessionlog.open_log("abc12345deadbeef", chat="chat-1", os="windows",
|
|
project="toolkit", owner="someone", task="build a GUI")
|
|
check("a log is opened for the session", log is not None)
|
|
|
|
log.write("plan", count=3)
|
|
log.write("note", turn=1, text="I'll write the skeleton first.")
|
|
log.write("write_file", path="Z:\\Toolkit.ps1", bytes=4821)
|
|
log.write("run", command="powershell -File Z:\\Toolkit.ps1", exit_code=0, seconds=2.1)
|
|
log.write("launch_app", path="Toolkit.ps1")
|
|
log.write("screenshot", width=1280, height=800, bytes=86000)
|
|
log.write("task", id=1, status="done")
|
|
log.close(outcome="finished", verified=True, commits=1)
|
|
|
|
files = [n for n in os.listdir(tmp) if n.endswith(".jsonl")]
|
|
check("it is one file per session", len(files) == 1, str(files))
|
|
check("named so the newest sorts last", files[0][:2] == "20" and "abc12345" in files[0],
|
|
files[0])
|
|
|
|
records = sessionlog.load(os.path.join(tmp, files[0]))
|
|
kinds = [r["kind"] for r in records]
|
|
check("it opens with the session and closes with the outcome",
|
|
kinds[0] == "start" and kinds[-1] == "end", str(kinds))
|
|
check("the chat is recorded, so a chat maps to a session",
|
|
records[0].get("chat") == "chat-1",
|
|
"without it the only way to tell two runs apart is guessing at timestamps")
|
|
check("every call is in order", kinds[1:-1] ==
|
|
["plan", "note", "write_file", "run", "launch_app", "screenshot", "task"],
|
|
str(kinds))
|
|
check("each one is stamped with how far into the session it happened",
|
|
all("at" in r for r in records))
|
|
|
|
# --- the census is the thing that answers the question ---------------
|
|
counts = sessionlog.census(records)
|
|
check("the census counts the calls, not the talk",
|
|
counts.get("screenshot") == 1 and counts.get("launch_app") == 1
|
|
and "note" not in counts, str(counts))
|
|
check("and a verb never called simply is not there", "click" not in counts, str(counts))
|
|
|
|
text = sessionlog.render(os.path.join(tmp, files[0]))
|
|
check("the rendering names the session and the chat",
|
|
"abc12345deadbeef" in text and "chat-1" in text)
|
|
check("it shows what the model said beside what it did",
|
|
"I'll write the skeleton first." in text and "Toolkit.ps1" in text)
|
|
check("it states plainly when a verb was never used",
|
|
"it never called click" in text, text[-300:])
|
|
check("and the screen verbs it DID use are not reported as missing",
|
|
"it never called screenshot" not in text)
|
|
|
|
# --- a log may never take a session down -----------------------------
|
|
log2 = sessionlog.open_log("x", os="ubuntu")
|
|
log2._handle.close() # the worst thing that can happen
|
|
log2.write("run", command="still fine") # must not raise
|
|
log2.close(outcome="finished") # nor this
|
|
check("a broken log is silent, not fatal", True)
|
|
|
|
sessionlog.LOG_DIR = "/var/lib/owuivm/sessions"
|
|
|
|
# --- the wiring, in the code that has to do it ---------------------------
|
|
vmd_src = open(os.path.join(VMD, "vmd.py"), encoding="utf-8").read()
|
|
check("every record goes to the log as well as to memory",
|
|
"STATE.log.write(kind, **fields)" in vmd_src,
|
|
"one hook, so a verb added later is logged without anyone remembering to")
|
|
check("the log opens with the session and carries the chat id",
|
|
"sessionlog.open_log(" in vmd_src and 'chat=str(args.get("chat_id"' in vmd_src)
|
|
check("and closes with what became of it",
|
|
"STATE.log.close(" in vmd_src and "outcome=" in vmd_src)
|
|
|
|
tree = ast.parse(vmd_src)
|
|
bodies = {n.name: ast.unparse(n) for n in ast.walk(tree)
|
|
if isinstance(n, ast.FunctionDef)}
|
|
verbs = {name for name in bodies if name.startswith("v_")}
|
|
# Directly, or through the one helper it delegates to - v_click and
|
|
# v_move are both `_pointer`, which is where the record lives.
|
|
records_directly = {name for name, body in bodies.items() if "_record(" in body}
|
|
|
|
|
|
def records(name: str) -> bool:
|
|
if name in records_directly:
|
|
return True
|
|
return any(f"{helper}(" in bodies[name]
|
|
for helper in records_directly if helper != name)
|
|
|
|
|
|
recorded = {name for name in verbs if records(name)}
|
|
# Verbs that legitimately record nothing: they change no state in the
|
|
# machine, or they ARE the record.
|
|
EXEMPT = {"v_session_status", "v_session_close", "v_session_open", "v_answer",
|
|
"v_report", "v_note", "v_reap", "v_snapshot_status", "v_session_cancel"}
|
|
missing = sorted(verbs - recorded - EXEMPT)
|
|
check("every verb that touches the machine is recorded", not missing,
|
|
f"{missing} would happen without leaving a trace")
|
|
|
|
check("the harness logs what the model said each turn",
|
|
'vmd.call("note"' in open(os.path.join(HERE, "..", "vmagent", "app", "loop.py"),
|
|
encoding="utf-8").read())
|
|
sys.path.insert(0, os.path.join(HERE, "..", "vmagent", "app"))
|
|
import tools as agent_tools # noqa: E402
|
|
|
|
offered = {s["function"]["name"] for s in agent_tools.schemas("ubuntu")} | \
|
|
{s["function"]["name"] for s in agent_tools.schemas("windows")}
|
|
check("and the model is never told that verb exists", "note" not in offered,
|
|
"it is the harness's record of the model, not a tool for it")
|
|
|
|
report_src = open(os.path.join(VMD, "report.py"), encoding="utf-8").read()
|
|
check("the tool result carries the counts too",
|
|
"Calls it made:" in report_src,
|
|
"so the reader sees the numbers without going to the log")
|
|
check("and says so when the screen was never looked at",
|
|
"It never looked at the screen" in report_src)
|
|
|
|
print()
|
|
if FAILS:
|
|
print(f"{len(FAILS)} failed")
|
|
for line in FAILS:
|
|
print(" -", line)
|
|
sys.exit(1)
|
|
print("what the agent did is written down, in order, and can be read back")
|