Grant/owui-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial Commit

Browse Source
This commit is contained in:
Grant
2026-09-15 15:24:03 -04:00
commit 4dc7086e7e
605 changed files with 101380 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_vm_session_log.py
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env python3
"""What the agent did is written down, and can be read back.
python3 tests/test_vm_session_log.py
WHY
Asked "did it ever open the app and look at it", the honest answer was
that the record could not say. Everything needed was ALREADY being
recorded - `_record()` has covered screenshot, click, key and type since
they were written - and all of it was discarded when the session ended.
What was left to read were the two records that cannot answer it:
* the tool result counted `run` and nothing else, so a session that
marked "clicked through the GUI" done without ever calling `click`
read exactly like one that did;
* Open WebUI's status history is SAMPLED every two seconds, so a verb
that returns fast is usually overwritten before it is ever seen.
Measured on a real session: 456 samples caught `launch_app` nine
times because it blocks for seconds, and caught not one of the
screenshots that demonstrably happened.
So the transcript is now written to disk as it happens, one file per
session, with the model's own words between the calls - and the tool
result carries the counts, so a claim that disagrees with them is
visible without opening anything.
"""
import ast
import json
import os
import sys
import tempfile
HERE = os.path.dirname(os.path.abspath(__file__))
VMD = os.path.join(HERE, "..", "vm", "vmd")
sys.path.insert(0, VMD)
FAILS: list[str] = []
def check(label: str, condition: bool, detail: str = "") -> None:
if condition:
print(f"ok {label}")
else:
FAILS.append(f"{label}: {detail}")
print(f"FAIL {label}\n {detail}")
with tempfile.TemporaryDirectory() as tmp:
os.environ["OWUITOOLS_VM_LOG_DIR"] = tmp
import sessionlog
sessionlog.LOG_DIR = tmp
log = sessionlog.open_log("abc12345deadbeef", chat="chat-1", os="windows",
project="toolkit", owner="someone", task="build a GUI")
check("a log is opened for the session", log is not None)
log.write("plan", count=3)
log.write("note", turn=1, text="I'll write the skeleton first.")
log.write("write_file", path="Z:\\Toolkit.ps1", bytes=4821)
log.write("run", command="powershell -File Z:\\Toolkit.ps1", exit_code=0, seconds=2.1)
log.write("launch_app", path="Toolkit.ps1")
log.write("screenshot", width=1280, height=800, bytes=86000)
log.write("task", id=1, status="done")
log.close(outcome="finished", verified=True, commits=1)
files = [n for n in os.listdir(tmp) if n.endswith(".jsonl")]
check("it is one file per session", len(files) == 1, str(files))
check("named so the newest sorts last", files[0][:2] == "20" and "abc12345" in files[0],
files[0])
records = sessionlog.load(os.path.join(tmp, files[0]))
kinds = [r["kind"] for r in records]
check("it opens with the session and closes with the outcome",
kinds[0] == "start" and kinds[-1] == "end", str(kinds))
check("the chat is recorded, so a chat maps to a session",
records[0].get("chat") == "chat-1",
"without it the only way to tell two runs apart is guessing at timestamps")
check("every call is in order", kinds[1:-1] ==
["plan", "note", "write_file", "run", "launch_app", "screenshot", "task"],
str(kinds))
check("each one is stamped with how far into the session it happened",
all("at" in r for r in records))
# --- the census is the thing that answers the question ---------------
counts = sessionlog.census(records)
check("the census counts the calls, not the talk",
counts.get("screenshot") == 1 and counts.get("launch_app") == 1
and "note" not in counts, str(counts))
check("and a verb never called simply is not there", "click" not in counts, str(counts))
text = sessionlog.render(os.path.join(tmp, files[0]))
check("the rendering names the session and the chat",
"abc12345deadbeef" in text and "chat-1" in text)
check("it shows what the model said beside what it did",
"I'll write the skeleton first." in text and "Toolkit.ps1" in text)
check("it states plainly when a verb was never used",
"it never called click" in text, text[-300:])
check("and the screen verbs it DID use are not reported as missing",
"it never called screenshot" not in text)
# --- a log may never take a session down -----------------------------
log2 = sessionlog.open_log("x", os="ubuntu")
log2._handle.close() # the worst thing that can happen
log2.write("run", command="still fine") # must not raise
log2.close(outcome="finished") # nor this
check("a broken log is silent, not fatal", True)
sessionlog.LOG_DIR = "/var/lib/owuivm/sessions"
# --- the wiring, in the code that has to do it ---------------------------
vmd_src = open(os.path.join(VMD, "vmd.py"), encoding="utf-8").read()
check("every record goes to the log as well as to memory",
"STATE.log.write(kind, **fields)" in vmd_src,
"one hook, so a verb added later is logged without anyone remembering to")
check("the log opens with the session and carries the chat id",
"sessionlog.open_log(" in vmd_src and 'chat=str(args.get("chat_id"' in vmd_src)
check("and closes with what became of it",
"STATE.log.close(" in vmd_src and "outcome=" in vmd_src)
tree = ast.parse(vmd_src)
bodies = {n.name: ast.unparse(n) for n in ast.walk(tree)
if isinstance(n, ast.FunctionDef)}
verbs = {name for name in bodies if name.startswith("v_")}
# Directly, or through the one helper it delegates to - v_click and
# v_move are both `_pointer`, which is where the record lives.
records_directly = {name for name, body in bodies.items() if "_record(" in body}
def records(name: str) -> bool:
if name in records_directly:
return True
return any(f"{helper}(" in bodies[name]
for helper in records_directly if helper != name)
recorded = {name for name in verbs if records(name)}
# Verbs that legitimately record nothing: they change no state in the
# machine, or they ARE the record.
EXEMPT = {"v_session_status", "v_session_close", "v_session_open", "v_answer",
"v_report", "v_note", "v_reap", "v_snapshot_status", "v_session_cancel"}
missing = sorted(verbs - recorded - EXEMPT)
check("every verb that touches the machine is recorded", not missing,
f"{missing} would happen without leaving a trace")
check("the harness logs what the model said each turn",
'vmd.call("note"' in open(os.path.join(HERE, "..", "vmagent", "app", "loop.py"),
encoding="utf-8").read())
sys.path.insert(0, os.path.join(HERE, "..", "vmagent", "app"))
import tools as agent_tools # noqa: E402
offered = {s["function"]["name"] for s in agent_tools.schemas("ubuntu")} | \
{s["function"]["name"] for s in agent_tools.schemas("windows")}
check("and the model is never told that verb exists", "note" not in offered,
"it is the harness's record of the model, not a tool for it")
report_src = open(os.path.join(VMD, "report.py"), encoding="utf-8").read()
check("the tool result carries the counts too",
"Calls it made:" in report_src,
"so the reader sees the numbers without going to the log")
check("and says so when the screen was never looked at",
"It never looked at the screen" in report_src)
print()
if FAILS:
print(f"{len(FAILS)} failed")
for line in FAILS:
print(" -", line)
sys.exit(1)
print("what the agent did is written down, in order, and can be read back")