131 lines
5.7 KiB
Python
131 lines
5.7 KiB
Python
#!/usr/bin/env python3
|
|
"""The text the two models read must agree with the code they drive.
|
|
|
|
python3 tests/test_vm_prompt_consistency.py
|
|
|
|
WHY THIS EXISTS
|
|
|
|
Prompt text rots quietly. When the CLI/GUI split was removed, the code
|
|
changed in one afternoon and the prose kept saying "in either mode" for
|
|
a week. When the handover screenshot was removed, the agent's prompt went
|
|
on promising "you will be handed a picture of it". Neither was caught,
|
|
because nothing reads the prompts the way the model does - as fact.
|
|
|
|
So this reads them. Every claim checked here is one that was found wrong
|
|
in a real prompt, tool description or report, and each check is scoped
|
|
to the text the model actually sees, not to comments about it.
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
ROOT = os.path.join(HERE, "..")
|
|
sys.path.insert(0, os.path.join(ROOT, "vmagent", "app"))
|
|
|
|
import tools # noqa: E402
|
|
|
|
FAILS: list[str] = []
|
|
|
|
|
|
def check(label: str, condition: bool, detail: str = "") -> None:
|
|
if condition:
|
|
print(f"ok {label}")
|
|
else:
|
|
FAILS.append(f"{label}: {detail}")
|
|
print(f"FAIL {label}\n {detail}")
|
|
|
|
|
|
def read(*parts: str) -> str:
|
|
return open(os.path.join(ROOT, *parts), encoding="utf-8").read()
|
|
|
|
|
|
prompt = read("configs", "agent", "prompt.md")
|
|
screen = read("configs", "agent", "prompt.screen.md")
|
|
ubuntu = read("configs", "agent", "prompt.ubuntu.md")
|
|
windows = read("configs", "agent", "prompt.windows.md")
|
|
neutral = prompt + screen
|
|
agent_facing = neutral + ubuntu + windows
|
|
|
|
# What the sub-agent's model is actually sent: descriptions, not comments.
|
|
schemas = tools.schemas("ubuntu") + tools.schemas("windows")
|
|
descriptions = "\n".join(
|
|
s["function"]["description"] + " " +
|
|
" ".join(str(p.get("description", "")) for p in s["function"]["parameters"]["properties"].values())
|
|
for s in schemas)
|
|
tool_names = {s["function"]["name"] for s in tools.schemas("ubuntu")}
|
|
|
|
# The main model's side: its system prompt section and the tool docstring.
|
|
system_prompt = read("configs", "assistant", "system-prompt.md")
|
|
vm_src = read("app", "owuitools", "routers", "vm.py")
|
|
docstring = re.search(r'async def vm_agent\(.*?"""(.*?)"""', vm_src, re.S)
|
|
main_facing = system_prompt + (docstring.group(1) if docstring else "")
|
|
|
|
report_src = read("vm", "vmd", "report.py")
|
|
filter_src = read("owui-functions", "vm_agent_progress.py")
|
|
|
|
# --- there is one kind of session, and no text may say otherwise ---------
|
|
STALE = ("either mode", "CLI mode", "GUI mode", "cli mode", "gui mode",
|
|
"CLI session", "GUI session", "in a desktop session")
|
|
for where, text in (("agent prompts", agent_facing), ("tool descriptions", descriptions),
|
|
("main model text", main_facing), ("report", report_src)):
|
|
hits = [s for s in STALE if s in text]
|
|
check(f"no CLI/GUI mode language in the {where}", not hits, str(hits))
|
|
|
|
valves = re.findall(r'description="([^"]*)"', filter_src[:8000])
|
|
check("the filter's valve text does not describe modes that no longer exist",
|
|
not any(("CLI" in v or "GUI" in v) for v in valves), str(valves))
|
|
|
|
# --- promises that were once true ----------------------------------------
|
|
check("the prompt no longer promises a handover screenshot",
|
|
"handed a picture" not in prompt and "picture of it along" not in prompt,
|
|
"the handover screenshot was removed; the model must not wait for one")
|
|
|
|
# --- every tool the prompt names exists -----------------------------------
|
|
named = set(re.findall(r"`([a-z]+(?:_[a-z]+)+)`", agent_facing))
|
|
missing = sorted(n for n in named if n not in tool_names and n not in {
|
|
"py_compile", # python3 -m py_compile: a command, not a tool
|
|
"in_progress", # a task status, not a tool
|
|
})
|
|
check("every tool the prompts name is a tool the agent has", not missing, str(missing))
|
|
|
|
# --- machine paths stay in the machine's own section ----------------------
|
|
for path in ("/tmp", "/mnt/project", "Z:\\", "C:\\Windows", "apt", "winget"):
|
|
# Word-bounded, or "apt" matches "captured".
|
|
check(f"the machine-neutral prompt does not say {path!r}",
|
|
not re.search(r"(?<![A-Za-z])" + re.escape(path) + r"(?![A-Za-z])", neutral),
|
|
"a Windows session planned to 'check /mnt/project' because the text in front "
|
|
"of it mentioned the Linux path")
|
|
check("Ubuntu is told where scratch goes", "/tmp" in ubuntu)
|
|
check("Windows is told where scratch goes", "C:\\Windows\\Temp" in windows)
|
|
|
|
# --- the same thing is not said four times --------------------------------
|
|
graphical = sum(1 for t in (prompt, screen, ubuntu, windows)
|
|
if "nobody is looking" in t or "nobody can see" in t)
|
|
check("'a program started with run cannot be seen' is said once, not four times",
|
|
graphical <= 1, f"said in {graphical} files")
|
|
|
|
# --- the release and the link ----------------------------------------------
|
|
check("the agent is told scratch is filtered on commit",
|
|
"held back" in prompt or "held back" in descriptions,
|
|
"a filter nobody mentions is indistinguishable from a bug")
|
|
check("git_commit's description says what it will not commit",
|
|
"never reach the release" in descriptions)
|
|
check("the main model is told the release excludes scratch",
|
|
"scratch are never published" in system_prompt)
|
|
check("the main model is forbidden from inventing a link",
|
|
"Never assemble a URL" in system_prompt and "Never build a URL" in main_facing,
|
|
"one reply sent the user to a different project's release")
|
|
check("the report states a missing link as a fact",
|
|
"There is no link to give" in report_src,
|
|
"tool output is the channel the model treats as fact")
|
|
|
|
print()
|
|
if FAILS:
|
|
print(f"{len(FAILS)} failed")
|
|
for line in FAILS:
|
|
print(" -", line)
|
|
sys.exit(1)
|
|
print("all prompt-consistency checks passed")
|