Initial Commit

tests/test_vm_prompt_consistency.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""The text the two models read must agree with the code they drive.
+
+    python3 tests/test_vm_prompt_consistency.py
+
+WHY THIS EXISTS
+
+Prompt text rots quietly. When the CLI/GUI split was removed, the code
+changed in one afternoon and the prose kept saying "in either mode" for
+a week. When the handover screenshot was removed, the agent's prompt went
+on promising "you will be handed a picture of it". Neither was caught,
+because nothing reads the prompts the way the model does - as fact.
+
+So this reads them. Every claim checked here is one that was found wrong
+in a real prompt, tool description or report, and each check is scoped
+to the text the model actually sees, not to comments about it.
+"""
+
+import os
+import re
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+ROOT = os.path.join(HERE, "..")
+sys.path.insert(0, os.path.join(ROOT, "vmagent", "app"))
+
+import tools  # noqa: E402
+
+FAILS: list[str] = []
+
+
+def check(label: str, condition: bool, detail: str = "") -> None:
+    if condition:
+        print(f"ok    {label}")
+    else:
+        FAILS.append(f"{label}: {detail}")
+        print(f"FAIL  {label}\n      {detail}")
+
+
+def read(*parts: str) -> str:
+    return open(os.path.join(ROOT, *parts), encoding="utf-8").read()
+
+
+prompt = read("configs", "agent", "prompt.md")
+screen = read("configs", "agent", "prompt.screen.md")
+ubuntu = read("configs", "agent", "prompt.ubuntu.md")
+windows = read("configs", "agent", "prompt.windows.md")
+neutral = prompt + screen
+agent_facing = neutral + ubuntu + windows
+
+# What the sub-agent's model is actually sent: descriptions, not comments.
+schemas = tools.schemas("ubuntu") + tools.schemas("windows")
+descriptions = "\n".join(
+    s["function"]["description"] + " " +
+    " ".join(str(p.get("description", "")) for p in s["function"]["parameters"]["properties"].values())
+    for s in schemas)
+tool_names = {s["function"]["name"] for s in tools.schemas("ubuntu")}
+
+# The main model's side: its system prompt section and the tool docstring.
+system_prompt = read("configs", "assistant", "system-prompt.md")
+vm_src = read("app", "owuitools", "routers", "vm.py")
+docstring = re.search(r'async def vm_agent\(.*?"""(.*?)"""', vm_src, re.S)
+main_facing = system_prompt + (docstring.group(1) if docstring else "")
+
+report_src = read("vm", "vmd", "report.py")
+filter_src = read("owui-functions", "vm_agent_progress.py")
+
+# --- there is one kind of session, and no text may say otherwise ---------
+STALE = ("either mode", "CLI mode", "GUI mode", "cli mode", "gui mode",
+         "CLI session", "GUI session", "in a desktop session")
+for where, text in (("agent prompts", agent_facing), ("tool descriptions", descriptions),
+                    ("main model text", main_facing), ("report", report_src)):
+    hits = [s for s in STALE if s in text]
+    check(f"no CLI/GUI mode language in the {where}", not hits, str(hits))
+
+valves = re.findall(r'description="([^"]*)"', filter_src[:8000])
+check("the filter's valve text does not describe modes that no longer exist",
+      not any(("CLI" in v or "GUI" in v) for v in valves), str(valves))
+
+# --- promises that were once true ----------------------------------------
+check("the prompt no longer promises a handover screenshot",
+      "handed a picture" not in prompt and "picture of it along" not in prompt,
+      "the handover screenshot was removed; the model must not wait for one")
+
+# --- every tool the prompt names exists -----------------------------------
+named = set(re.findall(r"`([a-z]+(?:_[a-z]+)+)`", agent_facing))
+missing = sorted(n for n in named if n not in tool_names and n not in {
+    "py_compile",   # python3 -m py_compile: a command, not a tool
+    "in_progress",  # a task status, not a tool
+})
+check("every tool the prompts name is a tool the agent has", not missing, str(missing))
+
+# --- machine paths stay in the machine's own section ----------------------
+for path in ("/tmp", "/mnt/project", "Z:\\", "C:\\Windows", "apt", "winget"):
+    # Word-bounded, or "apt" matches "captured".
+    check(f"the machine-neutral prompt does not say {path!r}",
+          not re.search(r"(?<![A-Za-z])" + re.escape(path) + r"(?![A-Za-z])", neutral),
+          "a Windows session planned to 'check /mnt/project' because the text in front "
+          "of it mentioned the Linux path")
+check("Ubuntu is told where scratch goes", "/tmp" in ubuntu)
+check("Windows is told where scratch goes", "C:\\Windows\\Temp" in windows)
+
+# --- the same thing is not said four times --------------------------------
+graphical = sum(1 for t in (prompt, screen, ubuntu, windows)
+                if "nobody is looking" in t or "nobody can see" in t)
+check("'a program started with run cannot be seen' is said once, not four times",
+      graphical <= 1, f"said in {graphical} files")
+
+# --- the release and the link ----------------------------------------------
+check("the agent is told scratch is filtered on commit",
+      "held back" in prompt or "held back" in descriptions,
+      "a filter nobody mentions is indistinguishable from a bug")
+check("git_commit's description says what it will not commit",
+      "never reach the release" in descriptions)
+check("the main model is told the release excludes scratch",
+      "scratch are never published" in system_prompt)
+check("the main model is forbidden from inventing a link",
+      "Never assemble a URL" in system_prompt and "Never build a URL" in main_facing,
+      "one reply sent the user to a different project's release")
+check("the report states a missing link as a fact",
+      "There is no link to give" in report_src,
+      "tool output is the channel the model treats as fact")
+
+print()
+if FAILS:
+    print(f"{len(FAILS)} failed")
+    for line in FAILS:
+        print("  -", line)
+    sys.exit(1)
+print("all prompt-consistency checks passed")