#!/usr/bin/env python3 """The sub-agent's tool definitions, now that they live in a config file. python3 tests/test_vm_agent_tool_config.py Pure Python, no container and no network. WHY THE GOLDEN FILE EXISTS configs/agent/tools.toml replaced ~380 lines of hand-written Python schemas. Every description in it is prompt text that took measurement to arrive at, so the move had to be provably lossless rather than approximately right - a dropped sentence in a tool description is not a crash, it is a session that goes wrong later for no visible reason. tests/golden/agent_tool_schemas.json is what the hand-written code produced, captured before it was touched. If this test passes, the config-driven builder is byte-identical to it for both machines and for the booting case. Regenerating the golden is therefore a deliberate act: do it only when you MEANT to change what the agent is told, and read the diff. """ import importlib import json import os import sys import tempfile ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") sys.path.insert(0, os.path.join(ROOT, "vmagent", "app")) import tools # noqa: E402 GOLDEN = os.path.join(ROOT, "tests", "golden", "agent_tool_schemas.json") FAILS: list[str] = [] def check(label: str, condition: bool, detail: str = "") -> None: if condition: print(f"ok {label}") else: FAILS.append(f"{label}: {detail}") print(f"FAIL {label}\n {detail}") def reload_with(**env): """Re-import tools.py against a different config, then put it back.""" saved = {k: os.environ.get(k) for k in env} os.environ.update({k: v for k, v in env.items() if v is not None}) try: return importlib.reload(tools) finally: for key, value in saved.items(): if value is None: os.environ.pop(key, None) else: os.environ[key] = value # --- the golden ------------------------------------------------------ with open(GOLDEN, encoding="utf-8") as handle: golden = json.load(handle) for key, want in golden.items(): got = tools.schemas("ubuntu", booting=True) if key == "booting" else tools.schemas(key) check(f"{key}: identical to the hand-written schemas", got == want, f"{len(got)} tools vs {len(want)}; " f"first difference at " f"{next((w['function']['name'] for g, w in zip(got, want) if g != w), 'length')}") # --- the registry ---------------------------------------------------- registry = tools.machines() check("configs/vms.toml defines both machines", set(registry) == {"ubuntu", "windows"}, str(sorted(registry))) check("every machine has the three prompt fields", all(all(m.get(k) for k in ("shell", "project", "example")) for m in registry.values()), str(registry)) # A placeholder that survives into a rendered schema is prompt text the # model reads as literal "{project}", which is worse than either machine's # real value. Cheap to assert, and invisible if it ever happens. for os_name in ("ubuntu", "windows"): blob = json.dumps(tools.schemas(os_name)) leftover = [p for p in ("{shell}", "{project}", "{example}") if p in blob] check(f"{os_name}: no unsubstituted placeholders", not leftover, str(leftover)) check("the two machines differ in exactly the tools that mention the shell", [t["function"]["name"] for t, w in zip(tools.schemas("ubuntu"), tools.schemas("windows")) if t != w] == ["run", "write_file"], "only run and write_file should vary by machine") # --- what the config may and may not do ------------------------------- with tempfile.TemporaryDirectory() as tmp: # A name with no dispatch behind it must be refused when it is read, # naming the file - not offered to the model and then failing. with open(os.path.join(tmp, "tools.toml"), "w", encoding="utf-8") as handle: handle.write('[[tool]]\nname = "unfreeze_base_image"\ngroup = "cli"\n' 'enabled = true\nrequired = []\ndescription = "no"\n') refused = False try: reload_with(OWUITOOLS_AGENT_CONFIG_DIR=tmp, OWUITOOLS_VMS_CONFIG=os.path.join(ROOT, "configs", "vms.toml")) tools.schemas("ubuntu") except tools.ToolConfigError as exc: refused = "unfreeze_base_image" in str(exc) and "DISPATCHABLE" in str(exc) except Exception: # noqa: BLE001 refused = False check("config cannot operator the agent a verb that is not in DISPATCHABLE", refused, "a name with no dispatch must raise ToolConfigError naming the file") # enabled = false removes a tool. with open(os.path.join(tmp, "tools.toml"), "w", encoding="utf-8") as handle: handle.write('[[tool]]\nname = "run"\ngroup = "cli"\nenabled = false\n' 'required = []\ndescription = "x"\n' '[[tool]]\nname = "screenshot"\ngroup = "gui"\nenabled = true\n' 'required = []\ndescription = "y"\n') reload_with(OWUITOOLS_AGENT_CONFIG_DIR=tmp, OWUITOOLS_VMS_CONFIG=os.path.join(ROOT, "configs", "vms.toml")) names = [t["function"]["name"] for t in tools.schemas("ubuntu")] check("enabled = false removes the tool", names == ["screenshot"], str(names)) # A lookup the SERVER has switched off must not be offered here: it # would 404, which reads as a broken tool rather than an absent one. with open(os.path.join(tmp, "tools.toml"), "w", encoding="utf-8") as handle: handle.write('[[tool]]\nname = "search_papers"\ngroup = "research"\nenabled = true\n' 'required = []\ndescription = "x"\n' '[[tool]]\nname = "read_page"\ngroup = "cli"\nenabled = true\n' 'required = []\ndescription = "y"\n' '[[tool]]\nname = "check_internet"\ngroup = "cli"\nenabled = true\n' 'required = []\ndescription = "z"\n') server = os.path.join(tmp, "server-tools.toml") with open(server, "w", encoding="utf-8") as handle: # fetch_page is what the agent's read_page is a call to. handle.write('[tools]\nsearch_papers = false\nfetch_page = false\n') reload_with(OWUITOOLS_AGENT_CONFIG_DIR=tmp, OWUITOOLS_VMS_CONFIG=os.path.join(ROOT, "configs", "vms.toml"), OWUITOOLS_TOOLS_CONFIG=server) names = [t["function"]["name"] for t in tools.schemas("ubuntu")] check("a lookup the server switched off is not offered to the agent", names == ["check_internet"], str(names)) check("and read_page follows the server's fetch_page, not its own name", "read_page" not in names, str(names)) reload_with(OWUITOOLS_AGENT_CONFIG_DIR=None) # --- dispatch still lines up ----------------------------------------- check("every offered tool is dispatchable", {t["function"]["name"] for t in tools.schemas("ubuntu")} <= tools.DISPATCHABLE, "schemas() offered something call() cannot route") # vmd's allowlist is the real gate - the agent asking for a lookup vmd # does not list gets a refusal, not a result. The two must be one set. sys.path.insert(0, os.path.join(ROOT, "vm", "vmd")) import websearch # noqa: E402 check("the research group matches vmd's own allowlist", tools.RESEARCH_NAMES == frozenset(websearch.RESEARCH_TOOLS), f"agent={sorted(tools.RESEARCH_NAMES)} vmd={sorted(websearch.RESEARCH_TOOLS)}") print() if FAILS: print(f"{len(FAILS)} FAILED") for line in FAILS: print(f" - {line}") raise SystemExit(1) print("the externalised tool definitions are faithful")