169 lines
7.5 KiB
Python
169 lines
7.5 KiB
Python
#!/usr/bin/env python3
|
|
"""The sub-agent's tool definitions, now that they live in a config file.
|
|
|
|
python3 tests/test_vm_agent_tool_config.py
|
|
|
|
Pure Python, no container and no network.
|
|
|
|
WHY THE GOLDEN FILE EXISTS
|
|
|
|
configs/agent/tools.toml replaced ~380 lines of hand-written Python
|
|
schemas. Every description in it is prompt text that took measurement to
|
|
arrive at, so the move had to be provably lossless rather than
|
|
approximately right - a dropped sentence in a tool description is not a
|
|
crash, it is a session that goes wrong later for no visible reason.
|
|
|
|
tests/golden/agent_tool_schemas.json is what the hand-written code
|
|
produced, captured before it was touched. If this test passes, the
|
|
config-driven builder is byte-identical to it for both machines and for
|
|
the booting case.
|
|
|
|
Regenerating the golden is therefore a deliberate act: do it only when
|
|
you MEANT to change what the agent is told, and read the diff.
|
|
"""
|
|
|
|
import importlib
|
|
import json
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
|
|
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
|
sys.path.insert(0, os.path.join(ROOT, "vmagent", "app"))
|
|
|
|
import tools # noqa: E402
|
|
|
|
GOLDEN = os.path.join(ROOT, "tests", "golden", "agent_tool_schemas.json")
|
|
FAILS: list[str] = []
|
|
|
|
|
|
def check(label: str, condition: bool, detail: str = "") -> None:
|
|
if condition:
|
|
print(f"ok {label}")
|
|
else:
|
|
FAILS.append(f"{label}: {detail}")
|
|
print(f"FAIL {label}\n {detail}")
|
|
|
|
|
|
def reload_with(**env):
|
|
"""Re-import tools.py against a different config, then put it back."""
|
|
saved = {k: os.environ.get(k) for k in env}
|
|
os.environ.update({k: v for k, v in env.items() if v is not None})
|
|
try:
|
|
return importlib.reload(tools)
|
|
finally:
|
|
for key, value in saved.items():
|
|
if value is None:
|
|
os.environ.pop(key, None)
|
|
else:
|
|
os.environ[key] = value
|
|
|
|
|
|
# --- the golden ------------------------------------------------------
|
|
with open(GOLDEN, encoding="utf-8") as handle:
|
|
golden = json.load(handle)
|
|
|
|
for key, want in golden.items():
|
|
got = tools.schemas("ubuntu", booting=True) if key == "booting" else tools.schemas(key)
|
|
check(f"{key}: identical to the hand-written schemas", got == want,
|
|
f"{len(got)} tools vs {len(want)}; "
|
|
f"first difference at "
|
|
f"{next((w['function']['name'] for g, w in zip(got, want) if g != w), 'length')}")
|
|
|
|
# --- the registry ----------------------------------------------------
|
|
registry = tools.machines()
|
|
check("configs/vms.toml defines both machines",
|
|
set(registry) == {"ubuntu", "windows"}, str(sorted(registry)))
|
|
check("every machine has the three prompt fields",
|
|
all(all(m.get(k) for k in ("shell", "project", "example")) for m in registry.values()),
|
|
str(registry))
|
|
|
|
# A placeholder that survives into a rendered schema is prompt text the
|
|
# model reads as literal "{project}", which is worse than either machine's
|
|
# real value. Cheap to assert, and invisible if it ever happens.
|
|
for os_name in ("ubuntu", "windows"):
|
|
blob = json.dumps(tools.schemas(os_name))
|
|
leftover = [p for p in ("{shell}", "{project}", "{example}") if p in blob]
|
|
check(f"{os_name}: no unsubstituted placeholders", not leftover, str(leftover))
|
|
|
|
check("the two machines differ in exactly the tools that mention the shell",
|
|
[t["function"]["name"]
|
|
for t, w in zip(tools.schemas("ubuntu"), tools.schemas("windows")) if t != w]
|
|
== ["run", "write_file"],
|
|
"only run and write_file should vary by machine")
|
|
|
|
# --- what the config may and may not do -------------------------------
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
# A name with no dispatch behind it must be refused when it is read,
|
|
# naming the file - not offered to the model and then failing.
|
|
with open(os.path.join(tmp, "tools.toml"), "w", encoding="utf-8") as handle:
|
|
handle.write('[[tool]]\nname = "unfreeze_base_image"\ngroup = "cli"\n'
|
|
'enabled = true\nrequired = []\ndescription = "no"\n')
|
|
refused = False
|
|
try:
|
|
reload_with(OWUITOOLS_AGENT_CONFIG_DIR=tmp,
|
|
OWUITOOLS_VMS_CONFIG=os.path.join(ROOT, "configs", "vms.toml"))
|
|
tools.schemas("ubuntu")
|
|
except tools.ToolConfigError as exc:
|
|
refused = "unfreeze_base_image" in str(exc) and "DISPATCHABLE" in str(exc)
|
|
except Exception: # noqa: BLE001
|
|
refused = False
|
|
check("config cannot operator the agent a verb that is not in DISPATCHABLE", refused,
|
|
"a name with no dispatch must raise ToolConfigError naming the file")
|
|
|
|
# enabled = false removes a tool.
|
|
with open(os.path.join(tmp, "tools.toml"), "w", encoding="utf-8") as handle:
|
|
handle.write('[[tool]]\nname = "run"\ngroup = "cli"\nenabled = false\n'
|
|
'required = []\ndescription = "x"\n'
|
|
'[[tool]]\nname = "screenshot"\ngroup = "gui"\nenabled = true\n'
|
|
'required = []\ndescription = "y"\n')
|
|
reload_with(OWUITOOLS_AGENT_CONFIG_DIR=tmp,
|
|
OWUITOOLS_VMS_CONFIG=os.path.join(ROOT, "configs", "vms.toml"))
|
|
names = [t["function"]["name"] for t in tools.schemas("ubuntu")]
|
|
check("enabled = false removes the tool", names == ["screenshot"], str(names))
|
|
|
|
# A lookup the SERVER has switched off must not be offered here: it
|
|
# would 404, which reads as a broken tool rather than an absent one.
|
|
with open(os.path.join(tmp, "tools.toml"), "w", encoding="utf-8") as handle:
|
|
handle.write('[[tool]]\nname = "search_papers"\ngroup = "research"\nenabled = true\n'
|
|
'required = []\ndescription = "x"\n'
|
|
'[[tool]]\nname = "read_page"\ngroup = "cli"\nenabled = true\n'
|
|
'required = []\ndescription = "y"\n'
|
|
'[[tool]]\nname = "check_internet"\ngroup = "cli"\nenabled = true\n'
|
|
'required = []\ndescription = "z"\n')
|
|
server = os.path.join(tmp, "server-tools.toml")
|
|
with open(server, "w", encoding="utf-8") as handle:
|
|
# fetch_page is what the agent's read_page is a call to.
|
|
handle.write('[tools]\nsearch_papers = false\nfetch_page = false\n')
|
|
reload_with(OWUITOOLS_AGENT_CONFIG_DIR=tmp,
|
|
OWUITOOLS_VMS_CONFIG=os.path.join(ROOT, "configs", "vms.toml"),
|
|
OWUITOOLS_TOOLS_CONFIG=server)
|
|
names = [t["function"]["name"] for t in tools.schemas("ubuntu")]
|
|
check("a lookup the server switched off is not offered to the agent",
|
|
names == ["check_internet"], str(names))
|
|
check("and read_page follows the server's fetch_page, not its own name",
|
|
"read_page" not in names, str(names))
|
|
|
|
reload_with(OWUITOOLS_AGENT_CONFIG_DIR=None)
|
|
|
|
# --- dispatch still lines up -----------------------------------------
|
|
check("every offered tool is dispatchable",
|
|
{t["function"]["name"] for t in tools.schemas("ubuntu")} <= tools.DISPATCHABLE,
|
|
"schemas() offered something call() cannot route")
|
|
# vmd's allowlist is the real gate - the agent asking for a lookup vmd
|
|
# does not list gets a refusal, not a result. The two must be one set.
|
|
sys.path.insert(0, os.path.join(ROOT, "vm", "vmd"))
|
|
import websearch # noqa: E402
|
|
|
|
check("the research group matches vmd's own allowlist",
|
|
tools.RESEARCH_NAMES == frozenset(websearch.RESEARCH_TOOLS),
|
|
f"agent={sorted(tools.RESEARCH_NAMES)} vmd={sorted(websearch.RESEARCH_TOOLS)}")
|
|
|
|
print()
|
|
if FAILS:
|
|
print(f"{len(FAILS)} FAILED")
|
|
for line in FAILS:
|
|
print(f" - {line}")
|
|
raise SystemExit(1)
|
|
print("the externalised tool definitions are faithful")
|