Initial Commit
This commit is contained in:
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The sub-agent's tool definitions, now that they live in a config file.
|
||||
|
||||
python3 tests/test_vm_agent_tool_config.py
|
||||
|
||||
Pure Python, no container and no network.
|
||||
|
||||
WHY THE GOLDEN FILE EXISTS
|
||||
|
||||
configs/agent/tools.toml replaced ~380 lines of hand-written Python
|
||||
schemas. Every description in it is prompt text that took measurement to
|
||||
arrive at, so the move had to be provably lossless rather than
|
||||
approximately right - a dropped sentence in a tool description is not a
|
||||
crash, it is a session that goes wrong later for no visible reason.
|
||||
|
||||
tests/golden/agent_tool_schemas.json is what the hand-written code
|
||||
produced, captured before it was touched. If this test passes, the
|
||||
config-driven builder is byte-identical to it for both machines and for
|
||||
the booting case.
|
||||
|
||||
Regenerating the golden is therefore a deliberate act: do it only when
|
||||
you MEANT to change what the agent is told, and read the diff.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
||||
sys.path.insert(0, os.path.join(ROOT, "vmagent", "app"))
|
||||
|
||||
import tools # noqa: E402
|
||||
|
||||
GOLDEN = os.path.join(ROOT, "tests", "golden", "agent_tool_schemas.json")
|
||||
FAILS: list[str] = []
|
||||
|
||||
|
||||
def check(label: str, condition: bool, detail: str = "") -> None:
|
||||
if condition:
|
||||
print(f"ok {label}")
|
||||
else:
|
||||
FAILS.append(f"{label}: {detail}")
|
||||
print(f"FAIL {label}\n {detail}")
|
||||
|
||||
|
||||
def reload_with(**env):
|
||||
"""Re-import tools.py against a different config, then put it back."""
|
||||
saved = {k: os.environ.get(k) for k in env}
|
||||
os.environ.update({k: v for k, v in env.items() if v is not None})
|
||||
try:
|
||||
return importlib.reload(tools)
|
||||
finally:
|
||||
for key, value in saved.items():
|
||||
if value is None:
|
||||
os.environ.pop(key, None)
|
||||
else:
|
||||
os.environ[key] = value
|
||||
|
||||
|
||||
# --- the golden ------------------------------------------------------
|
||||
with open(GOLDEN, encoding="utf-8") as handle:
|
||||
golden = json.load(handle)
|
||||
|
||||
for key, want in golden.items():
|
||||
got = tools.schemas("ubuntu", booting=True) if key == "booting" else tools.schemas(key)
|
||||
check(f"{key}: identical to the hand-written schemas", got == want,
|
||||
f"{len(got)} tools vs {len(want)}; "
|
||||
f"first difference at "
|
||||
f"{next((w['function']['name'] for g, w in zip(got, want) if g != w), 'length')}")
|
||||
|
||||
# --- the registry ----------------------------------------------------
|
||||
registry = tools.machines()
|
||||
check("configs/vms.toml defines both machines",
|
||||
set(registry) == {"ubuntu", "windows"}, str(sorted(registry)))
|
||||
check("every machine has the three prompt fields",
|
||||
all(all(m.get(k) for k in ("shell", "project", "example")) for m in registry.values()),
|
||||
str(registry))
|
||||
|
||||
# A placeholder that survives into a rendered schema is prompt text the
|
||||
# model reads as literal "{project}", which is worse than either machine's
|
||||
# real value. Cheap to assert, and invisible if it ever happens.
|
||||
for os_name in ("ubuntu", "windows"):
|
||||
blob = json.dumps(tools.schemas(os_name))
|
||||
leftover = [p for p in ("{shell}", "{project}", "{example}") if p in blob]
|
||||
check(f"{os_name}: no unsubstituted placeholders", not leftover, str(leftover))
|
||||
|
||||
check("the two machines differ in exactly the tools that mention the shell",
|
||||
[t["function"]["name"]
|
||||
for t, w in zip(tools.schemas("ubuntu"), tools.schemas("windows")) if t != w]
|
||||
== ["run", "write_file"],
|
||||
"only run and write_file should vary by machine")
|
||||
|
||||
# --- what the config may and may not do -------------------------------
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
# A name with no dispatch behind it must be refused when it is read,
|
||||
# naming the file - not offered to the model and then failing.
|
||||
with open(os.path.join(tmp, "tools.toml"), "w", encoding="utf-8") as handle:
|
||||
handle.write('[[tool]]\nname = "unfreeze_base_image"\ngroup = "cli"\n'
|
||||
'enabled = true\nrequired = []\ndescription = "no"\n')
|
||||
refused = False
|
||||
try:
|
||||
reload_with(OWUITOOLS_AGENT_CONFIG_DIR=tmp,
|
||||
OWUITOOLS_VMS_CONFIG=os.path.join(ROOT, "configs", "vms.toml"))
|
||||
tools.schemas("ubuntu")
|
||||
except tools.ToolConfigError as exc:
|
||||
refused = "unfreeze_base_image" in str(exc) and "DISPATCHABLE" in str(exc)
|
||||
except Exception: # noqa: BLE001
|
||||
refused = False
|
||||
check("config cannot operator the agent a verb that is not in DISPATCHABLE", refused,
|
||||
"a name with no dispatch must raise ToolConfigError naming the file")
|
||||
|
||||
# enabled = false removes a tool.
|
||||
with open(os.path.join(tmp, "tools.toml"), "w", encoding="utf-8") as handle:
|
||||
handle.write('[[tool]]\nname = "run"\ngroup = "cli"\nenabled = false\n'
|
||||
'required = []\ndescription = "x"\n'
|
||||
'[[tool]]\nname = "screenshot"\ngroup = "gui"\nenabled = true\n'
|
||||
'required = []\ndescription = "y"\n')
|
||||
reload_with(OWUITOOLS_AGENT_CONFIG_DIR=tmp,
|
||||
OWUITOOLS_VMS_CONFIG=os.path.join(ROOT, "configs", "vms.toml"))
|
||||
names = [t["function"]["name"] for t in tools.schemas("ubuntu")]
|
||||
check("enabled = false removes the tool", names == ["screenshot"], str(names))
|
||||
|
||||
# A lookup the SERVER has switched off must not be offered here: it
|
||||
# would 404, which reads as a broken tool rather than an absent one.
|
||||
with open(os.path.join(tmp, "tools.toml"), "w", encoding="utf-8") as handle:
|
||||
handle.write('[[tool]]\nname = "search_papers"\ngroup = "research"\nenabled = true\n'
|
||||
'required = []\ndescription = "x"\n'
|
||||
'[[tool]]\nname = "read_page"\ngroup = "cli"\nenabled = true\n'
|
||||
'required = []\ndescription = "y"\n'
|
||||
'[[tool]]\nname = "check_internet"\ngroup = "cli"\nenabled = true\n'
|
||||
'required = []\ndescription = "z"\n')
|
||||
server = os.path.join(tmp, "server-tools.toml")
|
||||
with open(server, "w", encoding="utf-8") as handle:
|
||||
# fetch_page is what the agent's read_page is a call to.
|
||||
handle.write('[tools]\nsearch_papers = false\nfetch_page = false\n')
|
||||
reload_with(OWUITOOLS_AGENT_CONFIG_DIR=tmp,
|
||||
OWUITOOLS_VMS_CONFIG=os.path.join(ROOT, "configs", "vms.toml"),
|
||||
OWUITOOLS_TOOLS_CONFIG=server)
|
||||
names = [t["function"]["name"] for t in tools.schemas("ubuntu")]
|
||||
check("a lookup the server switched off is not offered to the agent",
|
||||
names == ["check_internet"], str(names))
|
||||
check("and read_page follows the server's fetch_page, not its own name",
|
||||
"read_page" not in names, str(names))
|
||||
|
||||
reload_with(OWUITOOLS_AGENT_CONFIG_DIR=None)
|
||||
|
||||
# --- dispatch still lines up -----------------------------------------
|
||||
check("every offered tool is dispatchable",
|
||||
{t["function"]["name"] for t in tools.schemas("ubuntu")} <= tools.DISPATCHABLE,
|
||||
"schemas() offered something call() cannot route")
|
||||
# vmd's allowlist is the real gate - the agent asking for a lookup vmd
|
||||
# does not list gets a refusal, not a result. The two must be one set.
|
||||
sys.path.insert(0, os.path.join(ROOT, "vm", "vmd"))
|
||||
import websearch # noqa: E402
|
||||
|
||||
check("the research group matches vmd's own allowlist",
|
||||
tools.RESEARCH_NAMES == frozenset(websearch.RESEARCH_TOOLS),
|
||||
f"agent={sorted(tools.RESEARCH_NAMES)} vmd={sorted(websearch.RESEARCH_TOOLS)}")
|
||||
|
||||
print()
|
||||
if FAILS:
|
||||
print(f"{len(FAILS)} FAILED")
|
||||
for line in FAILS:
|
||||
print(f" - {line}")
|
||||
raise SystemExit(1)
|
||||
print("the externalised tool definitions are faithful")
|
||||
Reference in New Issue
Block a user