Grant/owui-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial Commit

Browse Source
This commit is contained in:
Grant
2026-09-15 15:24:03 -04:00
commit 4dc7086e7e
605 changed files with 101380 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vmagent/app/tools.py
+363
View File
@@ -0,0 +1,363 @@
"""The sub-agent's tools: the schemas it reads, and the dispatch to vmd.
WHERE THE DEFINITIONS LIVE
In configs/agent/tools.toml, not here. Every description is prompt text
the model pays for on every turn, and the wording of a tool description
is the single most effective lever on how a tool gets used - so it
belongs in a file you can edit and restart, rather than one you have to
rebuild an image to change.
What stays in code is everything that is behaviour rather than wording:
which vmd verb a name reaches (`call`, below), which results are worth
keeping verbatim (`TERSE`), and DISPATCHABLE - the allowlist of verbs the
agent is allowed to reach at all. That last one is why the config file
cannot widen the agent's reach: a name in the TOML with nothing in
DISPATCHABLE is refused when it is read, loudly, rather than offered to
the model and then failing somewhere less obvious.
Two things are deliberately NOT tools:
* anything that takes a raw QMP command, a host path, or a network
setting. vmd exposes typed verbs; this is where that decision is
visible.
* anything the machine can answer about itself. DNS, HTTP headers and
TLS certificates are one curl, dig or openssl away inside a machine
that has internet, and running them THERE is what the machine is
for. The lookup suite - web_search, read_page and the research
group - does exist, on the host, because reading documentation off a
screen costs a screenshot per scroll and reasoning it out from first
principles has cost whole sessions.
"""
from __future__ import annotations
import json
import logging
import os
import tomllib
from typing import Any
log = logging.getLogger(__name__)
_HERE = os.path.dirname(os.path.abspath(__file__))
# In the container configs/agent is bind-mounted at /configs/agent and the
# env var says so. Outside it - the tests, and anyone reading the repo -
# fall back to the tree's own copy, so nothing needs an environment set up
# to import this module.
CONFIG_DIR = (os.environ.get("OWUITOOLS_AGENT_CONFIG_DIR")
or os.path.join(_HERE, "..", "..", "configs", "agent"))
VMS_CONFIG = os.environ.get("OWUITOOLS_VMS_CONFIG",
os.path.join(CONFIG_DIR, "..", "vms.toml"))
# The SERVER's tool switch, not this file's. See _server_disabled().
SERVER_TOOLS_CONFIG = os.environ.get("OWUITOOLS_TOOLS_CONFIG",
os.path.join(CONFIG_DIR, "..", "tools.toml"))
# ---------------------------------------------------------------------
# what the agent is allowed to reach
# ---------------------------------------------------------------------
# The allowlist, and the reason it is a list and not a comment: the tool
# surface is a config file now, and vmd has verbs the agent must never
# call - closing a session, saving a machine, unfreezing a base image.
# vmd would refuse them on its own authority, but a refusal at read time
# names the file and the line, and a refusal at run time looks like a
# broken tool. Adding a name here is a deliberate act.
DISPATCHABLE = frozenset({
# planning
"plan_tasks", "update_task", "add_task",
# the machine
"run", "write_file", "append_file", "edit_file", "read_file",
"web_search", "read_page", "list_dir", "check_internet", "git_commit",
"reboot_vm", "ask_operator", "request_unfreeze", "wait", "finish",
# the lookup suite, dispatched as one `research` verb
"search_stackexchange", "read_man_page", "lookup_package", "wikipedia_lookup",
"lookup_rfc", "lookup_cve", "search_papers", "find_datasheet",
# the screen
"open_page", "launch_app", "screenshot", "click", "move", "key", "type", "scroll",
})
# Agent tool -> the tool server operation it is ultimately a call to.
# Only the proxies appear here; everything else runs in the machine.
# Used to keep this file honest about configs/tools.toml: a lookup the
# server has been told not to offer must not be offered here either, or
# the agent gets a tool that 404s, which reads as broken rather than
# absent. Note read_page is the server's fetch_page under another name.
SERVER_TOOL = {
"web_search": "web_search",
"read_page": "fetch_page",
"search_stackexchange": "search_stackexchange",
"read_man_page": "read_man_page",
"lookup_package": "lookup_package",
"wikipedia_lookup": "wikipedia_lookup",
"lookup_rfc": "lookup_rfc",
"lookup_cve": "lookup_cve",
"search_papers": "search_papers",
"find_datasheet": "find_datasheet",
}
# The order the groups are offered in, and it is not arbitrary: planning
# first because plan_tasks must be the first call, then the command line
# it spends most of its turns in, then the lookup suite, then the screen.
GROUP_ORDER = ("task", "cli", "research", "gui")
# What is offered while the machine is still booting. Only the planning
# verbs: there is nothing to run yet, and offering tools that would be
# refused teaches the model to retry rather than to wait.
BOOTING_GROUPS = ("task",)
class ToolConfigError(RuntimeError):
"""The tool definitions could not be read. Phrased for a human."""
# ---------------------------------------------------------------------
# loading
# ---------------------------------------------------------------------
_machines_cache: dict[str, dict[str, str]] | None = None
_entries_cache: list[dict[str, Any]] | None = None
_schema_cache: dict[tuple[str, bool], list[dict[str, Any]]] = {}
def _read(path: str) -> dict:
try:
with open(path, "rb") as handle:
return tomllib.load(handle)
except FileNotFoundError as exc:
raise ToolConfigError(
f"{path} is not there. It is bind-mounted from configs/ by "
"docker-compose.yml; without it the agent has no tools at all."
) from exc
except (OSError, tomllib.TOMLDecodeError) as exc:
raise ToolConfigError(f"{path} could not be read: {exc}") from exc
def machines() -> dict[str, dict[str, str]]:
"""The machine registry, keyed by id."""
global _machines_cache
if _machines_cache is None:
found = {}
for entry in _read(VMS_CONFIG).get("machine", []):
machine_id = str(entry.get("id", "")).strip()
if not machine_id:
raise ToolConfigError(f"{VMS_CONFIG} has a machine with no id")
found[machine_id] = {key: str(entry.get(key, "")) for key in
("name", "iso", "shell", "project", "example")}
if not found:
raise ToolConfigError(f"{VMS_CONFIG} defines no machines")
_machines_cache = found
return _machines_cache
def _server_disabled() -> set[str]:
"""Tool-server operations this deployment has switched off.
Read leniently and on purpose: this file's job is to avoid offering a
lookup that would 404, and if the server's config cannot be read the
right guess is that everything is on - which is what it ships as.
"""
try:
with open(SERVER_TOOLS_CONFIG, "rb") as handle:
table = tomllib.load(handle).get("tools", {})
except (OSError, tomllib.TOMLDecodeError):
return set()
return {str(name) for name, on in table.items() if not on}
def _entries() -> list[dict[str, Any]]:
"""Every tool in the config file, validated, in file order."""
global _entries_cache
if _entries_cache is not None:
return _entries_cache
path = os.path.join(CONFIG_DIR, "tools.toml")
raw = _read(path).get("tool", [])
if not raw:
raise ToolConfigError(f"{path} defines no tools")
seen: set[str] = set()
entries: list[dict[str, Any]] = []
for entry in raw:
name = str(entry.get("name", "")).strip()
if not name:
raise ToolConfigError(f"{path} has a tool with no name")
if name in seen:
raise ToolConfigError(f"{path} defines '{name}' twice")
seen.add(name)
if name not in DISPATCHABLE:
raise ToolConfigError(
f"{path} offers '{name}', which is not a verb the agent may call. "
f"Adding one means adding it to DISPATCHABLE in {__file__} and "
"implementing it in vmd - it is not something config can operator."
)
group = str(entry.get("group", "")).strip()
if group not in GROUP_ORDER:
raise ToolConfigError(
f"{path}: '{name}' has group '{group}'; expected one of {GROUP_ORDER}")
entries.append(entry)
missing = sorted(DISPATCHABLE - seen)
if missing:
# Not fatal: a verb with no entry is simply not offered, which is a
# legitimate thing to want. Worth a line, because the other reading
# is that someone deleted a tool by accident.
log.info("%d dispatchable verb(s) are not offered by %s: %s",
len(missing), path, ", ".join(missing))
_entries_cache = entries
return entries
def _render(entry: dict[str, Any], machine: dict[str, str]) -> dict[str, Any]:
"""One config entry as an Ollama tool schema, for one machine.
Substitution is done over the serialised form rather than field by
field, because the placeholders appear in parameter descriptions as
well as in the tool's own. Measured against the hand-written schemas
this replaced: byte-identical for both machines.
"""
schema = {
"type": "function",
"function": {
"name": entry["name"],
"description": str(entry.get("description", "")).strip(),
"parameters": {
"type": "object",
"properties": entry.get("parameters", {}),
"required": list(entry.get("required", [])),
},
},
}
text = json.dumps(schema)
for key in ("shell", "project", "example"):
text = text.replace("{%s}" % key, json.dumps(machine.get(key, ""))[1:-1])
return json.loads(text)
def schemas(os_name: str, booting: bool = False) -> list[dict[str, Any]]:
"""Everything a session can do on this machine. There is one kind.
THERE USED TO BE TWO, AND THE SPLIT COST MORE THAN IT SAVED.
A session was `cli` or `gui`, chosen by the CALLING model, and it got
the screen tools or the lookup suite but never both. Three things went
wrong with that and all three were measured:
* the caller chose wrong. Asked to build a GUI application it picked
the command line twice, with the rule in the tool docstring and in
the system prompt both times, and only a refusal stopped it - a
whole mechanism defending a choice that need not have existed.
* a `gui` session had no cheap way to look anything up, so it
reasoned instead: eleven minutes and eight compactions deriving
something the documentation states outright.
* a `cli` session could build a window and never see it, so "it
parses" was the best answer it could give.
Now every session has the command line, the lookup suite and the
screen. The agent decides what it needs, which it is in a far better
position to know, and a job that never looks at the screen simply
never calls those tools. The cost is their schemas in the context; the
thing it replaces was costing whole sessions.
"""
key = (os_name, booting)
if key in _schema_cache:
return list(_schema_cache[key])
registry = machines()
machine = registry.get(os_name) or next(iter(registry.values()))
wanted = BOOTING_GROUPS if booting else GROUP_ORDER
switched_off = _server_disabled()
built: list[dict[str, Any]] = []
for group in wanted:
for entry in _entries():
if entry.get("group") != group or not entry.get("enabled", True):
continue
if SERVER_TOOL.get(entry["name"]) in switched_off:
log.info("not offering %s: the tool server has %s switched off",
entry["name"], SERVER_TOOL[entry["name"]])
continue
built.append(_render(entry, machine))
_schema_cache[key] = built
return list(built)
def research_names() -> frozenset[str]:
"""The lookup tools, which vmd takes as one `research` verb."""
return frozenset(e["name"] for e in _entries() if e.get("group") == "research")
# Kept as a module-level name because `call` below reads it on every
# dispatch and the old hand-written list was one too.
RESEARCH_NAMES = research_names()
# ---------------------------------------------------------------------
# dispatch
# ---------------------------------------------------------------------
# Tools whose result is not worth keeping verbatim in the transcript: the
# task table is re-rendered every turn anyway, so twenty copies of it is
# twenty copies of the same thing.
TERSE = frozenset({"plan_tasks", "update_task", "add_task", "wait", "move"})
def call(vmd, name: str, arguments: dict[str, Any]) -> tuple[str, bytes | None]:
"""Run one tool. Returns (text for the transcript, image bytes or None).
Every failure comes back as TEXT, never as an exception: a refusal is
something the agent should read and act on, and an exception would
end the session over a mistyped path. Same policy as the api
container's @tool decorator, for the same reason.
"""
from client import VmdError, VmdUnavailable # noqa: PLC0415
import base64 # noqa: PLC0415
try:
# The lookup suite is one verb on vmd's side with an allowlist,
# rather than eight verbs that each do the same thing.
if name in RESEARCH_NAMES:
result = vmd.call("research", tool=name, args=arguments)
else:
result = vmd.call(name, **arguments)
except VmdError as exc:
return f"REFUSED: {exc}", None
except VmdUnavailable as exc:
return f"FAILED: {exc}", None
except Exception as exc: # noqa: BLE001
return f"FAILED: {type(exc).__name__}: {exc}", None
if name == "screenshot":
png = base64.b64decode(result.get("png_b64", ""))
return (f"Screenshot taken: {result.get('width')}x{result.get('height')} pixels. "
"Click coordinates are in these pixels from the top left."), png
if name == "web_search":
return result.get("results", "") or "NO RESULTS", None
if name in RESEARCH_NAMES:
return result.get("text", "") or "(no result)", None
if name == "read_page":
return result.get("text", "") or "(the page had no readable text)", None
if name == "open_page":
return (f"Opened {result.get('url')} in {result.get('browser')}. It takes a few "
"seconds to render - take a screenshot to see it."), None
if name == "run":
parts = [f"exit code: {result['exit_code']} ({result['seconds']}s)"]
if result.get("stdout"):
parts.append("stdout:\n" + result["stdout"])
if result.get("stderr"):
parts.append("stderr:\n" + result["stderr"])
if not result.get("stdout") and not result.get("stderr"):
parts.append("(no output)")
if result.get("truncated"):
parts.append("[output was truncated - re-run with grep or tail if you need the rest]")
return "\n".join(parts), None
if name in TERSE:
return result.get("tasks") or result.get("status_line") or "ok", None
import json as _json # noqa: PLC0415
return _json.dumps(result, indent=2)[:20000], None