Grant/owui-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial Commit

Browse Source
This commit is contained in:
Grant
2026-09-15 15:24:03 -04:00
commit 4dc7086e7e
605 changed files with 101380 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/owuitools/routers/vm.py
+516
View File
@@ -0,0 +1,516 @@
"""vm_agent - hand a task to a sub-agent with a real machine in front of it.
WHAT THE MAIN MODEL SEES
One tool, and deliberately one. Everything about driving a machine - the
screenshots, the key codes, the task bookkeeping, the transcript - lives
with the sub-agent and never enters this model's context. What arrives
here is a short report. Change 53 measured the existing 21 tool schemas
at ~6,900 tokens, paid on every turn of every conversation, so a second
tool would have to earn its place and none does.
WHY THIS ROUTER BLOCKS FOR SO LONG
A session is minutes to hours: a slow model with thinking on, driving a
real machine. Three separate clocks had to be raised for that to be
possible at all (see README), and the reason for preferring one long call
over polling is measured rather than aesthetic - Ollama runs with
OLLAMA_NUM_PARALLEL=1, so every return to the main model finds its KV
cache evicted and pays a cold prefill of its whole context. One long
block costs that once; ten polls cost it ten times.
While it blocks, the user is not staring at nothing: the sub-agent posts
status and screenshots straight into the chat message, over a channel
that never touches this model's context.
DISPATCH IS BY FILE, NOT BY HTTP
The api container lives behind netguard's default-deny filter, whose
whole allowlist is squid and valkey. Reaching the agent over HTTP would
mean editing that file. The Octave sandbox already solved this with a
shared volume, a .ready marker written last, and a per-job nonce, so the
same protocol is used here rather than a new hole.
"""
from __future__ import annotations
import asyncio
import contextlib
import json
import os
import re
import secrets
import time
import shutil
import uuid
from fastapi import APIRouter, Depends, Header, Request
from pydantic import Field
from ..budget import cap
from ..limits import Identity, identity
from ..security import require_auth
from ..settings import settings
from ..toolbase import ToolModel, tool
from ..vm_preflight import question, should_ask
router = APIRouter(prefix="/tools", tags=["vm"], dependencies=[Depends(require_auth)])
# Deliberately a SECOND router, on its own prefix and out of the schema.
#
# The live-progress feed is read by a filter running inside Open WebUI,
# not by the model. Anything under /tools with an operationId becomes a
# tool the model can call, and a tool whose whole job is "show me the
# progress of the thing you are already waiting for" is a tool it would
# call in a loop. Keeping it off /tools and out of the spec means it
# cannot: the model is never told it exists.
#
# Same bearer token as everything else - it is reachable only from
# openwebui_default, and the filter holds a tool-server key, not an
# Open WebUI admin key. That distinction is the point of this endpoint
# existing at all; see vmagent/app/events.py.
progress_router = APIRouter(prefix="/vm", tags=["vm"],
dependencies=[Depends(require_auth)],
include_in_schema=False)
@progress_router.get("/progress/{chat_id}")
async def vm_progress(chat_id: str) -> dict:
"""What the running session is doing, for the Open WebUI filter.
Returns `{}` when there is nothing - no session, a chat that never
started one, or a file not yet written. The caller polls, so "nothing
yet" and "nothing ever" are the same answer and neither is an error.
"""
if not re.fullmatch(r"[0-9a-fA-F-]{36}", chat_id or ""):
return {}
path = os.path.join(JOBS_DIR, "progress", f"{chat_id}.json")
try:
raw = await asyncio.to_thread(
lambda: open(path, encoding="utf-8", errors="replace").read())
return json.loads(raw)
except (OSError, ValueError):
return {}
JOBS_DIR = os.environ.get("OWUITOOLS_VM_JOBS_DIR", "/vmjobs")
CONFIRM_DIR = os.path.join(JOBS_DIR, "confirm")
# How long to wait for somebody to be watching before giving up on asking
# them. If the progress filter is not installed there is nobody to show
# the question to, and a tool that hangs for ten minutes in that case is
# worse than one that simply runs.
CONFIRM_PICKUP_SECONDS = 20.0
# And how long to wait for the person, once we know they were asked.
CONFIRM_ANSWER_SECONDS = 900.0
POLL_INTERVAL = 1.0
# Just under the gateway's own ceiling for this route, so the app is what
# answers rather than nginx - the same discipline the PDF extractor
# follows against the 240 s default.
WAIT_SECONDS = int(os.environ.get("OWUITOOLS_VM_WAIT_SECONDS", "7000"))
PROJECT_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,29}[a-z0-9]$")
class VmAgentRequest(ToolModel):
task: str | None = Field(
default=None,
description="What the agent should do, in plain prose: the user's request restated, "
"plus any constraints and what 'working' means. NOT an implementation - "
"do not write the code here, the agent writes it. A few hundred words "
"at most.",
)
os: str | None = Field(
default=None,
description="Which machine: 'ubuntu' for shell and Python, 'windows' for PowerShell, "
"batch and Python. Ask the user if the task does not say.",
)
project: str | None = Field(
default=None,
description="Folder and repository name for the work that is kept: lowercase letters, "
"digits and hyphens. Reuse the same name to continue earlier work.",
)
session_id: str | None = Field(
default=None,
description="Only when continuing a session that asked a question. Pass it back with "
"the user's reply in 'answer'.",
)
answer: str | None = Field(
default=None, description="The user's reply to a question the agent asked.",
)
cancel: bool = Field(
default=False, description="Stop a running session. Needs session_id.",
)
@progress_router.get("/confirm/{chat_id}", include_in_schema=False)
async def vm_confirm_pending(chat_id: str) -> dict:
"""The question waiting to be put to the person in this chat, if any.
Reading it counts as picking it up: the tool is sitting in a short
wait to find out whether anybody is listening, and this is how it
finds out. If nothing ever reads this, the job simply runs - a
confirmation nobody can see must not be able to stop work.
"""
if not re.fullmatch(r"[0-9a-fA-F-]{36}", chat_id or ""):
return {}
path = os.path.join(CONFIRM_DIR, f"{chat_id}.json")
try:
raw = await asyncio.to_thread(
lambda: open(path, encoding="utf-8", errors="replace").read())
pending = json.loads(raw)
except (OSError, ValueError):
return {}
if not pending.get("seen"):
pending["seen"] = True
with contextlib.suppress(OSError):
await asyncio.to_thread(_write_json, path, pending)
return pending
@progress_router.post("/confirm/{chat_id}", include_in_schema=False)
async def vm_confirm_answer(chat_id: str, answer: dict) -> dict:
"""What the person decided. One of: run, change, cancel."""
if not re.fullmatch(r"[0-9a-fA-F-]{36}", chat_id or ""):
return {"ok": False}
decision = str(answer.get("decision") or "").strip().lower()
if decision not in ("run", "change", "cancel"):
return {"ok": False}
payload = {"id": str(answer.get("id") or "")[:64], "decision": decision,
"text": str(answer.get("text") or "")[:2000]}
with contextlib.suppress(OSError):
await asyncio.to_thread(
_write_json, os.path.join(CONFIRM_DIR, f"{chat_id}.answer.json"), payload)
return {"ok": True}
def _write_json(path: str, payload: dict) -> None:
os.makedirs(os.path.dirname(path), exist_ok=True)
tmp = path + ".tmp"
with open(tmp, "w", encoding="utf-8") as handle:
json.dump(payload, handle)
os.replace(tmp, path)
async def _ask_before_starting(task: str, os_name: str, interface: str,
project: str | None, chat_id: str | None) -> tuple[str, str]:
"""Put the job to the person before a machine is started for it.
WHY THIS IS HERE AND NOT IN THE MODEL'S HANDS.
Starting a machine takes minutes and destroys itself afterwards, so a
job sent with the wrong wording costs the whole session. The person
who asked is the one who can tell at a glance whether the wording is
right - so they are shown it, verbatim, before anything starts.
It is done here, inside the tool, for the same reason: the model is
simply blocked in its call while the question is answered, and does
not have to be trusted to ask, to wait, or to report the answer
faithfully. It cannot skip this the way it could skip an instruction.
Returns (decision, text). Decisions: run, change, cancel.
"""
if not chat_id or not re.fullmatch(r"[0-9a-fA-F-]{36}", chat_id):
return ("run", "") # nobody to ask - see the pickup note
request_id = uuid.uuid4().hex
path = os.path.join(CONFIRM_DIR, f"{chat_id}.json")
answer_path = os.path.join(CONFIRM_DIR, f"{chat_id}.answer.json")
with contextlib.suppress(OSError):
await asyncio.to_thread(lambda: os.path.exists(answer_path) and os.remove(answer_path))
machine = "Windows" if os_name == "windows" else "Ubuntu"
try:
await asyncio.to_thread(_write_json, path, {
"id": request_id, "task": task, "machine": machine, "os": os_name,
"interface": interface, "project": project or "",
"asked_at": time.time(), "seen": False,
})
except OSError:
return ("run", "")
try:
deadline = time.monotonic() + CONFIRM_PICKUP_SECONDS
picked_up = False
while True:
await asyncio.sleep(0.5)
try:
raw = await asyncio.to_thread(
lambda: open(answer_path, encoding="utf-8", errors="replace").read())
reply = json.loads(raw)
if reply.get("id") == request_id:
return (str(reply.get("decision") or "run"), str(reply.get("text") or ""))
except (OSError, ValueError):
pass
if not picked_up:
with contextlib.suppress(OSError, ValueError):
current = json.loads(await asyncio.to_thread(
lambda: open(path, encoding="utf-8", errors="replace").read()))
if current.get("seen"):
picked_up = True
deadline = time.monotonic() + CONFIRM_ANSWER_SECONDS
if time.monotonic() > deadline:
# Nobody was watching, or nobody answered. Either way the
# work is what was asked for; run it.
return ("run", "")
finally:
for leftover in (path, answer_path):
with contextlib.suppress(OSError):
await asyncio.to_thread(lambda p=leftover: os.path.exists(p) and os.remove(p))
@router.post("/vm_agent", operation_id="vm_agent", response_model=str,
summary="Test scripts on a throwaway Ubuntu or Windows machine")
@tool
async def vm_agent(body: VmAgentRequest, request: Request,
who: Identity = Depends(identity),
x_openwebui_message_id: str | None = Header(default=None,
include_in_schema=False)) -> str:
"""Hand a job to a helper that has a real, disposable Ubuntu or Windows machine.
USE IT when the answer has to be true of a real machine rather than of the documentation.
The user asking you to prove it - "make sure it works", "test it properly", "check it for
real", "I don't trust the docs" - is the clearest signal, but so is any question about
behaviour in a case the documentation does not spell out. If an honest answer would have to
say "the documentation implies" or "it should", that is the answer you cannot give: run it
and report what happened.
HAND THE WORK OVER BEFORE YOU DO IT, NOT AFTER. When the user asks for something that
has to be built and run - a script, a tool, a service - call this FIRST, as the opening
move of your reply. Do not design it, do not write it, do not work out the approach: the
agent does all of that on the machine, where it can run what it writes.
The `task` is the user's request in your own words, plus anything they said that
constrains it and a sentence on what "working" means. A few hundred words. If you find
yourself putting code in it, stop - you are doing the agent's job, and you will run out
of room before the call is even finished. Measured: one attempt spent its entire output
budget writing a PowerShell GUI into the task and the tool call was never emitted at
all, so nothing ran and the user got a promise instead of a program.
DO NOT USE IT when a document already answers the question and nobody has asked for proof.
"What is X", "how is X configured", "which version ships X" are reading questions and take
seconds; a machine takes minutes. `read_man_page`, `web_search` and `fetch_page` are for
"what does X do" - this one is for "does X do that here".
The helper has a command line AND a desktop. It decides which it needs - most jobs are
shell work and never touch the screen, and one that has to see a window or click something
can. You do not choose, and nothing is lost by asking for something visual.
**Ask the user which machine when the task does not say.** Python runs on both, `.sh` only
on Ubuntu, `.ps1`/`.bat` only on Windows. Guessing costs the whole session.
Write the job as instructions, not as paths: the helper is told where its project folder is
on the machine it gets, and saying `/mnt/project` at it on Windows only confuses matters.
The machine reaches the internet but nothing on this network, and is destroyed afterwards -
only the named project folder survives, published to Gitea. Before it starts, the user is
shown the job you wrote and can change or stop it. One report comes back at the end.
WHEN THE RESULT CARRIES A DOWNLOAD LINK, GIVE THE LINK AND DO NOT REPRINT THE FILE. Each
session publishes what it built as a release - test scripts and scratch are never
published - and the result gives you that page: it lists the files and carries a zip
that saves to disk, which a link into the repository does not. Pasting the file back is
a wall of code the user must scroll past to reach the one line they wanted. Say what it
does and what the test showed, in a few sentences, and link it. Quote at most a few
lines, and only to make a specific point about them.
GIVE ONLY THE LINK THE RESULT GIVES YOU. If there is none, say the work was not published
and why. Never build a URL from the project name: that has sent a user to a different
project's release.
"""
if not os.path.isdir(JOBS_DIR):
return ("FAILED: the VM sandbox is not available on this server. Nothing was run. "
"Tell the user rather than retrying.")
if body.cancel:
return await _cancel(body.session_id)
# THE SCHEMA NO LONGER DEMANDS A NEW SESSION'S FIELDS FOR EVERY CALL.
#
# `task`, `os` and `interface` were required outright, so a cancel -
# and an answer to a question the agent had asked - was rejected with
# "task: Field required" before it reached any of this, even though
# the tool's own instructions say to call back with the session_id.
# They are optional in the schema now and required HERE, where a new
# session actually needs them, with a refusal that says which is
# missing. Found by using it.
missing = [name for name, value in (("task", body.task), ("os", body.os))
if not (value or "").strip()]
if missing:
return (f"CANNOT DO THAT: starting a session needs {', '.join(missing)}. "
"Give the job and the machine ('ubuntu' or 'windows'), then call again.")
os_name = (body.os or "").strip().lower()
if os_name not in ("ubuntu", "windows"):
return ("CANNOT DO THAT: 'os' must be 'ubuntu' or 'windows'. Ubuntu runs shell and "
"Python; Windows runs PowerShell, batch and Python. If the task could run on "
"either, ask the user which they want before calling again.")
# ONE KIND OF SESSION. The helper always has a desktop; whether it
# looks at it is its own business, and it was never a decision the
# calling model got right. It chose a command line for "build me a
# GUI" twice, with the rule in the docstring and in the system prompt
# both times, and the refusal that finally fixed it was a whole
# mechanism to protect a choice that did not need to exist.
interface = "gui"
project = (body.project or "").strip() or None
if project and not PROJECT_RE.fullmatch(project):
return (f"CANNOT DO THAT: '{project}' is not a usable project name. Use 2-31 "
"characters, lowercase letters, digits and hyphens, starting and ending with "
"a letter or digit. The name becomes a folder and a Gitea repository.")
task = (body.task or "").strip()
if len(task) < 10:
return ("CANNOT DO THAT: say what the agent should do. It cannot see this "
"conversation - the task text is everything it knows.")
# The one pre-flight refusal, and it exists because the cost of being
# wrong is a whole destroyed session rather than a retry. See
# vm_preflight.py for why it lives in tool output and not the prompt.
if not body.session_id and should_ask(task, os_name):
return question(os_name)
# THE PERSON SEES THE JOB BEFORE A MACHINE IS STARTED FOR IT.
#
# They can let it run, change the wording, or drop it. The model is
# blocked here while they decide and is told the outcome as an
# ordinary tool result - it has no part in asking and cannot skip it.
decision, note = await _ask_before_starting(
task, os_name, interface, project, getattr(who, "chat_id", None))
if decision == "cancel":
return ("STOPPED BY THE USER. No machine was started and nothing ran. "
"Reply with one short sentence acknowledging that, and nothing else - "
"do not summarise, suggest, or offer to continue unless they ask.")
if decision == "change":
return ("NOT YET - the user read the job and wants it changed before it runs.\n\n"
f"What they said: {note or '(they did not say what to change)'}\n\n"
"Rewrite the task with that taken into account and call vm_agent again. "
"Do not ask them to repeat themselves, and do not run the old wording.")
return await _submit(task, os_name, interface, project, body, who,
x_openwebui_message_id, request)
async def _submit(task: str, os_name: str, interface: str, project: str | None,
body: VmAgentRequest, who: Identity,
message_id: str | None, request: Request) -> str:
job_id = uuid.uuid4().hex
nonce = secrets.token_hex(16)
job_dir = os.path.join(JOBS_DIR, "in", job_id)
run_dir = os.path.join(JOBS_DIR, "run", job_id)
out_path = os.path.join(JOBS_DIR, "out", f"{job_id}.out")
payload = {
"task": task, "os": os_name, "interface": interface, "project": project,
"chat_id": who.chat_id or "", "message_id": message_id or "",
"owner": who.email or who.user_id or "",
"answer": (body.answer or "")[:4000],
}
def write() -> None:
# THE PREVIOUS RUN'S LEFTOVERS GO FIRST, AND THIS IS NOT TIDYING.
#
# The live-progress file is keyed by CHAT, not by run, so a second
# vm_agent call in the same conversation inherits the first one's
# final state - `done: true`. Measured: the watcher for a 45-minute
# run started, polled once, read the previous run's "done", said
# "The machine has been destroyed" and exited FOUR SECONDS IN. The
# run then worked for three quarters of an hour with nobody
# watching, and the person reasonably concluded the tool had given
# up. Deleting it here is authoritative because it happens before
# the job is queued, so there is no window for the watcher to read
# a stale file; the filter also ignores anything older than itself,
# because one guard on a race is not a guard.
for stale in (
os.path.join(JOBS_DIR, "progress", f"{payload['chat_id']}.json"),
os.path.join(CONFIRM_DIR, f"{payload['chat_id']}.answer.json"),
):
if payload["chat_id"]:
with contextlib.suppress(OSError):
os.unlink(stale)
os.makedirs(job_dir, exist_ok=True)
with open(os.path.join(job_dir, "request.json"), "w", encoding="utf-8") as handle:
json.dump(payload, handle)
with open(os.path.join(job_dir, "meta"), "w", encoding="utf-8") as handle:
handle.write(f"NONCE={nonce}\n")
# Written last: its existence is what tells the runner the
# directory is complete, so the two never race.
open(os.path.join(JOBS_DIR, "in", f"{job_id}.ready"), "w").close()
await asyncio.to_thread(write)
waited = 0.0
while waited < WAIT_SECONDS:
if await asyncio.to_thread(os.path.exists, out_path):
raw = await asyncio.to_thread(
lambda: open(out_path, encoding="utf-8", errors="replace").read())
await asyncio.to_thread(lambda: os.remove(out_path))
return _render(raw, nonce, job_id)
# The user pressed stop, or closed the tab. Without this the
# machine keeps running and the next caller waits behind it - and
# "cancel recorded, nothing killed" is precisely the bug the
# previous sub-agent stack shipped.
if await request.is_disconnected():
await asyncio.to_thread(_touch_cancel, run_dir)
return ("The VM session was cancelled because this request was abandoned. "
"The machine has been stopped.")
await asyncio.sleep(POLL_INTERVAL)
waited += POLL_INTERVAL
await asyncio.to_thread(_touch_cancel, run_dir)
return (f"FAILED: the VM session was still running after {WAIT_SECONDS // 60} minutes and "
"has been stopped. Anything it committed to the project is kept. Tell the user "
"what was attempted rather than starting again automatically.")
def _touch_cancel(run_dir: str) -> None:
try:
os.makedirs(run_dir, exist_ok=True)
open(os.path.join(run_dir, "CANCEL"), "w").close()
except OSError:
pass
async def _cancel(session_id: str | None) -> str:
if not session_id or not re.fullmatch(r"[0-9a-f]{32}", session_id):
return ("CANNOT DO THAT: cancelling needs the session_id from the report that asked "
"you to call back. If there is no session running, there is nothing to stop.")
await asyncio.to_thread(_touch_cancel, os.path.join(JOBS_DIR, "run", session_id))
return "The VM session has been told to stop. Its machine is being destroyed."
def _render(raw: str, nonce: str, job_id: str) -> str:
try:
result = json.loads(raw)
except ValueError:
return "FAILED: the VM agent returned something unreadable. Nothing is known about the run."
# Same check the Octave runner's nonce exists for: a result that is
# not stamped with the nonce this request generated did not come from
# this request.
if result.get("nonce") != nonce:
return ("FAILED: the VM agent's result did not match this request and has been "
"discarded. Nothing is known about the run.")
report = str(result.get("report") or "").strip()
state = str(result.get("state") or "done")
# A tool result that IS a data: URI is handed to the model as a vision
# input by Open WebUI, and budget.cap() passes it through untouched.
# A guest that printed one as its only output would be choosing a
# picture for this model to look at, so it can never be the whole
# answer here. vmd scrubs these too; this is the boundary check.
if report.startswith("data:"):
report = "[the agent's report was discarded: it was a bare data: URI]"
if state == "question":
report += f"\n\n_Session `{job_id}` is waiting. Call `vm_agent` again with " \
f"`session_id` and the user's `answer`._"
elif state == "unfreeze":
report += f"\n\n_Session `{job_id}` is waiting on the operator._"
steps, seconds = result.get("steps", 0), result.get("seconds", 0)
report += f"\n\n_{steps} agent steps, {seconds:.0f}s._"
return cap(report)