Grant/owui-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial Commit

Browse Source
This commit is contained in:
Grant
2026-09-15 15:24:03 -04:00
commit 4dc7086e7e
605 changed files with 101380 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
owui-functions/vm_agent_progress.py
+788
View File
@@ -0,0 +1,788 @@
"""
title: VM Agent Progress
author: owuitools
version: 0.1.0
description: While vm_agent is running, shows what the sub-agent is doing and, when it looks at the machine's screen, what it sees.
"""
# WHY THIS EXISTS
#
# A vm_agent call is one tool call that takes minutes to hours: a slow
# model with thinking on, driving a real VM. Open WebUI shows "Calling
# vm_agent" and then nothing at all until it returns. That is the single
# longest silence this deployment can produce, and it is silent precisely
# when the most is happening.
#
# WHY IT IS A FILTER AND NOT THE AGENT POSTING DIRECTLY
#
# The first design had the agent POST to
# /api/v1/chats/{id}/messages/{id}/event. That endpoint refuses a key
# whose user neither owns the chat nor is an admin - and anyone on this
# instance may invoke vm_agent, so it had to be an ADMIN key. An admin
# key is the ability to read every conversation on the instance and
# create users, and it would have had to live somewhere outside Open
# WebUI to be usable.
#
# Here there is no such key. This code runs INSIDE Open WebUI, in the
# same process as the chat pipeline, and __event_emitter__ is already
# bound to the requesting user's own session. The only credential it
# holds is the owuitools tool-server bearer token - which grants exactly
# the tools the model can already call, and nothing else.
#
# The cost is that progress is polled rather than pushed: a 2-second poll
# for the length of a session. That is the whole of the trade.
#
# WHY NONE OF THIS REACHES THE MODEL
#
# Open WebUI stores a message's status history and its `embeds` as
# SIBLING fields of the message, and rebuilds the payload sent to the
# model from an allowlist - role, thinking, content, tool_calls,
# tool_call_id, images (utils/payload.py). Neither `statusHistory` nor
# `embeds` is on that list. So a screenshot shown here is seen by the
# person watching and never by the model that called the tool, which is
# the entire reason for doing it this way rather than returning images
# from the tool itself.
#
# TWO SHARP EDGES IN THE stream() HOOK, both learned the hard way in
# live_progress_status.py and both still true here:
# * a falsy return value DROPS the chunk. Every path returns `event`.
# * an exception is re-raised by Open WebUI and breaks the response
# mid-flight. The whole body is wrapped.
from __future__ import annotations
import asyncio
import contextlib
import json
import re
import time
import aiohttp
from pydantic import BaseModel, Field
# Must match vmagent/app/events.py. Twenty lines is enough to see a
# command and what it said, and short enough that the view does not become
# the conversation.
TERMINAL_LINES = 20
# How often the whole view is re-sent even when it has not changed, so a
# viewer who reloads or joins late resyncs without waiting for the next
# change. See the note in _watch.
RESEND_SECONDS = 10.0
# How long a session may say nothing at all before the status line starts
# counting instead of repeating itself. See the note in _watch.
QUIET_SECONDS = 25.0
# The frame tells the page how tall it is. See _embed's note for why the
# other route - the page measuring the frame - cannot work here.
#
# It measures the CONTENT, not the document. `html{overflow:hidden}` makes
# `documentElement.scrollHeight` report the frame's current height whenever
# the content is shorter than it, so a box measured that way can grow but
# can never shrink again: measured, a seven-line list in a 150px frame
# reported 150 rather than its real 135. The wrapper's own bounding box has
# no such floor. A zero is never sent - a frame in a background tab has no
# layout and measures 0, and the observer reports the real height as soon
# as the tab is looked at.
# The one script every box carries: it measures itself, remembers whether
# it has been clicked shut, and reveals the live line at the rate the
# tokens arrived.
#
# WHY STATE LIVES IN window.name, WHICH LOOKS LIKE A HACK AND IS THE ONLY
# THING THAT WORKS HERE.
#
# The view is re-sent every couple of seconds, and each send replaces the
# frame's `srcdoc` - a new Document every time. The frame is sandboxed
# without `allow-same-origin`, so it has an opaque origin: localStorage
# and cookies throw, IndexedDB is useless, and there is no listener in
# Open WebUI to postMessage to. `window.name` belongs to the browsing
# CONTEXT rather than the document, and the context survives the swap.
# Measured in this exact sandbox: a frame reloaded three times read back
# the name its two predecessors had written. Without it a box would
# spring open again a second after being closed.
#
# The state is keyed by which box this is, because the boxes change
# places: the screen box appears mid-session and pushes the feed down one
# slot, and the feed would otherwise inherit the screen's state.
_VIEW_SCRIPT = """
<script>
(function(){
var b=document.getElementById('b');if(!b){return;}
var cfg={};try{cfg=JSON.parse(document.getElementById('cfg').textContent);}catch(e){}
var kind=cfg.k||'';
var st={};try{st=JSON.parse(window.name||'{}')||{};}catch(e){st={};}
if(st.k!==kind){st={k:kind};}
function save(){try{window.name=JSON.stringify(st);}catch(e){}}
function el(id){return document.getElementById(id);}
function post(){var h=Math.ceil(b.getBoundingClientRect().height);
if(h>0){parent.postMessage({type:'iframe:height',height:h},'*');}}
/* Clicking a box halves it; clicking again puts it back. */
function apply(){
var on=!!st.c;
if(kind==='tasks'){
var f=el('full'),c=el('cur');
if(f){f.style.display=on?'none':'';}
if(c){c.style.display=on?'':'none';}
}else if(kind==='screen'){
var w=el('wrap'),i=el('shot');
if(w){w.style.background=on?'#000':'';}
if(i){i.style.width=on?'50%':'100%';i.style.margin=on?'0 auto':'';}
}else if(kind==='feed'){
var r=el('rows');
if(r){
if(on){var first=r.firstElementChild;
var rh=first?first.getBoundingClientRect().height:17;
r.style.maxHeight=Math.round(rh*Math.ceil((cfg.n||20)/2))+'px';}
else{r.style.maxHeight='';}}
}}
b.style.cursor='pointer';
b.addEventListener('click',function(){st.c=st.c?0:1;save();apply();post();});
apply();
/* THE LIVE LINE, REVEALED AT THE RATE IT ARRIVED.
The server sends the tail of what the model is saying, at most once a
second, so this line used to jump a clause at a time - motion that
said nothing about how fast the tokens were actually coming. It is now
appended a few characters per frame, and the speed it moves IS the
token rate. The text so far rides in window.name, so a replaced
document carries on the same line instead of retyping it. */
if(kind==='tasks'){
var span=el('t');
var target=cfg.s||'';
var shown=(typeof st.d==='string')?st.d:'';
var pending='';
if(target){
if(shown&&shown.indexOf(target)>=0){pending='';}
else{
/* The server sends a sliding window of the tail, so what is new is
whatever follows the longest overlap between the two. */
var o=0,max=Math.min(shown.length,target.length);
for(var k=max;k>0;k--){
if(shown.slice(shown.length-k)===target.slice(0,k)){o=k;break;}}
if(o===0){shown='';}
pending=target.slice(o);}}
var CAP=400;
function paint(){if(shown.length>CAP){shown=shown.slice(-CAP);}
if(span){span.textContent=shown;}}
paint();
if(pending){
var TICK=40,DUR=1500;
var per=Math.max(1,Math.ceil(pending.length/(DUR/TICK)));
var timer=setInterval(function(){
if(!pending.length){clearInterval(timer);return;}
shown+=pending.slice(0,per);pending=pending.slice(per);
paint();st.d=shown;save();},TICK);}
st.d=shown;save();}
post();addEventListener('load',post);
if(window.ResizeObserver){new ResizeObserver(post).observe(b);}
setTimeout(post,60);setTimeout(post,400);
})();
</script>
"""
class Filter:
class Valves(BaseModel):
enabled: bool = Field(
default=True,
description="Turn the whole filter off without deleting it.",
)
owuitools_url: str = Field(
default="http://owuitools-gw:8000",
description="The owuitools gateway, as Open WebUI reaches it. This is the same "
"address as the tool server connection and is not published to the host.",
)
api_key: str = Field(
default="",
description="The owuitools bearer token - the SAME value as the tool server "
"connection's Auth field, not an Open WebUI API key. It grants the tools the "
"model can already call and nothing more. Without it this filter does nothing.",
)
poll_seconds: float = Field(
default=2.0,
description="How often to ask for progress while a session is running.",
)
show_terminal: bool = Field(
default=True,
description="Show the last 20 lines of what the agent ran.",
)
show_screenshots: bool = Field(
default=True,
description="Show the machine's screen when the agent takes a screenshot. Each frame is stored "
"on the message, so turning this off makes the chat record smaller.",
)
max_minutes: float = Field(
default=125.0,
description="Give up watching after this long. A backstop so a watcher can never "
"outlive the session it is watching, whatever goes wrong.",
)
def __init__(self) -> None:
self.valves = self.Valves()
# One watcher per message id. A chat can only have one assistant
# message in flight, but a user with two tabs open has two chats.
self._watchers: dict[str, asyncio.Task] = {}
# -- Open WebUI hooks -----------------------------------------------
async def inlet(self, body: dict, __event_emitter__=None, __event_call__=None,
__metadata__=None) -> dict:
# A previous turn's watcher has no business surviving into this
# one; inlet is the only hook guaranteed to run at the start.
try:
self._stop(self._message_id(__metadata__))
except Exception: # noqa: BLE001
pass
return body
async def stream(self, event: dict, __event_emitter__=None, __event_call__=None,
__metadata__=None) -> dict:
try:
if self.valves.enabled and self._is_vm_agent_call(event):
self._start(__event_emitter__, __event_call__, __metadata__)
except Exception: # noqa: BLE001
# Never break the response. See the header.
pass
return event
async def outlet(self, body: dict, __event_emitter__=None, __event_call__=None,
__metadata__=None) -> dict:
try:
self._stop(self._message_id(__metadata__))
except Exception: # noqa: BLE001
pass
return body
# -- detection -------------------------------------------------------
@staticmethod
def _is_vm_agent_call(event: dict) -> bool:
"""Has the model just started writing a vm_agent call?
Tool calls arrive token by token, so the name may appear in a
delta with nothing else in it. Starting the watcher more than
once is harmless - _start is idempotent per message.
"""
for choice in (event or {}).get("choices") or []:
delta = choice.get("delta") or {}
for call in delta.get("tool_calls") or []:
if (call.get("function") or {}).get("name") == "vm_agent":
return True
return False
@staticmethod
def _message_id(metadata) -> str:
return str((metadata or {}).get("message_id") or "")
# -- the watcher ------------------------------------------------------
def _start(self, emitter, caller, metadata) -> None:
message_id = self._message_id(metadata)
chat_id = str((metadata or {}).get("chat_id") or "")
if not emitter or not message_id or not chat_id:
return
if not self.valves.api_key:
return
if message_id in self._watchers and not self._watchers[message_id].done():
return
self._watchers[message_id] = asyncio.create_task(
self._watch(emitter, caller, chat_id))
def _stop(self, message_id: str) -> None:
task = self._watchers.pop(message_id, None)
if task and not task.done():
task.cancel()
async def _watch(self, emitter, caller, chat_id: str) -> None:
url = f"{self.valves.owuitools_url.rstrip('/')}/vm/progress/{chat_id}"
headers = {"Authorization": f"Bearer {self.valves.api_key}"}
deadline = asyncio.get_running_loop().time() + self.valves.max_minutes * 60
began = asyncio.get_running_loop().time()
# Wall clock, because that is what the progress file is stamped
# with; the loop clock above is a monotonic reading and the two
# are not comparable.
began_at = time.time()
last_status = ""
last_view = None
shown_anything = False
# WHY THE WHOLE VIEW IS RE-SENT EVEN WHEN NOTHING HAS MOVED.
#
# These updates are pushed to whoever is connected. Someone who
# reloads the page, or opens the chat on another device, joins
# partway through and gets only what is NEXT - and a session can
# sit on one step for minutes ("Waiting for the machine to finish
# booting" held for four polls in a row, measured), so a viewer who
# arrives just after a change sees an empty message for as long as
# that step lasts and concludes nothing is happening.
#
# So the current view is re-sent on a slow heartbeat regardless of
# whether it changed. `replace: true` makes that idempotent - it
# overwrites the one live view rather than appending a frame - so
# the cost is one small message every RESEND_SECONDS and the
# benefit is that reattaching fixes itself.
next_resend = 0.0
ended_cleanly = False
try:
timeout = aiohttp.ClientTimeout(total=10)
async with aiohttp.ClientSession(timeout=timeout) as session:
await self._status(emitter, "Starting a machine…")
asked: set[str] = set()
while asyncio.get_running_loop().time() < deadline:
await asyncio.sleep(self.valves.poll_seconds)
# Before a machine is started, the tool waits for the
# person to see the job. Putting that question is this
# watcher's job: it is already here, it already has the
# chat, and doing it here means the model plays no part
# in it and cannot skip it.
if caller is not None:
with contextlib.suppress(Exception):
await self._maybe_ask(session, caller, chat_id, headers, asked)
try:
async with session.get(url, headers=headers) as response:
if response.status != 200:
continue
data = json.loads(await response.text())
except (aiohttp.ClientError, ValueError, asyncio.TimeoutError):
# A poll that fails is a poll; the session is not
# affected by it and neither is the next one.
continue
# ANYTHING OLDER THAN THIS WATCHER BELONGS TO A PREVIOUS RUN.
#
# The progress file is named after the CHAT, so a second
# vm_agent call in the same conversation finds the first
# one's final state sitting there - including `done`.
# Measured: a watcher for a 45-minute run read that on
# its first poll, announced the machine had been
# destroyed and exited four seconds in, while the run
# carried on unobserved to sequence 1035.
#
# The api deletes the file before queueing the job, which
# closes the race properly; this is the second guard,
# because a race with one guard is a race. `updated` is
# written on every flush, so a file from the last run is
# simply older than the watcher reading it.
if float(data.get("updated") or 0) < began_at:
data = {}
if not data:
# NOTHING YET, AND SAYING SO BEATS SAYING NOTHING.
#
# "Starting a machine" is the first thing this
# emits, before there is any progress to read. If
# the file never appears - the job queued behind
# another session, the runner restarting, a boot
# that is simply slow - that sentence is the last
# word the reader gets, and a frozen line is
# indistinguishable from a broken one. Reported as
# exactly that: a toast stuck on "Starting a
# machine" with no boxes.
#
# So it counts, and says how long. A number that
# moves is the difference between waiting and
# wondering.
waited = int(asyncio.get_running_loop().time() - began)
if waited >= QUIET_SECONDS:
shown_anything = True
await self._status(
emitter,
f"Starting a machine - {waited}s so far, nothing back "
"from it yet")
continue
now = asyncio.get_running_loop().time()
resend = now >= next_resend
if resend:
next_resend = now + RESEND_SECONDS
status = str(data.get("status") or "")
if status and (status != last_status or resend):
last_status = status
shown_anything = True
await self._status(emitter, status)
# One embed holds the whole view - the task list and,
# under it, whichever of the two the session has. It is
# rebuilt when any part of it moves, and `replace: true`
# means the message carries one of these rather than a
# reel of every frame ever taken.
view = (str(data.get("tasks") or ""),
str(data.get("speech") or ""),
int(data.get("shot_seq") or 0),
int(data.get("terminal_seq") or 0))
if view != last_view or resend:
last_view = view
await self._embed(emitter, data, self.valves)
if data.get("done"):
ended_cleanly = True
break
except asyncio.CancelledError:
raise
except Exception: # noqa: BLE001
pass
finally:
# Only say the machine is gone if the session actually said it
# was done. This block also runs when the watcher is cancelled
# or hits its own deadline - and in those cases the session is
# very much still running, so the old wording stated something
# false and wiped the view while there was still work to show.
with contextlib.suppress(Exception):
await self._embed(emitter, {}, self.valves)
if shown_anything:
with contextlib.suppress(Exception):
await self._status(
emitter,
"The machine has been destroyed." if ended_cleanly
else "No longer watching this session.",
done=True)
# WHAT THE PERSON IS SHOWN, AND WHY IT IS NOT THE JOB ITSELF.
#
# The job the model writes is an instruction sheet for another machine:
# numbered steps, exact commands, the wording that stops an agent
# improvising. Measured, a real one ran to 2,600 characters. Nobody
# reads that to decide whether to press OK - and worse, Open WebUI does
# not scroll the dialog, so a job that tall pushed OK and Cancel 150px
# below the bottom of an 800px window and the question could not be
# answered at all.
#
# So the dialog gets a short summary of what the job will DO.
#
# IT IS BUILT HERE, FROM THE JOB, rather than written by the model.
# This is the one screen where a summary that drifts from the text is
# a real problem: the person approves the summary and the machine is
# sent the job. A summary pulled out of the job's own headings cannot
# describe work that is not in it, and asking the model for one would
# be both a second thing to trust and more work for it to do.
SUMMARY_STEPS = 5 # bullets before the rest become a count
SUMMARY_HEAD = 150 # characters of the opening line
SUMMARY_STEP = 64 # characters of any one bullet - ONE LINE each
# A step is a numbered line or a STEP heading. A bullet is only a step
# when there are no headings - measured on a real job, the bullets were
# the data the steps operated on ("- /tmp/logtest/recent.log -> mtime =
# now"), and treating them as steps pushed the actual steps out of the
# summary entirely.
_STEP_NUM = re.compile(
r"^\s*(?:step\s*\d+|\d+\s*[.)])\s*[:.\u2013\u2014-]*\s*(.+)$", re.IGNORECASE)
_BULLET = re.compile(r"^\s*[-*\u2022]\s*(.+)$")
# Sentences end at . ! ? and nothing else. Splitting on a colon as well
# turned "Run: ps -p 1 -o comm=" into the bullet "Run:", which says
# nothing - the useful half is always on the other side of the colon.
_SENTENCE = re.compile(r"(?<=[.!?])\s")
# "Create a clean test directory, e.g. /tmp/logtest" is one sentence.
# Splitting on every full stop cut that bullet off at "e.g".
_NOT_AN_END = ("e.g", "i.e", "etc", "vs", "no", "fig", "approx", "cf")
# The word before the stop carries whatever punctuation was around it -
# "(i.e." and "e.g.," both have to reduce to the abbreviation itself.
_LAST_WORD = re.compile(r"[^a-z.]", re.IGNORECASE)
_LABEL = re.compile(r"^(goal|task|objective|summary)\s*[:-]\s*", re.IGNORECASE)
@classmethod
def _headline(cls, text: str, cap: int) -> str:
"""The first sentence of something, cut to length without cutting a word."""
text = " ".join(text.split())
parts = cls._SENTENCE.split(text)
first = parts[0]
# Keep absorbing while the "sentence" ended on an abbreviation.
while (len(parts) > 1
and cls._LAST_WORD.sub("", first.rsplit(" ", 1)[-1]).strip(".").lower()
in cls._NOT_AN_END):
parts = parts[1:]
first = f"{first} {parts[0]}"
first = first.strip()
if len(first) <= cap:
return first.rstrip(".")
cut = first[:cap]
space = cut.rfind(" ")
return (cut[:space] if space > cap // 2 else cut).rstrip(" ,;:-") + "..."
@classmethod
def _summarise(cls, task: str) -> str:
"""What this job will do, in a few lines, taken from the job itself."""
task = task.strip()
if not task:
return "(the job was empty)"
lines = [line for line in task.splitlines() if line.strip()]
def heads(pattern) -> list:
found = []
for index, line in enumerate(lines):
match = pattern.match(line)
if not match:
continue
body = match.group(1)
# A step that is only a label - "Report:", "Then:" - says
# nothing by itself, so it takes what is under it.
if body.rstrip().endswith(":") and len(body.split()) <= 3:
following = lines[index + 1] if index + 1 < len(lines) else ""
under = cls._BULLET.match(following)
body = f"{body.rstrip()} {(under.group(1) if under else following).strip()}"
body = body.strip()
step = cls._headline(body, cls.SUMMARY_STEP)
if step:
found.append(step)
return found
steps = heads(cls._STEP_NUM) or heads(cls._BULLET)
# An opening line that is itself a step is the first bullet, not the
# headline; anything else - "Goal: ...", a sentence of context - is.
head = ""
if not (cls._STEP_NUM.match(lines[0]) or
(not heads(cls._STEP_NUM) and cls._BULLET.match(lines[0]))):
head = cls._headline(cls._LABEL.sub("", lines[0]), cls.SUMMARY_HEAD)
if not steps:
# No headings to work from, so the summary is the first few
# sentences of the prose - still shorter than the whole thing.
rest = cls._SENTENCE.split(" ".join(task.split()))[1:]
steps = [step for step in
(cls._headline(part, cls.SUMMARY_STEP)
for part in rest[:cls.SUMMARY_STEPS]) if step]
if not steps:
return head or cls._headline(task, cls.SUMMARY_HEAD)
# Open WebUI renders this dialog as markdown, and a plain newline there
# is a SOFT break: consecutive lines are joined into one paragraph.
# Measured, that ran two steps together on one wrapped line. A "- "
# list item at column 0 is the one form the renderer keeps on its own
# line - a leading indent turns it into a nested or literal block, and
# a bullet character of my own is not markdown at all and gets joined.
out = [head] if head else []
shown = steps[:cls.SUMMARY_STEPS]
out += [f"- {step}" for step in shown]
left = len(steps) - len(shown)
if left > 0:
out.append(f"- and {left} more step{'' if left == 1 else 's'}")
return "\n".join(out)
async def _maybe_ask(self, session, caller, chat_id: str, headers: dict,
asked: set) -> None:
"""Show the person the job, and send back what they decide.
Deliberately one dialog rather than three buttons. Open WebUI can
show a box with a message and a text field, and that covers all
three answers in one step without the person having to explain
themselves twice:
leave it empty and press OK -> it runs as written
type what to change -> it comes back rewritten
press Cancel -> nothing is started
Typing the change here is what keeps the model out of it. The old
shape would have been "the model asks what you want changed",
which is another round trip and another chance to lose the sense
of what was said.
"""
base = self.valves.owuitools_url.rstrip("/")
async with session.get(f"{base}/vm/confirm/{chat_id}", headers=headers) as resp:
if resp.status != 200:
return
pending = json.loads(await resp.text())
request_id = str(pending.get("id") or "")
if not request_id or request_id in asked:
return
asked.add(request_id)
machine = pending.get("machine") or "test"
# "a Ubuntu machine" reads as a typo. The two names are fixed, so
# the article is decided by the name rather than by a rule.
article = "an" if machine[:1].upper() in "AEIOU" else "a"
project = pending.get("project") or ""
where = f"Kept in '{project}'." if project else "Nothing will be kept."
# EVERY LINE HERE IS PAID FOR TWICE.
#
# Open WebUI does not scroll this dialog, so anything that does not
# fit pushes OK and Cancel off the bottom and the question cannot
# be answered at all. Measured in a 631px-tall window: a 21-line
# message put the buttons 65px past the edge. That is not an
# unusual window - it is a browser that is not maximised.
#
# So the framing is one line, the summary is five one-line bullets,
# and the instructions are one line, because the text box under
# them already says "Empty = run it. Or say what to change."
message = (
f"A throwaway {machine} machine, destroyed afterwards, no access to your "
f"network. {where}\n\n"
"WHAT IT WILL DO\n"
f"{self._summarise(str(pending.get('task') or ''))}\n\n"
"Empty + OK runs it \u00b7 type what to change \u00b7 Cancel drops it."
)
decision, text = "cancel", ""
try:
reply = await caller({
"type": "input",
"data": {"title": f"Start {article} {machine} machine?",
"message": message,
"placeholder": "Empty = run it. Or say what to change."},
})
except Exception: # noqa: BLE001
# If the question could not be put, the job is what was asked
# for. A confirmation that fails must not stop the work.
reply = ""
if reply is None or reply is False:
decision = "cancel"
else:
text = str(reply).strip()
decision = "change" if text else "run"
with contextlib.suppress(Exception):
await session.post(f"{base}/vm/confirm/{chat_id}", headers=headers,
json={"id": request_id, "decision": decision, "text": text})
# -- emitting ----------------------------------------------------------
@staticmethod
async def _status(emitter, description: str, done: bool = False) -> None:
await emitter({"type": "status",
"data": {"description": f"VM · {description}" if not done else description,
"done": done, "hidden": False}})
@staticmethod
async def _embed(emitter, data: dict, valves) -> None:
"""Replace the message's live view: the task list, then the machine.
TWO BOXES, NOT ONE, AND NEITHER OF THEM SCROLLS.
Open WebUI renders each entry of `embeds` in its OWN iframe, so
two entries is two boxes - which is what was asked for.
HOW A BOX GETS THE RIGHT HEIGHT, which is not what it looks like.
Open WebUI tries twice to size one of these frames:
1. on load it reads `contentDocument.body.scrollHeight + 20`;
2. it listens for `postMessage({type:"iframe:height", height})`
from the frame and sets that height exactly.
The first one CANNOT WORK HERE and fails silently. The frame is
sandboxed without `allow-same-origin`, so `contentDocument` throws
a cross-origin error, and the read sits inside a bare `catch {}`.
The height is then never set at all and the frame keeps its
stylesheet default of 150px - which is the whole reason the box
had a scrollbar. Measured: a seven-line task list rendering 168px
tall inside a 150px frame.
So the frame measures itself and posts its own height, which is
route 2 and needs no same-origin. `allowScripts` is hard-coded on
for embeds, so the script runs. It reports on load, on every
resize, and once more shortly after - the last of those is for a
screenshot that has not finished decoding when the rest is ready.
"""
tasks = str(data.get("tasks") or "")
speech = str(data.get("speech") or "")
status = str(data.get("status") or "")
shot = str(data.get("shot") or "")
terminal = list(data.get("terminal") or [])
if not tasks and not speech and not shot and not terminal:
await emitter({"type": "embeds", "data": {"replace": True, "embeds": []}})
return
def esc(text: str) -> str:
return (text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;"))
def frame(inner: str, background: str, kind: str, **config) -> str:
# The config goes in as JSON rather than as generated JavaScript:
# it carries the model's own words, and a quote or a backslash in
# those would otherwise end the script. `</` is escaped because
# the HTML parser ends this block at the first `</script>`,
# wherever it appears - and `\/` is a legal JSON escape.
payload = json.dumps({"k": kind, **config}).replace("</", "<\\/")
return (
"<style>html,body{margin:0;padding:0;overflow:hidden}"
"*{box-sizing:border-box}</style>"
f'<div id="b" style="font:12px/1.45 ui-monospace,SFMono-Regular,Menlo,'
f'monospace;color:#c9c9c9;background:{background};border-radius:6px;'
f'padding:8px 10px;overflow:hidden">{inner}</div>'
f'<script type="application/json" id="cfg">{payload}</script>'
+ _VIEW_SCRIPT
)
embeds = []
# BOX ONE: what the model is saying, then the list it is working to.
#
# The live line sits above the task list and carries no label. It
# is the model's own voice mid-turn, and a turn on this model runs
# tens of seconds - without it the box is motionless for most of a
# session and "still thinking" looks the same as "wedged". One
# line, and the script above reveals it a few characters at a time
# so that its speed is the token rate rather than a poll artefact.
#
# It is right-anchored: the server sends the TAIL of the sentence,
# so the end is the part that is moving and the end is what stays
# against the edge while the rest slides off to the left.
#
# Clicked, the box becomes those two lines and nothing else: the
# live line, and the step the session is on.
if tasks or speech:
rows = ('<div id="live" style="display:flex;justify-content:flex-end;'
'overflow:hidden;min-height:17px;color:#8f8f8f">'
f'<span id="t" style="white-space:pre">{esc(speech)}</span></div>')
if tasks:
rows += '<div id="full">'
rows += '<div style="height:6px"></div>'
rows += '<div style="color:#9aa0a6">Task List:</div>'
rows += "".join(
f'<div style="white-space:pre-wrap;word-break:break-word">{esc(line)}</div>'
for line in tasks.splitlines()
)
rows += "</div>"
rows += ('<div id="cur" style="display:none;white-space:pre;overflow:hidden;'
f'text-overflow:ellipsis;color:#9aa0a6">{esc(status)}</div>')
embeds.append(frame(rows, "#1b1b1b", "tasks", s=speech))
# BOX TWO, ONLY ONCE THERE IS SOMETHING TO SHOW.
#
# It used to appear for every desktop session, from the handover
# onwards, which meant a picture of an idle desktop sat in the
# chat before anything had happened - and most jobs never look at
# the screen at all. The operator's words: pop the preview up when
# the agent takes a screenshot. So this box exists only when a
# screenshot does, and its arrival means something.
#
# Clicked, the picture goes to half size. It keeps its shape, so
# half the height is half the width, and the black the box is
# filled with shows either side of it.
if valves.show_screenshots and shot:
caption = esc(str(data.get("caption") or ""))[:200]
embeds.append(frame(
f'<div id="wrap"><img id="shot" src="{shot}" style="width:100%;'
'height:auto;display:block;border-radius:4px" '
'alt="the machine\'s screen"></div>'
f'<div style="padding-top:4px;color:#888">{caption}</div>',
"#101010", "screen"))
# BOX THREE: the command feed, always, at one height. It used to be
# halved when a screen was shown above it; the screen is now the
# exception rather than the rule, so the feed is not shrunk for
# something that may never appear. Clicked, it shows half as many
# lines - the newest half, because the rows sit at the bottom of
# the box and it is the top that is cut off.
if valves.show_terminal:
height = TERMINAL_LINES
# Exactly `height` rows, always, so the box does not change
# size as output arrives; each row clips at the right edge
# instead of wrapping or scrolling sideways.
lines = terminal[-height:]
lines = [""] * (height - len(lines)) + lines
# `flex:none` is load-bearing. The rows sit in a flex column so
# that a collapsed feed keeps its NEWEST lines against the
# bottom and loses the oldest off the top. Flex items shrink by
# default, so without this the rows squeezed into the smaller
# box and the text overlapped itself instead of overflowing -
# measured, twenty lines crushed into ten lines of space.
rows = "".join(
'<div style="white-space:pre;overflow:hidden;text-overflow:ellipsis;'
f'flex:none">{esc(line) or "&nbsp;"}</div>'
for line in lines
)
embeds.append(frame(
'<div id="rows" style="display:flex;flex-direction:column;'
f'justify-content:flex-end;overflow:hidden">{rows}</div>',
"#101010", "feed", n=height))
await emitter({"type": "embeds", "data": {"replace": True, "embeds": embeds}})