Grant/owui-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial Commit

Browse Source
This commit is contained in:
Grant
2026-09-15 15:24:03 -04:00
commit 4dc7086e7e
605 changed files with 101380 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
owui-functions/answer_sources.py
+169
View File
@@ -0,0 +1,169 @@
"""
title: Answer Sources
author: owuitools
version: 0.2.0
description: Ends a tool-assisted answer with the URLs of the documents the tools actually read, when the answer names none of them.
"""
# WHY THIS EXISTS
#
# Across three graded rounds of forty prompts, the rule "end a tool-assisted
# answer with the URL you drew on" was the one instruction that never landed
# reliably: it lived in the system prompt (ignored), then in a `Sources:`
# trailer on every tool result (lifted into the answer on roughly half of
# the opportunities). The remaining half is the same failure each time - a
# correct answer, read from the right page, that does not say so - and it is
# the single largest bucket of lost points left.
#
# The model has the information; it just does not write it down. So this
# filter writes it down. After the turn, it collects the documents the tools
# report having READ (their `Sources:` trailers - fetched pages, Wikipedia
# articles, RFCs, manual pages, package records) and, if the answer carries
# none of those URLs and no sources line of its own, appends one.
#
# It never adds a search result: a URL that appeared in a `web_search`
# listing was not opened, and attributing an answer to it would be exactly
# the mislabelling the rounds penalise. Only tool outputs that end with a
# `Sources:` line contribute, and only successful reads produce that line.
#
# The line goes into BOTH content and the last message item of output, for
# the reason inline_tool_images.py records: Open WebUI renders from output
# when it is present. Everything fails closed - on any surprise the message
# is returned untouched.
import re
from pydantic import BaseModel, Field
_MARKER = "<!-- answer-sources -->"
_SOURCES_LINE = re.compile(r"^Sources:\s*(.+?)\s*$", re.MULTILINE)
_URL = re.compile(r"https?://[^\s<>\"')\]]+")
_DOI = re.compile(r"\bdoi:\s*(10\.\d{4,9}/\S+)", re.IGNORECASE)
# A line the model wrote itself that already serves as attribution. "Key
# sources:", "Sources used:" and "**References:**" all count; a research
# answer with its own labelled sources block got a second, worse one
# appended under it because the label did not start with the word.
_OWN_SOURCES = re.compile(
r"^\s*(?:[*_#>\-\s]*)(?:[\w-]+\s+){0,2}(?:sources?|references?|citations?|bibliography)"
r"(?:\s+(?:used|consulted|read|cited))?\s*[*_]*\s*[:\-–—]",
re.IGNORECASE | re.MULTILINE,
)
_BODY_DOI = re.compile(r"\b10\.\d{4,9}/[^\s<>\"')\]]+", re.IGNORECASE)
def _normalise(url: str) -> str:
url = url.strip().rstrip(".,;:")
url = re.sub(r"^https?://(www\.)?", "", url, flags=re.IGNORECASE)
return url.rstrip("/").lower()
def _documents_from(text: str) -> list[tuple[str, set[str]]]:
"""One (url, aliases) per `Sources:` line in a tool result, in order.
A trailer reads `Sources: <CITE AS line with a DOI> · <page URL>`. That is
one document, not two: counted as two it filled the cap twice as fast, and
the doi.org form and the page form of the same paper were both listed. The
page URL is what was read and is what gets shown; the DOI is kept as an
alias so a bare doi: in the answer counts as naming it.
"""
docs: list[tuple[str, set[str]]] = []
for match in _SOURCES_LINE.finditer(text or ""):
urls, dois = [], []
for entry in match.group(1).split(" · "):
entry = " ".join(entry.split())
urls += [u.rstrip(".,;:)") for u in _URL.findall(entry)]
dois += [d.group(1).rstrip(".,;:)") for d in _DOI.finditer(entry)]
for u in list(urls):
if "doi.org/" in u.lower():
dois.append(u.split("doi.org/", 1)[1])
page = next((u for u in reversed(urls) if "doi.org/" not in u.lower()), None)
shown = page or (urls[-1] if urls else None) or (f"https://doi.org/{dois[0]}" if dois else None)
if shown is None:
continue
aliases = {_normalise(u) for u in urls} | {d.lower() for d in dois}
docs.append((shown, aliases))
return docs
class Filter:
class Valves(BaseModel):
priority: int = Field(default=10, description="Run after the other outlet filters.")
enabled: bool = Field(default=True, description="Turn the filter off without deleting it.")
max_sources: int = Field(
default=8,
description="Most documents to list, in reading order. A long research turn can "
"read twenty pages; one document per Sources: line, so a paper's DOI and its page "
"URL count once.",
)
label: str = Field(default="Sources", description="Label for the appended line.")
def __init__(self):
self.valves = self.Valves()
async def outlet(self, body: dict, __user__=None, __metadata__=None) -> dict:
if not self.valves.enabled:
return body
try:
messages = body.get("messages") or []
target = None
for message in reversed(messages):
if message.get("role") == "assistant" and message.get("output"):
target = message
break
if target is None:
return body
content = target.get("content") or ""
if not content.strip() or _MARKER in content:
return body
# Documents the tools read this turn, in the order they were read.
entries: dict[str, str] = {}
aliases: set[str] = set()
for item in target.get("output") or []:
if item.get("type") != "function_call_output":
continue
for part in item.get("output") or []:
if part.get("type") not in ("input_text", "output_text", "text"):
continue
for url, names in _documents_from(part.get("text") or ""):
key = _normalise(url)
if key not in entries and not (names & aliases):
entries[key] = url
aliases |= names | {key}
if not entries:
return body
# The answer already attributes itself: it names one of the read
# URLs or DOIs, or carries a sources line of its own. Leave it
# alone - a second sources block under the model's own is
# clutter, and the model's choice of which page to cite is not
# ours to edit.
lowered = content.lower()
if any(key in lowered for key in entries):
return body
if any(d.lower() in aliases for d in _BODY_DOI.findall(content)):
return body
if _OWN_SOURCES.search(content):
return body
urls = list(entries.values())[: max(1, self.valves.max_sources)]
label = (self.valves.label or "Sources").strip()
visible = "\n\n---\n" + f"{label}: " + " · ".join(urls)
target["content"] = content.rstrip() + visible + "\n" + _MARKER
output_items = target.get("output") or []
message_items = [o for o in output_items if o.get("type") == "message"]
if message_items:
last = message_items[-1]
parts = last.get("content") or []
texts = [p for p in parts if p.get("type") == "output_text"]
if texts:
texts[-1]["text"] = (texts[-1].get("text") or "").rstrip() + visible
else:
parts.append({"type": "output_text", "text": visible.lstrip()})
last["content"] = parts
except Exception:
return body
return body
owui-functions/inline_tool_images.py
+199
View File
@@ -0,0 +1,199 @@
"""
title: Inline Tool Images
author: owuitools
version: 0.1.0
description: Shows images returned by tool calls (e.g. matlab_run figures) inline in the chat.
"""
# WHY THIS EXISTS
#
# When an OpenAPI tool server returns a result that IS a 'data:image/...'
# URI, Open WebUI lifts it out and - on the native tool-calling path -
# routes it to the MODEL as a vision input:
#
# utils/middleware.py
# if file_item['type'] == 'image' and file_item['url'].startswith('data:'):
# output_parts.append({'type': 'input_image', ...}) # LLM-only
# else:
# display_files.append(file_item) # user sees this
#
# So the model can see the picture and the user cannot. Only a NON-data
# URL reaches the frontend. This filter closes that gap without touching
# the tool server, the network policy, or the model: the image is already
# sitting in the finished message, so we upload it through Open WebUI's
# own helper - which stores it, links it to the message and hands back a
# real file URL - and append that URL to the message as markdown.
#
# The markdown has to go into BOTH content and the last message item of
# output. Open WebUI renders from output when it is present and falls
# back to content only when it is not (get_output_text), so editing
# content alone stores the image and shows nothing.
#
# Everything here fails closed. If an import moves in a future Open WebUI
# release, or an upload fails, the message is returned untouched: no
# picture is a nuisance, a corrupted answer is not.
import json
from pydantic import BaseModel, Field
# Marker left in the appended block so a second pass over the same message
# cannot upload the same image twice. Each upload mints a NEW url, so
# checking for a previous url would never match.
#
# It goes in `content` ONLY. Open WebUI renders the `output` text, and its
# markdown renderer does not hide HTML comments - the marker appeared under
# the image as literal "<!-- inline-tool-images -->". content is not
# rendered, so the guard can live there and stay invisible.
_MARKER = "<!-- inline-tool-images -->"
class Filter:
class Valves(BaseModel):
enabled: bool = Field(
default=True,
description="Turn the whole filter off without deleting it.",
)
max_images: int = Field(
default=4,
description="Most images to attach to one message. A runaway loop that plots on "
"every iteration should not paste twenty figures into the chat.",
)
strip_base64_from_output: bool = Field(
default=True,
description="Remove the base64 copy from the stored message once it has been "
"uploaded. The model has already seen it by this point, and leaving it behind "
"keeps roughly 11 KB per figure in the chat record.",
)
caption: str = Field(
default="figure",
description="Alt text used for the appended images.",
)
def __init__(self):
self.valves = self.Valves()
async def outlet(
self,
body: dict,
__request__=None,
__user__=None,
__metadata__=None,
) -> dict:
if not self.valves.enabled or __request__ is None:
return body
try:
from open_webui.models.users import Users
from open_webui.utils.files import get_image_url_from_base64
except Exception:
return body
try:
messages = body.get("messages") or []
target = None
for message in reversed(messages):
if message.get("role") == "assistant" and message.get("output"):
target = message
break
if target is None:
return body
content = target.get("content") or ""
output_items = target.get("output") or []
if _MARKER in content or _MARKER in str(output_items):
return body
# Which figure did each tool call ask for? A model that plots,
# looks at the result, spots a problem and re-plots plainly means
# the second one - but both images are sitting in the output, and
# appending both showed the user a picture the model had already
# rejected. Grouping by the requested filename separates a
# REFINEMENT (same name, keep the last) from a genuinely second
# figure (different name, keep both).
figure_by_call = {}
for item in output_items:
if item.get("type") != "function_call":
continue
call_id = item.get("call_id")
if not call_id:
continue
args = item.get("arguments")
if isinstance(args, str):
try:
args = json.loads(args)
except Exception:
args = {}
if isinstance(args, dict):
figure_by_call[call_id] = args.get("return_figure") or ""
# Keyed, insertion-ordered, last write wins.
found_by_key = {}
for item in output_items:
if item.get("type") != "function_call_output":
continue
call_id = item.get("call_id")
for part in item.get("output") or []:
if part.get("type") != "input_image":
continue
url = part.get("image_url") or ""
if not url.startswith("data:image/"):
continue
# Falling back to the call id keeps every image when the
# filename cannot be resolved: showing one image too many
# beats silently dropping the only one.
key = figure_by_call.get(call_id) or "call:%s" % call_id
found_by_key[key] = (part, url)
found = list(found_by_key.values())
if not found:
return body
user_id = (__user__ or {}).get("id")
user = await Users.get_user_by_id(user_id) if user_id else None
if user is None:
return body
metadata = {
"chat_id": body.get("chat_id"),
"message_id": target.get("id"),
}
urls = []
for part, data_uri in found[: max(1, self.valves.max_images)]:
try:
url = await get_image_url_from_base64(__request__, data_uri, metadata, user)
except Exception:
url = None
if not url:
continue
urls.append(url)
if self.valves.strip_base64_from_output:
part["image_url"] = url
if not urls:
return body
caption = self.valves.caption or "figure"
block = "\n".join(f"![{caption}]({url})" for url in urls)
visible = f"\n\n{block}"
target["content"] = content.rstrip() + visible + "\n" + _MARKER
# The one that actually shows: the visible answer is the last
# message item in output, and everything before it is
# reasoning and tool traffic.
message_items = [o for o in output_items if o.get("type") == "message"]
if message_items:
last = message_items[-1]
parts = last.get("content") or []
texts = [p for p in parts if p.get("type") == "output_text"]
if texts:
texts[-1]["text"] = (texts[-1].get("text") or "").rstrip() + visible
else:
parts.append({"type": "output_text", "text": visible.lstrip()})
last["content"] = parts
except Exception:
return body
return body
owui-functions/live_progress_status.py
+570
View File
@@ -0,0 +1,570 @@
"""
title: Live Progress Status
author: owuitools
version: 0.2.0
description: Shows what the model is doing right now - processing the prompt, thinking, and each tool call as it is written.
"""
# WHY THIS EXISTS
#
# Until the first token arrives, Open WebUI shows nothing at all. On this
# deployment the prompt is ~9.4k tokens of system prompt plus 20 tool
# schemas, and a forced re-prefill takes many seconds, so "nothing at all"
# is the most common thing on screen. Worse, the model spends ~70% of its
# generated characters thinking, and tool calls are written token by token
# into a block that is collapsed by default - so even once it IS producing
# output, there is no visible signal of which phase it is in.
#
# This filter emits Open WebUI status events on phase CHANGES only:
#
# Processing prompt <- inlet, before the model is called
# Thinking <- first reasoning delta
# Thought: <opening of it> <- when a reasoning block ends
# Calling web_search <- a tool call is being written
# Writing response <- first content delta
#
# Together those read as a narrative of the whole turn, which is what makes
# the collapsed tool blocks acceptable: the chain of thought and the tool
# sequence stay visible without any tool input or output on screen.
#
# WHERE THE NUMBERS COME FROM
#
# The status line used to claim "prompt processed in N s" for a figure that
# was really inlet-to-first-token wall clock: Open WebUI middleware, queue
# wait and model load included. Graded runs put it 2-3x either side of
# Ollama's own prefill for the same turn, so the one number on screen was
# the one number that was wrong; 35 of 40 rubrics flagged it.
#
# Ollama's real figures are available, and this filter now reads them
# instead of guessing. Open WebUI's Ollama->OpenAI conversion builds a
# usage object on the final chunk of each round:
#
# utils/response.py convert_ollama_usage_to_openai()
# 'prompt_eval_count', 'prompt_eval_duration', (nanoseconds)
# 'eval_count', 'eval_duration', 'total_duration', 'load_duration'
# utils/misc.py openai_chat_chunk_message_template()
# template['usage'] = usage <- TOP LEVEL, beside 'choices'
#
# and stream filters are handed that chunk verbatim, before any
# normalisation (middleware.py, filter_type='stream'). The same numbers
# reach outlet on the stored message's 'usage', so outlet can fall back to
# it - with one caveat that is worth knowing when the two disagree:
# merge_usage() sums only input_tokens/output_tokens and keeps the LATEST
# value of every provider-specific key, so a fallback prefill is the last
# round's warm re-prefill, while the stream path deliberately keeps the
# FIRST one, which is the cold prefill the user actually waited through.
#
# WHY THIS CANNOT AFFECT THE MODEL'S OUTPUT
#
# Status events travel over the websocket and are stored on a SIBLING key
# of the message ('statusHistory'), never in 'content'. The payload sent to
# the model is rebuilt from scratch by an explicit allowlist:
#
# utils/payload.py convert_messages_openai_to_ollama()
# new_message = {'role': message['role']}
# # then only: thinking, content, tool_calls, tool_call_id, images
#
# 'statusHistory' is not in that list, and the backend never references it
# outside persistence. The token sequence is therefore byte-identical with
# this filter on or off - no prompt change, no prefix-cache perturbation.
# The one thing that does reach the chat record is the empty-answer note
# below, and only on a turn that produced no answer to disturb.
#
# TWO SHARP EDGES IN THE STREAM HOOK, both load-bearing:
#
# 1. The return value REPLACES the chunk, and a FALSY return DROPS it:
# data = wrap_item(JSONCodec.dumps(event)) if event else None
# So stream() must return `event` on every single path. Returning None
# would silently delete the model's output.
# 2. Exceptions from a stream filter are re-raised by the caller
# (utils/filter.py logs then `raise`), which would break the response
# mid-flight. So the whole body is wrapped and always falls through to
# `return event`.
#
# The module object is cached across requests (load_from_db=False for
# stream), so per-request state must be keyed by message id rather than
# held on self - six users share this instance.
import time
from pydantic import BaseModel, Field
# Per-message progress state, keyed by message id. Bounded below so a
# request whose outlet never runs (client disconnect, cancelled generation)
# cannot leak entries forever.
_STATE: dict[str, dict] = {}
_MAX_TRACKED = 64
# Shown in the message itself when the turn ended with no answer text. One
# graded run spent 140 s and 8,201 tokens repeating a single sentence inside
# its reasoning block, emitted nothing, and the indicator said "Done" - so
# the failure looked exactly like a success that had scrolled off screen.
# 0.11.3's filter API is inlet/stream/request/outlet only, with no hook that
# re-enters the chat pipeline, so telling the user to regenerate is the most
# this filter can do.
_EMPTY_NOTE = (
"_No answer was produced: the model ended its turn while still reasoning. "
"Regenerate to try again._"
)
def _new_state() -> dict:
"""The per-message progress record. One factory, used by both creation
sites - they were duplicated literals once and silently drifted."""
return {
"started": time.monotonic(),
"phase": None,
"announced": set(),
"round": 0,
"prompt_reported": False,
"tools": 0,
"reasoning": [],
# Ollama's own figures, filled in from the usage chunk. prefill is
# first-wins (the cold one); generated tokens accumulate over every
# round of a tool-calling turn.
"prefill_s": None,
"prompt_tokens": None,
"gen_tokens": 0,
}
def _prune() -> None:
while len(_STATE) > _MAX_TRACKED:
# dicts are insertion-ordered, so this drops the oldest.
_STATE.pop(next(iter(_STATE)), None)
def _number(value) -> float | None:
"""Usage values arrive as whatever the provider sent. Anything that is
not a real number (None, 'N/A', a string) is treated as absent."""
if isinstance(value, bool) or not isinstance(value, (int, float)):
return None
return float(value)
class Filter:
class Valves(BaseModel):
enabled: bool = Field(
default=True,
description="Turn the whole filter off without deleting it.",
)
show_prompt_processing: bool = Field(
default=True,
description="Show 'Processing prompt' before the first token. This is the phase "
"that currently shows nothing at all, and on a cold prompt it is the longest.",
)
show_thinking: bool = Field(
default=True,
description="Show 'Thinking' once the model starts producing reasoning tokens.",
)
show_tool_calls: bool = Field(
default=True,
description="Show 'Calling <tool>' as each tool call is written. Tool calls are "
"still counted for the final line when this is off.",
)
report_prompt_time: bool = Field(
default=True,
description="Append timing to the status line: how long the first token took to "
"arrive, and - on the final line, from Ollama's own counters - how long the "
"prefill took and how many tokens went in and came out.",
)
thought_excerpt_chars: int = Field(
default=160,
description="When a block of reasoning ends, post an excerpt of it as a status so "
"the timeline reads as a narrative of what the model was doing. 0 disables it. "
"The renderer clamps a status to one line, so values much above ~180 are wasted.",
)
clear_when_done: bool = Field(
default=False,
description="Hide the status line entirely once the response finishes. Off by "
"default so the finished message keeps a short record of what it did, which "
"matches how Open WebUI's own built-in tools behave.",
)
def __init__(self):
self.valves = self.Valves()
# ---------------------------------------------------------------- utils
async def _emit(self, emitter, description: str, *, done: bool = False, hidden: bool = False) -> None:
if emitter is None:
return
try:
await emitter(
{
"type": "status",
"data": {"description": description, "done": done, "hidden": hidden},
}
)
except Exception:
# A status line is a nicety; never let it disturb the response.
pass
def _message_id(self, metadata) -> str | None:
if isinstance(metadata, dict):
return metadata.get("message_id") or metadata.get("chat_id")
return None
def _state(self, message_id: str | None) -> dict | None:
if not message_id:
return None
state = _STATE.get(message_id)
if state is None:
# stream() can legitimately run without inlet having recorded
# anything, so build a usable state rather than giving up.
state = _new_state()
_STATE[message_id] = state
_prune()
return state
def _record_usage(self, state: dict, usage: dict) -> None:
"""Take Ollama's own prefill and token counts off a usage chunk.
Prefill is first-wins because a tool-calling turn prefills once per
round and only the first one is the wait the user sat through; every
later round is served warm off the prefix cache. Generated tokens
accumulate, because all of them were generated on this turn.
"""
if state["prefill_s"] is None:
duration = _number(usage.get("prompt_eval_duration"))
if duration and duration > 0:
state["prefill_s"] = duration / 1e9
if state["prompt_tokens"] is None:
count = _number(usage.get("prompt_eval_count"))
if count and count > 0:
state["prompt_tokens"] = int(count)
generated = _number(usage.get("eval_count"))
if generated and generated > 0:
state["gen_tokens"] += int(generated)
def _final_usage(self, state: dict | None, message: dict | None) -> tuple:
"""(prefill_s, prompt_tokens, gen_tokens) for the final line.
The stream numbers are preferred; the stored message's usage is the
fallback for the paths where stream() never saw a usage chunk - a
non-streaming response, or a filter loaded after the stream began.
"""
prefill_s = state["prefill_s"] if state else None
prompt_tokens = state["prompt_tokens"] if state else None
gen_tokens = state["gen_tokens"] if state else 0
usage = message.get("usage") if isinstance(message, dict) else None
if isinstance(usage, dict):
if prefill_s is None:
duration = _number(usage.get("prompt_eval_duration"))
if duration and duration > 0:
prefill_s = duration / 1e9
if prompt_tokens is None:
count = _number(usage.get("prompt_eval_count")) or _number(usage.get("prompt_tokens"))
if count and count > 0:
prompt_tokens = int(count)
if not gen_tokens:
# output_tokens is the one Open WebUI sums across rounds;
# the Ollama-native keys hold the last round only.
generated = (
_number(usage.get("output_tokens"))
or _number(usage.get("completion_tokens"))
or _number(usage.get("eval_count"))
)
if generated and generated > 0:
gen_tokens = int(generated)
return prefill_s, prompt_tokens, gen_tokens
def _usage_clause(self, prefill_s, prompt_tokens, gen_tokens) -> str:
"""The measured half of the final line. Empty when nothing was
measured - an absent clause is honest, an invented one is not."""
if not self.valves.report_prompt_time:
return ""
parts = []
if prefill_s is not None:
if prompt_tokens:
parts.append(f"prefill {prefill_s:.1f} s for {prompt_tokens:,} tokens")
else:
parts.append(f"prefill {prefill_s:.1f} s")
if gen_tokens:
parts.append(f"{gen_tokens:,} generated")
return "".join(f" · {part}" for part in parts)
async def _flush_thought(self, state: dict, emitter) -> None:
"""Post an excerpt of the reasoning block that just ended, then drop it.
This is what makes the collapsed view usable: the tool payloads stay
shut, but the status timeline still reads as a narrative of what the
model was thinking between calls. The buffer is cleared either way, so
a disabled excerpt cannot accumulate text for the life of the request.
The excerpt shows both ends of the block, not just the opening. Head
truncation once surfaced "…I should verify with actual execution since
I have tools. Actually, I" - cutting the sentence exactly where the
model reversed itself, so a user watching live would have believed it
was about to run code it had in fact just decided not to run. The
conclusion of a thought lives at its end.
"""
buffered = state["reasoning"]
state["reasoning"] = []
limit = self.valves.thought_excerpt_chars
if limit <= 0 or not buffered:
return
# Collapse to a single line - the renderer clamps to one line anyway,
# and newlines in a status read as ragged whitespace.
text = " ".join("".join(buffered).split())
if not text:
return
if len(text) > limit:
head_budget = max(1, int(limit * 0.55))
tail_budget = max(1, int(limit * 0.40))
# Prefer breaking at a word boundary near each edge.
cut = text.rfind(" ", int(head_budget * 0.6), head_budget)
head = text[: cut if cut != -1 else head_budget].rstrip()
start = len(text) - tail_budget
space = text.find(" ", start)
if space != -1 and space + 1 < len(text):
start = space + 1
text = f"{head} … {text[start:].lstrip()}"
await self._emit(emitter, f"Thought: {text}", done=True)
def _suffix(self, state: dict) -> str:
"""Report, once, how long the first token took to arrive.
Deliberately not called prefill: this is inlet-to-first-token wall
clock and includes Open WebUI's own work, the queue and any model
load. The real prefill is not known until the usage chunk at the end
of the round, and it goes on the final line instead.
"""
if state["prompt_reported"] or not self.valves.report_prompt_time:
return ""
state["prompt_reported"] = True
elapsed = time.monotonic() - state["started"]
if elapsed < 0.5:
return ""
return f" · first token after {elapsed:.1f} s"
def _mark_empty(self, message: dict) -> None:
"""Leave a visible note on a turn that produced no answer text.
Both places, as inline_tool_images.py does it for its images: Open
WebUI renders `output` when it is present and falls back to `content`
only when it is not (misc.py get_output_text), so writing content
alone stores the note and shows nothing.
"""
content = message.get("content") or ""
message["content"] = (content.rstrip() + "\n\n" + _EMPTY_NOTE).lstrip()
output_items = message.get("output")
if not isinstance(output_items, list):
return
message_items = [
item for item in output_items if isinstance(item, dict) and item.get("type") == "message"
]
if message_items:
last = message_items[-1]
parts = last.get("content")
if not isinstance(parts, list):
parts = []
texts = [p for p in parts if isinstance(p, dict) and p.get("type") == "output_text"]
if texts:
texts[-1]["text"] = (texts[-1].get("text") or "").rstrip() + "\n\n" + _EMPTY_NOTE
else:
parts.append({"type": "output_text", "text": _EMPTY_NOTE})
last["content"] = parts
else:
# An output list of nothing but reasoning and tool traffic renders
# as an empty message, and the note in `content` would never be
# reached. Give it the one item the renderer reads.
output_items.append(
{
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": _EMPTY_NOTE}],
}
)
# ---------------------------------------------------------------- hooks
async def inlet(self, body: dict, __event_emitter__=None, __metadata__=None) -> dict:
"""Mark the start of prompt processing. `body` is returned untouched."""
if not self.valves.enabled:
return body
try:
message_id = self._message_id(__metadata__)
continued = False
if message_id:
state = _STATE.get(message_id)
# inlet runs a SECOND time for the same assistant message when
# the turn contains an interactive tool call - the native
# ask_user widget re-enters the pipeline. Wiping the state
# there zeroed the tool counter mid-turn, and a turn with
# three real calls reported "Done · 2 tool calls". Keep what
# has already been counted; reset only the per-round fields.
if state and (state["tools"] or state["phase"]):
continued = True
state["round"] += 1
state["phase"] = None
state["started"] = time.monotonic()
state["prompt_reported"] = False
else:
_STATE[message_id] = _new_state()
_prune()
if self.valves.show_prompt_processing:
await self._emit(
__event_emitter__,
"Processing prompt (continued)" if continued else "Processing prompt",
done=False,
)
except Exception:
pass
# Never modified. This filter must not touch the prompt.
return body
async def stream(self, event: dict, __event_emitter__=None, __metadata__=None) -> dict:
"""Announce phase changes. MUST return `event` on every path."""
if not self.valves.enabled:
return event
try:
if not isinstance(event, dict):
return event
state = self._state(self._message_id(__metadata__))
if state is None:
return event
# Read before the `choices` checks below: the usage chunk carries
# usage at the top level and an empty delta, and some providers
# send it with no choices at all.
usage = event.get("usage")
if isinstance(usage, dict):
self._record_usage(state, usage)
choices = event.get("choices")
if not isinstance(choices, list) or not choices:
return event
first = choices[0]
if not isinstance(first, dict):
return event
delta = first.get("delta")
if not isinstance(delta, dict):
return event
tool_calls = delta.get("tool_calls")
if tool_calls:
for call in tool_calls if isinstance(tool_calls, list) else []:
if not isinstance(call, dict):
continue
function = call.get("function")
name = function.get("name") if isinstance(function, dict) else None
if not name:
continue
# A tool id is the reliable key; index plus round is the
# fallback, so the same tool called twice still counts twice.
key = call.get("id") or f"{state['round']}:{call.get('index', 0)}"
if key in state["announced"]:
continue
state["announced"].add(key)
# Counting, de-duplication and the phase change all sit
# OUTSIDE the display valve: they used to be inside it, so
# turning the announcements off silently zeroed the final
# tool count as well, and stalled the round counter that
# the de-duplication key depends on. Only the announcement
# itself is optional.
state["tools"] += 1
# The reasoning that led to this call belongs before it.
await self._flush_thought(state, __event_emitter__)
state["phase"] = "tool"
if self.valves.show_tool_calls:
await self._emit(
__event_emitter__,
f"Calling {name}{self._suffix(state)}",
done=False,
)
return event
reasoning = delta.get("reasoning_content")
if reasoning:
if isinstance(reasoning, str) and self.valves.thought_excerpt_chars > 0:
# Only keep what an excerpt could possibly need, so a long
# think cannot grow this without bound.
if sum(len(p) for p in state["reasoning"]) < 4096:
state["reasoning"].append(reasoning)
if self.valves.show_thinking and state["phase"] != "thinking":
if state["phase"] == "tool":
state["round"] += 1
state["phase"] = "thinking"
await self._emit(
__event_emitter__, f"Thinking{self._suffix(state)}", done=False
)
elif delta.get("content"):
if state["phase"] != "writing":
if state["phase"] == "tool":
state["round"] += 1
await self._flush_thought(state, __event_emitter__)
state["phase"] = "writing"
await self._emit(
__event_emitter__,
f"Writing response{self._suffix(state)}",
done=False,
)
except Exception:
pass
return event
async def outlet(self, body: dict, __event_emitter__=None, __metadata__=None) -> dict:
"""Settle the status line and drop the per-message state."""
if not self.valves.enabled:
return body
try:
message_id = self._message_id(__metadata__)
state = _STATE.pop(message_id, None) if message_id else None
# A response that ended while still reasoning still has something
# worth showing.
if state and not self.valves.clear_when_done:
await self._flush_thought(state, __event_emitter__)
messages = body.get("messages")
message = None
if isinstance(messages, list) and messages and isinstance(messages[-1], dict):
message = messages[-1]
prefill_s, prompt_tokens, gen_tokens = self._final_usage(state, message)
# Open WebUI builds outlet's `content` as content OR the rendered
# text of `output`, so blank here means the turn really produced
# no answer - not that the answer lives somewhere else.
empty = (
message is not None
and message.get("role", "assistant") == "assistant"
and not (message.get("content") or "").strip()
)
if empty:
self._mark_empty(message)
if self.valves.clear_when_done and not empty:
# StatusHistory hides the whole block when the LATEST status
# is hidden, so this clears the line entirely.
await self._emit(__event_emitter__, "", done=True, hidden=True)
else:
if empty:
# Never hidden, even with clear_when_done: a turn that
# produced nothing is the one turn whose status matters.
# The token count is dropped rather than printed as 0 when
# no usage arrived - "(0 tokens)" would be a measurement
# nobody took.
spent = f" ({gen_tokens:,} tokens)" if gen_tokens else ""
summary = (
"Done · no answer produced — the model stopped inside its "
f"reasoning{spent}. Press Regenerate."
)
else:
tools = state["tools"] if state else 0
if tools == 1:
summary = "Done · 1 tool call"
elif tools > 1:
summary = f"Done · {tools} tool calls"
else:
summary = "Done"
summary += self._usage_clause(prefill_s, prompt_tokens, gen_tokens)
await self._emit(__event_emitter__, summary, done=True)
except Exception:
pass
return body
owui-functions/quantitative_question_nudge.py
+118
View File
@@ -0,0 +1,118 @@
"""
title: Quantitative Question Nudge
author: owuitools
version: 0.2.0
description: Raises tool-use salience on measurement-bearing questions that are phrased casually.
"""
# WHY THIS EXISTS
#
# The system prompt already says that reasoning tasks need tools and that
# ruling a cause out is itself a claim. It does not fire on questions that
# are phrased conversationally. Measured case:
#
# "im logging vibration off a motor at 500Hz, motor runs 3600 rpm but
# the biggest fft peak is at 40Hz. is my sensor broken?"
#
# Zero tool calls, and the answer stated "aliasing cannot move a 60 Hz peak
# to 40 Hz - so 40 Hz is a REAL vibration". True of the fundamental and
# wrong as an elimination: the 9th harmonic at 540 Hz folds to exactly
# 40.0 Hz at that sample rate. That sends someone hunting a phantom
# resonance instead of fitting an anti-alias filter. Adding a paragraph to
# the system prompt did not change it - which is this suite's most repeated
# finding: static prompt text loses to the shape of the conversation.
#
# An inlet note is a different lever. It arrives as the most recent thing
# in the conversation rather than as line 40 of a long preamble, so it is
# read as part of the current task. It is invisible to the user: inlet
# rewrites what goes to the model, not what is displayed.
#
# Deliberately narrow. It fires only when a message carries an actual
# measurement - a number with a unit - because that is what distinguishes
# "is my sensor broken" from "is Postgres a good choice". A nudge on every
# message would be one more paragraph the model learns to skim.
#
# WHY IT NAMES THE TOOLS
#
# The first version said "check it with a tool" and left the choice open.
# On a compound-interest question the model duly reached for a tool - and
# reached for web_search, to look up a formula it already knew, then did
# the actual arithmetic in its head and got it wrong. Tool-seeking without
# tool-selection. The note now says which tool does what, and says out loud
# that searching for a formula you know is not the point.
import re
from pydantic import BaseModel, Field
# A number bound to a unit. Bare numbers are excluded on purpose: "give me
# 3 options" is not a measurement. Currency and durations are in because
# "$4,000 at 5% for 10 years" is as quantitative as anything in volts, and
# the arithmetic is exactly where in-head answers go wrong.
_MEASUREMENT = re.compile(
r"[$£€]\s?\d"
r"|\b\d+(?:\.\d+)?\s?"
r"(?:hz|khz|mhz|ghz|rpm|v|kv|mv|a|ma|w|kw|mw|va|kva|ohm|ohms|"
r"db|dbm|s|ms|us|ns|m|mm|cm|km|kg|g|n|nm|pa|kpa|bar|psi|"
r"c|f|k|%|gb|mb|kb|tb|bps|kbps|mbps|gbps|awg|bit|bits|"
r"years?|yrs?|months?|weeks?|days?)\b",
re.I,
)
_NOTE = (
"Reminder for this question: it contains measurements, so it is a technical question "
"however casually it is worded. Before asserting any number, mechanism or standard, check "
"it with a tool: the calculators (ohms_law_power, component_calc, wiring_calc, pcb_calc, "
"subnet_calc) for what they cover, matlab_run for any other arithmetic - compounding, unit "
"conversions, statistics, counting - and web_search then fetch_page for facts and "
"standards, not for formulas you already know. In particular, if you are about to rule a "
"cause OUT, enumerate the variants of that cause and test the ones you have not considered "
"- a correct elimination of the obvious version and a missed elimination of a subtler "
"version look identical in the answer, and send the user to fix the wrong thing."
)
class Filter:
class Valves(BaseModel):
enabled: bool = Field(
default=True,
description="Turn the nudge off without deleting the filter.",
)
min_chars: int = Field(
default=40,
description="Skip very short messages. 'what is 3 dB' needs no reminder.",
)
def __init__(self):
self.valves = self.Valves()
async def inlet(self, body: dict, __user__=None) -> dict:
if not self.valves.enabled:
return body
try:
messages = body.get("messages") or []
if not messages:
return body
last_user = None
for message in reversed(messages):
if message.get("role") == "user":
last_user = message
break
if last_user is None:
return body
text = last_user.get("content")
if not isinstance(text, str) or len(text) < self.valves.min_chars:
return body
if not _MEASUREMENT.search(text):
return body
# Appended AFTER the user's turn, not before it. Inserted
# before, this was read as more preamble and changed nothing;
# the last message in the list is the one that gets weight.
messages.append({"role": "system", "content": _NOTE})
body["messages"] = messages
except Exception:
return body
return body
owui-functions/vm_agent_progress.py
+788
View File
@@ -0,0 +1,788 @@
"""
title: VM Agent Progress
author: owuitools
version: 0.1.0
description: While vm_agent is running, shows what the sub-agent is doing and, when it looks at the machine's screen, what it sees.
"""
# WHY THIS EXISTS
#
# A vm_agent call is one tool call that takes minutes to hours: a slow
# model with thinking on, driving a real VM. Open WebUI shows "Calling
# vm_agent" and then nothing at all until it returns. That is the single
# longest silence this deployment can produce, and it is silent precisely
# when the most is happening.
#
# WHY IT IS A FILTER AND NOT THE AGENT POSTING DIRECTLY
#
# The first design had the agent POST to
# /api/v1/chats/{id}/messages/{id}/event. That endpoint refuses a key
# whose user neither owns the chat nor is an admin - and anyone on this
# instance may invoke vm_agent, so it had to be an ADMIN key. An admin
# key is the ability to read every conversation on the instance and
# create users, and it would have had to live somewhere outside Open
# WebUI to be usable.
#
# Here there is no such key. This code runs INSIDE Open WebUI, in the
# same process as the chat pipeline, and __event_emitter__ is already
# bound to the requesting user's own session. The only credential it
# holds is the owuitools tool-server bearer token - which grants exactly
# the tools the model can already call, and nothing else.
#
# The cost is that progress is polled rather than pushed: a 2-second poll
# for the length of a session. That is the whole of the trade.
#
# WHY NONE OF THIS REACHES THE MODEL
#
# Open WebUI stores a message's status history and its `embeds` as
# SIBLING fields of the message, and rebuilds the payload sent to the
# model from an allowlist - role, thinking, content, tool_calls,
# tool_call_id, images (utils/payload.py). Neither `statusHistory` nor
# `embeds` is on that list. So a screenshot shown here is seen by the
# person watching and never by the model that called the tool, which is
# the entire reason for doing it this way rather than returning images
# from the tool itself.
#
# TWO SHARP EDGES IN THE stream() HOOK, both learned the hard way in
# live_progress_status.py and both still true here:
# * a falsy return value DROPS the chunk. Every path returns `event`.
# * an exception is re-raised by Open WebUI and breaks the response
# mid-flight. The whole body is wrapped.
from __future__ import annotations
import asyncio
import contextlib
import json
import re
import time
import aiohttp
from pydantic import BaseModel, Field
# Must match vmagent/app/events.py. Twenty lines is enough to see a
# command and what it said, and short enough that the view does not become
# the conversation.
TERMINAL_LINES = 20
# How often the whole view is re-sent even when it has not changed, so a
# viewer who reloads or joins late resyncs without waiting for the next
# change. See the note in _watch.
RESEND_SECONDS = 10.0
# How long a session may say nothing at all before the status line starts
# counting instead of repeating itself. See the note in _watch.
QUIET_SECONDS = 25.0
# The frame tells the page how tall it is. See _embed's note for why the
# other route - the page measuring the frame - cannot work here.
#
# It measures the CONTENT, not the document. `html{overflow:hidden}` makes
# `documentElement.scrollHeight` report the frame's current height whenever
# the content is shorter than it, so a box measured that way can grow but
# can never shrink again: measured, a seven-line list in a 150px frame
# reported 150 rather than its real 135. The wrapper's own bounding box has
# no such floor. A zero is never sent - a frame in a background tab has no
# layout and measures 0, and the observer reports the real height as soon
# as the tab is looked at.
# The one script every box carries: it measures itself, remembers whether
# it has been clicked shut, and reveals the live line at the rate the
# tokens arrived.
#
# WHY STATE LIVES IN window.name, WHICH LOOKS LIKE A HACK AND IS THE ONLY
# THING THAT WORKS HERE.
#
# The view is re-sent every couple of seconds, and each send replaces the
# frame's `srcdoc` - a new Document every time. The frame is sandboxed
# without `allow-same-origin`, so it has an opaque origin: localStorage
# and cookies throw, IndexedDB is useless, and there is no listener in
# Open WebUI to postMessage to. `window.name` belongs to the browsing
# CONTEXT rather than the document, and the context survives the swap.
# Measured in this exact sandbox: a frame reloaded three times read back
# the name its two predecessors had written. Without it a box would
# spring open again a second after being closed.
#
# The state is keyed by which box this is, because the boxes change
# places: the screen box appears mid-session and pushes the feed down one
# slot, and the feed would otherwise inherit the screen's state.
_VIEW_SCRIPT = """
<script>
(function(){
var b=document.getElementById('b');if(!b){return;}
var cfg={};try{cfg=JSON.parse(document.getElementById('cfg').textContent);}catch(e){}
var kind=cfg.k||'';
var st={};try{st=JSON.parse(window.name||'{}')||{};}catch(e){st={};}
if(st.k!==kind){st={k:kind};}
function save(){try{window.name=JSON.stringify(st);}catch(e){}}
function el(id){return document.getElementById(id);}
function post(){var h=Math.ceil(b.getBoundingClientRect().height);
if(h>0){parent.postMessage({type:'iframe:height',height:h},'*');}}
/* Clicking a box halves it; clicking again puts it back. */
function apply(){
var on=!!st.c;
if(kind==='tasks'){
var f=el('full'),c=el('cur');
if(f){f.style.display=on?'none':'';}
if(c){c.style.display=on?'':'none';}
}else if(kind==='screen'){
var w=el('wrap'),i=el('shot');
if(w){w.style.background=on?'#000':'';}
if(i){i.style.width=on?'50%':'100%';i.style.margin=on?'0 auto':'';}
}else if(kind==='feed'){
var r=el('rows');
if(r){
if(on){var first=r.firstElementChild;
var rh=first?first.getBoundingClientRect().height:17;
r.style.maxHeight=Math.round(rh*Math.ceil((cfg.n||20)/2))+'px';}
else{r.style.maxHeight='';}}
}}
b.style.cursor='pointer';
b.addEventListener('click',function(){st.c=st.c?0:1;save();apply();post();});
apply();
/* THE LIVE LINE, REVEALED AT THE RATE IT ARRIVED.
The server sends the tail of what the model is saying, at most once a
second, so this line used to jump a clause at a time - motion that
said nothing about how fast the tokens were actually coming. It is now
appended a few characters per frame, and the speed it moves IS the
token rate. The text so far rides in window.name, so a replaced
document carries on the same line instead of retyping it. */
if(kind==='tasks'){
var span=el('t');
var target=cfg.s||'';
var shown=(typeof st.d==='string')?st.d:'';
var pending='';
if(target){
if(shown&&shown.indexOf(target)>=0){pending='';}
else{
/* The server sends a sliding window of the tail, so what is new is
whatever follows the longest overlap between the two. */
var o=0,max=Math.min(shown.length,target.length);
for(var k=max;k>0;k--){
if(shown.slice(shown.length-k)===target.slice(0,k)){o=k;break;}}
if(o===0){shown='';}
pending=target.slice(o);}}
var CAP=400;
function paint(){if(shown.length>CAP){shown=shown.slice(-CAP);}
if(span){span.textContent=shown;}}
paint();
if(pending){
var TICK=40,DUR=1500;
var per=Math.max(1,Math.ceil(pending.length/(DUR/TICK)));
var timer=setInterval(function(){
if(!pending.length){clearInterval(timer);return;}
shown+=pending.slice(0,per);pending=pending.slice(per);
paint();st.d=shown;save();},TICK);}
st.d=shown;save();}
post();addEventListener('load',post);
if(window.ResizeObserver){new ResizeObserver(post).observe(b);}
setTimeout(post,60);setTimeout(post,400);
})();
</script>
"""
class Filter:
class Valves(BaseModel):
enabled: bool = Field(
default=True,
description="Turn the whole filter off without deleting it.",
)
owuitools_url: str = Field(
default="http://owuitools-gw:8000",
description="The owuitools gateway, as Open WebUI reaches it. This is the same "
"address as the tool server connection and is not published to the host.",
)
api_key: str = Field(
default="",
description="The owuitools bearer token - the SAME value as the tool server "
"connection's Auth field, not an Open WebUI API key. It grants the tools the "
"model can already call and nothing more. Without it this filter does nothing.",
)
poll_seconds: float = Field(
default=2.0,
description="How often to ask for progress while a session is running.",
)
show_terminal: bool = Field(
default=True,
description="Show the last 20 lines of what the agent ran.",
)
show_screenshots: bool = Field(
default=True,
description="Show the machine's screen when the agent takes a screenshot. Each frame is stored "
"on the message, so turning this off makes the chat record smaller.",
)
max_minutes: float = Field(
default=125.0,
description="Give up watching after this long. A backstop so a watcher can never "
"outlive the session it is watching, whatever goes wrong.",
)
def __init__(self) -> None:
self.valves = self.Valves()
# One watcher per message id. A chat can only have one assistant
# message in flight, but a user with two tabs open has two chats.
self._watchers: dict[str, asyncio.Task] = {}
# -- Open WebUI hooks -----------------------------------------------
async def inlet(self, body: dict, __event_emitter__=None, __event_call__=None,
__metadata__=None) -> dict:
# A previous turn's watcher has no business surviving into this
# one; inlet is the only hook guaranteed to run at the start.
try:
self._stop(self._message_id(__metadata__))
except Exception: # noqa: BLE001
pass
return body
async def stream(self, event: dict, __event_emitter__=None, __event_call__=None,
__metadata__=None) -> dict:
try:
if self.valves.enabled and self._is_vm_agent_call(event):
self._start(__event_emitter__, __event_call__, __metadata__)
except Exception: # noqa: BLE001
# Never break the response. See the header.
pass
return event
async def outlet(self, body: dict, __event_emitter__=None, __event_call__=None,
__metadata__=None) -> dict:
try:
self._stop(self._message_id(__metadata__))
except Exception: # noqa: BLE001
pass
return body
# -- detection -------------------------------------------------------
@staticmethod
def _is_vm_agent_call(event: dict) -> bool:
"""Has the model just started writing a vm_agent call?
Tool calls arrive token by token, so the name may appear in a
delta with nothing else in it. Starting the watcher more than
once is harmless - _start is idempotent per message.
"""
for choice in (event or {}).get("choices") or []:
delta = choice.get("delta") or {}
for call in delta.get("tool_calls") or []:
if (call.get("function") or {}).get("name") == "vm_agent":
return True
return False
@staticmethod
def _message_id(metadata) -> str:
return str((metadata or {}).get("message_id") or "")
# -- the watcher ------------------------------------------------------
def _start(self, emitter, caller, metadata) -> None:
message_id = self._message_id(metadata)
chat_id = str((metadata or {}).get("chat_id") or "")
if not emitter or not message_id or not chat_id:
return
if not self.valves.api_key:
return
if message_id in self._watchers and not self._watchers[message_id].done():
return
self._watchers[message_id] = asyncio.create_task(
self._watch(emitter, caller, chat_id))
def _stop(self, message_id: str) -> None:
task = self._watchers.pop(message_id, None)
if task and not task.done():
task.cancel()
async def _watch(self, emitter, caller, chat_id: str) -> None:
url = f"{self.valves.owuitools_url.rstrip('/')}/vm/progress/{chat_id}"
headers = {"Authorization": f"Bearer {self.valves.api_key}"}
deadline = asyncio.get_running_loop().time() + self.valves.max_minutes * 60
began = asyncio.get_running_loop().time()
# Wall clock, because that is what the progress file is stamped
# with; the loop clock above is a monotonic reading and the two
# are not comparable.
began_at = time.time()
last_status = ""
last_view = None
shown_anything = False
# WHY THE WHOLE VIEW IS RE-SENT EVEN WHEN NOTHING HAS MOVED.
#
# These updates are pushed to whoever is connected. Someone who
# reloads the page, or opens the chat on another device, joins
# partway through and gets only what is NEXT - and a session can
# sit on one step for minutes ("Waiting for the machine to finish
# booting" held for four polls in a row, measured), so a viewer who
# arrives just after a change sees an empty message for as long as
# that step lasts and concludes nothing is happening.
#
# So the current view is re-sent on a slow heartbeat regardless of
# whether it changed. `replace: true` makes that idempotent - it
# overwrites the one live view rather than appending a frame - so
# the cost is one small message every RESEND_SECONDS and the
# benefit is that reattaching fixes itself.
next_resend = 0.0
ended_cleanly = False
try:
timeout = aiohttp.ClientTimeout(total=10)
async with aiohttp.ClientSession(timeout=timeout) as session:
await self._status(emitter, "Starting a machine…")
asked: set[str] = set()
while asyncio.get_running_loop().time() < deadline:
await asyncio.sleep(self.valves.poll_seconds)
# Before a machine is started, the tool waits for the
# person to see the job. Putting that question is this
# watcher's job: it is already here, it already has the
# chat, and doing it here means the model plays no part
# in it and cannot skip it.
if caller is not None:
with contextlib.suppress(Exception):
await self._maybe_ask(session, caller, chat_id, headers, asked)
try:
async with session.get(url, headers=headers) as response:
if response.status != 200:
continue
data = json.loads(await response.text())
except (aiohttp.ClientError, ValueError, asyncio.TimeoutError):
# A poll that fails is a poll; the session is not
# affected by it and neither is the next one.
continue
# ANYTHING OLDER THAN THIS WATCHER BELONGS TO A PREVIOUS RUN.
#
# The progress file is named after the CHAT, so a second
# vm_agent call in the same conversation finds the first
# one's final state sitting there - including `done`.
# Measured: a watcher for a 45-minute run read that on
# its first poll, announced the machine had been
# destroyed and exited four seconds in, while the run
# carried on unobserved to sequence 1035.
#
# The api deletes the file before queueing the job, which
# closes the race properly; this is the second guard,
# because a race with one guard is a race. `updated` is
# written on every flush, so a file from the last run is
# simply older than the watcher reading it.
if float(data.get("updated") or 0) < began_at:
data = {}
if not data:
# NOTHING YET, AND SAYING SO BEATS SAYING NOTHING.
#
# "Starting a machine" is the first thing this
# emits, before there is any progress to read. If
# the file never appears - the job queued behind
# another session, the runner restarting, a boot
# that is simply slow - that sentence is the last
# word the reader gets, and a frozen line is
# indistinguishable from a broken one. Reported as
# exactly that: a toast stuck on "Starting a
# machine" with no boxes.
#
# So it counts, and says how long. A number that
# moves is the difference between waiting and
# wondering.
waited = int(asyncio.get_running_loop().time() - began)
if waited >= QUIET_SECONDS:
shown_anything = True
await self._status(
emitter,
f"Starting a machine - {waited}s so far, nothing back "
"from it yet")
continue
now = asyncio.get_running_loop().time()
resend = now >= next_resend
if resend:
next_resend = now + RESEND_SECONDS
status = str(data.get("status") or "")
if status and (status != last_status or resend):
last_status = status
shown_anything = True
await self._status(emitter, status)
# One embed holds the whole view - the task list and,
# under it, whichever of the two the session has. It is
# rebuilt when any part of it moves, and `replace: true`
# means the message carries one of these rather than a
# reel of every frame ever taken.
view = (str(data.get("tasks") or ""),
str(data.get("speech") or ""),
int(data.get("shot_seq") or 0),
int(data.get("terminal_seq") or 0))
if view != last_view or resend:
last_view = view
await self._embed(emitter, data, self.valves)
if data.get("done"):
ended_cleanly = True
break
except asyncio.CancelledError:
raise
except Exception: # noqa: BLE001
pass
finally:
# Only say the machine is gone if the session actually said it
# was done. This block also runs when the watcher is cancelled
# or hits its own deadline - and in those cases the session is
# very much still running, so the old wording stated something
# false and wiped the view while there was still work to show.
with contextlib.suppress(Exception):
await self._embed(emitter, {}, self.valves)
if shown_anything:
with contextlib.suppress(Exception):
await self._status(
emitter,
"The machine has been destroyed." if ended_cleanly
else "No longer watching this session.",
done=True)
# WHAT THE PERSON IS SHOWN, AND WHY IT IS NOT THE JOB ITSELF.
#
# The job the model writes is an instruction sheet for another machine:
# numbered steps, exact commands, the wording that stops an agent
# improvising. Measured, a real one ran to 2,600 characters. Nobody
# reads that to decide whether to press OK - and worse, Open WebUI does
# not scroll the dialog, so a job that tall pushed OK and Cancel 150px
# below the bottom of an 800px window and the question could not be
# answered at all.
#
# So the dialog gets a short summary of what the job will DO.
#
# IT IS BUILT HERE, FROM THE JOB, rather than written by the model.
# This is the one screen where a summary that drifts from the text is
# a real problem: the person approves the summary and the machine is
# sent the job. A summary pulled out of the job's own headings cannot
# describe work that is not in it, and asking the model for one would
# be both a second thing to trust and more work for it to do.
SUMMARY_STEPS = 5 # bullets before the rest become a count
SUMMARY_HEAD = 150 # characters of the opening line
SUMMARY_STEP = 64 # characters of any one bullet - ONE LINE each
# A step is a numbered line or a STEP heading. A bullet is only a step
# when there are no headings - measured on a real job, the bullets were
# the data the steps operated on ("- /tmp/logtest/recent.log -> mtime =
# now"), and treating them as steps pushed the actual steps out of the
# summary entirely.
_STEP_NUM = re.compile(
r"^\s*(?:step\s*\d+|\d+\s*[.)])\s*[:.\u2013\u2014-]*\s*(.+)$", re.IGNORECASE)
_BULLET = re.compile(r"^\s*[-*\u2022]\s*(.+)$")
# Sentences end at . ! ? and nothing else. Splitting on a colon as well
# turned "Run: ps -p 1 -o comm=" into the bullet "Run:", which says
# nothing - the useful half is always on the other side of the colon.
_SENTENCE = re.compile(r"(?<=[.!?])\s")
# "Create a clean test directory, e.g. /tmp/logtest" is one sentence.
# Splitting on every full stop cut that bullet off at "e.g".
_NOT_AN_END = ("e.g", "i.e", "etc", "vs", "no", "fig", "approx", "cf")
# The word before the stop carries whatever punctuation was around it -
# "(i.e." and "e.g.," both have to reduce to the abbreviation itself.
_LAST_WORD = re.compile(r"[^a-z.]", re.IGNORECASE)
_LABEL = re.compile(r"^(goal|task|objective|summary)\s*[:-]\s*", re.IGNORECASE)
@classmethod
def _headline(cls, text: str, cap: int) -> str:
"""The first sentence of something, cut to length without cutting a word."""
text = " ".join(text.split())
parts = cls._SENTENCE.split(text)
first = parts[0]
# Keep absorbing while the "sentence" ended on an abbreviation.
while (len(parts) > 1
and cls._LAST_WORD.sub("", first.rsplit(" ", 1)[-1]).strip(".").lower()
in cls._NOT_AN_END):
parts = parts[1:]
first = f"{first} {parts[0]}"
first = first.strip()
if len(first) <= cap:
return first.rstrip(".")
cut = first[:cap]
space = cut.rfind(" ")
return (cut[:space] if space > cap // 2 else cut).rstrip(" ,;:-") + "..."
@classmethod
def _summarise(cls, task: str) -> str:
"""What this job will do, in a few lines, taken from the job itself."""
task = task.strip()
if not task:
return "(the job was empty)"
lines = [line for line in task.splitlines() if line.strip()]
def heads(pattern) -> list:
found = []
for index, line in enumerate(lines):
match = pattern.match(line)
if not match:
continue
body = match.group(1)
# A step that is only a label - "Report:", "Then:" - says
# nothing by itself, so it takes what is under it.
if body.rstrip().endswith(":") and len(body.split()) <= 3:
following = lines[index + 1] if index + 1 < len(lines) else ""
under = cls._BULLET.match(following)
body = f"{body.rstrip()} {(under.group(1) if under else following).strip()}"
body = body.strip()
step = cls._headline(body, cls.SUMMARY_STEP)
if step:
found.append(step)
return found
steps = heads(cls._STEP_NUM) or heads(cls._BULLET)
# An opening line that is itself a step is the first bullet, not the
# headline; anything else - "Goal: ...", a sentence of context - is.
head = ""
if not (cls._STEP_NUM.match(lines[0]) or
(not heads(cls._STEP_NUM) and cls._BULLET.match(lines[0]))):
head = cls._headline(cls._LABEL.sub("", lines[0]), cls.SUMMARY_HEAD)
if not steps:
# No headings to work from, so the summary is the first few
# sentences of the prose - still shorter than the whole thing.
rest = cls._SENTENCE.split(" ".join(task.split()))[1:]
steps = [step for step in
(cls._headline(part, cls.SUMMARY_STEP)
for part in rest[:cls.SUMMARY_STEPS]) if step]
if not steps:
return head or cls._headline(task, cls.SUMMARY_HEAD)
# Open WebUI renders this dialog as markdown, and a plain newline there
# is a SOFT break: consecutive lines are joined into one paragraph.
# Measured, that ran two steps together on one wrapped line. A "- "
# list item at column 0 is the one form the renderer keeps on its own
# line - a leading indent turns it into a nested or literal block, and
# a bullet character of my own is not markdown at all and gets joined.
out = [head] if head else []
shown = steps[:cls.SUMMARY_STEPS]
out += [f"- {step}" for step in shown]
left = len(steps) - len(shown)
if left > 0:
out.append(f"- and {left} more step{'' if left == 1 else 's'}")
return "\n".join(out)
async def _maybe_ask(self, session, caller, chat_id: str, headers: dict,
asked: set) -> None:
"""Show the person the job, and send back what they decide.
Deliberately one dialog rather than three buttons. Open WebUI can
show a box with a message and a text field, and that covers all
three answers in one step without the person having to explain
themselves twice:
leave it empty and press OK -> it runs as written
type what to change -> it comes back rewritten
press Cancel -> nothing is started
Typing the change here is what keeps the model out of it. The old
shape would have been "the model asks what you want changed",
which is another round trip and another chance to lose the sense
of what was said.
"""
base = self.valves.owuitools_url.rstrip("/")
async with session.get(f"{base}/vm/confirm/{chat_id}", headers=headers) as resp:
if resp.status != 200:
return
pending = json.loads(await resp.text())
request_id = str(pending.get("id") or "")
if not request_id or request_id in asked:
return
asked.add(request_id)
machine = pending.get("machine") or "test"
# "a Ubuntu machine" reads as a typo. The two names are fixed, so
# the article is decided by the name rather than by a rule.
article = "an" if machine[:1].upper() in "AEIOU" else "a"
project = pending.get("project") or ""
where = f"Kept in '{project}'." if project else "Nothing will be kept."
# EVERY LINE HERE IS PAID FOR TWICE.
#
# Open WebUI does not scroll this dialog, so anything that does not
# fit pushes OK and Cancel off the bottom and the question cannot
# be answered at all. Measured in a 631px-tall window: a 21-line
# message put the buttons 65px past the edge. That is not an
# unusual window - it is a browser that is not maximised.
#
# So the framing is one line, the summary is five one-line bullets,
# and the instructions are one line, because the text box under
# them already says "Empty = run it. Or say what to change."
message = (
f"A throwaway {machine} machine, destroyed afterwards, no access to your "
f"network. {where}\n\n"
"WHAT IT WILL DO\n"
f"{self._summarise(str(pending.get('task') or ''))}\n\n"
"Empty + OK runs it \u00b7 type what to change \u00b7 Cancel drops it."
)
decision, text = "cancel", ""
try:
reply = await caller({
"type": "input",
"data": {"title": f"Start {article} {machine} machine?",
"message": message,
"placeholder": "Empty = run it. Or say what to change."},
})
except Exception: # noqa: BLE001
# If the question could not be put, the job is what was asked
# for. A confirmation that fails must not stop the work.
reply = ""
if reply is None or reply is False:
decision = "cancel"
else:
text = str(reply).strip()
decision = "change" if text else "run"
with contextlib.suppress(Exception):
await session.post(f"{base}/vm/confirm/{chat_id}", headers=headers,
json={"id": request_id, "decision": decision, "text": text})
# -- emitting ----------------------------------------------------------
@staticmethod
async def _status(emitter, description: str, done: bool = False) -> None:
await emitter({"type": "status",
"data": {"description": f"VM · {description}" if not done else description,
"done": done, "hidden": False}})
@staticmethod
async def _embed(emitter, data: dict, valves) -> None:
"""Replace the message's live view: the task list, then the machine.
TWO BOXES, NOT ONE, AND NEITHER OF THEM SCROLLS.
Open WebUI renders each entry of `embeds` in its OWN iframe, so
two entries is two boxes - which is what was asked for.
HOW A BOX GETS THE RIGHT HEIGHT, which is not what it looks like.
Open WebUI tries twice to size one of these frames:
1. on load it reads `contentDocument.body.scrollHeight + 20`;
2. it listens for `postMessage({type:"iframe:height", height})`
from the frame and sets that height exactly.
The first one CANNOT WORK HERE and fails silently. The frame is
sandboxed without `allow-same-origin`, so `contentDocument` throws
a cross-origin error, and the read sits inside a bare `catch {}`.
The height is then never set at all and the frame keeps its
stylesheet default of 150px - which is the whole reason the box
had a scrollbar. Measured: a seven-line task list rendering 168px
tall inside a 150px frame.
So the frame measures itself and posts its own height, which is
route 2 and needs no same-origin. `allowScripts` is hard-coded on
for embeds, so the script runs. It reports on load, on every
resize, and once more shortly after - the last of those is for a
screenshot that has not finished decoding when the rest is ready.
"""
tasks = str(data.get("tasks") or "")
speech = str(data.get("speech") or "")
status = str(data.get("status") or "")
shot = str(data.get("shot") or "")
terminal = list(data.get("terminal") or [])
if not tasks and not speech and not shot and not terminal:
await emitter({"type": "embeds", "data": {"replace": True, "embeds": []}})
return
def esc(text: str) -> str:
return (text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;"))
def frame(inner: str, background: str, kind: str, **config) -> str:
# The config goes in as JSON rather than as generated JavaScript:
# it carries the model's own words, and a quote or a backslash in
# those would otherwise end the script. `</` is escaped because
# the HTML parser ends this block at the first `</script>`,
# wherever it appears - and `\/` is a legal JSON escape.
payload = json.dumps({"k": kind, **config}).replace("</", "<\\/")
return (
"<style>html,body{margin:0;padding:0;overflow:hidden}"
"*{box-sizing:border-box}</style>"
f'<div id="b" style="font:12px/1.45 ui-monospace,SFMono-Regular,Menlo,'
f'monospace;color:#c9c9c9;background:{background};border-radius:6px;'
f'padding:8px 10px;overflow:hidden">{inner}</div>'
f'<script type="application/json" id="cfg">{payload}</script>'
+ _VIEW_SCRIPT
)
embeds = []
# BOX ONE: what the model is saying, then the list it is working to.
#
# The live line sits above the task list and carries no label. It
# is the model's own voice mid-turn, and a turn on this model runs
# tens of seconds - without it the box is motionless for most of a
# session and "still thinking" looks the same as "wedged". One
# line, and the script above reveals it a few characters at a time
# so that its speed is the token rate rather than a poll artefact.
#
# It is right-anchored: the server sends the TAIL of the sentence,
# so the end is the part that is moving and the end is what stays
# against the edge while the rest slides off to the left.
#
# Clicked, the box becomes those two lines and nothing else: the
# live line, and the step the session is on.
if tasks or speech:
rows = ('<div id="live" style="display:flex;justify-content:flex-end;'
'overflow:hidden;min-height:17px;color:#8f8f8f">'
f'<span id="t" style="white-space:pre">{esc(speech)}</span></div>')
if tasks:
rows += '<div id="full">'
rows += '<div style="height:6px"></div>'
rows += '<div style="color:#9aa0a6">Task List:</div>'
rows += "".join(
f'<div style="white-space:pre-wrap;word-break:break-word">{esc(line)}</div>'
for line in tasks.splitlines()
)
rows += "</div>"
rows += ('<div id="cur" style="display:none;white-space:pre;overflow:hidden;'
f'text-overflow:ellipsis;color:#9aa0a6">{esc(status)}</div>')
embeds.append(frame(rows, "#1b1b1b", "tasks", s=speech))
# BOX TWO, ONLY ONCE THERE IS SOMETHING TO SHOW.
#
# It used to appear for every desktop session, from the handover
# onwards, which meant a picture of an idle desktop sat in the
# chat before anything had happened - and most jobs never look at
# the screen at all. The operator's words: pop the preview up when
# the agent takes a screenshot. So this box exists only when a
# screenshot does, and its arrival means something.
#
# Clicked, the picture goes to half size. It keeps its shape, so
# half the height is half the width, and the black the box is
# filled with shows either side of it.
if valves.show_screenshots and shot:
caption = esc(str(data.get("caption") or ""))[:200]
embeds.append(frame(
f'<div id="wrap"><img id="shot" src="{shot}" style="width:100%;'
'height:auto;display:block;border-radius:4px" '
'alt="the machine\'s screen"></div>'
f'<div style="padding-top:4px;color:#888">{caption}</div>',
"#101010", "screen"))
# BOX THREE: the command feed, always, at one height. It used to be
# halved when a screen was shown above it; the screen is now the
# exception rather than the rule, so the feed is not shrunk for
# something that may never appear. Clicked, it shows half as many
# lines - the newest half, because the rows sit at the bottom of
# the box and it is the top that is cut off.
if valves.show_terminal:
height = TERMINAL_LINES
# Exactly `height` rows, always, so the box does not change
# size as output arrives; each row clips at the right edge
# instead of wrapping or scrolling sideways.
lines = terminal[-height:]
lines = [""] * (height - len(lines)) + lines
# `flex:none` is load-bearing. The rows sit in a flex column so
# that a collapsed feed keeps its NEWEST lines against the
# bottom and loses the oldest off the top. Flex items shrink by
# default, so without this the rows squeezed into the smaller
# box and the text overlapped itself instead of overflowing -
# measured, twenty lines crushed into ten lines of space.
rows = "".join(
'<div style="white-space:pre;overflow:hidden;text-overflow:ellipsis;'
f'flex:none">{esc(line) or "&nbsp;"}</div>'
for line in lines
)
embeds.append(frame(
'<div id="rows" style="display:flex;flex-direction:column;'
f'justify-content:flex-end;overflow:hidden">{rows}</div>',
"#101010", "feed", n=height))
await emitter({"type": "embeds", "data": {"replace": True, "embeds": embeds}})