200 lines
7.8 KiB
Python
200 lines
7.8 KiB
Python
"""
|
|
title: Inline Tool Images
|
|
author: owuitools
|
|
version: 0.1.0
|
|
description: Shows images returned by tool calls (e.g. matlab_run figures) inline in the chat.
|
|
"""
|
|
|
|
# WHY THIS EXISTS
|
|
#
|
|
# When an OpenAPI tool server returns a result that IS a 'data:image/...'
|
|
# URI, Open WebUI lifts it out and - on the native tool-calling path -
|
|
# routes it to the MODEL as a vision input:
|
|
#
|
|
# utils/middleware.py
|
|
# if file_item['type'] == 'image' and file_item['url'].startswith('data:'):
|
|
# output_parts.append({'type': 'input_image', ...}) # LLM-only
|
|
# else:
|
|
# display_files.append(file_item) # user sees this
|
|
#
|
|
# So the model can see the picture and the user cannot. Only a NON-data
|
|
# URL reaches the frontend. This filter closes that gap without touching
|
|
# the tool server, the network policy, or the model: the image is already
|
|
# sitting in the finished message, so we upload it through Open WebUI's
|
|
# own helper - which stores it, links it to the message and hands back a
|
|
# real file URL - and append that URL to the message as markdown.
|
|
#
|
|
# The markdown has to go into BOTH content and the last message item of
|
|
# output. Open WebUI renders from output when it is present and falls
|
|
# back to content only when it is not (get_output_text), so editing
|
|
# content alone stores the image and shows nothing.
|
|
#
|
|
# Everything here fails closed. If an import moves in a future Open WebUI
|
|
# release, or an upload fails, the message is returned untouched: no
|
|
# picture is a nuisance, a corrupted answer is not.
|
|
|
|
import json
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
# Marker left in the appended block so a second pass over the same message
|
|
# cannot upload the same image twice. Each upload mints a NEW url, so
|
|
# checking for a previous url would never match.
|
|
#
|
|
# It goes in `content` ONLY. Open WebUI renders the `output` text, and its
|
|
# markdown renderer does not hide HTML comments - the marker appeared under
|
|
# the image as literal "<!-- inline-tool-images -->". content is not
|
|
# rendered, so the guard can live there and stay invisible.
|
|
_MARKER = "<!-- inline-tool-images -->"
|
|
|
|
|
|
class Filter:
|
|
class Valves(BaseModel):
|
|
enabled: bool = Field(
|
|
default=True,
|
|
description="Turn the whole filter off without deleting it.",
|
|
)
|
|
max_images: int = Field(
|
|
default=4,
|
|
description="Most images to attach to one message. A runaway loop that plots on "
|
|
"every iteration should not paste twenty figures into the chat.",
|
|
)
|
|
strip_base64_from_output: bool = Field(
|
|
default=True,
|
|
description="Remove the base64 copy from the stored message once it has been "
|
|
"uploaded. The model has already seen it by this point, and leaving it behind "
|
|
"keeps roughly 11 KB per figure in the chat record.",
|
|
)
|
|
caption: str = Field(
|
|
default="figure",
|
|
description="Alt text used for the appended images.",
|
|
)
|
|
|
|
def __init__(self):
|
|
self.valves = self.Valves()
|
|
|
|
async def outlet(
|
|
self,
|
|
body: dict,
|
|
__request__=None,
|
|
__user__=None,
|
|
__metadata__=None,
|
|
) -> dict:
|
|
if not self.valves.enabled or __request__ is None:
|
|
return body
|
|
|
|
try:
|
|
from open_webui.models.users import Users
|
|
from open_webui.utils.files import get_image_url_from_base64
|
|
except Exception:
|
|
return body
|
|
|
|
try:
|
|
messages = body.get("messages") or []
|
|
target = None
|
|
for message in reversed(messages):
|
|
if message.get("role") == "assistant" and message.get("output"):
|
|
target = message
|
|
break
|
|
if target is None:
|
|
return body
|
|
|
|
content = target.get("content") or ""
|
|
output_items = target.get("output") or []
|
|
if _MARKER in content or _MARKER in str(output_items):
|
|
return body
|
|
|
|
# Which figure did each tool call ask for? A model that plots,
|
|
# looks at the result, spots a problem and re-plots plainly means
|
|
# the second one - but both images are sitting in the output, and
|
|
# appending both showed the user a picture the model had already
|
|
# rejected. Grouping by the requested filename separates a
|
|
# REFINEMENT (same name, keep the last) from a genuinely second
|
|
# figure (different name, keep both).
|
|
figure_by_call = {}
|
|
for item in output_items:
|
|
if item.get("type") != "function_call":
|
|
continue
|
|
call_id = item.get("call_id")
|
|
if not call_id:
|
|
continue
|
|
args = item.get("arguments")
|
|
if isinstance(args, str):
|
|
try:
|
|
args = json.loads(args)
|
|
except Exception:
|
|
args = {}
|
|
if isinstance(args, dict):
|
|
figure_by_call[call_id] = args.get("return_figure") or ""
|
|
|
|
# Keyed, insertion-ordered, last write wins.
|
|
found_by_key = {}
|
|
for item in output_items:
|
|
if item.get("type") != "function_call_output":
|
|
continue
|
|
call_id = item.get("call_id")
|
|
for part in item.get("output") or []:
|
|
if part.get("type") != "input_image":
|
|
continue
|
|
url = part.get("image_url") or ""
|
|
if not url.startswith("data:image/"):
|
|
continue
|
|
# Falling back to the call id keeps every image when the
|
|
# filename cannot be resolved: showing one image too many
|
|
# beats silently dropping the only one.
|
|
key = figure_by_call.get(call_id) or "call:%s" % call_id
|
|
found_by_key[key] = (part, url)
|
|
|
|
found = list(found_by_key.values())
|
|
if not found:
|
|
return body
|
|
|
|
user_id = (__user__ or {}).get("id")
|
|
user = await Users.get_user_by_id(user_id) if user_id else None
|
|
if user is None:
|
|
return body
|
|
|
|
metadata = {
|
|
"chat_id": body.get("chat_id"),
|
|
"message_id": target.get("id"),
|
|
}
|
|
|
|
urls = []
|
|
for part, data_uri in found[: max(1, self.valves.max_images)]:
|
|
try:
|
|
url = await get_image_url_from_base64(__request__, data_uri, metadata, user)
|
|
except Exception:
|
|
url = None
|
|
if not url:
|
|
continue
|
|
urls.append(url)
|
|
if self.valves.strip_base64_from_output:
|
|
part["image_url"] = url
|
|
|
|
if not urls:
|
|
return body
|
|
|
|
caption = self.valves.caption or "figure"
|
|
block = "\n".join(f"" for url in urls)
|
|
visible = f"\n\n{block}"
|
|
|
|
target["content"] = content.rstrip() + visible + "\n" + _MARKER
|
|
|
|
# The one that actually shows: the visible answer is the last
|
|
# message item in output, and everything before it is
|
|
# reasoning and tool traffic.
|
|
message_items = [o for o in output_items if o.get("type") == "message"]
|
|
if message_items:
|
|
last = message_items[-1]
|
|
parts = last.get("content") or []
|
|
texts = [p for p in parts if p.get("type") == "output_text"]
|
|
if texts:
|
|
texts[-1]["text"] = (texts[-1].get("text") or "").rstrip() + visible
|
|
else:
|
|
parts.append({"type": "output_text", "text": visible.lstrip()})
|
|
last["content"] = parts
|
|
except Exception:
|
|
return body
|
|
|
|
return body
|