Initial Commit
This commit is contained in:
@@ -0,0 +1,199 @@
|
||||
"""
|
||||
title: Inline Tool Images
|
||||
author: owuitools
|
||||
version: 0.1.0
|
||||
description: Shows images returned by tool calls (e.g. matlab_run figures) inline in the chat.
|
||||
"""
|
||||
|
||||
# WHY THIS EXISTS
|
||||
#
|
||||
# When an OpenAPI tool server returns a result that IS a 'data:image/...'
|
||||
# URI, Open WebUI lifts it out and - on the native tool-calling path -
|
||||
# routes it to the MODEL as a vision input:
|
||||
#
|
||||
# utils/middleware.py
|
||||
# if file_item['type'] == 'image' and file_item['url'].startswith('data:'):
|
||||
# output_parts.append({'type': 'input_image', ...}) # LLM-only
|
||||
# else:
|
||||
# display_files.append(file_item) # user sees this
|
||||
#
|
||||
# So the model can see the picture and the user cannot. Only a NON-data
|
||||
# URL reaches the frontend. This filter closes that gap without touching
|
||||
# the tool server, the network policy, or the model: the image is already
|
||||
# sitting in the finished message, so we upload it through Open WebUI's
|
||||
# own helper - which stores it, links it to the message and hands back a
|
||||
# real file URL - and append that URL to the message as markdown.
|
||||
#
|
||||
# The markdown has to go into BOTH content and the last message item of
|
||||
# output. Open WebUI renders from output when it is present and falls
|
||||
# back to content only when it is not (get_output_text), so editing
|
||||
# content alone stores the image and shows nothing.
|
||||
#
|
||||
# Everything here fails closed. If an import moves in a future Open WebUI
|
||||
# release, or an upload fails, the message is returned untouched: no
|
||||
# picture is a nuisance, a corrupted answer is not.
|
||||
|
||||
import json
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# Marker left in the appended block so a second pass over the same message
|
||||
# cannot upload the same image twice. Each upload mints a NEW url, so
|
||||
# checking for a previous url would never match.
|
||||
#
|
||||
# It goes in `content` ONLY. Open WebUI renders the `output` text, and its
|
||||
# markdown renderer does not hide HTML comments - the marker appeared under
|
||||
# the image as literal "<!-- inline-tool-images -->". content is not
|
||||
# rendered, so the guard can live there and stay invisible.
|
||||
_MARKER = "<!-- inline-tool-images -->"
|
||||
|
||||
|
||||
class Filter:
|
||||
class Valves(BaseModel):
|
||||
enabled: bool = Field(
|
||||
default=True,
|
||||
description="Turn the whole filter off without deleting it.",
|
||||
)
|
||||
max_images: int = Field(
|
||||
default=4,
|
||||
description="Most images to attach to one message. A runaway loop that plots on "
|
||||
"every iteration should not paste twenty figures into the chat.",
|
||||
)
|
||||
strip_base64_from_output: bool = Field(
|
||||
default=True,
|
||||
description="Remove the base64 copy from the stored message once it has been "
|
||||
"uploaded. The model has already seen it by this point, and leaving it behind "
|
||||
"keeps roughly 11 KB per figure in the chat record.",
|
||||
)
|
||||
caption: str = Field(
|
||||
default="figure",
|
||||
description="Alt text used for the appended images.",
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
self.valves = self.Valves()
|
||||
|
||||
async def outlet(
|
||||
self,
|
||||
body: dict,
|
||||
__request__=None,
|
||||
__user__=None,
|
||||
__metadata__=None,
|
||||
) -> dict:
|
||||
if not self.valves.enabled or __request__ is None:
|
||||
return body
|
||||
|
||||
try:
|
||||
from open_webui.models.users import Users
|
||||
from open_webui.utils.files import get_image_url_from_base64
|
||||
except Exception:
|
||||
return body
|
||||
|
||||
try:
|
||||
messages = body.get("messages") or []
|
||||
target = None
|
||||
for message in reversed(messages):
|
||||
if message.get("role") == "assistant" and message.get("output"):
|
||||
target = message
|
||||
break
|
||||
if target is None:
|
||||
return body
|
||||
|
||||
content = target.get("content") or ""
|
||||
output_items = target.get("output") or []
|
||||
if _MARKER in content or _MARKER in str(output_items):
|
||||
return body
|
||||
|
||||
# Which figure did each tool call ask for? A model that plots,
|
||||
# looks at the result, spots a problem and re-plots plainly means
|
||||
# the second one - but both images are sitting in the output, and
|
||||
# appending both showed the user a picture the model had already
|
||||
# rejected. Grouping by the requested filename separates a
|
||||
# REFINEMENT (same name, keep the last) from a genuinely second
|
||||
# figure (different name, keep both).
|
||||
figure_by_call = {}
|
||||
for item in output_items:
|
||||
if item.get("type") != "function_call":
|
||||
continue
|
||||
call_id = item.get("call_id")
|
||||
if not call_id:
|
||||
continue
|
||||
args = item.get("arguments")
|
||||
if isinstance(args, str):
|
||||
try:
|
||||
args = json.loads(args)
|
||||
except Exception:
|
||||
args = {}
|
||||
if isinstance(args, dict):
|
||||
figure_by_call[call_id] = args.get("return_figure") or ""
|
||||
|
||||
# Keyed, insertion-ordered, last write wins.
|
||||
found_by_key = {}
|
||||
for item in output_items:
|
||||
if item.get("type") != "function_call_output":
|
||||
continue
|
||||
call_id = item.get("call_id")
|
||||
for part in item.get("output") or []:
|
||||
if part.get("type") != "input_image":
|
||||
continue
|
||||
url = part.get("image_url") or ""
|
||||
if not url.startswith("data:image/"):
|
||||
continue
|
||||
# Falling back to the call id keeps every image when the
|
||||
# filename cannot be resolved: showing one image too many
|
||||
# beats silently dropping the only one.
|
||||
key = figure_by_call.get(call_id) or "call:%s" % call_id
|
||||
found_by_key[key] = (part, url)
|
||||
|
||||
found = list(found_by_key.values())
|
||||
if not found:
|
||||
return body
|
||||
|
||||
user_id = (__user__ or {}).get("id")
|
||||
user = await Users.get_user_by_id(user_id) if user_id else None
|
||||
if user is None:
|
||||
return body
|
||||
|
||||
metadata = {
|
||||
"chat_id": body.get("chat_id"),
|
||||
"message_id": target.get("id"),
|
||||
}
|
||||
|
||||
urls = []
|
||||
for part, data_uri in found[: max(1, self.valves.max_images)]:
|
||||
try:
|
||||
url = await get_image_url_from_base64(__request__, data_uri, metadata, user)
|
||||
except Exception:
|
||||
url = None
|
||||
if not url:
|
||||
continue
|
||||
urls.append(url)
|
||||
if self.valves.strip_base64_from_output:
|
||||
part["image_url"] = url
|
||||
|
||||
if not urls:
|
||||
return body
|
||||
|
||||
caption = self.valves.caption or "figure"
|
||||
block = "\n".join(f"" for url in urls)
|
||||
visible = f"\n\n{block}"
|
||||
|
||||
target["content"] = content.rstrip() + visible + "\n" + _MARKER
|
||||
|
||||
# The one that actually shows: the visible answer is the last
|
||||
# message item in output, and everything before it is
|
||||
# reasoning and tool traffic.
|
||||
message_items = [o for o in output_items if o.get("type") == "message"]
|
||||
if message_items:
|
||||
last = message_items[-1]
|
||||
parts = last.get("content") or []
|
||||
texts = [p for p in parts if p.get("type") == "output_text"]
|
||||
if texts:
|
||||
texts[-1]["text"] = (texts[-1].get("text") or "").rstrip() + visible
|
||||
else:
|
||||
parts.append({"type": "output_text", "text": visible.lstrip()})
|
||||
last["content"] = parts
|
||||
except Exception:
|
||||
return body
|
||||
|
||||
return body
|
||||
Reference in New Issue
Block a user