""" title: Inline Tool Images author: owuitools version: 0.1.0 description: Shows images returned by tool calls (e.g. matlab_run figures) inline in the chat. """ # WHY THIS EXISTS # # When an OpenAPI tool server returns a result that IS a 'data:image/...' # URI, Open WebUI lifts it out and - on the native tool-calling path - # routes it to the MODEL as a vision input: # # utils/middleware.py # if file_item['type'] == 'image' and file_item['url'].startswith('data:'): # output_parts.append({'type': 'input_image', ...}) # LLM-only # else: # display_files.append(file_item) # user sees this # # So the model can see the picture and the user cannot. Only a NON-data # URL reaches the frontend. This filter closes that gap without touching # the tool server, the network policy, or the model: the image is already # sitting in the finished message, so we upload it through Open WebUI's # own helper - which stores it, links it to the message and hands back a # real file URL - and append that URL to the message as markdown. # # The markdown has to go into BOTH content and the last message item of # output. Open WebUI renders from output when it is present and falls # back to content only when it is not (get_output_text), so editing # content alone stores the image and shows nothing. # # Everything here fails closed. If an import moves in a future Open WebUI # release, or an upload fails, the message is returned untouched: no # picture is a nuisance, a corrupted answer is not. import json from pydantic import BaseModel, Field # Marker left in the appended block so a second pass over the same message # cannot upload the same image twice. Each upload mints a NEW url, so # checking for a previous url would never match. # # It goes in `content` ONLY. Open WebUI renders the `output` text, and its # markdown renderer does not hide HTML comments - the marker appeared under # the image as literal "". content is not # rendered, so the guard can live there and stay invisible. _MARKER = "" class Filter: class Valves(BaseModel): enabled: bool = Field( default=True, description="Turn the whole filter off without deleting it.", ) max_images: int = Field( default=4, description="Most images to attach to one message. A runaway loop that plots on " "every iteration should not paste twenty figures into the chat.", ) strip_base64_from_output: bool = Field( default=True, description="Remove the base64 copy from the stored message once it has been " "uploaded. The model has already seen it by this point, and leaving it behind " "keeps roughly 11 KB per figure in the chat record.", ) caption: str = Field( default="figure", description="Alt text used for the appended images.", ) def __init__(self): self.valves = self.Valves() async def outlet( self, body: dict, __request__=None, __user__=None, __metadata__=None, ) -> dict: if not self.valves.enabled or __request__ is None: return body try: from open_webui.models.users import Users from open_webui.utils.files import get_image_url_from_base64 except Exception: return body try: messages = body.get("messages") or [] target = None for message in reversed(messages): if message.get("role") == "assistant" and message.get("output"): target = message break if target is None: return body content = target.get("content") or "" output_items = target.get("output") or [] if _MARKER in content or _MARKER in str(output_items): return body # Which figure did each tool call ask for? A model that plots, # looks at the result, spots a problem and re-plots plainly means # the second one - but both images are sitting in the output, and # appending both showed the user a picture the model had already # rejected. Grouping by the requested filename separates a # REFINEMENT (same name, keep the last) from a genuinely second # figure (different name, keep both). figure_by_call = {} for item in output_items: if item.get("type") != "function_call": continue call_id = item.get("call_id") if not call_id: continue args = item.get("arguments") if isinstance(args, str): try: args = json.loads(args) except Exception: args = {} if isinstance(args, dict): figure_by_call[call_id] = args.get("return_figure") or "" # Keyed, insertion-ordered, last write wins. found_by_key = {} for item in output_items: if item.get("type") != "function_call_output": continue call_id = item.get("call_id") for part in item.get("output") or []: if part.get("type") != "input_image": continue url = part.get("image_url") or "" if not url.startswith("data:image/"): continue # Falling back to the call id keeps every image when the # filename cannot be resolved: showing one image too many # beats silently dropping the only one. key = figure_by_call.get(call_id) or "call:%s" % call_id found_by_key[key] = (part, url) found = list(found_by_key.values()) if not found: return body user_id = (__user__ or {}).get("id") user = await Users.get_user_by_id(user_id) if user_id else None if user is None: return body metadata = { "chat_id": body.get("chat_id"), "message_id": target.get("id"), } urls = [] for part, data_uri in found[: max(1, self.valves.max_images)]: try: url = await get_image_url_from_base64(__request__, data_uri, metadata, user) except Exception: url = None if not url: continue urls.append(url) if self.valves.strip_base64_from_output: part["image_url"] = url if not urls: return body caption = self.valves.caption or "figure" block = "\n".join(f"![{caption}]({url})" for url in urls) visible = f"\n\n{block}" target["content"] = content.rstrip() + visible + "\n" + _MARKER # The one that actually shows: the visible answer is the last # message item in output, and everything before it is # reasoning and tool traffic. message_items = [o for o in output_items if o.get("type") == "message"] if message_items: last = message_items[-1] parts = last.get("content") or [] texts = [p for p in parts if p.get("type") == "output_text"] if texts: texts[-1]["text"] = (texts[-1].get("text") or "").rstrip() + visible else: parts.append({"type": "output_text", "text": visible.lstrip()}) last["content"] = parts except Exception: return body return body