Grant/owui-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial Commit

Browse Source
This commit is contained in:
Grant
2026-09-15 15:24:03 -04:00
commit 4dc7086e7e
605 changed files with 101380 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/owuitools/textwin.py
+291
View File
@@ -0,0 +1,291 @@
"""Jump to a region inside a long document instead of reading from the top.
Split out of `routers/reference.py`, where `read_man_page` and `lookup_rfc`
already had this, so `fetch_page` and `wikipedia_lookup` can use it too. Those
two had no way in at all: the only way to reach page 200 of a reference manual
or the results section of a Wikipedia article was to page through with
`offset`, six thousand characters at a time, and the model reliably gave up
and answered from what it had read - or from memory.
The one behavioural addition over the original is table-of-contents
avoidance. In an RFC or a vendor manual, a section number or a heading matches
the contents listing first, which is the single place the term is guaranteed
to appear and guaranteed to be useless. Two of four `lookup_rfc` calls in a
graded run were spent that way, each returning five thousand characters of dot
leaders. A contents line is skipped only when a body match exists further on:
if every match looks like a contents entry, the first one is still returned,
because a wrong region beats reporting a term absent when it is plainly there.
Two later additions, both from the second graded round, where `search=` twice
decided the fate of a source the model had correctly chosen:
* **A hit now names the section it landed in.** Asked for "Constitution" inside
the Canberra article, the jump matched "1 Constitution Avenue, Civic" - a
street - and returned a window about self-government, bushfires and climate.
The model read it as having checked the constitutional basis for the capital,
stopped hedging, and published a section number it had not verified and which
was wrong. The tool was honest about *where* it jumped; it said nothing about
*what* it jumped into, and a character offset does not tell you that you are
reading the wrong subject. The enclosing heading does.
* **A miss now shows the opening of the document.** Asked for "list of senses"
in the Sense article, the tool returned only "that term does not occur" - and
the model abandoned 54,000 characters of exactly the right article, answered
from memory, and got one entry wrong that the article would have corrected.
One search string missing is not the same finding as the document being
useless, and returning nothing made the two indistinguishable.
"""
from __future__ import annotations
import re
# "2.3.4 Size limits . . . . . . . . 10" / "B.4 Cipher Suites ....... 155"
_TOC_LEADERS = re.compile(r"(?:\.\s*){3,}\d+\s*$")
# The leaderless form, where the page number is just pushed to the right
# margin. Guarded by line length below - body prose that ends in a number is
# usually longer than a contents entry.
_TOC_COLUMNS = re.compile(r"\S[ \t]{3,}\d{1,4}[ \t]*$")
# Headings, in the four shapes this suite's documents actually use: Wikipedia
# plaintext extracts ("== History =="), markdown ("## Options"), numbered
# specification sections ("7.3.2 Clock configuration register") and the
# flush-left capitals of an RFC or man page ("SEE ALSO").
_HEADINGS = (
re.compile(r"^\s*(={2,})\s*(?P<h>.+?)\s*\1\s*$"),
re.compile(r"^\s{0,3}#{1,6}\s+(?P<h>\S.{0,78}?)\s*$"),
re.compile(r"^(?P<h>\d+(?:\.\d+){0,4}\.?[ \t]+[A-Za-z]\S.{0,70}?)\s*$"),
re.compile(r"^(?P<h>[A-Z][A-Z0-9 ,'/&()+-]{2,58}[A-Z0-9)])[ \t]*$"),
# The contents listing of an RFC or a manual, so a jump that lands in
# it is told so rather than told nothing.
re.compile(r"^\s*(?P<h>(?:Table of )?Contents)\s*:?\s*$", re.IGNORECASE),
)
_FLUSH_CAPS = _HEADINGS[3]
# A numbered contents entry: "15.5.19. 418 (Unused)", "7.3.2 Clock configuration".
_NUMBERED_ENTRY = re.compile(r"^\s*(?:\d+(?:\.\d+)*\.?|[A-Z](?:\.\d+)+\.?)[ \t]+\S")
def _recurs_later(text: str, line: str, after: int) -> bool:
"""Whether the same line (whitespace-insensitive) appears again after `after`."""
words = line.split()
if len(words) < 2:
return False
pattern = r"^[ \t]*" + r"\s+".join(re.escape(w) for w in words) + r"[ \t]*$"
return re.search(pattern, text[after:], re.MULTILINE) is not None
def _is_running_line(text: str, line: str) -> bool:
"""A line that repeats every page - "RM0008", a document title - is a header, not a heading."""
bare = line.strip()
if len(bare) < 2:
return True
return text.count("\n" + line.rstrip() + "\n") >= 4
def enclosing_heading(text: str, pos: int) -> str | None:
"""The nearest heading above `pos`, or None.
Bounded to the preceding 20,000 characters: past that the answer stops
being useful anyway, and an unbounded backward scan on a 500 KB RFC is
paid for on every jump.
"""
head = text[max(0, pos - 20_000):pos]
deferred = None
for line in reversed(head.splitlines()):
if not line.strip() or _is_toc_line(line):
continue
for pattern in _HEADINGS:
if match := pattern.match(line):
found = re.sub(r"\s+", " ", match.group("h")).strip(" =#.")
if not 2 <= len(found) <= 80:
continue
# Two things the flush-caps shape matched on a reference
# manual's PDF text and reported as sections: the running
# page header ("RM0008", on every page) and a register
# bit-field label ("XTPRE"). Neither is a heading. A line
# that repeats throughout the document is a header; a
# single capitalised word is kept only as a last resort,
# when no structured heading sits above it in the window
# - which in a man page (NAME, OPTIONS) is the normal case
# and in a numbered manual never is.
if pattern is _FLUSH_CAPS:
if _is_running_line(text, line):
break
if " " not in found:
if not found.isalpha():
break
deferred = deferred or found
break
elif _is_running_line(text, line):
break
return found
return deferred
_XREF_CUE = re.compile(
r"(?:refer(?:s|red)?\s+to|see(?:\s+also)?|described\s+in|defined\s+in|details?\s+in|"
r"listed\s+in|shown\s+in|given\s+in|specified\s+in)\s+(?:section|chapter|table|figure|clause)?"
r"\s*[\dA-Z]*(?:\.\d+)*\.?\s*[:\-]?\s*$",
re.IGNORECASE,
)
def _is_cross_reference(text: str, pos: int) -> bool:
"""Whether the match at `pos` is preceded on its line by a "refer to Section" cue."""
start = text.rfind("\n", 0, pos) + 1
return _XREF_CUE.search(text[start:pos]) is not None
def _loose_words(needle: str) -> str:
"""The needle as a pattern that tolerates whitespace inside its words."""
words = []
for word in needle.split():
words.append(r"\s*".join(re.escape(ch) for ch in word))
return r"\s+".join(words)
def _line_around(text: str, pos: int) -> str:
start = text.rfind("\n", 0, pos) + 1
end = text.find("\n", pos)
return text[start : end if end != -1 else len(text)]
def _is_toc_line(line: str) -> bool:
if _TOC_LEADERS.search(line):
return True
return bool(_TOC_COLUMNS.search(line)) and len(line.strip()) < 90
def find_in_document(
text: str,
needle: str,
*,
window: int = 3800,
start_at: int = 0,
continue_hint: str = "call again with offset= to read more",
) -> str | None:
"""Return the region of `text` around `needle`, or None if it is not there.
Four patterns are tried in order, because the same term means different
things depending on where it sits: an indented line is an option entry or
a numbered section, a flush-left line is a heading, and a bare substring is
a mention in prose. The last pattern allows any whitespace where the needle
has a space, because a multi-word phrase wraps across lines in a rendered
man page or RFC - searching ip-route for "sending a packet along this path"
reported it absent while the page reads "...along this\\n path."
"""
needle = (needle or "").strip()
if not needle:
return None
escaped = re.escape(needle)
patterns = (
# A heading-shaped line: optional section number, then the needle
# with whitespace tolerated inside words. PDF text layers split
# words at column boundaries - "Clock configurati on register" -
# and an exact search for the heading then lands on every
# cross-reference to it ("refer to Section 4.3.2: Clock
# configuration register") and never on the heading itself.
(rf"^[ \t]*(?:\d+(?:\.\d+)*\.?[ \t]+)?{_loose_words(needle)}[ \t]*:?[ \t]*$",
re.MULTILINE | re.IGNORECASE),
(rf"^[ \t]+{escaped}\b", re.MULTILINE), # " -Z user"
(rf"^[ \t]*{escaped}\b", re.MULTILINE), # "EXAMPLES"
(escaped, re.IGNORECASE), # anywhere in prose
(r"\s+".join(re.escape(w) for w in needle.split()), re.IGNORECASE),
)
first_any = None
chosen = None
skipped = 0
def contents_entry(m) -> bool:
line = _line_around(text, m.start())
if _is_toc_line(line):
return True
# The leaderless, page-numberless form used by every RFC published
# in the v3 text format since 2019: " 15.5.19. 418 (Unused)".
# It carries no marker at all; what identifies it is that the same
# line appears again later, as the heading it points to. Checked
# only in the opening quarter of the document, where a contents
# listing lives, so the scan stays cheap on a 500 KB specification.
return (m.start() < len(text) // 4
and _NUMBERED_ENTRY.match(line) is not None
and _recurs_later(text, line, m.end()))
for pattern, flags in patterns:
matches = [m for m in re.finditer(pattern, text, flags) if m.start() >= start_at]
if not matches:
continue
if first_any is None:
first_any = matches[0]
body = [m for m in matches[:40] if not contents_entry(m)] + matches[40:]
if body:
chosen = body[0]
skipped = sum(1 for m in matches if m.start() < chosen.start())
break
if chosen is None:
chosen = first_any
if chosen is None:
return None
start = max(0, chosen.start() - 200)
end = min(len(text), chosen.start() + window)
excerpt = text[start:end]
note = ""
if skipped:
note = (f"; skipped {skipped} table-of-contents "
f"{'entry' if skipped == 1 else 'entries'}")
if _is_cross_reference(text, chosen.start()):
note += ("; this line is a cross-reference to the section, not the section itself - "
"the heading may be split across words in this document's text layer; try "
"the section number, or a distinctive term from its first paragraph")
# Naming the section is the difference between "the string is here" and
# "the subject is here". A match on a word used in another sense lands
# under a heading that says so.
if section := enclosing_heading(text, chosen.start()):
note += f"; this is inside the section '{section}'"
header = (f"[Jumped to '{needle}' at character {chosen.start():,} "
f"of {len(text):,}{note}]")
prefix = "…\n" if start > 0 else ""
suffix = ""
if end < len(text):
suffix = (f"\n…\n\n[Showing the region around '{needle}'. The full document is "
f"{len(text):,} characters - {continue_hint}.]")
return f"{header}\n\n{prefix}{excerpt}{suffix}"
def search_window(
text: str,
needle: str,
*,
window: int = 3800,
start_at: int = 0,
continue_hint: str = "call again with offset= to read more",
miss_note: str,
) -> str:
"""`find_in_document`, but a miss returns the opening of the document too.
A caller that gets back only "that term is not here" has spent a call and
holds nothing, and the observed response to holding nothing is to stop
using the source rather than to try another term. The document is already
fetched and already in memory; showing its first window costs one round of
tokens and keeps the call from being a dead end.
"""
found = find_in_document(text, needle, window=window, start_at=start_at,
continue_hint=continue_hint)
if found is not None:
return found
size = max(500, min(window, len(text)))
lead = text[:size]
tail = ""
if size < len(text):
tail = (f"\n…\n\n[This is the opening only - {len(text) - size:,} characters follow. "
f"To read on, {continue_hint}.]")
return (f"{miss_note}\n\n[Showing the opening of the document instead, so the call is not "
f"wasted. The document itself is fine; only the search term missed.]\n\n{lead}{tail}")