292 lines
13 KiB
Python
292 lines
13 KiB
Python
"""Jump to a region inside a long document instead of reading from the top.
|
|
|
|
Split out of `routers/reference.py`, where `read_man_page` and `lookup_rfc`
|
|
already had this, so `fetch_page` and `wikipedia_lookup` can use it too. Those
|
|
two had no way in at all: the only way to reach page 200 of a reference manual
|
|
or the results section of a Wikipedia article was to page through with
|
|
`offset`, six thousand characters at a time, and the model reliably gave up
|
|
and answered from what it had read - or from memory.
|
|
|
|
The one behavioural addition over the original is table-of-contents
|
|
avoidance. In an RFC or a vendor manual, a section number or a heading matches
|
|
the contents listing first, which is the single place the term is guaranteed
|
|
to appear and guaranteed to be useless. Two of four `lookup_rfc` calls in a
|
|
graded run were spent that way, each returning five thousand characters of dot
|
|
leaders. A contents line is skipped only when a body match exists further on:
|
|
if every match looks like a contents entry, the first one is still returned,
|
|
because a wrong region beats reporting a term absent when it is plainly there.
|
|
|
|
Two later additions, both from the second graded round, where `search=` twice
|
|
decided the fate of a source the model had correctly chosen:
|
|
|
|
* **A hit now names the section it landed in.** Asked for "Constitution" inside
|
|
the Canberra article, the jump matched "1 Constitution Avenue, Civic" - a
|
|
street - and returned a window about self-government, bushfires and climate.
|
|
The model read it as having checked the constitutional basis for the capital,
|
|
stopped hedging, and published a section number it had not verified and which
|
|
was wrong. The tool was honest about *where* it jumped; it said nothing about
|
|
*what* it jumped into, and a character offset does not tell you that you are
|
|
reading the wrong subject. The enclosing heading does.
|
|
|
|
* **A miss now shows the opening of the document.** Asked for "list of senses"
|
|
in the Sense article, the tool returned only "that term does not occur" - and
|
|
the model abandoned 54,000 characters of exactly the right article, answered
|
|
from memory, and got one entry wrong that the article would have corrected.
|
|
One search string missing is not the same finding as the document being
|
|
useless, and returning nothing made the two indistinguishable.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
# "2.3.4 Size limits . . . . . . . . 10" / "B.4 Cipher Suites ....... 155"
|
|
_TOC_LEADERS = re.compile(r"(?:\.\s*){3,}\d+\s*$")
|
|
# The leaderless form, where the page number is just pushed to the right
|
|
# margin. Guarded by line length below - body prose that ends in a number is
|
|
# usually longer than a contents entry.
|
|
_TOC_COLUMNS = re.compile(r"\S[ \t]{3,}\d{1,4}[ \t]*$")
|
|
|
|
|
|
# Headings, in the four shapes this suite's documents actually use: Wikipedia
|
|
# plaintext extracts ("== History =="), markdown ("## Options"), numbered
|
|
# specification sections ("7.3.2 Clock configuration register") and the
|
|
# flush-left capitals of an RFC or man page ("SEE ALSO").
|
|
_HEADINGS = (
|
|
re.compile(r"^\s*(={2,})\s*(?P<h>.+?)\s*\1\s*$"),
|
|
re.compile(r"^\s{0,3}#{1,6}\s+(?P<h>\S.{0,78}?)\s*$"),
|
|
re.compile(r"^(?P<h>\d+(?:\.\d+){0,4}\.?[ \t]+[A-Za-z]\S.{0,70}?)\s*$"),
|
|
re.compile(r"^(?P<h>[A-Z][A-Z0-9 ,'/&()+-]{2,58}[A-Z0-9)])[ \t]*$"),
|
|
# The contents listing of an RFC or a manual, so a jump that lands in
|
|
# it is told so rather than told nothing.
|
|
re.compile(r"^\s*(?P<h>(?:Table of )?Contents)\s*:?\s*$", re.IGNORECASE),
|
|
)
|
|
_FLUSH_CAPS = _HEADINGS[3]
|
|
|
|
# A numbered contents entry: "15.5.19. 418 (Unused)", "7.3.2 Clock configuration".
|
|
_NUMBERED_ENTRY = re.compile(r"^\s*(?:\d+(?:\.\d+)*\.?|[A-Z](?:\.\d+)+\.?)[ \t]+\S")
|
|
|
|
|
|
def _recurs_later(text: str, line: str, after: int) -> bool:
|
|
"""Whether the same line (whitespace-insensitive) appears again after `after`."""
|
|
words = line.split()
|
|
if len(words) < 2:
|
|
return False
|
|
pattern = r"^[ \t]*" + r"\s+".join(re.escape(w) for w in words) + r"[ \t]*$"
|
|
return re.search(pattern, text[after:], re.MULTILINE) is not None
|
|
|
|
|
|
def _is_running_line(text: str, line: str) -> bool:
|
|
"""A line that repeats every page - "RM0008", a document title - is a header, not a heading."""
|
|
bare = line.strip()
|
|
if len(bare) < 2:
|
|
return True
|
|
return text.count("\n" + line.rstrip() + "\n") >= 4
|
|
|
|
|
|
def enclosing_heading(text: str, pos: int) -> str | None:
|
|
"""The nearest heading above `pos`, or None.
|
|
|
|
Bounded to the preceding 20,000 characters: past that the answer stops
|
|
being useful anyway, and an unbounded backward scan on a 500 KB RFC is
|
|
paid for on every jump.
|
|
"""
|
|
head = text[max(0, pos - 20_000):pos]
|
|
deferred = None
|
|
for line in reversed(head.splitlines()):
|
|
if not line.strip() or _is_toc_line(line):
|
|
continue
|
|
for pattern in _HEADINGS:
|
|
if match := pattern.match(line):
|
|
found = re.sub(r"\s+", " ", match.group("h")).strip(" =#.")
|
|
if not 2 <= len(found) <= 80:
|
|
continue
|
|
# Two things the flush-caps shape matched on a reference
|
|
# manual's PDF text and reported as sections: the running
|
|
# page header ("RM0008", on every page) and a register
|
|
# bit-field label ("XTPRE"). Neither is a heading. A line
|
|
# that repeats throughout the document is a header; a
|
|
# single capitalised word is kept only as a last resort,
|
|
# when no structured heading sits above it in the window
|
|
# - which in a man page (NAME, OPTIONS) is the normal case
|
|
# and in a numbered manual never is.
|
|
if pattern is _FLUSH_CAPS:
|
|
if _is_running_line(text, line):
|
|
break
|
|
if " " not in found:
|
|
if not found.isalpha():
|
|
break
|
|
deferred = deferred or found
|
|
break
|
|
elif _is_running_line(text, line):
|
|
break
|
|
return found
|
|
return deferred
|
|
|
|
|
|
_XREF_CUE = re.compile(
|
|
r"(?:refer(?:s|red)?\s+to|see(?:\s+also)?|described\s+in|defined\s+in|details?\s+in|"
|
|
r"listed\s+in|shown\s+in|given\s+in|specified\s+in)\s+(?:section|chapter|table|figure|clause)?"
|
|
r"\s*[\dA-Z]*(?:\.\d+)*\.?\s*[:\-]?\s*$",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
def _is_cross_reference(text: str, pos: int) -> bool:
|
|
"""Whether the match at `pos` is preceded on its line by a "refer to Section" cue."""
|
|
start = text.rfind("\n", 0, pos) + 1
|
|
return _XREF_CUE.search(text[start:pos]) is not None
|
|
|
|
|
|
def _loose_words(needle: str) -> str:
|
|
"""The needle as a pattern that tolerates whitespace inside its words."""
|
|
words = []
|
|
for word in needle.split():
|
|
words.append(r"\s*".join(re.escape(ch) for ch in word))
|
|
return r"\s+".join(words)
|
|
|
|
|
|
def _line_around(text: str, pos: int) -> str:
|
|
start = text.rfind("\n", 0, pos) + 1
|
|
end = text.find("\n", pos)
|
|
return text[start : end if end != -1 else len(text)]
|
|
|
|
|
|
def _is_toc_line(line: str) -> bool:
|
|
if _TOC_LEADERS.search(line):
|
|
return True
|
|
return bool(_TOC_COLUMNS.search(line)) and len(line.strip()) < 90
|
|
|
|
|
|
def find_in_document(
|
|
text: str,
|
|
needle: str,
|
|
*,
|
|
window: int = 3800,
|
|
start_at: int = 0,
|
|
continue_hint: str = "call again with offset= to read more",
|
|
) -> str | None:
|
|
"""Return the region of `text` around `needle`, or None if it is not there.
|
|
|
|
Four patterns are tried in order, because the same term means different
|
|
things depending on where it sits: an indented line is an option entry or
|
|
a numbered section, a flush-left line is a heading, and a bare substring is
|
|
a mention in prose. The last pattern allows any whitespace where the needle
|
|
has a space, because a multi-word phrase wraps across lines in a rendered
|
|
man page or RFC - searching ip-route for "sending a packet along this path"
|
|
reported it absent while the page reads "...along this\\n path."
|
|
"""
|
|
needle = (needle or "").strip()
|
|
if not needle:
|
|
return None
|
|
|
|
escaped = re.escape(needle)
|
|
patterns = (
|
|
# A heading-shaped line: optional section number, then the needle
|
|
# with whitespace tolerated inside words. PDF text layers split
|
|
# words at column boundaries - "Clock configurati on register" -
|
|
# and an exact search for the heading then lands on every
|
|
# cross-reference to it ("refer to Section 4.3.2: Clock
|
|
# configuration register") and never on the heading itself.
|
|
(rf"^[ \t]*(?:\d+(?:\.\d+)*\.?[ \t]+)?{_loose_words(needle)}[ \t]*:?[ \t]*$",
|
|
re.MULTILINE | re.IGNORECASE),
|
|
(rf"^[ \t]+{escaped}\b", re.MULTILINE), # " -Z user"
|
|
(rf"^[ \t]*{escaped}\b", re.MULTILINE), # "EXAMPLES"
|
|
(escaped, re.IGNORECASE), # anywhere in prose
|
|
(r"\s+".join(re.escape(w) for w in needle.split()), re.IGNORECASE),
|
|
)
|
|
|
|
first_any = None
|
|
chosen = None
|
|
skipped = 0
|
|
|
|
def contents_entry(m) -> bool:
|
|
line = _line_around(text, m.start())
|
|
if _is_toc_line(line):
|
|
return True
|
|
# The leaderless, page-numberless form used by every RFC published
|
|
# in the v3 text format since 2019: " 15.5.19. 418 (Unused)".
|
|
# It carries no marker at all; what identifies it is that the same
|
|
# line appears again later, as the heading it points to. Checked
|
|
# only in the opening quarter of the document, where a contents
|
|
# listing lives, so the scan stays cheap on a 500 KB specification.
|
|
return (m.start() < len(text) // 4
|
|
and _NUMBERED_ENTRY.match(line) is not None
|
|
and _recurs_later(text, line, m.end()))
|
|
|
|
for pattern, flags in patterns:
|
|
matches = [m for m in re.finditer(pattern, text, flags) if m.start() >= start_at]
|
|
if not matches:
|
|
continue
|
|
if first_any is None:
|
|
first_any = matches[0]
|
|
body = [m for m in matches[:40] if not contents_entry(m)] + matches[40:]
|
|
if body:
|
|
chosen = body[0]
|
|
skipped = sum(1 for m in matches if m.start() < chosen.start())
|
|
break
|
|
|
|
if chosen is None:
|
|
chosen = first_any
|
|
if chosen is None:
|
|
return None
|
|
|
|
start = max(0, chosen.start() - 200)
|
|
end = min(len(text), chosen.start() + window)
|
|
excerpt = text[start:end]
|
|
|
|
note = ""
|
|
if skipped:
|
|
note = (f"; skipped {skipped} table-of-contents "
|
|
f"{'entry' if skipped == 1 else 'entries'}")
|
|
if _is_cross_reference(text, chosen.start()):
|
|
note += ("; this line is a cross-reference to the section, not the section itself - "
|
|
"the heading may be split across words in this document's text layer; try "
|
|
"the section number, or a distinctive term from its first paragraph")
|
|
# Naming the section is the difference between "the string is here" and
|
|
# "the subject is here". A match on a word used in another sense lands
|
|
# under a heading that says so.
|
|
if section := enclosing_heading(text, chosen.start()):
|
|
note += f"; this is inside the section '{section}'"
|
|
header = (f"[Jumped to '{needle}' at character {chosen.start():,} "
|
|
f"of {len(text):,}{note}]")
|
|
|
|
prefix = "…\n" if start > 0 else ""
|
|
suffix = ""
|
|
if end < len(text):
|
|
suffix = (f"\n…\n\n[Showing the region around '{needle}'. The full document is "
|
|
f"{len(text):,} characters - {continue_hint}.]")
|
|
return f"{header}\n\n{prefix}{excerpt}{suffix}"
|
|
|
|
|
|
def search_window(
|
|
text: str,
|
|
needle: str,
|
|
*,
|
|
window: int = 3800,
|
|
start_at: int = 0,
|
|
continue_hint: str = "call again with offset= to read more",
|
|
miss_note: str,
|
|
) -> str:
|
|
"""`find_in_document`, but a miss returns the opening of the document too.
|
|
|
|
A caller that gets back only "that term is not here" has spent a call and
|
|
holds nothing, and the observed response to holding nothing is to stop
|
|
using the source rather than to try another term. The document is already
|
|
fetched and already in memory; showing its first window costs one round of
|
|
tokens and keeps the call from being a dead end.
|
|
"""
|
|
found = find_in_document(text, needle, window=window, start_at=start_at,
|
|
continue_hint=continue_hint)
|
|
if found is not None:
|
|
return found
|
|
|
|
size = max(500, min(window, len(text)))
|
|
lead = text[:size]
|
|
tail = ""
|
|
if size < len(text):
|
|
tail = (f"\n…\n\n[This is the opening only - {len(text) - size:,} characters follow. "
|
|
f"To read on, {continue_hint}.]")
|
|
return (f"{miss_note}\n\n[Showing the opening of the document instead, so the call is not "
|
|
f"wasted. The document itself is fine; only the search term missed.]\n\n{lead}{tail}")
|