156 lines
7.5 KiB
Python
156 lines
7.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Building a file in pieces, because designing it whole is what fails.
|
|
|
|
python3 tests/test_vm_incremental_build.py
|
|
|
|
WHY
|
|
|
|
The agent had `write_file` and nothing else. Every change to a file meant
|
|
reproducing the whole file, and a file you must reproduce whole is a file
|
|
you must design whole before you can write any of it.
|
|
|
|
Measured: a session spent twenty-two minutes and roughly fifty thousand
|
|
tokens inside ONE turn, designing a WPF application in its head, still on
|
|
task one of twelve, having made no tool call for twenty minutes. It was
|
|
not stuck and it was not looping. It was doing the only thing the tools
|
|
allowed.
|
|
|
|
The fix is the shape of the tools, not a rule in the prompt - this stack
|
|
has now three times proved a prompt rule loses to what the tools make
|
|
easy. So: `append_file` to build forward, `edit_file` to revise, and a cap
|
|
on `write_file` that makes those the path of least resistance.
|
|
|
|
`edit_file` is modelled on the one the author of this file uses all day:
|
|
exact-text replacement that FAILS LOUDLY when the text is missing or
|
|
ambiguous. That failure is the feature. Text that is not there means a
|
|
mistaken belief about the file, and a refusal now is far cheaper than a
|
|
silent half-edit found three sections later.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
sys.path.insert(0, os.path.join(HERE, "..", "vm", "vmd"))
|
|
sys.path.insert(0, os.path.join(HERE, "..", "vmagent", "app"))
|
|
|
|
FAILS: list[str] = []
|
|
|
|
|
|
def check(label: str, condition: bool, detail: str = "") -> None:
|
|
if condition:
|
|
print(f"ok {label}")
|
|
else:
|
|
FAILS.append(label)
|
|
print(f"FAIL {label}\n {detail}")
|
|
|
|
|
|
import tools # noqa: E402
|
|
|
|
VMD_SRC = open(os.path.join(HERE, "..", "vm", "vmd", "vmd.py"), encoding="utf-8").read()
|
|
|
|
# --- the tools exist and say what they are for ---------------------------
|
|
for os_name in ("ubuntu", "windows"):
|
|
names = [t["function"]["name"] for t in tools.schemas(os_name)]
|
|
check(f"{os_name}: append_file is offered", "append_file" in names)
|
|
check(f"{os_name}: edit_file is offered", "edit_file" in names)
|
|
|
|
schema = {t["function"]["name"]: t["function"] for t in tools.schemas("ubuntu")}
|
|
check("edit_file asks for exact text", "EXACTLY once" in schema["edit_file"]["description"])
|
|
check("and says a refusal is information, not an obstacle",
|
|
"wrong" in schema["edit_file"]["description"])
|
|
check("write_file says it is for starting, not for everything",
|
|
"skeleton" in schema["write_file"]["description"]
|
|
and "16 KB" in schema["write_file"]["description"])
|
|
check("append_file says what it is for",
|
|
"section" in schema["append_file"]["description"])
|
|
check("edit_file can replace every occurrence when that is meant",
|
|
"replace_all" in schema["edit_file"]["parameters"]["properties"])
|
|
check("but replace_all is not required, so the default is the safe one",
|
|
"replace_all" not in schema["edit_file"]["parameters"]["required"])
|
|
|
|
# --- the cap is a forcing function, and says so --------------------------
|
|
import re # noqa: E402
|
|
|
|
cap = int(re.search(r"MAX_WRITE_BYTES = ([\d_]+)", VMD_SRC).group(1).replace("_", ""))
|
|
check("one write is capped well below the file limit", 4000 < cap < 64000, f"{cap}")
|
|
check("the refusal explains why rather than just refusing",
|
|
"design a whole program" in schema["write_file"]["description"]
|
|
or "designing a whole program" in schema["write_file"]["description"])
|
|
check("and the refusal names the way past it",
|
|
"append_file" in VMD_SRC.split("MAX_WRITE_BYTES:")[1][:900]
|
|
and "edit_file" in VMD_SRC.split("MAX_WRITE_BYTES:")[1][:900])
|
|
check("generated data still has an escape hatch",
|
|
"command in the machine" in VMD_SRC.split("MAX_WRITE_BYTES:")[1][:900],
|
|
"a big data file is not the same mistake as a big program")
|
|
|
|
# --- edit_file's refusals ------------------------------------------------
|
|
body = VMD_SRC[VMD_SRC.index("def v_edit_file"):VMD_SRC.index("def v_list_dir")]
|
|
check("an edit whose text is not in the file is refused",
|
|
"not in the file" in body)
|
|
check("an ambiguous edit is refused rather than guessed at",
|
|
"ambiguous" in body and "found > 1" in body)
|
|
check("a no-op edit is refused", "identical" in body)
|
|
check("the file is read before it is written",
|
|
body.index("agent.read_file") < body.index("agent.write_file"),
|
|
"an edit that writes without reading is a guess")
|
|
check("and the agent is told to check it still parses",
|
|
"still parses" in body)
|
|
|
|
# --- append_file ---------------------------------------------------------
|
|
ap = VMD_SRC[VMD_SRC.index("def v_append_file"):VMD_SRC.index("def v_edit_file")]
|
|
check("appending to a file that is not there creates it",
|
|
"existing = b\"\"" in ap)
|
|
check("an append is capped like a write", "MAX_WRITE_BYTES" in ap)
|
|
|
|
# --- the directory bug this uncovered ------------------------------------
|
|
#
|
|
# write_file ran `mkdir -p "$(dirname ...)"` on every guest. On Windows
|
|
# that reaches PowerShell, where mkdir takes no -p and dirname is not a
|
|
# command - and the result was discarded, so it failed silently and any
|
|
# nested path failed to write. A top-level file worked, because Z:\
|
|
# already exists, which is the kind of half-working that survives testing.
|
|
parent = VMD_SRC[VMD_SRC.index("def _make_parent"):VMD_SRC.index("def v_write_file")]
|
|
check("the parent directory is made with the right shell for the machine",
|
|
"windows" in parent and "New-Item" in parent and "mkdir -p" in parent,
|
|
"this is the third bash-to-PowerShell bug in this subsystem")
|
|
# Two goes at this failed the same way. The second used `Split-Path -Parent
|
|
# -LiteralPath`, which PowerShell 5.1 rejects as an ambiguous parameter
|
|
# set: $p came back null, Test-Path and New-Item both bound null and did
|
|
# nothing, and because those are NON-TERMINATING errors PowerShell still
|
|
# exited 0. Measured: PARENT=[], EXISTS_AFTER=, exit 0, no directory.
|
|
parent_code = parent.split('"""')[2] # past the docstring; the code only
|
|
check("the path is split by vmd, not parsed by the guest",
|
|
"Split-Path" not in parent_code and "dirname" not in parent_code
|
|
and "rsplit(" in parent_code,
|
|
"vmd built this path, so it can take the parent without asking a shell")
|
|
check("and success is confirmed rather than read off an exit code",
|
|
"MADE" in parent_code and "stdout" in parent_code,
|
|
"PowerShell exits 0 on a non-terminating error, so zero proves nothing")
|
|
check("a file at the share root needs no folder made",
|
|
"len(parent) <= 2" in parent_code)
|
|
check("both writers use it",
|
|
VMD_SRC.count("_make_parent(sess, agent, path)") == 2,
|
|
"append_file would have reintroduced it")
|
|
|
|
# --- one turn cannot run away --------------------------------------------
|
|
ollama_src = open(os.path.join(HERE, "..", "vmagent", "app", "ollama.py"),
|
|
encoding="utf-8").read()
|
|
check("a single turn has an output ceiling", "num_predict" in ollama_src,
|
|
"there was none: one turn ran 22 minutes and ~50,000 tokens")
|
|
|
|
# --- the feed shows the new verbs ----------------------------------------
|
|
import loop # noqa: E402
|
|
|
|
for verb, args in (("append_file", {"path": "a.ps1", "content": "x" * 40}),
|
|
("edit_file", {"path": "a.ps1", "old": "foo", "new": "bar"})):
|
|
check(f"{verb} appears in the command feed", loop._terminal_for(verb, args) != [])
|
|
check("an append reads as an append, not a write",
|
|
">>" in loop._terminal_for("append_file", {"path": "a", "content": "x"})[0])
|
|
|
|
print()
|
|
if FAILS:
|
|
print(f"FAILED {len(FAILS)}")
|
|
raise SystemExit(1)
|
|
print("a file can be built a section at a time, and a wrong belief about it is refused")
|