"""Re-runs the agent's own verification, so the claim is not the evidence. WHAT KILLED THE PREVIOUS ATTEMPT The archived sub-agent orchestrator was retired for accuracy, and its post-mortem names the mechanism exactly: the coding agent "hallucinated third-party library APIs and wrote self-confirming tests around them". The tests passed. The summary said so. Nothing in the pipeline was in a position to disagree, because the only account of what happened came from the thing being accounted for. WHAT THIS DOES INSTEAD finish() requires a verification command. After the agent's last step, vmd runs that command ITSELF, in a fresh shell, and the exit code that reaches the model is the one vmd saw - not the one the agent reported. It is the same move as the per-job nonce in octave/run.sh, applied to truth rather than to job identity: the fact in tool output is produced by the side that is not being checked. If the re-run disagrees with the agent, the agent's summary is still shown - but labelled a claim, with the real output beside it. Suppressing it would lose information; presenting it as a finding would repeat the original mistake. WHAT THIS DOES NOT SOLVE, STATED PLAINLY A self-confirming test still passes when re-run, because it is still the agent's test. All that can be done cheaply is to NOTICE the shape - a verification command whose subject is a file created during this same session - and say so. That is a flag on the result, not a refusal, and it does not catch an agent that quietly edits an existing file instead. Mitigated, not eliminated. """ from __future__ import annotations import os import re import shlex from dataclasses import dataclass, field from guest import GuestError, GuestTimeout VERIFY_TIMEOUT_S = 180.0 MAX_OUTPUT = 4000 @dataclass class Verification: command: str expected: str ran: bool = False exit_code: int | None = None stdout: str = "" stderr: str = "" error: str = "" self_confirming: list[str] = field(default_factory=list) @property def ok(self) -> bool: return self.ran and self.exit_code == 0 def render(self) -> str: """vmd's own words. This is what the model reads as fact.""" if not self.command: return "Verification: none was offered." lines = [f"Verification command: `{self.command}`"] if not self.ran: lines.append(f" NOT RUN: {self.error}") lines.append(" Treat everything below as an unverified claim.") return "\n".join(lines) verdict = "passed" if self.ok else "FAILED" lines.append(f" Re-run by the supervisor, not by the agent: {verdict} " f"(exit {self.exit_code}).") if self.expected: lines.append(f" The agent said to expect: {self.expected}") out = (self.stdout or "").strip() err = (self.stderr or "").strip() if out: lines.append(" Output:\n" + _indent(out[:MAX_OUTPUT])) if err: lines.append(" Errors:\n" + _indent(err[:MAX_OUTPUT])) if not self.ok: lines.append( " The check the agent chose did not pass when run again. " "Its summary below is a claim, not a result." ) if self.self_confirming: lines.append( " NOTE: this check runs " + ", ".join(f"`{f}`" for f in self.self_confirming) + ", which the agent created during this session. A test written " "alongside the code it tests can pass without the code being right." ) return "\n".join(lines) def _indent(text: str) -> str: return "\n".join(f" {line}" for line in text.splitlines()) def _looks_self_confirming(command: str, created: set[str]) -> list[str]: """Which session-created files does this command name? Deliberately crude - a substring match on basenames. It is meant to catch `pytest test_thing.py` where the agent wrote test_thing.py ten minutes ago, not to be a parser. A false positive costs one extra sentence in the report; a parser would cost a maintenance burden and still miss the interesting cases. """ if not created: return [] try: words = shlex.split(command) except ValueError: words = re.split(r"\s+", command) hits: list[str] = [] for path in created: base = os.path.basename(path) if not base: continue if any(base == os.path.basename(w) or base in w for w in words): hits.append(base) return sorted(set(hits)) def working_directory_prelude(session) -> str: """Put a command in the project folder, on either machine. THE AGENT AND ITS SUPERVISOR HAVE TO AGREE ABOUT WHERE "HERE" IS. Only the supervisor used to do this. The agent's own `run` inherited whatever directory the guest agent happened to start in - on Windows that is C:\\Windows\\system32 - so a session could write a file with a relative path, read it back, run it, and pass every check it made, while the supervisor re-ran the same command one directory away and found nothing. Measured: a Windows session did exactly that, the verification failed on a file that existed, and the whole job had to be run a second time. It is also where the work has to be in order to survive: the project folder is the only thing that outlives the machine. THE PRELUDE IS SHELL, SO IT HAS TO BE THE RIGHT SHELL. This was `cd ... 2>/dev/null || cd /;` for both machines, and PowerShell has no `||` - measured on the first real Windows session, where correct work was reported to the user as a claim because the supervisor's own wrapper failed to parse. """ if session.os_name == "windows": cwd = "Z:\\" if session.share else "C:\\" return f"Set-Location -LiteralPath '{cwd}' -ErrorAction SilentlyContinue; " cwd = "/mnt/project" if session.share else "/root" return f"cd {shlex.quote(cwd)} 2>/dev/null || cd /; " def run(session, command: str, expected: str = "", files_created: set[str] | None = None) -> Verification: """Run the agent's check again, in a shell it did not prepare.""" result = Verification(command=(command or "").strip(), expected=(expected or "").strip()) if not result.command: result.error = "the agent offered no verification command" return result result.self_confirming = _looks_self_confirming(result.command, files_created or set()) try: with session.guest(timeout=30.0) as agent: # A fresh login shell, in the project directory, with none of # whatever environment the agent had built up around itself. # The point is to reproduce the check, not the circumstances # the agent arranged for it. The SAME prelude the agent's own # `run` uses - see working_directory_prelude for why that # matters more than it looks. outcome = agent.shell( working_directory_prelude(session) + result.command, timeout_s=VERIFY_TIMEOUT_S) except GuestTimeout: result.error = ( f"the check was still running after {VERIFY_TIMEOUT_S:.0f}s and was abandoned" ) return result except (GuestError, OSError) as exc: result.error = f"the machine could not be reached to run it ({exc})" return result result.ran = True result.exit_code = outcome["exit_code"] result.stdout = outcome["stdout"] result.stderr = outcome["stderr"] return result