95 lines
3.5 KiB
Python
Executable File
95 lines
3.5 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Unit tests for vm_agent's one pre-flight refusal.
|
|
|
|
python3 tests/test_vm_preflight.py
|
|
|
|
WHY THIS IS NOT A SMOKE CASE
|
|
|
|
The interesting half - a task that names its machine and is therefore let
|
|
through - starts a real VM session when exercised over HTTP, and a
|
|
session is minutes long by design. The smoke suite has a 90 s per-case
|
|
timeout, so only the refusals can live there; the "does not fire" side
|
|
has to be tested against the function.
|
|
|
|
WHAT IT IS FOR
|
|
|
|
A Python script runs on both machines, a .sh only on Ubuntu, a .ps1 only
|
|
on Windows. Guessing wrong destroys a whole session, because the machine
|
|
does not survive it. The system prompt asks the model to check with the
|
|
user first, but a prompt rule loses to an explicit user demand - the
|
|
finding restated four times in TUNING.md - so the tool refuses in its own
|
|
output, which is what this model treats as fact.
|
|
|
|
The refusal has to be narrow. Firing when the caller has already said
|
|
which machine would be worse than not firing at all: it would train the
|
|
model to add a magic word rather than to ask.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "app"))
|
|
|
|
from owuitools.vm_preflight import CROSS_PLATFORM, names_its_machine, should_ask # noqa: E402
|
|
|
|
FAILS: list[str] = []
|
|
|
|
|
|
def check(label: str, condition: bool, detail: str = "") -> None:
|
|
if condition:
|
|
print(f"ok {label}")
|
|
else:
|
|
FAILS.append(f"{label}: {detail}")
|
|
print(f"FAIL {label}\n {detail}")
|
|
|
|
|
|
fires = should_ask
|
|
|
|
|
|
# --- must stop and ask: cross-platform, no machine named -----------------
|
|
for task in (
|
|
"write a python script that prints the first 20 primes and run it",
|
|
"check whether this pip package installs cleanly",
|
|
"run the node build and tell me if it passes",
|
|
"does this Java program compile",
|
|
"build the rust crate and report the warnings",
|
|
):
|
|
check(f"asks: {task[:46]!r}", fires(task, "ubuntu"), "did not fire")
|
|
|
|
# --- must NOT fire: the machine is already named -------------------------
|
|
for task, os_name in (
|
|
("write a bash script for ubuntu that prints the first 20 primes", "ubuntu"),
|
|
("run this python script on windows with powershell", "windows"),
|
|
("check the python version in the ubuntu container image", "ubuntu"),
|
|
("test this .ps1 file", "windows"),
|
|
("apt install python3-venv and check it works", "ubuntu"),
|
|
("run the python tests under linux", "ubuntu"),
|
|
("run install.bat and see if python is on PATH afterwards", "windows"),
|
|
):
|
|
check(f"lets through: {task[:42]!r}", not fires(task, os_name), "fired anyway")
|
|
|
|
# --- must NOT fire: nothing cross-platform in the task -------------------
|
|
for task, os_name in (
|
|
("write a script that renames every file in the folder", "ubuntu"),
|
|
("check how much disk space is free", "windows"),
|
|
("compile the C program and run it", "ubuntu"),
|
|
):
|
|
check(f"no trigger: {task[:42]!r}", not fires(task, os_name), "fired on a task with no trigger")
|
|
|
|
# --- the check must not be fooled by the word appearing in another word --
|
|
check("does not fire on 'pygmy' or 'pyramid'",
|
|
not CROSS_PLATFORM.search("draw a pyramid of pygmy goats"),
|
|
"matched a substring inside an unrelated word")
|
|
check("does not fire on 'gopher' for 'go'",
|
|
not CROSS_PLATFORM.search("write a gopher client"),
|
|
"'go' matched inside 'gopher'")
|
|
|
|
print()
|
|
print("=" * 60)
|
|
if FAILS:
|
|
print(f"FAILED {len(FAILS)}")
|
|
for f in FAILS:
|
|
print(" -", f)
|
|
sys.exit(1)
|
|
print("all vm_agent pre-flight cases pass")
|