"""Git and Gitea, run on the host against a tree the guest controls. THE WORKING TREE IS HOSTILE INPUT The project share is mounted inside the VM, and anything running there can write anywhere in it - including `.git/`. That makes an ordinary `git commit` on the host a remote code execution primitive: * `.git/hooks/pre-commit` is executed, as whoever ran git. * `.git/config` can set `core.hooksPath`, or a `filter.*.clean` command that runs on `git add`. * `.gitattributes` can point at such a filter. None of that is exotic; it is how git is designed to work. So every git invocation here is wrapped by _git(), which pins the settings that decide whether code runs, on the COMMAND LINE - where `-c` beats anything the repository's own config says - and passes --no-verify where hooks would otherwise be consulted anyway. THE TOKEN NEVER GOES NEAR THE GUEST The remote is stored without credentials and the token is supplied at push time through GIT_ASKPASS. A `.git/config` the guest rewrites therefore has nothing to leak, and a `git remote -v` inside the VM shows a plain URL. The push itself goes to the LAN address rather than the public hostname: it never leaves this host, so it cannot be intercepted between here and the VPS that fronts Gitea. """ from __future__ import annotations import json import os import re import subprocess import time import tempfile import urllib.error import urllib.request from urllib.parse import quote # Settings, read once. Everything here is optional: a session with no # Gitea configured still runs, it just has nowhere to publish, and says so. GITEA_INTERNAL = os.environ.get("OWUITOOLS_GITEA_URL", "http://192.168.1.10:7200").rstrip("/") GITEA_PUBLIC = os.environ.get("OWUITOOLS_GITEA_PUBLIC_URL", "https://git.example.com").rstrip("/") GITEA_TOKEN = os.environ.get("OWUITOOLS_GITEA_TOKEN", "") GITEA_ORG = os.environ.get("OWUITOOLS_GITEA_ORG", "vmlab") GITEA_BOT = os.environ.get("OWUITOOLS_GITEA_BOT", "vmagent") BOT_NAME = "owuitools VM agent" BOT_EMAIL = "vmagent@example.com" # Applied to every git call. `-c` on the command line takes precedence # over the repository's own config, which is the whole point: the tree # belongs to the guest and its .git/config is not to be trusted. _SAFE_FLAGS = [ "-c", "core.hooksPath=/var/empty", # no hooks, whatever .git/config says "-c", "core.fsmonitor=false", # fsmonitor is another command to run "-c", "protocol.ext.allow=never", # ext:: URLs are a shell "-c", "protocol.file.allow=never", "-c", "uploadpack.allowFilter=false", "-c", f"user.name={BOT_NAME}", "-c", f"user.email={BOT_EMAIL}", "-c", "commit.gpgsign=false", "-c", "gc.auto=0", ] _SAFE_ENV = { # Neither the host's git config nor a stray ~/.gitconfig gets a say. "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_CONFIG_SYSTEM": "/dev/null", "GIT_TERMINAL_PROMPT": "0", "GIT_ALLOW_PROTOCOL": "http:https", } class GitError(RuntimeError): """Phrased for the model to read.""" def configured() -> bool: return bool(GITEA_TOKEN) def _git(repo: str, *args: str, env_extra: dict[str, str] | None = None, timeout: float = 120.0) -> subprocess.CompletedProcess: env = {"PATH": "/usr/bin:/bin", "HOME": "/nonexistent", **_SAFE_ENV} if env_extra: env.update(env_extra) return subprocess.run( ["/usr/bin/git", "-C", repo, *_SAFE_FLAGS, *args], capture_output=True, text=True, timeout=timeout, env=env, check=False, ) def init(repo_path: str, project_name: str) -> None: """Make the project directory a repository if it is not one already.""" if os.path.isdir(os.path.join(repo_path, ".git")): return result = _git(repo_path, "init", "-q", "-b", "main") if result.returncode != 0: raise GitError(f"could not initialise the project repository: {result.stderr.strip()}") # A guest that fills the share with build output should not be # committing it, and the agent should not have to remember to say so. ignore = os.path.join(repo_path, ".gitignore") if not os.path.exists(ignore): with open(ignore, "w") as handle: handle.write( "# Written when the project was created. Edit freely.\n" "__pycache__/\n*.pyc\n.venv/\nvenv/\nnode_modules/\n" "*.log\n*.tmp\n.DS_Store\n" ) def has_commits(repo_path: str) -> bool: """Is there anything in this repository's history to publish? Separate from has_changes, which is about the working tree. A session that changed nothing can still have something to push: the commit a previous session made before it was cancelled. """ return _git(repo_path, "rev-parse", "--verify", "HEAD").returncode == 0 # --- what gets published, and what stays behind on the array ---------- # # MEASURED, AND THE REASON THIS EXISTS. Release v3 of the project # `windows-troubleshooting-toolkit` carried ten files - `_build.ps1`, # `check_disp3.ps1`, `test_thread2.ps1`, `csharp_out.txt` and more of the # same - and did NOT carry the 34 KB toolkit the session had been asked # for. Two things conspired: `git add -A` commits whatever the agent # happened to leave in the folder, and the asset list sorts `_build` # ahead of `TroubleshootingToolkit` and then stops at ten. # # SO THE FILTER BELONGS AT THE COMMIT, NOT AT THE RELEASE. Gitea builds # the "Source code (zip)" from the tree the tag points at, and that # archive is the one thing on a release page that actually downloads - # measured: `Content-Disposition: attachment`, where every uploaded # asset, a `.zip` included, comes back `inline`. A release is therefore # only ever as clean as the commit behind it. Filtering here cleans the # repository, the asset list and the archive in one move. # # THE PATTERNS ARE DELIBERATELY NARROW. `check_*` and `find_*` are NOT # here, however scratch-like `check_disp2.ps1` looks: `find_duplicates.py` # was a real deliverable, and a job asked to "check disk health" would # reasonably produce `check_disk.py`. What is here is either a scratch # convention nobody names a deliverable after (a leading underscore), a # build product, or the thing the operator named outright - "ambiguous # testing tools / scripts". SCRATCH_PATTERNS = ( # Scratch by convention: a leading underscore, at any depth. "**/_*", # Test scripts and their captured output. "**/test_*", "**/test-*", "**/*_test.*", "**/*-test.*", "**/smoke.*", "**/smoke_*", "**/probe_*", "**/scratch*", "**/tmp_*", "**/*_tmp.*", "**/*_out.txt", "**/*_out.log", "**/*_out.json", "**/*.out", # Build products. Source stays; what the compiler made does not. "**/*.exe", "**/*.dll", "**/*.obj", "**/*.o", "**/*.pdb", "**/*.class", # Whole scratch trees. `test*/` rather than `*test*/` on purpose: # the broader form also swallows `latest/`. "**/test*/**", "**/*_test/**", "**/*-test/**", "**/tmp/**", "**/scratch/**", ) def _scratch_specs() -> list[str]: """The exclusion as git pathspecs. `glob` magic so `**/` means "at any depth, including the top level"; without it git treats the pattern as a literal path prefix. """ return [f":(exclude,glob){pattern}" for pattern in SCRATCH_PATTERNS] def _would_stage(repo_path: str, *specs: str) -> set[str]: """What `git add -A` would do, without doing it. `--dry-run` prints `add 'path'` for anything new or modified and `remove 'path'` for anything deleted, which is exactly the set that would end up in the commit. """ result = _git(repo_path, "add", "-A", "--dry-run", "--", ".", *specs) names: set[str] = set() for line in result.stdout.splitlines(): line = line.strip() for prefix in ("add '", "remove '"): if line.startswith(prefix) and line.endswith("'"): names.add(line[len(prefix):-1]) break return names def held_back(repo_path: str) -> list[str]: """Files in the folder that will not be published, so it can be said. A filter nobody is told about is indistinguishable from a bug: the agent writes `test_thing.ps1`, commits, and watches it not appear. This is what `git_commit` reports back, in tool output - the one channel this stack has repeatedly measured the model to treat as fact. """ return sorted(_would_stage(repo_path) - _would_stage(repo_path, *_scratch_specs())) def _tracked_scratch(repo_path: str) -> list[str]: """Scratch that an EARLIER session already committed. Without this, a project resumed under its old name keeps publishing the junk it accumulated before the filter existed - and the operator asked for the release to be clean, not for it to start being clean only for projects that happen to be new. """ result = _git(repo_path, "ls-files", "-z", "--", ".", *_scratch_specs()) if result.returncode != 0: return [] keep = {n for n in result.stdout.split("\0") if n} result = _git(repo_path, "ls-files", "-z") if result.returncode != 0: return [] return sorted({n for n in result.stdout.split("\0") if n} - keep) def _substantive(names: set[str]) -> set[str]: """Dotfiles are bookkeeping, not the work. `init` writes a `.gitignore` into every new project, and without this that one file is enough to make a folder holding nothing but tests look like a folder with something publishable in it - which is exactly the case the fallback below exists to catch. """ return {n for n in names if not os.path.basename(n).startswith(".")} def _staging_plan(repo_path: str) -> tuple[list[str], set[str]]: """Which exclusions apply here, and what they would leave. One place, because `has_changes` and `commit` have to agree: if they disagree, the end-of-session backstop decides there is something to commit and then stages nothing, and git calls that an error. """ specs = _scratch_specs() staged = _would_stage(repo_path, *specs) if not _substantive(staged) and not tracked_files(repo_path, limit=1): # Everything in the folder reads as scratch, and nothing # publishable has ever been committed here. That combination # almost always means the tests WERE the job - publishing them is # better than publishing an empty repository. specs = [] staged = _would_stage(repo_path) return specs, staged def has_changes(repo_path: str) -> bool: """Is there anything worth publishing? Deliberately not "is the working tree dirty". A session that left nothing but scratch behind has nothing to commit, and saying otherwise makes the end-of-session backstop try a commit that stages nothing and fails. """ specs, staged = _staging_plan(repo_path) if staged: return True # Nothing new, but a previous session may have published scratch that # this one should stop publishing. return bool(specs) and bool(_tracked_scratch(repo_path)) def commit(repo_path: str, message: str, author_note: str = "") -> str | None: """Commit everything in the tree. Returns the short sha, or None. The message is passed with `-F -` on stdin rather than `-m`, because it comes from the agent, which got it from a model, which read it off a guest's output. There is no shell in the path either way, but a message that starts with a dash should be a message, not an option. """ init(repo_path, os.path.basename(repo_path)) if not has_changes(repo_path): return None # SCRATCH IS NOT COMMITTED, SO IT CANNOT BE PUBLISHED. # # See SCRATCH_PATTERNS. The one exception is the project that would # otherwise end up empty: if everything in the folder looks like a # test and nothing publishable has ever been committed, then the # tests WERE the job, and publishing nothing would be worse than # publishing them. Checked against the committed tree rather than # against this session alone - a session that only produced scratch # for an existing project has genuinely produced nothing to publish. specs, _ = _staging_plan(repo_path) # Scratch an earlier session committed, before this filter existed or # under a name it did not catch. --cached: it leaves the file on the # array and only stops publishing it. if specs: stale = _tracked_scratch(repo_path) if stale: _git(repo_path, "rm", "-q", "--cached", "--", *stale) add = _git(repo_path, "add", "-A", "--", ".", *specs) if add.returncode != 0: raise GitError(f"could not stage the changes: {add.stderr.strip()}") body = message.strip()[:2000] or "Update from a VM session" if author_note: # Who asked for this, recorded where it cannot be edited by the # next session. The project share is writable by every caller, so # attribution has to live in the commit rather than in a file. body += f"\n\nRequested-By: {author_note}" result = subprocess.run( ["/usr/bin/git", "-C", repo_path, *_SAFE_FLAGS, "commit", "--no-verify", "-q", "-F", "-"], input=body, capture_output=True, text=True, timeout=120, env={"PATH": "/usr/bin:/bin", "HOME": "/nonexistent", **_SAFE_ENV}, check=False, ) if result.returncode != 0: raise GitError(f"the commit failed: {(result.stderr or result.stdout).strip()}") return _git(repo_path, "rev-parse", "--short", "HEAD").stdout.strip() or None SUMMARY_CAP = 140 # "On the Ubuntu machine," / "On this Windows machine:" and the like. _LEADING_MACHINE = re.compile( r"^on (?:the|this|a)?\s*\w*\s*machine[,:]?\s*", re.IGNORECASE) # The verb that asked for the thing, not a property of the thing. Only # verbs that CREATE are stripped: a project whose job was to "test" or # "check" something is accurately described by that word. _LEADING_VERB = re.compile( r"^(?:please\s+)?(?:write|create|build|make|implement|produce|add|develop)\s+" r"(?:me\s+)?(?:a|an|the)?\s*", re.IGNORECASE) def short_description(text: str, project_name: str) -> str: """One plain line saying what this project is, for the repository page. The operator's words: it "just has to accurately describe what the app does briefly - not sell it to me". So this is built from the job the session was given rather than written by a model: a description that is generated cannot be a pitch, and cannot describe work that was not asked for. First sentence, one line, hard cap. """ text = " ".join((text or "").split()) # A job that opens by shouting at the agent - "IMPORTANT: put this in # the project folder" - is a correction, not a description. Strip the # label; what follows is still closer to the subject than the warning. for label in ("goal:", "task:", "objective:", "important:", "note:", "warning:"): if text.lower().startswith(label): text = text[len(label):].lstrip() # A job opens by addressing the machine and telling it to build # something. A repository description is a label on the thing that was # built, so both of those go: "On the Ubuntu machine, write a small # Python script that ..." becomes "Small Python script that ...". text = _LEADING_MACHINE.sub("", text, count=1) text = _LEADING_VERB.sub("", text, count=1) text = text[:1].upper() + text[1:] if text else text for stop in (". ", "! ", "? "): at = text.find(stop) if at > 20: text = text[:at] break text = text.rstrip(" .") if len(text) > SUMMARY_CAP: cut = text[:SUMMARY_CAP] space = cut.rfind(" ") text = (cut[:space] if space > SUMMARY_CAP // 2 else cut).rstrip(" ,;:-") + "..." return text or f"Work from a VM session for project '{project_name}'." def raw_file_url(web_url: str, path: str) -> str: """A link that downloads one file, rather than one that shows a repository. The operator asked for a link to the script, and got a link to the repo - which is a page to navigate, not a file to fetch. Gitea serves the file itself at /raw/branch//. """ return f"{web_url}/raw/branch/main/{quote(path)}" def tracked_files(repo_path: str, limit: int = 12) -> list[str]: """What is actually committed, so a link is never offered for a missing file. `git ls-files` rather than a directory walk: the walk would also list whatever the guest left lying around uncommitted, and a link to that is a 404 with the operator's name on it. """ result = _git(repo_path, "ls-files", "-z") if result.returncode != 0: return [] names = [n for n in result.stdout.split("\0") if n and not n.startswith(".")] names.sort(key=lambda n: (n.count("/"), n.lower())) return names[:limit] def ensure_remote_repo(project_name: str, description: str = "") -> str: """Create the Gitea repository if it is missing; return its web URL.""" if not configured(): raise GitError("no Gitea token is configured, so there is nowhere to push.") api = f"{GITEA_INTERNAL}/api/v1" web = f"{GITEA_PUBLIC}/{GITEA_ORG}/{project_name}" status, body = _api("GET", f"{api}/repos/{quote(GITEA_ORG)}/{quote(project_name)}") if status == 200: # A repository created before there was anything to say about it # carries the placeholder. Replace that once, but never overwrite a # description somebody has since written by hand. if description: try: current = json.loads(body).get("description") or "" except ValueError: current = "" if not current or current.startswith("Created by the owuitools VM agent"): _api("PATCH", f"{api}/repos/{quote(GITEA_ORG)}/{quote(project_name)}", {"description": description}) return web status, body = _api("POST", f"{api}/orgs/{quote(GITEA_ORG)}/repos", { "name": project_name, "description": description or f"Created by the owuitools VM agent for project '{project_name}'.", # Public INSIDE an org whose visibility is Limited: any signed-in # Gitea user can read it, anonymous gets a 404. That is the # arrangement asked for, and the same one the previous sub-agent # stack used. "private": False, "auto_init": False, "default_branch": "main", }) # 409 is "it already exists", which is the outcome we wanted anyway. if status not in (201, 409): raise GitError(f"Gitea refused to create the repository ({status}): {body[:200]}") return web def _api(method: str, url: str, payload: dict | None = None) -> tuple[int, str]: """One small HTTP call, on the stdlib. Deliberately not httpx: vmd is the privileged half of this system and runs on the host's own interpreter, so every third-party import is a package that has to be present, patched and trusted there. Two JSON calls to one LAN address do not justify that. """ data = json.dumps(payload).encode() if payload is not None else None request = urllib.request.Request(url, data=data, method=method) request.add_header("Authorization", f"token {GITEA_TOKEN}") request.add_header("Accept", "application/json") if data is not None: request.add_header("Content-Type", "application/json") # No proxy handler: the systemd unit allows exactly one destination, # and an inherited http_proxy would send this somewhere else entirely. opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) try: with opener.open(request, timeout=20.0) as response: return response.status, response.read().decode("utf-8", "replace") except urllib.error.HTTPError as exc: return exc.code, exc.read().decode("utf-8", "replace") except urllib.error.URLError as exc: raise GitError(f"could not reach Gitea at {GITEA_INTERNAL}: {exc.reason}") from exc # --- releases --------------------------------------------------------- # # WHY A RELEASE AND NOT A LINK TO THE FILE. # # A link into the repository serves the file as `text/plain` with # `Content-Disposition: inline` - measured - so a browser shows the source # instead of saving it. Release assets are served as attachments, so they # download. A release is also a record of one session: every run that kept # anything leaves a numbered, dated entry with the files as they were at # the end of it, which a branch does not give you. # # Nothing in here may take a session down. The work is committed and pushed # by the time this runs; a release that fails to publish is a missing # convenience, not lost work, and it is reported as one. RELEASE_ASSET_MAX = 25 * 1024 * 1024 # per file RELEASE_ASSETS_MAX = 25 # per release # Was 10, which is how the toolkit came to be missing from its own # release: ten scratch files sorted ahead of it and used the budget # up. Scratch is no longer committed, so the cap now only has to be # high enough that a real project never reaches it. def _next_tag(api: str, project_name: str) -> str: """v1, v2, v3 ... one per session that kept something. Counted from what is published rather than held anywhere, so it stays right across a vmd restart. Only one session runs at a time, enforced by systemd, so there is nobody to race with. """ status, body = _api( "GET", f"{api}/repos/{quote(GITEA_ORG)}/{quote(project_name)}/releases?limit=50") existing = set() if status == 200: try: for entry in json.loads(body): existing.add(str(entry.get("tag_name") or "")) except ValueError: pass number = len(existing) + 1 while f"v{number}" in existing: number += 1 return f"v{number}" def publish_release(repo_path: str, project_name: str, web_url: str, summary: str = "") -> str: """Publish this session's files as a downloadable release. Returns the URL of the releases page, or "" if nothing was published. """ if not configured(): return "" api = f"{GITEA_INTERNAL}/api/v1" tag = _next_tag(api, project_name) payload = { "tag_name": tag, "target_commitish": "main", "name": tag, # Plain and short, for the same reason the repository # description is: this is a label, not a pitch. "body": summary or "Files as they were at the end of this session.", "draft": False, "prerelease": False, } status, body = _api( "POST", f"{api}/repos/{quote(GITEA_ORG)}/{quote(project_name)}/releases", payload) # "repo is empty" AFTER A SUCCESSFUL PUSH IS A RACE, NOT A REFUSAL. # # Gitea clears a repository's is_empty flag when it processes the push # it has just accepted, and the release API checks that flag. Measured # on the FIRST session of a new project - the common case - the push # succeeded, the commit was on the server, and the release came back # 422 "repo is empty" a moment later. A repository that already had # commits never showed it, which is why this survived two sessions. for attempt in range(4): if status not in (422,) or "empty" not in body.lower(): break time.sleep(1.5 * (attempt + 1)) status, body = _api( "POST", f"{api}/repos/{quote(GITEA_ORG)}/{quote(project_name)}/releases", payload) if status not in (201, 200): raise GitError(f"Gitea refused to create the release ({status}): {body[:200]}") try: release_id = int(json.loads(body).get("id")) except (ValueError, TypeError, AttributeError) as exc: raise GitError(f"Gitea's release reply could not be read: {exc}") from exc for name in tracked_files(repo_path, limit=RELEASE_ASSETS_MAX): full = os.path.join(repo_path, name) try: if os.path.getsize(full) > RELEASE_ASSET_MAX: continue with open(full, "rb") as handle: blob = handle.read() except OSError: continue # The asset name is flat - a release has no directories - so a # nested file keeps its path in the name rather than colliding # with a same-named file from another folder. _api_upload( f"{api}/repos/{quote(GITEA_ORG)}/{quote(project_name)}" f"/releases/{release_id}/assets?name={quote(name.replace('/', '_'), safe='')}", name.replace("/", "_"), blob) return f"{web_url}/releases" def _api_upload(url: str, filename: str, blob: bytes) -> tuple[int, str]: """One multipart POST, on the stdlib, for the same reason as _api.""" boundary = "----owuivm" + os.urandom(16).hex() safe = filename.replace('"', "").replace("\\", "") body = ( f"--{boundary}\r\n" f'Content-Disposition: form-data; name="attachment"; filename="{safe}"\r\n' "Content-Type: application/octet-stream\r\n\r\n" ).encode() + blob + f"\r\n--{boundary}--\r\n".encode() request = urllib.request.Request(url, data=body, method="POST") request.add_header("Authorization", f"token {GITEA_TOKEN}") request.add_header("Content-Type", f"multipart/form-data; boundary={boundary}") opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) try: with opener.open(request, timeout=60.0) as response: return response.status, response.read().decode("utf-8", "replace") except urllib.error.HTTPError as exc: return exc.code, exc.read().decode("utf-8", "replace") except urllib.error.URLError as exc: raise GitError(f"could not reach Gitea at {GITEA_INTERNAL}: {exc.reason}") from exc def push(repo_path: str, project_name: str, description: str = "") -> str: """Push main to Gitea and return the web URL.""" web = ensure_remote_repo(project_name, description) remote = f"{GITEA_INTERNAL}/{GITEA_ORG}/{project_name}.git" # Set rather than add, every time: whatever the guest may have written # into .git/config, the remote we push to is the one chosen here. _git(repo_path, "remote", "remove", "origin") result = _git(repo_path, "remote", "add", "origin", remote) if result.returncode != 0: raise GitError(f"could not set the push target: {result.stderr.strip()}") # GIT_ASKPASS rather than a credential in the URL: a URL ends up in # .git/config, in the reflog and in any error message git prints. with tempfile.NamedTemporaryFile("w", suffix=".sh", delete=False) as script: script.write("#!/bin/sh\ncase \"$1\" in Username*) echo \"$GIT_USER\" ;; *) echo \"$GIT_PASS\" ;; esac\n") askpass = script.name try: os.chmod(askpass, 0o700) result = _git( repo_path, "push", "-u", "origin", "main", env_extra={ "GIT_ASKPASS": askpass, "GIT_USER": GITEA_BOT, "GIT_PASS": GITEA_TOKEN, }, timeout=300.0, ) finally: try: os.unlink(askpass) except OSError: pass if result.returncode != 0: detail = (result.stderr or result.stdout).strip() # The token would appear in a URL-bearing error; it is not in the # URL, but scrub anyway rather than rely on that staying true. if GITEA_TOKEN: detail = detail.replace(GITEA_TOKEN, "") raise GitError(f"the push failed: {detail[:400]}") return web