636 lines
27 KiB
Python
636 lines
27 KiB
Python
"""Git and Gitea, run on the host against a tree the guest controls.
|
|
|
|
THE WORKING TREE IS HOSTILE INPUT
|
|
|
|
The project share is mounted inside the VM, and anything running there
|
|
can write anywhere in it - including `.git/`. That makes an ordinary
|
|
`git commit` on the host a remote code execution primitive:
|
|
|
|
* `.git/hooks/pre-commit` is executed, as whoever ran git.
|
|
* `.git/config` can set `core.hooksPath`, or a `filter.*.clean`
|
|
command that runs on `git add`.
|
|
* `.gitattributes` can point at such a filter.
|
|
|
|
None of that is exotic; it is how git is designed to work. So every git
|
|
invocation here is wrapped by _git(), which pins the settings that decide
|
|
whether code runs, on the COMMAND LINE - where `-c` beats anything the
|
|
repository's own config says - and passes --no-verify where hooks would
|
|
otherwise be consulted anyway.
|
|
|
|
THE TOKEN NEVER GOES NEAR THE GUEST
|
|
|
|
The remote is stored without credentials and the token is supplied at
|
|
push time through GIT_ASKPASS. A `.git/config` the guest rewrites
|
|
therefore has nothing to leak, and a `git remote -v` inside the VM shows
|
|
a plain URL. The push itself goes to the LAN address rather than the
|
|
public hostname: it never leaves this host, so it cannot be intercepted
|
|
between here and the VPS that fronts Gitea.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import time
|
|
import tempfile
|
|
import urllib.error
|
|
import urllib.request
|
|
from urllib.parse import quote
|
|
|
|
# Settings, read once. Everything here is optional: a session with no
|
|
# Gitea configured still runs, it just has nowhere to publish, and says so.
|
|
GITEA_INTERNAL = os.environ.get("OWUITOOLS_GITEA_URL", "http://192.168.1.10:7200").rstrip("/")
|
|
GITEA_PUBLIC = os.environ.get("OWUITOOLS_GITEA_PUBLIC_URL", "https://git.example.com").rstrip("/")
|
|
GITEA_TOKEN = os.environ.get("OWUITOOLS_GITEA_TOKEN", "")
|
|
GITEA_ORG = os.environ.get("OWUITOOLS_GITEA_ORG", "vmlab")
|
|
GITEA_BOT = os.environ.get("OWUITOOLS_GITEA_BOT", "vmagent")
|
|
|
|
BOT_NAME = "owuitools VM agent"
|
|
BOT_EMAIL = "vmagent@example.com"
|
|
|
|
# Applied to every git call. `-c` on the command line takes precedence
|
|
# over the repository's own config, which is the whole point: the tree
|
|
# belongs to the guest and its .git/config is not to be trusted.
|
|
_SAFE_FLAGS = [
|
|
"-c", "core.hooksPath=/var/empty", # no hooks, whatever .git/config says
|
|
"-c", "core.fsmonitor=false", # fsmonitor is another command to run
|
|
"-c", "protocol.ext.allow=never", # ext:: URLs are a shell
|
|
"-c", "protocol.file.allow=never",
|
|
"-c", "uploadpack.allowFilter=false",
|
|
"-c", f"user.name={BOT_NAME}",
|
|
"-c", f"user.email={BOT_EMAIL}",
|
|
"-c", "commit.gpgsign=false",
|
|
"-c", "gc.auto=0",
|
|
]
|
|
|
|
_SAFE_ENV = {
|
|
# Neither the host's git config nor a stray ~/.gitconfig gets a say.
|
|
"GIT_CONFIG_GLOBAL": "/dev/null",
|
|
"GIT_CONFIG_SYSTEM": "/dev/null",
|
|
"GIT_TERMINAL_PROMPT": "0",
|
|
"GIT_ALLOW_PROTOCOL": "http:https",
|
|
}
|
|
|
|
|
|
class GitError(RuntimeError):
|
|
"""Phrased for the model to read."""
|
|
|
|
|
|
def configured() -> bool:
|
|
return bool(GITEA_TOKEN)
|
|
|
|
|
|
def _git(repo: str, *args: str, env_extra: dict[str, str] | None = None,
|
|
timeout: float = 120.0) -> subprocess.CompletedProcess:
|
|
env = {"PATH": "/usr/bin:/bin", "HOME": "/nonexistent", **_SAFE_ENV}
|
|
if env_extra:
|
|
env.update(env_extra)
|
|
return subprocess.run(
|
|
["/usr/bin/git", "-C", repo, *_SAFE_FLAGS, *args],
|
|
capture_output=True, text=True, timeout=timeout, env=env, check=False,
|
|
)
|
|
|
|
|
|
def init(repo_path: str, project_name: str) -> None:
|
|
"""Make the project directory a repository if it is not one already."""
|
|
if os.path.isdir(os.path.join(repo_path, ".git")):
|
|
return
|
|
result = _git(repo_path, "init", "-q", "-b", "main")
|
|
if result.returncode != 0:
|
|
raise GitError(f"could not initialise the project repository: {result.stderr.strip()}")
|
|
# A guest that fills the share with build output should not be
|
|
# committing it, and the agent should not have to remember to say so.
|
|
ignore = os.path.join(repo_path, ".gitignore")
|
|
if not os.path.exists(ignore):
|
|
with open(ignore, "w") as handle:
|
|
handle.write(
|
|
"# Written when the project was created. Edit freely.\n"
|
|
"__pycache__/\n*.pyc\n.venv/\nvenv/\nnode_modules/\n"
|
|
"*.log\n*.tmp\n.DS_Store\n"
|
|
)
|
|
|
|
|
|
def has_commits(repo_path: str) -> bool:
|
|
"""Is there anything in this repository's history to publish?
|
|
|
|
Separate from has_changes, which is about the working tree. A session
|
|
that changed nothing can still have something to push: the commit a
|
|
previous session made before it was cancelled.
|
|
"""
|
|
return _git(repo_path, "rev-parse", "--verify", "HEAD").returncode == 0
|
|
|
|
|
|
# --- what gets published, and what stays behind on the array ----------
|
|
#
|
|
# MEASURED, AND THE REASON THIS EXISTS. Release v3 of the project
|
|
# `windows-troubleshooting-toolkit` carried ten files - `_build.ps1`,
|
|
# `check_disp3.ps1`, `test_thread2.ps1`, `csharp_out.txt` and more of the
|
|
# same - and did NOT carry the 34 KB toolkit the session had been asked
|
|
# for. Two things conspired: `git add -A` commits whatever the agent
|
|
# happened to leave in the folder, and the asset list sorts `_build`
|
|
# ahead of `TroubleshootingToolkit` and then stops at ten.
|
|
#
|
|
# SO THE FILTER BELONGS AT THE COMMIT, NOT AT THE RELEASE. Gitea builds
|
|
# the "Source code (zip)" from the tree the tag points at, and that
|
|
# archive is the one thing on a release page that actually downloads -
|
|
# measured: `Content-Disposition: attachment`, where every uploaded
|
|
# asset, a `.zip` included, comes back `inline`. A release is therefore
|
|
# only ever as clean as the commit behind it. Filtering here cleans the
|
|
# repository, the asset list and the archive in one move.
|
|
#
|
|
# THE PATTERNS ARE DELIBERATELY NARROW. `check_*` and `find_*` are NOT
|
|
# here, however scratch-like `check_disp2.ps1` looks: `find_duplicates.py`
|
|
# was a real deliverable, and a job asked to "check disk health" would
|
|
# reasonably produce `check_disk.py`. What is here is either a scratch
|
|
# convention nobody names a deliverable after (a leading underscore), a
|
|
# build product, or the thing the operator named outright - "ambiguous
|
|
# testing tools / scripts".
|
|
SCRATCH_PATTERNS = (
|
|
# Scratch by convention: a leading underscore, at any depth.
|
|
"**/_*",
|
|
# Test scripts and their captured output.
|
|
"**/test_*", "**/test-*", "**/*_test.*", "**/*-test.*",
|
|
"**/smoke.*", "**/smoke_*",
|
|
"**/probe_*", "**/scratch*", "**/tmp_*", "**/*_tmp.*",
|
|
"**/*_out.txt", "**/*_out.log", "**/*_out.json", "**/*.out",
|
|
# Build products. Source stays; what the compiler made does not.
|
|
"**/*.exe", "**/*.dll", "**/*.obj", "**/*.o", "**/*.pdb", "**/*.class",
|
|
# Whole scratch trees. `test*/` rather than `*test*/` on purpose:
|
|
# the broader form also swallows `latest/`.
|
|
"**/test*/**", "**/*_test/**", "**/*-test/**",
|
|
"**/tmp/**", "**/scratch/**",
|
|
)
|
|
|
|
|
|
def _scratch_specs() -> list[str]:
|
|
"""The exclusion as git pathspecs.
|
|
|
|
`glob` magic so `**/` means "at any depth, including the top level";
|
|
without it git treats the pattern as a literal path prefix.
|
|
"""
|
|
return [f":(exclude,glob){pattern}" for pattern in SCRATCH_PATTERNS]
|
|
|
|
|
|
def _would_stage(repo_path: str, *specs: str) -> set[str]:
|
|
"""What `git add -A` would do, without doing it.
|
|
|
|
`--dry-run` prints `add 'path'` for anything new or modified and
|
|
`remove 'path'` for anything deleted, which is exactly the set that
|
|
would end up in the commit.
|
|
"""
|
|
result = _git(repo_path, "add", "-A", "--dry-run", "--", ".", *specs)
|
|
names: set[str] = set()
|
|
for line in result.stdout.splitlines():
|
|
line = line.strip()
|
|
for prefix in ("add '", "remove '"):
|
|
if line.startswith(prefix) and line.endswith("'"):
|
|
names.add(line[len(prefix):-1])
|
|
break
|
|
return names
|
|
|
|
|
|
def held_back(repo_path: str) -> list[str]:
|
|
"""Files in the folder that will not be published, so it can be said.
|
|
|
|
A filter nobody is told about is indistinguishable from a bug: the
|
|
agent writes `test_thing.ps1`, commits, and watches it not appear.
|
|
This is what `git_commit` reports back, in tool output - the one
|
|
channel this stack has repeatedly measured the model to treat as
|
|
fact.
|
|
"""
|
|
return sorted(_would_stage(repo_path) - _would_stage(repo_path, *_scratch_specs()))
|
|
|
|
|
|
def _tracked_scratch(repo_path: str) -> list[str]:
|
|
"""Scratch that an EARLIER session already committed.
|
|
|
|
Without this, a project resumed under its old name keeps publishing
|
|
the junk it accumulated before the filter existed - and the operator
|
|
asked for the release to be clean, not for it to start being clean
|
|
only for projects that happen to be new.
|
|
"""
|
|
result = _git(repo_path, "ls-files", "-z", "--", ".", *_scratch_specs())
|
|
if result.returncode != 0:
|
|
return []
|
|
keep = {n for n in result.stdout.split("\0") if n}
|
|
result = _git(repo_path, "ls-files", "-z")
|
|
if result.returncode != 0:
|
|
return []
|
|
return sorted({n for n in result.stdout.split("\0") if n} - keep)
|
|
|
|
|
|
def _substantive(names: set[str]) -> set[str]:
|
|
"""Dotfiles are bookkeeping, not the work.
|
|
|
|
`init` writes a `.gitignore` into every new project, and without this
|
|
that one file is enough to make a folder holding nothing but tests
|
|
look like a folder with something publishable in it - which is
|
|
exactly the case the fallback below exists to catch.
|
|
"""
|
|
return {n for n in names if not os.path.basename(n).startswith(".")}
|
|
|
|
|
|
def _staging_plan(repo_path: str) -> tuple[list[str], set[str]]:
|
|
"""Which exclusions apply here, and what they would leave.
|
|
|
|
One place, because `has_changes` and `commit` have to agree: if they
|
|
disagree, the end-of-session backstop decides there is something to
|
|
commit and then stages nothing, and git calls that an error.
|
|
"""
|
|
specs = _scratch_specs()
|
|
staged = _would_stage(repo_path, *specs)
|
|
if not _substantive(staged) and not tracked_files(repo_path, limit=1):
|
|
# Everything in the folder reads as scratch, and nothing
|
|
# publishable has ever been committed here. That combination
|
|
# almost always means the tests WERE the job - publishing them is
|
|
# better than publishing an empty repository.
|
|
specs = []
|
|
staged = _would_stage(repo_path)
|
|
return specs, staged
|
|
|
|
|
|
def has_changes(repo_path: str) -> bool:
|
|
"""Is there anything worth publishing?
|
|
|
|
Deliberately not "is the working tree dirty". A session that left
|
|
nothing but scratch behind has nothing to commit, and saying
|
|
otherwise makes the end-of-session backstop try a commit that stages
|
|
nothing and fails.
|
|
"""
|
|
specs, staged = _staging_plan(repo_path)
|
|
if staged:
|
|
return True
|
|
# Nothing new, but a previous session may have published scratch that
|
|
# this one should stop publishing.
|
|
return bool(specs) and bool(_tracked_scratch(repo_path))
|
|
|
|
|
|
def commit(repo_path: str, message: str, author_note: str = "") -> str | None:
|
|
"""Commit everything in the tree. Returns the short sha, or None.
|
|
|
|
The message is passed with `-F -` on stdin rather than `-m`, because
|
|
it comes from the agent, which got it from a model, which read it off
|
|
a guest's output. There is no shell in the path either way, but a
|
|
message that starts with a dash should be a message, not an option.
|
|
"""
|
|
init(repo_path, os.path.basename(repo_path))
|
|
if not has_changes(repo_path):
|
|
return None
|
|
|
|
# SCRATCH IS NOT COMMITTED, SO IT CANNOT BE PUBLISHED.
|
|
#
|
|
# See SCRATCH_PATTERNS. The one exception is the project that would
|
|
# otherwise end up empty: if everything in the folder looks like a
|
|
# test and nothing publishable has ever been committed, then the
|
|
# tests WERE the job, and publishing nothing would be worse than
|
|
# publishing them. Checked against the committed tree rather than
|
|
# against this session alone - a session that only produced scratch
|
|
# for an existing project has genuinely produced nothing to publish.
|
|
specs, _ = _staging_plan(repo_path)
|
|
|
|
# Scratch an earlier session committed, before this filter existed or
|
|
# under a name it did not catch. --cached: it leaves the file on the
|
|
# array and only stops publishing it.
|
|
if specs:
|
|
stale = _tracked_scratch(repo_path)
|
|
if stale:
|
|
_git(repo_path, "rm", "-q", "--cached", "--", *stale)
|
|
|
|
add = _git(repo_path, "add", "-A", "--", ".", *specs)
|
|
if add.returncode != 0:
|
|
raise GitError(f"could not stage the changes: {add.stderr.strip()}")
|
|
|
|
body = message.strip()[:2000] or "Update from a VM session"
|
|
if author_note:
|
|
# Who asked for this, recorded where it cannot be edited by the
|
|
# next session. The project share is writable by every caller, so
|
|
# attribution has to live in the commit rather than in a file.
|
|
body += f"\n\nRequested-By: {author_note}"
|
|
|
|
result = subprocess.run(
|
|
["/usr/bin/git", "-C", repo_path, *_SAFE_FLAGS,
|
|
"commit", "--no-verify", "-q", "-F", "-"],
|
|
input=body, capture_output=True, text=True, timeout=120,
|
|
env={"PATH": "/usr/bin:/bin", "HOME": "/nonexistent", **_SAFE_ENV}, check=False,
|
|
)
|
|
if result.returncode != 0:
|
|
raise GitError(f"the commit failed: {(result.stderr or result.stdout).strip()}")
|
|
return _git(repo_path, "rev-parse", "--short", "HEAD").stdout.strip() or None
|
|
|
|
|
|
SUMMARY_CAP = 140
|
|
|
|
# "On the Ubuntu machine," / "On this Windows machine:" and the like.
|
|
_LEADING_MACHINE = re.compile(
|
|
r"^on (?:the|this|a)?\s*\w*\s*machine[,:]?\s*", re.IGNORECASE)
|
|
# The verb that asked for the thing, not a property of the thing. Only
|
|
# verbs that CREATE are stripped: a project whose job was to "test" or
|
|
# "check" something is accurately described by that word.
|
|
_LEADING_VERB = re.compile(
|
|
r"^(?:please\s+)?(?:write|create|build|make|implement|produce|add|develop)\s+"
|
|
r"(?:me\s+)?(?:a|an|the)?\s*", re.IGNORECASE)
|
|
|
|
|
|
def short_description(text: str, project_name: str) -> str:
|
|
"""One plain line saying what this project is, for the repository page.
|
|
|
|
The operator's words: it "just has to accurately describe what the app
|
|
does briefly - not sell it to me". So this is built from the job the
|
|
session was given rather than written by a model: a description that is
|
|
generated cannot be a pitch, and cannot describe work that was not
|
|
asked for. First sentence, one line, hard cap.
|
|
"""
|
|
text = " ".join((text or "").split())
|
|
# A job that opens by shouting at the agent - "IMPORTANT: put this in
|
|
# the project folder" - is a correction, not a description. Strip the
|
|
# label; what follows is still closer to the subject than the warning.
|
|
for label in ("goal:", "task:", "objective:", "important:", "note:", "warning:"):
|
|
if text.lower().startswith(label):
|
|
text = text[len(label):].lstrip()
|
|
# A job opens by addressing the machine and telling it to build
|
|
# something. A repository description is a label on the thing that was
|
|
# built, so both of those go: "On the Ubuntu machine, write a small
|
|
# Python script that ..." becomes "Small Python script that ...".
|
|
text = _LEADING_MACHINE.sub("", text, count=1)
|
|
text = _LEADING_VERB.sub("", text, count=1)
|
|
text = text[:1].upper() + text[1:] if text else text
|
|
for stop in (". ", "! ", "? "):
|
|
at = text.find(stop)
|
|
if at > 20:
|
|
text = text[:at]
|
|
break
|
|
text = text.rstrip(" .")
|
|
if len(text) > SUMMARY_CAP:
|
|
cut = text[:SUMMARY_CAP]
|
|
space = cut.rfind(" ")
|
|
text = (cut[:space] if space > SUMMARY_CAP // 2 else cut).rstrip(" ,;:-") + "..."
|
|
return text or f"Work from a VM session for project '{project_name}'."
|
|
|
|
|
|
def raw_file_url(web_url: str, path: str) -> str:
|
|
"""A link that downloads one file, rather than one that shows a repository.
|
|
|
|
The operator asked for a link to the script, and got a link to the repo
|
|
- which is a page to navigate, not a file to fetch. Gitea serves the
|
|
file itself at /raw/branch/<branch>/<path>.
|
|
"""
|
|
return f"{web_url}/raw/branch/main/{quote(path)}"
|
|
|
|
|
|
def tracked_files(repo_path: str, limit: int = 12) -> list[str]:
|
|
"""What is actually committed, so a link is never offered for a missing file.
|
|
|
|
`git ls-files` rather than a directory walk: the walk would also list
|
|
whatever the guest left lying around uncommitted, and a link to that is
|
|
a 404 with the operator's name on it.
|
|
"""
|
|
result = _git(repo_path, "ls-files", "-z")
|
|
if result.returncode != 0:
|
|
return []
|
|
names = [n for n in result.stdout.split("\0") if n and not n.startswith(".")]
|
|
names.sort(key=lambda n: (n.count("/"), n.lower()))
|
|
return names[:limit]
|
|
|
|
|
|
def ensure_remote_repo(project_name: str, description: str = "") -> str:
|
|
"""Create the Gitea repository if it is missing; return its web URL."""
|
|
if not configured():
|
|
raise GitError("no Gitea token is configured, so there is nowhere to push.")
|
|
api = f"{GITEA_INTERNAL}/api/v1"
|
|
web = f"{GITEA_PUBLIC}/{GITEA_ORG}/{project_name}"
|
|
|
|
status, body = _api("GET", f"{api}/repos/{quote(GITEA_ORG)}/{quote(project_name)}")
|
|
if status == 200:
|
|
# A repository created before there was anything to say about it
|
|
# carries the placeholder. Replace that once, but never overwrite a
|
|
# description somebody has since written by hand.
|
|
if description:
|
|
try:
|
|
current = json.loads(body).get("description") or ""
|
|
except ValueError:
|
|
current = ""
|
|
if not current or current.startswith("Created by the owuitools VM agent"):
|
|
_api("PATCH", f"{api}/repos/{quote(GITEA_ORG)}/{quote(project_name)}",
|
|
{"description": description})
|
|
return web
|
|
|
|
status, body = _api("POST", f"{api}/orgs/{quote(GITEA_ORG)}/repos", {
|
|
"name": project_name,
|
|
"description": description or
|
|
f"Created by the owuitools VM agent for project '{project_name}'.",
|
|
# Public INSIDE an org whose visibility is Limited: any signed-in
|
|
# Gitea user can read it, anonymous gets a 404. That is the
|
|
# arrangement asked for, and the same one the previous sub-agent
|
|
# stack used.
|
|
"private": False,
|
|
"auto_init": False,
|
|
"default_branch": "main",
|
|
})
|
|
# 409 is "it already exists", which is the outcome we wanted anyway.
|
|
if status not in (201, 409):
|
|
raise GitError(f"Gitea refused to create the repository ({status}): {body[:200]}")
|
|
return web
|
|
|
|
|
|
def _api(method: str, url: str, payload: dict | None = None) -> tuple[int, str]:
|
|
"""One small HTTP call, on the stdlib.
|
|
|
|
Deliberately not httpx: vmd is the privileged half of this system and
|
|
runs on the host's own interpreter, so every third-party import is a
|
|
package that has to be present, patched and trusted there. Two JSON
|
|
calls to one LAN address do not justify that.
|
|
"""
|
|
data = json.dumps(payload).encode() if payload is not None else None
|
|
request = urllib.request.Request(url, data=data, method=method)
|
|
request.add_header("Authorization", f"token {GITEA_TOKEN}")
|
|
request.add_header("Accept", "application/json")
|
|
if data is not None:
|
|
request.add_header("Content-Type", "application/json")
|
|
# No proxy handler: the systemd unit allows exactly one destination,
|
|
# and an inherited http_proxy would send this somewhere else entirely.
|
|
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
try:
|
|
with opener.open(request, timeout=20.0) as response:
|
|
return response.status, response.read().decode("utf-8", "replace")
|
|
except urllib.error.HTTPError as exc:
|
|
return exc.code, exc.read().decode("utf-8", "replace")
|
|
except urllib.error.URLError as exc:
|
|
raise GitError(f"could not reach Gitea at {GITEA_INTERNAL}: {exc.reason}") from exc
|
|
|
|
|
|
# --- releases ---------------------------------------------------------
|
|
#
|
|
# WHY A RELEASE AND NOT A LINK TO THE FILE.
|
|
#
|
|
# A link into the repository serves the file as `text/plain` with
|
|
# `Content-Disposition: inline` - measured - so a browser shows the source
|
|
# instead of saving it. Release assets are served as attachments, so they
|
|
# download. A release is also a record of one session: every run that kept
|
|
# anything leaves a numbered, dated entry with the files as they were at
|
|
# the end of it, which a branch does not give you.
|
|
#
|
|
# Nothing in here may take a session down. The work is committed and pushed
|
|
# by the time this runs; a release that fails to publish is a missing
|
|
# convenience, not lost work, and it is reported as one.
|
|
|
|
RELEASE_ASSET_MAX = 25 * 1024 * 1024 # per file
|
|
RELEASE_ASSETS_MAX = 25 # per release
|
|
# Was 10, which is how the toolkit came to be missing from its own
|
|
# release: ten scratch files sorted ahead of it and used the budget
|
|
# up. Scratch is no longer committed, so the cap now only has to be
|
|
# high enough that a real project never reaches it.
|
|
|
|
|
|
def _next_tag(api: str, project_name: str) -> str:
|
|
"""v1, v2, v3 ... one per session that kept something.
|
|
|
|
Counted from what is published rather than held anywhere, so it stays
|
|
right across a vmd restart. Only one session runs at a time, enforced
|
|
by systemd, so there is nobody to race with.
|
|
"""
|
|
status, body = _api(
|
|
"GET", f"{api}/repos/{quote(GITEA_ORG)}/{quote(project_name)}/releases?limit=50")
|
|
existing = set()
|
|
if status == 200:
|
|
try:
|
|
for entry in json.loads(body):
|
|
existing.add(str(entry.get("tag_name") or ""))
|
|
except ValueError:
|
|
pass
|
|
number = len(existing) + 1
|
|
while f"v{number}" in existing:
|
|
number += 1
|
|
return f"v{number}"
|
|
|
|
|
|
def publish_release(repo_path: str, project_name: str, web_url: str,
|
|
summary: str = "") -> str:
|
|
"""Publish this session's files as a downloadable release.
|
|
|
|
Returns the URL of the releases page, or "" if nothing was published.
|
|
"""
|
|
if not configured():
|
|
return ""
|
|
api = f"{GITEA_INTERNAL}/api/v1"
|
|
tag = _next_tag(api, project_name)
|
|
|
|
payload = {
|
|
"tag_name": tag,
|
|
"target_commitish": "main",
|
|
"name": tag,
|
|
# Plain and short, for the same reason the repository
|
|
# description is: this is a label, not a pitch.
|
|
"body": summary or "Files as they were at the end of this session.",
|
|
"draft": False,
|
|
"prerelease": False,
|
|
}
|
|
status, body = _api(
|
|
"POST", f"{api}/repos/{quote(GITEA_ORG)}/{quote(project_name)}/releases", payload)
|
|
# "repo is empty" AFTER A SUCCESSFUL PUSH IS A RACE, NOT A REFUSAL.
|
|
#
|
|
# Gitea clears a repository's is_empty flag when it processes the push
|
|
# it has just accepted, and the release API checks that flag. Measured
|
|
# on the FIRST session of a new project - the common case - the push
|
|
# succeeded, the commit was on the server, and the release came back
|
|
# 422 "repo is empty" a moment later. A repository that already had
|
|
# commits never showed it, which is why this survived two sessions.
|
|
for attempt in range(4):
|
|
if status not in (422,) or "empty" not in body.lower():
|
|
break
|
|
time.sleep(1.5 * (attempt + 1))
|
|
status, body = _api(
|
|
"POST", f"{api}/repos/{quote(GITEA_ORG)}/{quote(project_name)}/releases",
|
|
payload)
|
|
if status not in (201, 200):
|
|
raise GitError(f"Gitea refused to create the release ({status}): {body[:200]}")
|
|
try:
|
|
release_id = int(json.loads(body).get("id"))
|
|
except (ValueError, TypeError, AttributeError) as exc:
|
|
raise GitError(f"Gitea's release reply could not be read: {exc}") from exc
|
|
|
|
for name in tracked_files(repo_path, limit=RELEASE_ASSETS_MAX):
|
|
full = os.path.join(repo_path, name)
|
|
try:
|
|
if os.path.getsize(full) > RELEASE_ASSET_MAX:
|
|
continue
|
|
with open(full, "rb") as handle:
|
|
blob = handle.read()
|
|
except OSError:
|
|
continue
|
|
# The asset name is flat - a release has no directories - so a
|
|
# nested file keeps its path in the name rather than colliding
|
|
# with a same-named file from another folder.
|
|
_api_upload(
|
|
f"{api}/repos/{quote(GITEA_ORG)}/{quote(project_name)}"
|
|
f"/releases/{release_id}/assets?name={quote(name.replace('/', '_'), safe='')}",
|
|
name.replace("/", "_"), blob)
|
|
return f"{web_url}/releases"
|
|
|
|
|
|
def _api_upload(url: str, filename: str, blob: bytes) -> tuple[int, str]:
|
|
"""One multipart POST, on the stdlib, for the same reason as _api."""
|
|
boundary = "----owuivm" + os.urandom(16).hex()
|
|
safe = filename.replace('"', "").replace("\\", "")
|
|
body = (
|
|
f"--{boundary}\r\n"
|
|
f'Content-Disposition: form-data; name="attachment"; filename="{safe}"\r\n'
|
|
"Content-Type: application/octet-stream\r\n\r\n"
|
|
).encode() + blob + f"\r\n--{boundary}--\r\n".encode()
|
|
request = urllib.request.Request(url, data=body, method="POST")
|
|
request.add_header("Authorization", f"token {GITEA_TOKEN}")
|
|
request.add_header("Content-Type", f"multipart/form-data; boundary={boundary}")
|
|
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
try:
|
|
with opener.open(request, timeout=60.0) as response:
|
|
return response.status, response.read().decode("utf-8", "replace")
|
|
except urllib.error.HTTPError as exc:
|
|
return exc.code, exc.read().decode("utf-8", "replace")
|
|
except urllib.error.URLError as exc:
|
|
raise GitError(f"could not reach Gitea at {GITEA_INTERNAL}: {exc.reason}") from exc
|
|
|
|
|
|
def push(repo_path: str, project_name: str, description: str = "") -> str:
|
|
"""Push main to Gitea and return the web URL."""
|
|
web = ensure_remote_repo(project_name, description)
|
|
remote = f"{GITEA_INTERNAL}/{GITEA_ORG}/{project_name}.git"
|
|
|
|
# Set rather than add, every time: whatever the guest may have written
|
|
# into .git/config, the remote we push to is the one chosen here.
|
|
_git(repo_path, "remote", "remove", "origin")
|
|
result = _git(repo_path, "remote", "add", "origin", remote)
|
|
if result.returncode != 0:
|
|
raise GitError(f"could not set the push target: {result.stderr.strip()}")
|
|
|
|
# GIT_ASKPASS rather than a credential in the URL: a URL ends up in
|
|
# .git/config, in the reflog and in any error message git prints.
|
|
with tempfile.NamedTemporaryFile("w", suffix=".sh", delete=False) as script:
|
|
script.write("#!/bin/sh\ncase \"$1\" in Username*) echo \"$GIT_USER\" ;; *) echo \"$GIT_PASS\" ;; esac\n")
|
|
askpass = script.name
|
|
try:
|
|
os.chmod(askpass, 0o700)
|
|
result = _git(
|
|
repo_path, "push", "-u", "origin", "main",
|
|
env_extra={
|
|
"GIT_ASKPASS": askpass,
|
|
"GIT_USER": GITEA_BOT,
|
|
"GIT_PASS": GITEA_TOKEN,
|
|
},
|
|
timeout=300.0,
|
|
)
|
|
finally:
|
|
try:
|
|
os.unlink(askpass)
|
|
except OSError:
|
|
pass
|
|
|
|
if result.returncode != 0:
|
|
detail = (result.stderr or result.stdout).strip()
|
|
# The token would appear in a URL-bearing error; it is not in the
|
|
# URL, but scrub anyway rather than rely on that staying true.
|
|
if GITEA_TOKEN:
|
|
detail = detail.replace(GITEA_TOKEN, "<token>")
|
|
raise GitError(f"the push failed: {detail[:400]}")
|
|
return web
|