Install Pipelines natively by default on macOS and Linux

Add scripts/setup-native.py and make ./install (and --native) build the server, the CLI
and ntfy from source into ~/.local/share/pipelines, detect an existing host OpenCode
and Pi instead of installing them, install the packaged authoring skill, migrate an
existing Docker database and ntfy auth store, save the host client default and start
both services. ./install --docker keeps the previous Docker server + ntfy
installation unchanged.

ntfy is compiled from its pinned Go module on every host because upstream publishes
no macOS server binary (its darwin asset is built with -tags noserver); its cgo
SQLite driver is why a C compiler is now required natively. The ntfy credential
bootstrap and the client default writer are shared by both installers,
installation.Load resolves symlinked prefixes, the Docker migration copies the
SQLite write-ahead log so the newest definitions and runs are not lost, and the
account subcommands read the generated configuration through NTFY_CONFIG_FILE.

Verified on macOS: 115 Python tests, go build/vet/test, and scratch installs that
migrate 9 pipelines and 31 runs, build ntfy v2.30.0 and start both services healthy
on spare ports before stopping them.
This commit is contained in:
2026-10-13 17:20:14 +01:00
parent 9b305df725
commit a70e3aef4b
21 changed files with 1474 additions and 153 deletions
+62 -13
View File
@@ -1,10 +1,10 @@
#!/usr/bin/env python3
"""Bounded resolution of current harness releases for the Docker image.
"""Bounded resolution of current harness and ntfy releases for installation.
Network I/O is isolated behind an injectable ``fetch`` callable so unit tests
never touch the network. The resolver:
* discovers supported V2 releases from one bounded GitHub request to
* discovers supported V2 OpenCode releases from one bounded GitHub request to
``git/matching-refs/tags/v2``, which the official REST docs
(https://docs.github.com/en/rest/git/refs#list-matching-references) document
as a complete array with no pagination parameters. There is no page/per_page
@@ -15,9 +15,14 @@ never touch the network. The resolver:
ones; it never uses the moving ``releases/latest`` anchor;
* resolves the selected tag's immutable ``sha256`` digest from GHCR and fails
closed when it is missing or invalid, never falling back to an older version;
* resolves the exact Pi package version and validates its Node engine
constraint fail-closed;
* resolves the exact ntfy release tag that the native installer builds its
server from, because upstream publishes only a client-only binary for macOS
and the server is therefore compiled from the pinned Go module on every host;
* validates untrusted metadata, versions, digests and fixture values;
* refuses non-registry hosts and unexpected redirects;
* validates the Pi Node engine constraint fail-closed.
* refuses any host outside the explicit allowlist (the container registry, the
GitHub API and the npm registry) and refuses any redirect that leaves it.
The selected versions/digests are data, not evergreen hardcoded answers.
"""
@@ -38,10 +43,18 @@ NPM = "https://registry.npmjs.org"
OPENCODE_REPOSITORY = "anomalyco/opencode"
OPENCODE_IMAGE = "ghcr.io/anomalyco/opencode"
PI_PACKAGE = "@earendil-works/pi-coding-agent"
NTFY_REPOSITORY = "binwiederhier/ntfy"
# The ntfy Go module the native installer builds a server from. Upstream only
# publishes a client-only binary for macOS, so both platforms compile the server
# from this module at the resolved tag (Go verifies the module via its checksum
# database).
NTFY_MODULE = "heckel.io/ntfy/v2"
MAX_REFS = 1000 # Explicit ceiling on the documented complete matching-refs array.
MAX_BODY = 1 << 20 # Generous ceiling for tag lists and manifest metadata.
SUPPORTED_MAJOR = 2
HTTP_TIMEOUT = 5.0
# Registry, GitHub API and npm metadata hosts only. The native installer builds
# ntfy from its Go module, so no release-asset download hosts are needed.
ALLOWED_HOSTS = frozenset({"ghcr.io", "api.github.com", "registry.npmjs.org"})
GITHUB_HEADERS = {"Accept": "application/vnd.github+json", "User-Agent": "pipelines-setup"}
MANIFEST_ACCEPT = ", ".join([
@@ -72,19 +85,26 @@ def parse_version(text):
return tuple(int(part) for part in match.groups())
class _NoRedirect(urllib.request.HTTPRedirectHandler):
"""Registry metadata lookups must never be silently redirected elsewhere."""
class _AllowlistRedirect(urllib.request.HTTPRedirectHandler):
"""Follow only redirects that stay on an explicitly allowed HTTPS host."""
def redirect_request(self, req, fp, code, msg, headers, newurl):
raise ReleaseError(f"refusing unexpected redirect from {req.full_url!r} to {newurl!r}")
if not is_allowlisted(newurl):
raise ReleaseError(f"refusing unexpected redirect from {req.full_url!r} to {newurl!r}")
return super().redirect_request(req, fp, code, msg, headers, newurl)
_OPENER = urllib.request.build_opener(_NoRedirect)
_OPENER = urllib.request.build_opener(_AllowlistRedirect)
def is_allowlisted(url):
"""True only for HTTPS URLs on an explicitly allowed host (shared guard)."""
parsed = urllib.parse.urlsplit(url)
return parsed.scheme == "https" and parsed.hostname in ALLOWED_HOSTS
def _default_fetch(url, headers=None):
parsed = urllib.parse.urlsplit(url)
if parsed.scheme != "https" or parsed.hostname not in ALLOWED_HOSTS:
if not is_allowlisted(url):
raise ReleaseError(f"refusing non-registry host {url!r}")
request = urllib.request.Request(url, headers=headers or {})
try:
@@ -118,15 +138,16 @@ def _json(fetch, url, headers=None):
return data
def _github_v2_tags(fetch):
def _github_matching_refs(fetch, repository, tag_prefix, normalize=lambda tag: tag):
"""One bounded request for the documented complete matching-refs array.
GitHub's "List matching references" endpoint documents a complete array and
no pagination parameters, so a single ``MAX_BODY``/``MAX_REFS``-bounded GET
is sufficient. Exact stable versions become candidates; version-shaped tags
with a prerelease/build suffix are ignored; other malformed tags fail closed.
``normalize`` maps a raw tag to the candidate key (OpenCode strips the ``v``).
"""
url = f"{GITHUB}/repos/{OPENCODE_REPOSITORY}/git/matching-refs/tags/v{SUPPORTED_MAJOR}"
url = f"{GITHUB}/repos/{repository}/git/matching-refs/tags/{tag_prefix}"
data, _ = _response_json(fetch, url, GITHUB_HEADERS)
if not isinstance(data, list):
raise ReleaseError("GitHub matching-refs payload is not a list")
@@ -141,7 +162,7 @@ def _github_v2_tags(fetch):
ref = entry.get("ref")
if not isinstance(ref, str) or not ref.startswith("refs/tags/"):
raise ReleaseError(f"GitHub matching-refs entry has an unexpected ref {ref!r}")
tag = registry_tag(ref[len("refs/tags/"):])
tag = normalize(ref[len("refs/tags/"):])
version = parse_version(tag)
if version is None:
if VERSION_SUFFIX.fullmatch(tag):
@@ -152,6 +173,10 @@ def _github_v2_tags(fetch):
return candidates
def _github_v2_tags(fetch):
return _github_matching_refs(fetch, OPENCODE_REPOSITORY, f"v{SUPPORTED_MAJOR}", registry_tag)
def _ghcr_token(fetch):
data = _json(fetch, f"{GHCR}/token?scope=repository:{OPENCODE_REPOSITORY}:pull")
if not isinstance(data, dict):
@@ -247,6 +272,30 @@ def resolve_pi(fetch=_default_fetch, node_version=None):
return {"package": PI_PACKAGE, "version": version, "engine": engine}
def _github_ntfy_tags(fetch):
"""One bounded request for the complete ntfy vX.Y.Z matching-refs array."""
return _github_matching_refs(fetch, NTFY_REPOSITORY, "v")
def resolve_ntfy(fetch=_default_fetch):
"""Resolve the exact ntfy release the native installer builds from source.
The selected version comes from the complete official matching-refs tag list
(never the moving ``latest`` release and never an older fallback); a missing
or malformed tag list fails closed. The native installer then compiles the
server from the pinned ``heckel.io/ntfy/v2`` module, so no release assets or
their checksums are downloaded here.
"""
candidates = _github_ntfy_tags(fetch)
if not candidates:
raise ReleaseError("no stable ntfy release found in the official tag metadata")
tag = max(candidates, key=lambda name: candidates[name])
version = registry_tag(tag)
if parse_version(version) is None:
raise ReleaseError(f"resolved ntfy tag is not an exact version: {tag!r}")
return {"repository": NTFY_REPOSITORY, "module": NTFY_MODULE, "version": version, "tag": tag}
def resolve(fetch=_default_fetch, node_version=None):
return {"opencode": resolve_opencode(fetch), "pi": resolve_pi(fetch, node_version)}