a70e3aef4b
Add scripts/setup-native.py and make ./install (and --native) build the server, the CLI and ntfy from source into ~/.local/share/pipelines, detect an existing host OpenCode and Pi instead of installing them, install the packaged authoring skill, migrate an existing Docker database and ntfy auth store, save the host client default and start both services. ./install --docker keeps the previous Docker server + ntfy installation unchanged. ntfy is compiled from its pinned Go module on every host because upstream publishes no macOS server binary (its darwin asset is built with -tags noserver); its cgo SQLite driver is why a C compiler is now required natively. The ntfy credential bootstrap and the client default writer are shared by both installers, installation.Load resolves symlinked prefixes, the Docker migration copies the SQLite write-ahead log so the newest definitions and runs are not lost, and the account subcommands read the generated configuration through NTFY_CONFIG_FILE. Verified on macOS: 115 Python tests, go build/vet/test, and scratch installs that migrate 9 pipelines and 31 runs, build ntfy v2.30.0 and start both services healthy on spare ports before stopping them.
330 lines
14 KiB
Python
330 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Bounded resolution of current harness and ntfy releases for installation.
|
|
|
|
Network I/O is isolated behind an injectable ``fetch`` callable so unit tests
|
|
never touch the network. The resolver:
|
|
|
|
* discovers supported V2 OpenCode releases from one bounded GitHub request to
|
|
``git/matching-refs/tags/v2``, which the official REST docs
|
|
(https://docs.github.com/en/rest/git/refs#list-matching-references) document
|
|
as a complete array with no pagination parameters. There is no page/per_page
|
|
query, no ``Link`` cursor and no tags/list cross-check: an explicit
|
|
``MAX_REFS`` count and ``MAX_BODY`` byte cap bound the single response;
|
|
* treats the official V2 tag metadata (not the registry's insertion order) as
|
|
the ordering, ignoring prerelease/build-suffixed tags and rejecting malformed
|
|
ones; it never uses the moving ``releases/latest`` anchor;
|
|
* resolves the selected tag's immutable ``sha256`` digest from GHCR and fails
|
|
closed when it is missing or invalid, never falling back to an older version;
|
|
* resolves the exact Pi package version and validates its Node engine
|
|
constraint fail-closed;
|
|
* resolves the exact ntfy release tag that the native installer builds its
|
|
server from, because upstream publishes only a client-only binary for macOS
|
|
and the server is therefore compiled from the pinned Go module on every host;
|
|
* validates untrusted metadata, versions, digests and fixture values;
|
|
* refuses any host outside the explicit allowlist (the container registry, the
|
|
GitHub API and the npm registry) and refuses any redirect that leaves it.
|
|
|
|
The selected versions/digests are data, not evergreen hardcoded answers.
|
|
"""
|
|
import json
|
|
import re
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
|
|
class ReleaseError(ValueError):
|
|
"""Resolution failed; callers must keep the last verified pins."""
|
|
|
|
|
|
GHCR = "https://ghcr.io"
|
|
GITHUB = "https://api.github.com"
|
|
NPM = "https://registry.npmjs.org"
|
|
OPENCODE_REPOSITORY = "anomalyco/opencode"
|
|
OPENCODE_IMAGE = "ghcr.io/anomalyco/opencode"
|
|
PI_PACKAGE = "@earendil-works/pi-coding-agent"
|
|
NTFY_REPOSITORY = "binwiederhier/ntfy"
|
|
# The ntfy Go module the native installer builds a server from. Upstream only
|
|
# publishes a client-only binary for macOS, so both platforms compile the server
|
|
# from this module at the resolved tag (Go verifies the module via its checksum
|
|
# database).
|
|
NTFY_MODULE = "heckel.io/ntfy/v2"
|
|
MAX_REFS = 1000 # Explicit ceiling on the documented complete matching-refs array.
|
|
MAX_BODY = 1 << 20 # Generous ceiling for tag lists and manifest metadata.
|
|
SUPPORTED_MAJOR = 2
|
|
HTTP_TIMEOUT = 5.0
|
|
# Registry, GitHub API and npm metadata hosts only. The native installer builds
|
|
# ntfy from its Go module, so no release-asset download hosts are needed.
|
|
ALLOWED_HOSTS = frozenset({"ghcr.io", "api.github.com", "registry.npmjs.org"})
|
|
GITHUB_HEADERS = {"Accept": "application/vnd.github+json", "User-Agent": "pipelines-setup"}
|
|
MANIFEST_ACCEPT = ", ".join([
|
|
"application/vnd.oci.image.index.v1+json",
|
|
"application/vnd.docker.distribution.manifest.list.v2+json",
|
|
"application/vnd.oci.image.manifest.v1+json",
|
|
"application/vnd.docker.distribution.manifest.v2+json",
|
|
])
|
|
VERSION = re.compile(r"v?(\d+)\.(\d+)\.(\d+)")
|
|
# A version-shaped tag carrying a prerelease/build suffix is ignored, not fatal.
|
|
VERSION_SUFFIX = re.compile(r"v?(\d+)\.(\d+)\.(\d+)[-+].*")
|
|
DIGEST = re.compile(r"sha256:[0-9a-f]{64}")
|
|
TAG = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,127}")
|
|
ENGINE = re.compile(r"(>=|<=|>|<|=)?\s*v?(\d+\.\d+\.\d+)")
|
|
|
|
|
|
def registry_tag(tag):
|
|
"""Normalize an official ``vX.Y.Z`` git tag to its registry tag name."""
|
|
return tag[1:] if tag.startswith("v") else tag
|
|
|
|
|
|
def parse_version(text):
|
|
if not isinstance(text, str):
|
|
return None
|
|
match = VERSION.fullmatch(text.strip())
|
|
if not match:
|
|
return None
|
|
return tuple(int(part) for part in match.groups())
|
|
|
|
|
|
class _AllowlistRedirect(urllib.request.HTTPRedirectHandler):
|
|
"""Follow only redirects that stay on an explicitly allowed HTTPS host."""
|
|
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
if not is_allowlisted(newurl):
|
|
raise ReleaseError(f"refusing unexpected redirect from {req.full_url!r} to {newurl!r}")
|
|
return super().redirect_request(req, fp, code, msg, headers, newurl)
|
|
|
|
|
|
_OPENER = urllib.request.build_opener(_AllowlistRedirect)
|
|
|
|
|
|
def is_allowlisted(url):
|
|
"""True only for HTTPS URLs on an explicitly allowed host (shared guard)."""
|
|
parsed = urllib.parse.urlsplit(url)
|
|
return parsed.scheme == "https" and parsed.hostname in ALLOWED_HOSTS
|
|
|
|
|
|
def _default_fetch(url, headers=None):
|
|
if not is_allowlisted(url):
|
|
raise ReleaseError(f"refusing non-registry host {url!r}")
|
|
request = urllib.request.Request(url, headers=headers or {})
|
|
try:
|
|
with _OPENER.open(request, timeout=HTTP_TIMEOUT) as response:
|
|
lowered = {key.lower(): value for key, value in response.headers.items()}
|
|
return response.status, lowered, response.read(MAX_BODY + 1)
|
|
except urllib.error.HTTPError as error:
|
|
lowered = {key.lower(): value for key, value in error.headers.items()}
|
|
return error.code, lowered, error.read(MAX_BODY + 1)
|
|
|
|
|
|
def _response_json(fetch, url, headers=None):
|
|
"""Fetch a bounded JSON body and return it with its lower-cased headers."""
|
|
status, response_headers, body = fetch(url, headers or {})
|
|
if status != 200:
|
|
raise ReleaseError(f"HTTP {status} from {url}")
|
|
if not isinstance(body, (bytes, bytearray)):
|
|
raise ReleaseError(f"non-bytes response from {url}")
|
|
if len(body) > MAX_BODY:
|
|
raise ReleaseError(f"response body exceeds {MAX_BODY} bytes from {url}")
|
|
try:
|
|
data = json.loads(body)
|
|
except (TypeError, ValueError):
|
|
raise ReleaseError(f"invalid JSON from {url}") from None
|
|
lowered = {key.lower(): value for key, value in (response_headers or {}).items()}
|
|
return data, lowered
|
|
|
|
|
|
def _json(fetch, url, headers=None):
|
|
data, _ = _response_json(fetch, url, headers)
|
|
return data
|
|
|
|
|
|
def _github_matching_refs(fetch, repository, tag_prefix, normalize=lambda tag: tag):
|
|
"""One bounded request for the documented complete matching-refs array.
|
|
|
|
GitHub's "List matching references" endpoint documents a complete array and
|
|
no pagination parameters, so a single ``MAX_BODY``/``MAX_REFS``-bounded GET
|
|
is sufficient. Exact stable versions become candidates; version-shaped tags
|
|
with a prerelease/build suffix are ignored; other malformed tags fail closed.
|
|
``normalize`` maps a raw tag to the candidate key (OpenCode strips the ``v``).
|
|
"""
|
|
url = f"{GITHUB}/repos/{repository}/git/matching-refs/tags/{tag_prefix}"
|
|
data, _ = _response_json(fetch, url, GITHUB_HEADERS)
|
|
if not isinstance(data, list):
|
|
raise ReleaseError("GitHub matching-refs payload is not a list")
|
|
if len(data) > MAX_REFS:
|
|
raise ReleaseError(
|
|
f"GitHub matching-refs returned {len(data)} refs, exceeding the {MAX_REFS}-ref cap; "
|
|
"refusing to select a release without proof the tag list is complete")
|
|
candidates = {}
|
|
for entry in data:
|
|
if not isinstance(entry, dict):
|
|
raise ReleaseError("GitHub matching-refs entry is not an object")
|
|
ref = entry.get("ref")
|
|
if not isinstance(ref, str) or not ref.startswith("refs/tags/"):
|
|
raise ReleaseError(f"GitHub matching-refs entry has an unexpected ref {ref!r}")
|
|
tag = normalize(ref[len("refs/tags/"):])
|
|
version = parse_version(tag)
|
|
if version is None:
|
|
if VERSION_SUFFIX.fullmatch(tag):
|
|
# Prerelease/build tags never win; skip them instead of failing.
|
|
continue
|
|
raise ReleaseError(f"GitHub matching-refs tag {tag!r} is not an exact version")
|
|
candidates[tag] = version
|
|
return candidates
|
|
|
|
|
|
def _github_v2_tags(fetch):
|
|
return _github_matching_refs(fetch, OPENCODE_REPOSITORY, f"v{SUPPORTED_MAJOR}", registry_tag)
|
|
|
|
|
|
def _ghcr_token(fetch):
|
|
data = _json(fetch, f"{GHCR}/token?scope=repository:{OPENCODE_REPOSITORY}:pull")
|
|
if not isinstance(data, dict):
|
|
raise ReleaseError("registry token payload is not an object")
|
|
token = data.get("token")
|
|
if not isinstance(token, str) or not token:
|
|
raise ReleaseError("registry did not issue a pull token")
|
|
return token
|
|
|
|
|
|
def _manifest_digest(fetch, token, tag):
|
|
if not isinstance(tag, str) or not TAG.fullmatch(tag):
|
|
raise ReleaseError(f"refusing to resolve an invalid registry tag {tag!r}")
|
|
url = f"{GHCR}/v2/{OPENCODE_REPOSITORY}/manifests/{urllib.parse.quote(tag, safe='')}"
|
|
status, headers, _ = fetch(url, {"Authorization": "Bearer " + token, "Accept": MANIFEST_ACCEPT})
|
|
if status != 200:
|
|
raise ReleaseError(f"HTTP {status} resolving digest for {tag}")
|
|
digest = headers.get("docker-content-digest", "")
|
|
if not DIGEST.fullmatch(digest):
|
|
raise ReleaseError(f"registry returned no usable content digest for {tag}")
|
|
return digest
|
|
|
|
|
|
def resolve_opencode(fetch=_default_fetch):
|
|
candidates = _github_v2_tags(fetch)
|
|
supported = {tag: version for tag, version in candidates.items() if version[0] == SUPPORTED_MAJOR}
|
|
if not supported:
|
|
raise ReleaseError(f"no supported OpenCode V{SUPPORTED_MAJOR} release found in the official tag metadata")
|
|
tag = max(supported, key=lambda name: supported[name])
|
|
token = _ghcr_token(fetch)
|
|
# The highest supported stable tag is authoritative; the manifest lookup
|
|
# below proves publication and fails closed if its digest is missing, never
|
|
# falling back to an older version.
|
|
return {
|
|
"image": OPENCODE_IMAGE,
|
|
"version": tag,
|
|
"digest": _manifest_digest(fetch, token, tag),
|
|
"anchor": f"github:repos/{OPENCODE_REPOSITORY}/git/matching-refs/tags/v{SUPPORTED_MAJOR}",
|
|
"candidates": sorted(supported),
|
|
}
|
|
|
|
|
|
def _validate_engine(engine):
|
|
text = (engine or "").strip()
|
|
if not text:
|
|
return
|
|
if not ENGINE.fullmatch(text):
|
|
raise ReleaseError(f"unsupported Pi Node engine constraint {engine!r}; refusing to accept an unchecked runtime")
|
|
|
|
|
|
def node_satisfies(engine, node_version):
|
|
"""Fail-closed check for the simple constraints Pi publishes."""
|
|
node = parse_version(node_version)
|
|
if node is None:
|
|
raise ReleaseError(f"could not parse Node version {node_version!r}")
|
|
text = (engine or "").strip()
|
|
if not text:
|
|
return True
|
|
match = ENGINE.fullmatch(text)
|
|
if not match:
|
|
raise ReleaseError(f"unsupported Pi Node engine constraint {engine!r}; refusing to accept an unchecked runtime")
|
|
operator = match.group(1) or "="
|
|
bound = parse_version(match.group(2))
|
|
if operator == ">=":
|
|
return node >= bound
|
|
if operator == "<=":
|
|
return node <= bound
|
|
if operator == ">":
|
|
return node > bound
|
|
if operator == "<":
|
|
return node < bound
|
|
return node == bound
|
|
|
|
|
|
def resolve_pi(fetch=_default_fetch, node_version=None):
|
|
url = f"{NPM}/@earendil-works%2fpi-coding-agent/latest"
|
|
data = _json(fetch, url)
|
|
if not isinstance(data, dict):
|
|
raise ReleaseError("npm Pi payload is not an object")
|
|
version = data.get("version")
|
|
if parse_version(version) is None:
|
|
raise ReleaseError(f"npm returned a non-version Pi release {version!r}")
|
|
engines = data.get("engines") or {}
|
|
if not isinstance(engines, dict):
|
|
raise ReleaseError("npm Pi engines metadata is malformed")
|
|
engine = engines.get("node", "")
|
|
if engine is not None and not isinstance(engine, str):
|
|
raise ReleaseError("npm Pi node engine constraint is malformed")
|
|
engine = engine or ""
|
|
_validate_engine(engine)
|
|
if node_version is not None and not node_satisfies(engine, node_version):
|
|
raise ReleaseError(f"Pi {version} requires Node {engine} but the image provides {node_version}")
|
|
return {"package": PI_PACKAGE, "version": version, "engine": engine}
|
|
|
|
|
|
def _github_ntfy_tags(fetch):
|
|
"""One bounded request for the complete ntfy vX.Y.Z matching-refs array."""
|
|
return _github_matching_refs(fetch, NTFY_REPOSITORY, "v")
|
|
|
|
|
|
def resolve_ntfy(fetch=_default_fetch):
|
|
"""Resolve the exact ntfy release the native installer builds from source.
|
|
|
|
The selected version comes from the complete official matching-refs tag list
|
|
(never the moving ``latest`` release and never an older fallback); a missing
|
|
or malformed tag list fails closed. The native installer then compiles the
|
|
server from the pinned ``heckel.io/ntfy/v2`` module, so no release assets or
|
|
their checksums are downloaded here.
|
|
"""
|
|
candidates = _github_ntfy_tags(fetch)
|
|
if not candidates:
|
|
raise ReleaseError("no stable ntfy release found in the official tag metadata")
|
|
tag = max(candidates, key=lambda name: candidates[name])
|
|
version = registry_tag(tag)
|
|
if parse_version(version) is None:
|
|
raise ReleaseError(f"resolved ntfy tag is not an exact version: {tag!r}")
|
|
return {"repository": NTFY_REPOSITORY, "module": NTFY_MODULE, "version": version, "tag": tag}
|
|
|
|
|
|
def resolve(fetch=_default_fetch, node_version=None):
|
|
return {"opencode": resolve_opencode(fetch), "pi": resolve_pi(fetch, node_version)}
|
|
|
|
|
|
def load_fixture(path):
|
|
"""Test/offline seam: read an explicit resolution instead of the network."""
|
|
with open(path, encoding="utf-8") as handle:
|
|
data = json.load(handle)
|
|
if not isinstance(data, dict):
|
|
raise ReleaseError("release fixture must be an object")
|
|
opencode, pi = data.get("opencode"), data.get("pi")
|
|
if not isinstance(opencode, dict) or not isinstance(pi, dict):
|
|
raise ReleaseError("release fixture must contain opencode and pi objects")
|
|
if parse_version(opencode.get("version")) is None:
|
|
raise ReleaseError("release fixture opencode version is not an exact version")
|
|
if not DIGEST.fullmatch(str(opencode.get("digest", ""))):
|
|
raise ReleaseError("release fixture opencode digest is not a sha256 digest")
|
|
if opencode.get("image", OPENCODE_IMAGE) != OPENCODE_IMAGE:
|
|
raise ReleaseError("release fixture opencode image is not the supported registry image")
|
|
if parse_version(pi.get("version")) is None:
|
|
raise ReleaseError("release fixture pi version is not an exact version")
|
|
_validate_engine(pi.get("engine", ""))
|
|
return data
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import sys
|
|
try:
|
|
print(json.dumps(resolve(), indent=2))
|
|
except ReleaseError as error:
|
|
sys.exit(f"Release resolution failed: {error}")
|