Merge branch 'worktree-agent-a5eba3669bc0e09ad' into worktree-agent-ac54c8919e940163a

This commit is contained in:
2026-11-10 01:02:08 +00:00
17 changed files with 1010 additions and 26 deletions
+13 -6
View File
@@ -27,6 +27,9 @@ from urllib.parse import urlsplit
REPO = Path(__file__).resolve().parents[1]
APP = REPO / "app"
DEFAULT_PREFIX = Path.home() / ".local/share/pipelines"
_signing_spec = importlib.util.spec_from_file_location("pipelines_update_signing", Path(__file__).with_name("update_signing.py"))
update_signing = importlib.util.module_from_spec(_signing_spec)
_signing_spec.loader.exec_module(update_signing)
def default_cache():
return Path(os.environ.get("PIPELINES_APP_CACHE") or Path.home() / ".cache/pipelines/app-builds")
@@ -346,7 +349,10 @@ def build_apk(prefix, server_url=None, notes="", runner=subprocess.run, log=prin
server = server_url or phone_server_url(manifest)
# The build number is the source key's build date, so the same sources
# reuse one APK and every rebuild is newer than the last.
arguments = (f"--dart-define=PIPELINES_SERVER_URL={server}",)
# The feed is signed and the app embeds the keys it trusts (see
# scripts/update_signing.py), so a server cannot hand phones another APK.
signer = update_signing.Signer()
arguments = (f"--dart-define=PIPELINES_SERVER_URL={server}", *update_signing.dart_defines())
host_flutter, sdk = native_flutter(flutter), android_sdk_dir()
use_native = native if native is not None else bool(host_flutter and sdk)
if use_native:
@@ -365,9 +371,9 @@ def build_apk(prefix, server_url=None, notes="", runner=subprocess.run, log=prin
target.mkdir(parents=True, exist_ok=True)
shutil.copyfile(output / "pipelines.apk", target / "pipelines.apk.next")
os.replace(target / "pipelines.apk.next", target / "pipelines.apk")
(target / "version.json").write_text(json.dumps({
update_signing.publish_feed(target, {
"versionCode": code, "versionName": f"0.1.{code}", "apk": "pipelines.apk", "notes": notes,
"signingCertificateSha256": fingerprint, "sha256": apk_sha256}) + "\n")
"signingCertificateSha256": fingerprint, "sha256": apk_sha256}, signer)
log(f"APK ready: {target / 'pipelines.apk'} (build {code}, server {server}).")
log("Phones get it from Settings → App updates → Check for updates, or download "
f"{server.replace('wss://', 'https://').replace('ws://', 'http://').removesuffix('/connect')}/updates/pipelines.apk")
@@ -417,13 +423,14 @@ def main(argv=None):
bundle, archive = desktop.build_desktop(args.kind, app=APP, cache=default_cache(), runner=subprocess.run,
source_digest=digest, docker_build=build, docker_ready=docker_available,
server_url=args.server_url, flutter=args.flutter, output=args.output,
prune_cache=prune)
prune_cache=prune, update_keys=update_signing.dart_defines())
if args.publish:
manifest = load_manifest(args.prefix)
updates = (manifest.get("web") or {}).get("updates")
if not updates:
raise ValueError("This installation has no app update directory; rerun ./install")
desktop.publish_desktop(args.kind, bundle, archive, Path(updates), notes=args.notes)
desktop.publish_desktop(args.kind, bundle, archive, Path(updates), notes=args.notes,
signer=update_signing.Signer())
return
if args.kind == "apk":
build_apk(args.prefix, args.server_url, args.notes, flutter=args.flutter, native=args.native)
@@ -435,7 +442,7 @@ def main(argv=None):
web = manifest.get("web") or {}
if not web.get("root"):
raise ValueError("This installation does not serve the app; rerun ./install")
install_tree(build("web"), Path(web["root"]))
install_tree(build("web", arguments=update_signing.dart_defines()), Path(web["root"]))
print(f"Web app installed at {web['root']}; reload the page.")
print_web_urls(manifest)
+10 -5
View File
@@ -120,8 +120,8 @@ def _macos_build(*, app, cache, source_digest, runner, arguments, flutter, log):
def build_desktop(kind, *, app, cache, runner, source_digest, docker_build,
docker_ready, server_url=None, flutter=None, output=None, log=print,
prune_cache=None):
arguments = _arguments(server_url)
prune_cache=None, update_keys=()):
arguments = _arguments(server_url) + tuple(update_keys)
if kind == "macos":
entry, bundle, archive = _macos_build(app=app, cache=cache, runner=runner,
source_digest=source_digest, arguments=arguments, flutter=flutter, log=log)
@@ -154,8 +154,10 @@ def build_desktop(kind, *, app, cache, runner, source_digest, docker_build,
return bundle, archive
def publish_desktop(kind, bundle, archive, updates, *, notes="", log=print):
"""Publish archive before its feed; versioned names preserve in-flight reads."""
def publish_desktop(kind, bundle, archive, updates, *, signer, notes="", log=print):
"""Publish archive before its feed; versioned names preserve in-flight reads.
[signer] returns the detached `version.json.sig` for the feed bytes."""
entry = bundle.parent if kind == "macos" else bundle
complete = json.loads((entry / ".complete").read_text())
code = int(complete["built"]) // 60
@@ -176,8 +178,11 @@ def publish_desktop(kind, bundle, archive, updates, *, notes="", log=print):
previous = json.loads(feed.read_text())
if previous.get("versionCode", 0) > code:
raise ValueError("Refusing to publish an older build over a newer update")
data = (json.dumps(metadata) + "\n").encode()
# The signature lands before the feed it covers, after the archive.
for destination, content in ((target / filename, archive.read_bytes()),
(feed, (json.dumps(metadata) + "\n").encode())):
(target / "version.json.sig", signer(data)),
(feed, data)):
descriptor, temporary = tempfile.mkstemp(prefix=destination.name + ".", dir=target)
try:
with os.fdopen(descriptor, "wb") as output:
+2 -1
View File
@@ -403,7 +403,8 @@ class Setup:
raise ValueError("Docker is not running, so the web app cannot be built. The server is never "
"installed without it; start Docker and rerun ./install.")
try:
built = app_build.build("web", runner=self.runner)
built = app_build.build("web", runner=self.runner,
arguments=app_build.update_signing.dart_defines(self.environ))
except RuntimeError as error:
raise ValueError(f"The web app build failed ({error}); nothing was installed") from None
if not (Path(built) / "index.html").is_file():
+5
View File
@@ -83,6 +83,11 @@ class AppBuildTests(unittest.TestCase):
feed = json.loads((updates / "version.json").read_text())
self.assertEqual(feed["signingCertificateSha256"], "ab" * 32)
self.assertEqual(feed["sha256"], app_build.hashlib.sha256((updates / "pipelines.apk").read_bytes()).hexdigest())
# Signed with the installation key generated (0600) in the scratch home.
key = Path(self.home.name) / ".config/pipelines/update-signing-key.pem"
self.assertEqual(key.stat().st_mode & 0o777, 0o600)
signature = json.loads((updates / "version.json.sig").read_text())
self.assertEqual(signature["publicKey"], app_build.update_signing.public_key(key))
(output / "signing-certificate-sha256.txt").write_text("invalid")
with patch.object(app_build, "load_manifest", return_value=manifest), patch.object(app_build, "build", return_value=output):
with self.assertRaisesRegex(ValueError, "exactly one"):
+15 -3
View File
@@ -160,24 +160,36 @@ class DesktopBuildTests(unittest.TestCase):
def test_linux_publication_has_matching_hash_and_rejects_downgrade(self):
bundle, archive = self.build("linux")
updates = self.root / "updates"
metadata = desktop.publish_desktop("linux", bundle, archive, updates, log=lambda *_: None)
signed = []
signer = lambda data: signed.append(data) or b"signature document\n"
metadata = desktop.publish_desktop("linux", bundle, archive, updates, signer=signer, log=lambda *_: None)
target = updates / "linux-x64"
self.assertEqual(json.loads((target / "version.json").read_text()), metadata)
# The detached signature covers the exact published feed bytes.
self.assertEqual(signed, [(target / "version.json").read_bytes()])
self.assertEqual((target / "version.json.sig").read_bytes(), b"signature document\n")
self.assertEqual(metadata["architectures"], ["x64"])
self.assertEqual(metadata["sha256"], desktop.hashlib.sha256((target / metadata["file"]).read_bytes()).hexdigest())
metadata["versionCode"] += 1
(target / "version.json").write_text(json.dumps(metadata))
with self.assertRaisesRegex(ValueError, "older build"):
desktop.publish_desktop("linux", bundle, archive, updates)
desktop.publish_desktop("linux", bundle, archive, updates, signer=signer)
@patch.object(desktop.platform, "system", return_value="Darwin")
def test_macos_feed_uses_actual_architectures_and_native_build_stamp(self, _):
bundle, archive = self.build("macos", flutter="/fixture/flutter")
metadata = desktop.publish_desktop("macos", bundle, archive, self.root / "updates", log=lambda *_: None)
metadata = desktop.publish_desktop("macos", bundle, archive, self.root / "updates",
signer=lambda data: b"{}\n", log=lambda *_: None)
self.assertTrue((self.root / "updates/macos/version.json.sig").is_file())
self.assertEqual(metadata["architectures"], ["arm64", "x64"])
command = next(c for c in self.runner.calls if c[1:3] == ["build", "macos"])
self.assertIn(f"--dart-define=PIPELINES_NATIVE_BUILD_CODE={metadata['versionCode']}", command)
def test_desktop_builds_embed_the_trusted_update_keys(self):
self.build("linux", update_keys=("--dart-define=PIPELINES_UPDATE_PUBLIC_KEY=abc",))
run = next(c for c in self.runner.calls if c[:2] == ["docker", "run"])
self.assertIn("--dart-define=PIPELINES_UPDATE_PUBLIC_KEY=abc", run)
def test_desktop_server_prefill_rejects_embedded_credentials(self):
with self.assertRaisesRegex(ValueError, "embedded credentials"):
self.build("linux", server_url="wss://secret@example.com/connect")
+117
View File
@@ -0,0 +1,117 @@
import base64
import importlib.util
import json
import os
from pathlib import Path
import subprocess
import tempfile
import unittest
from unittest.mock import patch
def load(name):
spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(name + ".py"))
result = importlib.util.module_from_spec(spec)
spec.loader.exec_module(result)
return result
signing = load("update_signing")
FIXTURE = Path(__file__).resolve().parents[1] / "app/test/features/updates/fixtures/signed_feed"
def verify(tool, public, data, signature):
"""Independent check with `openssl pkeyutl -verify` against the raw key."""
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / "pub.der").write_bytes(signing.SPKI_PREFIX + base64.b64decode(public))
(root / "data").write_bytes(data)
(root / "sig").write_bytes(signature)
result = subprocess.run([tool, "pkeyutl", "-verify", "-pubin", "-keyform", "DER", "-inkey",
str(root / "pub.der"), "-rawin", "-in", str(root / "data"),
"-sigfile", str(root / "sig")], capture_output=True)
return result.returncode == 0
class UpdateSigningTests(unittest.TestCase):
def setUp(self):
self.home = tempfile.TemporaryDirectory()
self.addCleanup(self.home.cleanup)
patcher = patch.object(signing.Path, "home", return_value=Path(self.home.name))
patcher.start()
self.addCleanup(patcher.stop)
self.environ = {}
self.tool = signing.openssl(self.environ)
def test_key_is_generated_once_private_under_the_config_directory(self):
key = signing.ensure_key(self.environ)
self.assertEqual(key, Path(self.home.name) / ".config/pipelines/update-signing-key.pem")
self.assertEqual(key.stat().st_mode & 0o777, 0o600)
self.assertEqual(key.parent.stat().st_mode & 0o777, 0o700)
before = key.read_bytes()
self.assertEqual(signing.ensure_key(self.environ), key)
self.assertEqual(key.read_bytes(), before)
self.assertEqual([p.name for p in key.parent.iterdir()], [key.name])
def test_readable_or_symlinked_key_is_refused(self):
key = signing.ensure_key(self.environ)
os.chmod(key, 0o644)
with self.assertRaisesRegex(ValueError, "chmod 600"):
signing.ensure_key(self.environ)
os.chmod(key, 0o600)
link = key.with_name("link.pem")
link.symlink_to(key)
with self.assertRaisesRegex(ValueError, "regular file"):
signing.ensure_key({"PIPELINES_UPDATE_SIGNING_KEY": str(link)})
def test_openssl_without_ed25519_is_skipped_or_reported(self):
missing = {"PIPELINES_OPENSSL": "/nonexistent/openssl"}
self.assertEqual(signing.openssl(missing), self.tool)
with patch.object(signing, "CANDIDATES", ()), \
patch.object(signing.shutil, "which", return_value="/usr/bin/false"):
with self.assertRaisesRegex(ValueError, "OpenSSL 3"):
signing.openssl({})
def test_signature_verifies_and_tampering_does_not(self):
key = signing.ensure_key(self.environ)
data = b'{"versionCode": 7}\n'
document = json.loads(signing.signature_document(data, key, self.environ))
self.assertEqual(document["algorithm"], "ed25519")
public = document["publicKey"]
signature = base64.b64decode(document["signature"])
self.assertEqual((len(base64.b64decode(public)), len(signature)), (32, 64))
self.assertTrue(verify(self.tool, public, data, signature))
self.assertFalse(verify(self.tool, public, b'{"versionCode": 8}\n', signature))
def test_publish_writes_feed_and_detached_signature(self):
target = Path(self.home.name) / "updates"
target.mkdir()
data = signing.publish_feed(target, {"versionCode": 3}, signing.Signer(environ=self.environ))
self.assertEqual((target / "version.json").read_bytes(), data)
document = json.loads((target / "version.json.sig").read_text())
self.assertTrue(verify(self.tool, document["publicKey"], data, base64.b64decode(document["signature"])))
self.assertEqual((target / "version.json").stat().st_mode & 0o777, 0o644)
def test_build_embeds_own_key_first_then_listed_trusted_keys(self):
own = signing.public_key(signing.ensure_key(self.environ))
other = base64.b64encode(bytes(range(32))).decode()
listed = Path(self.home.name) / ".config/pipelines/update-trusted-keys"
listed.write_text(f"# tank\n{other}\n\n{own}\n")
self.assertEqual(signing.dart_defines(self.environ),
(f"--dart-define=PIPELINES_UPDATE_PUBLIC_KEY={own},{other}",))
listed.write_text("not-a-key\n")
with self.assertRaisesRegex(ValueError, "update-trusted-keys:1"):
signing.trusted_keys(self.environ)
def test_committed_dart_fixture_is_a_valid_python_signature(self):
# The Dart verifier test reads this fixture; publish_feed produced it
# with a throwaway key that was never kept.
public = (FIXTURE / "public_key.txt").read_text().strip()
document = json.loads((FIXTURE / "version.json.sig").read_text())
self.assertEqual(document["publicKey"], public)
self.assertTrue(verify(self.tool, public, (FIXTURE / "version.json").read_bytes(),
base64.b64decode(document["signature"])))
if __name__ == "__main__":
unittest.main()
+189
View File
@@ -0,0 +1,189 @@
"""Ed25519 signing for the app update feed.
Each installation owns one private key, generated once into a 0600 file under
~/.config/pipelines/ (never in the repository). App builds embed the public key
(`--dart-define=PIPELINES_UPDATE_PUBLIC_KEY`, comma separated) together with any
other servers' keys listed one per line in
~/.config/pipelines/update-trusted-keys. Publishing writes `version.json` and a
detached `version.json.sig` over its exact bytes; the app verifies that
signature with an embedded key before it trusts the artifact sha256 inside.
The OpenSSL 3 command line does the cryptography (the system Python has no
`cryptography` package and macOS's LibreSSL has no Ed25519), so the tool is
looked up explicitly: PIPELINES_OPENSSL, Homebrew's openssl@3, then PATH.
"""
import base64
import json
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
KEY_NAME = "update-signing-key.pem"
TRUSTED_NAME = "update-trusted-keys"
SIGNATURE_SUFFIX = ".sig"
DEFINE = "PIPELINES_UPDATE_PUBLIC_KEY"
CANDIDATES = ("/opt/homebrew/opt/openssl@3/bin/openssl", "/usr/local/opt/openssl@3/bin/openssl")
# DER SubjectPublicKeyInfo for Ed25519 is this 12-byte prefix plus the raw key.
SPKI_PREFIX = bytes.fromhex("302a300506032b6570032100")
def config_dir():
return Path.home() / ".config/pipelines"
def key_path(environ=None):
environ = os.environ if environ is None else environ
explicit = environ.get("PIPELINES_UPDATE_SIGNING_KEY")
return Path(explicit).expanduser() if explicit else config_dir() / KEY_NAME
def _supports_ed25519(tool):
try:
result = subprocess.run([tool, "genpkey", "-algorithm", "ed25519"],
capture_output=True, timeout=30)
except (OSError, subprocess.TimeoutExpired):
return False
return result.returncode == 0 and b"PRIVATE KEY" in result.stdout
def openssl(environ=None):
"""An OpenSSL executable with Ed25519 support, or a clear error."""
environ = os.environ if environ is None else environ
found = [environ.get("PIPELINES_OPENSSL"), *CANDIDATES, shutil.which("openssl")]
for tool in dict.fromkeys(t for t in found if t):
if _supports_ed25519(tool):
return tool
raise ValueError("Update signing needs OpenSSL 3 with Ed25519 (LibreSSL lacks it): "
"install openssl@3 (brew install openssl@3) or set PIPELINES_OPENSSL")
def _check_private(path):
if path.is_symlink() or not path.is_file():
raise ValueError(f"Update signing key must be a regular file: {path}")
if path.stat().st_mode & 0o077:
raise ValueError(f"Update signing key must be private (chmod 600): {path}")
def ensure_key(environ=None):
"""The installation's signing key, generated once (0600) when absent."""
path = key_path(environ)
if path.exists() or path.is_symlink():
_check_private(path)
return path
tool = openssl(environ)
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
descriptor, temporary = tempfile.mkstemp(prefix=".update-key-", dir=path.parent)
os.close(descriptor)
try:
os.chmod(temporary, 0o600)
result = subprocess.run([tool, "genpkey", "-algorithm", "ed25519", "-out", temporary],
capture_output=True, text=True, timeout=30)
if result.returncode:
raise ValueError("Could not generate the update signing key: " + result.stderr.strip())
os.chmod(temporary, 0o600)
# Never replace a key another process created meanwhile.
os.link(temporary, path)
finally:
Path(temporary).unlink(missing_ok=True)
_check_private(path)
return path
def public_key(key, environ=None):
"""The raw 32-byte public key, base64 encoded."""
result = subprocess.run([openssl(environ), "pkey", "-in", str(key), "-pubout", "-outform", "DER"],
capture_output=True, timeout=30)
der = result.stdout
if result.returncode or len(der) != 44 or not der.startswith(SPKI_PREFIX):
raise ValueError(f"Could not read the Ed25519 public key from {key}")
return base64.b64encode(der[len(SPKI_PREFIX):]).decode()
def valid_public_key(text):
try:
return len(base64.b64decode(text, validate=True)) == 32
except ValueError:
return False
def trusted_keys(environ=None):
"""This installation's key first, then other servers' keys it trusts."""
keys = [public_key(ensure_key(environ), environ)]
listed = config_dir() / TRUSTED_NAME
if listed.is_file():
for number, line in enumerate(listed.read_text().splitlines(), 1):
line = line.split("#", 1)[0].strip()
if not line:
continue
if not valid_public_key(line):
raise ValueError(f"{listed}:{number} is not a base64 Ed25519 public key")
keys.append(line)
return list(dict.fromkeys(keys))
def dart_defines(environ=None):
"""The build argument embedding the trusted update keys."""
return (f"--dart-define={DEFINE}={','.join(trusted_keys(environ))}",)
def sign(data, key, environ=None):
"""A raw 64-byte Ed25519 signature over [data]."""
with tempfile.NamedTemporaryFile(prefix="pipelines-feed-") as message:
message.write(data)
message.flush()
result = subprocess.run([openssl(environ), "pkeyutl", "-sign", "-rawin", "-inkey", str(key),
"-in", message.name], capture_output=True, timeout=30)
if result.returncode or len(result.stdout) != 64:
raise ValueError("Could not sign the update feed: " + result.stderr.decode(errors="replace").strip())
return result.stdout
def signature_document(data, key, environ=None):
"""The detached `version.json.sig` content for feed bytes [data]."""
return (json.dumps({"algorithm": "ed25519", "publicKey": public_key(key, environ),
"signature": base64.b64encode(sign(data, key, environ)).decode()}) + "\n").encode()
class Signer:
"""Signs feeds with one key file; the installer's key unless given."""
def __init__(self, key=None, environ=None):
self.environ = environ
self.key = Path(key) if key else ensure_key(environ)
_check_private(self.key)
def __call__(self, data):
return signature_document(data, self.key, self.environ)
def write_atomic(path, content, mode=0o644):
descriptor, temporary = tempfile.mkstemp(prefix=path.name + ".", dir=path.parent)
try:
with os.fdopen(descriptor, "wb") as output:
output.write(content)
output.flush()
os.fsync(output.fileno())
os.chmod(temporary, mode)
os.replace(temporary, path)
finally:
Path(temporary).unlink(missing_ok=True)
def publish_feed(directory, metadata, signer):
"""Writes the signature, then the feed it covers, both atomically.
A reader racing a publish may see a new signature with the old feed for a
moment; that verification fails and the next check succeeds."""
data = (json.dumps(metadata) + "\n").encode()
signature = signer(data)
write_atomic(directory / ("version.json" + SIGNATURE_SUFFIX), signature)
write_atomic(directory / "version.json", data)
return data
if __name__ == "__main__":
# Prints this installation's public key, for another server's
# update-trusted-keys file (generating the key on first use).
print(public_key(ensure_key()))