Build Linux and macOS clients with platform update feeds

This commit is contained in:
2026-11-08 16:45:42 +00:00
parent b7f439a94f
commit 36f73bb89d
17 changed files with 913 additions and 9 deletions
+42 -6
View File
@@ -13,6 +13,7 @@ unchanged app is never rebuilt.
"""
import argparse
import hashlib
import importlib.util
import json
import os
from pathlib import Path
@@ -34,10 +35,12 @@ KEEP = 3
# Inputs that change what a build produces. Generated and machine-local paths
# (build/, .dart_tool/, android/local.properties) are left out.
SOURCES = {
"web": ("lib", "web", "assets", "pubspec.yaml", "pubspec.lock", ".flutter-version", "docker"),
"apk": ("lib", "android", "assets", "pubspec.yaml", "pubspec.lock", ".flutter-version", "docker"),
"web": ("lib", "web", "assets", "fonts", "pubspec.yaml", "pubspec.lock", ".flutter-version", "docker"),
"apk": ("lib", "android", "assets", "fonts", "pubspec.yaml", "pubspec.lock", ".flutter-version", "docker"),
"linux": ("lib", "linux", "assets", "fonts", "pubspec.yaml", "pubspec.lock", ".flutter-version", "docker"),
"macos": ("lib", "macos", "assets", "fonts", "pubspec.yaml", "pubspec.lock", ".flutter-version"),
}
SKIPPED = {"build", ".dart_tool", ".gradle", ".cxx", "local.properties", ".DS_Store"}
SKIPPED = {"build", ".dart_tool", ".gradle", ".cxx", "local.properties", ".DS_Store", "ephemeral", "Pods", ".symlinks"}
# This is part of the web cache key so entries made before release metadata was
# generated cannot be mistaken for complete web releases.
WEB_CACHE_SCHEMA = "web-release-metadata-v1"
@@ -54,6 +57,12 @@ def digest(kind, app=APP, extra=()):
hasher.update(kind.encode())
if kind == "web":
hasher.update(b"\0cache-schema\0" + WEB_CACHE_SCHEMA.encode())
if kind in ("linux", "macos"):
hasher.update(b"\0cache-schema\0desktop-v1")
for script in (Path(__file__), Path(__file__).with_name("app_desktop_build.py")):
hasher.update(b"\0builder\0" + script.read_bytes())
if kind == "linux":
hasher.update(b"\0platform\0linux/amd64")
for item in extra:
hasher.update(b"\0arg\0" + item.encode())
for name in SOURCES[kind]:
@@ -107,12 +116,12 @@ def docker_available(runner=subprocess.run):
def image(kind, runner, app=APP):
"""Builds (or reuses, via Docker's layer cache) the build image for [kind]."""
target = "android" if kind == "apk" else "web"
target = {"apk": "android", "linux": "linux"}.get(kind, "web")
version = flutter_version(app)
tag = f"pipelines-flutter-{target}:{version}"
args = ["build", "--target", target, "--build-arg", f"FLUTTER_VERSION={version}", "-t", tag,
"-f", str(app / "docker/Dockerfile")]
if kind == "apk":
if kind in ("apk", "linux"):
# Android's build tools (aapt2) ship for x86-64 Linux only.
args += ["--platform", "linux/amd64"]
docker(runner, *args, str(app / "docker"))
@@ -140,6 +149,8 @@ def build(kind, runner=subprocess.run, app=APP, cache=None, arguments=(), log=pr
key = digest(kind, app, arguments)
entry = cache / f"{kind}-{key}"
usable = web_cache_complete(entry) if kind == "web" else (entry / ".complete").is_file()
if kind == "linux":
usable = usable and (entry / "pipelines-app").is_file() and (entry / "data/flutter_assets").is_dir() and (entry / "lib").is_dir()
if usable:
log(f"Reusing the {kind} build {key} (app sources unchanged).")
return entry
@@ -152,6 +163,8 @@ def build(kind, runner=subprocess.run, app=APP, cache=None, arguments=(), log=pr
command = ["run", "--rm", "--user", f"{os.getuid()}:{os.getgid()}",
"-v", f"{app}:/src:ro", "-v", f"{work}:/out",
"-v", "pipelines-flutter-pub:/home/builder/.pub-cache"]
if kind == "linux":
command += ["--platform", "linux/amd64"]
if kind == "apk":
command += ["--platform", "linux/amd64",
"-v", "pipelines-flutter-gradle:/home/builder/.gradle",
@@ -165,6 +178,8 @@ def build(kind, runner=subprocess.run, app=APP, cache=None, arguments=(), log=pr
# this avoids circular hashing while keeping the compiled app and
# its published metadata on the same identity.
build_arguments += (f"--dart-define=PIPELINES_WEB_BUILD_ID={web_build_id(key)}",)
if kind == "linux":
build_arguments += (f"--dart-define=PIPELINES_NATIVE_BUILD_CODE={built // 60}",)
if kind == "apk":
# Android installs only a higher build number over the last one, so
# each build is stamped when it is made (minutes since the epoch);
@@ -172,6 +187,9 @@ def build(kind, runner=subprocess.run, app=APP, cache=None, arguments=(), log=pr
code = built // 60
stamp = (f"--build-number={code}", f"--build-name=0.1.{code}")
docker(runner, *command, tag, "pipelines-app-build", kind, *build_arguments, *stamp)
if kind == "linux" and not ((work / "pipelines-app").is_file() and
(work / "data/flutter_assets").is_dir() and (work / "lib").is_dir()):
raise RuntimeError("Linux build did not produce its complete executable/data/lib bundle")
if kind == "web":
(work / "release.json").write_text(json.dumps(web_release_metadata(key)) + "\n")
(work / ".complete").write_text(json.dumps({"key": key, "built": built}) + "\n")
@@ -270,11 +288,29 @@ def print_web_urls(manifest, lan_host=None):
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("kind", choices=("web", "apk"))
parser.add_argument("kind", choices=("web", "apk", "linux", "macos"))
parser.add_argument("--prefix", type=Path, default=DEFAULT_PREFIX)
parser.add_argument("--server-url", help="APK default server (default: this installation's web address)")
parser.add_argument("--notes", default="", help="APK release notes shown in Check for updates")
parser.add_argument("--publish", action="store_true", help="Publish desktop archive and update feed to this installation")
parser.add_argument("--output", type=Path, help="Desktop archive output directory (default: build cache)")
parser.add_argument("--flutter", help="Native Flutter executable for macOS (or PIPELINES_FLUTTER/PATH)")
args = parser.parse_args(argv)
if args.kind in ("linux", "macos"):
spec = importlib.util.spec_from_file_location("pipelines_desktop_build", Path(__file__).with_name("app_desktop_build.py"))
desktop = importlib.util.module_from_spec(spec)
spec.loader.exec_module(desktop)
bundle, archive = desktop.build_desktop(args.kind, app=APP, cache=default_cache(), runner=subprocess.run,
source_digest=digest, docker_build=build, docker_ready=docker_available,
server_url=args.server_url, flutter=args.flutter, output=args.output,
prune_cache=prune)
if args.publish:
manifest = load_manifest(args.prefix)
updates = (manifest.get("web") or {}).get("updates")
if not updates:
raise ValueError("This installation has no app update directory; rerun ./install")
desktop.publish_desktop(args.kind, bundle, archive, Path(updates), notes=args.notes)
return
if not docker_available():
raise RuntimeError("Docker is required to build the app; start Docker and retry")
if args.kind == "apk":
+192
View File
@@ -0,0 +1,192 @@
"""Standalone desktop client artifacts; never installs or restarts a server."""
import hashlib
import json
import os
from pathlib import Path
import platform
import shutil
import subprocess
import tarfile
import tempfile
import time
from urllib.parse import urlsplit
def _run(runner, command, **kwargs):
try:
result = runner(command, **kwargs)
except subprocess.TimeoutExpired as error:
raise RuntimeError(f"{command[0]} timed out") from error
if result.returncode:
raise RuntimeError(f"{command[0]} failed; see its build output")
return result
def _arguments(server_url):
if not server_url:
return ()
parsed = urlsplit(server_url)
if parsed.scheme not in ("ws", "wss", "http", "https") or not parsed.hostname or parsed.username or parsed.password or parsed.fragment:
raise ValueError("--server-url must be a server address without embedded credentials or fragment")
return (f"--dart-define=PIPELINES_SERVER_URL={server_url}",)
def _linux_archive(entry):
archive = entry / "pipelines-linux-x64.tar.gz"
if archive.is_file():
return archive
staged = archive.with_suffix(".gz.next")
try:
with tarfile.open(staged, "w:gz") as output:
for child in sorted(entry.iterdir()):
if child.name not in (".complete", archive.name, staged.name):
output.add(child, arcname=f"pipelines-linux-x64/{child.name}")
os.replace(staged, archive)
finally:
staged.unlink(missing_ok=True)
return archive
def _macos_build(*, app, cache, source_digest, runner, arguments, flutter, log):
if platform.system() != "Darwin":
raise ValueError("macOS clients must be built natively on macOS with Xcode")
selected = flutter or os.environ.get("PIPELINES_FLUTTER") or shutil.which("flutter")
if not selected:
raise ValueError("Flutter is required; pass --flutter /path/to/flutter or set PIPELINES_FLUTTER")
selected = str(Path(selected).expanduser())
version = _run(runner, [selected, "--version", "--machine"], capture_output=True, text=True, timeout=60)
try:
actual = json.loads(version.stdout)["frameworkVersion"]
except (ValueError, KeyError, TypeError) as error:
raise ValueError("The selected Flutter SDK did not report its version") from error
pinned = (app / ".flutter-version").read_text().strip()
if actual != pinned:
raise ValueError(f"Flutter {pinned} is required; selected SDK is {actual}. No SDK was changed.")
xcode = _run(runner, ["xcodebuild", "-version"], capture_output=True, text=True, timeout=30)
identity = ("native-macos-v1", platform.machine(), xcode.stdout.strip(), actual)
key = source_digest("macos", app=app, extra=arguments + identity)
entry = cache / f"macos-{key}"
bundle = entry / "Pipelines.app"
archive = entry / "Pipelines-macos.zip"
if (entry / ".complete").is_file() and (bundle / "Contents/MacOS/Pipelines").is_file() and archive.is_file():
log(f"Reusing native macOS build {key}.")
return entry, bundle, archive
cache.mkdir(parents=True, exist_ok=True)
staging = Path(tempfile.mkdtemp(prefix="macos-", dir=cache))
try:
with tempfile.TemporaryDirectory(prefix="pipelines-macos-source-") as temporary:
work = Path(temporary) / "app"
shutil.copytree(app, work, symlinks=True, ignore=shutil.ignore_patterns(
"build", ".dart_tool", "Pods", "ephemeral", ".symlinks", ".gradle", ".cxx", "local.properties", ".DS_Store"))
_run(runner, [selected, "pub", "get"], cwd=work, timeout=300)
built = int(time.time())
_run(runner, [selected, "build", "macos", "--release", "--no-pub", *arguments,
f"--build-number={built // 60}",
f"--dart-define=PIPELINES_NATIVE_BUILD_CODE={built // 60}"], cwd=work, timeout=1200,
env={**os.environ, "FLUTTER_XCODE_ARCHS": {"aarch64": "arm64", "x86_64": "x86_64"}.get(platform.machine(), platform.machine())})
native = work / "build/macos/Build/Products/Release/Pipelines.app"
if not (native / "Contents/MacOS/Pipelines").is_file():
raise RuntimeError("Flutter did not produce a complete Pipelines.app")
arches = _run(runner, ["lipo", "-archs", str(native / "Contents/MacOS/Pipelines")],
capture_output=True, text=True, timeout=30).stdout.split()
architectures = sorted({{"x86_64": "x64", "arm64": "arm64"}.get(a, a) for a in arches})
if not architectures or any(a not in ("x64", "arm64") for a in architectures):
raise RuntimeError("macOS bundle has unsupported or missing architectures")
# Framework symlinks and executable modes are part of the bundle.
shutil.copytree(native, staging / "Pipelines.app", symlinks=True)
_run(runner, ["ditto", "-c", "-k", "--keepParent", str(staging / "Pipelines.app"),
str(staging / archive.name)], timeout=120)
if not (staging / archive.name).is_file():
raise RuntimeError("macOS archive was not created")
(staging / ".complete").write_text(json.dumps({"key": key, "built": built, "architectures": architectures}) + "\n")
if entry.exists():
previous = entry.with_name(entry.name + ".previous")
if previous.exists():
raise RuntimeError(f"A previous build is retained at {previous}; no files were replaced")
entry.rename(previous)
try:
staging.rename(entry)
except BaseException:
previous.rename(entry)
raise
shutil.rmtree(previous)
else:
staging.rename(entry)
finally:
if staging.exists():
shutil.rmtree(staging)
return entry, bundle, archive
def build_desktop(kind, *, app, cache, runner, source_digest, docker_build,
docker_ready, server_url=None, flutter=None, output=None, log=print,
prune_cache=None):
arguments = _arguments(server_url)
if kind == "macos":
entry, bundle, archive = _macos_build(app=app, cache=cache, runner=runner,
source_digest=source_digest, arguments=arguments, flutter=flutter, log=log)
elif kind == "linux":
if not docker_ready(runner):
raise RuntimeError("Docker is required for the Linux x64 client; start Docker and retry")
entry = docker_build("linux", app=app, cache=cache, runner=runner, arguments=arguments, log=log)
if not (entry / "pipelines-app").is_file() or not (entry / "data/flutter_assets").is_dir() or not (entry / "lib").is_dir():
raise RuntimeError("Linux build is missing its executable, data or libraries")
bundle, archive = entry, _linux_archive(entry)
else:
raise ValueError(f"Unknown desktop target: {kind}")
if prune_cache is not None:
prune_cache(cache, kind, keep=entry)
if output is not None:
output = Path(output).expanduser()
output.mkdir(parents=True, exist_ok=True)
published = output / archive.name
descriptor, temporary = tempfile.mkstemp(prefix=archive.name + ".", dir=output)
os.close(descriptor)
try:
shutil.copy2(archive, temporary)
os.replace(temporary, published)
finally:
Path(temporary).unlink(missing_ok=True)
archive = published
log(f"Desktop client bundle: {bundle}")
log(f"Desktop client archive: {archive}")
log("No server was installed or restarted. Choose your server in the app.")
return bundle, archive
def publish_desktop(kind, bundle, archive, updates, *, notes="", log=print):
"""Publish archive before its feed; versioned names preserve in-flight reads."""
entry = bundle.parent if kind == "macos" else bundle
complete = json.loads((entry / ".complete").read_text())
code = int(complete["built"]) // 60
architectures = complete.get("architectures") if kind == "macos" else ["x64"]
if not architectures or any(a not in ("x64", "arm64") for a in architectures):
raise ValueError("Build metadata has no supported architecture")
digest = hashlib.sha256(archive.read_bytes()).hexdigest()
suffix = ".zip" if kind == "macos" else ".tar.gz"
stem = "Pipelines-macos" if kind == "macos" else "pipelines-linux-x64"
filename = f"{stem}-{code}-{digest[:8]}{suffix}"
target = updates / ("macos" if kind == "macos" else "linux-x64")
target.mkdir(parents=True, exist_ok=True)
metadata = {"versionCode": code, "versionName": f"0.1.{code}",
"platform": kind, "architectures": architectures, "file": filename,
"sha256": digest, "notes": notes}
feed = target / "version.json"
if feed.exists():
previous = json.loads(feed.read_text())
if previous.get("versionCode", 0) > code:
raise ValueError("Refusing to publish an older build over a newer update")
for destination, content in ((target / filename, archive.read_bytes()),
(feed, (json.dumps(metadata) + "\n").encode())):
descriptor, temporary = tempfile.mkstemp(prefix=destination.name + ".", dir=target)
try:
with os.fdopen(descriptor, "wb") as output:
output.write(content)
output.flush()
os.fsync(output.fileno())
os.chmod(temporary, 0o644)
os.replace(temporary, destination)
finally:
Path(temporary).unlink(missing_ok=True)
log(f"Published {kind} update {code} at {feed}")
return metadata
+178
View File
@@ -0,0 +1,178 @@
import importlib.util
import json
from pathlib import Path
import subprocess
import tarfile
import tempfile
import unittest
from unittest.mock import patch
def load(name):
spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(name + ".py"))
result = importlib.util.module_from_spec(spec)
spec.loader.exec_module(result)
return result
core = load("app_build")
desktop = load("app_desktop_build")
class Runner:
def __init__(self):
self.calls = []
self.native_env = None
self.fail_native = False
self.version = "3.44.6"
def __call__(self, command, **kwargs):
self.calls.append(command)
stdout = ""
code = 0
if "--machine" in command:
stdout = json.dumps({"frameworkVersion": self.version})
elif command[:2] == ["xcodebuild", "-version"]:
stdout = "Xcode 27.0\nBuild version 27A266a"
elif command[:2] == ["docker", "run"]:
output = Path(next(arg[:-5] for arg in command if arg.endswith(":/out")))
(output / "pipelines-app").write_bytes(b"linux-client")
(output / "pipelines-app").chmod(0o755)
(output / "data/flutter_assets").mkdir(parents=True)
(output / "data/flutter_assets/font.ttf").write_bytes(b"font")
(output / "lib").mkdir()
(output / "lib/engine.so").write_bytes(b"engine")
elif command[1:3] == ["build", "macos"]:
self.native_env = kwargs.get("env")
if self.fail_native:
code = 1
else:
app = Path(kwargs["cwd"]) / "build/macos/Build/Products/Release/Pipelines.app"
(app / "Contents/MacOS").mkdir(parents=True)
(app / "Contents/MacOS/Pipelines").write_bytes(b"macos-client")
(app / "Contents/MacOS/Pipelines").chmod(0o755)
framework = app / "Contents/Frameworks/Fixture.framework"
(framework / "Versions/A").mkdir(parents=True)
(framework / "Versions/A/Fixture").write_bytes(b"framework")
(framework / "Versions/Current").symlink_to("A")
(framework / "Fixture").symlink_to("Versions/Current/Fixture")
elif command[:2] == ["lipo", "-archs"]:
stdout = "x86_64 arm64"
elif command[0] == "ditto":
Path(command[-1]).write_bytes(b"archive")
return subprocess.CompletedProcess(command, code, stdout=stdout, stderr="")
class DesktopBuildTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary.cleanup)
self.root = Path(self.temporary.name)
self.app = self.root / "app"
for directory in ("lib", "fonts", "linux", "macos", "docker"):
(self.app / directory).mkdir(parents=True, exist_ok=True)
(self.app / "lib/main.dart").write_text("main() {}")
(self.app / ".flutter-version").write_text("3.44.6\n")
self.runner = Runner()
def build(self, kind, **kwargs):
return desktop.build_desktop(kind, app=self.app, cache=self.root / "cache",
runner=self.runner, source_digest=core.digest, docker_build=core.build,
docker_ready=lambda *_: True, log=lambda *_: None, **kwargs)
def test_linux_is_x64_and_packages_full_bundle_without_installation(self):
bundle, archive = self.build("linux", output=self.root / "artifacts")
builds = [c for c in self.runner.calls if c[:2] == ["docker", "build"]]
self.assertEqual(builds[0][builds[0].index("--target") + 1], "linux")
for command in [builds[0], next(c for c in self.runner.calls if c[:2] == ["docker", "run"])]:
self.assertEqual(command[command.index("--platform") + 1], "linux/amd64")
with tarfile.open(archive) as package:
self.assertTrue({"pipelines-linux-x64/pipelines-app", "pipelines-linux-x64/lib/engine.so",
"pipelines-linux-x64/data/flutter_assets/font.ttf"}.issubset(package.getnames()))
self.assertEqual(package.getmember("pipelines-linux-x64/pipelines-app").mode & 0o111, 0o111)
self.assertTrue(bundle.is_dir())
before = len(self.runner.calls)
self.build("linux")
self.assertEqual(len(self.runner.calls), before)
def test_font_and_target_sources_invalidate_correct_caches(self):
keys = {kind: core.digest(kind, self.app) for kind in core.SOURCES}
(self.app / "fonts/font.ttf").write_bytes(b"new font")
for kind in core.SOURCES:
self.assertNotEqual(keys[kind], core.digest(kind, self.app), kind)
linux = core.digest("linux", self.app)
macos = core.digest("macos", self.app)
(self.app / "linux/CMakeLists.txt").write_text("changed native runner")
self.assertNotEqual(linux, core.digest("linux", self.app))
self.assertEqual(macos, core.digest("macos", self.app))
generated = self.app / "macos/Flutter/ephemeral"
generated.mkdir(parents=True)
(generated / "local-path").write_text("machine-specific generated value")
self.assertEqual(macos, core.digest("macos", self.app))
@patch.object(desktop.platform, "system", return_value="Darwin")
def test_native_macos_preserves_framework_symlinks_and_reuses_build(self, _):
bundle, archive = self.build("macos", flutter="/fixture/flutter")
self.assertTrue((bundle / "Contents/Frameworks/Fixture.framework/Versions/Current").is_symlink())
self.assertEqual((bundle / "Contents/MacOS/Pipelines").stat().st_mode & 0o111, 0o111)
self.assertTrue(archive.is_file())
self.build("macos", flutter="/fixture/flutter")
self.assertEqual(sum(c[1:3] == ["build", "macos"] for c in self.runner.calls), 1)
self.assertFalse(any(c[0] == "docker" for c in self.runner.calls))
self.assertFalse((self.app / "build").exists())
expected = {"aarch64": "arm64", "x86_64": "x86_64"}.get(desktop.platform.machine(), desktop.platform.machine())
self.assertEqual(self.runner.native_env["FLUTTER_XCODE_ARCHS"], expected)
@patch.object(desktop.platform, "system", return_value="Darwin")
def test_failed_native_build_preserves_previous_artifact(self, _):
bundle, archive = self.build("macos", flutter="/fixture/flutter")
previous = archive.read_bytes()
(self.app / "lib/main.dart").write_text("changed application")
self.runner.fail_native = True
with self.assertRaisesRegex(RuntimeError, "failed"):
self.build("macos", flutter="/fixture/flutter")
self.assertEqual(archive.read_bytes(), previous)
self.assertTrue((bundle / "Contents/MacOS/Pipelines").is_file())
@patch.object(desktop.platform, "system", return_value="Darwin")
def test_wrong_sdk_fails_before_building_or_changing_any_sdk(self, _):
self.runner.version = "3.47.0"
with self.assertRaisesRegex(ValueError, "3.44.6 is required"):
self.build("macos", flutter="/fixture/flutter")
self.assertEqual(len(self.runner.calls), 1)
@patch.object(desktop.platform, "system", return_value="Linux")
def test_macos_refuses_non_macos_host_without_docker_fallback(self, _):
with self.assertRaisesRegex(ValueError, "natively on macOS"):
self.build("macos")
self.assertEqual(self.runner.calls, [])
def test_linux_publication_has_matching_hash_and_rejects_downgrade(self):
bundle, archive = self.build("linux")
updates = self.root / "updates"
metadata = desktop.publish_desktop("linux", bundle, archive, updates, log=lambda *_: None)
target = updates / "linux-x64"
self.assertEqual(json.loads((target / "version.json").read_text()), metadata)
self.assertEqual(metadata["architectures"], ["x64"])
self.assertEqual(metadata["sha256"], desktop.hashlib.sha256((target / metadata["file"]).read_bytes()).hexdigest())
metadata["versionCode"] += 1
(target / "version.json").write_text(json.dumps(metadata))
with self.assertRaisesRegex(ValueError, "older build"):
desktop.publish_desktop("linux", bundle, archive, updates)
@patch.object(desktop.platform, "system", return_value="Darwin")
def test_macos_feed_uses_actual_architectures_and_native_build_stamp(self, _):
bundle, archive = self.build("macos", flutter="/fixture/flutter")
metadata = desktop.publish_desktop("macos", bundle, archive, self.root / "updates", log=lambda *_: None)
self.assertEqual(metadata["architectures"], ["arm64", "x64"])
command = next(c for c in self.runner.calls if c[1:3] == ["build", "macos"])
self.assertIn(f"--dart-define=PIPELINES_NATIVE_BUILD_CODE={metadata['versionCode']}", command)
def test_desktop_server_prefill_rejects_embedded_credentials(self):
with self.assertRaisesRegex(ValueError, "embedded credentials"):
self.build("linux", server_url="wss://secret@example.com/connect")
self.assertEqual(self.runner.calls, [])
if __name__ == "__main__":
unittest.main()