"""Builds the Flutter app in Docker and reuses each build until its source changes. `web` is the browser client the installer serves behind the web login; `apk` is the optional Android app, offered to phones through Check for updates. python3 scripts/app_build.py web [--prefix PREFIX] python3 scripts/app_build.py apk [--prefix PREFIX] [--server-url URL] Each build is keyed by a digest of the app's sources, the Flutter version, the build image definition and the build arguments. A finished build is kept under ~/.cache/pipelines/app-builds and reused whenever that key repeats, so an unchanged app is never rebuilt. """ import argparse import hashlib import json import os from pathlib import Path import shutil import subprocess import sys import tempfile import time from urllib.parse import urlsplit REPO = Path(__file__).resolve().parents[1] APP = REPO / "app" DEFAULT_PREFIX = Path.home() / ".local/share/pipelines" def default_cache(): return Path(os.environ.get("PIPELINES_APP_CACHE") or Path.home() / ".cache/pipelines/app-builds") # Kept per build kind; older entries are pruned after a new build. KEEP = 3 # Inputs that change what a build produces. Generated and machine-local paths # (build/, .dart_tool/, android/local.properties) are left out. SOURCES = { "web": ("lib", "web", "assets", "pubspec.yaml", "pubspec.lock", ".flutter-version", "docker"), "apk": ("lib", "android", "assets", "pubspec.yaml", "pubspec.lock", ".flutter-version", "docker"), } SKIPPED = {"build", ".dart_tool", ".gradle", ".cxx", "local.properties", ".DS_Store"} def flutter_version(app=APP): return (app / ".flutter-version").read_text().strip() def digest(kind, app=APP, extra=()): """The build key: every source byte and path, plus [extra] arguments.""" hasher = hashlib.sha256() hasher.update(kind.encode()) for item in extra: hasher.update(b"\0arg\0" + item.encode()) for name in SOURCES[kind]: root = app / name paths = [root] if root.is_file() else sorted(p for p in root.rglob("*") if p.is_file()) if root.is_dir() else [] for path in paths: relative = path.relative_to(app) if SKIPPED.intersection(relative.parts): continue hasher.update(b"\0path\0" + str(relative).encode() + b"\0") hasher.update(path.read_bytes()) return hasher.hexdigest()[:20] def docker(runner, *args, quiet=False): result = runner(["docker", *args], capture_output=quiet, text=True) if result.returncode: detail = (result.stderr or "").strip().splitlines()[-1:] if quiet else [] raise RuntimeError("docker " + args[0] + " failed" + (": " + detail[0] if detail else "")) return result def docker_available(runner=subprocess.run): try: return runner(["docker", "info", "--format", "{{.ServerVersion}}"], capture_output=True, text=True, timeout=20).returncode == 0 except (OSError, subprocess.SubprocessError): return False def image(kind, runner, app=APP): """Builds (or reuses, via Docker's layer cache) the build image for [kind].""" target = "android" if kind == "apk" else "web" version = flutter_version(app) tag = f"pipelines-flutter-{target}:{version}" args = ["build", "--target", target, "--build-arg", f"FLUTTER_VERSION={version}", "-t", tag, "-f", str(app / "docker/Dockerfile")] if kind == "apk": # Android's build tools (aapt2) ship for x86-64 Linux only. args += ["--platform", "linux/amd64"] docker(runner, *args, str(app / "docker")) return tag def signing_dir(): """One debug signing key for every APK, so a phone accepts each update. Seeded from the host's own Android debug key when there is one, so APKs built before Docker builds existed keep updating.""" directory = Path.home() / ".config/pipelines/android" directory.mkdir(mode=0o700, parents=True, exist_ok=True) keystore = directory / "debug.keystore" host = Path.home() / ".android/debug.keystore" if not keystore.exists() and host.is_file(): shutil.copyfile(host, keystore) os.chmod(keystore, 0o600) return directory def build(kind, runner=subprocess.run, app=APP, cache=None, arguments=(), log=print): """The directory holding the finished [kind] build, built only when needed.""" cache = Path(cache) if cache else default_cache() key = digest(kind, app, arguments) entry = cache / f"{kind}-{key}" if (entry / ".complete").is_file(): log(f"Reusing the {kind} build {key} (app sources unchanged).") return entry log(f"Building the app ({kind}) in Docker with Flutter {flutter_version(app)}; " "the first build downloads the toolchain and takes several minutes.") tag = image(kind, runner, app) cache.mkdir(parents=True, exist_ok=True) work = Path(tempfile.mkdtemp(prefix=f"{kind}-", dir=cache)) try: command = ["run", "--rm", "--user", f"{os.getuid()}:{os.getgid()}", "-v", f"{app}:/src:ro", "-v", f"{work}:/out", "-v", "pipelines-flutter-pub:/home/builder/.pub-cache"] if kind == "apk": command += ["--platform", "linux/amd64", "-v", "pipelines-flutter-gradle:/home/builder/.gradle", "-v", "pipelines-android-sdk:/opt/android-sdk", "-v", f"{signing_dir()}:/home/builder/.android"] built = int(time.time()) stamp = () if kind == "apk": # Android installs only a higher build number over the last one, so # each build is stamped when it is made (minutes since the epoch); # the stamp is not part of the reuse key. code = built // 60 stamp = (f"--build-number={code}", f"--build-name=0.1.{code}") docker(runner, *command, tag, "pipelines-app-build", kind, *arguments, *stamp) (work / ".complete").write_text(json.dumps({"key": key, "built": built}) + "\n") if entry.exists(): shutil.rmtree(entry) work.rename(entry) except BaseException: shutil.rmtree(work, ignore_errors=True) raise prune(cache, kind, keep=entry) return entry def prune(cache, kind, keep): entries = sorted((p for p in cache.glob(f"{kind}-*") if (p / ".complete").is_file()), key=lambda p: p.stat().st_mtime, reverse=True) for stale in [p for p in entries if p != keep][KEEP - 1:]: shutil.rmtree(stale, ignore_errors=True) def install_tree(source, target): """Replaces [target] with a copy of [source] without a half-written moment.""" target.parent.mkdir(parents=True, exist_ok=True) staging = target.with_name(target.name + ".next") shutil.rmtree(staging, ignore_errors=True) shutil.copytree(source, staging, ignore=shutil.ignore_patterns(".complete")) previous = target.with_name(target.name + ".previous") shutil.rmtree(previous, ignore_errors=True) if target.exists(): target.rename(previous) staging.rename(target) shutil.rmtree(previous, ignore_errors=True) def phone_server_url(manifest): """The phone's default server: the installation's authenticated web origin.""" public = (manifest.get("opencode_env") or {}).get("PIPELINES_WEB_PUBLIC_URL", "") parsed = urlsplit(public) if parsed.scheme not in ("http", "https") or not parsed.netloc: raise ValueError("The installation records no web address; pass --server-url ws(s)://host:port/connect") return f"{'wss' if parsed.scheme == 'https' else 'ws'}://{parsed.netloc}/connect" def load_manifest(prefix): path = Path(prefix).expanduser().resolve() / "pipelines-install.json" if not path.is_file(): raise ValueError(f"No native Pipelines installation at {prefix}; run ./install first") return json.loads(path.read_text()) def build_apk(prefix, server_url=None, notes="", runner=subprocess.run, log=print): """Builds the APK for this installation and offers it at /updates/.""" manifest = load_manifest(prefix) web = manifest.get("web") or {} updates = web.get("updates") if not updates: raise ValueError("This installation predates app updates; rerun ./install, then ./install --apk") server = server_url or phone_server_url(manifest) # The build number is the source key's build date, so the same sources # reuse one APK and every rebuild is newer than the last. arguments = (f"--dart-define=PIPELINES_SERVER_URL={server}",) output = build("apk", runner, arguments=arguments, log=log) built = json.loads((output / ".complete").read_text())["built"] code = built // 60 target = Path(updates) target.mkdir(parents=True, exist_ok=True) shutil.copyfile(output / "pipelines.apk", target / "pipelines.apk.next") os.replace(target / "pipelines.apk.next", target / "pipelines.apk") (target / "version.json").write_text(json.dumps({ "versionCode": code, "versionName": f"0.1.{code}", "apk": "pipelines.apk", "notes": notes}) + "\n") log(f"APK ready: {target / 'pipelines.apk'} (build {code}, server {server}).") log("Phones get it from Settings → App updates → Check for updates, or download " f"{server.replace('wss://', 'https://').replace('ws://', 'http://').removesuffix('/connect')}/updates/pipelines.apk") return target / "pipelines.apk" def main(argv=None): parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("kind", choices=("web", "apk")) parser.add_argument("--prefix", type=Path, default=DEFAULT_PREFIX) parser.add_argument("--server-url", help="APK default server (default: this installation's web address)") parser.add_argument("--notes", default="", help="APK release notes shown in Check for updates") args = parser.parse_args(argv) if not docker_available(): raise RuntimeError("Docker is required to build the app; start Docker and retry") if args.kind == "apk": build_apk(args.prefix, args.server_url, args.notes) return manifest = load_manifest(args.prefix) web = manifest.get("web") or {} if not web.get("root"): raise ValueError("This installation does not serve the app; rerun ./install") install_tree(build("web"), Path(web["root"])) print(f"Web app installed at {web['root']}; reload the page.") if __name__ == "__main__": try: main() except (ValueError, RuntimeError, OSError) as error: sys.exit(str(error))