335 lines
16 KiB
Python
335 lines
16 KiB
Python
"""Builds the Flutter app in Docker and reuses each build until its source changes.
|
|
|
|
`web` is the browser client the installer serves behind the web login; `apk`
|
|
is the optional Android app, offered to phones through Check for updates.
|
|
|
|
python3 scripts/app_build.py web [--prefix PREFIX]
|
|
python3 scripts/app_build.py apk [--prefix PREFIX] [--server-url URL]
|
|
|
|
Each build is keyed by a digest of the app's sources, the Flutter version, the
|
|
build image definition and the build arguments. A finished build is kept under
|
|
~/.cache/pipelines/app-builds and reused whenever that key repeats, so an
|
|
unchanged app is never rebuilt.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from urllib.parse import urlsplit
|
|
|
|
REPO = Path(__file__).resolve().parents[1]
|
|
APP = REPO / "app"
|
|
DEFAULT_PREFIX = Path.home() / ".local/share/pipelines"
|
|
|
|
def default_cache():
|
|
return Path(os.environ.get("PIPELINES_APP_CACHE") or Path.home() / ".cache/pipelines/app-builds")
|
|
# Kept per build kind; older entries are pruned after a new build.
|
|
KEEP = 3
|
|
# Inputs that change what a build produces. Generated and machine-local paths
|
|
# (build/, .dart_tool/, android/local.properties) are left out.
|
|
SOURCES = {
|
|
"web": ("lib", "web", "assets", "fonts", "pubspec.yaml", "pubspec.lock", ".flutter-version", "docker"),
|
|
"apk": ("lib", "android", "assets", "fonts", "pubspec.yaml", "pubspec.lock", ".flutter-version", "docker"),
|
|
"linux": ("lib", "linux", "assets", "fonts", "pubspec.yaml", "pubspec.lock", ".flutter-version", "docker"),
|
|
"macos": ("lib", "macos", "assets", "fonts", "pubspec.yaml", "pubspec.lock", ".flutter-version"),
|
|
}
|
|
SKIPPED = {"build", ".dart_tool", ".gradle", ".cxx", "local.properties", ".DS_Store", "ephemeral", "Pods", ".symlinks"}
|
|
# This is part of the web cache key so entries made before release metadata was
|
|
# generated cannot be mistaken for complete web releases.
|
|
WEB_CACHE_SCHEMA = "web-release-metadata-v1"
|
|
WEB_RELEASE_SCHEMA = 1
|
|
|
|
|
|
def flutter_version(app=APP):
|
|
return (app / ".flutter-version").read_text().strip()
|
|
|
|
|
|
def digest(kind, app=APP, extra=()):
|
|
"""The build key: every source byte and path, plus [extra] arguments."""
|
|
hasher = hashlib.sha256()
|
|
hasher.update(kind.encode())
|
|
if kind == "web":
|
|
hasher.update(b"\0cache-schema\0" + WEB_CACHE_SCHEMA.encode())
|
|
if kind in ("linux", "macos"):
|
|
hasher.update(b"\0cache-schema\0desktop-v1")
|
|
for script in (Path(__file__), Path(__file__).with_name("app_desktop_build.py")):
|
|
hasher.update(b"\0builder\0" + script.read_bytes())
|
|
if kind == "linux":
|
|
hasher.update(b"\0platform\0linux/amd64")
|
|
for item in extra:
|
|
hasher.update(b"\0arg\0" + item.encode())
|
|
for name in SOURCES[kind]:
|
|
root = app / name
|
|
paths = [root] if root.is_file() else sorted(p for p in root.rglob("*") if p.is_file()) if root.is_dir() else []
|
|
for path in paths:
|
|
relative = path.relative_to(app)
|
|
if SKIPPED.intersection(relative.parts) or (relative.parts[:2] == ("linux", "flutter") and relative.name in {
|
|
"generated_plugin_registrant.cc", "generated_plugin_registrant.h", "generated_plugins.cmake"}):
|
|
continue
|
|
hasher.update(b"\0path\0" + str(relative).encode() + b"\0")
|
|
hasher.update(path.read_bytes())
|
|
return hasher.hexdigest()[:20]
|
|
|
|
|
|
def web_build_id(key):
|
|
"""Stable identity embedded in the web app and published beside it."""
|
|
return f"web-{key}"
|
|
|
|
|
|
def web_release_metadata(key):
|
|
return {"schema": WEB_RELEASE_SCHEMA, "platform": "web", "buildId": web_build_id(key)}
|
|
|
|
|
|
def web_cache_complete(entry):
|
|
"""Whether [entry] contains metadata from the current web build schema."""
|
|
marker = entry / "release.json"
|
|
if not (entry / ".complete").is_file() or not marker.is_file():
|
|
return False
|
|
try:
|
|
complete = json.loads((entry / ".complete").read_text())
|
|
return json.loads(marker.read_text()) == web_release_metadata(complete["key"])
|
|
except (KeyError, TypeError, ValueError, json.JSONDecodeError):
|
|
return False
|
|
|
|
|
|
def docker(runner, *args, quiet=False):
|
|
result = runner(["docker", *args], capture_output=quiet, text=True)
|
|
if result.returncode:
|
|
detail = (result.stderr or "").strip().splitlines()[-1:] if quiet else []
|
|
raise RuntimeError("docker " + args[0] + " failed" + (": " + detail[0] if detail else ""))
|
|
return result
|
|
|
|
|
|
def docker_available(runner=subprocess.run):
|
|
try:
|
|
return runner(["docker", "info", "--format", "{{.ServerVersion}}"],
|
|
capture_output=True, text=True, timeout=20).returncode == 0
|
|
except (OSError, subprocess.SubprocessError):
|
|
return False
|
|
|
|
|
|
def image(kind, runner, app=APP):
|
|
"""Builds (or reuses, via Docker's layer cache) the build image for [kind]."""
|
|
target = {"apk": "android", "linux": "linux"}.get(kind, "web")
|
|
version = flutter_version(app)
|
|
tag = f"pipelines-flutter-{target}:{version}"
|
|
args = ["build", "--target", target, "--build-arg", f"FLUTTER_VERSION={version}", "-t", tag,
|
|
"-f", str(app / "docker/Dockerfile")]
|
|
if kind in ("apk", "linux"):
|
|
# Android's build tools (aapt2) ship for x86-64 Linux only.
|
|
args += ["--platform", "linux/amd64"]
|
|
docker(runner, *args, str(app / "docker"))
|
|
return tag
|
|
|
|
|
|
def signing_dir():
|
|
"""One debug signing key for every APK, so a phone accepts each update.
|
|
|
|
Seeded from the host's own Android debug key when there is one, so APKs
|
|
built before Docker builds existed keep updating."""
|
|
directory = Path.home() / ".config/pipelines/android"
|
|
directory.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
keystore = directory / "debug.keystore"
|
|
host = Path.home() / ".android/debug.keystore"
|
|
if not keystore.exists() and host.is_file():
|
|
shutil.copyfile(host, keystore)
|
|
os.chmod(keystore, 0o600)
|
|
return directory
|
|
|
|
|
|
def build(kind, runner=subprocess.run, app=APP, cache=None, arguments=(), log=print):
|
|
"""The directory holding the finished [kind] build, built only when needed."""
|
|
cache = Path(cache) if cache else default_cache()
|
|
key = digest(kind, app, arguments)
|
|
entry = cache / f"{kind}-{key}"
|
|
usable = web_cache_complete(entry) if kind == "web" else (entry / ".complete").is_file()
|
|
if kind == "linux":
|
|
usable = usable and (entry / "pipelines-app").is_file() and (entry / "data/flutter_assets").is_dir() and (entry / "lib").is_dir()
|
|
if usable:
|
|
log(f"Reusing the {kind} build {key} (app sources unchanged).")
|
|
return entry
|
|
log(f"Building the app ({kind}) in Docker with Flutter {flutter_version(app)}; "
|
|
"the first build downloads the toolchain and takes several minutes.")
|
|
tag = image(kind, runner, app)
|
|
cache.mkdir(parents=True, exist_ok=True)
|
|
work = Path(tempfile.mkdtemp(prefix=f"{kind}-", dir=cache))
|
|
try:
|
|
command = ["run", "--rm", "--user", f"{os.getuid()}:{os.getgid()}",
|
|
"-v", f"{app}:/src:ro", "-v", f"{work}:/out",
|
|
"-v", "pipelines-flutter-pub:/home/builder/.pub-cache"]
|
|
if kind == "linux":
|
|
command += ["--platform", "linux/amd64"]
|
|
if kind == "apk":
|
|
command += ["--platform", "linux/amd64",
|
|
"-v", "pipelines-flutter-gradle:/home/builder/.gradle",
|
|
"-v", "pipelines-android-sdk:/opt/android-sdk",
|
|
"-v", f"{signing_dir()}:/home/builder/.android"]
|
|
built = int(time.time())
|
|
stamp = ()
|
|
build_arguments = arguments
|
|
if kind == "web":
|
|
# The define is derived from the source key, not included in it:
|
|
# this avoids circular hashing while keeping the compiled app and
|
|
# its published metadata on the same identity.
|
|
build_arguments += (f"--dart-define=PIPELINES_WEB_BUILD_ID={web_build_id(key)}",)
|
|
if kind == "linux":
|
|
build_arguments += (f"--dart-define=PIPELINES_NATIVE_BUILD_CODE={built // 60}",)
|
|
if kind == "apk":
|
|
# Android installs only a higher build number over the last one, so
|
|
# each build is stamped when it is made (minutes since the epoch);
|
|
# the stamp is not part of the reuse key.
|
|
code = built // 60
|
|
stamp = (f"--build-number={code}", f"--build-name=0.1.{code}")
|
|
docker(runner, *command, tag, "pipelines-app-build", kind, *build_arguments, *stamp)
|
|
if kind == "linux" and not ((work / "pipelines-app").is_file() and
|
|
(work / "data/flutter_assets").is_dir() and (work / "lib").is_dir()):
|
|
raise RuntimeError("Linux build did not produce its complete executable/data/lib bundle")
|
|
if kind == "web":
|
|
(work / "release.json").write_text(json.dumps(web_release_metadata(key)) + "\n")
|
|
(work / ".complete").write_text(json.dumps({"key": key, "built": built}) + "\n")
|
|
if entry.exists():
|
|
shutil.rmtree(entry)
|
|
work.rename(entry)
|
|
except BaseException:
|
|
shutil.rmtree(work, ignore_errors=True)
|
|
raise
|
|
prune(cache, kind, keep=entry)
|
|
return entry
|
|
|
|
|
|
def prune(cache, kind, keep):
|
|
entries = sorted((p for p in cache.glob(f"{kind}-*") if (p / ".complete").is_file()),
|
|
key=lambda p: p.stat().st_mtime, reverse=True)
|
|
for stale in [p for p in entries if p != keep][KEEP - 1:]:
|
|
shutil.rmtree(stale, ignore_errors=True)
|
|
|
|
|
|
def install_tree(source, target):
|
|
"""Replaces [target] with a copy of [source] without a half-written moment."""
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
staging = target.with_name(target.name + ".next")
|
|
shutil.rmtree(staging, ignore_errors=True)
|
|
shutil.copytree(source, staging, ignore=shutil.ignore_patterns(".complete"))
|
|
previous = target.with_name(target.name + ".previous")
|
|
shutil.rmtree(previous, ignore_errors=True)
|
|
if target.exists():
|
|
target.rename(previous)
|
|
staging.rename(target)
|
|
shutil.rmtree(previous, ignore_errors=True)
|
|
|
|
|
|
def phone_server_url(manifest):
|
|
"""The phone's default server: the installation's authenticated web origin."""
|
|
public = (manifest.get("opencode_env") or {}).get("PIPELINES_WEB_PUBLIC_URL", "")
|
|
parsed = urlsplit(public)
|
|
if parsed.scheme not in ("http", "https") or not parsed.netloc:
|
|
raise ValueError("The installation records no web address; pass --server-url ws(s)://host:port/connect")
|
|
return f"{'wss' if parsed.scheme == 'https' else 'ws'}://{parsed.netloc}/connect"
|
|
|
|
|
|
def load_manifest(prefix):
|
|
path = Path(prefix).expanduser().resolve() / "pipelines-install.json"
|
|
if not path.is_file():
|
|
raise ValueError(f"No native Pipelines installation at {prefix}; run ./install first")
|
|
return json.loads(path.read_text())
|
|
|
|
|
|
def build_apk(prefix, server_url=None, notes="", runner=subprocess.run, log=print):
|
|
"""Builds the APK for this installation and offers it at /updates/."""
|
|
manifest = load_manifest(prefix)
|
|
web = manifest.get("web") or {}
|
|
updates = web.get("updates")
|
|
if not updates:
|
|
raise ValueError("This installation predates app updates; rerun ./install, then ./install --apk")
|
|
server = server_url or phone_server_url(manifest)
|
|
# The build number is the source key's build date, so the same sources
|
|
# reuse one APK and every rebuild is newer than the last.
|
|
arguments = (f"--dart-define=PIPELINES_SERVER_URL={server}",)
|
|
output = build("apk", runner, arguments=arguments, log=log)
|
|
built = json.loads((output / ".complete").read_text())["built"]
|
|
code = built // 60
|
|
target = Path(updates)
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
shutil.copyfile(output / "pipelines.apk", target / "pipelines.apk.next")
|
|
os.replace(target / "pipelines.apk.next", target / "pipelines.apk")
|
|
(target / "version.json").write_text(json.dumps({
|
|
"versionCode": code, "versionName": f"0.1.{code}", "apk": "pipelines.apk", "notes": notes}) + "\n")
|
|
log(f"APK ready: {target / 'pipelines.apk'} (build {code}, server {server}).")
|
|
log("Phones get it from Settings → App updates → Check for updates, or download "
|
|
f"{server.replace('wss://', 'https://').replace('ws://', 'http://').removesuffix('/connect')}/updates/pipelines.apk")
|
|
return target / "pipelines.apk"
|
|
|
|
|
|
def print_web_urls(manifest, lan_host=None):
|
|
"""Keep browser access details as the final installer output."""
|
|
web = manifest.get("web") or {}
|
|
if not web.get("port"):
|
|
return
|
|
host = web.get("host", "0.0.0.0")
|
|
local = "localhost" if host in ("0.0.0.0", "127.0.0.1") else host
|
|
urls = [f"http://{local}:{web['port']}/"]
|
|
if host == "0.0.0.0":
|
|
lan_host = lan_host or urlsplit(manifest.get("public_url", "")).hostname
|
|
if lan_host and lan_host not in ("localhost", "127.0.0.1", "0.0.0.0"):
|
|
urls.append(f"http://{lan_host}:{web['port']}/")
|
|
public = manifest.get("opencode_env", {}).get("PIPELINES_WEB_PUBLIC_URL")
|
|
if public:
|
|
urls.append(public.rstrip("/") + "/")
|
|
print("\nOpen Pipelines in your browser:")
|
|
for url in dict.fromkeys(urls):
|
|
print(" " + url)
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
parser.add_argument("kind", choices=("web", "apk", "linux", "macos"))
|
|
parser.add_argument("--prefix", type=Path, default=DEFAULT_PREFIX)
|
|
parser.add_argument("--server-url", help="APK default server (default: this installation's web address)")
|
|
parser.add_argument("--notes", default="", help="APK release notes shown in Check for updates")
|
|
parser.add_argument("--publish", action="store_true", help="Publish desktop archive and update feed to this installation")
|
|
parser.add_argument("--output", type=Path, help="Desktop archive output directory (default: build cache)")
|
|
parser.add_argument("--flutter", help="Native Flutter executable for macOS (or PIPELINES_FLUTTER/PATH)")
|
|
args = parser.parse_args(argv)
|
|
if args.kind in ("linux", "macos"):
|
|
spec = importlib.util.spec_from_file_location("pipelines_desktop_build", Path(__file__).with_name("app_desktop_build.py"))
|
|
desktop = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(desktop)
|
|
bundle, archive = desktop.build_desktop(args.kind, app=APP, cache=default_cache(), runner=subprocess.run,
|
|
source_digest=digest, docker_build=build, docker_ready=docker_available,
|
|
server_url=args.server_url, flutter=args.flutter, output=args.output,
|
|
prune_cache=prune)
|
|
if args.publish:
|
|
manifest = load_manifest(args.prefix)
|
|
updates = (manifest.get("web") or {}).get("updates")
|
|
if not updates:
|
|
raise ValueError("This installation has no app update directory; rerun ./install")
|
|
desktop.publish_desktop(args.kind, bundle, archive, Path(updates), notes=args.notes)
|
|
return
|
|
if not docker_available():
|
|
raise RuntimeError("Docker is required to build the app; start Docker and retry")
|
|
if args.kind == "apk":
|
|
build_apk(args.prefix, args.server_url, args.notes)
|
|
print_web_urls(load_manifest(args.prefix))
|
|
return
|
|
manifest = load_manifest(args.prefix)
|
|
web = manifest.get("web") or {}
|
|
if not web.get("root"):
|
|
raise ValueError("This installation does not serve the app; rerun ./install")
|
|
install_tree(build("web"), Path(web["root"]))
|
|
print(f"Web app installed at {web['root']}; reload the page.")
|
|
print_web_urls(manifest)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except (ValueError, RuntimeError, OSError) as error:
|
|
sys.exit(str(error))
|