feat(install): build the Flutter web app in Docker, optional ./install --apk, chosen web password, startup report

This commit is contained in:
2026-10-23 21:45:14 +01:00
parent 3136132144
commit 7958d1830e
16 changed files with 776 additions and 29 deletions
+231
View File
@@ -0,0 +1,231 @@
"""Builds the Flutter app in Docker and reuses each build until its source changes.
`web` is the browser client the installer serves behind the web login; `apk`
is the optional Android app, offered to phones through Check for updates.
python3 scripts/app_build.py web [--prefix PREFIX]
python3 scripts/app_build.py apk [--prefix PREFIX] [--server-url URL]
Each build is keyed by a digest of the app's sources, the Flutter version, the
build image definition and the build arguments. A finished build is kept under
~/.cache/pipelines/app-builds and reused whenever that key repeats, so an
unchanged app is never rebuilt.
"""
import argparse
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
import time
from urllib.parse import urlsplit
REPO = Path(__file__).resolve().parents[1]
APP = REPO / "app"
DEFAULT_PREFIX = Path.home() / ".local/share/pipelines"
def default_cache():
return Path(os.environ.get("PIPELINES_APP_CACHE") or Path.home() / ".cache/pipelines/app-builds")
# Kept per build kind; older entries are pruned after a new build.
KEEP = 3
# Inputs that change what a build produces. Generated and machine-local paths
# (build/, .dart_tool/, android/local.properties) are left out.
SOURCES = {
"web": ("lib", "web", "assets", "pubspec.yaml", "pubspec.lock", ".flutter-version", "docker"),
"apk": ("lib", "android", "assets", "pubspec.yaml", "pubspec.lock", ".flutter-version", "docker"),
}
SKIPPED = {"build", ".dart_tool", ".gradle", ".cxx", "local.properties", ".DS_Store"}
def flutter_version(app=APP):
return (app / ".flutter-version").read_text().strip()
def digest(kind, app=APP, extra=()):
"""The build key: every source byte and path, plus [extra] arguments."""
hasher = hashlib.sha256()
hasher.update(kind.encode())
for item in extra:
hasher.update(b"\0arg\0" + item.encode())
for name in SOURCES[kind]:
root = app / name
paths = [root] if root.is_file() else sorted(p for p in root.rglob("*") if p.is_file()) if root.is_dir() else []
for path in paths:
relative = path.relative_to(app)
if SKIPPED.intersection(relative.parts):
continue
hasher.update(b"\0path\0" + str(relative).encode() + b"\0")
hasher.update(path.read_bytes())
return hasher.hexdigest()[:20]
def docker(runner, *args, quiet=False):
result = runner(["docker", *args], capture_output=quiet, text=True)
if result.returncode:
detail = (result.stderr or "").strip().splitlines()[-1:] if quiet else []
raise RuntimeError("docker " + args[0] + " failed" + (": " + detail[0] if detail else ""))
return result
def docker_available(runner=subprocess.run):
try:
return runner(["docker", "info", "--format", "{{.ServerVersion}}"],
capture_output=True, text=True, timeout=20).returncode == 0
except (OSError, subprocess.SubprocessError):
return False
def image(kind, runner, app=APP):
"""Builds (or reuses, via Docker's layer cache) the build image for [kind]."""
target = "android" if kind == "apk" else "web"
version = flutter_version(app)
tag = f"pipelines-flutter-{target}:{version}"
args = ["build", "--target", target, "--build-arg", f"FLUTTER_VERSION={version}", "-t", tag,
"-f", str(app / "docker/Dockerfile")]
if kind == "apk":
# Android's build tools (aapt2) ship for x86-64 Linux only.
args += ["--platform", "linux/amd64"]
docker(runner, *args, str(app / "docker"))
return tag
def signing_dir():
"""One debug signing key for every APK, so a phone accepts each update.
Seeded from the host's own Android debug key when there is one, so APKs
built before Docker builds existed keep updating."""
directory = Path.home() / ".config/pipelines/android"
directory.mkdir(mode=0o700, parents=True, exist_ok=True)
keystore = directory / "debug.keystore"
host = Path.home() / ".android/debug.keystore"
if not keystore.exists() and host.is_file():
shutil.copyfile(host, keystore)
os.chmod(keystore, 0o600)
return directory
def build(kind, runner=subprocess.run, app=APP, cache=None, arguments=(), log=print):
"""The directory holding the finished [kind] build, built only when needed."""
cache = Path(cache) if cache else default_cache()
key = digest(kind, app, arguments)
entry = cache / f"{kind}-{key}"
if (entry / ".complete").is_file():
log(f"Reusing the {kind} build {key} (app sources unchanged).")
return entry
log(f"Building the app ({kind}) in Docker with Flutter {flutter_version(app)}; "
"the first build downloads the toolchain and takes several minutes.")
tag = image(kind, runner, app)
cache.mkdir(parents=True, exist_ok=True)
work = Path(tempfile.mkdtemp(prefix=f"{kind}-", dir=cache))
try:
command = ["run", "--rm", "--user", f"{os.getuid()}:{os.getgid()}",
"-v", f"{app}:/src:ro", "-v", f"{work}:/out",
"-v", "pipelines-flutter-pub:/home/builder/.pub-cache"]
if kind == "apk":
command += ["--platform", "linux/amd64",
"-v", "pipelines-flutter-gradle:/home/builder/.gradle",
"-v", "pipelines-android-sdk:/opt/android-sdk",
"-v", f"{signing_dir()}:/home/builder/.android"]
docker(runner, *command, tag, "pipelines-app-build", kind, *arguments)
(work / ".complete").write_text(json.dumps({"key": key, "built": int(time.time())}) + "\n")
if entry.exists():
shutil.rmtree(entry)
work.rename(entry)
except BaseException:
shutil.rmtree(work, ignore_errors=True)
raise
prune(cache, kind, keep=entry)
return entry
def prune(cache, kind, keep):
entries = sorted((p for p in cache.glob(f"{kind}-*") if (p / ".complete").is_file()),
key=lambda p: p.stat().st_mtime, reverse=True)
for stale in [p for p in entries if p != keep][KEEP - 1:]:
shutil.rmtree(stale, ignore_errors=True)
def install_tree(source, target):
"""Replaces [target] with a copy of [source] without a half-written moment."""
target.parent.mkdir(parents=True, exist_ok=True)
staging = target.with_name(target.name + ".next")
shutil.rmtree(staging, ignore_errors=True)
shutil.copytree(source, staging, ignore=shutil.ignore_patterns(".complete"))
previous = target.with_name(target.name + ".previous")
shutil.rmtree(previous, ignore_errors=True)
if target.exists():
target.rename(previous)
staging.rename(target)
shutil.rmtree(previous, ignore_errors=True)
def phone_server_url(manifest):
"""The phone's default server: the installation's authenticated web origin."""
public = (manifest.get("opencode_env") or {}).get("PIPELINES_WEB_PUBLIC_URL", "")
parsed = urlsplit(public)
if parsed.scheme not in ("http", "https") or not parsed.netloc:
raise ValueError("The installation records no web address; pass --server-url ws(s)://host:port/connect")
return f"{'wss' if parsed.scheme == 'https' else 'ws'}://{parsed.netloc}/connect"
def load_manifest(prefix):
path = Path(prefix).expanduser().resolve() / "pipelines-install.json"
if not path.is_file():
raise ValueError(f"No native Pipelines installation at {prefix}; run ./install first")
return json.loads(path.read_text())
def build_apk(prefix, server_url=None, notes="", runner=subprocess.run, log=print):
"""Builds the APK for this installation and offers it at /updates/."""
manifest = load_manifest(prefix)
web = manifest.get("web") or {}
updates = web.get("updates")
if not updates:
raise ValueError("This installation predates app updates; rerun ./install, then ./install --apk")
server = server_url or phone_server_url(manifest)
# The build number is the source key's build date, so the same sources
# reuse one APK and every rebuild is newer than the last.
arguments = (f"--dart-define=PIPELINES_SERVER_URL={server}",)
output = build("apk", runner, arguments=arguments, log=log)
built = json.loads((output / ".complete").read_text())["built"]
code = built // 60
target = Path(updates)
target.mkdir(parents=True, exist_ok=True)
shutil.copyfile(output / "pipelines.apk", target / "pipelines.apk.next")
os.replace(target / "pipelines.apk.next", target / "pipelines.apk")
(target / "version.json").write_text(json.dumps({
"versionCode": code, "versionName": f"0.1.{code}", "apk": "pipelines.apk", "notes": notes}) + "\n")
log(f"APK ready: {target / 'pipelines.apk'} (build {code}, server {server}).")
log("Phones get it from Settings → App updates → Check for updates, or download "
f"{server.replace('wss://', 'https://').replace('ws://', 'http://').removesuffix('/connect')}/updates/pipelines.apk")
return target / "pipelines.apk"
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("kind", choices=("web", "apk"))
parser.add_argument("--prefix", type=Path, default=DEFAULT_PREFIX)
parser.add_argument("--server-url", help="APK default server (default: this installation's web address)")
parser.add_argument("--notes", default="", help="APK release notes shown in Check for updates")
args = parser.parse_args(argv)
if not docker_available():
raise RuntimeError("Docker is required to build the app; start Docker and retry")
if args.kind == "apk":
build_apk(args.prefix, args.server_url, args.notes)
return
manifest = load_manifest(args.prefix)
web = manifest.get("web") or {}
if not web.get("root"):
raise ValueError("This installation does not serve the app; rerun ./install")
install_tree(build("web"), Path(web["root"]))
print(f"Web app installed at {web['root']}; reload the page.")
if __name__ == "__main__":
try:
main()
except (ValueError, RuntimeError, OSError) as error:
sys.exit(str(error))
+47 -1
View File
@@ -268,6 +268,47 @@ def ntfy_stop(config):
path.unlink(missing_ok=True)
def server_report(config):
"""The server's own startup report: the last `== Pipelines server` block
it wrote to its log."""
log = Path(config["prefix"]) / "logs/server.log"
try:
with log.open("rb") as handle:
handle.seek(max(0, log.stat().st_size - 64 * 1024))
lines = handle.read().decode("utf-8", "replace").splitlines()
except OSError:
return []
starts = [i for i, line in enumerate(lines) if line.startswith("== Pipelines server")]
if not starts:
return []
block = []
for line in lines[starts[-1]:]:
block.append(line)
if line == "==":
break
return block
def report(config):
"""Prints what is running: server, backends, web app and phone access."""
for line in server_report(config):
print(line)
web = config.get("web")
if web:
public = (config.get("opencode_env") or {}).get("PIPELINES_WEB_PUBLIC_URL")
print("== Access")
address = public or f"http://{web['host']}:{web['port']}"
print(f" {'Web':<10} {address} (password in {web['auth_file']})")
root = Path(web["root"]) if web.get("root") else None
built = root is not None and (root / "index.html").is_file()
print(f" {'Web app':<10} {'installed' if built else 'not built; run ./install --web'}")
updates = Path(web["updates"]) if web.get("updates") else None
apk = updates is not None and (updates / "pipelines.apk").is_file()
print(f" {'Phone app':<10} " + (f"{updates / 'pipelines.apk'} (Settings → Check for updates)"
if apk else "not built; run ./install --apk"))
print("==")
def supervise(config):
root = Path(config["prefix"])
with (root / "supervisor.lock").open("a") as lock:
@@ -320,7 +361,9 @@ def supervise(config):
with (root / "logs/web.log").open("a") as web_log:
web_child = subprocess.Popen(
[web["node"], web["script"], "--port", str(web["port"]),
"--host", web["host"], "--pipelines", f"127.0.0.1:{config['server_port']}"],
"--host", web["host"], "--pipelines", f"127.0.0.1:{config['server_port']}",
*(["--root", web["root"]] if web.get("root") else []),
*(["--updates", web["updates"]] if web.get("updates") else [])],
env=web_env, stdout=web_log, stderr=web_log)
while child.poll() is None and not stopped:
if web_child is not None and web_child.poll() is not None:
@@ -419,11 +462,14 @@ def main():
parser.error("unexpected arguments: " + " ".join(rest))
if args.action == "start":
start(config)
report(config)
elif args.action == "stop":
stop(config)
elif args.action == "restart":
stop(config); start(config)
report(config)
elif args.action in ("status", "doctor"):
report(config)
print("Pipeline server:", "healthy" if healthy(f"http://127.0.0.1:{config['server_port']}/healthz") else "stopped/unhealthy")
print("ntfy:", "healthy" if healthy(ntfy_health_url(config)) else "stopped/unhealthy")
print("Phone server:", config["public_url"], "· topic:", config["topic"], "· username:", config["username"])
+45 -2
View File
@@ -17,6 +17,7 @@ import ipaddress
import json
import os
import platform
import getpass
import plistlib
import secrets
import shlex
@@ -41,6 +42,7 @@ configure = module("configure_compose", "configure-compose.py")
lan = module("lan_address", "lan-address.py")
release_lookup = module("resolve_releases", "resolve-releases.py")
tokens = module("save_ntfy_token", "save-ntfy-token.py")
app_build = module("app_build", "app_build.py")
DEFAULT_PREFIX = Path.home() / ".local/share/pipelines"
MARKER = "pipelines-install.json"
@@ -325,14 +327,53 @@ class Setup:
if auth.stat().st_mode & 0o077:
raise ValueError(f"Web auth file must be private (chmod 600): {auth}")
else:
password = self.choose_password()
auth.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
self._write_atomic(auth, (json.dumps({"password": secrets.token_urlsafe(18),
self._write_atomic(auth, (json.dumps({"password": password,
"secret": secrets.token_urlsafe(48)}) + "\n").encode(), 0o600)
target = prefix / "lib/web"
shutil.copytree(Path(__file__).resolve().parents[1] / "web", target, dirs_exist_ok=True,
ignore=shutil.ignore_patterns("test", "node_modules", ".DS_Store"))
updates = prefix / "lib/app-updates"
updates.mkdir(parents=True, exist_ok=True)
return {"node": node, "script": str(target / "serve.mjs"),
"auth_file": str(auth), "host": args.web_host, "port": args.web_port}
"auth_file": str(auth), "host": args.web_host, "port": args.web_port,
"root": self.install_app_web(prefix, args), "updates": str(updates)}
def choose_password(self):
"""The web sign-in password for a new computer, chosen by the operator.
PIPELINES_WEB_PASSWORD answers it without a terminal (automation)."""
given = self.environ.get("PIPELINES_WEB_PASSWORD")
if given is not None:
if len(given) < 8:
raise ValueError("PIPELINES_WEB_PASSWORD must be at least 8 characters")
return given
if not sys.stdin.isatty():
raise ValueError("Choose the web sign-in password: rerun ./install in a terminal, "
"or set PIPELINES_WEB_PASSWORD (at least 8 characters)")
print("This computer has no web sign-in yet. Choose the password for the web app and phone.")
while True:
first = getpass.getpass("Web password (at least 8 characters): ")
if len(first) < 8:
print("Too short; use at least 8 characters.")
continue
if getpass.getpass("Repeat the password: ") != first:
print("The passwords did not match; try again.")
continue
return first
def install_app_web(self, prefix, args):
"""Builds the Flutter web app in Docker (reused while its sources are
unchanged) and installs it for the web proxy. Without Docker the web
page explains how to build it later (./install --web)."""
target = prefix / "lib/app-web"
if not app_build.docker_available(self.runner):
print("WARNING: Docker is not running, so the web app was not built. "
"Start Docker, then run ./install --web.")
return str(target)
app_build.install_tree(app_build.build("web", runner=self.runner), target)
return str(target)
# --- build and skill -------------------------------------------------
@@ -750,6 +791,8 @@ class Setup:
f"(sign-in password in {web['auth_file']}; existing credentials retained).")
print("Login autostart is already enabled. To enable/start it again, run:\n "
+ shlex.join([str(prefix / "bin/pipelines"), "service", "enable"]))
print("Android app (optional, slower): ./install --apk builds it in Docker and offers it at "
f"{opencode_env['PIPELINES_WEB_PUBLIC_URL']}/updates/.")
print("Configured server role: " + ("single-user authoring enabled (web pipeline creation/publication allowed)."
if allow_authoring else "execution-only (pipeline publication disabled)."))
if args.bind_address == "0.0.0.0":
+29 -2
View File
@@ -177,13 +177,38 @@ class ManageNativeTests(unittest.TestCase):
self.assertEqual(calls[0], ["launchctl", "enable", f"gui/{os.getuid()}/net.a-wan.pipelines"])
self.assertIn(["launchctl", "kickstart", f"gui/{os.getuid()}/net.a-wan.pipelines"], calls)
def test_report_prints_the_server_block_and_access(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / "logs").mkdir()
(root / "logs/server.log").write_text(
"== Pipelines server v1\n API x\n==\nold\n"
"== Pipelines server v2\n API y\n== Backends\n Chat opencode\n==\nlater line\n")
(root / "app-web").mkdir()
(root / "app-web/index.html").write_text("app")
(root / "updates").mkdir()
config = self.config(root)
config["web"] = {"host": "0.0.0.0", "port": 8082, "auth_file": "/private/auth.json",
"root": str(root / "app-web"), "updates": str(root / "updates")}
config["opencode_env"] = {"PIPELINES_WEB_PUBLIC_URL": "http://10.0.0.2:8082"}
output = io.StringIO()
with redirect_stdout(output):
manage.report(config)
text = output.getvalue()
self.assertIn("== Pipelines server v2\n API y\n== Backends\n Chat opencode\n==\n", text)
self.assertNotIn("v1", text)
self.assertIn("http://10.0.0.2:8082 (password in /private/auth.json)", text)
self.assertIn("Web app installed", text)
self.assertIn("not built; run ./install --apk", text)
def test_supervisor_spawns_authenticated_web_and_cleans_up_on_exit(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / "logs").mkdir()
config = self.config(root)
config["web"] = {"node": "/bin/node", "script": "/prefix/lib/web/serve.mjs",
"host": "0.0.0.0", "port": 8082, "auth_file": "/private/auth.json"}
"host": "0.0.0.0", "port": 8082, "auth_file": "/private/auth.json",
"root": "/prefix/lib/app-web", "updates": "/prefix/lib/app-updates"}
ntfy, server, web = MagicMock(), MagicMock(), MagicMock()
ntfy.poll.return_value = None
server.poll.return_value = None
@@ -196,7 +221,9 @@ class ManageNativeTests(unittest.TestCase):
manage.supervise(config)
call = popen.call_args_list[1]
self.assertEqual(call.args[0], ["/bin/node", "/prefix/lib/web/serve.mjs", "--port", "8082",
"--host", "0.0.0.0", "--pipelines", "127.0.0.1:8080"])
"--host", "0.0.0.0", "--pipelines", "127.0.0.1:8080",
"--root", "/prefix/lib/app-web",
"--updates", "/prefix/lib/app-updates"])
self.assertNotIn("PIPELINES_WEB_AUTH", call.kwargs["env"])
self.assertEqual(call.kwargs["env"]["PIPELINES_WEB_AUTH_FILE"], "/private/auth.json")
web.terminate.assert_called_once()
+31 -2
View File
@@ -116,7 +116,8 @@ class SetupNativeTests(unittest.TestCase):
# user configuration during tests.
return setup_native.Setup(directory, prefix=Path(directory) / "prefix", runner=recorder,
resolver=resolution,
environ={"XDG_CONFIG_HOME": str(Path(directory) / "config")})
environ={"XDG_CONFIG_HOME": str(Path(directory) / "config"),
"PIPELINES_WEB_PASSWORD": "chosen-password"})
@staticmethod
def which_all(opencode="/usr/bin/opencode", pi=None, cc="/usr/bin/cc"):
@@ -202,11 +203,39 @@ class SetupNativeTests(unittest.TestCase):
installer.configure_web(prefix, setup_native.arguments([]))
self.assertEqual(auth.read_bytes(), before)
self.assertEqual(auth.stat().st_mode & 0o777, 0o600)
self.assertGreaterEqual(len(json.loads(before)["password"]), 8)
self.assertEqual(json.loads(before)["password"], "chosen-password")
self.assertTrue(Path(web["script"]).is_file())
# The web app is built in Docker once and installed for the proxy.
self.assertTrue(installer.runner.has("docker", "run", "pipelines-app-build", "web"))
self.assertEqual(web["root"], str(prefix / "lib/app-web"))
self.assertTrue(Path(web["root"]).is_dir())
self.assertTrue(Path(web["updates"]).is_dir())
self.assertTrue((prefix / "lib/web/src/proxySecurity.js").is_file())
self.assertEqual(web["port"], 8082)
def test_new_web_password_must_be_chosen(self):
with tempfile.TemporaryDirectory() as directory:
installer = self.installer(directory, Recorder())
prefix = Path(directory) / "prefix"
installer.prepare_layout(prefix)
installer.environ.pop("PIPELINES_WEB_PASSWORD")
with patch.object(setup_native.shutil, "which", side_effect=self.which_all()), \
patch.object(setup_native.sys.stdin, "isatty", return_value=False):
with self.assertRaisesRegex(ValueError, "PIPELINES_WEB_PASSWORD"):
installer.configure_web(prefix, setup_native.arguments([]))
installer.environ["PIPELINES_WEB_PASSWORD"] = "short"
with patch.object(setup_native.shutil, "which", side_effect=self.which_all()):
with self.assertRaisesRegex(ValueError, "at least 8"):
installer.configure_web(prefix, setup_native.arguments([]))
installer.environ.pop("PIPELINES_WEB_PASSWORD")
with patch.object(setup_native.shutil, "which", side_effect=self.which_all()), \
patch.object(setup_native.sys.stdin, "isatty", return_value=True), \
patch.object(setup_native.getpass, "getpass",
side_effect=["short", "typed-password", "other-password",
"typed-password", "typed-password"]):
web = installer.configure_web(prefix, setup_native.arguments([]))
self.assertEqual(json.loads(Path(web["auth_file"]).read_text())["password"], "typed-password")
def test_invalid_web_auth_is_retained(self):
with tempfile.TemporaryDirectory() as directory:
installer = self.installer(directory, Recorder())