diff --git a/app/.flutter-version b/app/.flutter-version new file mode 100644 index 00000000..d1bf8a44 --- /dev/null +++ b/app/.flutter-version @@ -0,0 +1 @@ +3.44.6 diff --git a/app/docker/Dockerfile b/app/docker/Dockerfile new file mode 100644 index 00000000..58d13e72 --- /dev/null +++ b/app/docker/Dockerfile @@ -0,0 +1,43 @@ +# Flutter build images for the Pipelines app. `web` builds the web client the +# installer serves; `android` adds a JDK and the Android SDK for the optional +# APK. The Flutter version is the app's own pin (app/.flutter-version). +# +# Built and run by scripts/app_build.py; the container runs as the invoking +# user, so the SDK and cache directories are world-writable. +FROM debian:bookworm-slim AS web +ARG FLUTTER_VERSION +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl git unzip xz-utils zip \ + && rm -rf /var/lib/apt/lists/* +RUN test -n "$FLUTTER_VERSION" \ + && git clone --depth 1 --branch "$FLUTTER_VERSION" https://github.com/flutter/flutter.git /opt/flutter \ + && git config --system --add safe.directory '*' +ENV PATH=/opt/flutter/bin:$PATH \ + HOME=/home/builder \ + PUB_CACHE=/home/builder/.pub-cache \ + FLUTTER_SUPPRESS_ANALYTICS=true +RUN mkdir -p /home/builder/.pub-cache \ + && flutter --disable-analytics && flutter config --no-cli-animations \ + && flutter precache --web --no-android --no-ios --no-linux --no-macos --no-windows --no-fuchsia \ + && chmod -R a+rwX /opt/flutter /home/builder +COPY build.sh /usr/local/bin/pipelines-app-build +WORKDIR /tmp + +FROM web AS android +ARG ANDROID_CMDLINE_TOOLS=13114758 +RUN apt-get update \ + && apt-get install -y --no-install-recommends openjdk-17-jdk-headless \ + && rm -rf /var/lib/apt/lists/* +ENV ANDROID_HOME=/opt/android-sdk \ + ANDROID_SDK_ROOT=/opt/android-sdk \ + GRADLE_USER_HOME=/home/builder/.gradle +ENV PATH=$ANDROID_HOME/cmdline-tools/latest/bin:$PATH +RUN mkdir -p $ANDROID_HOME/cmdline-tools /home/builder/.gradle /home/builder/.android \ + && curl -fsSL -o /tmp/tools.zip "https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_CMDLINE_TOOLS}_latest.zip" \ + && unzip -q /tmp/tools.zip -d /tmp/tools && mv /tmp/tools/cmdline-tools $ANDROID_HOME/cmdline-tools/latest \ + && rm -rf /tmp/tools /tmp/tools.zip \ + && yes | sdkmanager --licenses >/dev/null \ + && sdkmanager "platform-tools" >/dev/null \ + && flutter config --android-sdk $ANDROID_HOME \ + && flutter precache --android \ + && chmod -R a+rwX /opt/android-sdk /opt/flutter /home/builder diff --git a/app/docker/build.sh b/app/docker/build.sh new file mode 100755 index 00000000..0e8e6506 --- /dev/null +++ b/app/docker/build.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# Builds the app from the read-only source at /src into /out. +# Usage: pipelines-app-build web|apk [flutter build arguments...] +set -eu +kind="$1" +shift +rm -rf /tmp/work && mkdir -p /tmp/work +# A private copy: the host checkout's build outputs and machine paths stay out. +(cd /src && tar --exclude=./build --exclude=./.dart_tool --exclude=./android/.gradle \ + --exclude=./android/local.properties --exclude=./android/app/.cxx --exclude=./ios/Pods \ + -cf - .) | tar -C /tmp/work -xf - +cd /tmp/work +flutter pub get +case "$kind" in + web) + flutter build web --release --no-web-resources-cdn "$@" + cp -R build/web/. /out/ + ;; + apk) + flutter build apk --release "$@" + cp build/app/outputs/flutter-apk/app-release.apk /out/pipelines.apk + ;; + *) + echo "unknown build: $kind" >&2 + exit 2 + ;; +esac diff --git a/app/lib/features/updates/app_updates.dart b/app/lib/features/updates/app_updates.dart index 30251606..260f7438 100644 --- a/app/lib/features/updates/app_updates.dart +++ b/app/lib/features/updates/app_updates.dart @@ -38,15 +38,29 @@ class AppUpdate { } } -/// Where a server's app builds live: the preview origin beside it (same host, -/// port 8096) under `/updates/`. -Uri? updatesBaseFor(Uri endpoint) { +/// Where a build set its update feed (`--dart-define=PIPELINES_UPDATES_URL`), +/// blank for the default. +const configuredUpdatesUrl = String.fromEnvironment('PIPELINES_UPDATES_URL'); + +/// The update feed for [endpoint]: `/updates/` on the server's own web origin +/// (the installed web proxy serves the APK there), or [configured] when the +/// build named another feed. +Uri? updatesBaseFor(Uri endpoint, {String configured = configuredUpdatesUrl}) { + final override = configured.trim(); + if (override.isNotEmpty) { + final parsed = Uri.tryParse(override); + if (parsed != null && parsed.hasScheme && parsed.host.isNotEmpty) { + return parsed.path.endsWith('/') + ? parsed + : parsed.replace(path: '${parsed.path}/'); + } + } if (endpoint.host.isEmpty) return null; final secure = endpoint.scheme == 'wss' || endpoint.scheme == 'https'; return Uri( scheme: secure ? 'https' : 'http', host: endpoint.host, - port: 8096, + port: endpoint.hasPort ? endpoint.port : null, path: '/updates/', ); } diff --git a/app/test/features/updates/app_updates_test.dart b/app/test/features/updates/app_updates_test.dart index f5d173d5..bc2126b9 100644 --- a/app/test/features/updates/app_updates_test.dart +++ b/app/test/features/updates/app_updates_test.dart @@ -2,9 +2,29 @@ import 'package:flutter_test/flutter_test.dart'; import 'package:pipelines_app/features/updates/app_updates.dart'; void main() { - test('builds live beside the server, under the preview origin', () { + test('builds live on the server\'s own web origin', () { expect( - updatesBaseFor(Uri.parse('wss://host.ts.net:8443/connect')).toString(), + updatesBaseFor( + Uri.parse('wss://host.ts.net:8443/connect'), + configured: '', + ).toString(), + 'https://host.ts.net:8443/updates/', + ); + expect( + updatesBaseFor( + Uri.parse('ws://192.168.1.5:8082/connect'), + configured: '', + ).toString(), + 'http://192.168.1.5:8082/updates/', + ); + }); + + test('a build can name another update feed', () { + expect( + updatesBaseFor( + Uri.parse('wss://host.ts.net:8443/connect'), + configured: 'https://host.ts.net:8096/updates', + ).toString(), 'https://host.ts.net:8096/updates/', ); }); diff --git a/app/tool/publish_apk.sh b/app/tool/publish_apk.sh index 51d3dc2f..23f95df1 100644 --- a/app/tool/publish_apk.sh +++ b/app/tool/publish_apk.sh @@ -18,7 +18,8 @@ export ANDROID_HOME="${ANDROID_HOME:-$HOME/Library/Android/sdk}" code=$(( $(date +%s) / 60 )) name="0.1.$code" "$FLUTTER" build apk --release --build-number="$code" --build-name="$name" \ - --dart-define=PIPELINES_SERVER_URL="$server" + --dart-define=PIPELINES_SERVER_URL="$server" \ + --dart-define=PIPELINES_UPDATES_URL="${PIPELINES_UPDATES_URL:-https://alans-macbook-pro.tail6bf4dc.ts.net:8096/updates/}" mkdir -p build/updates cp build/app/outputs/flutter-apk/app-release.apk build/updates/pipelines.apk cat > build/updates/version.json < 12 { + commit = commit[:12] + } + if s.Build.Dirty != nil && *s.Build.Dirty { + commit += " (modified)" + } + row := func(label, value string) { fmt.Fprintf(w, " %-10s %s\n", label, value) } + fmt.Fprintf(w, "== Pipelines server %s\n", version) + if commit != "" { + row("Commit", commit) + } + row("API", "http://"+s.Addr+" (WebSocket /connect)") + if s.WebURL != "" { + row("Web", s.WebURL) + } + row("Database", s.Database) + row("Artifacts", s.Artifacts) + if s.Authoring { + row("Authoring", "enabled (pipelines can be published)") + } else { + row("Authoring", "disabled (execution only)") + } + fmt.Fprintln(w, "== Backends") + opencode := s.OpenCode.Health + switch s.OpenCode.Health { + case "endpoint": + opencode = "attached to " + s.OpenCode.URL + case "spawned": + opencode = "own instance at " + s.OpenCode.URL + case "fallback": + opencode = s.OpenCode.URL + " unreachable; using the server's own instance" + } + row("OpenCode", opencode) + if len(s.Chat) == 0 { + row("Chat", "unavailable") + } else { + row("Chat", strings.Join(s.Chat, ", ")) + } + if s.HarnessErr != nil { + row("Harnesses", "unreadable: "+s.HarnessErr.Error()) + } else if len(s.Harnesses.Profiles) == 0 { + row("Harnesses", "none configured") + } else { + names := make([]string, 0, len(s.Harnesses.Profiles)) + for name := range s.Harnesses.Profiles { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + profile := s.Harnesses.Profiles[name] + label := "" + if name == s.Harnesses.Default { + label = "Default" + } + detail := name + if model := firstNonEmpty(profile.PipelineModel, profile.Model); model != "" { + detail += " · " + model + } + if profile.Mode != "" { + detail += " · " + profile.Mode + } + if profile.Executable != "" { + detail += " · " + profile.Executable + } + if label == "" { + label = "Harness" + } + row(label, detail) + } + } + fmt.Fprintln(w, "==") +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + return "" +} + +func printBanner(s startup) { + s.Harnesses, s.HarnessErr = harness.List() + s.WebURL = os.Getenv("PIPELINES_WEB_PUBLIC_URL") + banner(os.Stdout, s) +} diff --git a/cmd/pipelines-server/main.go b/cmd/pipelines-server/main.go index d6c44b46..59c3d648 100644 --- a/cmd/pipelines-server/main.go +++ b/cmd/pipelines-server/main.go @@ -12,6 +12,7 @@ import ( "syscall" "time" + "git.a-wan.net/alan/pipelines/internal/buildinfo" "git.a-wan.net/alan/pipelines/internal/opencode" "git.a-wan.net/alan/pipelines/internal/paths" "git.a-wan.net/alan/pipelines/internal/pipeline" @@ -142,6 +143,12 @@ func main() { log.Printf("shutdown: %v", err) } }() + var chatNames []string + if len(chats.Backends()) > 0 { + chatNames = chats.Backends() + } + printBanner(startup{Build: buildinfo.Current(), Addr: *addr, Database: *db, Artifacts: artifactsRoot, + Authoring: *allowAuthoring, OpenCode: openCodeStatus, Chat: chatNames}) log.Printf("listening on %s (WebSocket endpoint: /connect)", *addr) if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { log.Fatal(err) diff --git a/install.sh b/install.sh index 627d5d31..73bad717 100755 --- a/install.sh +++ b/install.sh @@ -17,6 +17,17 @@ case "$mode" in if [ "$#" -gt 0 ]; then shift; fi exec python3 scripts/setup-docker.py "$@" ;; + --web|web) + # Rebuild the web app in Docker (reused while app sources are unchanged). + shift + exec python3 scripts/app_build.py web "$@" + ;; + --apk|apk) + # The optional Android app: built in Docker with the Android SDK (slower), + # then offered to phones at /updates/ on the web address. + shift + exec python3 scripts/app_build.py apk "$@" + ;; --native|native) shift exec python3 scripts/setup-native.py "$@" diff --git a/scripts/app_build.py b/scripts/app_build.py new file mode 100644 index 00000000..f80e3208 --- /dev/null +++ b/scripts/app_build.py @@ -0,0 +1,231 @@ +"""Builds the Flutter app in Docker and reuses each build until its source changes. + +`web` is the browser client the installer serves behind the web login; `apk` +is the optional Android app, offered to phones through Check for updates. + + python3 scripts/app_build.py web [--prefix PREFIX] + python3 scripts/app_build.py apk [--prefix PREFIX] [--server-url URL] + +Each build is keyed by a digest of the app's sources, the Flutter version, the +build image definition and the build arguments. A finished build is kept under +~/.cache/pipelines/app-builds and reused whenever that key repeats, so an +unchanged app is never rebuilt. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import time +from urllib.parse import urlsplit + +REPO = Path(__file__).resolve().parents[1] +APP = REPO / "app" +DEFAULT_PREFIX = Path.home() / ".local/share/pipelines" + +def default_cache(): + return Path(os.environ.get("PIPELINES_APP_CACHE") or Path.home() / ".cache/pipelines/app-builds") +# Kept per build kind; older entries are pruned after a new build. +KEEP = 3 +# Inputs that change what a build produces. Generated and machine-local paths +# (build/, .dart_tool/, android/local.properties) are left out. +SOURCES = { + "web": ("lib", "web", "assets", "pubspec.yaml", "pubspec.lock", ".flutter-version", "docker"), + "apk": ("lib", "android", "assets", "pubspec.yaml", "pubspec.lock", ".flutter-version", "docker"), +} +SKIPPED = {"build", ".dart_tool", ".gradle", ".cxx", "local.properties", ".DS_Store"} + + +def flutter_version(app=APP): + return (app / ".flutter-version").read_text().strip() + + +def digest(kind, app=APP, extra=()): + """The build key: every source byte and path, plus [extra] arguments.""" + hasher = hashlib.sha256() + hasher.update(kind.encode()) + for item in extra: + hasher.update(b"\0arg\0" + item.encode()) + for name in SOURCES[kind]: + root = app / name + paths = [root] if root.is_file() else sorted(p for p in root.rglob("*") if p.is_file()) if root.is_dir() else [] + for path in paths: + relative = path.relative_to(app) + if SKIPPED.intersection(relative.parts): + continue + hasher.update(b"\0path\0" + str(relative).encode() + b"\0") + hasher.update(path.read_bytes()) + return hasher.hexdigest()[:20] + + +def docker(runner, *args, quiet=False): + result = runner(["docker", *args], capture_output=quiet, text=True) + if result.returncode: + detail = (result.stderr or "").strip().splitlines()[-1:] if quiet else [] + raise RuntimeError("docker " + args[0] + " failed" + (": " + detail[0] if detail else "")) + return result + + +def docker_available(runner=subprocess.run): + try: + return runner(["docker", "info", "--format", "{{.ServerVersion}}"], + capture_output=True, text=True, timeout=20).returncode == 0 + except (OSError, subprocess.SubprocessError): + return False + + +def image(kind, runner, app=APP): + """Builds (or reuses, via Docker's layer cache) the build image for [kind].""" + target = "android" if kind == "apk" else "web" + version = flutter_version(app) + tag = f"pipelines-flutter-{target}:{version}" + args = ["build", "--target", target, "--build-arg", f"FLUTTER_VERSION={version}", "-t", tag, + "-f", str(app / "docker/Dockerfile")] + if kind == "apk": + # Android's build tools (aapt2) ship for x86-64 Linux only. + args += ["--platform", "linux/amd64"] + docker(runner, *args, str(app / "docker")) + return tag + + +def signing_dir(): + """One debug signing key for every APK, so a phone accepts each update. + + Seeded from the host's own Android debug key when there is one, so APKs + built before Docker builds existed keep updating.""" + directory = Path.home() / ".config/pipelines/android" + directory.mkdir(mode=0o700, parents=True, exist_ok=True) + keystore = directory / "debug.keystore" + host = Path.home() / ".android/debug.keystore" + if not keystore.exists() and host.is_file(): + shutil.copyfile(host, keystore) + os.chmod(keystore, 0o600) + return directory + + +def build(kind, runner=subprocess.run, app=APP, cache=None, arguments=(), log=print): + """The directory holding the finished [kind] build, built only when needed.""" + cache = Path(cache) if cache else default_cache() + key = digest(kind, app, arguments) + entry = cache / f"{kind}-{key}" + if (entry / ".complete").is_file(): + log(f"Reusing the {kind} build {key} (app sources unchanged).") + return entry + log(f"Building the app ({kind}) in Docker with Flutter {flutter_version(app)}; " + "the first build downloads the toolchain and takes several minutes.") + tag = image(kind, runner, app) + cache.mkdir(parents=True, exist_ok=True) + work = Path(tempfile.mkdtemp(prefix=f"{kind}-", dir=cache)) + try: + command = ["run", "--rm", "--user", f"{os.getuid()}:{os.getgid()}", + "-v", f"{app}:/src:ro", "-v", f"{work}:/out", + "-v", "pipelines-flutter-pub:/home/builder/.pub-cache"] + if kind == "apk": + command += ["--platform", "linux/amd64", + "-v", "pipelines-flutter-gradle:/home/builder/.gradle", + "-v", "pipelines-android-sdk:/opt/android-sdk", + "-v", f"{signing_dir()}:/home/builder/.android"] + docker(runner, *command, tag, "pipelines-app-build", kind, *arguments) + (work / ".complete").write_text(json.dumps({"key": key, "built": int(time.time())}) + "\n") + if entry.exists(): + shutil.rmtree(entry) + work.rename(entry) + except BaseException: + shutil.rmtree(work, ignore_errors=True) + raise + prune(cache, kind, keep=entry) + return entry + + +def prune(cache, kind, keep): + entries = sorted((p for p in cache.glob(f"{kind}-*") if (p / ".complete").is_file()), + key=lambda p: p.stat().st_mtime, reverse=True) + for stale in [p for p in entries if p != keep][KEEP - 1:]: + shutil.rmtree(stale, ignore_errors=True) + + +def install_tree(source, target): + """Replaces [target] with a copy of [source] without a half-written moment.""" + target.parent.mkdir(parents=True, exist_ok=True) + staging = target.with_name(target.name + ".next") + shutil.rmtree(staging, ignore_errors=True) + shutil.copytree(source, staging, ignore=shutil.ignore_patterns(".complete")) + previous = target.with_name(target.name + ".previous") + shutil.rmtree(previous, ignore_errors=True) + if target.exists(): + target.rename(previous) + staging.rename(target) + shutil.rmtree(previous, ignore_errors=True) + + +def phone_server_url(manifest): + """The phone's default server: the installation's authenticated web origin.""" + public = (manifest.get("opencode_env") or {}).get("PIPELINES_WEB_PUBLIC_URL", "") + parsed = urlsplit(public) + if parsed.scheme not in ("http", "https") or not parsed.netloc: + raise ValueError("The installation records no web address; pass --server-url ws(s)://host:port/connect") + return f"{'wss' if parsed.scheme == 'https' else 'ws'}://{parsed.netloc}/connect" + + +def load_manifest(prefix): + path = Path(prefix).expanduser().resolve() / "pipelines-install.json" + if not path.is_file(): + raise ValueError(f"No native Pipelines installation at {prefix}; run ./install first") + return json.loads(path.read_text()) + + +def build_apk(prefix, server_url=None, notes="", runner=subprocess.run, log=print): + """Builds the APK for this installation and offers it at /updates/.""" + manifest = load_manifest(prefix) + web = manifest.get("web") or {} + updates = web.get("updates") + if not updates: + raise ValueError("This installation predates app updates; rerun ./install, then ./install --apk") + server = server_url or phone_server_url(manifest) + # The build number is the source key's build date, so the same sources + # reuse one APK and every rebuild is newer than the last. + arguments = (f"--dart-define=PIPELINES_SERVER_URL={server}",) + output = build("apk", runner, arguments=arguments, log=log) + built = json.loads((output / ".complete").read_text())["built"] + code = built // 60 + target = Path(updates) + target.mkdir(parents=True, exist_ok=True) + shutil.copyfile(output / "pipelines.apk", target / "pipelines.apk.next") + os.replace(target / "pipelines.apk.next", target / "pipelines.apk") + (target / "version.json").write_text(json.dumps({ + "versionCode": code, "versionName": f"0.1.{code}", "apk": "pipelines.apk", "notes": notes}) + "\n") + log(f"APK ready: {target / 'pipelines.apk'} (build {code}, server {server}).") + log("Phones get it from Settings → App updates → Check for updates, or download " + f"{server.replace('wss://', 'https://').replace('ws://', 'http://').removesuffix('/connect')}/updates/pipelines.apk") + return target / "pipelines.apk" + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("kind", choices=("web", "apk")) + parser.add_argument("--prefix", type=Path, default=DEFAULT_PREFIX) + parser.add_argument("--server-url", help="APK default server (default: this installation's web address)") + parser.add_argument("--notes", default="", help="APK release notes shown in Check for updates") + args = parser.parse_args(argv) + if not docker_available(): + raise RuntimeError("Docker is required to build the app; start Docker and retry") + if args.kind == "apk": + build_apk(args.prefix, args.server_url, args.notes) + return + manifest = load_manifest(args.prefix) + web = manifest.get("web") or {} + if not web.get("root"): + raise ValueError("This installation does not serve the app; rerun ./install") + install_tree(build("web"), Path(web["root"])) + print(f"Web app installed at {web['root']}; reload the page.") + + +if __name__ == "__main__": + try: + main() + except (ValueError, RuntimeError, OSError) as error: + sys.exit(str(error)) diff --git a/scripts/manage.py b/scripts/manage.py index 3ee93ddd..ae74db17 100644 --- a/scripts/manage.py +++ b/scripts/manage.py @@ -268,6 +268,47 @@ def ntfy_stop(config): path.unlink(missing_ok=True) +def server_report(config): + """The server's own startup report: the last `== Pipelines server` block + it wrote to its log.""" + log = Path(config["prefix"]) / "logs/server.log" + try: + with log.open("rb") as handle: + handle.seek(max(0, log.stat().st_size - 64 * 1024)) + lines = handle.read().decode("utf-8", "replace").splitlines() + except OSError: + return [] + starts = [i for i, line in enumerate(lines) if line.startswith("== Pipelines server")] + if not starts: + return [] + block = [] + for line in lines[starts[-1]:]: + block.append(line) + if line == "==": + break + return block + + +def report(config): + """Prints what is running: server, backends, web app and phone access.""" + for line in server_report(config): + print(line) + web = config.get("web") + if web: + public = (config.get("opencode_env") or {}).get("PIPELINES_WEB_PUBLIC_URL") + print("== Access") + address = public or f"http://{web['host']}:{web['port']}" + print(f" {'Web':<10} {address} (password in {web['auth_file']})") + root = Path(web["root"]) if web.get("root") else None + built = root is not None and (root / "index.html").is_file() + print(f" {'Web app':<10} {'installed' if built else 'not built; run ./install --web'}") + updates = Path(web["updates"]) if web.get("updates") else None + apk = updates is not None and (updates / "pipelines.apk").is_file() + print(f" {'Phone app':<10} " + (f"{updates / 'pipelines.apk'} (Settings → Check for updates)" + if apk else "not built; run ./install --apk")) + print("==") + + def supervise(config): root = Path(config["prefix"]) with (root / "supervisor.lock").open("a") as lock: @@ -320,7 +361,9 @@ def supervise(config): with (root / "logs/web.log").open("a") as web_log: web_child = subprocess.Popen( [web["node"], web["script"], "--port", str(web["port"]), - "--host", web["host"], "--pipelines", f"127.0.0.1:{config['server_port']}"], + "--host", web["host"], "--pipelines", f"127.0.0.1:{config['server_port']}", + *(["--root", web["root"]] if web.get("root") else []), + *(["--updates", web["updates"]] if web.get("updates") else [])], env=web_env, stdout=web_log, stderr=web_log) while child.poll() is None and not stopped: if web_child is not None and web_child.poll() is not None: @@ -419,11 +462,14 @@ def main(): parser.error("unexpected arguments: " + " ".join(rest)) if args.action == "start": start(config) + report(config) elif args.action == "stop": stop(config) elif args.action == "restart": stop(config); start(config) + report(config) elif args.action in ("status", "doctor"): + report(config) print("Pipeline server:", "healthy" if healthy(f"http://127.0.0.1:{config['server_port']}/healthz") else "stopped/unhealthy") print("ntfy:", "healthy" if healthy(ntfy_health_url(config)) else "stopped/unhealthy") print("Phone server:", config["public_url"], "· topic:", config["topic"], "· username:", config["username"]) diff --git a/scripts/setup-native.py b/scripts/setup-native.py index 26beaf89..598cc8d5 100644 --- a/scripts/setup-native.py +++ b/scripts/setup-native.py @@ -17,6 +17,7 @@ import ipaddress import json import os import platform +import getpass import plistlib import secrets import shlex @@ -41,6 +42,7 @@ configure = module("configure_compose", "configure-compose.py") lan = module("lan_address", "lan-address.py") release_lookup = module("resolve_releases", "resolve-releases.py") tokens = module("save_ntfy_token", "save-ntfy-token.py") +app_build = module("app_build", "app_build.py") DEFAULT_PREFIX = Path.home() / ".local/share/pipelines" MARKER = "pipelines-install.json" @@ -325,14 +327,53 @@ class Setup: if auth.stat().st_mode & 0o077: raise ValueError(f"Web auth file must be private (chmod 600): {auth}") else: + password = self.choose_password() auth.parent.mkdir(mode=0o700, parents=True, exist_ok=True) - self._write_atomic(auth, (json.dumps({"password": secrets.token_urlsafe(18), + self._write_atomic(auth, (json.dumps({"password": password, "secret": secrets.token_urlsafe(48)}) + "\n").encode(), 0o600) target = prefix / "lib/web" shutil.copytree(Path(__file__).resolve().parents[1] / "web", target, dirs_exist_ok=True, ignore=shutil.ignore_patterns("test", "node_modules", ".DS_Store")) + updates = prefix / "lib/app-updates" + updates.mkdir(parents=True, exist_ok=True) return {"node": node, "script": str(target / "serve.mjs"), - "auth_file": str(auth), "host": args.web_host, "port": args.web_port} + "auth_file": str(auth), "host": args.web_host, "port": args.web_port, + "root": self.install_app_web(prefix, args), "updates": str(updates)} + + def choose_password(self): + """The web sign-in password for a new computer, chosen by the operator. + + PIPELINES_WEB_PASSWORD answers it without a terminal (automation).""" + given = self.environ.get("PIPELINES_WEB_PASSWORD") + if given is not None: + if len(given) < 8: + raise ValueError("PIPELINES_WEB_PASSWORD must be at least 8 characters") + return given + if not sys.stdin.isatty(): + raise ValueError("Choose the web sign-in password: rerun ./install in a terminal, " + "or set PIPELINES_WEB_PASSWORD (at least 8 characters)") + print("This computer has no web sign-in yet. Choose the password for the web app and phone.") + while True: + first = getpass.getpass("Web password (at least 8 characters): ") + if len(first) < 8: + print("Too short; use at least 8 characters.") + continue + if getpass.getpass("Repeat the password: ") != first: + print("The passwords did not match; try again.") + continue + return first + + def install_app_web(self, prefix, args): + """Builds the Flutter web app in Docker (reused while its sources are + unchanged) and installs it for the web proxy. Without Docker the web + page explains how to build it later (./install --web).""" + target = prefix / "lib/app-web" + if not app_build.docker_available(self.runner): + print("WARNING: Docker is not running, so the web app was not built. " + "Start Docker, then run ./install --web.") + return str(target) + app_build.install_tree(app_build.build("web", runner=self.runner), target) + return str(target) # --- build and skill ------------------------------------------------- @@ -750,6 +791,8 @@ class Setup: f"(sign-in password in {web['auth_file']}; existing credentials retained).") print("Login autostart is already enabled. To enable/start it again, run:\n " + shlex.join([str(prefix / "bin/pipelines"), "service", "enable"])) + print("Android app (optional, slower): ./install --apk builds it in Docker and offers it at " + f"{opencode_env['PIPELINES_WEB_PUBLIC_URL']}/updates/.") print("Configured server role: " + ("single-user authoring enabled (web pipeline creation/publication allowed)." if allow_authoring else "execution-only (pipeline publication disabled).")) if args.bind_address == "0.0.0.0": diff --git a/scripts/test_manage_native.py b/scripts/test_manage_native.py index 4738e557..0f248de7 100644 --- a/scripts/test_manage_native.py +++ b/scripts/test_manage_native.py @@ -177,13 +177,38 @@ class ManageNativeTests(unittest.TestCase): self.assertEqual(calls[0], ["launchctl", "enable", f"gui/{os.getuid()}/net.a-wan.pipelines"]) self.assertIn(["launchctl", "kickstart", f"gui/{os.getuid()}/net.a-wan.pipelines"], calls) + def test_report_prints_the_server_block_and_access(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "logs").mkdir() + (root / "logs/server.log").write_text( + "== Pipelines server v1\n API x\n==\nold\n" + "== Pipelines server v2\n API y\n== Backends\n Chat opencode\n==\nlater line\n") + (root / "app-web").mkdir() + (root / "app-web/index.html").write_text("app") + (root / "updates").mkdir() + config = self.config(root) + config["web"] = {"host": "0.0.0.0", "port": 8082, "auth_file": "/private/auth.json", + "root": str(root / "app-web"), "updates": str(root / "updates")} + config["opencode_env"] = {"PIPELINES_WEB_PUBLIC_URL": "http://10.0.0.2:8082"} + output = io.StringIO() + with redirect_stdout(output): + manage.report(config) + text = output.getvalue() + self.assertIn("== Pipelines server v2\n API y\n== Backends\n Chat opencode\n==\n", text) + self.assertNotIn("v1", text) + self.assertIn("http://10.0.0.2:8082 (password in /private/auth.json)", text) + self.assertIn("Web app installed", text) + self.assertIn("not built; run ./install --apk", text) + def test_supervisor_spawns_authenticated_web_and_cleans_up_on_exit(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) (root / "logs").mkdir() config = self.config(root) config["web"] = {"node": "/bin/node", "script": "/prefix/lib/web/serve.mjs", - "host": "0.0.0.0", "port": 8082, "auth_file": "/private/auth.json"} + "host": "0.0.0.0", "port": 8082, "auth_file": "/private/auth.json", + "root": "/prefix/lib/app-web", "updates": "/prefix/lib/app-updates"} ntfy, server, web = MagicMock(), MagicMock(), MagicMock() ntfy.poll.return_value = None server.poll.return_value = None @@ -196,7 +221,9 @@ class ManageNativeTests(unittest.TestCase): manage.supervise(config) call = popen.call_args_list[1] self.assertEqual(call.args[0], ["/bin/node", "/prefix/lib/web/serve.mjs", "--port", "8082", - "--host", "0.0.0.0", "--pipelines", "127.0.0.1:8080"]) + "--host", "0.0.0.0", "--pipelines", "127.0.0.1:8080", + "--root", "/prefix/lib/app-web", + "--updates", "/prefix/lib/app-updates"]) self.assertNotIn("PIPELINES_WEB_AUTH", call.kwargs["env"]) self.assertEqual(call.kwargs["env"]["PIPELINES_WEB_AUTH_FILE"], "/private/auth.json") web.terminate.assert_called_once() diff --git a/scripts/test_setup_native.py b/scripts/test_setup_native.py index a8b12e50..01f1e713 100644 --- a/scripts/test_setup_native.py +++ b/scripts/test_setup_native.py @@ -116,7 +116,8 @@ class SetupNativeTests(unittest.TestCase): # user configuration during tests. return setup_native.Setup(directory, prefix=Path(directory) / "prefix", runner=recorder, resolver=resolution, - environ={"XDG_CONFIG_HOME": str(Path(directory) / "config")}) + environ={"XDG_CONFIG_HOME": str(Path(directory) / "config"), + "PIPELINES_WEB_PASSWORD": "chosen-password"}) @staticmethod def which_all(opencode="/usr/bin/opencode", pi=None, cc="/usr/bin/cc"): @@ -202,11 +203,39 @@ class SetupNativeTests(unittest.TestCase): installer.configure_web(prefix, setup_native.arguments([])) self.assertEqual(auth.read_bytes(), before) self.assertEqual(auth.stat().st_mode & 0o777, 0o600) - self.assertGreaterEqual(len(json.loads(before)["password"]), 8) + self.assertEqual(json.loads(before)["password"], "chosen-password") self.assertTrue(Path(web["script"]).is_file()) + # The web app is built in Docker once and installed for the proxy. + self.assertTrue(installer.runner.has("docker", "run", "pipelines-app-build", "web")) + self.assertEqual(web["root"], str(prefix / "lib/app-web")) + self.assertTrue(Path(web["root"]).is_dir()) + self.assertTrue(Path(web["updates"]).is_dir()) self.assertTrue((prefix / "lib/web/src/proxySecurity.js").is_file()) self.assertEqual(web["port"], 8082) + def test_new_web_password_must_be_chosen(self): + with tempfile.TemporaryDirectory() as directory: + installer = self.installer(directory, Recorder()) + prefix = Path(directory) / "prefix" + installer.prepare_layout(prefix) + installer.environ.pop("PIPELINES_WEB_PASSWORD") + with patch.object(setup_native.shutil, "which", side_effect=self.which_all()), \ + patch.object(setup_native.sys.stdin, "isatty", return_value=False): + with self.assertRaisesRegex(ValueError, "PIPELINES_WEB_PASSWORD"): + installer.configure_web(prefix, setup_native.arguments([])) + installer.environ["PIPELINES_WEB_PASSWORD"] = "short" + with patch.object(setup_native.shutil, "which", side_effect=self.which_all()): + with self.assertRaisesRegex(ValueError, "at least 8"): + installer.configure_web(prefix, setup_native.arguments([])) + installer.environ.pop("PIPELINES_WEB_PASSWORD") + with patch.object(setup_native.shutil, "which", side_effect=self.which_all()), \ + patch.object(setup_native.sys.stdin, "isatty", return_value=True), \ + patch.object(setup_native.getpass, "getpass", + side_effect=["short", "typed-password", "other-password", + "typed-password", "typed-password"]): + web = installer.configure_web(prefix, setup_native.arguments([])) + self.assertEqual(json.loads(Path(web["auth_file"]).read_text())["password"], "typed-password") + def test_invalid_web_auth_is_retained(self): with tempfile.TemporaryDirectory() as directory: installer = self.installer(directory, Recorder()) diff --git a/web/serve.mjs b/web/serve.mjs index 891dfd40..4afc9b62 100644 --- a/web/serve.mjs +++ b/web/serve.mjs @@ -1,10 +1,12 @@ -// Zero-dependency static file server + WebSocket proxy for the spike. +// The authenticated web front door for Pipelines: the sign-in page, the Flutter +// web app, and a proxy to the Pipelines server, with no dependencies. // node serve.mjs [--port 8090] [--host 0.0.0.0] [--pipelines 127.0.0.1:8080] +// [--root DIR] [--updates DIR] // -// Serves this directory, proxies GET/Upgrade /connect and GET/HEAD /artifacts/ to -// the Pipelines server, so the page, the WebSocket and artifact files share one -// origin. The API and its artifacts live on the server's port; the page is served -// from another, so artifact URLs resolved against the page would 404 without this. +// Serves the app (--root, default ../app/build/web) and proxies /connect, +// /artifacts/ and /chat-uploads to the Pipelines server, so the page, the +// WebSocket and artifact files share one origin. --updates serves the Android +// app's update feed at /updates/. // // Why the proxy exists: internal/server/websocket.go calls // `websocket.Accept(w, r, nil)` (github.com/coder/websocket), which rejects an @@ -17,7 +19,7 @@ // The default host is 127.0.0.1. Use --host 0.0.0.0 only on a trusted LAN/VPN. import { createServer, request } from "node:http"; import { createHmac, timingSafeEqual } from "node:crypto"; -import { readFileSync } from "node:fs"; +import { createReadStream, readFileSync } from "node:fs"; import { homedir } from "node:os"; import { readFile, stat } from "node:fs/promises"; import { extname, join, normalize, resolve, sep } from "node:path"; @@ -25,7 +27,13 @@ import { fileURLToPath } from "node:url"; import net from "node:net"; import { sameOriginUpgrade } from "./src/proxySecurity.js"; -const ROOT = resolve(fileURLToPath(new URL(".", import.meta.url))); +// The proxy's own directory holds the sign-in page's icons. +const HERE = resolve(fileURLToPath(new URL(".", import.meta.url))); +// The page root: `--root DIR` (the installer's Flutter web build), by default +// the checkout's app/build/web. `--updates DIR` serves the Android app's +// version.json and APK at /updates/, behind the same sign-in. +const ROOT = resolve(flag(process.argv, "--root", "") || join(HERE, "..", "app", "build", "web")); +const UPDATES = flag(process.argv, "--updates", "") ? resolve(flag(process.argv, "--updates", "")) : null; // --------------------------------------------------------------------------- // Access control @@ -239,6 +247,14 @@ const MIME = { ".png": "image/png", ".ico": "image/x-icon", ".map": "application/json; charset=utf-8", + ".wasm": "application/wasm", + ".otf": "font/otf", + ".ttf": "font/ttf", + ".woff": "font/woff", + ".woff2": "font/woff2", + ".bin": "application/octet-stream", + ".frag": "application/octet-stream", + ".apk": "application/vnd.android.package-archive", }; // A path is asset-looking when its last segment ends in a known file type, and a @@ -288,9 +304,46 @@ function resolvePath(urlPath) { return target; } +/** One file of the Android app's update feed, or 404. */ +async function serveUpdate(req, res, pathname) { + let name; + try { + name = decodeURIComponent(pathname.slice("/updates/".length)); + } catch { + name = ""; + } + const path = resolve(join(UPDATES, name)); + const info = name && path.startsWith(UPDATES + sep) ? await stat(path).catch(() => null) : null; + if (!info?.isFile()) { + res.writeHead(404, { "Content-Type": "text/plain; charset=utf-8" }).end("Not found"); + return; + } + res.writeHead(200, { + "Content-Type": MIME[extname(path)] ?? "application/octet-stream", + "Content-Length": info.size, + "Cache-Control": "no-store", + }); + if (req.method === "HEAD") { + res.end(); + return; + } + createReadStream(path).pipe(res); +} + /** Send index.html (the page shell). The router reads the real URL in the browser. */ async function serveIndex(res) { - const html = await readFile(join(ROOT, "index.html"), "utf8"); + const html = await readFile(join(ROOT, "index.html"), "utf8").catch(() => null); + if (html === null) { + // The installer builds the app in Docker; without it there is no page yet. + res.writeHead(503, { "Content-Type": "text/html; charset=utf-8", "Cache-Control": "no-store" }); + res.end( + 'Pipelines' + + '' + + "

The web app is not built yet

" + + "

On the server, start Docker and run ./install --web, then reload.

", + ); + return; + } // A deep URL such as /runs/440 would otherwise resolve the shell's relative // `./src/app.js` against /runs/. A root keeps the shell's own asset URLs // absolute without changing the address bar (and without editing index.html). @@ -328,14 +381,16 @@ function proxyArtifact(req, res) { const server = createServer(async (req, res) => { const pathname = (req.url ?? "/").split("?")[0]; + // Brand icons are public so the sign-in page itself can show the app icon. + if (req.method === "GET" && (pathname === "/favicon.svg" || pathname === "/apple-touch-icon.png")) { + const body = await readFile(join(HERE, pathname.slice(1))).catch(() => null); + if (body) { + res.writeHead(200, { "Content-Type": MIME[extname(pathname)], "Cache-Control": "no-store" }).end(body); + return; + } + } if (auth) { - // Brand assets are public so the sign-in page itself can show the app icon. - if ( - req.method === "GET" && - (pathname === "/favicon.svg" || pathname === "/apple-touch-icon.png") - ) { - // fall through to the static handler below - } else if (pathname === "/login" && req.method === "GET") { + if (pathname === "/login" && req.method === "GET") { if (isAuthed(req)) { res.writeHead(302, { Location: "/" }).end(); return; @@ -392,6 +447,10 @@ const server = createServer(async (req, res) => { proxyArtifact(req, res); return; } + if (UPDATES && pathname.startsWith("/updates/") && (req.method === "GET" || req.method === "HEAD")) { + await serveUpdate(req, res, pathname); + return; + } if (pathname === "/favicon.ico") { res.writeHead(204).end(); // no icon; keeps the console free of a 404 return; @@ -497,4 +556,5 @@ server.listen(port, host, () => { console.log(`web spike: http://${shown}:${port}/ (bound ${host}:${port})`); console.log(`proxying /connect (ws) and /artifacts/ (http) -> ${targetHost}:${targetPort}`); console.log(auth ? "auth: sign-in required" : "auth: DISABLED (tests)"); + console.log(`serving ${ROOT}${UPDATES ? `; app updates from ${UPDATES}` : ""}`); }); diff --git a/web/test/serve.test.mjs b/web/test/serve.test.mjs new file mode 100644 index 00000000..e06972a0 --- /dev/null +++ b/web/test/serve.test.mjs @@ -0,0 +1,64 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { mkdtemp, mkdir, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import net from "node:net"; + +const here = fileURLToPath(new URL("..", import.meta.url)); + +async function freePort() { + const server = net.createServer().listen(0, "127.0.0.1"); + await new Promise((done) => server.once("listening", done)); + const { port } = server.address(); + await new Promise((done) => server.close(done)); + return port; +} + +async function start(args) { + const port = await freePort(); + const child = spawn(process.execPath, ["serve.mjs", "--port", String(port), ...args], { + cwd: here, + env: { ...process.env, PIPELINES_WEB_AUTH: "disabled" }, + stdio: ["ignore", "pipe", "inherit"], + }); + await new Promise((done) => child.stdout.once("data", done)); + return { base: `http://127.0.0.1:${port}`, stop: () => child.kill() }; +} + +test("serves the app root, deep routes and the update feed", async (t) => { + const dir = await mkdtemp(join(tmpdir(), "serve-")); + const root = join(dir, "app"); + const updates = join(dir, "updates"); + await mkdir(join(root, "canvaskit"), { recursive: true }); + await mkdir(updates); + await writeFile(join(root, "index.html"), "app"); + await writeFile(join(root, "canvaskit", "canvaskit.wasm"), "wasm"); + await writeFile(join(updates, "version.json"), '{"versionCode":1}'); + await writeFile(join(dir, "secret.txt"), "no"); + const server = await start(["--root", root, "--updates", updates]); + t.after(server.stop); + + const index = await fetch(`${server.base}/runs/12`); + assert.equal(index.status, 200); + assert.match(await index.text(), /app/); + const wasm = await fetch(`${server.base}/canvaskit/canvaskit.wasm`); + assert.equal(wasm.headers.get("content-type"), "application/wasm"); + const feed = await fetch(`${server.base}/updates/version.json`); + assert.deepEqual(await feed.json(), { versionCode: 1 }); + assert.equal((await fetch(`${server.base}/updates/%2e%2e%2fsecret.txt`)).status, 404); + assert.equal((await fetch(`${server.base}/updates/missing.apk`)).status, 404); + const icon = await fetch(`${server.base}/favicon.svg`); + assert.equal(icon.status, 200); +}); + +test("explains how to build the app when it is missing", async (t) => { + const dir = await mkdtemp(join(tmpdir(), "serve-")); + const server = await start(["--root", join(dir, "absent")]); + t.after(server.stop); + const page = await fetch(`${server.base}/`); + assert.equal(page.status, 503); + assert.match(await page.text(), /\.\/install --web/); +});