feat(install): build the Flutter web app in Docker, optional ./install --apk, chosen web password, startup report

This commit is contained in:
2026-10-23 21:45:14 +01:00
parent 3136132144
commit 7958d1830e
16 changed files with 776 additions and 29 deletions
+1
View File
@@ -0,0 +1 @@
3.44.6
+43
View File
@@ -0,0 +1,43 @@
# Flutter build images for the Pipelines app. `web` builds the web client the
# installer serves; `android` adds a JDK and the Android SDK for the optional
# APK. The Flutter version is the app's own pin (app/.flutter-version).
#
# Built and run by scripts/app_build.py; the container runs as the invoking
# user, so the SDK and cache directories are world-writable.
FROM debian:bookworm-slim AS web
ARG FLUTTER_VERSION
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates curl git unzip xz-utils zip \
&& rm -rf /var/lib/apt/lists/*
RUN test -n "$FLUTTER_VERSION" \
&& git clone --depth 1 --branch "$FLUTTER_VERSION" https://github.com/flutter/flutter.git /opt/flutter \
&& git config --system --add safe.directory '*'
ENV PATH=/opt/flutter/bin:$PATH \
HOME=/home/builder \
PUB_CACHE=/home/builder/.pub-cache \
FLUTTER_SUPPRESS_ANALYTICS=true
RUN mkdir -p /home/builder/.pub-cache \
&& flutter --disable-analytics && flutter config --no-cli-animations \
&& flutter precache --web --no-android --no-ios --no-linux --no-macos --no-windows --no-fuchsia \
&& chmod -R a+rwX /opt/flutter /home/builder
COPY build.sh /usr/local/bin/pipelines-app-build
WORKDIR /tmp
FROM web AS android
ARG ANDROID_CMDLINE_TOOLS=13114758
RUN apt-get update \
&& apt-get install -y --no-install-recommends openjdk-17-jdk-headless \
&& rm -rf /var/lib/apt/lists/*
ENV ANDROID_HOME=/opt/android-sdk \
ANDROID_SDK_ROOT=/opt/android-sdk \
GRADLE_USER_HOME=/home/builder/.gradle
ENV PATH=$ANDROID_HOME/cmdline-tools/latest/bin:$PATH
RUN mkdir -p $ANDROID_HOME/cmdline-tools /home/builder/.gradle /home/builder/.android \
&& curl -fsSL -o /tmp/tools.zip "https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_CMDLINE_TOOLS}_latest.zip" \
&& unzip -q /tmp/tools.zip -d /tmp/tools && mv /tmp/tools/cmdline-tools $ANDROID_HOME/cmdline-tools/latest \
&& rm -rf /tmp/tools /tmp/tools.zip \
&& yes | sdkmanager --licenses >/dev/null \
&& sdkmanager "platform-tools" >/dev/null \
&& flutter config --android-sdk $ANDROID_HOME \
&& flutter precache --android \
&& chmod -R a+rwX /opt/android-sdk /opt/flutter /home/builder
+27
View File
@@ -0,0 +1,27 @@
#!/bin/sh
# Builds the app from the read-only source at /src into /out.
# Usage: pipelines-app-build web|apk [flutter build arguments...]
set -eu
kind="$1"
shift
rm -rf /tmp/work && mkdir -p /tmp/work
# A private copy: the host checkout's build outputs and machine paths stay out.
(cd /src && tar --exclude=./build --exclude=./.dart_tool --exclude=./android/.gradle \
--exclude=./android/local.properties --exclude=./android/app/.cxx --exclude=./ios/Pods \
-cf - .) | tar -C /tmp/work -xf -
cd /tmp/work
flutter pub get
case "$kind" in
web)
flutter build web --release --no-web-resources-cdn "$@"
cp -R build/web/. /out/
;;
apk)
flutter build apk --release "$@"
cp build/app/outputs/flutter-apk/app-release.apk /out/pipelines.apk
;;
*)
echo "unknown build: $kind" >&2
exit 2
;;
esac
+18 -4
View File
@@ -38,15 +38,29 @@ class AppUpdate {
}
}
/// Where a server's app builds live: the preview origin beside it (same host,
/// port 8096) under `/updates/`.
Uri? updatesBaseFor(Uri endpoint) {
/// Where a build set its update feed (`--dart-define=PIPELINES_UPDATES_URL`),
/// blank for the default.
const configuredUpdatesUrl = String.fromEnvironment('PIPELINES_UPDATES_URL');
/// The update feed for [endpoint]: `/updates/` on the server's own web origin
/// (the installed web proxy serves the APK there), or [configured] when the
/// build named another feed.
Uri? updatesBaseFor(Uri endpoint, {String configured = configuredUpdatesUrl}) {
final override = configured.trim();
if (override.isNotEmpty) {
final parsed = Uri.tryParse(override);
if (parsed != null && parsed.hasScheme && parsed.host.isNotEmpty) {
return parsed.path.endsWith('/')
? parsed
: parsed.replace(path: '${parsed.path}/');
}
}
if (endpoint.host.isEmpty) return null;
final secure = endpoint.scheme == 'wss' || endpoint.scheme == 'https';
return Uri(
scheme: secure ? 'https' : 'http',
host: endpoint.host,
port: 8096,
port: endpoint.hasPort ? endpoint.port : null,
path: '/updates/',
);
}
@@ -2,9 +2,29 @@ import 'package:flutter_test/flutter_test.dart';
import 'package:pipelines_app/features/updates/app_updates.dart';
void main() {
test('builds live beside the server, under the preview origin', () {
test('builds live on the server\'s own web origin', () {
expect(
updatesBaseFor(Uri.parse('wss://host.ts.net:8443/connect')).toString(),
updatesBaseFor(
Uri.parse('wss://host.ts.net:8443/connect'),
configured: '',
).toString(),
'https://host.ts.net:8443/updates/',
);
expect(
updatesBaseFor(
Uri.parse('ws://192.168.1.5:8082/connect'),
configured: '',
).toString(),
'http://192.168.1.5:8082/updates/',
);
});
test('a build can name another update feed', () {
expect(
updatesBaseFor(
Uri.parse('wss://host.ts.net:8443/connect'),
configured: 'https://host.ts.net:8096/updates',
).toString(),
'https://host.ts.net:8096/updates/',
);
});
+2 -1
View File
@@ -18,7 +18,8 @@ export ANDROID_HOME="${ANDROID_HOME:-$HOME/Library/Android/sdk}"
code=$(( $(date +%s) / 60 ))
name="0.1.$code"
"$FLUTTER" build apk --release --build-number="$code" --build-name="$name" \
--dart-define=PIPELINES_SERVER_URL="$server"
--dart-define=PIPELINES_SERVER_URL="$server" \
--dart-define=PIPELINES_UPDATES_URL="${PIPELINES_UPDATES_URL:-https://alans-macbook-pro.tail6bf4dc.ts.net:8096/updates/}"
mkdir -p build/updates
cp build/app/outputs/flutter-apk/app-release.apk build/updates/pipelines.apk
cat > build/updates/version.json <<JSON
+123
View File
@@ -0,0 +1,123 @@
package main
import (
"fmt"
"io"
"os"
"sort"
"strings"
"git.a-wan.net/alan/pipelines/internal/buildinfo"
"git.a-wan.net/alan/pipelines/internal/harness"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// startup is what the server reports about itself once it is configured.
type startup struct {
Build buildinfo.Info
Addr string
Database string
Artifacts string
Authoring bool
OpenCode protocol.ServiceStatus
Chat []string
Harnesses harness.ListView
HarnessErr error
WebURL string
}
// banner writes the server and backend summary. `pipelines start` prints the
// latest one from the server log, so its first and last lines are markers.
func banner(w io.Writer, s startup) {
version := s.Build.Version
if version == "" {
version = "dev"
}
commit := s.Build.Commit
if len(commit) > 12 {
commit = commit[:12]
}
if s.Build.Dirty != nil && *s.Build.Dirty {
commit += " (modified)"
}
row := func(label, value string) { fmt.Fprintf(w, " %-10s %s\n", label, value) }
fmt.Fprintf(w, "== Pipelines server %s\n", version)
if commit != "" {
row("Commit", commit)
}
row("API", "http://"+s.Addr+" (WebSocket /connect)")
if s.WebURL != "" {
row("Web", s.WebURL)
}
row("Database", s.Database)
row("Artifacts", s.Artifacts)
if s.Authoring {
row("Authoring", "enabled (pipelines can be published)")
} else {
row("Authoring", "disabled (execution only)")
}
fmt.Fprintln(w, "== Backends")
opencode := s.OpenCode.Health
switch s.OpenCode.Health {
case "endpoint":
opencode = "attached to " + s.OpenCode.URL
case "spawned":
opencode = "own instance at " + s.OpenCode.URL
case "fallback":
opencode = s.OpenCode.URL + " unreachable; using the server's own instance"
}
row("OpenCode", opencode)
if len(s.Chat) == 0 {
row("Chat", "unavailable")
} else {
row("Chat", strings.Join(s.Chat, ", "))
}
if s.HarnessErr != nil {
row("Harnesses", "unreadable: "+s.HarnessErr.Error())
} else if len(s.Harnesses.Profiles) == 0 {
row("Harnesses", "none configured")
} else {
names := make([]string, 0, len(s.Harnesses.Profiles))
for name := range s.Harnesses.Profiles {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
profile := s.Harnesses.Profiles[name]
label := ""
if name == s.Harnesses.Default {
label = "Default"
}
detail := name
if model := firstNonEmpty(profile.PipelineModel, profile.Model); model != "" {
detail += " · " + model
}
if profile.Mode != "" {
detail += " · " + profile.Mode
}
if profile.Executable != "" {
detail += " · " + profile.Executable
}
if label == "" {
label = "Harness"
}
row(label, detail)
}
}
fmt.Fprintln(w, "==")
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
if value != "" {
return value
}
}
return ""
}
func printBanner(s startup) {
s.Harnesses, s.HarnessErr = harness.List()
s.WebURL = os.Getenv("PIPELINES_WEB_PUBLIC_URL")
banner(os.Stdout, s)
}
+7
View File
@@ -12,6 +12,7 @@ import (
"syscall"
"time"
"git.a-wan.net/alan/pipelines/internal/buildinfo"
"git.a-wan.net/alan/pipelines/internal/opencode"
"git.a-wan.net/alan/pipelines/internal/paths"
"git.a-wan.net/alan/pipelines/internal/pipeline"
@@ -142,6 +143,12 @@ func main() {
log.Printf("shutdown: %v", err)
}
}()
var chatNames []string
if len(chats.Backends()) > 0 {
chatNames = chats.Backends()
}
printBanner(startup{Build: buildinfo.Current(), Addr: *addr, Database: *db, Artifacts: artifactsRoot,
Authoring: *allowAuthoring, OpenCode: openCodeStatus, Chat: chatNames})
log.Printf("listening on %s (WebSocket endpoint: /connect)", *addr)
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
log.Fatal(err)
+11
View File
@@ -17,6 +17,17 @@ case "$mode" in
if [ "$#" -gt 0 ]; then shift; fi
exec python3 scripts/setup-docker.py "$@"
;;
--web|web)
# Rebuild the web app in Docker (reused while app sources are unchanged).
shift
exec python3 scripts/app_build.py web "$@"
;;
--apk|apk)
# The optional Android app: built in Docker with the Android SDK (slower),
# then offered to phones at /updates/ on the web address.
shift
exec python3 scripts/app_build.py apk "$@"
;;
--native|native)
shift
exec python3 scripts/setup-native.py "$@"
+231
View File
@@ -0,0 +1,231 @@
"""Builds the Flutter app in Docker and reuses each build until its source changes.
`web` is the browser client the installer serves behind the web login; `apk`
is the optional Android app, offered to phones through Check for updates.
python3 scripts/app_build.py web [--prefix PREFIX]
python3 scripts/app_build.py apk [--prefix PREFIX] [--server-url URL]
Each build is keyed by a digest of the app's sources, the Flutter version, the
build image definition and the build arguments. A finished build is kept under
~/.cache/pipelines/app-builds and reused whenever that key repeats, so an
unchanged app is never rebuilt.
"""
import argparse
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
import time
from urllib.parse import urlsplit
REPO = Path(__file__).resolve().parents[1]
APP = REPO / "app"
DEFAULT_PREFIX = Path.home() / ".local/share/pipelines"
def default_cache():
return Path(os.environ.get("PIPELINES_APP_CACHE") or Path.home() / ".cache/pipelines/app-builds")
# Kept per build kind; older entries are pruned after a new build.
KEEP = 3
# Inputs that change what a build produces. Generated and machine-local paths
# (build/, .dart_tool/, android/local.properties) are left out.
SOURCES = {
"web": ("lib", "web", "assets", "pubspec.yaml", "pubspec.lock", ".flutter-version", "docker"),
"apk": ("lib", "android", "assets", "pubspec.yaml", "pubspec.lock", ".flutter-version", "docker"),
}
SKIPPED = {"build", ".dart_tool", ".gradle", ".cxx", "local.properties", ".DS_Store"}
def flutter_version(app=APP):
return (app / ".flutter-version").read_text().strip()
def digest(kind, app=APP, extra=()):
"""The build key: every source byte and path, plus [extra] arguments."""
hasher = hashlib.sha256()
hasher.update(kind.encode())
for item in extra:
hasher.update(b"\0arg\0" + item.encode())
for name in SOURCES[kind]:
root = app / name
paths = [root] if root.is_file() else sorted(p for p in root.rglob("*") if p.is_file()) if root.is_dir() else []
for path in paths:
relative = path.relative_to(app)
if SKIPPED.intersection(relative.parts):
continue
hasher.update(b"\0path\0" + str(relative).encode() + b"\0")
hasher.update(path.read_bytes())
return hasher.hexdigest()[:20]
def docker(runner, *args, quiet=False):
result = runner(["docker", *args], capture_output=quiet, text=True)
if result.returncode:
detail = (result.stderr or "").strip().splitlines()[-1:] if quiet else []
raise RuntimeError("docker " + args[0] + " failed" + (": " + detail[0] if detail else ""))
return result
def docker_available(runner=subprocess.run):
try:
return runner(["docker", "info", "--format", "{{.ServerVersion}}"],
capture_output=True, text=True, timeout=20).returncode == 0
except (OSError, subprocess.SubprocessError):
return False
def image(kind, runner, app=APP):
"""Builds (or reuses, via Docker's layer cache) the build image for [kind]."""
target = "android" if kind == "apk" else "web"
version = flutter_version(app)
tag = f"pipelines-flutter-{target}:{version}"
args = ["build", "--target", target, "--build-arg", f"FLUTTER_VERSION={version}", "-t", tag,
"-f", str(app / "docker/Dockerfile")]
if kind == "apk":
# Android's build tools (aapt2) ship for x86-64 Linux only.
args += ["--platform", "linux/amd64"]
docker(runner, *args, str(app / "docker"))
return tag
def signing_dir():
"""One debug signing key for every APK, so a phone accepts each update.
Seeded from the host's own Android debug key when there is one, so APKs
built before Docker builds existed keep updating."""
directory = Path.home() / ".config/pipelines/android"
directory.mkdir(mode=0o700, parents=True, exist_ok=True)
keystore = directory / "debug.keystore"
host = Path.home() / ".android/debug.keystore"
if not keystore.exists() and host.is_file():
shutil.copyfile(host, keystore)
os.chmod(keystore, 0o600)
return directory
def build(kind, runner=subprocess.run, app=APP, cache=None, arguments=(), log=print):
"""The directory holding the finished [kind] build, built only when needed."""
cache = Path(cache) if cache else default_cache()
key = digest(kind, app, arguments)
entry = cache / f"{kind}-{key}"
if (entry / ".complete").is_file():
log(f"Reusing the {kind} build {key} (app sources unchanged).")
return entry
log(f"Building the app ({kind}) in Docker with Flutter {flutter_version(app)}; "
"the first build downloads the toolchain and takes several minutes.")
tag = image(kind, runner, app)
cache.mkdir(parents=True, exist_ok=True)
work = Path(tempfile.mkdtemp(prefix=f"{kind}-", dir=cache))
try:
command = ["run", "--rm", "--user", f"{os.getuid()}:{os.getgid()}",
"-v", f"{app}:/src:ro", "-v", f"{work}:/out",
"-v", "pipelines-flutter-pub:/home/builder/.pub-cache"]
if kind == "apk":
command += ["--platform", "linux/amd64",
"-v", "pipelines-flutter-gradle:/home/builder/.gradle",
"-v", "pipelines-android-sdk:/opt/android-sdk",
"-v", f"{signing_dir()}:/home/builder/.android"]
docker(runner, *command, tag, "pipelines-app-build", kind, *arguments)
(work / ".complete").write_text(json.dumps({"key": key, "built": int(time.time())}) + "\n")
if entry.exists():
shutil.rmtree(entry)
work.rename(entry)
except BaseException:
shutil.rmtree(work, ignore_errors=True)
raise
prune(cache, kind, keep=entry)
return entry
def prune(cache, kind, keep):
entries = sorted((p for p in cache.glob(f"{kind}-*") if (p / ".complete").is_file()),
key=lambda p: p.stat().st_mtime, reverse=True)
for stale in [p for p in entries if p != keep][KEEP - 1:]:
shutil.rmtree(stale, ignore_errors=True)
def install_tree(source, target):
"""Replaces [target] with a copy of [source] without a half-written moment."""
target.parent.mkdir(parents=True, exist_ok=True)
staging = target.with_name(target.name + ".next")
shutil.rmtree(staging, ignore_errors=True)
shutil.copytree(source, staging, ignore=shutil.ignore_patterns(".complete"))
previous = target.with_name(target.name + ".previous")
shutil.rmtree(previous, ignore_errors=True)
if target.exists():
target.rename(previous)
staging.rename(target)
shutil.rmtree(previous, ignore_errors=True)
def phone_server_url(manifest):
"""The phone's default server: the installation's authenticated web origin."""
public = (manifest.get("opencode_env") or {}).get("PIPELINES_WEB_PUBLIC_URL", "")
parsed = urlsplit(public)
if parsed.scheme not in ("http", "https") or not parsed.netloc:
raise ValueError("The installation records no web address; pass --server-url ws(s)://host:port/connect")
return f"{'wss' if parsed.scheme == 'https' else 'ws'}://{parsed.netloc}/connect"
def load_manifest(prefix):
path = Path(prefix).expanduser().resolve() / "pipelines-install.json"
if not path.is_file():
raise ValueError(f"No native Pipelines installation at {prefix}; run ./install first")
return json.loads(path.read_text())
def build_apk(prefix, server_url=None, notes="", runner=subprocess.run, log=print):
"""Builds the APK for this installation and offers it at /updates/."""
manifest = load_manifest(prefix)
web = manifest.get("web") or {}
updates = web.get("updates")
if not updates:
raise ValueError("This installation predates app updates; rerun ./install, then ./install --apk")
server = server_url or phone_server_url(manifest)
# The build number is the source key's build date, so the same sources
# reuse one APK and every rebuild is newer than the last.
arguments = (f"--dart-define=PIPELINES_SERVER_URL={server}",)
output = build("apk", runner, arguments=arguments, log=log)
built = json.loads((output / ".complete").read_text())["built"]
code = built // 60
target = Path(updates)
target.mkdir(parents=True, exist_ok=True)
shutil.copyfile(output / "pipelines.apk", target / "pipelines.apk.next")
os.replace(target / "pipelines.apk.next", target / "pipelines.apk")
(target / "version.json").write_text(json.dumps({
"versionCode": code, "versionName": f"0.1.{code}", "apk": "pipelines.apk", "notes": notes}) + "\n")
log(f"APK ready: {target / 'pipelines.apk'} (build {code}, server {server}).")
log("Phones get it from Settings → App updates → Check for updates, or download "
f"{server.replace('wss://', 'https://').replace('ws://', 'http://').removesuffix('/connect')}/updates/pipelines.apk")
return target / "pipelines.apk"
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("kind", choices=("web", "apk"))
parser.add_argument("--prefix", type=Path, default=DEFAULT_PREFIX)
parser.add_argument("--server-url", help="APK default server (default: this installation's web address)")
parser.add_argument("--notes", default="", help="APK release notes shown in Check for updates")
args = parser.parse_args(argv)
if not docker_available():
raise RuntimeError("Docker is required to build the app; start Docker and retry")
if args.kind == "apk":
build_apk(args.prefix, args.server_url, args.notes)
return
manifest = load_manifest(args.prefix)
web = manifest.get("web") or {}
if not web.get("root"):
raise ValueError("This installation does not serve the app; rerun ./install")
install_tree(build("web"), Path(web["root"]))
print(f"Web app installed at {web['root']}; reload the page.")
if __name__ == "__main__":
try:
main()
except (ValueError, RuntimeError, OSError) as error:
sys.exit(str(error))
+47 -1
View File
@@ -268,6 +268,47 @@ def ntfy_stop(config):
path.unlink(missing_ok=True)
def server_report(config):
"""The server's own startup report: the last `== Pipelines server` block
it wrote to its log."""
log = Path(config["prefix"]) / "logs/server.log"
try:
with log.open("rb") as handle:
handle.seek(max(0, log.stat().st_size - 64 * 1024))
lines = handle.read().decode("utf-8", "replace").splitlines()
except OSError:
return []
starts = [i for i, line in enumerate(lines) if line.startswith("== Pipelines server")]
if not starts:
return []
block = []
for line in lines[starts[-1]:]:
block.append(line)
if line == "==":
break
return block
def report(config):
"""Prints what is running: server, backends, web app and phone access."""
for line in server_report(config):
print(line)
web = config.get("web")
if web:
public = (config.get("opencode_env") or {}).get("PIPELINES_WEB_PUBLIC_URL")
print("== Access")
address = public or f"http://{web['host']}:{web['port']}"
print(f" {'Web':<10} {address} (password in {web['auth_file']})")
root = Path(web["root"]) if web.get("root") else None
built = root is not None and (root / "index.html").is_file()
print(f" {'Web app':<10} {'installed' if built else 'not built; run ./install --web'}")
updates = Path(web["updates"]) if web.get("updates") else None
apk = updates is not None and (updates / "pipelines.apk").is_file()
print(f" {'Phone app':<10} " + (f"{updates / 'pipelines.apk'} (Settings → Check for updates)"
if apk else "not built; run ./install --apk"))
print("==")
def supervise(config):
root = Path(config["prefix"])
with (root / "supervisor.lock").open("a") as lock:
@@ -320,7 +361,9 @@ def supervise(config):
with (root / "logs/web.log").open("a") as web_log:
web_child = subprocess.Popen(
[web["node"], web["script"], "--port", str(web["port"]),
"--host", web["host"], "--pipelines", f"127.0.0.1:{config['server_port']}"],
"--host", web["host"], "--pipelines", f"127.0.0.1:{config['server_port']}",
*(["--root", web["root"]] if web.get("root") else []),
*(["--updates", web["updates"]] if web.get("updates") else [])],
env=web_env, stdout=web_log, stderr=web_log)
while child.poll() is None and not stopped:
if web_child is not None and web_child.poll() is not None:
@@ -419,11 +462,14 @@ def main():
parser.error("unexpected arguments: " + " ".join(rest))
if args.action == "start":
start(config)
report(config)
elif args.action == "stop":
stop(config)
elif args.action == "restart":
stop(config); start(config)
report(config)
elif args.action in ("status", "doctor"):
report(config)
print("Pipeline server:", "healthy" if healthy(f"http://127.0.0.1:{config['server_port']}/healthz") else "stopped/unhealthy")
print("ntfy:", "healthy" if healthy(ntfy_health_url(config)) else "stopped/unhealthy")
print("Phone server:", config["public_url"], "· topic:", config["topic"], "· username:", config["username"])
+45 -2
View File
@@ -17,6 +17,7 @@ import ipaddress
import json
import os
import platform
import getpass
import plistlib
import secrets
import shlex
@@ -41,6 +42,7 @@ configure = module("configure_compose", "configure-compose.py")
lan = module("lan_address", "lan-address.py")
release_lookup = module("resolve_releases", "resolve-releases.py")
tokens = module("save_ntfy_token", "save-ntfy-token.py")
app_build = module("app_build", "app_build.py")
DEFAULT_PREFIX = Path.home() / ".local/share/pipelines"
MARKER = "pipelines-install.json"
@@ -325,14 +327,53 @@ class Setup:
if auth.stat().st_mode & 0o077:
raise ValueError(f"Web auth file must be private (chmod 600): {auth}")
else:
password = self.choose_password()
auth.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
self._write_atomic(auth, (json.dumps({"password": secrets.token_urlsafe(18),
self._write_atomic(auth, (json.dumps({"password": password,
"secret": secrets.token_urlsafe(48)}) + "\n").encode(), 0o600)
target = prefix / "lib/web"
shutil.copytree(Path(__file__).resolve().parents[1] / "web", target, dirs_exist_ok=True,
ignore=shutil.ignore_patterns("test", "node_modules", ".DS_Store"))
updates = prefix / "lib/app-updates"
updates.mkdir(parents=True, exist_ok=True)
return {"node": node, "script": str(target / "serve.mjs"),
"auth_file": str(auth), "host": args.web_host, "port": args.web_port}
"auth_file": str(auth), "host": args.web_host, "port": args.web_port,
"root": self.install_app_web(prefix, args), "updates": str(updates)}
def choose_password(self):
"""The web sign-in password for a new computer, chosen by the operator.
PIPELINES_WEB_PASSWORD answers it without a terminal (automation)."""
given = self.environ.get("PIPELINES_WEB_PASSWORD")
if given is not None:
if len(given) < 8:
raise ValueError("PIPELINES_WEB_PASSWORD must be at least 8 characters")
return given
if not sys.stdin.isatty():
raise ValueError("Choose the web sign-in password: rerun ./install in a terminal, "
"or set PIPELINES_WEB_PASSWORD (at least 8 characters)")
print("This computer has no web sign-in yet. Choose the password for the web app and phone.")
while True:
first = getpass.getpass("Web password (at least 8 characters): ")
if len(first) < 8:
print("Too short; use at least 8 characters.")
continue
if getpass.getpass("Repeat the password: ") != first:
print("The passwords did not match; try again.")
continue
return first
def install_app_web(self, prefix, args):
"""Builds the Flutter web app in Docker (reused while its sources are
unchanged) and installs it for the web proxy. Without Docker the web
page explains how to build it later (./install --web)."""
target = prefix / "lib/app-web"
if not app_build.docker_available(self.runner):
print("WARNING: Docker is not running, so the web app was not built. "
"Start Docker, then run ./install --web.")
return str(target)
app_build.install_tree(app_build.build("web", runner=self.runner), target)
return str(target)
# --- build and skill -------------------------------------------------
@@ -750,6 +791,8 @@ class Setup:
f"(sign-in password in {web['auth_file']}; existing credentials retained).")
print("Login autostart is already enabled. To enable/start it again, run:\n "
+ shlex.join([str(prefix / "bin/pipelines"), "service", "enable"]))
print("Android app (optional, slower): ./install --apk builds it in Docker and offers it at "
f"{opencode_env['PIPELINES_WEB_PUBLIC_URL']}/updates/.")
print("Configured server role: " + ("single-user authoring enabled (web pipeline creation/publication allowed)."
if allow_authoring else "execution-only (pipeline publication disabled)."))
if args.bind_address == "0.0.0.0":
+29 -2
View File
@@ -177,13 +177,38 @@ class ManageNativeTests(unittest.TestCase):
self.assertEqual(calls[0], ["launchctl", "enable", f"gui/{os.getuid()}/net.a-wan.pipelines"])
self.assertIn(["launchctl", "kickstart", f"gui/{os.getuid()}/net.a-wan.pipelines"], calls)
def test_report_prints_the_server_block_and_access(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / "logs").mkdir()
(root / "logs/server.log").write_text(
"== Pipelines server v1\n API x\n==\nold\n"
"== Pipelines server v2\n API y\n== Backends\n Chat opencode\n==\nlater line\n")
(root / "app-web").mkdir()
(root / "app-web/index.html").write_text("app")
(root / "updates").mkdir()
config = self.config(root)
config["web"] = {"host": "0.0.0.0", "port": 8082, "auth_file": "/private/auth.json",
"root": str(root / "app-web"), "updates": str(root / "updates")}
config["opencode_env"] = {"PIPELINES_WEB_PUBLIC_URL": "http://10.0.0.2:8082"}
output = io.StringIO()
with redirect_stdout(output):
manage.report(config)
text = output.getvalue()
self.assertIn("== Pipelines server v2\n API y\n== Backends\n Chat opencode\n==\n", text)
self.assertNotIn("v1", text)
self.assertIn("http://10.0.0.2:8082 (password in /private/auth.json)", text)
self.assertIn("Web app installed", text)
self.assertIn("not built; run ./install --apk", text)
def test_supervisor_spawns_authenticated_web_and_cleans_up_on_exit(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / "logs").mkdir()
config = self.config(root)
config["web"] = {"node": "/bin/node", "script": "/prefix/lib/web/serve.mjs",
"host": "0.0.0.0", "port": 8082, "auth_file": "/private/auth.json"}
"host": "0.0.0.0", "port": 8082, "auth_file": "/private/auth.json",
"root": "/prefix/lib/app-web", "updates": "/prefix/lib/app-updates"}
ntfy, server, web = MagicMock(), MagicMock(), MagicMock()
ntfy.poll.return_value = None
server.poll.return_value = None
@@ -196,7 +221,9 @@ class ManageNativeTests(unittest.TestCase):
manage.supervise(config)
call = popen.call_args_list[1]
self.assertEqual(call.args[0], ["/bin/node", "/prefix/lib/web/serve.mjs", "--port", "8082",
"--host", "0.0.0.0", "--pipelines", "127.0.0.1:8080"])
"--host", "0.0.0.0", "--pipelines", "127.0.0.1:8080",
"--root", "/prefix/lib/app-web",
"--updates", "/prefix/lib/app-updates"])
self.assertNotIn("PIPELINES_WEB_AUTH", call.kwargs["env"])
self.assertEqual(call.kwargs["env"]["PIPELINES_WEB_AUTH_FILE"], "/private/auth.json")
web.terminate.assert_called_once()
+31 -2
View File
@@ -116,7 +116,8 @@ class SetupNativeTests(unittest.TestCase):
# user configuration during tests.
return setup_native.Setup(directory, prefix=Path(directory) / "prefix", runner=recorder,
resolver=resolution,
environ={"XDG_CONFIG_HOME": str(Path(directory) / "config")})
environ={"XDG_CONFIG_HOME": str(Path(directory) / "config"),
"PIPELINES_WEB_PASSWORD": "chosen-password"})
@staticmethod
def which_all(opencode="/usr/bin/opencode", pi=None, cc="/usr/bin/cc"):
@@ -202,11 +203,39 @@ class SetupNativeTests(unittest.TestCase):
installer.configure_web(prefix, setup_native.arguments([]))
self.assertEqual(auth.read_bytes(), before)
self.assertEqual(auth.stat().st_mode & 0o777, 0o600)
self.assertGreaterEqual(len(json.loads(before)["password"]), 8)
self.assertEqual(json.loads(before)["password"], "chosen-password")
self.assertTrue(Path(web["script"]).is_file())
# The web app is built in Docker once and installed for the proxy.
self.assertTrue(installer.runner.has("docker", "run", "pipelines-app-build", "web"))
self.assertEqual(web["root"], str(prefix / "lib/app-web"))
self.assertTrue(Path(web["root"]).is_dir())
self.assertTrue(Path(web["updates"]).is_dir())
self.assertTrue((prefix / "lib/web/src/proxySecurity.js").is_file())
self.assertEqual(web["port"], 8082)
def test_new_web_password_must_be_chosen(self):
with tempfile.TemporaryDirectory() as directory:
installer = self.installer(directory, Recorder())
prefix = Path(directory) / "prefix"
installer.prepare_layout(prefix)
installer.environ.pop("PIPELINES_WEB_PASSWORD")
with patch.object(setup_native.shutil, "which", side_effect=self.which_all()), \
patch.object(setup_native.sys.stdin, "isatty", return_value=False):
with self.assertRaisesRegex(ValueError, "PIPELINES_WEB_PASSWORD"):
installer.configure_web(prefix, setup_native.arguments([]))
installer.environ["PIPELINES_WEB_PASSWORD"] = "short"
with patch.object(setup_native.shutil, "which", side_effect=self.which_all()):
with self.assertRaisesRegex(ValueError, "at least 8"):
installer.configure_web(prefix, setup_native.arguments([]))
installer.environ.pop("PIPELINES_WEB_PASSWORD")
with patch.object(setup_native.shutil, "which", side_effect=self.which_all()), \
patch.object(setup_native.sys.stdin, "isatty", return_value=True), \
patch.object(setup_native.getpass, "getpass",
side_effect=["short", "typed-password", "other-password",
"typed-password", "typed-password"]):
web = installer.configure_web(prefix, setup_native.arguments([]))
self.assertEqual(json.loads(Path(web["auth_file"]).read_text())["password"], "typed-password")
def test_invalid_web_auth_is_retained(self):
with tempfile.TemporaryDirectory() as directory:
installer = self.installer(directory, Recorder())
+75 -15
View File
@@ -1,10 +1,12 @@
// Zero-dependency static file server + WebSocket proxy for the spike.
// The authenticated web front door for Pipelines: the sign-in page, the Flutter
// web app, and a proxy to the Pipelines server, with no dependencies.
// node serve.mjs [--port 8090] [--host 0.0.0.0] [--pipelines 127.0.0.1:8080]
// [--root DIR] [--updates DIR]
//
// Serves this directory, proxies GET/Upgrade /connect and GET/HEAD /artifacts/ to
// the Pipelines server, so the page, the WebSocket and artifact files share one
// origin. The API and its artifacts live on the server's port; the page is served
// from another, so artifact URLs resolved against the page would 404 without this.
// Serves the app (--root, default ../app/build/web) and proxies /connect,
// /artifacts/ and /chat-uploads to the Pipelines server, so the page, the
// WebSocket and artifact files share one origin. --updates serves the Android
// app's update feed at /updates/.
//
// Why the proxy exists: internal/server/websocket.go calls
// `websocket.Accept(w, r, nil)` (github.com/coder/websocket), which rejects an
@@ -17,7 +19,7 @@
// The default host is 127.0.0.1. Use --host 0.0.0.0 only on a trusted LAN/VPN.
import { createServer, request } from "node:http";
import { createHmac, timingSafeEqual } from "node:crypto";
import { readFileSync } from "node:fs";
import { createReadStream, readFileSync } from "node:fs";
import { homedir } from "node:os";
import { readFile, stat } from "node:fs/promises";
import { extname, join, normalize, resolve, sep } from "node:path";
@@ -25,7 +27,13 @@ import { fileURLToPath } from "node:url";
import net from "node:net";
import { sameOriginUpgrade } from "./src/proxySecurity.js";
const ROOT = resolve(fileURLToPath(new URL(".", import.meta.url)));
// The proxy's own directory holds the sign-in page's icons.
const HERE = resolve(fileURLToPath(new URL(".", import.meta.url)));
// The page root: `--root DIR` (the installer's Flutter web build), by default
// the checkout's app/build/web. `--updates DIR` serves the Android app's
// version.json and APK at /updates/, behind the same sign-in.
const ROOT = resolve(flag(process.argv, "--root", "") || join(HERE, "..", "app", "build", "web"));
const UPDATES = flag(process.argv, "--updates", "") ? resolve(flag(process.argv, "--updates", "")) : null;
// ---------------------------------------------------------------------------
// Access control
@@ -239,6 +247,14 @@ const MIME = {
".png": "image/png",
".ico": "image/x-icon",
".map": "application/json; charset=utf-8",
".wasm": "application/wasm",
".otf": "font/otf",
".ttf": "font/ttf",
".woff": "font/woff",
".woff2": "font/woff2",
".bin": "application/octet-stream",
".frag": "application/octet-stream",
".apk": "application/vnd.android.package-archive",
};
// A path is asset-looking when its last segment ends in a known file type, and a
@@ -288,9 +304,46 @@ function resolvePath(urlPath) {
return target;
}
/** One file of the Android app's update feed, or 404. */
async function serveUpdate(req, res, pathname) {
let name;
try {
name = decodeURIComponent(pathname.slice("/updates/".length));
} catch {
name = "";
}
const path = resolve(join(UPDATES, name));
const info = name && path.startsWith(UPDATES + sep) ? await stat(path).catch(() => null) : null;
if (!info?.isFile()) {
res.writeHead(404, { "Content-Type": "text/plain; charset=utf-8" }).end("Not found");
return;
}
res.writeHead(200, {
"Content-Type": MIME[extname(path)] ?? "application/octet-stream",
"Content-Length": info.size,
"Cache-Control": "no-store",
});
if (req.method === "HEAD") {
res.end();
return;
}
createReadStream(path).pipe(res);
}
/** Send index.html (the page shell). The router reads the real URL in the browser. */
async function serveIndex(res) {
const html = await readFile(join(ROOT, "index.html"), "utf8");
const html = await readFile(join(ROOT, "index.html"), "utf8").catch(() => null);
if (html === null) {
// The installer builds the app in Docker; without it there is no page yet.
res.writeHead(503, { "Content-Type": "text/html; charset=utf-8", "Cache-Control": "no-store" });
res.end(
'<!doctype html><meta charset="utf-8"><title>Pipelines</title>' +
'<body style="font-family:system-ui;padding:2rem;background:#101013;color:#cfd9e6">' +
"<h1>The web app is not built yet</h1>" +
"<p>On the server, start Docker and run <code>./install --web</code>, then reload.</p>",
);
return;
}
// A deep URL such as /runs/440 would otherwise resolve the shell's relative
// `./src/app.js` against /runs/. A root <base> keeps the shell's own asset URLs
// absolute without changing the address bar (and without editing index.html).
@@ -328,14 +381,16 @@ function proxyArtifact(req, res) {
const server = createServer(async (req, res) => {
const pathname = (req.url ?? "/").split("?")[0];
// Brand icons are public so the sign-in page itself can show the app icon.
if (req.method === "GET" && (pathname === "/favicon.svg" || pathname === "/apple-touch-icon.png")) {
const body = await readFile(join(HERE, pathname.slice(1))).catch(() => null);
if (body) {
res.writeHead(200, { "Content-Type": MIME[extname(pathname)], "Cache-Control": "no-store" }).end(body);
return;
}
}
if (auth) {
// Brand assets are public so the sign-in page itself can show the app icon.
if (
req.method === "GET" &&
(pathname === "/favicon.svg" || pathname === "/apple-touch-icon.png")
) {
// fall through to the static handler below
} else if (pathname === "/login" && req.method === "GET") {
if (pathname === "/login" && req.method === "GET") {
if (isAuthed(req)) {
res.writeHead(302, { Location: "/" }).end();
return;
@@ -392,6 +447,10 @@ const server = createServer(async (req, res) => {
proxyArtifact(req, res);
return;
}
if (UPDATES && pathname.startsWith("/updates/") && (req.method === "GET" || req.method === "HEAD")) {
await serveUpdate(req, res, pathname);
return;
}
if (pathname === "/favicon.ico") {
res.writeHead(204).end(); // no icon; keeps the console free of a 404
return;
@@ -497,4 +556,5 @@ server.listen(port, host, () => {
console.log(`web spike: http://${shown}:${port}/ (bound ${host}:${port})`);
console.log(`proxying /connect (ws) and /artifacts/ (http) -> ${targetHost}:${targetPort}`);
console.log(auth ? "auth: sign-in required" : "auth: DISABLED (tests)");
console.log(`serving ${ROOT}${UPDATES ? `; app updates from ${UPDATES}` : ""}`);
});
+64
View File
@@ -0,0 +1,64 @@
import test from "node:test";
import assert from "node:assert/strict";
import { spawn } from "node:child_process";
import { mkdtemp, mkdir, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import net from "node:net";
const here = fileURLToPath(new URL("..", import.meta.url));
async function freePort() {
const server = net.createServer().listen(0, "127.0.0.1");
await new Promise((done) => server.once("listening", done));
const { port } = server.address();
await new Promise((done) => server.close(done));
return port;
}
async function start(args) {
const port = await freePort();
const child = spawn(process.execPath, ["serve.mjs", "--port", String(port), ...args], {
cwd: here,
env: { ...process.env, PIPELINES_WEB_AUTH: "disabled" },
stdio: ["ignore", "pipe", "inherit"],
});
await new Promise((done) => child.stdout.once("data", done));
return { base: `http://127.0.0.1:${port}`, stop: () => child.kill() };
}
test("serves the app root, deep routes and the update feed", async (t) => {
const dir = await mkdtemp(join(tmpdir(), "serve-"));
const root = join(dir, "app");
const updates = join(dir, "updates");
await mkdir(join(root, "canvaskit"), { recursive: true });
await mkdir(updates);
await writeFile(join(root, "index.html"), "<html><head></head><body>app</body></html>");
await writeFile(join(root, "canvaskit", "canvaskit.wasm"), "wasm");
await writeFile(join(updates, "version.json"), '{"versionCode":1}');
await writeFile(join(dir, "secret.txt"), "no");
const server = await start(["--root", root, "--updates", updates]);
t.after(server.stop);
const index = await fetch(`${server.base}/runs/12`);
assert.equal(index.status, 200);
assert.match(await index.text(), /app/);
const wasm = await fetch(`${server.base}/canvaskit/canvaskit.wasm`);
assert.equal(wasm.headers.get("content-type"), "application/wasm");
const feed = await fetch(`${server.base}/updates/version.json`);
assert.deepEqual(await feed.json(), { versionCode: 1 });
assert.equal((await fetch(`${server.base}/updates/%2e%2e%2fsecret.txt`)).status, 404);
assert.equal((await fetch(`${server.base}/updates/missing.apk`)).status, 404);
const icon = await fetch(`${server.base}/favicon.svg`);
assert.equal(icon.status, 200);
});
test("explains how to build the app when it is missing", async (t) => {
const dir = await mkdtemp(join(tmpdir(), "serve-"));
const server = await start(["--root", join(dir, "absent")]);
t.after(server.stop);
const page = await fetch(`${server.base}/`);
assert.equal(page.status, 503);
assert.match(await page.text(), /\.\/install --web/);
});