diff --git a/docs/install.md b/docs/install.md index bc0e3476..7b23adb5 100644 --- a/docs/install.md +++ b/docs/install.md @@ -10,7 +10,7 @@ See [configured harnesses](harnesses-and-control.md) for scope and verification ```sh git clone https://git.a-wan.net/alan/pipelines.git cd pipelines -./install # native: build server, CLI and ntfy on this host +./install # native: server, authenticated web, CLI, ntfy + login autostart ./install --docker # explicit Docker server + ntfy ``` @@ -18,13 +18,37 @@ cd pipelines `./install`, `./install --native` and `./install native` run `scripts/setup-native.py`. It only detects the tools it needs and never installs -or upgrades them: **Go, python3, git, curl and a C compiler (`cc`/`clang`/`gcc`)** +or upgrades them: **Go (version required by `go.mod`), Python 3.9+, Node.js 18+, +git, curl and a C compiler (`cc`/`clang`/`gcc`)** must already be on PATH. The C compiler is required because ntfy stores its cache and auth database in SQLite through the cgo driver `mattn/go-sqlite3`. It also detects an existing `opencode` (required) and `pi` (optional) on PATH and records their absolute paths; it never downloads, installs or upgrades OpenCode or Pi (harness installation stays Docker-only). +Run as your normal logged-in user, not root. macOS requires `launchctl` and an +active GUI login; Linux requires `systemctl` and a working `systemd --user` +session. Unsupported platforms and missing tools fail instead of silently +installing an unsupervised server. Go downloads dependencies and the pinned ntfy +source during a fresh build; the installer does not install system prerequisites. + +**`./install` is the only setup command.** It also copies the dependency-free web +client into `/lib/web`, configures its authenticated Node proxy and starts +all three processes under one login service. No npm install or separate manual +`node serve.mjs` step is needed. The default browser URL is +`http://localhost:8082/` on this computer, or `http://HOST:8082/` on a trusted +LAN/VPN (`--web-port` / `--web-host` override it). Setup prints the actual URL. +The unauthenticated Go API defaults to **127.0.0.1:8080**; expose only the web +proxy. Use TLS termination or an encrypted VPN for remote access; plain HTTP +does not encrypt the sign-in password. + +Existing web credentials in `PIPELINES_WEB_AUTH_FILE`, or +`~/.config/pipelines/web-auth.json`, are reused unchanged. A fresh setup generates +a private 0600 file there with `password` and a cookie-signing `secret`; read its +`password` field locally to sign in (never share the whole file). An invalid or +non-private existing file stops setup instead of resetting credentials. The +service never inherits the test-only `PIPELINES_WEB_AUTH=disabled` bypass. + Setup resolves `PIPELINES_PREFIX` or defaults to `~/.local/share/pipelines`, creates `bin/`, `lib/`, `data/`, `logs/` and `secrets/` (`secrets` and `data` at 0700), copies `scripts/manage.py` to `/lib/manage.py`, builds exactly @@ -46,8 +70,8 @@ other binary, and it installs no unrelated tool; the manifest keeps The native install also writes **host** harness profiles for the harnesses it detects into XDG config home's `pipelines/harnesses.json` (no -`docker_directory`), defaulting to the detected OpenCode profile rather than a -stale `pi` default and taking models from `--model` or the existing valid profile. +`docker_directory`), preserving a valid host default (or choosing OpenCode instead +of a stale Docker default) and taking models from `--model` or the existing valid profile. When migrating a Docker install it clears the recorded Docker link instead of leaving the config Docker-linked. It never installs, upgrades or downloads a harness. If no valid model can be resolved it writes nothing and points you at @@ -83,20 +107,34 @@ present but its volumes cannot be inspected (for example a stopped daemon), setu fails clearly instead of silently skipping migration, so an existing Docker database is never shadowed by an empty native start. -Start and manage it with the manifest's prefix on PATH: +Setup already starts it and enables login autostart. Optional CLI management +requires the prefix selector (the executable path alone does not select it): ```sh -PIPELINES_PREFIX=~/.local/share/pipelines ~/.local/share/pipelines/bin/pipelines start -PIPELINES_PREFIX=~/.local/share/pipelines ~/.local/share/pipelines/bin/pipelines status +export PIPELINES_PREFIX="$HOME/.local/share/pipelines" +export PATH="$PIPELINES_PREFIX/bin:$PATH" +pipelines doctor +# pipelines stop / start / restart / logs ``` `start`, `stop`, `restart`, `doctor`, `logs` and `uninstall` reach `/lib/manage.py`. Ports default to 8080 (server) and 8090 (ntfy) and can be changed with `--port`/`--ntfy-port`; setup starts both services itself, so -those ports must be free (a running Docker installation holds them — stop it +those ports and the web port (8082 by default) must be free (a running Docker installation holds them — stop it first; the migration still needs its volumes and image). ntfy runs from `/bin/ntfy` with the generated config; per-user launchd (macOS) or -`systemd --user` (Linux) registration and the supervised fallback both remain. +`systemd --user` (Linux) starts ` /lib/manage.py supervise --prefix +`, which owns server, ntfy and web lifecycles. macOS writes +`~/Library/LaunchAgents/net.a-wan.pipelines.plist` and uses `launchctl enable`, +`bootstrap`, and `kickstart` in `gui/UID`. Linux writes +`$XDG_CONFIG_HOME/systemd/user/net.a-wan.pipelines.service` (default `~/.config`) +and runs `systemctl --user daemon-reload` then `enable --now +net.a-wan.pipelines.service`. This is login autostart, not boot-without-login; +Linux lingering is an explicit administrator decision. Do **not** run the separate +`pipelines service enable` for this installation: it configures a standalone +developer server, not the manifest database/ntfy/web supervisor. Logs are in +`/logs/{server,ntfy,web,supervisor}.log` (Linux supervisor output also goes +to `journalctl --user -u net.a-wan.pipelines.service`). Dated evidence (2026-10-13, macOS arm64): a scratch-prefix run of `scripts/setup-native.py` resolved ntfy v2.30.0, compiled it from source (`ntfy @@ -108,6 +146,11 @@ ntfy accounts from the live Docker volumes, and wrote the manifest. `pipelines stop` released both ports. launchd/systemd registration and real phone delivery are **not tested**. +The newer integrated web/autostart path is covered by scratch-prefix installer +mocks, rendered launchd/systemd definitions and mocked lifecycle/auth-preservation +checks. It has **not** been live-installed or tested through a real login on +either OS; Linux coverage here is mocked, not native Linux verification. + ## Docker installation (explicit) `./install --docker` (or `./install docker`) runs the unchanged @@ -120,7 +163,7 @@ available, using `GOBIN` or `GOPATH/bin`; put that directory on PATH. It does no install a host server. Reconfiguration uses `./install --opencode` or `./install --pi` inside the existing Docker installation and never falls back to host execution. It resolves and verifies the same current releases and refuses to configure against an -image that does not match them (rebuild with `./install` first); it does not restart +image that does not match them (rebuild with `./install --docker` first); it does not restart or redeploy services. Native Go builds embed VCS identity; Docker setup passes the checkout revision and @@ -131,7 +174,7 @@ arguments plus the resolved version/digest arguments, without restarting service A plain `docker compose build` cannot resolve the OpenCode image digest, so it must be given `OPENCODE_VERSION`, `OPENCODE_DIGEST` and `PI_VERSION` through Docker `--build-arg` flags (and `PIPELINES_VERSION`, `PIPELINES_COMMIT` and -`PIPELINES_DIRTY` outside a Git checkout); `./install` supplies all of them. After deployment, +`PIPELINES_DIRTY` outside a Git checkout); `./install --docker` supplies all of them. After deployment, `pipelines --url ws://HOST:PORT/connect ping --json` identifies the running server. Docker setup checks that the started server's pong matches the source identity passed to the build and fails setup if metadata is missing or mismatched. @@ -206,8 +249,8 @@ volumes use `nocopy` so Docker cannot reset a freshly prepared empty volume's ownership to the image UID. Setup verifies that the server can enter the host checkout before reporting completion. -For fresh configuration you can supply `--ntfy-url https://ntfy.example.com ---bind-address 127.0.0.1 --timezone Europe/London --port 8081` directly to `./install`. +For fresh Docker configuration you can supply `--ntfy-url https://ntfy.example.com +--bind-address 127.0.0.1 --timezone Europe/London --port 8081` to `./install --docker`. The defaults are ready for direct phone access on your trusted LAN. Only use plain HTTP on a trusted network. Other existing `.env` settings are not replaced. These Docker flags apply to `./install --docker`; the explicit Docker mode fails @@ -310,9 +353,9 @@ pipelines serve # foreground developer/legacy server; Ctrl+C stops it pipelines ``` -The server binds to `0.0.0.0` by default (the LAN, so the phone client can reach -it), matching the Docker install; record `127.0.0.1` on the manifest with -`./install --bind-address 127.0.0.1` for a loopback-only server. The Pipelines API +The native installer binds the Go server to loopback by default and exposes the +authenticated web proxy instead. An explicit `--bind-address` changes the manifest. +The Pipelines API has no authentication, so keep that port on a trusted LAN/VPN. Change into your repository and run a pipeline there; the client sends its current directory to the server. The server's own working directory does not restrict which repositories @@ -401,6 +444,11 @@ No login or provider credentials are copied or exported by Pipelines. ## Optional notifications Installation configures this automatically via `./install`. +ntfy is independent of Android: it is an optional notification channel for any +compatible subscriber, including a separate ntfy app. A web-only user does not +need an Android client, and observing that the browser is the only client is not +a reason to disable an existing ntfy configuration. Browser-closed web push is +**not implemented**; installing/configuring ntfy does not add that capability. For a developer/legacy host server, use an existing ntfy server or the Docker setup in [docs/ntfy.md](ntfy.md). Configure `PIPELINES_NTFY_URL`, `PIPELINES_NTFY_PUBLIC_URL`, and a publishing token via `PIPELINES_NTFY_TOKEN_FILE` in the server's environment. Keep secrets out of @@ -409,8 +457,15 @@ expose an unauthenticated pipeline server or HTTP credentials to untrusted netwo ## Uninstall -Disable autostart with `pipelines service disable` if configured, or stop the -foreground server first, then: +For the native installer, select the manifest and run `pipelines uninstall`: + +```sh +PIPELINES_PREFIX="$HOME/.local/share/pipelines" "$HOME/.local/share/pipelines/bin/pipelines" uninstall +``` + +This stops its supervisor and removes its login definition and managed binaries; +data and credentials are retained. The following applies only to standalone +developer binaries: disable `pipelines service disable` if configured, then: ```sh pipelines uninstall @@ -423,7 +478,7 @@ and optional ntfy data remain untouched. Use the permissions appropriate to your chosen bin directory. Custom database paths cannot be inferred, so uninstallation does not purge data. -## Existing managed installations +## Existing legacy managed installations The previous Python-supervised setup is retained only for explicit legacy use. Set `PIPELINES_PREFIX` to its existing directory for its lifecycle commands; diff --git a/install.sh b/install.sh index 563b04fd..627d5d31 100755 --- a/install.sh +++ b/install.sh @@ -1,6 +1,11 @@ #!/bin/sh set -eu cd "$(dirname "$0")" +if ! command -v python3 >/dev/null 2>&1; then + echo "Python 3.9 or newer is required; install it and rerun ./install." >&2 + exit 1 +fi +python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3, 9) else "Python 3.9 or newer is required")' mode=${1:-} case "$mode" in --opencode|--pi) diff --git a/scripts/manage.py b/scripts/manage.py index 82e90547..5934a784 100644 --- a/scripts/manage.py +++ b/scripts/manage.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""One per-user lifecycle for the native server and its private Docker ntfy.""" +"""One per-user lifecycle for the server, authenticated web proxy and ntfy.""" import argparse import fcntl import json @@ -110,6 +110,7 @@ def start(config): if registration(config): if platform.system() == "Darwin": target = f"gui/{os.getuid()}/{config['label']}" + run(["launchctl", "enable", target], quiet=True) if run(["launchctl", "print", target], quiet=True, check=False).returncode: run(["launchctl", "bootstrap", f"gui/{os.getuid()}", registration(config)], quiet=True) run(["launchctl", "kickstart", target], quiet=True) @@ -122,6 +123,20 @@ def start(config): stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True) log.close() wait_health(f"http://127.0.0.1:{config['server_port']}/healthz", timeout=75) + if config.get("web"): + web = config["web"] + host = "127.0.0.1" if web["host"] == "0.0.0.0" else web["host"] + end = time.monotonic() + 20 + while True: + try: + with urllib.request.urlopen(f"http://{host}:{web['port']}/login", timeout=2) as response: + if response.status == 200: + break + except OSError: + pass + if time.monotonic() >= end: + raise RuntimeError("Web proxy did not become ready; see logs/web.log") + time.sleep(0.25) def stop(config): @@ -194,11 +209,12 @@ def supervise(config): stopped = False child = None ntfy_child = None + web_child = None def stopping(*_): nonlocal stopped stopped = True - for process in (child, ntfy_child): + for process in (child, ntfy_child, web_child): if process is not None and process.poll() is None: process.terminate() @@ -222,12 +238,31 @@ def supervise(config): with (root / "logs/server.log").open("a") as log: child = subprocess.Popen([str(root / "bin/pipelines-server"), "-addr", server_address(config), "-db", str(root / "data/pipelines.db")], - cwd=config["workspace"], env=env, stdout=log, stderr=log) + cwd=config["workspace"], env=env, stdout=log, stderr=log) + if config.get("web"): + web = config["web"] + web_env = dict(env) + web_env.pop("PIPELINES_WEB_AUTH", None) + web_env["PIPELINES_WEB_AUTH_FILE"] = web["auth_file"] + with (root / "logs/web.log").open("a") as web_log: + web_child = subprocess.Popen( + [web["node"], web["script"], "--port", str(web["port"]), + "--host", web["host"], "--pipelines", f"127.0.0.1:{config['server_port']}"], + env=web_env, stdout=web_log, stderr=web_log) while child.poll() is None and not stopped: + if web_child is not None and web_child.poll() is not None: + raise RuntimeError("Authenticated web proxy exited; see logs/web.log") time.sleep(0.25) if not stopped and child.returncode: raise RuntimeError("Pipeline server exited; see logs/server.log") finally: + if web_child is not None and web_child.poll() is None: + web_child.terminate() + try: + web_child.wait(timeout=10) + except subprocess.TimeoutExpired: + web_child.kill() + web_child.wait() if child is not None and child.poll() is None: child.terminate() try: diff --git a/scripts/setup-native.py b/scripts/setup-native.py index 46cb9d21..6045383d 100644 --- a/scripts/setup-native.py +++ b/scripts/setup-native.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Install Pipelines natively on the host (macOS and Linux). -The installer only detects the tools it needs (go, python3, git, curl and a C +The installer only detects the tools it needs (go, python3, node, git, curl and a C compiler for ntfy's SQLite driver); it never installs or upgrades them, and it never installs, upgrades or downloads OpenCode or Pi. It builds the server, CLI and ntfy from source into a per-user prefix, generates an ntfy configuration that @@ -16,6 +16,9 @@ import importlib.util import ipaddress import json import os +import platform +import plistlib +import secrets import shutil import subprocess import sys @@ -40,7 +43,8 @@ tokens = module("save_ntfy_token", "save-ntfy-token.py") DEFAULT_PREFIX = Path.home() / ".local/share/pipelines" MARKER = "pipelines-install.json" LABEL = "net.a-wan.pipelines" -REQUIRED_TOOLS = ("go", "python3", "git", "curl") +SERVICE_MARKER = "Managed by Pipelines native installer" +REQUIRED_TOOLS = ("go", "python3", "git", "curl", "node") # ntfy's SQLite driver (mattn/go-sqlite3) is cgo, so a C compiler is required to # build the server. Upstream's macOS release asset is client-only, so both # platforms compile ntfy from the pinned Go module. @@ -48,11 +52,8 @@ C_COMPILERS = ("cc", "clang", "gcc") NTFY_BUILD_TAGS = "sqlite_omit_load_extension,osusergo,netgo" DEFAULT_NTFY_PORT = 8090 DEFAULT_SERVER_PORT = 8080 -# The phone client reaches the server over the LAN, matching the Docker install -# (which published 0.0.0.0). Use --bind-address 127.0.0.1 for a loopback-only -# server; the Pipelines API has no authentication, so keep it on a trusted -# LAN/VPN either way. -DEFAULT_BIND_ADDRESS = "0.0.0.0" +# Only the authenticated web proxy is exposed; the control API stays loopback. +DEFAULT_BIND_ADDRESS = "127.0.0.1" class Setup: @@ -100,6 +101,51 @@ class Setup: pi = shutil.which("pi") return {"opencode": str(Path(opencode).resolve()), "pi": str(Path(pi).resolve()) if pi else ""} + def service_path(self): + system = platform.system() + if system == "Darwin": + path = Path.home() / "Library/LaunchAgents" / (LABEL + ".plist") + tool = "launchctl" + elif system == "Linux": + root = Path(self.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config") + path = root / "systemd/user" / (LABEL + ".service") + tool = "systemctl" + else: + raise ValueError("Native installation supports macOS (launchd) and Linux (systemd --user) only") + if not shutil.which(tool): + raise ValueError(f"{tool} is required for login autostart; run setup in a logged-in user session") + if path.is_symlink() or (path.exists() and SERVICE_MARKER not in path.read_text()): + raise ValueError(f"Refusing to replace an unmanaged autostart definition: {path}") + return path + + def register_service(self, path, manifest): + """Register the paired native services, not a second standalone server.""" + prefix = Path(manifest["prefix"]) + command = [manifest["python"], str(prefix / "lib/manage.py"), + "supervise", "--prefix", str(prefix)] + if platform.system() == "Darwin": + content = plistlib.dumps({ + "Label": LABEL, "Comment": SERVICE_MARKER, + "ProgramArguments": command, "RunAtLoad": True, + "KeepAlive": {"SuccessfulExit": False}, "ThrottleInterval": 5, + "EnvironmentVariables": {"HOME": str(Path.home()), "PATH": manifest["path"]}, + "StandardOutPath": str(prefix / "logs/supervisor.log"), + "StandardErrorPath": str(prefix / "logs/supervisor.log"), "Umask": 0o077, + }) + else: + def quote(value): + if any(char in value for char in "\r\n\x00"): + raise ValueError("Autostart paths cannot contain line breaks or NUL") + return '"' + value.replace('\\', '\\\\').replace('"', '\\"').replace('%', '%%').replace('$', '$$') + '"' + + content = (f"# {SERVICE_MARKER}\n[Unit]\nDescription=Pipelines native services\n" + "\n[Service]\nType=simple\nExecStart=" + " ".join(map(quote, command)) + + "\nRestart=on-failure\nRestartSec=5\nTimeoutStopSec=90\nUMask=0077\n" + "\n[Install]\nWantedBy=default.target\n").encode() + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + self._write_atomic(path, content, 0o600) + manifest["registration"] = str(path) + # --- host harness profiles ------------------------------------------ MODEL_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_./:-" @@ -154,15 +200,19 @@ class Setup: if not harness_model or not pipeline_model: continue # Host mode: no docker_directory, executable is the detected host binary. - merged[name] = {"executable": harness[name], "model": harness_model, "pipeline_model": pipeline_model} + profile = dict(current) + profile.pop("docker_directory", None) + profile.update(executable=harness[name], model=harness_model, pipeline_model=pipeline_model) + merged[name] = profile configured.append(name) if not configured: print("Host harness profiles were not written: no valid model was found for " + ", ".join(order) + ". Run 'pipelines harness setup opencode --host' to configure authoring.") return None - # Prefer the primary detected harness, so a stale Docker-era default ("pi") - # never survives a native install. - default = "opencode" if "opencode" in configured else configured[0] + # Keep a configured host default; only a stale Docker default must change. + default = existing.get("default") + if default not in configured or profiles.get(default, {}).get("docker_directory"): + default = "opencode" if "opencode" in configured else configured[0] data = dict(existing) data["default"] = default data["profiles"] = merged @@ -193,6 +243,36 @@ class Setup: os.chmod(target, 0o700) return target + def configure_web(self, prefix, args): + node = str(Path(shutil.which("node")).resolve()) + version = self.runner([node, "--version"], capture_output=True, text=True) + try: + major = int(version.stdout.strip().removeprefix("v").split(".")[0]) + except (ValueError, AttributeError): + major = 0 + if version.returncode or major < 18: + raise ValueError("Node.js 18 or newer is required for the authenticated web proxy") + # Reuse the proxy's existing credential location; never disable its gate. + auth = Path(self.environ.get("PIPELINES_WEB_AUTH_FILE") + or Path.home() / ".config/pipelines/web-auth.json").expanduser().resolve() + if auth.exists(): + data = json.loads(auth.read_text()) + if (not isinstance(data, dict) or not isinstance(data.get("password"), str) + or len(data["password"]) < 8 or not isinstance(data.get("secret"), str) + or len(data["secret"]) < 32): + raise ValueError(f"Invalid existing web auth file; retained unchanged: {auth}") + if auth.stat().st_mode & 0o077: + raise ValueError(f"Web auth file must be private (chmod 600): {auth}") + else: + auth.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + self._write_atomic(auth, (json.dumps({"password": secrets.token_urlsafe(18), + "secret": secrets.token_urlsafe(48)}) + "\n").encode(), 0o600) + target = prefix / "lib/web" + shutil.copytree(Path(__file__).resolve().parents[1] / "web", target, dirs_exist_ok=True, + ignore=shutil.ignore_patterns("test", "node_modules", ".DS_Store")) + return {"node": node, "script": str(target / "serve.mjs"), + "auth_file": str(auth), "host": args.web_host, "port": args.web_port} + # --- build and skill ------------------------------------------------- def build(self, prefix): @@ -493,20 +573,26 @@ class Setup: raise ValueError("Invalid ntfy port; choose a value between 1024 and 65535") if args.ntfy_port == args.port: raise ValueError("The server and ntfy ports must differ") + if not 1024 <= args.web_port <= 65535 or args.web_port in (args.port, args.ntfy_port): + raise ValueError("Web port must be between 1024 and 65535 and differ from server/ntfy ports") try: ipaddress.IPv4Address(args.bind_address) + ipaddress.IPv4Address(args.web_host) except ValueError: - raise ValueError("Invalid bind address; use an IPv4 address such as 0.0.0.0 or 127.0.0.1") from None + raise ValueError("Invalid API/web bind address; use an IPv4 address such as 0.0.0.0 or 127.0.0.1") from None prefix = self.resolve_prefix() - self.prepare_layout(prefix) harness = self.harness() + service_path = self.service_path() # Reuse an already built ntfy binary and skip the release lookup, so a # reinstall with the same version makes no network requests at all. installed = self._existing_ntfy_version(prefix) if (prefix / "bin/ntfy").is_file() and installed: resolution = {"module": release_lookup.NTFY_MODULE, "version": installed, "tag": installed} else: + self.require_compiler() resolution = self.resolve_ntfy() + self.prepare_layout(prefix) + web = self.configure_web(prefix, args) self.build(prefix) self.check_path_shadow(prefix) self.install_skill(prefix) @@ -561,7 +647,9 @@ class Setup: "ntfy": {"mode": "native", "bin": str(prefix / "bin/ntfy"), "config": str(config_path), "base_url": base_url, "version": resolution.get("tag") or resolution.get("version")}, + "web": web, } + self.register_service(service_path, manifest) self.write_manifest(prefix, manifest) # Save the address the host CLI/TUI dials by default, so a plain # `pipelines` (or its TUI) reaches this installation instead of a stale @@ -579,16 +667,17 @@ class Setup: if started.returncode: raise ValueError( "The native services were installed but could not be started; run " - f"'{prefix}/bin/pipelines start' to see the error. Check that ports " - f"{args.port} and {args.ntfy_port} are free (a running Docker installation uses them).") - print(f"ntfy is configured for {base_url} and both services are running.") + f"'PIPELINES_PREFIX={prefix} {prefix}/bin/pipelines start' to see the error. " + "Check your login service manager and that ports " + f"{args.port}, {args.ntfy_port} and {args.web_port} are free (a running Docker installation may use them).") + print(f"ntfy is configured for {base_url}; server, web proxy and ntfy have login autostart ({service_path}).") + print(f"Web: http://{lan_ip if args.web_host == '0.0.0.0' else args.web_host}:{args.web_port}/ " + f"(sign-in password in {web['auth_file']}; existing credentials retained).") if args.bind_address == "0.0.0.0": print(f"Phone client server URL: ws://{lan_ip}:{args.port}/connect " "(use a workspace path that exists on this server).") else: - print(f"Server bound to {args.bind_address}:{args.port}; a phone on the LAN needs a LAN bind " - "(re-run with --bind-address 0.0.0.0).") - print("Create the phone account password if it is a new account, then subscribe to the pipelines topic.") + print(f"Unauthenticated API bound to {args.bind_address}:{args.port}; use the authenticated web URL.") def arguments(argv=None): @@ -597,9 +686,10 @@ def arguments(argv=None): parser.add_argument("--ntfy-url", help="Override the auto-detected LAN ntfy address") parser.add_argument("--timezone", help="Override the detected timezone (fallback: UTC)") parser.add_argument("--port", type=int, default=DEFAULT_SERVER_PORT, help="Pipelines server port") + parser.add_argument("--web-port", type=int, default=8082, help="Authenticated web proxy port (default 8082)") + parser.add_argument("--web-host", default="0.0.0.0", help="Web proxy bind address (default 0.0.0.0)") parser.add_argument("--bind-address", default=DEFAULT_BIND_ADDRESS, - help="Server bind address (default 0.0.0.0, the LAN, so the phone client can reach it; " - "use 127.0.0.1 for a loopback-only server)") + help="Unauthenticated server bind address (default 127.0.0.1; expose only the web proxy)") parser.add_argument("--ntfy-port", type=int, default=DEFAULT_NTFY_PORT, help="ntfy listen port") parser.add_argument("--model", help="Record an explicit default model; otherwise host OpenCode configuration is reused") parser.add_argument("--replace-data", action="store_true", diff --git a/scripts/test_manage_native.py b/scripts/test_manage_native.py index 2dc59f66..8be55efe 100644 --- a/scripts/test_manage_native.py +++ b/scripts/test_manage_native.py @@ -4,7 +4,7 @@ from pathlib import Path import signal import tempfile import unittest -from unittest.mock import patch +from unittest.mock import MagicMock, patch spec = importlib.util.spec_from_file_location("manage_native", Path(__file__).with_name("manage.py")) manage = importlib.util.module_from_spec(spec) @@ -37,6 +37,61 @@ class ManageNativeTests(unittest.TestCase): self.assertEqual(manage.ntfy_mode({"ntfy": {"mode": "native"}}), "native") self.assertEqual(manage.ntfy_health_url({"ntfy_port": 8090}), "http://127.0.0.1:8090/v1/health") + def test_registered_start_commands_on_both_platforms(self): + for system in ("Darwin", "Linux"): + with self.subTest(system=system), tempfile.TemporaryDirectory() as directory: + config = self.config(Path(directory)) + config["registration"] = str(Path(directory) / "service") + with patch.object(manage.platform, "system", return_value=system), \ + patch.object(manage, "run") as run, patch.object(manage, "wait_health"): + manage.start(config) + calls = [call.args[0] for call in run.call_args_list] + if system == "Linux": + self.assertEqual(calls, [["systemctl", "--user", "daemon-reload"], + ["systemctl", "--user", "enable", "--now", "net.a-wan.pipelines.service"]]) + else: + self.assertEqual(calls[0], ["launchctl", "enable", f"gui/{os.getuid()}/net.a-wan.pipelines"]) + self.assertIn(["launchctl", "kickstart", f"gui/{os.getuid()}/net.a-wan.pipelines"], calls) + + def test_supervisor_spawns_authenticated_web_and_cleans_up_on_exit(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "logs").mkdir() + config = self.config(root) + config["web"] = {"node": "/bin/node", "script": "/prefix/lib/web/serve.mjs", + "host": "0.0.0.0", "port": 8082, "auth_file": "/private/auth.json"} + ntfy, server, web = MagicMock(), MagicMock(), MagicMock() + ntfy.poll.return_value = None + server.poll.return_value = None + web.poll.side_effect = [1, None] + with patch.object(manage, "ntfy_start", return_value=ntfy), \ + patch.object(manage, "wait_health"), patch.object(manage.signal, "signal"), \ + patch.dict(os.environ, {"PIPELINES_WEB_AUTH": "disabled"}), \ + patch.object(manage.subprocess, "Popen", side_effect=[server, web]) as popen: + with self.assertRaisesRegex(RuntimeError, "web proxy exited"): + manage.supervise(config) + call = popen.call_args_list[1] + self.assertEqual(call.args[0], ["/bin/node", "/prefix/lib/web/serve.mjs", "--port", "8082", + "--host", "0.0.0.0", "--pipelines", "127.0.0.1:8080"]) + self.assertNotIn("PIPELINES_WEB_AUTH", call.kwargs["env"]) + self.assertEqual(call.kwargs["env"]["PIPELINES_WEB_AUTH_FILE"], "/private/auth.json") + web.terminate.assert_called_once() + server.terminate.assert_called_once() + ntfy.terminate.assert_called_once() + + def test_start_checks_web_login_readiness(self): + with tempfile.TemporaryDirectory() as directory: + config = self.config(Path(directory)) + config["registration"] = "/scratch/service" + config["web"] = {"host": "0.0.0.0", "port": 8123} + response = MagicMock() + response.__enter__.return_value.status = 200 + with patch.object(manage.platform, "system", return_value="Linux"), \ + patch.object(manage, "run"), patch.object(manage, "wait_health"), \ + patch.object(manage.urllib.request, "urlopen", return_value=response) as open_url: + manage.start(config) + open_url.assert_called_once_with("http://127.0.0.1:8123/login", timeout=2) + def test_server_address_uses_the_manifest_bind_and_defaults_to_loopback(self): self.assertEqual(manage.server_address({"server_port": 8080, "bind_address": "0.0.0.0"}), "0.0.0.0:8080") self.assertEqual(manage.server_address({"server_port": 8080, "bind_address": "10.0.0.75"}), "10.0.0.75:8080") diff --git a/scripts/test_setup_native.py b/scripts/test_setup_native.py index eeca6091..be10124b 100644 --- a/scripts/test_setup_native.py +++ b/scripts/test_setup_native.py @@ -2,6 +2,7 @@ import importlib.util import io import json import os +import plistlib from contextlib import redirect_stdout from pathlib import Path import subprocess @@ -56,6 +57,8 @@ class Recorder: self.calls.append(command) self.kwargs.append(kwargs) stdout, code = "", 0 + if command[0].endswith("node") and command[1:] == ["--version"]: + stdout = "v22.19.0\n" if command[0] == "docker": if command[1:3] == ["volume", "ls"]: if self.fail_volume_ls: @@ -92,6 +95,14 @@ class Recorder: class SetupNativeTests(unittest.TestCase): + def setUp(self): + # All LaunchAgent files, including full-run tests, stay in scratch HOME. + self.home = tempfile.TemporaryDirectory() + self.addCleanup(self.home.cleanup) + home = patch.object(setup_native.Path, "home", return_value=Path(self.home.name)) + home.start() + self.addCleanup(home.stop) + def installer(self, directory, recorder): # A disposable XDG config home keeps the client default out of the real # user configuration during tests. @@ -102,7 +113,8 @@ class SetupNativeTests(unittest.TestCase): @staticmethod def which_all(opencode="/usr/bin/opencode", pi=None, cc="/usr/bin/cc"): mapping = {"go": "/usr/bin/go", "python3": "/usr/bin/python3", - "git": "/usr/bin/git", "curl": "/usr/bin/curl"} + "git": "/usr/bin/git", "curl": "/usr/bin/curl", "node": "/usr/bin/node", + "launchctl": "/bin/launchctl", "systemctl": "/usr/bin/systemctl"} if opencode: mapping["opencode"] = opencode if pi: @@ -122,9 +134,91 @@ class SetupNativeTests(unittest.TestCase): message = str(raised.exception) self.assertIn("git", message) self.assertIn("curl", message) + self.assertIn("node", message) self.assertIn("never installs tools", message) self.assertEqual(recorder.calls, []) + def test_registration_uses_native_supervisor_on_both_platforms(self): + for system in ("Darwin", "Linux"): + with self.subTest(system=system), tempfile.TemporaryDirectory() as directory: + installer = self.installer(directory, Recorder()) + prefix = Path(directory) / "prefix with spaces" + manifest = {"prefix": str(prefix), "python": "/usr/bin/python3", "path": "/bin"} + with patch.object(setup_native.platform, "system", return_value=system), \ + patch.object(setup_native.shutil, "which", side_effect=self.which_all()): + path = installer.service_path() + installer.register_service(path, manifest) + self.assertEqual(manifest["registration"], str(path)) + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + if system == "Darwin": + data = plistlib.loads(path.read_bytes()) + self.assertEqual(data["ProgramArguments"], ["/usr/bin/python3", str(prefix / "lib/manage.py"), + "supervise", "--prefix", str(prefix)]) + self.assertTrue(data["RunAtLoad"]) + else: + text = path.read_text() + self.assertIn('"supervise" "--prefix"', text) + self.assertIn('"' + str(prefix) + '"', text) + self.assertIn("WantedBy=default.target", text) + + def test_unmanaged_service_is_never_replaced(self): + with tempfile.TemporaryDirectory() as directory: + installer = self.installer(directory, Recorder()) + with patch.object(setup_native.platform, "system", return_value="Linux"), \ + patch.object(setup_native.shutil, "which", side_effect=self.which_all()): + path = installer.service_path() + path.parent.mkdir(parents=True) + path.write_text("operator service") + with self.assertRaisesRegex(ValueError, "unmanaged"): + installer.service_path() + self.assertEqual(path.read_text(), "operator service") + + def test_missing_service_manager_fails_before_build_or_config(self): + with tempfile.TemporaryDirectory() as directory: + installer = self.installer(directory, Recorder()) + with patch.object(setup_native.platform, "system", return_value="Linux"), \ + patch.object(setup_native.shutil, "which", return_value=None): + with self.assertRaisesRegex(ValueError, "systemctl is required"): + installer.service_path() + self.assertFalse((Path(directory) / "prefix").exists()) + + def test_web_install_copies_assets_and_preserves_credentials(self): + with tempfile.TemporaryDirectory() as directory: + installer = self.installer(directory, Recorder()) + prefix = Path(directory) / "prefix" + installer.prepare_layout(prefix) + with patch.object(setup_native.shutil, "which", side_effect=self.which_all()): + web = installer.configure_web(prefix, setup_native.arguments([])) + auth = Path(web["auth_file"]) + before = auth.read_bytes() + installer.configure_web(prefix, setup_native.arguments([])) + self.assertEqual(auth.read_bytes(), before) + self.assertEqual(auth.stat().st_mode & 0o777, 0o600) + self.assertGreaterEqual(len(json.loads(before)["password"]), 8) + self.assertTrue(Path(web["script"]).is_file()) + self.assertTrue((prefix / "lib/web/src/proxySecurity.js").is_file()) + self.assertEqual(web["port"], 8082) + + def test_invalid_web_auth_is_retained(self): + with tempfile.TemporaryDirectory() as directory: + installer = self.installer(directory, Recorder()) + auth = Path(directory) / "auth.json" + auth.write_text('{"password":"short"}') + installer.environ["PIPELINES_WEB_AUTH_FILE"] = str(auth) + with patch.object(setup_native.shutil, "which", side_effect=self.which_all()): + with self.assertRaisesRegex(ValueError, "Invalid existing web auth"): + installer.configure_web(Path(directory) / "prefix", setup_native.arguments([])) + self.assertEqual(auth.read_text(), '{"password":"short"}') + + def test_old_node_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + def old_node(command, **kwargs): + return subprocess.CompletedProcess(command, 0, stdout="v16.0.0\n") + installer = self.installer(directory, old_node) + with patch.object(setup_native.shutil, "which", side_effect=self.which_all()): + with self.assertRaisesRegex(ValueError, "Node.js 18"): + installer.configure_web(Path(directory) / "prefix", setup_native.arguments([])) + def test_build_installs_only_server_and_cli_with_trimpath(self): with tempfile.TemporaryDirectory() as directory: recorder = Recorder() @@ -176,11 +270,13 @@ class SetupNativeTests(unittest.TestCase): self.assertEqual(config["format"], 1) self.assertEqual(config["prefix"], str(prefix.resolve())) self.assertEqual(config["server_port"], 8080) - self.assertEqual(config["bind_address"], "0.0.0.0") + self.assertEqual(config["bind_address"], "127.0.0.1") self.assertEqual(config["ntfy"]["mode"], "native") self.assertEqual(config["topic"], "pipelines") self.assertEqual(config["opencode"], "/usr/bin/opencode") self.assertEqual(config["pi"], "") + self.assertTrue(Path(config["registration"]).is_file()) + self.assertTrue(recorder.has("start", "--prefix", str(prefix.resolve()))) self.assertEqual((prefix / setup_native.MARKER).stat().st_mode & 0o777, 0o600) token_path = prefix / "secrets/publisher-token" self.assertEqual(token_path.read_text(), TOKEN + "\n") @@ -412,6 +508,21 @@ class SetupNativeTests(unittest.TestCase): self.assertTrue(installer._valid_model(saved["profiles"][name]["model"])) self.assertEqual(saved["profiles"]["opencode"]["executable"], "/usr/bin/opencode") + def test_configure_harnesses_preserves_configured_host_default(self): + with tempfile.TemporaryDirectory() as directory: + installer = self.installer(directory, Recorder()) + path = installer.harness_config_path() + path.parent.mkdir(parents=True) + profiles = {name: {"model": "fixture/interactive", "pipeline_model": "fixture/stages"} + for name in ("opencode", "pi")} + profiles["pi"]["future"] = "preserve" + path.write_text(json.dumps({"default": "pi", "profiles": profiles})) + installer.configure_harnesses({"opencode": "/usr/bin/opencode", "pi": "/usr/bin/pi"}) + saved = json.loads(path.read_text()) + self.assertEqual(saved["default"], "pi") + self.assertEqual(saved["profiles"]["pi"]["model"], "fixture/interactive") + self.assertEqual(saved["profiles"]["pi"]["future"], "preserve") + def _fake_pipelines(self, path, version, commit): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(f'#!/bin/sh\necho \'{{"version":"{version}","commit":"{commit}"}}\'\n')