Files
pipelines/scripts/setup-native.py
T
alan ad6cdab418 Listen on loopback and tailnet by default; make the plain-HTTP LAN opt-in
The web proxy binds the wildcard and drops connections that are not
loopback or Tailscale at both ends. A LAN listener is opt-in (web.lan) and
refuses sign-in and signed-in pages over plain HTTP unless
web.lan_plain_login is set. Existing installs keep their bind address with
a warning. Without the LAN the mDNS advert carries only the tailnet and
public routes (lan=0).
2026-11-10 01:02:02 +00:00

1003 lines
53 KiB
Python

#!/usr/bin/env python3
"""Install Pipelines natively on the host (macOS and Linux).
The installer only detects the tools it needs (go, python3, node, git, curl and a C
compiler for ntfy's SQLite driver); it never installs or upgrades them, and it
never installs, upgrades or downloads OpenCode or Pi. It builds the server, CLI
and ntfy from source into a per-user prefix, generates an ntfy configuration that
mirrors compose.yaml, creates the publishing/phone accounts and writes the
manifest scripts/manage.py loads. ntfy is compiled from its pinned Go module on
every host because upstream publishes no macOS server binary. It reuses existing
host OpenCode configuration; it writes host (non-Docker) harness profiles for the
harnesses it detects and never invents a model.
"""
import argparse
import importlib.util
import ipaddress
import json
import os
import platform
import getpass
import plistlib
import secrets
import time
import shlex
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
from urllib.parse import urlsplit
from zoneinfo import ZoneInfoNotFoundError
def module(name, file):
spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(file))
result = importlib.util.module_from_spec(spec)
spec.loader.exec_module(result)
return result
configure = module("configure_compose", "configure-compose.py")
lan = module("lan_address", "lan-address.py")
release_lookup = module("resolve_releases", "resolve-releases.py")
tokens = module("save_ntfy_token", "save-ntfy-token.py")
app_build = module("app_build", "app_build.py")
web_access = module("web_access", "web_access.py")
DEFAULT_PREFIX = Path.home() / ".local/share/pipelines"
MARKER = "pipelines-install.json"
LABEL = "net.a-wan.pipelines"
SERVICE_MARKER = "Managed by Pipelines native installer"
REQUIRED_TOOLS = ("go", "python3", "git", "curl", "node")
# ntfy's SQLite driver (mattn/go-sqlite3) is cgo, so a C compiler is required to
# build the server. Upstream's macOS release asset is client-only, so both
# platforms compile ntfy from the pinned Go module.
C_COMPILERS = ("cc", "clang", "gcc")
NTFY_BUILD_TAGS = "sqlite_omit_load_extension,osusergo,netgo"
DEFAULT_NTFY_PORT = 8090
DEFAULT_SERVER_PORT = 8080
# Only the authenticated web proxy is exposed; the control API stays loopback.
DEFAULT_BIND_ADDRESS = "127.0.0.1"
class Setup:
def __init__(self, directory, prefix=None, runner=subprocess.run, resolver=None, environ=None,
popen=subprocess.Popen):
self.directory = Path(directory)
self.prefix_argument = prefix
self.runner = runner
self.popen = popen
self.resolver = resolver or release_lookup.resolve_ntfy
self.environ = environ if environ is not None else os.environ
# --- detection -------------------------------------------------------
def require_compiler(self):
"""ntfy is built locally, so a C toolchain must already be present."""
for compiler in C_COMPILERS:
path = shutil.which(compiler)
if path:
return str(Path(path).resolve())
raise ValueError(
"No C compiler found (tried " + ", ".join(C_COMPILERS)
+ "). ntfy's SQLite driver is cgo, so the server is compiled locally; "
"install a C toolchain (Xcode Command Line Tools on macOS, gcc or clang on Linux) and rerun.")
def require_tools(self):
found, missing = {}, []
for tool in REQUIRED_TOOLS:
path = shutil.which(tool)
if path:
found[tool] = str(Path(path).resolve())
else:
missing.append(tool)
if missing:
raise ValueError(
"Missing required tool(s): " + ", ".join(missing)
+ ". Install them yourself and rerun; this installer never installs tools or uses a package manager.")
return found["python3"]
def harness(self):
"""Detect existing harnesses only; never install, upgrade or download them."""
opencode = shutil.which("opencode")
if not opencode:
raise ValueError(
"OpenCode was not found on PATH. It is required: it runs pipeline stages and chat by default. "
"Install it yourself (this installer never installs, upgrades or downloads it): "
"curl -fsSL https://opencode.ai/install | bash -- then rerun ./install.")
pi = shutil.which("pi")
return {"opencode": str(Path(opencode).resolve()), "pi": str(Path(pi).resolve()) if pi else ""}
def service_path(self):
system = platform.system()
if system == "Darwin":
path = Path.home() / "Library/LaunchAgents" / (LABEL + ".plist")
tool = "launchctl"
elif system == "Linux":
root = Path(self.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config")
path = root / "systemd/user" / (LABEL + ".service")
tool = "systemctl"
else:
raise ValueError("Native installation supports macOS (launchd) and Linux (systemd --user) only")
if not shutil.which(tool):
raise ValueError(f"{tool} is required for login autostart; run setup in a logged-in user session")
if path.is_symlink() or (path.exists() and SERVICE_MARKER not in path.read_text()):
raise ValueError(f"Refusing to replace an unmanaged autostart definition: {path}")
return path
def register_service(self, path, manifest):
"""Register the paired native services, not a second standalone server."""
prefix = Path(manifest["prefix"])
command = [manifest["python"], str(prefix / "lib/manage.py"),
"supervise", "--prefix", str(prefix)]
if platform.system() == "Darwin":
content = plistlib.dumps({
"Label": LABEL, "Comment": SERVICE_MARKER,
"ProgramArguments": command, "RunAtLoad": True,
"KeepAlive": {"SuccessfulExit": False}, "ThrottleInterval": 5,
"EnvironmentVariables": {"HOME": str(Path.home()), "PATH": manifest["path"]},
"StandardOutPath": str(prefix / "logs/supervisor.log"),
"StandardErrorPath": str(prefix / "logs/supervisor.log"), "Umask": 0o077,
})
else:
def quote(value):
if any(char in value for char in "\r\n\x00"):
raise ValueError("Autostart paths cannot contain line breaks or NUL")
return '"' + value.replace('\\', '\\\\').replace('"', '\\"').replace('%', '%%').replace('$', '$$') + '"'
content = (f"# {SERVICE_MARKER}\n[Unit]\nDescription=Pipelines native services\n"
"\n[Service]\nType=simple\nExecStart=" + " ".join(map(quote, command))
+ "\nRestart=on-failure\nRestartSec=5\nTimeoutStopSec=90\nUMask=0077\n"
# The supervisor stops its own children. Only it is
# signalled, so a Claude Code chat host (its own
# session) outlives a restart as it does under launchd.
"KillMode=process\n"
"\n[Install]\nWantedBy=default.target\n").encode()
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
self._write_atomic(path, content, 0o600)
manifest["registration"] = str(path)
# --- host harness profiles ------------------------------------------
MODEL_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_./:-"
@classmethod
def _valid_model(cls, value):
if not isinstance(value, str) or len(value) > 501 or "/" not in value:
return False
provider, model = value.split("/", 1)
if not provider or not model or value in ("default/default", "provider/model"):
return False
return all(char in cls.MODEL_CHARS for char in value)
def harness_config_path(self):
root = Path(self.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config")
return root / "pipelines" / "harnesses.json"
def _existing_harness_config(self):
path = self.harness_config_path()
if path.is_symlink() or not path.is_file():
return {}
try:
data = json.loads(path.read_text())
except (OSError, ValueError):
return {}
return data if isinstance(data, dict) else {}
def opencode_default_model(self, executable):
"""Read V2's resolved Model.Info, with OpenCode's own JSONC/auth handling.
Run in HOME, not the install checkout, so a repository's model override
does not accidentally become the host installation's global default.
No login, sessions, prompts or inference are involved.
"""
try:
result = self.runner([executable, "api", "get", "/api/model/default"],
cwd=str(Path.home()), env=dict(self.environ),
capture_output=True, text=True, timeout=30)
if result.returncode or len(result.stdout or "") > 4 * 1024 * 1024:
return ""
response = json.loads(result.stdout)
data = response.get("data") if isinstance(response, dict) else None
if not isinstance(data, dict) or data.get("enabled") is not True or data.get("status") == "deprecated":
return ""
provider, model = data.get("providerID"), data.get("modelID")
if not isinstance(provider, str) or "/" in provider or not isinstance(model, str):
return ""
value = provider + "/" + model
return value if self._valid_model(value) else ""
except (OSError, ValueError, subprocess.SubprocessError):
# Never echo raw CLI output: provider discovery can include secrets.
return ""
def configure_harnesses(self, harness, model="", allow_picker=True):
"""Write host (non-Docker) profiles for the harnesses detected on PATH.
The native install launches host harnesses, so a Docker-era
``docker_directory`` must be cleared rather than kept. Models come from
existing valid profile, the explicit ``--model`` (also recorded as
PIPELINES_DEFAULT_MODEL), or OpenCode's resolved host default; they are
never invented. Unrelated keys and
profiles for harnesses that are not detected are preserved, matching
harness.Save. No harness is installed, upgraded or downloaded.
"""
if model and not self._valid_model(model):
raise ValueError("Invalid --model; supply an actual PROVIDER/MODEL, not a default placeholder")
existing = self._existing_harness_config()
profiles = existing.get("profiles")
if not isinstance(profiles, dict):
profiles = {}
merged = {name: dict(value) for name, value in profiles.items() if isinstance(value, dict)}
order = [name for name in ("opencode", "pi") if harness.get(name)]
configured = []
for name in order:
current = merged.get(name, {})
harness_model = current.get("model") if self._valid_model(current.get("model")) else ""
pipeline_model = current.get("pipeline_model") if self._valid_model(current.get("pipeline_model")) else ""
if name == "opencode" and self._valid_model(model):
harness_model = harness_model or model
pipeline_model = pipeline_model or model
if name == "opencode" and (not harness_model or not pipeline_model):
resolved = self.opencode_default_model(harness[name])
harness_model = harness_model or resolved
pipeline_model = pipeline_model or resolved
if not harness_model or not pipeline_model:
continue
# Host mode: no docker_directory, executable is the detected host binary.
profile = dict(current)
profile.pop("docker_directory", None)
profile.update(executable=harness[name], model=harness_model, pipeline_model=pipeline_model)
merged[name] = profile
configured.append(name)
if not configured:
if allow_picker and sys.stdin.isatty():
print("No usable OpenCode default model was found. Choose the defaults below to finish ./install.")
command = [str(self.resolve_prefix() / "bin/pipelines"), "harness", "setup", "opencode",
"--host", "--executable", harness["opencode"]]
current = merged.get("opencode", {})
for field, flag in (("model", "--model"), ("pipeline_model", "--pipeline-model")):
if self._valid_model(current.get(field)):
command.extend([flag, current[field]])
result = self.runner(command, cwd=str(Path.home()), env=dict(self.environ))
if result.returncode:
raise ValueError("Model selection did not complete; native setup is not ready. Rerun ./install or supply --model PROVIDER/MODEL.")
return self.configure_harnesses(harness, model, allow_picker=False)
raise ValueError("No usable configured harness model was found; native setup is not ready. "
"Rerun ./install in a terminal to choose models, or use ./install --model PROVIDER/MODEL. "
"Existing credentials and harness settings were retained.")
# Keep a configured host default; only a stale Docker default must change.
default = existing.get("default")
if default not in configured or profiles.get(default, {}).get("docker_directory"):
default = "opencode" if "opencode" in configured else configured[0]
data = dict(existing)
data["default"] = default
data["profiles"] = merged
path = self.harness_config_path()
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
self._write_atomic(path, (json.dumps(data, indent=2) + "\n").encode(), 0o600)
print(f"Host harness profiles written for {', '.join(configured)}; default {default} ({path}).")
return path
def resolve_prefix(self):
default = Path.home() / ".local/share/pipelines"
raw = self.prefix_argument or self.environ.get("PIPELINES_PREFIX") or default
return Path(raw).expanduser().resolve()
def authoring_role(self, prefix, requested):
"""Fresh single-user installs author; never widen an existing server's role."""
if requested is not None:
return requested
path = prefix / MARKER
if not path.exists() and not path.is_symlink():
return True
if path.is_symlink() or not path.is_file():
raise ValueError("Refusing to read authoring role from a non-regular installation manifest")
existing = json.loads(path.read_text())
if not isinstance(existing, dict) or not isinstance(existing.get("allow_authoring", False), bool):
raise ValueError("Invalid existing authoring role; no permission changes were made")
return existing.get("allow_authoring", False)
def preserve_network_settings(self, prefix, args):
"""A plain reinstall keeps the installed addresses; flags override them."""
path = prefix / MARKER
if not path.exists() and not path.is_symlink():
return
if path.is_symlink() or not path.is_file():
raise ValueError("Refusing to read network settings from a non-regular installation manifest")
existing = json.loads(path.read_text())
web = existing.get("web") or {}
saved = {"port": existing.get("server_port"),
"ntfy_port": existing.get("ntfy_port"),
"bind_address": existing.get("bind_address"),
"web_port": web.get("port"), "web_host": web.get("host"),
"web_lan": web.get("lan"), "web_lan_plain_login": web.get("lan_plain_login")}
supplied = {argument.split("=", 1)[0] for argument in args.argv}
for name, value in saved.items():
flag = name.replace("_", "-")
if value is not None and not {"--" + flag, "--no-" + flag} & supplied:
setattr(args, name, value)
# --- prefix layout ---------------------------------------------------
def prepare_layout(self, prefix):
prefix.mkdir(parents=True, exist_ok=True)
for name in ("bin", "lib", "data", "logs", "secrets"):
(prefix / name).mkdir(exist_ok=True)
(prefix / "secrets").chmod(0o700)
(prefix / "data").chmod(0o700)
source = Path(__file__).with_name("manage.py")
target = prefix / "lib/manage.py"
if target.is_symlink():
raise ValueError("Refusing to replace a symlinked lib/manage.py")
shutil.copyfile(source, target)
os.chmod(target, 0o700)
return target
def configure_web(self, prefix, args, web_build=None):
# Tool-manager shims dispatch by their invoked name (for example Volta).
# Resolving the symlink would run volta-shim instead of node.
node = str(Path(shutil.which("node")).absolute())
version = self.runner([node, "--version"], capture_output=True, text=True)
try:
major = int(version.stdout.strip().removeprefix("v").split(".")[0])
except (ValueError, AttributeError):
major = 0
if version.returncode or major < 18:
raise ValueError("Node.js 18 or newer is required for the authenticated web proxy")
# Reuse the proxy's existing credential location; never disable its gate.
auth = Path(self.environ.get("PIPELINES_WEB_AUTH_FILE")
or Path.home() / ".config/pipelines/web-auth.json").expanduser().resolve()
if auth.exists():
data = json.loads(auth.read_text())
if (not isinstance(data, dict) or not isinstance(data.get("password"), str)
or len(data["password"]) < 8 or not isinstance(data.get("secret"), str)
or len(data["secret"]) < 32):
raise ValueError(f"Invalid existing web auth file; retained unchanged: {auth}")
if auth.stat().st_mode & 0o077:
raise ValueError(f"Web auth file must be private (chmod 600): {auth}")
else:
password = self.choose_password()
auth.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
self._write_atomic(auth, (json.dumps({"password": password,
"secret": secrets.token_urlsafe(48)}) + "\n").encode(), 0o600)
target = prefix / "lib/web"
shutil.copytree(Path(__file__).resolve().parents[1] / "web", target, dirs_exist_ok=True,
ignore=shutil.ignore_patterns("test", "node_modules", ".DS_Store"))
updates = prefix / "lib/app-updates"
updates.mkdir(parents=True, exist_ok=True)
return {"node": node, "script": str(target / "serve.mjs"),
"auth_file": str(auth), "host": args.web_host, "port": args.web_port,
"lan": args.web_lan, "lan_plain_login": args.web_lan_plain_login,
"root": self.install_app_web(prefix, args, web_build), "updates": str(updates)}
def choose_password(self):
"""The web sign-in password for a new computer, chosen by the operator.
PIPELINES_WEB_PASSWORD answers it without a terminal (automation)."""
given = self.environ.get("PIPELINES_WEB_PASSWORD")
if given is not None:
if len(given) < 8:
raise ValueError("PIPELINES_WEB_PASSWORD must be at least 8 characters")
return given
if not sys.stdin.isatty():
raise ValueError("Choose the web sign-in password: rerun ./install in a terminal, "
"or set PIPELINES_WEB_PASSWORD (at least 8 characters)")
print("This computer has no web sign-in yet. Choose the password for the web app and phone.")
while True:
first = getpass.getpass("Web password (at least 8 characters): ")
if len(first) < 8:
print("Too short; use at least 8 characters.")
continue
if getpass.getpass("Repeat the password: ") != first:
print("The passwords did not match; try again.")
continue
return first
def build_app_web(self):
"""Builds the Flutter web app in Docker (reused while its sources are
unchanged). The web app ships with the server: the install stops here,
before anything is replaced, when it cannot be built."""
if not app_build.docker_available(self.runner):
raise ValueError("Docker is not running, so the web app cannot be built. The server is never "
"installed without it; start Docker and rerun ./install.")
try:
built = app_build.build("web", runner=self.runner)
except RuntimeError as error:
raise ValueError(f"The web app build failed ({error}); nothing was installed") from None
if not (Path(built) / "index.html").is_file():
raise ValueError(f"The web app build in {built} has no index.html; nothing was installed")
return built
def install_app_web(self, prefix, args, built=None):
"""Installs the built web app for the web proxy."""
target = prefix / "lib/app-web"
app_build.install_tree(built or self.build_app_web(), target)
return str(target)
# --- one web app ---------------------------------------------------
# A standalone web proxy agent from before the native install. It served
# a second copy of the web app, so the tailnet address showed an old build.
LEGACY_WEB_LABEL = "net.awan.pipelines.web"
def retire_legacy_web(self):
"""Stops the pre-native standalone web agent; its plist is kept as
.disabled. The installed web proxy is the only web app."""
if platform.system() != "Darwin":
return
plist = Path.home() / f"Library/LaunchAgents/{self.LEGACY_WEB_LABEL}.plist"
if not plist.is_file():
return
self.runner(["launchctl", "bootout", f"gui/{os.getuid()}/{self.LEGACY_WEB_LABEL}"],
capture_output=True, text=True)
plist.rename(plist.with_name(plist.name + ".disabled"))
print(f"Retired the old standalone web agent {self.LEGACY_WEB_LABEL} (kept as {plist.name}.disabled).")
def serve_on_tailnet(self, public_url, web_port):
"""Points the tailnet HTTPS address recorded as the public web URL at
the installed web proxy, so every address serves the same web app."""
parsed = urlsplit(public_url)
if parsed.scheme != "https" or not (parsed.hostname or "").endswith(".ts.net"):
return
tailscale = shutil.which("tailscale")
if not tailscale:
raise ValueError(f"The public web URL {public_url} is a tailnet address but tailscale is not on PATH")
port = parsed.port or 443
result = self.runner([tailscale, "serve", "--bg", f"--https={port}", f"http://127.0.0.1:{web_port}"],
capture_output=True, text=True)
if result.returncode:
raise ValueError(f"tailscale serve could not point {public_url} at the web app: "
+ ((result.stderr or "").strip().splitlines() or ["failed"])[-1])
print(f"Tailnet: {public_url} serves the installed web app (port {web_port}).")
# --- build and skill -------------------------------------------------
# Detachable hosts retain Claude processes and Codex interaction callbacks
# across server restarts; the server finds them beside its own executable.
COMMANDS = ("pipelines-server", "pipelines", "pipelines-claude-host", "pipelines-codex-host")
def build(self, prefix):
"""Builds the server and CLI into bin/.next; install_binaries moves
them into place once the web app is installed too."""
staging = prefix / "bin/.next"
shutil.rmtree(staging, ignore_errors=True)
staging.mkdir(parents=True)
for command in self.COMMANDS:
result = self.runner(
["go", "build", "-trimpath", "-o", str(staging / command), "./cmd/" + command],
cwd=str(self.directory))
if result.returncode:
shutil.rmtree(staging, ignore_errors=True)
raise ValueError(f"go build failed for ./cmd/{command}; nothing was installed")
def install_binaries(self, prefix):
staging = prefix / "bin/.next"
for command in self.COMMANDS:
os.replace(staging / command, prefix / "bin" / command)
shutil.rmtree(staging, ignore_errors=True)
# --- stale-CLI-on-PATH guard -----------------------------------------
def check_path_shadow(self, prefix):
"""Warn when a different ``pipelines`` earlier on PATH shadows this install.
Detection is read-only and bounded: one ``shutil.which`` check plus one
bounded ``version --json`` read per binary. The installer never modifies,
moves, deletes or upgrades the other binary; it only names the shadowing
path and both versions so an older CLI is not used silently in front of
the freshly built server. Naming both versions requires reading each
binary once, so a detected shadow costs exactly two short, read-only
subprocesses (none when the prefix binary is the one PATH already
resolves). Returns the shadowing path, or None when the prefix binary is
the one PATH resolves.
"""
installed = prefix / "bin/pipelines"
found = shutil.which("pipelines")
if not found:
return None
found_path = Path(found).resolve()
try:
if found_path == installed.resolve():
return None
except OSError:
pass
other = self._binary_version(found_path)
current = self._binary_version(installed)
print(
f"WARNING: a different pipelines resolves earlier on PATH than the freshly "
f"installed CLI: {found_path} (version {other}) shadows {installed} "
f"(version {current}); the installer left it untouched. Put {prefix / 'bin'} "
"first on PATH so the CLI matches the running server.")
return found_path
def _binary_version(self, path):
"""Read a binary's ``version --json`` once; never a hot path."""
if not Path(path).is_file():
return "unknown"
try:
result = self.runner([str(path), "version", "--json"], capture_output=True, text=True)
except OSError:
return "unknown"
if result.returncode:
return "unknown"
text = (result.stdout or "").strip()
try:
data = json.loads(text)
except ValueError:
return text or "unknown"
if not isinstance(data, dict):
return "unknown"
version = data.get("version") or "unknown"
commit = data.get("commit")
return f"{version} (commit {commit})" if commit else str(version)
def install_skill(self, prefix):
result = self.runner([str(prefix / "bin/pipelines"), "skill", "install"], cwd=str(self.directory))
if result.returncode:
raise ValueError("pipelines skill install failed; the native install is incomplete")
# --- ntfy server build and configuration ----------------------------
def resolve_ntfy(self):
try:
return self.resolver()
except release_lookup.ReleaseError as error:
raise ValueError(
f"Could not resolve the ntfy release ({error}); refusing to build an unverified version") from None
def build_ntfy(self, prefix, resolution):
"""Compile the ntfy server from the pinned Go module on this host.
Upstream publishes no macOS server binary (its darwin asset is built with
``-tags noserver``), so every platform builds here. The module is fetched
through the Go toolchain, which verifies it against the public checksum
database, and cgo is required for ntfy's SQLite driver.
"""
path = prefix / "bin/ntfy"
if path.is_symlink():
raise ValueError("Refusing to replace a symlinked bin/ntfy")
version = resolution.get("tag") or resolution.get("version")
module = resolution.get("module") or release_lookup.NTFY_MODULE
# A previously built binary for the same resolved version is reused.
if path.is_file() and self._existing_ntfy_version(prefix) == version:
return path
self.require_compiler()
pinned = f"{module}@{version}"
result = self.runner(["go", "mod", "download", pinned], cwd=str(self.directory))
if result.returncode:
raise ValueError(f"go mod download {pinned} failed; the native install is incomplete")
cache = self.runner(["go", "env", "GOMODCACHE"], cwd=str(self.directory),
capture_output=True, text=True)
if cache.returncode or not (cache.stdout or "").strip():
raise ValueError("Could not locate the Go module cache to build ntfy from")
source = Path(cache.stdout.strip()) / f"{module}@{version}"
if not source.is_dir():
raise ValueError(f"the Go module cache has no source for {pinned}")
# Keep source work in the system temp directory, but stage the output
# beside the destination: /tmp may be a different filesystem on Linux.
with tempfile.TemporaryDirectory(prefix="pipelines-ntfy-") as temporary, \
tempfile.TemporaryDirectory(prefix=".ntfy-", dir=path.parent) as staging:
checkout = Path(temporary) / "src"
shutil.copytree(source, checkout, symlinks=True)
# The module cache is read-only and carries no built web assets; the
# upstream Makefile creates the same placeholders for local builds.
self.runner(["chmod", "-R", "u+w", str(checkout)])
(checkout / "server/docs").mkdir(parents=True, exist_ok=True)
(checkout / "server/site").mkdir(parents=True, exist_ok=True)
(checkout / "server/docs/index.html").touch()
(checkout / "server/site/app.html").touch()
built = Path(staging) / "ntfy"
env = dict(self.environ)
env["CGO_ENABLED"] = "1"
build = self.runner(
["go", "build", "-trimpath", "-tags", NTFY_BUILD_TAGS,
"-ldflags", f"-s -w -X main.version={version}", "-o", str(built), "."],
cwd=str(checkout), env=env)
if build.returncode:
raise ValueError(
"ntfy server build failed; a working C compiler is required for its SQLite driver")
if not built.is_file():
raise ValueError("ntfy server build produced no binary")
os.chmod(built, 0o755)
os.replace(built, path)
return path
def _existing_ntfy_version(self, prefix):
manifest = prefix / MARKER
if manifest.is_symlink() or not manifest.is_file():
return None
try:
data = json.loads(manifest.read_text())
except (OSError, ValueError):
return None
return (data.get("ntfy") or {}).get("version") if isinstance(data, dict) else None
def write_ntfy_config(self, prefix, base_url, ntfy_port):
directory = prefix / "data/ntfy"
directory.mkdir(mode=0o700, exist_ok=True)
directory.chmod(0o700)
path = directory / "server.yml"
if path.is_symlink():
raise ValueError("Refusing to replace a symlinked ntfy configuration")
text = (
f'base-url: "{base_url}"\n'
f'listen-http: ":{ntfy_port}"\n'
f'cache-file: "{directory}/cache.db"\n'
'cache-duration: "168h"\n'
f'auth-file: "{directory}/auth.db"\n'
'auth-default-access: "deny-all"\n'
'enable-login: true\n'
f'attachment-cache-dir: "{directory}/attachments"\n'
'attachment-expiry-duration: "168h"\n'
'attachment-total-size-limit: "1G"\n'
'attachment-file-size-limit: "1M"\n'
)
self._write_atomic(path, text.encode(), 0o600)
return path
# --- ntfy users and publishing token --------------------------------
def ntfy(self, prefix, config_path, args, capture=False, password=None):
# The account subcommands locate the server configuration through
# NTFY_CONFIG_FILE; --config is a `serve`-only flag.
env = dict(self.environ)
env["NTFY_CONFIG_FILE"] = str(config_path)
if password is not None:
# `ntfy user add` takes the password from here instead of a prompt.
env["NTFY_PASSWORD"] = password
result = self.runner([str(prefix / "bin/ntfy"), *args], capture_output=capture, text=True, env=env)
if result.returncode:
step = " ".join(args[:2])
raise ValueError(f"ntfy {step} failed; the native install is incomplete")
return result
def ensure_ntfy_auth_db(self, prefix, config_path):
"""ntfy creates its account database the first time its server runs;
the account commands refuse to run before that, so a fresh install
runs it once."""
auth = config_path.parent / "auth.db"
if auth.exists():
return
env = dict(self.environ)
process = self.popen([str(prefix / "bin/ntfy"), "serve", "--config", str(config_path)], env=env)
try:
deadline = time.monotonic() + 20
while not auth.exists():
if process.poll() is not None or time.monotonic() > deadline:
raise ValueError("ntfy did not create its account database; is its port free?")
time.sleep(0.2)
finally:
if process.poll() is None:
process.terminate()
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
process.kill()
process.wait()
def bootstrap_ntfy(self, prefix, config_path, reader_password=None):
def run_ntfy(args, capture=False, interactive=False, password=None):
return self.ntfy(prefix, config_path, list(args), capture=capture, password=password)
self.ensure_ntfy_auth_db(prefix, config_path)
# The publisher authenticates with its token; its password is never
# needed, so it is random. The phone signs in with the web password.
tokens.bootstrap_accounts(run_ntfy, prefix / "secrets/publisher-token", 0o600,
publisher_password=secrets.token_urlsafe(24),
reader_password=reader_password)
# --- Docker migration ------------------------------------------------
def docker_volumes(self):
try:
result = self.runner(["docker", "volume", "ls", "--format", "{{.Name}}"],
capture_output=True, text=True)
except FileNotFoundError:
# No Docker CLI at all: there is no Docker installation to migrate.
return ()
except OSError as error:
raise ValueError(
f"Could not inspect Docker volumes for migration ({error}); refusing to start empty") from None
if result.returncode:
# A present-but-unusable Docker daemon must not silently skip migration:
# Docker data may still exist, so fail instead of starting empty.
raise ValueError(
"Could not inspect Docker volumes for migration; refusing a possibly empty start")
names = set(result.stdout.split())
return tuple(name for name in ("pipelines_pipelines-data", "pipelines_ntfy-data") if name in names)
def _migration_image(self):
result = self.runner(["docker", "image", "ls", "--format", "{{.Repository}}:{{.Tag}}"],
capture_output=True, text=True)
if result.returncode:
return None
for line in result.stdout.splitlines():
name = line.strip()
repository = name.rsplit(":", 1)[0]
if repository == "pipelines" or repository.endswith("/pipelines") or repository.endswith("-pipelines"):
return name
return None
def _docker_copy_out(self, image, volume, inner, destination, replace):
if destination.exists() and not replace:
print(f"Keeping existing native {destination.name}; not overwriting it during migration.")
return False
name = f"pipelines-migrate-{os.getpid()}"
created = self.runner(["docker", "create", "--name", name, "--volume", f"{volume}:/mnt:ro", image],
capture_output=True, text=True)
if created.returncode:
raise ValueError(f"Could not open Docker volume {volume} for migration; install aborted")
try:
result = self.runner(["docker", "cp", f"{name}:/mnt/{inner}", str(destination)],
capture_output=True, text=True)
if result.returncode:
raise ValueError(f"Could not copy {inner} from Docker volume {volume}; install aborted")
# The live server keeps recent commits in the SQLite write-ahead log,
# so copy the -wal sidecar too or the newest definitions and runs are
# lost. SQLite recovers it on next open and rebuilds -shm itself.
for suffix in ("-wal",):
sidecar = Path(str(destination) + suffix)
sidecar.unlink(missing_ok=True)
copy = self.runner(["docker", "cp", f"{name}:/mnt/{inner}{suffix}", str(sidecar)],
capture_output=True, text=True)
if copy.returncode or not sidecar.exists() or sidecar.stat().st_size == 0:
sidecar.unlink(missing_ok=True)
finally:
self.runner(["docker", "rm", "-f", name], capture_output=True, text=True, check=False)
return True
def migrate_docker(self, prefix, replace=False):
volumes = self.docker_volumes()
if not volumes:
return False
image = self._migration_image()
if not image:
raise ValueError(
"Docker volumes exist but no local Pipelines image is available to read them; "
"refusing to pull an image. Build the Docker image or rerun without migration.")
(prefix / "data").mkdir(exist_ok=True)
if "pipelines_pipelines-data" in volumes:
self._docker_copy_out(image, "pipelines_pipelines-data", "pipelines.db",
prefix / "data/pipelines.db", replace)
if "pipelines_ntfy-data" in volumes:
directory = prefix / "data/ntfy"
directory.mkdir(mode=0o700, exist_ok=True)
self._docker_copy_out(image, "pipelines_ntfy-data", "auth.db",
directory / "auth.db", replace)
print("Existing Docker data was migrated where it did not already exist.")
return True
# --- manifest --------------------------------------------------------
def write_manifest(self, prefix, values):
path = prefix / MARKER
if path.is_symlink():
raise ValueError("Refusing to replace a symlinked installation manifest")
self._write_atomic(path, (json.dumps(values, indent=2, sort_keys=True) + "\n").encode(), 0o600)
return path
def _write_atomic(self, path, data, mode):
handle, temporary = tempfile.mkstemp(dir=str(path.parent), prefix=".pipelines-")
try:
os.chmod(temporary, mode)
with os.fdopen(handle, "wb") as file:
file.write(data)
if path.is_symlink():
raise ValueError(f"Refusing to replace a symlinked file {path}")
os.replace(temporary, path)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
# --- run -------------------------------------------------------------
def run(self, args):
python = self.require_tools()
if args.prefix:
self.prefix_argument = args.prefix
prefix = self.resolve_prefix()
self.preserve_network_settings(prefix, args)
if not 1024 <= args.port <= 65535:
raise ValueError("Invalid server port; choose a value between 1024 and 65535")
if not 1024 <= args.ntfy_port <= 65535:
raise ValueError("Invalid ntfy port; choose a value between 1024 and 65535")
if args.ntfy_port == args.port:
raise ValueError("The server and ntfy ports must differ")
if not 1024 <= args.web_port <= 65535 or args.web_port in (args.port, args.ntfy_port):
raise ValueError("Web port must be between 1024 and 65535 and differ from server/ntfy ports")
try:
ipaddress.IPv4Address(args.bind_address)
except ValueError:
raise ValueError("Invalid API bind address; use an IPv4 address such as 127.0.0.1") from None
listen, web_warnings = web_access.resolve(args.web_host, args.web_lan, args.web_lan_plain_login,
{argument.split("=", 1)[0] for argument in args.argv})
args.web_host, args.web_lan, args.web_lan_plain_login = listen["host"], listen["lan"], listen["lan_plain_login"]
allow_authoring = self.authoring_role(prefix, args.allow_authoring)
web_public_url = self.environ.get("PIPELINES_WEB_PUBLIC_URL", "")
previous = prefix / MARKER
if not web_public_url and previous.is_file() and not previous.is_symlink():
web_public_url = json.loads(previous.read_text()).get("opencode_env", {}).get("PIPELINES_WEB_PUBLIC_URL", "")
if web_public_url:
parsed = urlsplit(web_public_url)
if (parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username is not None
or parsed.password is not None or parsed.query or parsed.fragment or parsed.path not in ("", "/")
or any(ord(c) < 33 for c in web_public_url)):
raise ValueError("PIPELINES_WEB_PUBLIC_URL must be an HTTP(S) web origin without credentials, path, query or fragment")
harness = self.harness()
service_path = self.service_path()
# Reuse an already built ntfy binary and skip the release lookup, so a
# reinstall with the same version makes no network requests at all.
installed = self._existing_ntfy_version(prefix)
if (prefix / "bin/ntfy").is_file() and installed:
resolution = {"module": release_lookup.NTFY_MODULE, "version": installed, "tag": installed}
else:
self.require_compiler()
resolution = self.resolve_ntfy()
# All or nothing: the web app and the server are both built before
# either is installed, and a failed build leaves the old pair running.
web_build = self.build_app_web()
self.prepare_layout(prefix)
self.build(prefix)
web = self.configure_web(prefix, args, web_build)
self.install_binaries(prefix)
self.check_path_shadow(prefix)
self.install_skill(prefix)
# The native install launches host harnesses, so write host profiles for
# the detected harnesses and clear any Docker-era link. Uses only the
# explicit --model or the existing valid profile/host configuration.
harness_path = self.configure_harnesses(harness, args.model or "")
harness_config = json.loads(harness_path.read_text())
pipeline_model = harness_config["profiles"][harness_config["default"]]["pipeline_model"]
lan_ip = lan.detect_lan()
base_url = args.ntfy_url or f"http://{lan_ip}:{args.ntfy_port}"
config_path = self.write_ntfy_config(prefix, base_url, args.ntfy_port)
self.build_ntfy(prefix, resolution)
# Migrate existing Docker data before creating native ntfy accounts so a
# migrated auth store (the phone's credentials) is preserved rather than
# shadowed by a freshly bootstrapped one.
if args.fresh:
print("Starting empty (--fresh); existing Docker volumes are left untouched.")
else:
self.migrate_docker(prefix, args.replace_data)
self.bootstrap_ntfy(prefix, config_path,
reader_password=json.loads(Path(web["auth_file"]).read_text())["password"])
timezone = args.timezone
if not timezone:
try:
timezone = configure.host_timezone()
except (ValueError, ZoneInfoNotFoundError):
timezone = "UTC"
opencode_env = {}
for key in ("XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_STATE_HOME", "XDG_CACHE_HOME",
"OPENCODE_SERVER", "OPENCODE_DB"):
if self.environ.get(key):
opencode_env[key] = self.environ[key]
opencode_env["PIPELINES_DEFAULT_MODEL"] = args.model or pipeline_model
web_url = f"http://{web_access.default_url_host(web, lan_ip, web_access.tailnet_ipv4(self.runner))}:{args.web_port}"
opencode_env["PIPELINES_WEB_PUBLIC_URL"] = web_public_url or web_url
path_value = str(prefix / "bin") + os.pathsep + self.environ.get("PATH", "")
manifest = {
"format": 1,
"allow_authoring": allow_authoring,
"prefix": str(prefix),
"python": python,
"server_port": args.port,
"bind_address": args.bind_address,
"ntfy_port": args.ntfy_port,
"opencode": harness["opencode"],
"pi": harness["pi"],
"opencode_env": opencode_env,
"path": path_value,
"topic": "pipelines",
"public_url": base_url,
"username": "phone-reader",
"workspace": str(Path.home()),
"launcher": str(prefix / "bin/pipelines"),
"registration": None,
"label": LABEL,
"docker": "docker",
"project": "pipelines",
"timezone": timezone,
"ntfy": {"mode": "native", "bin": str(prefix / "bin/ntfy"),
"config": str(config_path), "base_url": base_url,
"version": resolution.get("tag") or resolution.get("version")},
"web": web,
# `pipelines update` re-runs this installer from here, the same way.
"source": str(self.directory),
"install_args": list(getattr(args, "argv", [])),
}
self.register_service(service_path, manifest)
self.write_manifest(prefix, manifest)
# Save the address the host CLI/TUI dials by default, so a plain
# `pipelines` (or its TUI) reaches this installation instead of a stale
# Docker address.
endpoint = f"ws://127.0.0.1:{args.port}/connect"
client_path = configure.configure_client(endpoint, self.environ)
print(f"Saved native client default: {endpoint} ({client_path}). Restart the TUI.")
override = self.environ.get("PIPELINES_URL", "")
if override and override != endpoint:
print("WARNING: PIPELINES_URL overrides the saved native address; unset it or pass --url explicitly.")
print(f"Native Pipelines installed at {prefix}."
f" ntfy {resolution.get('tag') or resolution.get('version')} was built from source.")
# Parity with the Docker installer: leave the server and ntfy running.
started = self.runner([python, str(prefix / "lib/manage.py"), "start", "--prefix", str(prefix)])
if started.returncode:
raise ValueError(
"The native services were installed but could not be started; run "
f"'PIPELINES_PREFIX={prefix} {prefix}/bin/pipelines start' to see the error. "
"Check your login service manager and that ports "
f"{args.port}, {args.ntfy_port} and {args.web_port} are free (a running Docker installation may use them).")
self.retire_legacy_web()
self.serve_on_tailnet(opencode_env["PIPELINES_WEB_PUBLIC_URL"], args.web_port)
print(f"ntfy is configured for {base_url}; server, web proxy and ntfy have login autostart ({service_path}).")
print("Phone notifications: subscribe to the 'pipelines' topic as phone-reader with the web password.")
print(f"Web: {web_url}/ (sign-in password in {web['auth_file']}; existing credentials retained).")
print("Web proxy access: " + ("loopback + tailnet" + (" + LAN" if args.web_lan else "")
if args.web_host == web_access.TAILNET else f"bound to {args.web_host}"))
for note in (*web_warnings, web_access.public_url_warning(opencode_env["PIPELINES_WEB_PUBLIC_URL"], web),
web_access.firewall_note(self.runner)):
if note:
print(note)
print("Login autostart is already enabled. To enable/start it again, run:\n "
+ shlex.join([str(prefix / "bin/pipelines"), "service", "enable"]))
print("Android app (optional, slower): ./install --apk builds it in Docker and offers it at "
f"{opencode_env['PIPELINES_WEB_PUBLIC_URL']}/updates/.")
print("Configured server role: " + ("single-user authoring enabled (web pipeline creation/publication allowed)."
if allow_authoring else "execution-only (pipeline publication disabled)."))
if args.bind_address == "0.0.0.0":
print(f"Phone client server URL: ws://{lan_ip}:{args.port}/connect "
"(use a workspace path that exists on this server).")
else:
print(f"Unauthenticated API bound to {args.bind_address}:{args.port}; use the authenticated web URL.")
app_build.print_web_urls(manifest, lan_host=lan_ip)
print("OpenCode is required (default stage and chat harness) and is set up by ./install.")
print("Next: set up your agent harnesses: pipelines harness setup claude|codex|opencode|pi")
def arguments(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--prefix", type=Path, help="Installation prefix (default: ~/.local/share/pipelines)")
parser.add_argument("--ntfy-url", help="Override the auto-detected LAN ntfy address")
parser.add_argument("--timezone", help="Override the detected timezone (fallback: UTC)")
parser.add_argument("--port", type=int, default=DEFAULT_SERVER_PORT, help="Pipelines server port")
parser.add_argument("--web-port", type=int, default=8082, help="Authenticated web proxy port (default 8082)")
parser.add_argument("--web-host", help="Web proxy access: 'tailnet' (default for new installs: loopback and "
"this machine's Tailscale addresses only) or an IPv4 bind address such as 0.0.0.0 "
"(plain HTTP on every interface); reinstalls keep the saved value")
parser.add_argument("--web-lan", action=argparse.BooleanOptionalAction, default=None,
help="Also accept plain-HTTP connections from the LAN (with --web-host tailnet); "
"sign-in over them still needs --web-lan-plain-login")
parser.add_argument("--web-lan-plain-login", action=argparse.BooleanOptionalAction, default=None,
help="Allow the password and signed-in pages over plain-HTTP LAN connections "
"(unencrypted; only on a trusted network)")
parser.add_argument("--bind-address", default=DEFAULT_BIND_ADDRESS,
help="Unauthenticated server bind address (default 127.0.0.1; expose only the web proxy)")
parser.add_argument("--ntfy-port", type=int, default=DEFAULT_NTFY_PORT, help="ntfy listen port")
parser.add_argument("--model", help="Explicit PROVIDER/MODEL for starter defaults and missing profile fields; "
"otherwise reuse saved/default OpenCode models or choose within this installer")
role = parser.add_mutually_exclusive_group()
role.add_argument("--allow-authoring", dest="allow_authoring", action="store_true", default=None,
help="Explicitly enable pipeline publication; fresh native installs default to enabled")
role.add_argument("--no-authoring", dest="allow_authoring", action="store_false",
help="Execution-only server; disable pipeline publication (reinstalls otherwise preserve the recorded role)")
parser.add_argument("--fresh", action="store_true",
help="Start with empty data even when an earlier Docker installation's volumes exist")
parser.add_argument("--replace-data", action="store_true",
help="Allow Docker migration to replace an existing native database/auth store")
parsed = parser.parse_args(argv)
parsed.argv = list(sys.argv[1:] if argv is None else argv)
return parsed
if __name__ == "__main__":
try:
Setup(Path(__file__).resolve().parents[1]).run(arguments())
except KeyboardInterrupt:
print("\nNative setup canceled; existing configuration and data were retained.", file=sys.stderr)
sys.exit(130)
except (ValueError, OSError, json.JSONDecodeError, subprocess.SubprocessError,
ZoneInfoNotFoundError) as error:
sys.exit(f"Native setup incomplete: {error}")