Files
pipelines/scripts/setup-native.py
T
alan da6daa9db5 Resolve configured host models during native installation
Read OpenCode V2's resolved default Model.Info through its authenticated CLI API from HOME when saved profile fields are missing. Preserve valid saved models, explicit --model and unrelated settings. Record the selected stage model in the native server environment so web starters have an actual default.

Run the existing read-only model picker inside ./install when a terminal needs a choice; fail noninteractive or cancelled/incomplete setup before claiming ready or starting services. Do not parse JSONC, open login flows, create sessions or run inference. Document the one-step configuration path.

Checks: 61 mocked native installer/lifecycle tests, 8 entrypoint tests, sh -n install install.sh, git diff --check. API/schema checked against official OpenCode V2 documentation. No operator CLI/service/model calls, live install, restart, deployment or native Linux verification.
2026-10-20 21:56:32 +01:00

782 lines
40 KiB
Python

#!/usr/bin/env python3
"""Install Pipelines natively on the host (macOS and Linux).
The installer only detects the tools it needs (go, python3, node, git, curl and a C
compiler for ntfy's SQLite driver); it never installs or upgrades them, and it
never installs, upgrades or downloads OpenCode or Pi. It builds the server, CLI
and ntfy from source into a per-user prefix, generates an ntfy configuration that
mirrors compose.yaml, creates the publishing/phone accounts and writes the
manifest scripts/manage.py loads. ntfy is compiled from its pinned Go module on
every host because upstream publishes no macOS server binary. It reuses existing
host OpenCode configuration; it writes host (non-Docker) harness profiles for the
harnesses it detects and never invents a model.
"""
import argparse
import importlib.util
import ipaddress
import json
import os
import platform
import plistlib
import secrets
import shlex
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
from zoneinfo import ZoneInfoNotFoundError
def module(name, file):
spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(file))
result = importlib.util.module_from_spec(spec)
spec.loader.exec_module(result)
return result
configure = module("configure_compose", "configure-compose.py")
lan = module("lan_address", "lan-address.py")
release_lookup = module("resolve_releases", "resolve-releases.py")
tokens = module("save_ntfy_token", "save-ntfy-token.py")
DEFAULT_PREFIX = Path.home() / ".local/share/pipelines"
MARKER = "pipelines-install.json"
LABEL = "net.a-wan.pipelines"
SERVICE_MARKER = "Managed by Pipelines native installer"
REQUIRED_TOOLS = ("go", "python3", "git", "curl", "node")
# ntfy's SQLite driver (mattn/go-sqlite3) is cgo, so a C compiler is required to
# build the server. Upstream's macOS release asset is client-only, so both
# platforms compile ntfy from the pinned Go module.
C_COMPILERS = ("cc", "clang", "gcc")
NTFY_BUILD_TAGS = "sqlite_omit_load_extension,osusergo,netgo"
DEFAULT_NTFY_PORT = 8090
DEFAULT_SERVER_PORT = 8080
# Only the authenticated web proxy is exposed; the control API stays loopback.
DEFAULT_BIND_ADDRESS = "127.0.0.1"
class Setup:
def __init__(self, directory, prefix=None, runner=subprocess.run, resolver=None, environ=None):
self.directory = Path(directory)
self.prefix_argument = prefix
self.runner = runner
self.resolver = resolver or release_lookup.resolve_ntfy
self.environ = environ if environ is not None else os.environ
# --- detection -------------------------------------------------------
def require_compiler(self):
"""ntfy is built locally, so a C toolchain must already be present."""
for compiler in C_COMPILERS:
path = shutil.which(compiler)
if path:
return str(Path(path).resolve())
raise ValueError(
"No C compiler found (tried " + ", ".join(C_COMPILERS)
+ "). ntfy's SQLite driver is cgo, so the server is compiled locally; "
"install a C toolchain (Xcode Command Line Tools on macOS, gcc or clang on Linux) and rerun.")
def require_tools(self):
found, missing = {}, []
for tool in REQUIRED_TOOLS:
path = shutil.which(tool)
if path:
found[tool] = str(Path(path).resolve())
else:
missing.append(tool)
if missing:
raise ValueError(
"Missing required tool(s): " + ", ".join(missing)
+ ". Install them yourself and rerun; this installer never installs tools or uses a package manager.")
return found["python3"]
def harness(self):
"""Detect existing harnesses only; never install, upgrade or download them."""
opencode = shutil.which("opencode")
if not opencode:
raise ValueError(
"OpenCode was not found on PATH. Install OpenCode yourself (this installer never installs, "
"upgrades or downloads it) and rerun; model selection happens in your existing OpenCode configuration.")
pi = shutil.which("pi")
return {"opencode": str(Path(opencode).resolve()), "pi": str(Path(pi).resolve()) if pi else ""}
def service_path(self):
system = platform.system()
if system == "Darwin":
path = Path.home() / "Library/LaunchAgents" / (LABEL + ".plist")
tool = "launchctl"
elif system == "Linux":
root = Path(self.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config")
path = root / "systemd/user" / (LABEL + ".service")
tool = "systemctl"
else:
raise ValueError("Native installation supports macOS (launchd) and Linux (systemd --user) only")
if not shutil.which(tool):
raise ValueError(f"{tool} is required for login autostart; run setup in a logged-in user session")
if path.is_symlink() or (path.exists() and SERVICE_MARKER not in path.read_text()):
raise ValueError(f"Refusing to replace an unmanaged autostart definition: {path}")
return path
def register_service(self, path, manifest):
"""Register the paired native services, not a second standalone server."""
prefix = Path(manifest["prefix"])
command = [manifest["python"], str(prefix / "lib/manage.py"),
"supervise", "--prefix", str(prefix)]
if platform.system() == "Darwin":
content = plistlib.dumps({
"Label": LABEL, "Comment": SERVICE_MARKER,
"ProgramArguments": command, "RunAtLoad": True,
"KeepAlive": {"SuccessfulExit": False}, "ThrottleInterval": 5,
"EnvironmentVariables": {"HOME": str(Path.home()), "PATH": manifest["path"]},
"StandardOutPath": str(prefix / "logs/supervisor.log"),
"StandardErrorPath": str(prefix / "logs/supervisor.log"), "Umask": 0o077,
})
else:
def quote(value):
if any(char in value for char in "\r\n\x00"):
raise ValueError("Autostart paths cannot contain line breaks or NUL")
return '"' + value.replace('\\', '\\\\').replace('"', '\\"').replace('%', '%%').replace('$', '$$') + '"'
content = (f"# {SERVICE_MARKER}\n[Unit]\nDescription=Pipelines native services\n"
"\n[Service]\nType=simple\nExecStart=" + " ".join(map(quote, command))
+ "\nRestart=on-failure\nRestartSec=5\nTimeoutStopSec=90\nUMask=0077\n"
"\n[Install]\nWantedBy=default.target\n").encode()
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
self._write_atomic(path, content, 0o600)
manifest["registration"] = str(path)
# --- host harness profiles ------------------------------------------
MODEL_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_./:-"
@classmethod
def _valid_model(cls, value):
if not isinstance(value, str) or len(value) > 501 or "/" not in value:
return False
provider, model = value.split("/", 1)
if not provider or not model or value in ("default/default", "provider/model"):
return False
return all(char in cls.MODEL_CHARS for char in value)
def harness_config_path(self):
root = Path(self.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config")
return root / "pipelines" / "harnesses.json"
def _existing_harness_config(self):
path = self.harness_config_path()
if path.is_symlink() or not path.is_file():
return {}
try:
data = json.loads(path.read_text())
except (OSError, ValueError):
return {}
return data if isinstance(data, dict) else {}
def opencode_default_model(self, executable):
"""Read V2's resolved Model.Info, with OpenCode's own JSONC/auth handling.
Run in HOME, not the install checkout, so a repository's model override
does not accidentally become the host installation's global default.
No login, sessions, prompts or inference are involved.
"""
try:
result = self.runner([executable, "api", "get", "/api/model/default"],
cwd=str(Path.home()), env=dict(self.environ),
capture_output=True, text=True, timeout=30)
if result.returncode or len(result.stdout or "") > 4 * 1024 * 1024:
return ""
response = json.loads(result.stdout)
data = response.get("data") if isinstance(response, dict) else None
if not isinstance(data, dict) or data.get("enabled") is not True or data.get("status") == "deprecated":
return ""
provider, model = data.get("providerID"), data.get("modelID")
if not isinstance(provider, str) or "/" in provider or not isinstance(model, str):
return ""
value = provider + "/" + model
return value if self._valid_model(value) else ""
except (OSError, ValueError, subprocess.SubprocessError):
# Never echo raw CLI output: provider discovery can include secrets.
return ""
def configure_harnesses(self, harness, model="", allow_picker=True):
"""Write host (non-Docker) profiles for the harnesses detected on PATH.
The native install launches host harnesses, so a Docker-era
``docker_directory`` must be cleared rather than kept. Models come from
existing valid profile, the explicit ``--model`` (also recorded as
PIPELINES_DEFAULT_MODEL), or OpenCode's resolved host default; they are
never invented. Unrelated keys and
profiles for harnesses that are not detected are preserved, matching
harness.Save. No harness is installed, upgraded or downloaded.
"""
if model and not self._valid_model(model):
raise ValueError("Invalid --model; supply an actual PROVIDER/MODEL, not a default placeholder")
existing = self._existing_harness_config()
profiles = existing.get("profiles")
if not isinstance(profiles, dict):
profiles = {}
merged = {name: dict(value) for name, value in profiles.items() if isinstance(value, dict)}
order = [name for name in ("opencode", "pi") if harness.get(name)]
configured = []
for name in order:
current = merged.get(name, {})
harness_model = current.get("model") if self._valid_model(current.get("model")) else ""
pipeline_model = current.get("pipeline_model") if self._valid_model(current.get("pipeline_model")) else ""
if name == "opencode" and self._valid_model(model):
harness_model = harness_model or model
pipeline_model = pipeline_model or model
if name == "opencode" and (not harness_model or not pipeline_model):
resolved = self.opencode_default_model(harness[name])
harness_model = harness_model or resolved
pipeline_model = pipeline_model or resolved
if not harness_model or not pipeline_model:
continue
# Host mode: no docker_directory, executable is the detected host binary.
profile = dict(current)
profile.pop("docker_directory", None)
profile.update(executable=harness[name], model=harness_model, pipeline_model=pipeline_model)
merged[name] = profile
configured.append(name)
if not configured:
if allow_picker and sys.stdin.isatty():
print("No usable OpenCode default model was found. Choose the defaults below to finish ./install.")
command = [str(self.resolve_prefix() / "bin/pipelines"), "harness", "setup", "opencode",
"--host", "--executable", harness["opencode"]]
current = merged.get("opencode", {})
for field, flag in (("model", "--model"), ("pipeline_model", "--pipeline-model")):
if self._valid_model(current.get(field)):
command.extend([flag, current[field]])
result = self.runner(command, cwd=str(Path.home()), env=dict(self.environ))
if result.returncode:
raise ValueError("Model selection did not complete; native setup is not ready. Rerun ./install or supply --model PROVIDER/MODEL.")
return self.configure_harnesses(harness, model, allow_picker=False)
raise ValueError("No usable configured harness model was found; native setup is not ready. "
"Rerun ./install in a terminal to choose models, or use ./install --model PROVIDER/MODEL. "
"Existing credentials and harness settings were retained.")
# Keep a configured host default; only a stale Docker default must change.
default = existing.get("default")
if default not in configured or profiles.get(default, {}).get("docker_directory"):
default = "opencode" if "opencode" in configured else configured[0]
data = dict(existing)
data["default"] = default
data["profiles"] = merged
path = self.harness_config_path()
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
self._write_atomic(path, (json.dumps(data, indent=2) + "\n").encode(), 0o600)
print(f"Host harness profiles written for {', '.join(configured)}; default {default} ({path}).")
return path
def resolve_prefix(self):
default = Path.home() / ".local/share/pipelines"
raw = self.prefix_argument or self.environ.get("PIPELINES_PREFIX") or default
return Path(raw).expanduser().resolve()
def authoring_role(self, prefix, requested):
"""Fresh single-user installs author; never widen an existing server's role."""
if requested is not None:
return requested
path = prefix / MARKER
if not path.exists() and not path.is_symlink():
return True
if path.is_symlink() or not path.is_file():
raise ValueError("Refusing to read authoring role from a non-regular installation manifest")
existing = json.loads(path.read_text())
if not isinstance(existing, dict) or not isinstance(existing.get("allow_authoring", False), bool):
raise ValueError("Invalid existing authoring role; no permission changes were made")
return existing.get("allow_authoring", False)
# --- prefix layout ---------------------------------------------------
def prepare_layout(self, prefix):
prefix.mkdir(parents=True, exist_ok=True)
for name in ("bin", "lib", "data", "logs", "secrets"):
(prefix / name).mkdir(exist_ok=True)
(prefix / "secrets").chmod(0o700)
(prefix / "data").chmod(0o700)
source = Path(__file__).with_name("manage.py")
target = prefix / "lib/manage.py"
if target.is_symlink():
raise ValueError("Refusing to replace a symlinked lib/manage.py")
shutil.copyfile(source, target)
os.chmod(target, 0o700)
return target
def configure_web(self, prefix, args):
node = str(Path(shutil.which("node")).resolve())
version = self.runner([node, "--version"], capture_output=True, text=True)
try:
major = int(version.stdout.strip().removeprefix("v").split(".")[0])
except (ValueError, AttributeError):
major = 0
if version.returncode or major < 18:
raise ValueError("Node.js 18 or newer is required for the authenticated web proxy")
# Reuse the proxy's existing credential location; never disable its gate.
auth = Path(self.environ.get("PIPELINES_WEB_AUTH_FILE")
or Path.home() / ".config/pipelines/web-auth.json").expanduser().resolve()
if auth.exists():
data = json.loads(auth.read_text())
if (not isinstance(data, dict) or not isinstance(data.get("password"), str)
or len(data["password"]) < 8 or not isinstance(data.get("secret"), str)
or len(data["secret"]) < 32):
raise ValueError(f"Invalid existing web auth file; retained unchanged: {auth}")
if auth.stat().st_mode & 0o077:
raise ValueError(f"Web auth file must be private (chmod 600): {auth}")
else:
auth.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
self._write_atomic(auth, (json.dumps({"password": secrets.token_urlsafe(18),
"secret": secrets.token_urlsafe(48)}) + "\n").encode(), 0o600)
target = prefix / "lib/web"
shutil.copytree(Path(__file__).resolve().parents[1] / "web", target, dirs_exist_ok=True,
ignore=shutil.ignore_patterns("test", "node_modules", ".DS_Store"))
return {"node": node, "script": str(target / "serve.mjs"),
"auth_file": str(auth), "host": args.web_host, "port": args.web_port}
# --- build and skill -------------------------------------------------
def build(self, prefix):
for command in ("pipelines-server", "pipelines"):
result = self.runner(
["go", "build", "-trimpath", "-o", str(prefix / "bin" / command), "./cmd/" + command],
cwd=str(self.directory))
if result.returncode:
raise ValueError(f"go build failed for ./cmd/{command}; the native install is incomplete")
# --- stale-CLI-on-PATH guard -----------------------------------------
def check_path_shadow(self, prefix):
"""Warn when a different ``pipelines`` earlier on PATH shadows this install.
Detection is read-only and bounded: one ``shutil.which`` check plus one
bounded ``version --json`` read per binary. The installer never modifies,
moves, deletes or upgrades the other binary; it only names the shadowing
path and both versions so an older CLI is not used silently in front of
the freshly built server. Naming both versions requires reading each
binary once, so a detected shadow costs exactly two short, read-only
subprocesses (none when the prefix binary is the one PATH already
resolves). Returns the shadowing path, or None when the prefix binary is
the one PATH resolves.
"""
installed = prefix / "bin/pipelines"
found = shutil.which("pipelines")
if not found:
return None
found_path = Path(found).resolve()
try:
if found_path == installed.resolve():
return None
except OSError:
pass
other = self._binary_version(found_path)
current = self._binary_version(installed)
print(
f"WARNING: a different pipelines resolves earlier on PATH than the freshly "
f"installed CLI: {found_path} (version {other}) shadows {installed} "
f"(version {current}); the installer left it untouched. Put {prefix / 'bin'} "
"first on PATH so the CLI matches the running server.")
return found_path
def _binary_version(self, path):
"""Read a binary's ``version --json`` once; never a hot path."""
if not Path(path).is_file():
return "unknown"
try:
result = self.runner([str(path), "version", "--json"], capture_output=True, text=True)
except OSError:
return "unknown"
if result.returncode:
return "unknown"
text = (result.stdout or "").strip()
try:
data = json.loads(text)
except ValueError:
return text or "unknown"
if not isinstance(data, dict):
return "unknown"
version = data.get("version") or "unknown"
commit = data.get("commit")
return f"{version} (commit {commit})" if commit else str(version)
def install_skill(self, prefix):
result = self.runner([str(prefix / "bin/pipelines"), "skill", "install"], cwd=str(self.directory))
if result.returncode:
raise ValueError("pipelines skill install failed; the native install is incomplete")
# --- ntfy server build and configuration ----------------------------
def resolve_ntfy(self):
try:
return self.resolver()
except release_lookup.ReleaseError as error:
raise ValueError(
f"Could not resolve the ntfy release ({error}); refusing to build an unverified version") from None
def build_ntfy(self, prefix, resolution):
"""Compile the ntfy server from the pinned Go module on this host.
Upstream publishes no macOS server binary (its darwin asset is built with
``-tags noserver``), so every platform builds here. The module is fetched
through the Go toolchain, which verifies it against the public checksum
database, and cgo is required for ntfy's SQLite driver.
"""
path = prefix / "bin/ntfy"
if path.is_symlink():
raise ValueError("Refusing to replace a symlinked bin/ntfy")
version = resolution.get("tag") or resolution.get("version")
module = resolution.get("module") or release_lookup.NTFY_MODULE
# A previously built binary for the same resolved version is reused.
if path.is_file() and self._existing_ntfy_version(prefix) == version:
return path
self.require_compiler()
pinned = f"{module}@{version}"
result = self.runner(["go", "mod", "download", pinned], cwd=str(self.directory))
if result.returncode:
raise ValueError(f"go mod download {pinned} failed; the native install is incomplete")
cache = self.runner(["go", "env", "GOMODCACHE"], cwd=str(self.directory),
capture_output=True, text=True)
if cache.returncode or not (cache.stdout or "").strip():
raise ValueError("Could not locate the Go module cache to build ntfy from")
source = Path(cache.stdout.strip()) / f"{module}@{version}"
if not source.is_dir():
raise ValueError(f"the Go module cache has no source for {pinned}")
with tempfile.TemporaryDirectory(prefix="pipelines-ntfy-") as temporary:
checkout = Path(temporary) / "src"
shutil.copytree(source, checkout, symlinks=True)
# The module cache is read-only and carries no built web assets; the
# upstream Makefile creates the same placeholders for local builds.
self.runner(["chmod", "-R", "u+w", str(checkout)])
(checkout / "server/docs").mkdir(parents=True, exist_ok=True)
(checkout / "server/site").mkdir(parents=True, exist_ok=True)
(checkout / "server/docs/index.html").touch()
(checkout / "server/site/app.html").touch()
built = Path(temporary) / "ntfy"
env = dict(self.environ)
env["CGO_ENABLED"] = "1"
build = self.runner(
["go", "build", "-trimpath", "-tags", NTFY_BUILD_TAGS,
"-ldflags", f"-s -w -X main.version={version}", "-o", str(built), "."],
cwd=str(checkout), env=env)
if build.returncode:
raise ValueError(
"ntfy server build failed; a working C compiler is required for its SQLite driver")
if not built.is_file():
raise ValueError("ntfy server build produced no binary")
os.chmod(built, 0o755)
os.replace(built, path)
return path
def _existing_ntfy_version(self, prefix):
manifest = prefix / MARKER
if manifest.is_symlink() or not manifest.is_file():
return None
try:
data = json.loads(manifest.read_text())
except (OSError, ValueError):
return None
return (data.get("ntfy") or {}).get("version") if isinstance(data, dict) else None
def write_ntfy_config(self, prefix, base_url, ntfy_port):
directory = prefix / "data/ntfy"
directory.mkdir(mode=0o700, exist_ok=True)
directory.chmod(0o700)
path = directory / "server.yml"
if path.is_symlink():
raise ValueError("Refusing to replace a symlinked ntfy configuration")
text = (
f'base-url: "{base_url}"\n'
f'listen-http: ":{ntfy_port}"\n'
f'cache-file: "{directory}/cache.db"\n'
'cache-duration: "168h"\n'
f'auth-file: "{directory}/auth.db"\n'
'auth-default-access: "deny-all"\n'
'enable-login: true\n'
f'attachment-cache-dir: "{directory}/attachments"\n'
'attachment-expiry-duration: "168h"\n'
'attachment-total-size-limit: "1G"\n'
'attachment-file-size-limit: "1M"\n'
)
self._write_atomic(path, text.encode(), 0o600)
return path
# --- ntfy users and publishing token --------------------------------
def ntfy(self, prefix, config_path, args, capture=False):
# The account subcommands locate the server configuration through
# NTFY_CONFIG_FILE; --config is a `serve`-only flag.
env = dict(self.environ)
env["NTFY_CONFIG_FILE"] = str(config_path)
result = self.runner([str(prefix / "bin/ntfy"), *args], capture_output=capture, text=True, env=env)
if result.returncode:
step = " ".join(args[:2])
raise ValueError(f"ntfy {step} failed; the native install is incomplete")
return result
def bootstrap_ntfy(self, prefix, config_path):
def run_ntfy(args, capture=False, interactive=False):
return self.ntfy(prefix, config_path, list(args), capture=capture)
tokens.bootstrap_accounts(run_ntfy, prefix / "secrets/publisher-token", 0o600)
# --- Docker migration ------------------------------------------------
def docker_volumes(self):
try:
result = self.runner(["docker", "volume", "ls", "--format", "{{.Name}}"],
capture_output=True, text=True)
except FileNotFoundError:
# No Docker CLI at all: there is no Docker installation to migrate.
return ()
except OSError as error:
raise ValueError(
f"Could not inspect Docker volumes for migration ({error}); refusing to start empty") from None
if result.returncode:
# A present-but-unusable Docker daemon must not silently skip migration:
# Docker data may still exist, so fail instead of starting empty.
raise ValueError(
"Could not inspect Docker volumes for migration; refusing a possibly empty start")
names = set(result.stdout.split())
return tuple(name for name in ("pipelines_pipelines-data", "pipelines_ntfy-data") if name in names)
def _migration_image(self):
result = self.runner(["docker", "image", "ls", "--format", "{{.Repository}}:{{.Tag}}"],
capture_output=True, text=True)
if result.returncode:
return None
for line in result.stdout.splitlines():
name = line.strip()
repository = name.rsplit(":", 1)[0]
if repository == "pipelines" or repository.endswith("/pipelines") or repository.endswith("-pipelines"):
return name
return None
def _docker_copy_out(self, image, volume, inner, destination, replace):
if destination.exists() and not replace:
print(f"Keeping existing native {destination.name}; not overwriting it during migration.")
return False
name = f"pipelines-migrate-{os.getpid()}"
created = self.runner(["docker", "create", "--name", name, "--volume", f"{volume}:/mnt:ro", image],
capture_output=True, text=True)
if created.returncode:
raise ValueError(f"Could not open Docker volume {volume} for migration; install aborted")
try:
result = self.runner(["docker", "cp", f"{name}:/mnt/{inner}", str(destination)],
capture_output=True, text=True)
if result.returncode:
raise ValueError(f"Could not copy {inner} from Docker volume {volume}; install aborted")
# The live server keeps recent commits in the SQLite write-ahead log,
# so copy the -wal sidecar too or the newest definitions and runs are
# lost. SQLite recovers it on next open and rebuilds -shm itself.
for suffix in ("-wal",):
sidecar = Path(str(destination) + suffix)
sidecar.unlink(missing_ok=True)
copy = self.runner(["docker", "cp", f"{name}:/mnt/{inner}{suffix}", str(sidecar)],
capture_output=True, text=True)
if copy.returncode or not sidecar.exists() or sidecar.stat().st_size == 0:
sidecar.unlink(missing_ok=True)
finally:
self.runner(["docker", "rm", "-f", name], capture_output=True, text=True, check=False)
return True
def migrate_docker(self, prefix, replace=False):
volumes = self.docker_volumes()
if not volumes:
return False
image = self._migration_image()
if not image:
raise ValueError(
"Docker volumes exist but no local Pipelines image is available to read them; "
"refusing to pull an image. Build the Docker image or rerun without migration.")
(prefix / "data").mkdir(exist_ok=True)
if "pipelines_pipelines-data" in volumes:
self._docker_copy_out(image, "pipelines_pipelines-data", "pipelines.db",
prefix / "data/pipelines.db", replace)
if "pipelines_ntfy-data" in volumes:
directory = prefix / "data/ntfy"
directory.mkdir(mode=0o700, exist_ok=True)
self._docker_copy_out(image, "pipelines_ntfy-data", "auth.db",
directory / "auth.db", replace)
print("Existing Docker data was migrated where it did not already exist.")
return True
# --- manifest --------------------------------------------------------
def write_manifest(self, prefix, values):
path = prefix / MARKER
if path.is_symlink():
raise ValueError("Refusing to replace a symlinked installation manifest")
self._write_atomic(path, (json.dumps(values, indent=2, sort_keys=True) + "\n").encode(), 0o600)
return path
def _write_atomic(self, path, data, mode):
handle, temporary = tempfile.mkstemp(dir=str(path.parent), prefix=".pipelines-")
try:
os.chmod(temporary, mode)
with os.fdopen(handle, "wb") as file:
file.write(data)
if path.is_symlink():
raise ValueError(f"Refusing to replace a symlinked file {path}")
os.replace(temporary, path)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
# --- run -------------------------------------------------------------
def run(self, args):
python = self.require_tools()
if args.prefix:
self.prefix_argument = args.prefix
if not 1024 <= args.port <= 65535:
raise ValueError("Invalid server port; choose a value between 1024 and 65535")
if not 1024 <= args.ntfy_port <= 65535:
raise ValueError("Invalid ntfy port; choose a value between 1024 and 65535")
if args.ntfy_port == args.port:
raise ValueError("The server and ntfy ports must differ")
if not 1024 <= args.web_port <= 65535 or args.web_port in (args.port, args.ntfy_port):
raise ValueError("Web port must be between 1024 and 65535 and differ from server/ntfy ports")
try:
ipaddress.IPv4Address(args.bind_address)
ipaddress.IPv4Address(args.web_host)
except ValueError:
raise ValueError("Invalid API/web bind address; use an IPv4 address such as 0.0.0.0 or 127.0.0.1") from None
prefix = self.resolve_prefix()
allow_authoring = self.authoring_role(prefix, args.allow_authoring)
harness = self.harness()
service_path = self.service_path()
# Reuse an already built ntfy binary and skip the release lookup, so a
# reinstall with the same version makes no network requests at all.
installed = self._existing_ntfy_version(prefix)
if (prefix / "bin/ntfy").is_file() and installed:
resolution = {"module": release_lookup.NTFY_MODULE, "version": installed, "tag": installed}
else:
self.require_compiler()
resolution = self.resolve_ntfy()
self.prepare_layout(prefix)
web = self.configure_web(prefix, args)
self.build(prefix)
self.check_path_shadow(prefix)
self.install_skill(prefix)
# The native install launches host harnesses, so write host profiles for
# the detected harnesses and clear any Docker-era link. Uses only the
# explicit --model or the existing valid profile/host configuration.
harness_path = self.configure_harnesses(harness, args.model or "")
harness_config = json.loads(harness_path.read_text())
pipeline_model = harness_config["profiles"][harness_config["default"]]["pipeline_model"]
lan_ip = lan.detect_lan()
base_url = args.ntfy_url or f"http://{lan_ip}:{args.ntfy_port}"
config_path = self.write_ntfy_config(prefix, base_url, args.ntfy_port)
self.build_ntfy(prefix, resolution)
# Migrate existing Docker data before creating native ntfy accounts so a
# migrated auth store (the phone's credentials) is preserved rather than
# shadowed by a freshly bootstrapped one.
self.migrate_docker(prefix, args.replace_data)
self.bootstrap_ntfy(prefix, config_path)
timezone = args.timezone
if not timezone:
try:
timezone = configure.host_timezone()
except (ValueError, ZoneInfoNotFoundError):
timezone = "UTC"
opencode_env = {}
for key in ("XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_STATE_HOME", "XDG_CACHE_HOME",
"OPENCODE_SERVER", "OPENCODE_DB"):
if self.environ.get(key):
opencode_env[key] = self.environ[key]
opencode_env["PIPELINES_DEFAULT_MODEL"] = args.model or pipeline_model
path_value = str(prefix / "bin") + os.pathsep + self.environ.get("PATH", "")
manifest = {
"format": 1,
"allow_authoring": allow_authoring,
"prefix": str(prefix),
"python": python,
"server_port": args.port,
"bind_address": args.bind_address,
"ntfy_port": args.ntfy_port,
"opencode": harness["opencode"],
"pi": harness["pi"],
"opencode_env": opencode_env,
"path": path_value,
"topic": "pipelines",
"public_url": base_url,
"username": "phone-reader",
"workspace": str(Path.home()),
"launcher": str(prefix / "bin/pipelines"),
"registration": None,
"label": LABEL,
"docker": "docker",
"project": "pipelines",
"timezone": timezone,
"ntfy": {"mode": "native", "bin": str(prefix / "bin/ntfy"),
"config": str(config_path), "base_url": base_url,
"version": resolution.get("tag") or resolution.get("version")},
"web": web,
}
self.register_service(service_path, manifest)
self.write_manifest(prefix, manifest)
# Save the address the host CLI/TUI dials by default, so a plain
# `pipelines` (or its TUI) reaches this installation instead of a stale
# Docker address.
endpoint = f"ws://127.0.0.1:{args.port}/connect"
client_path = configure.configure_client(endpoint, self.environ)
print(f"Saved native client default: {endpoint} ({client_path}). Restart the TUI.")
override = self.environ.get("PIPELINES_URL", "")
if override and override != endpoint:
print("WARNING: PIPELINES_URL overrides the saved native address; unset it or pass --url explicitly.")
print(f"Native Pipelines installed at {prefix}."
f" ntfy {resolution.get('tag') or resolution.get('version')} was built from source.")
# Parity with the Docker installer: leave the server and ntfy running.
started = self.runner([python, str(prefix / "lib/manage.py"), "start", "--prefix", str(prefix)])
if started.returncode:
raise ValueError(
"The native services were installed but could not be started; run "
f"'PIPELINES_PREFIX={prefix} {prefix}/bin/pipelines start' to see the error. "
"Check your login service manager and that ports "
f"{args.port}, {args.ntfy_port} and {args.web_port} are free (a running Docker installation may use them).")
print(f"ntfy is configured for {base_url}; server, web proxy and ntfy have login autostart ({service_path}).")
print(f"Web: http://{lan_ip if args.web_host == '0.0.0.0' else args.web_host}:{args.web_port}/ "
f"(sign-in password in {web['auth_file']}; existing credentials retained).")
print("Login autostart is already enabled. To enable/start it again, run:\n "
+ shlex.join([str(prefix / "bin/pipelines"), "service", "enable"]))
print("Configured server role: " + ("single-user authoring enabled (web pipeline creation/publication allowed)."
if allow_authoring else "execution-only (pipeline publication disabled)."))
if args.bind_address == "0.0.0.0":
print(f"Phone client server URL: ws://{lan_ip}:{args.port}/connect "
"(use a workspace path that exists on this server).")
else:
print(f"Unauthenticated API bound to {args.bind_address}:{args.port}; use the authenticated web URL.")
def arguments(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--prefix", type=Path, help="Installation prefix (default: ~/.local/share/pipelines)")
parser.add_argument("--ntfy-url", help="Override the auto-detected LAN ntfy address")
parser.add_argument("--timezone", help="Override the detected timezone (fallback: UTC)")
parser.add_argument("--port", type=int, default=DEFAULT_SERVER_PORT, help="Pipelines server port")
parser.add_argument("--web-port", type=int, default=8082, help="Authenticated web proxy port (default 8082)")
parser.add_argument("--web-host", default="0.0.0.0", help="Web proxy bind address (default 0.0.0.0)")
parser.add_argument("--bind-address", default=DEFAULT_BIND_ADDRESS,
help="Unauthenticated server bind address (default 127.0.0.1; expose only the web proxy)")
parser.add_argument("--ntfy-port", type=int, default=DEFAULT_NTFY_PORT, help="ntfy listen port")
parser.add_argument("--model", help="Explicit PROVIDER/MODEL for starter defaults and missing profile fields; "
"otherwise reuse saved/default OpenCode models or choose within this installer")
role = parser.add_mutually_exclusive_group()
role.add_argument("--allow-authoring", dest="allow_authoring", action="store_true", default=None,
help="Explicitly enable pipeline publication; fresh native installs default to enabled")
role.add_argument("--no-authoring", dest="allow_authoring", action="store_false",
help="Execution-only server; disable pipeline publication (reinstalls otherwise preserve the recorded role)")
parser.add_argument("--replace-data", action="store_true",
help="Allow Docker migration to replace an existing native database/auth store")
return parser.parse_args(argv)
if __name__ == "__main__":
try:
Setup(Path(__file__).resolve().parents[1]).run(arguments())
except KeyboardInterrupt:
print("\nNative setup canceled; existing configuration and data were retained.", file=sys.stderr)
sys.exit(130)
except (ValueError, OSError, json.JSONDecodeError, subprocess.SubprocessError,
ZoneInfoNotFoundError) as error:
sys.exit(f"Native setup incomplete: {error}")