#!/usr/bin/env python3 """Install Pipelines natively on the host (macOS and Linux). The installer only detects the tools it needs (go, python3, git, curl and a C compiler for ntfy's SQLite driver); it never installs or upgrades them, and it never installs, upgrades or downloads OpenCode or Pi. It builds the server, CLI and ntfy from source into a per-user prefix, generates an ntfy configuration that mirrors compose.yaml, creates the publishing/phone accounts and writes the manifest scripts/manage.py loads. ntfy is compiled from its pinned Go module on every host because upstream publishes no macOS server binary. It reuses existing host OpenCode configuration; it writes host (non-Docker) harness profiles for the harnesses it detects and never invents a model. """ import argparse import importlib.util import ipaddress import json import os import shutil import subprocess import sys import tempfile from pathlib import Path from zoneinfo import ZoneInfoNotFoundError def module(name, file): spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(file)) result = importlib.util.module_from_spec(spec) spec.loader.exec_module(result) return result configure = module("configure_compose", "configure-compose.py") lan = module("lan_address", "lan-address.py") release_lookup = module("resolve_releases", "resolve-releases.py") tokens = module("save_ntfy_token", "save-ntfy-token.py") DEFAULT_PREFIX = Path.home() / ".local/share/pipelines" MARKER = "pipelines-install.json" LABEL = "net.a-wan.pipelines" REQUIRED_TOOLS = ("go", "python3", "git", "curl") # ntfy's SQLite driver (mattn/go-sqlite3) is cgo, so a C compiler is required to # build the server. Upstream's macOS release asset is client-only, so both # platforms compile ntfy from the pinned Go module. C_COMPILERS = ("cc", "clang", "gcc") NTFY_BUILD_TAGS = "sqlite_omit_load_extension,osusergo,netgo" DEFAULT_NTFY_PORT = 8090 DEFAULT_SERVER_PORT = 8080 # The phone client reaches the server over the LAN, matching the Docker install # (which published 0.0.0.0). Use --bind-address 127.0.0.1 for a loopback-only # server; the Pipelines API has no authentication, so keep it on a trusted # LAN/VPN either way. DEFAULT_BIND_ADDRESS = "0.0.0.0" class Setup: def __init__(self, directory, prefix=None, runner=subprocess.run, resolver=None, environ=None): self.directory = Path(directory) self.prefix_argument = prefix self.runner = runner self.resolver = resolver or release_lookup.resolve_ntfy self.environ = environ if environ is not None else os.environ # --- detection ------------------------------------------------------- def require_compiler(self): """ntfy is built locally, so a C toolchain must already be present.""" for compiler in C_COMPILERS: path = shutil.which(compiler) if path: return str(Path(path).resolve()) raise ValueError( "No C compiler found (tried " + ", ".join(C_COMPILERS) + "). ntfy's SQLite driver is cgo, so the server is compiled locally; " "install a C toolchain (Xcode Command Line Tools on macOS, gcc or clang on Linux) and rerun.") def require_tools(self): found, missing = {}, [] for tool in REQUIRED_TOOLS: path = shutil.which(tool) if path: found[tool] = str(Path(path).resolve()) else: missing.append(tool) if missing: raise ValueError( "Missing required tool(s): " + ", ".join(missing) + ". Install them yourself and rerun; this installer never installs tools or uses a package manager.") return found["python3"] def harness(self): """Detect existing harnesses only; never install, upgrade or download them.""" opencode = shutil.which("opencode") if not opencode: raise ValueError( "OpenCode was not found on PATH. Install OpenCode yourself (this installer never installs, " "upgrades or downloads it) and rerun; model selection happens in your existing OpenCode configuration.") pi = shutil.which("pi") return {"opencode": str(Path(opencode).resolve()), "pi": str(Path(pi).resolve()) if pi else ""} # --- host harness profiles ------------------------------------------ MODEL_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_./:-" @classmethod def _valid_model(cls, value): if not isinstance(value, str) or len(value) > 501 or "/" not in value: return False provider, model = value.split("/", 1) if not provider or not model: return False return all(char in cls.MODEL_CHARS for char in value) def harness_config_path(self): root = Path(self.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config") return root / "pipelines" / "harnesses.json" def _existing_harness_config(self): path = self.harness_config_path() if path.is_symlink() or not path.is_file(): return {} try: data = json.loads(path.read_text()) except (OSError, ValueError): return {} return data if isinstance(data, dict) else {} def configure_harnesses(self, harness, model=""): """Write host (non-Docker) profiles for the harnesses detected on PATH. The native install launches host harnesses, so a Docker-era ``docker_directory`` must be cleared rather than kept. Models come from the explicit ``--model`` (recorded as PIPELINES_DEFAULT_MODEL) or the existing valid profile; they are never invented. Unrelated keys and profiles for harnesses that are not detected are preserved, matching harness.Save. No harness is installed, upgraded or downloaded. """ existing = self._existing_harness_config() profiles = existing.get("profiles") if not isinstance(profiles, dict): profiles = {} merged = {name: dict(value) for name, value in profiles.items() if isinstance(value, dict)} order = [name for name in ("opencode", "pi") if harness.get(name)] configured = [] for name in order: current = merged.get(name, {}) harness_model = current.get("model") if self._valid_model(current.get("model")) else "" pipeline_model = current.get("pipeline_model") if self._valid_model(current.get("pipeline_model")) else "" if name == "opencode" and self._valid_model(model): harness_model = harness_model or model pipeline_model = pipeline_model or model if not harness_model or not pipeline_model: continue # Host mode: no docker_directory, executable is the detected host binary. merged[name] = {"executable": harness[name], "model": harness_model, "pipeline_model": pipeline_model} configured.append(name) if not configured: print("Host harness profiles were not written: no valid model was found for " + ", ".join(order) + ". Run 'pipelines harness setup opencode --host' to configure authoring.") return None # Prefer the primary detected harness, so a stale Docker-era default ("pi") # never survives a native install. default = "opencode" if "opencode" in configured else configured[0] data = dict(existing) data["default"] = default data["profiles"] = merged path = self.harness_config_path() path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) self._write_atomic(path, (json.dumps(data, indent=2) + "\n").encode(), 0o600) print(f"Host harness profiles written for {', '.join(configured)}; default {default} ({path}).") return path def resolve_prefix(self): default = Path.home() / ".local/share/pipelines" raw = self.prefix_argument or self.environ.get("PIPELINES_PREFIX") or default return Path(raw).expanduser().resolve() # --- prefix layout --------------------------------------------------- def prepare_layout(self, prefix): prefix.mkdir(parents=True, exist_ok=True) for name in ("bin", "lib", "data", "logs", "secrets"): (prefix / name).mkdir(exist_ok=True) (prefix / "secrets").chmod(0o700) (prefix / "data").chmod(0o700) source = Path(__file__).with_name("manage.py") target = prefix / "lib/manage.py" if target.is_symlink(): raise ValueError("Refusing to replace a symlinked lib/manage.py") shutil.copyfile(source, target) os.chmod(target, 0o700) return target # --- build and skill ------------------------------------------------- def build(self, prefix): for command in ("pipelines-server", "pipelines"): result = self.runner( ["go", "build", "-trimpath", "-o", str(prefix / "bin" / command), "./cmd/" + command], cwd=str(self.directory)) if result.returncode: raise ValueError(f"go build failed for ./cmd/{command}; the native install is incomplete") def install_skill(self, prefix): result = self.runner([str(prefix / "bin/pipelines"), "skill", "install"], cwd=str(self.directory)) if result.returncode: raise ValueError("pipelines skill install failed; the native install is incomplete") # --- ntfy server build and configuration ---------------------------- def resolve_ntfy(self): try: return self.resolver() except release_lookup.ReleaseError as error: raise ValueError( f"Could not resolve the ntfy release ({error}); refusing to build an unverified version") from None def build_ntfy(self, prefix, resolution): """Compile the ntfy server from the pinned Go module on this host. Upstream publishes no macOS server binary (its darwin asset is built with ``-tags noserver``), so every platform builds here. The module is fetched through the Go toolchain, which verifies it against the public checksum database, and cgo is required for ntfy's SQLite driver. """ path = prefix / "bin/ntfy" if path.is_symlink(): raise ValueError("Refusing to replace a symlinked bin/ntfy") version = resolution.get("tag") or resolution.get("version") module = resolution.get("module") or release_lookup.NTFY_MODULE # A previously built binary for the same resolved version is reused. if path.is_file() and self._existing_ntfy_version(prefix) == version: return path self.require_compiler() pinned = f"{module}@{version}" result = self.runner(["go", "mod", "download", pinned], cwd=str(self.directory)) if result.returncode: raise ValueError(f"go mod download {pinned} failed; the native install is incomplete") cache = self.runner(["go", "env", "GOMODCACHE"], cwd=str(self.directory), capture_output=True, text=True) if cache.returncode or not (cache.stdout or "").strip(): raise ValueError("Could not locate the Go module cache to build ntfy from") source = Path(cache.stdout.strip()) / f"{module}@{version}" if not source.is_dir(): raise ValueError(f"the Go module cache has no source for {pinned}") with tempfile.TemporaryDirectory(prefix="pipelines-ntfy-") as temporary: checkout = Path(temporary) / "src" shutil.copytree(source, checkout, symlinks=True) # The module cache is read-only and carries no built web assets; the # upstream Makefile creates the same placeholders for local builds. self.runner(["chmod", "-R", "u+w", str(checkout)]) (checkout / "server/docs").mkdir(parents=True, exist_ok=True) (checkout / "server/site").mkdir(parents=True, exist_ok=True) (checkout / "server/docs/index.html").touch() (checkout / "server/site/app.html").touch() built = Path(temporary) / "ntfy" env = dict(self.environ) env["CGO_ENABLED"] = "1" build = self.runner( ["go", "build", "-trimpath", "-tags", NTFY_BUILD_TAGS, "-ldflags", f"-s -w -X main.version={version}", "-o", str(built), "."], cwd=str(checkout), env=env) if build.returncode: raise ValueError( "ntfy server build failed; a working C compiler is required for its SQLite driver") if not built.is_file(): raise ValueError("ntfy server build produced no binary") os.chmod(built, 0o755) os.replace(built, path) return path def _existing_ntfy_version(self, prefix): manifest = prefix / MARKER if manifest.is_symlink() or not manifest.is_file(): return None try: data = json.loads(manifest.read_text()) except (OSError, ValueError): return None return (data.get("ntfy") or {}).get("version") if isinstance(data, dict) else None def write_ntfy_config(self, prefix, base_url, ntfy_port): directory = prefix / "data/ntfy" directory.mkdir(mode=0o700, exist_ok=True) directory.chmod(0o700) path = directory / "server.yml" if path.is_symlink(): raise ValueError("Refusing to replace a symlinked ntfy configuration") text = ( f'base-url: "{base_url}"\n' f'listen-http: ":{ntfy_port}"\n' f'cache-file: "{directory}/cache.db"\n' 'cache-duration: "168h"\n' f'auth-file: "{directory}/auth.db"\n' 'auth-default-access: "deny-all"\n' 'enable-login: true\n' f'attachment-cache-dir: "{directory}/attachments"\n' 'attachment-expiry-duration: "168h"\n' 'attachment-total-size-limit: "1G"\n' 'attachment-file-size-limit: "1M"\n' ) self._write_atomic(path, text.encode(), 0o600) return path # --- ntfy users and publishing token -------------------------------- def ntfy(self, prefix, config_path, args, capture=False): # The account subcommands locate the server configuration through # NTFY_CONFIG_FILE; --config is a `serve`-only flag. env = dict(self.environ) env["NTFY_CONFIG_FILE"] = str(config_path) result = self.runner([str(prefix / "bin/ntfy"), *args], capture_output=capture, text=True, env=env) if result.returncode: step = " ".join(args[:2]) raise ValueError(f"ntfy {step} failed; the native install is incomplete") return result def bootstrap_ntfy(self, prefix, config_path): def run_ntfy(args, capture=False, interactive=False): return self.ntfy(prefix, config_path, list(args), capture=capture) tokens.bootstrap_accounts(run_ntfy, prefix / "secrets/publisher-token", 0o600) # --- Docker migration ------------------------------------------------ def docker_volumes(self): try: result = self.runner(["docker", "volume", "ls", "--format", "{{.Name}}"], capture_output=True, text=True) except FileNotFoundError: # No Docker CLI at all: there is no Docker installation to migrate. return () except OSError as error: raise ValueError( f"Could not inspect Docker volumes for migration ({error}); refusing to start empty") from None if result.returncode: # A present-but-unusable Docker daemon must not silently skip migration: # Docker data may still exist, so fail instead of starting empty. raise ValueError( "Could not inspect Docker volumes for migration; refusing a possibly empty start") names = set(result.stdout.split()) return tuple(name for name in ("pipelines_pipelines-data", "pipelines_ntfy-data") if name in names) def _migration_image(self): result = self.runner(["docker", "image", "ls", "--format", "{{.Repository}}:{{.Tag}}"], capture_output=True, text=True) if result.returncode: return None for line in result.stdout.splitlines(): name = line.strip() repository = name.rsplit(":", 1)[0] if repository == "pipelines" or repository.endswith("/pipelines") or repository.endswith("-pipelines"): return name return None def _docker_copy_out(self, image, volume, inner, destination, replace): if destination.exists() and not replace: print(f"Keeping existing native {destination.name}; not overwriting it during migration.") return False name = f"pipelines-migrate-{os.getpid()}" created = self.runner(["docker", "create", "--name", name, "--volume", f"{volume}:/mnt:ro", image], capture_output=True, text=True) if created.returncode: raise ValueError(f"Could not open Docker volume {volume} for migration; install aborted") try: result = self.runner(["docker", "cp", f"{name}:/mnt/{inner}", str(destination)], capture_output=True, text=True) if result.returncode: raise ValueError(f"Could not copy {inner} from Docker volume {volume}; install aborted") # The live server keeps recent commits in the SQLite write-ahead log, # so copy the -wal sidecar too or the newest definitions and runs are # lost. SQLite recovers it on next open and rebuilds -shm itself. for suffix in ("-wal",): sidecar = Path(str(destination) + suffix) sidecar.unlink(missing_ok=True) copy = self.runner(["docker", "cp", f"{name}:/mnt/{inner}{suffix}", str(sidecar)], capture_output=True, text=True) if copy.returncode or not sidecar.exists() or sidecar.stat().st_size == 0: sidecar.unlink(missing_ok=True) finally: self.runner(["docker", "rm", "-f", name], capture_output=True, text=True, check=False) return True def migrate_docker(self, prefix, replace=False): volumes = self.docker_volumes() if not volumes: return False image = self._migration_image() if not image: raise ValueError( "Docker volumes exist but no local Pipelines image is available to read them; " "refusing to pull an image. Build the Docker image or rerun without migration.") (prefix / "data").mkdir(exist_ok=True) if "pipelines_pipelines-data" in volumes: self._docker_copy_out(image, "pipelines_pipelines-data", "pipelines.db", prefix / "data/pipelines.db", replace) if "pipelines_ntfy-data" in volumes: directory = prefix / "data/ntfy" directory.mkdir(mode=0o700, exist_ok=True) self._docker_copy_out(image, "pipelines_ntfy-data", "auth.db", directory / "auth.db", replace) print("Existing Docker data was migrated where it did not already exist.") return True # --- manifest -------------------------------------------------------- def write_manifest(self, prefix, values): path = prefix / MARKER if path.is_symlink(): raise ValueError("Refusing to replace a symlinked installation manifest") self._write_atomic(path, (json.dumps(values, indent=2, sort_keys=True) + "\n").encode(), 0o600) return path def _write_atomic(self, path, data, mode): handle, temporary = tempfile.mkstemp(dir=str(path.parent), prefix=".pipelines-") try: os.chmod(temporary, mode) with os.fdopen(handle, "wb") as file: file.write(data) if path.is_symlink(): raise ValueError(f"Refusing to replace a symlinked file {path}") os.replace(temporary, path) finally: if os.path.exists(temporary): os.unlink(temporary) # --- run ------------------------------------------------------------- def run(self, args): python = self.require_tools() if args.prefix: self.prefix_argument = args.prefix if not 1024 <= args.port <= 65535: raise ValueError("Invalid server port; choose a value between 1024 and 65535") if not 1024 <= args.ntfy_port <= 65535: raise ValueError("Invalid ntfy port; choose a value between 1024 and 65535") if args.ntfy_port == args.port: raise ValueError("The server and ntfy ports must differ") try: ipaddress.IPv4Address(args.bind_address) except ValueError: raise ValueError("Invalid bind address; use an IPv4 address such as 0.0.0.0 or 127.0.0.1") from None prefix = self.resolve_prefix() self.prepare_layout(prefix) harness = self.harness() # Reuse an already built ntfy binary and skip the release lookup, so a # reinstall with the same version makes no network requests at all. installed = self._existing_ntfy_version(prefix) if (prefix / "bin/ntfy").is_file() and installed: resolution = {"module": release_lookup.NTFY_MODULE, "version": installed, "tag": installed} else: resolution = self.resolve_ntfy() self.build(prefix) self.install_skill(prefix) # The native install launches host harnesses, so write host profiles for # the detected harnesses and clear any Docker-era link. Uses only the # explicit --model or the existing valid profile/host configuration. self.configure_harnesses(harness, args.model or "") lan_ip = lan.detect_lan() base_url = args.ntfy_url or f"http://{lan_ip}:{args.ntfy_port}" config_path = self.write_ntfy_config(prefix, base_url, args.ntfy_port) self.build_ntfy(prefix, resolution) # Migrate existing Docker data before creating native ntfy accounts so a # migrated auth store (the phone's credentials) is preserved rather than # shadowed by a freshly bootstrapped one. self.migrate_docker(prefix, args.replace_data) self.bootstrap_ntfy(prefix, config_path) timezone = args.timezone if not timezone: try: timezone = configure.host_timezone() except (ValueError, ZoneInfoNotFoundError): timezone = "UTC" opencode_env = {} for key in ("XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_STATE_HOME", "XDG_CACHE_HOME", "OPENCODE_SERVER", "OPENCODE_DB"): if self.environ.get(key): opencode_env[key] = self.environ[key] if args.model: opencode_env["PIPELINES_DEFAULT_MODEL"] = args.model path_value = str(prefix / "bin") + os.pathsep + self.environ.get("PATH", "") manifest = { "format": 1, "prefix": str(prefix), "python": python, "server_port": args.port, "bind_address": args.bind_address, "ntfy_port": args.ntfy_port, "opencode": harness["opencode"], "pi": harness["pi"], "opencode_env": opencode_env, "path": path_value, "topic": "pipelines", "public_url": base_url, "username": "phone-reader", "workspace": str(Path.home()), "launcher": str(prefix / "bin/pipelines"), "registration": None, "label": LABEL, "docker": "docker", "project": "pipelines", "timezone": timezone, "ntfy": {"mode": "native", "bin": str(prefix / "bin/ntfy"), "config": str(config_path), "base_url": base_url, "version": resolution.get("tag") or resolution.get("version")}, } self.write_manifest(prefix, manifest) # Save the address the host CLI/TUI dials by default, so a plain # `pipelines` (or its TUI) reaches this installation instead of a stale # Docker address. endpoint = f"ws://127.0.0.1:{args.port}/connect" client_path = configure.configure_client(endpoint, self.environ) print(f"Saved native client default: {endpoint} ({client_path}). Restart the TUI.") override = self.environ.get("PIPELINES_URL", "") if override and override != endpoint: print("WARNING: PIPELINES_URL overrides the saved native address; unset it or pass --url explicitly.") print(f"Native Pipelines installed at {prefix}." f" ntfy {resolution.get('tag') or resolution.get('version')} was built from source.") # Parity with the Docker installer: leave the server and ntfy running. started = self.runner([python, str(prefix / "lib/manage.py"), "start", "--prefix", str(prefix)]) if started.returncode: raise ValueError( "The native services were installed but could not be started; run " f"'{prefix}/bin/pipelines start' to see the error. Check that ports " f"{args.port} and {args.ntfy_port} are free (a running Docker installation uses them).") print(f"ntfy is configured for {base_url} and both services are running.") if args.bind_address == "0.0.0.0": print(f"Phone client server URL: ws://{lan_ip}:{args.port}/connect " "(use a workspace path that exists on this server).") else: print(f"Server bound to {args.bind_address}:{args.port}; a phone on the LAN needs a LAN bind " "(re-run with --bind-address 0.0.0.0).") print("Create the phone account password if it is a new account, then subscribe to the pipelines topic.") def arguments(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--prefix", type=Path, help="Installation prefix (default: ~/.local/share/pipelines)") parser.add_argument("--ntfy-url", help="Override the auto-detected LAN ntfy address") parser.add_argument("--timezone", help="Override the detected timezone (fallback: UTC)") parser.add_argument("--port", type=int, default=DEFAULT_SERVER_PORT, help="Pipelines server port") parser.add_argument("--bind-address", default=DEFAULT_BIND_ADDRESS, help="Server bind address (default 0.0.0.0, the LAN, so the phone client can reach it; " "use 127.0.0.1 for a loopback-only server)") parser.add_argument("--ntfy-port", type=int, default=DEFAULT_NTFY_PORT, help="ntfy listen port") parser.add_argument("--model", help="Record an explicit default model; otherwise host OpenCode configuration is reused") parser.add_argument("--replace-data", action="store_true", help="Allow Docker migration to replace an existing native database/auth store") return parser.parse_args(argv) if __name__ == "__main__": try: Setup(Path(__file__).resolve().parents[1]).run(arguments()) except KeyboardInterrupt: print("\nNative setup canceled; existing configuration and data were retained.", file=sys.stderr) sys.exit(130) except (ValueError, OSError, json.JSONDecodeError, subprocess.SubprocessError, ZoneInfoNotFoundError) as error: sys.exit(f"Native setup incomplete: {error}")