#!/usr/bin/env python3 """Install Pipelines natively on the host (macOS and Linux). The installer only detects the tools it needs (go, python3, git, curl and a C compiler for ntfy's SQLite driver); it never installs or upgrades them, and it never installs, upgrades or downloads OpenCode or Pi. It builds the server, CLI and ntfy from source into a per-user prefix, generates an ntfy configuration that mirrors compose.yaml, creates the publishing/phone accounts and writes the manifest scripts/manage.py loads. ntfy is compiled from its pinned Go module on every host because upstream publishes no macOS server binary. It reuses existing host OpenCode configuration; model selection stays at runtime and is never invented here. """ import argparse import importlib.util import json import os import shutil import subprocess import sys import tempfile from pathlib import Path from zoneinfo import ZoneInfoNotFoundError def module(name, file): spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(file)) result = importlib.util.module_from_spec(spec) spec.loader.exec_module(result) return result configure = module("configure_compose", "configure-compose.py") lan = module("lan_address", "lan-address.py") release_lookup = module("resolve_releases", "resolve-releases.py") tokens = module("save_ntfy_token", "save-ntfy-token.py") DEFAULT_PREFIX = Path.home() / ".local/share/pipelines" MARKER = "pipelines-install.json" LABEL = "net.a-wan.pipelines" REQUIRED_TOOLS = ("go", "python3", "git", "curl") # ntfy's SQLite driver (mattn/go-sqlite3) is cgo, so a C compiler is required to # build the server. Upstream's macOS release asset is client-only, so both # platforms compile ntfy from the pinned Go module. C_COMPILERS = ("cc", "clang", "gcc") NTFY_BUILD_TAGS = "sqlite_omit_load_extension,osusergo,netgo" DEFAULT_NTFY_PORT = 8090 DEFAULT_SERVER_PORT = 8080 class Setup: def __init__(self, directory, prefix=None, runner=subprocess.run, resolver=None, environ=None): self.directory = Path(directory) self.prefix_argument = prefix self.runner = runner self.resolver = resolver or release_lookup.resolve_ntfy self.environ = environ if environ is not None else os.environ # --- detection ------------------------------------------------------- def require_compiler(self): """ntfy is built locally, so a C toolchain must already be present.""" for compiler in C_COMPILERS: path = shutil.which(compiler) if path: return str(Path(path).resolve()) raise ValueError( "No C compiler found (tried " + ", ".join(C_COMPILERS) + "). ntfy's SQLite driver is cgo, so the server is compiled locally; " "install a C toolchain (Xcode Command Line Tools on macOS, gcc or clang on Linux) and rerun.") def require_tools(self): found, missing = {}, [] for tool in REQUIRED_TOOLS: path = shutil.which(tool) if path: found[tool] = str(Path(path).resolve()) else: missing.append(tool) if missing: raise ValueError( "Missing required tool(s): " + ", ".join(missing) + ". Install them yourself and rerun; this installer never installs tools or uses a package manager.") return found["python3"] def harness(self): """Detect existing harnesses only; never install, upgrade or download them.""" opencode = shutil.which("opencode") if not opencode: raise ValueError( "OpenCode was not found on PATH. Install OpenCode yourself (this installer never installs, " "upgrades or downloads it) and rerun; model selection happens in your existing OpenCode configuration.") pi = shutil.which("pi") return {"opencode": str(Path(opencode).resolve()), "pi": str(Path(pi).resolve()) if pi else ""} def resolve_prefix(self): default = Path.home() / ".local/share/pipelines" raw = self.prefix_argument or self.environ.get("PIPELINES_PREFIX") or default return Path(raw).expanduser().resolve() # --- prefix layout --------------------------------------------------- def prepare_layout(self, prefix): prefix.mkdir(parents=True, exist_ok=True) for name in ("bin", "lib", "data", "logs", "secrets"): (prefix / name).mkdir(exist_ok=True) (prefix / "secrets").chmod(0o700) (prefix / "data").chmod(0o700) source = Path(__file__).with_name("manage.py") target = prefix / "lib/manage.py" if target.is_symlink(): raise ValueError("Refusing to replace a symlinked lib/manage.py") shutil.copyfile(source, target) os.chmod(target, 0o700) return target # --- build and skill ------------------------------------------------- def build(self, prefix): for command in ("pipelines-server", "pipelines"): result = self.runner( ["go", "build", "-trimpath", "-o", str(prefix / "bin" / command), "./cmd/" + command], cwd=str(self.directory)) if result.returncode: raise ValueError(f"go build failed for ./cmd/{command}; the native install is incomplete") def install_skill(self, prefix): result = self.runner([str(prefix / "bin/pipelines"), "skill", "install"], cwd=str(self.directory)) if result.returncode: raise ValueError("pipelines skill install failed; the native install is incomplete") # --- ntfy server build and configuration ---------------------------- def resolve_ntfy(self): try: return self.resolver() except release_lookup.ReleaseError as error: raise ValueError( f"Could not resolve the ntfy release ({error}); refusing to build an unverified version") from None def build_ntfy(self, prefix, resolution): """Compile the ntfy server from the pinned Go module on this host. Upstream publishes no macOS server binary (its darwin asset is built with ``-tags noserver``), so every platform builds here. The module is fetched through the Go toolchain, which verifies it against the public checksum database, and cgo is required for ntfy's SQLite driver. """ path = prefix / "bin/ntfy" if path.is_symlink(): raise ValueError("Refusing to replace a symlinked bin/ntfy") version = resolution.get("tag") or resolution.get("version") module = resolution.get("module") or release_lookup.NTFY_MODULE # A previously built binary for the same resolved version is reused. if path.is_file() and self._existing_ntfy_version(prefix) == version: return path self.require_compiler() pinned = f"{module}@{version}" result = self.runner(["go", "mod", "download", pinned], cwd=str(self.directory)) if result.returncode: raise ValueError(f"go mod download {pinned} failed; the native install is incomplete") cache = self.runner(["go", "env", "GOMODCACHE"], cwd=str(self.directory), capture_output=True, text=True) if cache.returncode or not (cache.stdout or "").strip(): raise ValueError("Could not locate the Go module cache to build ntfy from") source = Path(cache.stdout.strip()) / f"{module}@{version}" if not source.is_dir(): raise ValueError(f"the Go module cache has no source for {pinned}") with tempfile.TemporaryDirectory(prefix="pipelines-ntfy-") as temporary: checkout = Path(temporary) / "src" shutil.copytree(source, checkout, symlinks=True) # The module cache is read-only and carries no built web assets; the # upstream Makefile creates the same placeholders for local builds. self.runner(["chmod", "-R", "u+w", str(checkout)]) (checkout / "server/docs").mkdir(parents=True, exist_ok=True) (checkout / "server/site").mkdir(parents=True, exist_ok=True) (checkout / "server/docs/index.html").touch() (checkout / "server/site/app.html").touch() built = Path(temporary) / "ntfy" env = dict(self.environ) env["CGO_ENABLED"] = "1" build = self.runner( ["go", "build", "-trimpath", "-tags", NTFY_BUILD_TAGS, "-ldflags", f"-s -w -X main.version={version}", "-o", str(built), "."], cwd=str(checkout), env=env) if build.returncode: raise ValueError( "ntfy server build failed; a working C compiler is required for its SQLite driver") if not built.is_file(): raise ValueError("ntfy server build produced no binary") os.chmod(built, 0o755) os.replace(built, path) return path def _existing_ntfy_version(self, prefix): manifest = prefix / MARKER if manifest.is_symlink() or not manifest.is_file(): return None try: data = json.loads(manifest.read_text()) except (OSError, ValueError): return None return (data.get("ntfy") or {}).get("version") if isinstance(data, dict) else None def write_ntfy_config(self, prefix, base_url, ntfy_port): directory = prefix / "data/ntfy" directory.mkdir(mode=0o700, exist_ok=True) directory.chmod(0o700) path = directory / "server.yml" if path.is_symlink(): raise ValueError("Refusing to replace a symlinked ntfy configuration") text = ( f'base-url: "{base_url}"\n' f'listen-http: ":{ntfy_port}"\n' f'cache-file: "{directory}/cache.db"\n' 'cache-duration: "168h"\n' f'auth-file: "{directory}/auth.db"\n' 'auth-default-access: "deny-all"\n' 'enable-login: true\n' f'attachment-cache-dir: "{directory}/attachments"\n' 'attachment-expiry-duration: "168h"\n' 'attachment-total-size-limit: "1G"\n' 'attachment-file-size-limit: "1M"\n' ) self._write_atomic(path, text.encode(), 0o600) return path # --- ntfy users and publishing token -------------------------------- def ntfy(self, prefix, config_path, args, capture=False): # The account subcommands locate the server configuration through # NTFY_CONFIG_FILE; --config is a `serve`-only flag. env = dict(self.environ) env["NTFY_CONFIG_FILE"] = str(config_path) result = self.runner([str(prefix / "bin/ntfy"), *args], capture_output=capture, text=True, env=env) if result.returncode: step = " ".join(args[:2]) raise ValueError(f"ntfy {step} failed; the native install is incomplete") return result def bootstrap_ntfy(self, prefix, config_path): def run_ntfy(args, capture=False, interactive=False): return self.ntfy(prefix, config_path, list(args), capture=capture) tokens.bootstrap_accounts(run_ntfy, prefix / "secrets/publisher-token", 0o600) # --- Docker migration ------------------------------------------------ def docker_volumes(self): try: result = self.runner(["docker", "volume", "ls", "--format", "{{.Name}}"], capture_output=True, text=True) except FileNotFoundError: # No Docker CLI at all: there is no Docker installation to migrate. return () except OSError as error: raise ValueError( f"Could not inspect Docker volumes for migration ({error}); refusing to start empty") from None if result.returncode: # A present-but-unusable Docker daemon must not silently skip migration: # Docker data may still exist, so fail instead of starting empty. raise ValueError( "Could not inspect Docker volumes for migration; refusing a possibly empty start") names = set(result.stdout.split()) return tuple(name for name in ("pipelines_pipelines-data", "pipelines_ntfy-data") if name in names) def _migration_image(self): result = self.runner(["docker", "image", "ls", "--format", "{{.Repository}}:{{.Tag}}"], capture_output=True, text=True) if result.returncode: return None for line in result.stdout.splitlines(): name = line.strip() repository = name.rsplit(":", 1)[0] if repository == "pipelines" or repository.endswith("/pipelines") or repository.endswith("-pipelines"): return name return None def _docker_copy_out(self, image, volume, inner, destination, replace): if destination.exists() and not replace: print(f"Keeping existing native {destination.name}; not overwriting it during migration.") return False name = f"pipelines-migrate-{os.getpid()}" created = self.runner(["docker", "create", "--name", name, "--volume", f"{volume}:/mnt:ro", image], capture_output=True, text=True) if created.returncode: raise ValueError(f"Could not open Docker volume {volume} for migration; install aborted") try: result = self.runner(["docker", "cp", f"{name}:/mnt/{inner}", str(destination)], capture_output=True, text=True) if result.returncode: raise ValueError(f"Could not copy {inner} from Docker volume {volume}; install aborted") # The live server keeps recent commits in the SQLite write-ahead log, # so copy the -wal sidecar too or the newest definitions and runs are # lost. SQLite recovers it on next open and rebuilds -shm itself. for suffix in ("-wal",): sidecar = Path(str(destination) + suffix) sidecar.unlink(missing_ok=True) copy = self.runner(["docker", "cp", f"{name}:/mnt/{inner}{suffix}", str(sidecar)], capture_output=True, text=True) if copy.returncode or not sidecar.exists() or sidecar.stat().st_size == 0: sidecar.unlink(missing_ok=True) finally: self.runner(["docker", "rm", "-f", name], capture_output=True, text=True, check=False) return True def migrate_docker(self, prefix, replace=False): volumes = self.docker_volumes() if not volumes: return False image = self._migration_image() if not image: raise ValueError( "Docker volumes exist but no local Pipelines image is available to read them; " "refusing to pull an image. Build the Docker image or rerun without migration.") (prefix / "data").mkdir(exist_ok=True) if "pipelines_pipelines-data" in volumes: self._docker_copy_out(image, "pipelines_pipelines-data", "pipelines.db", prefix / "data/pipelines.db", replace) if "pipelines_ntfy-data" in volumes: directory = prefix / "data/ntfy" directory.mkdir(mode=0o700, exist_ok=True) self._docker_copy_out(image, "pipelines_ntfy-data", "auth.db", directory / "auth.db", replace) print("Existing Docker data was migrated where it did not already exist.") return True # --- manifest -------------------------------------------------------- def write_manifest(self, prefix, values): path = prefix / MARKER if path.is_symlink(): raise ValueError("Refusing to replace a symlinked installation manifest") self._write_atomic(path, (json.dumps(values, indent=2, sort_keys=True) + "\n").encode(), 0o600) return path def _write_atomic(self, path, data, mode): handle, temporary = tempfile.mkstemp(dir=str(path.parent), prefix=".pipelines-") try: os.chmod(temporary, mode) with os.fdopen(handle, "wb") as file: file.write(data) if path.is_symlink(): raise ValueError(f"Refusing to replace a symlinked file {path}") os.replace(temporary, path) finally: if os.path.exists(temporary): os.unlink(temporary) # --- run ------------------------------------------------------------- def run(self, args): python = self.require_tools() if args.prefix: self.prefix_argument = args.prefix if not 1024 <= args.port <= 65535: raise ValueError("Invalid server port; choose a value between 1024 and 65535") if not 1024 <= args.ntfy_port <= 65535: raise ValueError("Invalid ntfy port; choose a value between 1024 and 65535") if args.ntfy_port == args.port: raise ValueError("The server and ntfy ports must differ") prefix = self.resolve_prefix() self.prepare_layout(prefix) harness = self.harness() # Reuse an already built ntfy binary and skip the release lookup, so a # reinstall with the same version makes no network requests at all. installed = self._existing_ntfy_version(prefix) if (prefix / "bin/ntfy").is_file() and installed: resolution = {"module": release_lookup.NTFY_MODULE, "version": installed, "tag": installed} else: resolution = self.resolve_ntfy() self.build(prefix) self.install_skill(prefix) base_url = args.ntfy_url or f"http://{lan.detect_lan()}:{args.ntfy_port}" config_path = self.write_ntfy_config(prefix, base_url, args.ntfy_port) self.build_ntfy(prefix, resolution) # Migrate existing Docker data before creating native ntfy accounts so a # migrated auth store (the phone's credentials) is preserved rather than # shadowed by a freshly bootstrapped one. self.migrate_docker(prefix, args.replace_data) self.bootstrap_ntfy(prefix, config_path) timezone = args.timezone if not timezone: try: timezone = configure.host_timezone() except (ValueError, ZoneInfoNotFoundError): timezone = "UTC" opencode_env = {} for key in ("XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_STATE_HOME", "XDG_CACHE_HOME", "OPENCODE_SERVER", "OPENCODE_DB"): if self.environ.get(key): opencode_env[key] = self.environ[key] if args.model: opencode_env["PIPELINES_DEFAULT_MODEL"] = args.model path_value = str(prefix / "bin") + os.pathsep + self.environ.get("PATH", "") manifest = { "format": 1, "prefix": str(prefix), "python": python, "server_port": args.port, "ntfy_port": args.ntfy_port, "opencode": harness["opencode"], "pi": harness["pi"], "opencode_env": opencode_env, "path": path_value, "topic": "pipelines", "public_url": base_url, "username": "phone-reader", "workspace": str(Path.home()), "launcher": str(prefix / "bin/pipelines"), "registration": None, "label": LABEL, "docker": "docker", "project": "pipelines", "timezone": timezone, "ntfy": {"mode": "native", "bin": str(prefix / "bin/ntfy"), "config": str(config_path), "base_url": base_url, "version": resolution.get("tag") or resolution.get("version")}, } self.write_manifest(prefix, manifest) # Save the address the host CLI/TUI dials by default, so a plain # `pipelines` (or its TUI) reaches this installation instead of a stale # Docker address. endpoint = f"ws://127.0.0.1:{args.port}/connect" client_path = configure.configure_client(endpoint, self.environ) print(f"Saved native client default: {endpoint} ({client_path}). Restart the TUI.") override = self.environ.get("PIPELINES_URL", "") if override and override != endpoint: print("WARNING: PIPELINES_URL overrides the saved native address; unset it or pass --url explicitly.") print(f"Native Pipelines installed at {prefix}." f" ntfy {resolution.get('tag') or resolution.get('version')} was built from source.") # Parity with the Docker installer: leave the server and ntfy running. started = self.runner([python, str(prefix / "lib/manage.py"), "start", "--prefix", str(prefix)]) if started.returncode: raise ValueError( "The native services were installed but could not be started; run " f"'{prefix}/bin/pipelines start' to see the error. Check that ports " f"{args.port} and {args.ntfy_port} are free (a running Docker installation uses them).") print(f"ntfy is configured for {base_url} and both services are running.") print("Create the phone account password if it is a new account, then subscribe to the pipelines topic.") def arguments(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--prefix", type=Path, help="Installation prefix (default: ~/.local/share/pipelines)") parser.add_argument("--ntfy-url", help="Override the auto-detected LAN ntfy address") parser.add_argument("--timezone", help="Override the detected timezone (fallback: UTC)") parser.add_argument("--port", type=int, default=DEFAULT_SERVER_PORT, help="Pipelines server port") parser.add_argument("--ntfy-port", type=int, default=DEFAULT_NTFY_PORT, help="ntfy listen port") parser.add_argument("--model", help="Record an explicit default model; otherwise host OpenCode configuration is reused") parser.add_argument("--replace-data", action="store_true", help="Allow Docker migration to replace an existing native database/auth store") return parser.parse_args(argv) if __name__ == "__main__": try: Setup(Path(__file__).resolve().parents[1]).run(arguments()) except KeyboardInterrupt: print("\nNative setup canceled; existing configuration and data were retained.", file=sys.stderr) sys.exit(130) except (ValueError, OSError, json.JSONDecodeError, subprocess.SubprocessError, ZoneInfoNotFoundError) as error: sys.exit(f"Native setup incomplete: {error}")