ae6ba6ae10
Register the existing native supervisor with launchd/systemd, include the dependency-free authenticated web proxy, detect Node, and keep the API loopback by default. Preserve existing web credentials and configured host harness defaults. Clarify single-step setup, lifecycle selection, and optional ntfy limits. Checks: 44 mocked native installer/lifecycle tests; 8 entrypoint tests; 42 Docker installer tests; GOPROXY=off GOTOOLCHAIN=local go test ./internal/installation ./cmd/pipelines; sh -n install install.sh; git diff --check. No live install, deployment, restart, downloads, real autostart/login verification or native Linux verification.
709 lines
35 KiB
Python
709 lines
35 KiB
Python
#!/usr/bin/env python3
|
|
"""Install Pipelines natively on the host (macOS and Linux).
|
|
|
|
The installer only detects the tools it needs (go, python3, node, git, curl and a C
|
|
compiler for ntfy's SQLite driver); it never installs or upgrades them, and it
|
|
never installs, upgrades or downloads OpenCode or Pi. It builds the server, CLI
|
|
and ntfy from source into a per-user prefix, generates an ntfy configuration that
|
|
mirrors compose.yaml, creates the publishing/phone accounts and writes the
|
|
manifest scripts/manage.py loads. ntfy is compiled from its pinned Go module on
|
|
every host because upstream publishes no macOS server binary. It reuses existing
|
|
host OpenCode configuration; it writes host (non-Docker) harness profiles for the
|
|
harnesses it detects and never invents a model.
|
|
"""
|
|
import argparse
|
|
import importlib.util
|
|
import ipaddress
|
|
import json
|
|
import os
|
|
import platform
|
|
import plistlib
|
|
import secrets
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
from zoneinfo import ZoneInfoNotFoundError
|
|
|
|
|
|
def module(name, file):
|
|
spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(file))
|
|
result = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(result)
|
|
return result
|
|
|
|
|
|
configure = module("configure_compose", "configure-compose.py")
|
|
lan = module("lan_address", "lan-address.py")
|
|
release_lookup = module("resolve_releases", "resolve-releases.py")
|
|
tokens = module("save_ntfy_token", "save-ntfy-token.py")
|
|
|
|
DEFAULT_PREFIX = Path.home() / ".local/share/pipelines"
|
|
MARKER = "pipelines-install.json"
|
|
LABEL = "net.a-wan.pipelines"
|
|
SERVICE_MARKER = "Managed by Pipelines native installer"
|
|
REQUIRED_TOOLS = ("go", "python3", "git", "curl", "node")
|
|
# ntfy's SQLite driver (mattn/go-sqlite3) is cgo, so a C compiler is required to
|
|
# build the server. Upstream's macOS release asset is client-only, so both
|
|
# platforms compile ntfy from the pinned Go module.
|
|
C_COMPILERS = ("cc", "clang", "gcc")
|
|
NTFY_BUILD_TAGS = "sqlite_omit_load_extension,osusergo,netgo"
|
|
DEFAULT_NTFY_PORT = 8090
|
|
DEFAULT_SERVER_PORT = 8080
|
|
# Only the authenticated web proxy is exposed; the control API stays loopback.
|
|
DEFAULT_BIND_ADDRESS = "127.0.0.1"
|
|
|
|
|
|
class Setup:
|
|
def __init__(self, directory, prefix=None, runner=subprocess.run, resolver=None, environ=None):
|
|
self.directory = Path(directory)
|
|
self.prefix_argument = prefix
|
|
self.runner = runner
|
|
self.resolver = resolver or release_lookup.resolve_ntfy
|
|
self.environ = environ if environ is not None else os.environ
|
|
|
|
# --- detection -------------------------------------------------------
|
|
|
|
def require_compiler(self):
|
|
"""ntfy is built locally, so a C toolchain must already be present."""
|
|
for compiler in C_COMPILERS:
|
|
path = shutil.which(compiler)
|
|
if path:
|
|
return str(Path(path).resolve())
|
|
raise ValueError(
|
|
"No C compiler found (tried " + ", ".join(C_COMPILERS)
|
|
+ "). ntfy's SQLite driver is cgo, so the server is compiled locally; "
|
|
"install a C toolchain (Xcode Command Line Tools on macOS, gcc or clang on Linux) and rerun.")
|
|
|
|
def require_tools(self):
|
|
found, missing = {}, []
|
|
for tool in REQUIRED_TOOLS:
|
|
path = shutil.which(tool)
|
|
if path:
|
|
found[tool] = str(Path(path).resolve())
|
|
else:
|
|
missing.append(tool)
|
|
if missing:
|
|
raise ValueError(
|
|
"Missing required tool(s): " + ", ".join(missing)
|
|
+ ". Install them yourself and rerun; this installer never installs tools or uses a package manager.")
|
|
return found["python3"]
|
|
|
|
def harness(self):
|
|
"""Detect existing harnesses only; never install, upgrade or download them."""
|
|
opencode = shutil.which("opencode")
|
|
if not opencode:
|
|
raise ValueError(
|
|
"OpenCode was not found on PATH. Install OpenCode yourself (this installer never installs, "
|
|
"upgrades or downloads it) and rerun; model selection happens in your existing OpenCode configuration.")
|
|
pi = shutil.which("pi")
|
|
return {"opencode": str(Path(opencode).resolve()), "pi": str(Path(pi).resolve()) if pi else ""}
|
|
|
|
def service_path(self):
|
|
system = platform.system()
|
|
if system == "Darwin":
|
|
path = Path.home() / "Library/LaunchAgents" / (LABEL + ".plist")
|
|
tool = "launchctl"
|
|
elif system == "Linux":
|
|
root = Path(self.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config")
|
|
path = root / "systemd/user" / (LABEL + ".service")
|
|
tool = "systemctl"
|
|
else:
|
|
raise ValueError("Native installation supports macOS (launchd) and Linux (systemd --user) only")
|
|
if not shutil.which(tool):
|
|
raise ValueError(f"{tool} is required for login autostart; run setup in a logged-in user session")
|
|
if path.is_symlink() or (path.exists() and SERVICE_MARKER not in path.read_text()):
|
|
raise ValueError(f"Refusing to replace an unmanaged autostart definition: {path}")
|
|
return path
|
|
|
|
def register_service(self, path, manifest):
|
|
"""Register the paired native services, not a second standalone server."""
|
|
prefix = Path(manifest["prefix"])
|
|
command = [manifest["python"], str(prefix / "lib/manage.py"),
|
|
"supervise", "--prefix", str(prefix)]
|
|
if platform.system() == "Darwin":
|
|
content = plistlib.dumps({
|
|
"Label": LABEL, "Comment": SERVICE_MARKER,
|
|
"ProgramArguments": command, "RunAtLoad": True,
|
|
"KeepAlive": {"SuccessfulExit": False}, "ThrottleInterval": 5,
|
|
"EnvironmentVariables": {"HOME": str(Path.home()), "PATH": manifest["path"]},
|
|
"StandardOutPath": str(prefix / "logs/supervisor.log"),
|
|
"StandardErrorPath": str(prefix / "logs/supervisor.log"), "Umask": 0o077,
|
|
})
|
|
else:
|
|
def quote(value):
|
|
if any(char in value for char in "\r\n\x00"):
|
|
raise ValueError("Autostart paths cannot contain line breaks or NUL")
|
|
return '"' + value.replace('\\', '\\\\').replace('"', '\\"').replace('%', '%%').replace('$', '$$') + '"'
|
|
|
|
content = (f"# {SERVICE_MARKER}\n[Unit]\nDescription=Pipelines native services\n"
|
|
"\n[Service]\nType=simple\nExecStart=" + " ".join(map(quote, command))
|
|
+ "\nRestart=on-failure\nRestartSec=5\nTimeoutStopSec=90\nUMask=0077\n"
|
|
"\n[Install]\nWantedBy=default.target\n").encode()
|
|
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
self._write_atomic(path, content, 0o600)
|
|
manifest["registration"] = str(path)
|
|
|
|
# --- host harness profiles ------------------------------------------
|
|
|
|
MODEL_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_./:-"
|
|
|
|
@classmethod
|
|
def _valid_model(cls, value):
|
|
if not isinstance(value, str) or len(value) > 501 or "/" not in value:
|
|
return False
|
|
provider, model = value.split("/", 1)
|
|
if not provider or not model:
|
|
return False
|
|
return all(char in cls.MODEL_CHARS for char in value)
|
|
|
|
def harness_config_path(self):
|
|
root = Path(self.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config")
|
|
return root / "pipelines" / "harnesses.json"
|
|
|
|
def _existing_harness_config(self):
|
|
path = self.harness_config_path()
|
|
if path.is_symlink() or not path.is_file():
|
|
return {}
|
|
try:
|
|
data = json.loads(path.read_text())
|
|
except (OSError, ValueError):
|
|
return {}
|
|
return data if isinstance(data, dict) else {}
|
|
|
|
def configure_harnesses(self, harness, model=""):
|
|
"""Write host (non-Docker) profiles for the harnesses detected on PATH.
|
|
|
|
The native install launches host harnesses, so a Docker-era
|
|
``docker_directory`` must be cleared rather than kept. Models come from
|
|
the explicit ``--model`` (recorded as PIPELINES_DEFAULT_MODEL) or the
|
|
existing valid profile; they are never invented. Unrelated keys and
|
|
profiles for harnesses that are not detected are preserved, matching
|
|
harness.Save. No harness is installed, upgraded or downloaded.
|
|
"""
|
|
existing = self._existing_harness_config()
|
|
profiles = existing.get("profiles")
|
|
if not isinstance(profiles, dict):
|
|
profiles = {}
|
|
merged = {name: dict(value) for name, value in profiles.items() if isinstance(value, dict)}
|
|
order = [name for name in ("opencode", "pi") if harness.get(name)]
|
|
configured = []
|
|
for name in order:
|
|
current = merged.get(name, {})
|
|
harness_model = current.get("model") if self._valid_model(current.get("model")) else ""
|
|
pipeline_model = current.get("pipeline_model") if self._valid_model(current.get("pipeline_model")) else ""
|
|
if name == "opencode" and self._valid_model(model):
|
|
harness_model = harness_model or model
|
|
pipeline_model = pipeline_model or model
|
|
if not harness_model or not pipeline_model:
|
|
continue
|
|
# Host mode: no docker_directory, executable is the detected host binary.
|
|
profile = dict(current)
|
|
profile.pop("docker_directory", None)
|
|
profile.update(executable=harness[name], model=harness_model, pipeline_model=pipeline_model)
|
|
merged[name] = profile
|
|
configured.append(name)
|
|
if not configured:
|
|
print("Host harness profiles were not written: no valid model was found for "
|
|
+ ", ".join(order) + ". Run 'pipelines harness setup opencode --host' to configure authoring.")
|
|
return None
|
|
# Keep a configured host default; only a stale Docker default must change.
|
|
default = existing.get("default")
|
|
if default not in configured or profiles.get(default, {}).get("docker_directory"):
|
|
default = "opencode" if "opencode" in configured else configured[0]
|
|
data = dict(existing)
|
|
data["default"] = default
|
|
data["profiles"] = merged
|
|
path = self.harness_config_path()
|
|
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
self._write_atomic(path, (json.dumps(data, indent=2) + "\n").encode(), 0o600)
|
|
print(f"Host harness profiles written for {', '.join(configured)}; default {default} ({path}).")
|
|
return path
|
|
|
|
def resolve_prefix(self):
|
|
default = Path.home() / ".local/share/pipelines"
|
|
raw = self.prefix_argument or self.environ.get("PIPELINES_PREFIX") or default
|
|
return Path(raw).expanduser().resolve()
|
|
|
|
# --- prefix layout ---------------------------------------------------
|
|
|
|
def prepare_layout(self, prefix):
|
|
prefix.mkdir(parents=True, exist_ok=True)
|
|
for name in ("bin", "lib", "data", "logs", "secrets"):
|
|
(prefix / name).mkdir(exist_ok=True)
|
|
(prefix / "secrets").chmod(0o700)
|
|
(prefix / "data").chmod(0o700)
|
|
source = Path(__file__).with_name("manage.py")
|
|
target = prefix / "lib/manage.py"
|
|
if target.is_symlink():
|
|
raise ValueError("Refusing to replace a symlinked lib/manage.py")
|
|
shutil.copyfile(source, target)
|
|
os.chmod(target, 0o700)
|
|
return target
|
|
|
|
def configure_web(self, prefix, args):
|
|
node = str(Path(shutil.which("node")).resolve())
|
|
version = self.runner([node, "--version"], capture_output=True, text=True)
|
|
try:
|
|
major = int(version.stdout.strip().removeprefix("v").split(".")[0])
|
|
except (ValueError, AttributeError):
|
|
major = 0
|
|
if version.returncode or major < 18:
|
|
raise ValueError("Node.js 18 or newer is required for the authenticated web proxy")
|
|
# Reuse the proxy's existing credential location; never disable its gate.
|
|
auth = Path(self.environ.get("PIPELINES_WEB_AUTH_FILE")
|
|
or Path.home() / ".config/pipelines/web-auth.json").expanduser().resolve()
|
|
if auth.exists():
|
|
data = json.loads(auth.read_text())
|
|
if (not isinstance(data, dict) or not isinstance(data.get("password"), str)
|
|
or len(data["password"]) < 8 or not isinstance(data.get("secret"), str)
|
|
or len(data["secret"]) < 32):
|
|
raise ValueError(f"Invalid existing web auth file; retained unchanged: {auth}")
|
|
if auth.stat().st_mode & 0o077:
|
|
raise ValueError(f"Web auth file must be private (chmod 600): {auth}")
|
|
else:
|
|
auth.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
self._write_atomic(auth, (json.dumps({"password": secrets.token_urlsafe(18),
|
|
"secret": secrets.token_urlsafe(48)}) + "\n").encode(), 0o600)
|
|
target = prefix / "lib/web"
|
|
shutil.copytree(Path(__file__).resolve().parents[1] / "web", target, dirs_exist_ok=True,
|
|
ignore=shutil.ignore_patterns("test", "node_modules", ".DS_Store"))
|
|
return {"node": node, "script": str(target / "serve.mjs"),
|
|
"auth_file": str(auth), "host": args.web_host, "port": args.web_port}
|
|
|
|
# --- build and skill -------------------------------------------------
|
|
|
|
def build(self, prefix):
|
|
for command in ("pipelines-server", "pipelines"):
|
|
result = self.runner(
|
|
["go", "build", "-trimpath", "-o", str(prefix / "bin" / command), "./cmd/" + command],
|
|
cwd=str(self.directory))
|
|
if result.returncode:
|
|
raise ValueError(f"go build failed for ./cmd/{command}; the native install is incomplete")
|
|
|
|
# --- stale-CLI-on-PATH guard -----------------------------------------
|
|
|
|
def check_path_shadow(self, prefix):
|
|
"""Warn when a different ``pipelines`` earlier on PATH shadows this install.
|
|
|
|
Detection is read-only and bounded: one ``shutil.which`` check plus one
|
|
bounded ``version --json`` read per binary. The installer never modifies,
|
|
moves, deletes or upgrades the other binary; it only names the shadowing
|
|
path and both versions so an older CLI is not used silently in front of
|
|
the freshly built server. Naming both versions requires reading each
|
|
binary once, so a detected shadow costs exactly two short, read-only
|
|
subprocesses (none when the prefix binary is the one PATH already
|
|
resolves). Returns the shadowing path, or None when the prefix binary is
|
|
the one PATH resolves.
|
|
"""
|
|
installed = prefix / "bin/pipelines"
|
|
found = shutil.which("pipelines")
|
|
if not found:
|
|
return None
|
|
found_path = Path(found).resolve()
|
|
try:
|
|
if found_path == installed.resolve():
|
|
return None
|
|
except OSError:
|
|
pass
|
|
other = self._binary_version(found_path)
|
|
current = self._binary_version(installed)
|
|
print(
|
|
f"WARNING: a different pipelines resolves earlier on PATH than the freshly "
|
|
f"installed CLI: {found_path} (version {other}) shadows {installed} "
|
|
f"(version {current}); the installer left it untouched. Put {prefix / 'bin'} "
|
|
"first on PATH so the CLI matches the running server.")
|
|
return found_path
|
|
|
|
def _binary_version(self, path):
|
|
"""Read a binary's ``version --json`` once; never a hot path."""
|
|
if not Path(path).is_file():
|
|
return "unknown"
|
|
try:
|
|
result = self.runner([str(path), "version", "--json"], capture_output=True, text=True)
|
|
except OSError:
|
|
return "unknown"
|
|
if result.returncode:
|
|
return "unknown"
|
|
text = (result.stdout or "").strip()
|
|
try:
|
|
data = json.loads(text)
|
|
except ValueError:
|
|
return text or "unknown"
|
|
if not isinstance(data, dict):
|
|
return "unknown"
|
|
version = data.get("version") or "unknown"
|
|
commit = data.get("commit")
|
|
return f"{version} (commit {commit})" if commit else str(version)
|
|
|
|
def install_skill(self, prefix):
|
|
result = self.runner([str(prefix / "bin/pipelines"), "skill", "install"], cwd=str(self.directory))
|
|
if result.returncode:
|
|
raise ValueError("pipelines skill install failed; the native install is incomplete")
|
|
|
|
# --- ntfy server build and configuration ----------------------------
|
|
|
|
def resolve_ntfy(self):
|
|
try:
|
|
return self.resolver()
|
|
except release_lookup.ReleaseError as error:
|
|
raise ValueError(
|
|
f"Could not resolve the ntfy release ({error}); refusing to build an unverified version") from None
|
|
|
|
def build_ntfy(self, prefix, resolution):
|
|
"""Compile the ntfy server from the pinned Go module on this host.
|
|
|
|
Upstream publishes no macOS server binary (its darwin asset is built with
|
|
``-tags noserver``), so every platform builds here. The module is fetched
|
|
through the Go toolchain, which verifies it against the public checksum
|
|
database, and cgo is required for ntfy's SQLite driver.
|
|
"""
|
|
path = prefix / "bin/ntfy"
|
|
if path.is_symlink():
|
|
raise ValueError("Refusing to replace a symlinked bin/ntfy")
|
|
version = resolution.get("tag") or resolution.get("version")
|
|
module = resolution.get("module") or release_lookup.NTFY_MODULE
|
|
# A previously built binary for the same resolved version is reused.
|
|
if path.is_file() and self._existing_ntfy_version(prefix) == version:
|
|
return path
|
|
self.require_compiler()
|
|
pinned = f"{module}@{version}"
|
|
result = self.runner(["go", "mod", "download", pinned], cwd=str(self.directory))
|
|
if result.returncode:
|
|
raise ValueError(f"go mod download {pinned} failed; the native install is incomplete")
|
|
cache = self.runner(["go", "env", "GOMODCACHE"], cwd=str(self.directory),
|
|
capture_output=True, text=True)
|
|
if cache.returncode or not (cache.stdout or "").strip():
|
|
raise ValueError("Could not locate the Go module cache to build ntfy from")
|
|
source = Path(cache.stdout.strip()) / f"{module}@{version}"
|
|
if not source.is_dir():
|
|
raise ValueError(f"the Go module cache has no source for {pinned}")
|
|
with tempfile.TemporaryDirectory(prefix="pipelines-ntfy-") as temporary:
|
|
checkout = Path(temporary) / "src"
|
|
shutil.copytree(source, checkout, symlinks=True)
|
|
# The module cache is read-only and carries no built web assets; the
|
|
# upstream Makefile creates the same placeholders for local builds.
|
|
self.runner(["chmod", "-R", "u+w", str(checkout)])
|
|
(checkout / "server/docs").mkdir(parents=True, exist_ok=True)
|
|
(checkout / "server/site").mkdir(parents=True, exist_ok=True)
|
|
(checkout / "server/docs/index.html").touch()
|
|
(checkout / "server/site/app.html").touch()
|
|
built = Path(temporary) / "ntfy"
|
|
env = dict(self.environ)
|
|
env["CGO_ENABLED"] = "1"
|
|
build = self.runner(
|
|
["go", "build", "-trimpath", "-tags", NTFY_BUILD_TAGS,
|
|
"-ldflags", f"-s -w -X main.version={version}", "-o", str(built), "."],
|
|
cwd=str(checkout), env=env)
|
|
if build.returncode:
|
|
raise ValueError(
|
|
"ntfy server build failed; a working C compiler is required for its SQLite driver")
|
|
if not built.is_file():
|
|
raise ValueError("ntfy server build produced no binary")
|
|
os.chmod(built, 0o755)
|
|
os.replace(built, path)
|
|
return path
|
|
|
|
def _existing_ntfy_version(self, prefix):
|
|
manifest = prefix / MARKER
|
|
if manifest.is_symlink() or not manifest.is_file():
|
|
return None
|
|
try:
|
|
data = json.loads(manifest.read_text())
|
|
except (OSError, ValueError):
|
|
return None
|
|
return (data.get("ntfy") or {}).get("version") if isinstance(data, dict) else None
|
|
|
|
def write_ntfy_config(self, prefix, base_url, ntfy_port):
|
|
directory = prefix / "data/ntfy"
|
|
directory.mkdir(mode=0o700, exist_ok=True)
|
|
directory.chmod(0o700)
|
|
path = directory / "server.yml"
|
|
if path.is_symlink():
|
|
raise ValueError("Refusing to replace a symlinked ntfy configuration")
|
|
text = (
|
|
f'base-url: "{base_url}"\n'
|
|
f'listen-http: ":{ntfy_port}"\n'
|
|
f'cache-file: "{directory}/cache.db"\n'
|
|
'cache-duration: "168h"\n'
|
|
f'auth-file: "{directory}/auth.db"\n'
|
|
'auth-default-access: "deny-all"\n'
|
|
'enable-login: true\n'
|
|
f'attachment-cache-dir: "{directory}/attachments"\n'
|
|
'attachment-expiry-duration: "168h"\n'
|
|
'attachment-total-size-limit: "1G"\n'
|
|
'attachment-file-size-limit: "1M"\n'
|
|
)
|
|
self._write_atomic(path, text.encode(), 0o600)
|
|
return path
|
|
|
|
# --- ntfy users and publishing token --------------------------------
|
|
|
|
def ntfy(self, prefix, config_path, args, capture=False):
|
|
# The account subcommands locate the server configuration through
|
|
# NTFY_CONFIG_FILE; --config is a `serve`-only flag.
|
|
env = dict(self.environ)
|
|
env["NTFY_CONFIG_FILE"] = str(config_path)
|
|
result = self.runner([str(prefix / "bin/ntfy"), *args], capture_output=capture, text=True, env=env)
|
|
if result.returncode:
|
|
step = " ".join(args[:2])
|
|
raise ValueError(f"ntfy {step} failed; the native install is incomplete")
|
|
return result
|
|
|
|
def bootstrap_ntfy(self, prefix, config_path):
|
|
def run_ntfy(args, capture=False, interactive=False):
|
|
return self.ntfy(prefix, config_path, list(args), capture=capture)
|
|
|
|
tokens.bootstrap_accounts(run_ntfy, prefix / "secrets/publisher-token", 0o600)
|
|
|
|
# --- Docker migration ------------------------------------------------
|
|
|
|
def docker_volumes(self):
|
|
try:
|
|
result = self.runner(["docker", "volume", "ls", "--format", "{{.Name}}"],
|
|
capture_output=True, text=True)
|
|
except FileNotFoundError:
|
|
# No Docker CLI at all: there is no Docker installation to migrate.
|
|
return ()
|
|
except OSError as error:
|
|
raise ValueError(
|
|
f"Could not inspect Docker volumes for migration ({error}); refusing to start empty") from None
|
|
if result.returncode:
|
|
# A present-but-unusable Docker daemon must not silently skip migration:
|
|
# Docker data may still exist, so fail instead of starting empty.
|
|
raise ValueError(
|
|
"Could not inspect Docker volumes for migration; refusing a possibly empty start")
|
|
names = set(result.stdout.split())
|
|
return tuple(name for name in ("pipelines_pipelines-data", "pipelines_ntfy-data") if name in names)
|
|
|
|
def _migration_image(self):
|
|
result = self.runner(["docker", "image", "ls", "--format", "{{.Repository}}:{{.Tag}}"],
|
|
capture_output=True, text=True)
|
|
if result.returncode:
|
|
return None
|
|
for line in result.stdout.splitlines():
|
|
name = line.strip()
|
|
repository = name.rsplit(":", 1)[0]
|
|
if repository == "pipelines" or repository.endswith("/pipelines") or repository.endswith("-pipelines"):
|
|
return name
|
|
return None
|
|
|
|
def _docker_copy_out(self, image, volume, inner, destination, replace):
|
|
if destination.exists() and not replace:
|
|
print(f"Keeping existing native {destination.name}; not overwriting it during migration.")
|
|
return False
|
|
name = f"pipelines-migrate-{os.getpid()}"
|
|
created = self.runner(["docker", "create", "--name", name, "--volume", f"{volume}:/mnt:ro", image],
|
|
capture_output=True, text=True)
|
|
if created.returncode:
|
|
raise ValueError(f"Could not open Docker volume {volume} for migration; install aborted")
|
|
try:
|
|
result = self.runner(["docker", "cp", f"{name}:/mnt/{inner}", str(destination)],
|
|
capture_output=True, text=True)
|
|
if result.returncode:
|
|
raise ValueError(f"Could not copy {inner} from Docker volume {volume}; install aborted")
|
|
# The live server keeps recent commits in the SQLite write-ahead log,
|
|
# so copy the -wal sidecar too or the newest definitions and runs are
|
|
# lost. SQLite recovers it on next open and rebuilds -shm itself.
|
|
for suffix in ("-wal",):
|
|
sidecar = Path(str(destination) + suffix)
|
|
sidecar.unlink(missing_ok=True)
|
|
copy = self.runner(["docker", "cp", f"{name}:/mnt/{inner}{suffix}", str(sidecar)],
|
|
capture_output=True, text=True)
|
|
if copy.returncode or not sidecar.exists() or sidecar.stat().st_size == 0:
|
|
sidecar.unlink(missing_ok=True)
|
|
finally:
|
|
self.runner(["docker", "rm", "-f", name], capture_output=True, text=True, check=False)
|
|
return True
|
|
|
|
def migrate_docker(self, prefix, replace=False):
|
|
volumes = self.docker_volumes()
|
|
if not volumes:
|
|
return False
|
|
image = self._migration_image()
|
|
if not image:
|
|
raise ValueError(
|
|
"Docker volumes exist but no local Pipelines image is available to read them; "
|
|
"refusing to pull an image. Build the Docker image or rerun without migration.")
|
|
(prefix / "data").mkdir(exist_ok=True)
|
|
if "pipelines_pipelines-data" in volumes:
|
|
self._docker_copy_out(image, "pipelines_pipelines-data", "pipelines.db",
|
|
prefix / "data/pipelines.db", replace)
|
|
if "pipelines_ntfy-data" in volumes:
|
|
directory = prefix / "data/ntfy"
|
|
directory.mkdir(mode=0o700, exist_ok=True)
|
|
self._docker_copy_out(image, "pipelines_ntfy-data", "auth.db",
|
|
directory / "auth.db", replace)
|
|
print("Existing Docker data was migrated where it did not already exist.")
|
|
return True
|
|
|
|
# --- manifest --------------------------------------------------------
|
|
|
|
def write_manifest(self, prefix, values):
|
|
path = prefix / MARKER
|
|
if path.is_symlink():
|
|
raise ValueError("Refusing to replace a symlinked installation manifest")
|
|
self._write_atomic(path, (json.dumps(values, indent=2, sort_keys=True) + "\n").encode(), 0o600)
|
|
return path
|
|
|
|
def _write_atomic(self, path, data, mode):
|
|
handle, temporary = tempfile.mkstemp(dir=str(path.parent), prefix=".pipelines-")
|
|
try:
|
|
os.chmod(temporary, mode)
|
|
with os.fdopen(handle, "wb") as file:
|
|
file.write(data)
|
|
if path.is_symlink():
|
|
raise ValueError(f"Refusing to replace a symlinked file {path}")
|
|
os.replace(temporary, path)
|
|
finally:
|
|
if os.path.exists(temporary):
|
|
os.unlink(temporary)
|
|
|
|
# --- run -------------------------------------------------------------
|
|
|
|
def run(self, args):
|
|
python = self.require_tools()
|
|
if args.prefix:
|
|
self.prefix_argument = args.prefix
|
|
if not 1024 <= args.port <= 65535:
|
|
raise ValueError("Invalid server port; choose a value between 1024 and 65535")
|
|
if not 1024 <= args.ntfy_port <= 65535:
|
|
raise ValueError("Invalid ntfy port; choose a value between 1024 and 65535")
|
|
if args.ntfy_port == args.port:
|
|
raise ValueError("The server and ntfy ports must differ")
|
|
if not 1024 <= args.web_port <= 65535 or args.web_port in (args.port, args.ntfy_port):
|
|
raise ValueError("Web port must be between 1024 and 65535 and differ from server/ntfy ports")
|
|
try:
|
|
ipaddress.IPv4Address(args.bind_address)
|
|
ipaddress.IPv4Address(args.web_host)
|
|
except ValueError:
|
|
raise ValueError("Invalid API/web bind address; use an IPv4 address such as 0.0.0.0 or 127.0.0.1") from None
|
|
prefix = self.resolve_prefix()
|
|
harness = self.harness()
|
|
service_path = self.service_path()
|
|
# Reuse an already built ntfy binary and skip the release lookup, so a
|
|
# reinstall with the same version makes no network requests at all.
|
|
installed = self._existing_ntfy_version(prefix)
|
|
if (prefix / "bin/ntfy").is_file() and installed:
|
|
resolution = {"module": release_lookup.NTFY_MODULE, "version": installed, "tag": installed}
|
|
else:
|
|
self.require_compiler()
|
|
resolution = self.resolve_ntfy()
|
|
self.prepare_layout(prefix)
|
|
web = self.configure_web(prefix, args)
|
|
self.build(prefix)
|
|
self.check_path_shadow(prefix)
|
|
self.install_skill(prefix)
|
|
# The native install launches host harnesses, so write host profiles for
|
|
# the detected harnesses and clear any Docker-era link. Uses only the
|
|
# explicit --model or the existing valid profile/host configuration.
|
|
self.configure_harnesses(harness, args.model or "")
|
|
lan_ip = lan.detect_lan()
|
|
base_url = args.ntfy_url or f"http://{lan_ip}:{args.ntfy_port}"
|
|
config_path = self.write_ntfy_config(prefix, base_url, args.ntfy_port)
|
|
self.build_ntfy(prefix, resolution)
|
|
# Migrate existing Docker data before creating native ntfy accounts so a
|
|
# migrated auth store (the phone's credentials) is preserved rather than
|
|
# shadowed by a freshly bootstrapped one.
|
|
self.migrate_docker(prefix, args.replace_data)
|
|
self.bootstrap_ntfy(prefix, config_path)
|
|
timezone = args.timezone
|
|
if not timezone:
|
|
try:
|
|
timezone = configure.host_timezone()
|
|
except (ValueError, ZoneInfoNotFoundError):
|
|
timezone = "UTC"
|
|
opencode_env = {}
|
|
for key in ("XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_STATE_HOME", "XDG_CACHE_HOME",
|
|
"OPENCODE_SERVER", "OPENCODE_DB"):
|
|
if self.environ.get(key):
|
|
opencode_env[key] = self.environ[key]
|
|
if args.model:
|
|
opencode_env["PIPELINES_DEFAULT_MODEL"] = args.model
|
|
path_value = str(prefix / "bin") + os.pathsep + self.environ.get("PATH", "")
|
|
manifest = {
|
|
"format": 1,
|
|
"prefix": str(prefix),
|
|
"python": python,
|
|
"server_port": args.port,
|
|
"bind_address": args.bind_address,
|
|
"ntfy_port": args.ntfy_port,
|
|
"opencode": harness["opencode"],
|
|
"pi": harness["pi"],
|
|
"opencode_env": opencode_env,
|
|
"path": path_value,
|
|
"topic": "pipelines",
|
|
"public_url": base_url,
|
|
"username": "phone-reader",
|
|
"workspace": str(Path.home()),
|
|
"launcher": str(prefix / "bin/pipelines"),
|
|
"registration": None,
|
|
"label": LABEL,
|
|
"docker": "docker",
|
|
"project": "pipelines",
|
|
"timezone": timezone,
|
|
"ntfy": {"mode": "native", "bin": str(prefix / "bin/ntfy"),
|
|
"config": str(config_path), "base_url": base_url,
|
|
"version": resolution.get("tag") or resolution.get("version")},
|
|
"web": web,
|
|
}
|
|
self.register_service(service_path, manifest)
|
|
self.write_manifest(prefix, manifest)
|
|
# Save the address the host CLI/TUI dials by default, so a plain
|
|
# `pipelines` (or its TUI) reaches this installation instead of a stale
|
|
# Docker address.
|
|
endpoint = f"ws://127.0.0.1:{args.port}/connect"
|
|
client_path = configure.configure_client(endpoint, self.environ)
|
|
print(f"Saved native client default: {endpoint} ({client_path}). Restart the TUI.")
|
|
override = self.environ.get("PIPELINES_URL", "")
|
|
if override and override != endpoint:
|
|
print("WARNING: PIPELINES_URL overrides the saved native address; unset it or pass --url explicitly.")
|
|
print(f"Native Pipelines installed at {prefix}."
|
|
f" ntfy {resolution.get('tag') or resolution.get('version')} was built from source.")
|
|
# Parity with the Docker installer: leave the server and ntfy running.
|
|
started = self.runner([python, str(prefix / "lib/manage.py"), "start", "--prefix", str(prefix)])
|
|
if started.returncode:
|
|
raise ValueError(
|
|
"The native services were installed but could not be started; run "
|
|
f"'PIPELINES_PREFIX={prefix} {prefix}/bin/pipelines start' to see the error. "
|
|
"Check your login service manager and that ports "
|
|
f"{args.port}, {args.ntfy_port} and {args.web_port} are free (a running Docker installation may use them).")
|
|
print(f"ntfy is configured for {base_url}; server, web proxy and ntfy have login autostart ({service_path}).")
|
|
print(f"Web: http://{lan_ip if args.web_host == '0.0.0.0' else args.web_host}:{args.web_port}/ "
|
|
f"(sign-in password in {web['auth_file']}; existing credentials retained).")
|
|
if args.bind_address == "0.0.0.0":
|
|
print(f"Phone client server URL: ws://{lan_ip}:{args.port}/connect "
|
|
"(use a workspace path that exists on this server).")
|
|
else:
|
|
print(f"Unauthenticated API bound to {args.bind_address}:{args.port}; use the authenticated web URL.")
|
|
|
|
|
|
def arguments(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--prefix", type=Path, help="Installation prefix (default: ~/.local/share/pipelines)")
|
|
parser.add_argument("--ntfy-url", help="Override the auto-detected LAN ntfy address")
|
|
parser.add_argument("--timezone", help="Override the detected timezone (fallback: UTC)")
|
|
parser.add_argument("--port", type=int, default=DEFAULT_SERVER_PORT, help="Pipelines server port")
|
|
parser.add_argument("--web-port", type=int, default=8082, help="Authenticated web proxy port (default 8082)")
|
|
parser.add_argument("--web-host", default="0.0.0.0", help="Web proxy bind address (default 0.0.0.0)")
|
|
parser.add_argument("--bind-address", default=DEFAULT_BIND_ADDRESS,
|
|
help="Unauthenticated server bind address (default 127.0.0.1; expose only the web proxy)")
|
|
parser.add_argument("--ntfy-port", type=int, default=DEFAULT_NTFY_PORT, help="ntfy listen port")
|
|
parser.add_argument("--model", help="Record an explicit default model; otherwise host OpenCode configuration is reused")
|
|
parser.add_argument("--replace-data", action="store_true",
|
|
help="Allow Docker migration to replace an existing native database/auth store")
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
Setup(Path(__file__).resolve().parents[1]).run(arguments())
|
|
except KeyboardInterrupt:
|
|
print("\nNative setup canceled; existing configuration and data were retained.", file=sys.stderr)
|
|
sys.exit(130)
|
|
except (ValueError, OSError, json.JSONDecodeError, subprocess.SubprocessError,
|
|
ZoneInfoNotFoundError) as error:
|
|
sys.exit(f"Native setup incomplete: {error}")
|