a70e3aef4b
Add scripts/setup-native.py and make ./install (and --native) build the server, the CLI and ntfy from source into ~/.local/share/pipelines, detect an existing host OpenCode and Pi instead of installing them, install the packaged authoring skill, migrate an existing Docker database and ntfy auth store, save the host client default and start both services. ./install --docker keeps the previous Docker server + ntfy installation unchanged. ntfy is compiled from its pinned Go module on every host because upstream publishes no macOS server binary (its darwin asset is built with -tags noserver); its cgo SQLite driver is why a C compiler is now required natively. The ntfy credential bootstrap and the client default writer are shared by both installers, installation.Load resolves symlinked prefixes, the Docker migration copies the SQLite write-ahead log so the newest definitions and runs are not lost, and the account subcommands read the generated configuration through NTFY_CONFIG_FILE. Verified on macOS: 115 Python tests, go build/vet/test, and scratch installs that migrate 9 pipelines and 31 runs, build ntfy v2.30.0 and start both services healthy on spare ports before stopping them.
466 lines
22 KiB
Python
466 lines
22 KiB
Python
#!/usr/bin/env python3
|
|
"""Install Pipelines natively on the host (macOS and Linux).
|
|
|
|
The installer only detects the tools it needs (go, python3, git, curl and a C
|
|
compiler for ntfy's SQLite driver); it never installs or upgrades them, and it
|
|
never installs, upgrades or downloads OpenCode or Pi. It builds the server, CLI
|
|
and ntfy from source into a per-user prefix, generates an ntfy configuration that
|
|
mirrors compose.yaml, creates the publishing/phone accounts and writes the
|
|
manifest scripts/manage.py loads. ntfy is compiled from its pinned Go module on
|
|
every host because upstream publishes no macOS server binary. It reuses existing
|
|
host OpenCode configuration; model selection stays at runtime and is never
|
|
invented here.
|
|
"""
|
|
import argparse
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
from zoneinfo import ZoneInfoNotFoundError
|
|
|
|
|
|
def module(name, file):
|
|
spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(file))
|
|
result = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(result)
|
|
return result
|
|
|
|
|
|
configure = module("configure_compose", "configure-compose.py")
|
|
lan = module("lan_address", "lan-address.py")
|
|
release_lookup = module("resolve_releases", "resolve-releases.py")
|
|
tokens = module("save_ntfy_token", "save-ntfy-token.py")
|
|
|
|
DEFAULT_PREFIX = Path.home() / ".local/share/pipelines"
|
|
MARKER = "pipelines-install.json"
|
|
LABEL = "net.a-wan.pipelines"
|
|
REQUIRED_TOOLS = ("go", "python3", "git", "curl")
|
|
# ntfy's SQLite driver (mattn/go-sqlite3) is cgo, so a C compiler is required to
|
|
# build the server. Upstream's macOS release asset is client-only, so both
|
|
# platforms compile ntfy from the pinned Go module.
|
|
C_COMPILERS = ("cc", "clang", "gcc")
|
|
NTFY_BUILD_TAGS = "sqlite_omit_load_extension,osusergo,netgo"
|
|
DEFAULT_NTFY_PORT = 8090
|
|
DEFAULT_SERVER_PORT = 8080
|
|
|
|
|
|
class Setup:
|
|
def __init__(self, directory, prefix=None, runner=subprocess.run, resolver=None, environ=None):
|
|
self.directory = Path(directory)
|
|
self.prefix_argument = prefix
|
|
self.runner = runner
|
|
self.resolver = resolver or release_lookup.resolve_ntfy
|
|
self.environ = environ if environ is not None else os.environ
|
|
|
|
# --- detection -------------------------------------------------------
|
|
|
|
def require_compiler(self):
|
|
"""ntfy is built locally, so a C toolchain must already be present."""
|
|
for compiler in C_COMPILERS:
|
|
path = shutil.which(compiler)
|
|
if path:
|
|
return str(Path(path).resolve())
|
|
raise ValueError(
|
|
"No C compiler found (tried " + ", ".join(C_COMPILERS)
|
|
+ "). ntfy's SQLite driver is cgo, so the server is compiled locally; "
|
|
"install a C toolchain (Xcode Command Line Tools on macOS, gcc or clang on Linux) and rerun.")
|
|
|
|
def require_tools(self):
|
|
found, missing = {}, []
|
|
for tool in REQUIRED_TOOLS:
|
|
path = shutil.which(tool)
|
|
if path:
|
|
found[tool] = str(Path(path).resolve())
|
|
else:
|
|
missing.append(tool)
|
|
if missing:
|
|
raise ValueError(
|
|
"Missing required tool(s): " + ", ".join(missing)
|
|
+ ". Install them yourself and rerun; this installer never installs tools or uses a package manager.")
|
|
return found["python3"]
|
|
|
|
def harness(self):
|
|
"""Detect existing harnesses only; never install, upgrade or download them."""
|
|
opencode = shutil.which("opencode")
|
|
if not opencode:
|
|
raise ValueError(
|
|
"OpenCode was not found on PATH. Install OpenCode yourself (this installer never installs, "
|
|
"upgrades or downloads it) and rerun; model selection happens in your existing OpenCode configuration.")
|
|
pi = shutil.which("pi")
|
|
return {"opencode": str(Path(opencode).resolve()), "pi": str(Path(pi).resolve()) if pi else ""}
|
|
|
|
def resolve_prefix(self):
|
|
default = Path.home() / ".local/share/pipelines"
|
|
raw = self.prefix_argument or self.environ.get("PIPELINES_PREFIX") or default
|
|
return Path(raw).expanduser().resolve()
|
|
|
|
# --- prefix layout ---------------------------------------------------
|
|
|
|
def prepare_layout(self, prefix):
|
|
prefix.mkdir(parents=True, exist_ok=True)
|
|
for name in ("bin", "lib", "data", "logs", "secrets"):
|
|
(prefix / name).mkdir(exist_ok=True)
|
|
(prefix / "secrets").chmod(0o700)
|
|
(prefix / "data").chmod(0o700)
|
|
source = Path(__file__).with_name("manage.py")
|
|
target = prefix / "lib/manage.py"
|
|
if target.is_symlink():
|
|
raise ValueError("Refusing to replace a symlinked lib/manage.py")
|
|
shutil.copyfile(source, target)
|
|
os.chmod(target, 0o700)
|
|
return target
|
|
|
|
# --- build and skill -------------------------------------------------
|
|
|
|
def build(self, prefix):
|
|
for command in ("pipelines-server", "pipelines"):
|
|
result = self.runner(
|
|
["go", "build", "-trimpath", "-o", str(prefix / "bin" / command), "./cmd/" + command],
|
|
cwd=str(self.directory))
|
|
if result.returncode:
|
|
raise ValueError(f"go build failed for ./cmd/{command}; the native install is incomplete")
|
|
|
|
def install_skill(self, prefix):
|
|
result = self.runner([str(prefix / "bin/pipelines"), "skill", "install"], cwd=str(self.directory))
|
|
if result.returncode:
|
|
raise ValueError("pipelines skill install failed; the native install is incomplete")
|
|
|
|
# --- ntfy server build and configuration ----------------------------
|
|
|
|
def resolve_ntfy(self):
|
|
try:
|
|
return self.resolver()
|
|
except release_lookup.ReleaseError as error:
|
|
raise ValueError(
|
|
f"Could not resolve the ntfy release ({error}); refusing to build an unverified version") from None
|
|
|
|
def build_ntfy(self, prefix, resolution):
|
|
"""Compile the ntfy server from the pinned Go module on this host.
|
|
|
|
Upstream publishes no macOS server binary (its darwin asset is built with
|
|
``-tags noserver``), so every platform builds here. The module is fetched
|
|
through the Go toolchain, which verifies it against the public checksum
|
|
database, and cgo is required for ntfy's SQLite driver.
|
|
"""
|
|
path = prefix / "bin/ntfy"
|
|
if path.is_symlink():
|
|
raise ValueError("Refusing to replace a symlinked bin/ntfy")
|
|
version = resolution.get("tag") or resolution.get("version")
|
|
module = resolution.get("module") or release_lookup.NTFY_MODULE
|
|
# A previously built binary for the same resolved version is reused.
|
|
if path.is_file() and self._existing_ntfy_version(prefix) == version:
|
|
return path
|
|
self.require_compiler()
|
|
pinned = f"{module}@{version}"
|
|
result = self.runner(["go", "mod", "download", pinned], cwd=str(self.directory))
|
|
if result.returncode:
|
|
raise ValueError(f"go mod download {pinned} failed; the native install is incomplete")
|
|
cache = self.runner(["go", "env", "GOMODCACHE"], cwd=str(self.directory),
|
|
capture_output=True, text=True)
|
|
if cache.returncode or not (cache.stdout or "").strip():
|
|
raise ValueError("Could not locate the Go module cache to build ntfy from")
|
|
source = Path(cache.stdout.strip()) / f"{module}@{version}"
|
|
if not source.is_dir():
|
|
raise ValueError(f"the Go module cache has no source for {pinned}")
|
|
with tempfile.TemporaryDirectory(prefix="pipelines-ntfy-") as temporary:
|
|
checkout = Path(temporary) / "src"
|
|
shutil.copytree(source, checkout, symlinks=True)
|
|
# The module cache is read-only and carries no built web assets; the
|
|
# upstream Makefile creates the same placeholders for local builds.
|
|
self.runner(["chmod", "-R", "u+w", str(checkout)])
|
|
(checkout / "server/docs").mkdir(parents=True, exist_ok=True)
|
|
(checkout / "server/site").mkdir(parents=True, exist_ok=True)
|
|
(checkout / "server/docs/index.html").touch()
|
|
(checkout / "server/site/app.html").touch()
|
|
built = Path(temporary) / "ntfy"
|
|
env = dict(self.environ)
|
|
env["CGO_ENABLED"] = "1"
|
|
build = self.runner(
|
|
["go", "build", "-trimpath", "-tags", NTFY_BUILD_TAGS,
|
|
"-ldflags", f"-s -w -X main.version={version}", "-o", str(built), "."],
|
|
cwd=str(checkout), env=env)
|
|
if build.returncode:
|
|
raise ValueError(
|
|
"ntfy server build failed; a working C compiler is required for its SQLite driver")
|
|
if not built.is_file():
|
|
raise ValueError("ntfy server build produced no binary")
|
|
os.chmod(built, 0o755)
|
|
os.replace(built, path)
|
|
return path
|
|
|
|
def _existing_ntfy_version(self, prefix):
|
|
manifest = prefix / MARKER
|
|
if manifest.is_symlink() or not manifest.is_file():
|
|
return None
|
|
try:
|
|
data = json.loads(manifest.read_text())
|
|
except (OSError, ValueError):
|
|
return None
|
|
return (data.get("ntfy") or {}).get("version") if isinstance(data, dict) else None
|
|
|
|
def write_ntfy_config(self, prefix, base_url, ntfy_port):
|
|
directory = prefix / "data/ntfy"
|
|
directory.mkdir(mode=0o700, exist_ok=True)
|
|
directory.chmod(0o700)
|
|
path = directory / "server.yml"
|
|
if path.is_symlink():
|
|
raise ValueError("Refusing to replace a symlinked ntfy configuration")
|
|
text = (
|
|
f'base-url: "{base_url}"\n'
|
|
f'listen-http: ":{ntfy_port}"\n'
|
|
f'cache-file: "{directory}/cache.db"\n'
|
|
'cache-duration: "168h"\n'
|
|
f'auth-file: "{directory}/auth.db"\n'
|
|
'auth-default-access: "deny-all"\n'
|
|
'enable-login: true\n'
|
|
f'attachment-cache-dir: "{directory}/attachments"\n'
|
|
'attachment-expiry-duration: "168h"\n'
|
|
'attachment-total-size-limit: "1G"\n'
|
|
'attachment-file-size-limit: "1M"\n'
|
|
)
|
|
self._write_atomic(path, text.encode(), 0o600)
|
|
return path
|
|
|
|
# --- ntfy users and publishing token --------------------------------
|
|
|
|
def ntfy(self, prefix, config_path, args, capture=False):
|
|
# The account subcommands locate the server configuration through
|
|
# NTFY_CONFIG_FILE; --config is a `serve`-only flag.
|
|
env = dict(self.environ)
|
|
env["NTFY_CONFIG_FILE"] = str(config_path)
|
|
result = self.runner([str(prefix / "bin/ntfy"), *args], capture_output=capture, text=True, env=env)
|
|
if result.returncode:
|
|
step = " ".join(args[:2])
|
|
raise ValueError(f"ntfy {step} failed; the native install is incomplete")
|
|
return result
|
|
|
|
def bootstrap_ntfy(self, prefix, config_path):
|
|
def run_ntfy(args, capture=False, interactive=False):
|
|
return self.ntfy(prefix, config_path, list(args), capture=capture)
|
|
|
|
tokens.bootstrap_accounts(run_ntfy, prefix / "secrets/publisher-token", 0o600)
|
|
|
|
# --- Docker migration ------------------------------------------------
|
|
|
|
def docker_volumes(self):
|
|
try:
|
|
result = self.runner(["docker", "volume", "ls", "--format", "{{.Name}}"],
|
|
capture_output=True, text=True)
|
|
except FileNotFoundError:
|
|
# No Docker CLI at all: there is no Docker installation to migrate.
|
|
return ()
|
|
except OSError as error:
|
|
raise ValueError(
|
|
f"Could not inspect Docker volumes for migration ({error}); refusing to start empty") from None
|
|
if result.returncode:
|
|
# A present-but-unusable Docker daemon must not silently skip migration:
|
|
# Docker data may still exist, so fail instead of starting empty.
|
|
raise ValueError(
|
|
"Could not inspect Docker volumes for migration; refusing a possibly empty start")
|
|
names = set(result.stdout.split())
|
|
return tuple(name for name in ("pipelines_pipelines-data", "pipelines_ntfy-data") if name in names)
|
|
|
|
def _migration_image(self):
|
|
result = self.runner(["docker", "image", "ls", "--format", "{{.Repository}}:{{.Tag}}"],
|
|
capture_output=True, text=True)
|
|
if result.returncode:
|
|
return None
|
|
for line in result.stdout.splitlines():
|
|
name = line.strip()
|
|
repository = name.rsplit(":", 1)[0]
|
|
if repository == "pipelines" or repository.endswith("/pipelines") or repository.endswith("-pipelines"):
|
|
return name
|
|
return None
|
|
|
|
def _docker_copy_out(self, image, volume, inner, destination, replace):
|
|
if destination.exists() and not replace:
|
|
print(f"Keeping existing native {destination.name}; not overwriting it during migration.")
|
|
return False
|
|
name = f"pipelines-migrate-{os.getpid()}"
|
|
created = self.runner(["docker", "create", "--name", name, "--volume", f"{volume}:/mnt:ro", image],
|
|
capture_output=True, text=True)
|
|
if created.returncode:
|
|
raise ValueError(f"Could not open Docker volume {volume} for migration; install aborted")
|
|
try:
|
|
result = self.runner(["docker", "cp", f"{name}:/mnt/{inner}", str(destination)],
|
|
capture_output=True, text=True)
|
|
if result.returncode:
|
|
raise ValueError(f"Could not copy {inner} from Docker volume {volume}; install aborted")
|
|
# The live server keeps recent commits in the SQLite write-ahead log,
|
|
# so copy the -wal sidecar too or the newest definitions and runs are
|
|
# lost. SQLite recovers it on next open and rebuilds -shm itself.
|
|
for suffix in ("-wal",):
|
|
sidecar = Path(str(destination) + suffix)
|
|
sidecar.unlink(missing_ok=True)
|
|
copy = self.runner(["docker", "cp", f"{name}:/mnt/{inner}{suffix}", str(sidecar)],
|
|
capture_output=True, text=True)
|
|
if copy.returncode or not sidecar.exists() or sidecar.stat().st_size == 0:
|
|
sidecar.unlink(missing_ok=True)
|
|
finally:
|
|
self.runner(["docker", "rm", "-f", name], capture_output=True, text=True, check=False)
|
|
return True
|
|
|
|
def migrate_docker(self, prefix, replace=False):
|
|
volumes = self.docker_volumes()
|
|
if not volumes:
|
|
return False
|
|
image = self._migration_image()
|
|
if not image:
|
|
raise ValueError(
|
|
"Docker volumes exist but no local Pipelines image is available to read them; "
|
|
"refusing to pull an image. Build the Docker image or rerun without migration.")
|
|
(prefix / "data").mkdir(exist_ok=True)
|
|
if "pipelines_pipelines-data" in volumes:
|
|
self._docker_copy_out(image, "pipelines_pipelines-data", "pipelines.db",
|
|
prefix / "data/pipelines.db", replace)
|
|
if "pipelines_ntfy-data" in volumes:
|
|
directory = prefix / "data/ntfy"
|
|
directory.mkdir(mode=0o700, exist_ok=True)
|
|
self._docker_copy_out(image, "pipelines_ntfy-data", "auth.db",
|
|
directory / "auth.db", replace)
|
|
print("Existing Docker data was migrated where it did not already exist.")
|
|
return True
|
|
|
|
# --- manifest --------------------------------------------------------
|
|
|
|
def write_manifest(self, prefix, values):
|
|
path = prefix / MARKER
|
|
if path.is_symlink():
|
|
raise ValueError("Refusing to replace a symlinked installation manifest")
|
|
self._write_atomic(path, (json.dumps(values, indent=2, sort_keys=True) + "\n").encode(), 0o600)
|
|
return path
|
|
|
|
def _write_atomic(self, path, data, mode):
|
|
handle, temporary = tempfile.mkstemp(dir=str(path.parent), prefix=".pipelines-")
|
|
try:
|
|
os.chmod(temporary, mode)
|
|
with os.fdopen(handle, "wb") as file:
|
|
file.write(data)
|
|
if path.is_symlink():
|
|
raise ValueError(f"Refusing to replace a symlinked file {path}")
|
|
os.replace(temporary, path)
|
|
finally:
|
|
if os.path.exists(temporary):
|
|
os.unlink(temporary)
|
|
|
|
# --- run -------------------------------------------------------------
|
|
|
|
def run(self, args):
|
|
python = self.require_tools()
|
|
if args.prefix:
|
|
self.prefix_argument = args.prefix
|
|
if not 1024 <= args.port <= 65535:
|
|
raise ValueError("Invalid server port; choose a value between 1024 and 65535")
|
|
if not 1024 <= args.ntfy_port <= 65535:
|
|
raise ValueError("Invalid ntfy port; choose a value between 1024 and 65535")
|
|
if args.ntfy_port == args.port:
|
|
raise ValueError("The server and ntfy ports must differ")
|
|
prefix = self.resolve_prefix()
|
|
self.prepare_layout(prefix)
|
|
harness = self.harness()
|
|
# Reuse an already built ntfy binary and skip the release lookup, so a
|
|
# reinstall with the same version makes no network requests at all.
|
|
installed = self._existing_ntfy_version(prefix)
|
|
if (prefix / "bin/ntfy").is_file() and installed:
|
|
resolution = {"module": release_lookup.NTFY_MODULE, "version": installed, "tag": installed}
|
|
else:
|
|
resolution = self.resolve_ntfy()
|
|
self.build(prefix)
|
|
self.install_skill(prefix)
|
|
base_url = args.ntfy_url or f"http://{lan.detect_lan()}:{args.ntfy_port}"
|
|
config_path = self.write_ntfy_config(prefix, base_url, args.ntfy_port)
|
|
self.build_ntfy(prefix, resolution)
|
|
# Migrate existing Docker data before creating native ntfy accounts so a
|
|
# migrated auth store (the phone's credentials) is preserved rather than
|
|
# shadowed by a freshly bootstrapped one.
|
|
self.migrate_docker(prefix, args.replace_data)
|
|
self.bootstrap_ntfy(prefix, config_path)
|
|
timezone = args.timezone
|
|
if not timezone:
|
|
try:
|
|
timezone = configure.host_timezone()
|
|
except (ValueError, ZoneInfoNotFoundError):
|
|
timezone = "UTC"
|
|
opencode_env = {}
|
|
for key in ("XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_STATE_HOME", "XDG_CACHE_HOME",
|
|
"OPENCODE_SERVER", "OPENCODE_DB"):
|
|
if self.environ.get(key):
|
|
opencode_env[key] = self.environ[key]
|
|
if args.model:
|
|
opencode_env["PIPELINES_DEFAULT_MODEL"] = args.model
|
|
path_value = str(prefix / "bin") + os.pathsep + self.environ.get("PATH", "")
|
|
manifest = {
|
|
"format": 1,
|
|
"prefix": str(prefix),
|
|
"python": python,
|
|
"server_port": args.port,
|
|
"ntfy_port": args.ntfy_port,
|
|
"opencode": harness["opencode"],
|
|
"pi": harness["pi"],
|
|
"opencode_env": opencode_env,
|
|
"path": path_value,
|
|
"topic": "pipelines",
|
|
"public_url": base_url,
|
|
"username": "phone-reader",
|
|
"workspace": str(Path.home()),
|
|
"launcher": str(prefix / "bin/pipelines"),
|
|
"registration": None,
|
|
"label": LABEL,
|
|
"docker": "docker",
|
|
"project": "pipelines",
|
|
"timezone": timezone,
|
|
"ntfy": {"mode": "native", "bin": str(prefix / "bin/ntfy"),
|
|
"config": str(config_path), "base_url": base_url,
|
|
"version": resolution.get("tag") or resolution.get("version")},
|
|
}
|
|
self.write_manifest(prefix, manifest)
|
|
# Save the address the host CLI/TUI dials by default, so a plain
|
|
# `pipelines` (or its TUI) reaches this installation instead of a stale
|
|
# Docker address.
|
|
endpoint = f"ws://127.0.0.1:{args.port}/connect"
|
|
client_path = configure.configure_client(endpoint, self.environ)
|
|
print(f"Saved native client default: {endpoint} ({client_path}). Restart the TUI.")
|
|
override = self.environ.get("PIPELINES_URL", "")
|
|
if override and override != endpoint:
|
|
print("WARNING: PIPELINES_URL overrides the saved native address; unset it or pass --url explicitly.")
|
|
print(f"Native Pipelines installed at {prefix}."
|
|
f" ntfy {resolution.get('tag') or resolution.get('version')} was built from source.")
|
|
# Parity with the Docker installer: leave the server and ntfy running.
|
|
started = self.runner([python, str(prefix / "lib/manage.py"), "start", "--prefix", str(prefix)])
|
|
if started.returncode:
|
|
raise ValueError(
|
|
"The native services were installed but could not be started; run "
|
|
f"'{prefix}/bin/pipelines start' to see the error. Check that ports "
|
|
f"{args.port} and {args.ntfy_port} are free (a running Docker installation uses them).")
|
|
print(f"ntfy is configured for {base_url} and both services are running.")
|
|
print("Create the phone account password if it is a new account, then subscribe to the pipelines topic.")
|
|
|
|
|
|
def arguments(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--prefix", type=Path, help="Installation prefix (default: ~/.local/share/pipelines)")
|
|
parser.add_argument("--ntfy-url", help="Override the auto-detected LAN ntfy address")
|
|
parser.add_argument("--timezone", help="Override the detected timezone (fallback: UTC)")
|
|
parser.add_argument("--port", type=int, default=DEFAULT_SERVER_PORT, help="Pipelines server port")
|
|
parser.add_argument("--ntfy-port", type=int, default=DEFAULT_NTFY_PORT, help="ntfy listen port")
|
|
parser.add_argument("--model", help="Record an explicit default model; otherwise host OpenCode configuration is reused")
|
|
parser.add_argument("--replace-data", action="store_true",
|
|
help="Allow Docker migration to replace an existing native database/auth store")
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
Setup(Path(__file__).resolve().parents[1]).run(arguments())
|
|
except KeyboardInterrupt:
|
|
print("\nNative setup canceled; existing configuration and data were retained.", file=sys.stderr)
|
|
sys.exit(130)
|
|
except (ValueError, OSError, json.JSONDecodeError, subprocess.SubprocessError,
|
|
ZoneInfoNotFoundError) as error:
|
|
sys.exit(f"Native setup incomplete: {error}")
|