Install Pipelines natively by default on macOS and Linux

Add scripts/setup-native.py and make ./install (and --native) build the server, the CLI
and ntfy from source into ~/.local/share/pipelines, detect an existing host OpenCode
and Pi instead of installing them, install the packaged authoring skill, migrate an
existing Docker database and ntfy auth store, save the host client default and start
both services. ./install --docker keeps the previous Docker server + ntfy
installation unchanged.

ntfy is compiled from its pinned Go module on every host because upstream publishes
no macOS server binary (its darwin asset is built with -tags noserver); its cgo
SQLite driver is why a C compiler is now required natively. The ntfy credential
bootstrap and the client default writer are shared by both installers,
installation.Load resolves symlinked prefixes, the Docker migration copies the
SQLite write-ahead log so the newest definitions and runs are not lost, and the
account subcommands read the generated configuration through NTFY_CONFIG_FILE.

Verified on macOS: 115 Python tests, go build/vet/test, and scratch installs that
migrate 9 pipelines and 31 runs, build ntfy v2.30.0 and start both services healthy
on spare ports before stopping them.
This commit is contained in:
2026-10-13 17:20:14 +01:00
parent 9b305df725
commit a70e3aef4b
21 changed files with 1474 additions and 153 deletions
+362
View File
@@ -0,0 +1,362 @@
import importlib.util
import os
from pathlib import Path
import subprocess
import tempfile
import unittest
from unittest.mock import patch
spec = importlib.util.spec_from_file_location("setup_native", Path(__file__).with_name("setup-native.py"))
setup_native = importlib.util.module_from_spec(spec)
spec.loader.exec_module(setup_native)
manage_spec = importlib.util.spec_from_file_location("pipelines_manage", Path(__file__).with_name("manage.py"))
manage = importlib.util.module_from_spec(manage_spec)
manage_spec.loader.exec_module(manage)
TOKEN = "tk_" + "a" * 29
NTFY_VERSION = "2.11.0"
NTFY_MODULE = "heckel.io/ntfy/v2"
def resolution():
"""The shape resolve_ntfy returns: an exact tag to build the server from."""
return {"repository": "binwiederhier/ntfy", "module": NTFY_MODULE,
"version": NTFY_VERSION, "tag": "v" + NTFY_VERSION}
class Recorder:
def __init__(self, token=TOKEN, volumes=(), image="pipelines-pipelines:latest",
fail_create=False, fail_copy=False, fail_volume_ls=False, fail_build=False):
self.calls = []
self.token = token
self.volumes = volumes
self.image = image
self.fail_create = fail_create
self.fail_copy = fail_copy
self.fail_volume_ls = fail_volume_ls
self.fail_build = fail_build
self._module_cache = None
self.kwargs = []
def module_cache(self):
"""A stand-in Go module cache holding the pinned ntfy module source."""
if self._module_cache is None:
self._module_cache = tempfile.mkdtemp(prefix="gomodcache-")
source = Path(self._module_cache) / f"{NTFY_MODULE}@{resolution()['tag']}"
source.mkdir(parents=True)
(source / "go.mod").write_text(f"module {NTFY_MODULE}\n")
return self._module_cache
def __call__(self, command, **kwargs):
command = list(command)
self.calls.append(command)
self.kwargs.append(kwargs)
stdout, code = "", 0
if command[0] == "docker":
if command[1:3] == ["volume", "ls"]:
if self.fail_volume_ls:
code = 1
else:
stdout = "\n".join(self.volumes)
elif command[1:3] == ["image", "ls"]:
stdout = self.image + "\n" if self.image else ""
elif command[1] == "create" and self.fail_create:
code = 1
elif command[1] == "cp":
if self.fail_copy:
code = 1
else:
Path(command[3]).write_bytes(b"migrated")
elif command[0] == "go" and command[1:3] == ["env", "GOMODCACHE"]:
stdout = self.module_cache() + "\n"
elif command[0] == "go" and command[1:2] == ["build"]:
if self.fail_build:
code = 1
elif "-o" in command:
# Only the ntfy build is inspected by the installer; its output
# is created so the atomic install has a real file to move.
output = Path(command[command.index("-o") + 1])
if output.name == "ntfy":
output.write_bytes(b"#!/bin/sh\n")
output.chmod(0o755)
if command[0].endswith("ntfy") and "token" in command:
stdout = f"token {self.token} created for pipeline-publisher\n"
return subprocess.CompletedProcess(command, code, stdout=stdout, stderr="")
def has(self, *parts):
return any(all(part in call for part in parts) for call in self.calls)
class SetupNativeTests(unittest.TestCase):
def installer(self, directory, recorder):
# A disposable XDG config home keeps the client default out of the real
# user configuration during tests.
return setup_native.Setup(directory, prefix=Path(directory) / "prefix", runner=recorder,
resolver=resolution,
environ={"XDG_CONFIG_HOME": str(Path(directory) / "config")})
@staticmethod
def which_all(opencode="/usr/bin/opencode", pi=None, cc="/usr/bin/cc"):
mapping = {"go": "/usr/bin/go", "python3": "/usr/bin/python3",
"git": "/usr/bin/git", "curl": "/usr/bin/curl"}
if opencode:
mapping["opencode"] = opencode
if pi:
mapping["pi"] = pi
if cc:
mapping["cc"] = cc
return lambda name: mapping.get(name)
def test_require_tools_reports_each_missing_tool_without_installing(self):
with tempfile.TemporaryDirectory() as directory:
recorder = Recorder()
installer = self.installer(directory, recorder)
mapping = {"go": "/usr/bin/go", "python3": "/usr/bin/python3", "git": None, "curl": None}
with patch.object(setup_native.shutil, "which", side_effect=lambda name: mapping.get(name)):
with self.assertRaises(ValueError) as raised:
installer.require_tools()
message = str(raised.exception)
self.assertIn("git", message)
self.assertIn("curl", message)
self.assertIn("never installs tools", message)
self.assertEqual(recorder.calls, [])
def test_build_installs_only_server_and_cli_with_trimpath(self):
with tempfile.TemporaryDirectory() as directory:
recorder = Recorder()
installer = self.installer(directory, recorder)
prefix = Path(directory) / "prefix"
installer.prepare_layout(prefix)
installer.build(prefix)
builds = [call for call in recorder.calls if call[:2] == ["go", "build"]]
self.assertEqual(len(builds), 2)
for call in builds:
self.assertIn("-trimpath", call)
self.assertTrue(any(call[-1] == "./cmd/pipelines-server" for call in builds))
self.assertTrue(any(call[-1] == "./cmd/pipelines" for call in builds))
self.assertFalse(any("pipelines-tui" in part for call in recorder.calls for part in call))
def test_layout_creates_private_prefix_and_copies_manage(self):
with tempfile.TemporaryDirectory() as directory:
installer = self.installer(directory, Recorder())
prefix = Path(directory) / "prefix"
installer.prepare_layout(prefix)
for name in ("bin", "lib", "data", "logs", "secrets"):
self.assertTrue((prefix / name).is_dir(), name)
self.assertEqual((prefix / "secrets").stat().st_mode & 0o777, 0o700)
self.assertTrue((prefix / "lib/manage.py").is_file())
def test_prefix_honours_pipelines_prefix_environment(self):
with tempfile.TemporaryDirectory() as directory:
custom = Path(directory) / "custom"
installer = setup_native.Setup(directory, runner=Recorder(), environ={"PIPELINES_PREFIX": str(custom)})
self.assertEqual(installer.resolve_prefix(), custom.resolve())
def test_missing_opencode_fails_actionably(self):
with tempfile.TemporaryDirectory() as directory:
installer = self.installer(directory, Recorder())
with patch.object(setup_native.shutil, "which", side_effect=self.which_all(opencode=None)):
with self.assertRaisesRegex(ValueError, "OpenCode was not found"):
installer.harness()
def test_full_run_writes_manifest_tokens_and_config_without_harness_install(self):
with tempfile.TemporaryDirectory() as directory:
recorder = Recorder()
installer = self.installer(directory, recorder)
prefix = Path(directory) / "prefix"
with patch.object(setup_native.shutil, "which", side_effect=self.which_all()), \
patch.object(setup_native.lan, "detect_lan", return_value="192.168.1.20"), \
patch.object(setup_native.configure, "host_timezone", return_value="Europe/London"):
installer.run(setup_native.arguments([]))
config = manage.load(prefix)
self.assertEqual(config["format"], 1)
self.assertEqual(config["prefix"], str(prefix.resolve()))
self.assertEqual(config["server_port"], 8080)
self.assertEqual(config["ntfy"]["mode"], "native")
self.assertEqual(config["topic"], "pipelines")
self.assertEqual(config["opencode"], "/usr/bin/opencode")
self.assertEqual(config["pi"], "")
self.assertEqual((prefix / setup_native.MARKER).stat().st_mode & 0o777, 0o600)
token_path = prefix / "secrets/publisher-token"
self.assertEqual(token_path.read_text(), TOKEN + "\n")
self.assertEqual(token_path.stat().st_mode & 0o777, 0o600)
text = (prefix / "data/ntfy/server.yml").read_text()
for needle in ('listen-http:', 'base-url: "http://192.168.1.20:8090"', 'cache-file:', 'auth-file:',
'auth-default-access: "deny-all"', 'enable-login: true', 'attachment-cache-dir:',
'attachment-expiry-duration:', 'attachment-total-size-limit:',
'attachment-file-size-limit:'):
self.assertIn(needle, text)
self.assertNotIn("topic", text)
self.assertTrue(recorder.has(str(prefix.resolve() / "bin/pipelines"), "skill", "install"))
self.assertTrue(recorder.has("access", "pipeline-publisher", "pipelines", "write-only"))
self.assertTrue(recorder.has("access", "phone-reader", "pipelines", "read-only"))
self.assertFalse(any("opencode" in part for call in recorder.calls for part in call))
self.assertFalse(any("pipelines-tui" in part for call in recorder.calls for part in call))
self.assertFalse(any(TOKEN in part for call in recorder.calls for part in call))
def test_reinstall_keeps_existing_publisher_token(self):
with tempfile.TemporaryDirectory() as directory:
recorder = Recorder()
installer = self.installer(directory, recorder)
prefix = Path(directory) / "prefix"
with patch.object(setup_native.shutil, "which", side_effect=self.which_all()), \
patch.object(setup_native.lan, "detect_lan", return_value="192.168.1.20"), \
patch.object(setup_native.configure, "host_timezone", return_value="Europe/London"):
installer.run(setup_native.arguments([]))
before = (prefix / "secrets/publisher-token").read_text()
def refuse_resolution():
raise AssertionError("re-resolved ntfy despite a verified installed binary")
installer.resolver = refuse_resolution
recorder.calls.clear()
installer.run(setup_native.arguments([]))
self.assertEqual((prefix / "secrets/publisher-token").read_text(), before)
self.assertFalse(any("token" in call for call in recorder.calls), "reinstall re-created the token")
def test_ntfy_is_built_from_the_pinned_module(self):
with tempfile.TemporaryDirectory() as directory:
recorder = Recorder()
installer = self.installer(directory, recorder)
prefix = Path(directory) / "prefix"
installer.prepare_layout(prefix)
with patch.object(setup_native.shutil, "which", side_effect=self.which_all()):
path = installer.build_ntfy(prefix, resolution())
self.assertTrue(path.is_file())
self.assertTrue(path.stat().st_mode & 0o100)
self.assertTrue(recorder.has("go", "mod", "download", f"{NTFY_MODULE}@v{NTFY_VERSION}"))
build = next(call for call in recorder.calls if call[:2] == ["go", "build"] and call[-1] == ".")
self.assertIn("-trimpath", build)
self.assertIn(setup_native.NTFY_BUILD_TAGS, build)
self.assertTrue(any(f"-X main.version=v{NTFY_VERSION}" in part for part in build))
self.assertTrue(build[build.index("-o") + 1].endswith("/ntfy"))
kwargs = next(kw for call, kw in zip(recorder.calls, recorder.kwargs)
if call[:2] == ["go", "build"] and call[-1] == ".")
self.assertEqual(kwargs.get("env", {}).get("CGO_ENABLED"), "1")
def test_ntfy_build_is_reused_for_the_same_version(self):
with tempfile.TemporaryDirectory() as directory:
recorder = Recorder()
installer = self.installer(directory, recorder)
prefix = Path(directory) / "prefix"
installer.prepare_layout(prefix)
with patch.object(setup_native.shutil, "which", side_effect=self.which_all()):
installer.build_ntfy(prefix, resolution())
installer.write_manifest(prefix, {"ntfy": {"version": "v" + NTFY_VERSION}})
recorder.calls.clear()
installer.build_ntfy(prefix, resolution())
self.assertFalse(any(call[:2] == ["go", "build"] for call in recorder.calls),
"rebuilt ntfy for an already installed version")
def test_missing_c_compiler_fails_actionably(self):
installer = setup_native.Setup("/tmp", runner=Recorder(), resolver=resolution, environ={})
with patch.object(setup_native.shutil, "which", side_effect=self.which_all(cc=None)):
with self.assertRaisesRegex(ValueError, "No C compiler found"):
installer.require_compiler()
def test_ntfy_build_failure_is_reported(self):
with tempfile.TemporaryDirectory() as directory:
installer = self.installer(directory, Recorder(fail_build=True))
prefix = Path(directory) / "prefix"
installer.prepare_layout(prefix)
with patch.object(setup_native.shutil, "which", side_effect=self.which_all()):
with self.assertRaisesRegex(ValueError, "build failed"):
installer.build_ntfy(prefix, resolution())
def test_publisher_token_is_never_overwritten(self):
with tempfile.TemporaryDirectory() as directory:
prefix = Path(directory) / "prefix"
(prefix / "secrets").mkdir(parents=True)
path = prefix / "secrets/publisher-token"
setup_native.tokens.save_token(f"token {TOKEN} created", path, 0o600)
self.assertEqual(path.stat().st_mode & 0o777, 0o600)
with self.assertRaises(ValueError):
setup_native.tokens.save_token(f"token {TOKEN} created", path, 0o600)
self.assertEqual(path.read_text(), TOKEN + "\n")
def test_migration_copies_docker_data_only_when_absent(self):
with tempfile.TemporaryDirectory() as directory:
prefix = Path(directory) / "prefix"
(prefix / "data/ntfy").mkdir(parents=True)
recorder = Recorder(volumes=("pipelines_pipelines-data", "pipelines_ntfy-data"))
installer = self.installer(directory, recorder)
self.assertTrue(installer.migrate_docker(prefix, replace=False))
self.assertEqual((prefix / "data/pipelines.db").read_bytes(), b"migrated")
self.assertTrue(any(call[:2] == ["docker", "cp"] and call[2].endswith("-wal")
for call in installer.runner.calls),
"the SQLite write-ahead log sidecar must be copied with the database")
self.assertEqual((prefix / "data/ntfy/auth.db").read_bytes(), b"migrated")
def test_migration_keeps_existing_native_data_without_opt_in(self):
with tempfile.TemporaryDirectory() as directory:
prefix = Path(directory) / "prefix"
(prefix / "data").mkdir(parents=True)
(prefix / "data/pipelines.db").write_bytes(b"native")
recorder = Recorder(volumes=("pipelines_pipelines-data",))
installer = self.installer(directory, recorder)
self.assertTrue(installer.migrate_docker(prefix, replace=False))
self.assertEqual((prefix / "data/pipelines.db").read_bytes(), b"native")
self.assertFalse(any(call[1:2] == ["cp"] for call in recorder.calls))
def test_absent_docker_volumes_is_a_noop(self):
with tempfile.TemporaryDirectory() as directory:
recorder = Recorder(volumes=())
installer = self.installer(directory, recorder)
self.assertFalse(installer.migrate_docker(Path(directory) / "prefix", replace=False))
self.assertFalse(any(call[0] == "docker" and call[1] == "create" for call in recorder.calls))
def test_uninspectable_docker_volumes_abort_instead_of_skipping_migration(self):
with tempfile.TemporaryDirectory() as directory:
recorder = Recorder(fail_volume_ls=True)
installer = self.installer(directory, recorder)
with self.assertRaisesRegex(ValueError, "Could not inspect Docker volumes"):
installer.migrate_docker(Path(directory) / "prefix", replace=False)
def test_missing_docker_cli_is_a_noop_not_a_failure(self):
def no_docker(command, **kwargs):
raise FileNotFoundError(command[0])
with tempfile.TemporaryDirectory() as directory:
installer = setup_native.Setup(directory, prefix=Path(directory) / "prefix",
runner=no_docker, resolver=resolution, environ={})
self.assertEqual(installer.docker_volumes(), ())
def test_requested_migration_failure_aborts(self):
with tempfile.TemporaryDirectory() as directory:
prefix = Path(directory) / "prefix"
(prefix / "data").mkdir(parents=True)
recorder = Recorder(volumes=("pipelines_pipelines-data",), fail_copy=True)
installer = self.installer(directory, recorder)
with self.assertRaisesRegex(ValueError, "install aborted"):
installer.migrate_docker(prefix, replace=False)
def test_full_run_migrates_docker_auth_store_before_native_accounts(self):
with tempfile.TemporaryDirectory() as directory:
recorder = Recorder(volumes=("pipelines_ntfy-data",))
installer = self.installer(directory, recorder)
prefix = Path(directory) / "prefix"
with patch.object(setup_native.shutil, "which", side_effect=self.which_all()), \
patch.object(setup_native.lan, "detect_lan", return_value="192.168.1.20"), \
patch.object(setup_native.configure, "host_timezone", return_value="Europe/London"):
installer.run(setup_native.arguments([]))
# The migrated store must survive bootstrap instead of being shadowed.
self.assertEqual((prefix / "data/ntfy/auth.db").read_bytes(), b"migrated")
copy_index = next(i for i, call in enumerate(recorder.calls)
if call[0] == "docker" and call[1:2] == ["cp"])
add_index = next(i for i, call in enumerate(recorder.calls)
if "pipeline-publisher" in call and "user" in call)
self.assertLess(copy_index, add_index, "auth store migrated after native accounts were created")
def test_missing_local_image_aborts_migration(self):
with tempfile.TemporaryDirectory() as directory:
prefix = Path(directory) / "prefix"
(prefix / "data").mkdir(parents=True)
recorder = Recorder(volumes=("pipelines_pipelines-data",), image=None)
installer = self.installer(directory, recorder)
with self.assertRaisesRegex(ValueError, "no local Pipelines image"):
installer.migrate_docker(prefix, replace=False)
if __name__ == "__main__":
unittest.main()