a70e3aef4b
Add scripts/setup-native.py and make ./install (and --native) build the server, the CLI and ntfy from source into ~/.local/share/pipelines, detect an existing host OpenCode and Pi instead of installing them, install the packaged authoring skill, migrate an existing Docker database and ntfy auth store, save the host client default and start both services. ./install --docker keeps the previous Docker server + ntfy installation unchanged. ntfy is compiled from its pinned Go module on every host because upstream publishes no macOS server binary (its darwin asset is built with -tags noserver); its cgo SQLite driver is why a C compiler is now required natively. The ntfy credential bootstrap and the client default writer are shared by both installers, installation.Load resolves symlinked prefixes, the Docker migration copies the SQLite write-ahead log so the newest definitions and runs are not lost, and the account subcommands read the generated configuration through NTFY_CONFIG_FILE. Verified on macOS: 115 Python tests, go build/vet/test, and scratch installs that migrate 9 pipelines and 31 runs, build ntfy v2.30.0 and start both services healthy on spare ports before stopping them.
363 lines
19 KiB
Python
363 lines
19 KiB
Python
import importlib.util
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
spec = importlib.util.spec_from_file_location("setup_native", Path(__file__).with_name("setup-native.py"))
|
|
setup_native = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(setup_native)
|
|
|
|
manage_spec = importlib.util.spec_from_file_location("pipelines_manage", Path(__file__).with_name("manage.py"))
|
|
manage = importlib.util.module_from_spec(manage_spec)
|
|
manage_spec.loader.exec_module(manage)
|
|
|
|
TOKEN = "tk_" + "a" * 29
|
|
NTFY_VERSION = "2.11.0"
|
|
NTFY_MODULE = "heckel.io/ntfy/v2"
|
|
|
|
|
|
def resolution():
|
|
"""The shape resolve_ntfy returns: an exact tag to build the server from."""
|
|
return {"repository": "binwiederhier/ntfy", "module": NTFY_MODULE,
|
|
"version": NTFY_VERSION, "tag": "v" + NTFY_VERSION}
|
|
|
|
|
|
class Recorder:
|
|
def __init__(self, token=TOKEN, volumes=(), image="pipelines-pipelines:latest",
|
|
fail_create=False, fail_copy=False, fail_volume_ls=False, fail_build=False):
|
|
self.calls = []
|
|
self.token = token
|
|
self.volumes = volumes
|
|
self.image = image
|
|
self.fail_create = fail_create
|
|
self.fail_copy = fail_copy
|
|
self.fail_volume_ls = fail_volume_ls
|
|
self.fail_build = fail_build
|
|
self._module_cache = None
|
|
self.kwargs = []
|
|
|
|
def module_cache(self):
|
|
"""A stand-in Go module cache holding the pinned ntfy module source."""
|
|
if self._module_cache is None:
|
|
self._module_cache = tempfile.mkdtemp(prefix="gomodcache-")
|
|
source = Path(self._module_cache) / f"{NTFY_MODULE}@{resolution()['tag']}"
|
|
source.mkdir(parents=True)
|
|
(source / "go.mod").write_text(f"module {NTFY_MODULE}\n")
|
|
return self._module_cache
|
|
|
|
def __call__(self, command, **kwargs):
|
|
command = list(command)
|
|
self.calls.append(command)
|
|
self.kwargs.append(kwargs)
|
|
stdout, code = "", 0
|
|
if command[0] == "docker":
|
|
if command[1:3] == ["volume", "ls"]:
|
|
if self.fail_volume_ls:
|
|
code = 1
|
|
else:
|
|
stdout = "\n".join(self.volumes)
|
|
elif command[1:3] == ["image", "ls"]:
|
|
stdout = self.image + "\n" if self.image else ""
|
|
elif command[1] == "create" and self.fail_create:
|
|
code = 1
|
|
elif command[1] == "cp":
|
|
if self.fail_copy:
|
|
code = 1
|
|
else:
|
|
Path(command[3]).write_bytes(b"migrated")
|
|
elif command[0] == "go" and command[1:3] == ["env", "GOMODCACHE"]:
|
|
stdout = self.module_cache() + "\n"
|
|
elif command[0] == "go" and command[1:2] == ["build"]:
|
|
if self.fail_build:
|
|
code = 1
|
|
elif "-o" in command:
|
|
# Only the ntfy build is inspected by the installer; its output
|
|
# is created so the atomic install has a real file to move.
|
|
output = Path(command[command.index("-o") + 1])
|
|
if output.name == "ntfy":
|
|
output.write_bytes(b"#!/bin/sh\n")
|
|
output.chmod(0o755)
|
|
if command[0].endswith("ntfy") and "token" in command:
|
|
stdout = f"token {self.token} created for pipeline-publisher\n"
|
|
return subprocess.CompletedProcess(command, code, stdout=stdout, stderr="")
|
|
|
|
def has(self, *parts):
|
|
return any(all(part in call for part in parts) for call in self.calls)
|
|
|
|
|
|
class SetupNativeTests(unittest.TestCase):
|
|
def installer(self, directory, recorder):
|
|
# A disposable XDG config home keeps the client default out of the real
|
|
# user configuration during tests.
|
|
return setup_native.Setup(directory, prefix=Path(directory) / "prefix", runner=recorder,
|
|
resolver=resolution,
|
|
environ={"XDG_CONFIG_HOME": str(Path(directory) / "config")})
|
|
|
|
@staticmethod
|
|
def which_all(opencode="/usr/bin/opencode", pi=None, cc="/usr/bin/cc"):
|
|
mapping = {"go": "/usr/bin/go", "python3": "/usr/bin/python3",
|
|
"git": "/usr/bin/git", "curl": "/usr/bin/curl"}
|
|
if opencode:
|
|
mapping["opencode"] = opencode
|
|
if pi:
|
|
mapping["pi"] = pi
|
|
if cc:
|
|
mapping["cc"] = cc
|
|
return lambda name: mapping.get(name)
|
|
|
|
def test_require_tools_reports_each_missing_tool_without_installing(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
recorder = Recorder()
|
|
installer = self.installer(directory, recorder)
|
|
mapping = {"go": "/usr/bin/go", "python3": "/usr/bin/python3", "git": None, "curl": None}
|
|
with patch.object(setup_native.shutil, "which", side_effect=lambda name: mapping.get(name)):
|
|
with self.assertRaises(ValueError) as raised:
|
|
installer.require_tools()
|
|
message = str(raised.exception)
|
|
self.assertIn("git", message)
|
|
self.assertIn("curl", message)
|
|
self.assertIn("never installs tools", message)
|
|
self.assertEqual(recorder.calls, [])
|
|
|
|
def test_build_installs_only_server_and_cli_with_trimpath(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
recorder = Recorder()
|
|
installer = self.installer(directory, recorder)
|
|
prefix = Path(directory) / "prefix"
|
|
installer.prepare_layout(prefix)
|
|
installer.build(prefix)
|
|
builds = [call for call in recorder.calls if call[:2] == ["go", "build"]]
|
|
self.assertEqual(len(builds), 2)
|
|
for call in builds:
|
|
self.assertIn("-trimpath", call)
|
|
self.assertTrue(any(call[-1] == "./cmd/pipelines-server" for call in builds))
|
|
self.assertTrue(any(call[-1] == "./cmd/pipelines" for call in builds))
|
|
self.assertFalse(any("pipelines-tui" in part for call in recorder.calls for part in call))
|
|
|
|
def test_layout_creates_private_prefix_and_copies_manage(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
installer = self.installer(directory, Recorder())
|
|
prefix = Path(directory) / "prefix"
|
|
installer.prepare_layout(prefix)
|
|
for name in ("bin", "lib", "data", "logs", "secrets"):
|
|
self.assertTrue((prefix / name).is_dir(), name)
|
|
self.assertEqual((prefix / "secrets").stat().st_mode & 0o777, 0o700)
|
|
self.assertTrue((prefix / "lib/manage.py").is_file())
|
|
|
|
def test_prefix_honours_pipelines_prefix_environment(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
custom = Path(directory) / "custom"
|
|
installer = setup_native.Setup(directory, runner=Recorder(), environ={"PIPELINES_PREFIX": str(custom)})
|
|
self.assertEqual(installer.resolve_prefix(), custom.resolve())
|
|
|
|
def test_missing_opencode_fails_actionably(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
installer = self.installer(directory, Recorder())
|
|
with patch.object(setup_native.shutil, "which", side_effect=self.which_all(opencode=None)):
|
|
with self.assertRaisesRegex(ValueError, "OpenCode was not found"):
|
|
installer.harness()
|
|
|
|
def test_full_run_writes_manifest_tokens_and_config_without_harness_install(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
recorder = Recorder()
|
|
installer = self.installer(directory, recorder)
|
|
prefix = Path(directory) / "prefix"
|
|
with patch.object(setup_native.shutil, "which", side_effect=self.which_all()), \
|
|
patch.object(setup_native.lan, "detect_lan", return_value="192.168.1.20"), \
|
|
patch.object(setup_native.configure, "host_timezone", return_value="Europe/London"):
|
|
installer.run(setup_native.arguments([]))
|
|
config = manage.load(prefix)
|
|
self.assertEqual(config["format"], 1)
|
|
self.assertEqual(config["prefix"], str(prefix.resolve()))
|
|
self.assertEqual(config["server_port"], 8080)
|
|
self.assertEqual(config["ntfy"]["mode"], "native")
|
|
self.assertEqual(config["topic"], "pipelines")
|
|
self.assertEqual(config["opencode"], "/usr/bin/opencode")
|
|
self.assertEqual(config["pi"], "")
|
|
self.assertEqual((prefix / setup_native.MARKER).stat().st_mode & 0o777, 0o600)
|
|
token_path = prefix / "secrets/publisher-token"
|
|
self.assertEqual(token_path.read_text(), TOKEN + "\n")
|
|
self.assertEqual(token_path.stat().st_mode & 0o777, 0o600)
|
|
text = (prefix / "data/ntfy/server.yml").read_text()
|
|
for needle in ('listen-http:', 'base-url: "http://192.168.1.20:8090"', 'cache-file:', 'auth-file:',
|
|
'auth-default-access: "deny-all"', 'enable-login: true', 'attachment-cache-dir:',
|
|
'attachment-expiry-duration:', 'attachment-total-size-limit:',
|
|
'attachment-file-size-limit:'):
|
|
self.assertIn(needle, text)
|
|
self.assertNotIn("topic", text)
|
|
self.assertTrue(recorder.has(str(prefix.resolve() / "bin/pipelines"), "skill", "install"))
|
|
self.assertTrue(recorder.has("access", "pipeline-publisher", "pipelines", "write-only"))
|
|
self.assertTrue(recorder.has("access", "phone-reader", "pipelines", "read-only"))
|
|
self.assertFalse(any("opencode" in part for call in recorder.calls for part in call))
|
|
self.assertFalse(any("pipelines-tui" in part for call in recorder.calls for part in call))
|
|
self.assertFalse(any(TOKEN in part for call in recorder.calls for part in call))
|
|
|
|
def test_reinstall_keeps_existing_publisher_token(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
recorder = Recorder()
|
|
installer = self.installer(directory, recorder)
|
|
prefix = Path(directory) / "prefix"
|
|
with patch.object(setup_native.shutil, "which", side_effect=self.which_all()), \
|
|
patch.object(setup_native.lan, "detect_lan", return_value="192.168.1.20"), \
|
|
patch.object(setup_native.configure, "host_timezone", return_value="Europe/London"):
|
|
installer.run(setup_native.arguments([]))
|
|
before = (prefix / "secrets/publisher-token").read_text()
|
|
|
|
def refuse_resolution():
|
|
raise AssertionError("re-resolved ntfy despite a verified installed binary")
|
|
|
|
installer.resolver = refuse_resolution
|
|
recorder.calls.clear()
|
|
installer.run(setup_native.arguments([]))
|
|
self.assertEqual((prefix / "secrets/publisher-token").read_text(), before)
|
|
self.assertFalse(any("token" in call for call in recorder.calls), "reinstall re-created the token")
|
|
|
|
def test_ntfy_is_built_from_the_pinned_module(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
recorder = Recorder()
|
|
installer = self.installer(directory, recorder)
|
|
prefix = Path(directory) / "prefix"
|
|
installer.prepare_layout(prefix)
|
|
with patch.object(setup_native.shutil, "which", side_effect=self.which_all()):
|
|
path = installer.build_ntfy(prefix, resolution())
|
|
self.assertTrue(path.is_file())
|
|
self.assertTrue(path.stat().st_mode & 0o100)
|
|
self.assertTrue(recorder.has("go", "mod", "download", f"{NTFY_MODULE}@v{NTFY_VERSION}"))
|
|
build = next(call for call in recorder.calls if call[:2] == ["go", "build"] and call[-1] == ".")
|
|
self.assertIn("-trimpath", build)
|
|
self.assertIn(setup_native.NTFY_BUILD_TAGS, build)
|
|
self.assertTrue(any(f"-X main.version=v{NTFY_VERSION}" in part for part in build))
|
|
self.assertTrue(build[build.index("-o") + 1].endswith("/ntfy"))
|
|
kwargs = next(kw for call, kw in zip(recorder.calls, recorder.kwargs)
|
|
if call[:2] == ["go", "build"] and call[-1] == ".")
|
|
self.assertEqual(kwargs.get("env", {}).get("CGO_ENABLED"), "1")
|
|
|
|
def test_ntfy_build_is_reused_for_the_same_version(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
recorder = Recorder()
|
|
installer = self.installer(directory, recorder)
|
|
prefix = Path(directory) / "prefix"
|
|
installer.prepare_layout(prefix)
|
|
with patch.object(setup_native.shutil, "which", side_effect=self.which_all()):
|
|
installer.build_ntfy(prefix, resolution())
|
|
installer.write_manifest(prefix, {"ntfy": {"version": "v" + NTFY_VERSION}})
|
|
recorder.calls.clear()
|
|
installer.build_ntfy(prefix, resolution())
|
|
self.assertFalse(any(call[:2] == ["go", "build"] for call in recorder.calls),
|
|
"rebuilt ntfy for an already installed version")
|
|
|
|
def test_missing_c_compiler_fails_actionably(self):
|
|
installer = setup_native.Setup("/tmp", runner=Recorder(), resolver=resolution, environ={})
|
|
with patch.object(setup_native.shutil, "which", side_effect=self.which_all(cc=None)):
|
|
with self.assertRaisesRegex(ValueError, "No C compiler found"):
|
|
installer.require_compiler()
|
|
|
|
def test_ntfy_build_failure_is_reported(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
installer = self.installer(directory, Recorder(fail_build=True))
|
|
prefix = Path(directory) / "prefix"
|
|
installer.prepare_layout(prefix)
|
|
with patch.object(setup_native.shutil, "which", side_effect=self.which_all()):
|
|
with self.assertRaisesRegex(ValueError, "build failed"):
|
|
installer.build_ntfy(prefix, resolution())
|
|
|
|
def test_publisher_token_is_never_overwritten(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
prefix = Path(directory) / "prefix"
|
|
(prefix / "secrets").mkdir(parents=True)
|
|
path = prefix / "secrets/publisher-token"
|
|
setup_native.tokens.save_token(f"token {TOKEN} created", path, 0o600)
|
|
self.assertEqual(path.stat().st_mode & 0o777, 0o600)
|
|
with self.assertRaises(ValueError):
|
|
setup_native.tokens.save_token(f"token {TOKEN} created", path, 0o600)
|
|
self.assertEqual(path.read_text(), TOKEN + "\n")
|
|
|
|
def test_migration_copies_docker_data_only_when_absent(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
prefix = Path(directory) / "prefix"
|
|
(prefix / "data/ntfy").mkdir(parents=True)
|
|
recorder = Recorder(volumes=("pipelines_pipelines-data", "pipelines_ntfy-data"))
|
|
installer = self.installer(directory, recorder)
|
|
self.assertTrue(installer.migrate_docker(prefix, replace=False))
|
|
self.assertEqual((prefix / "data/pipelines.db").read_bytes(), b"migrated")
|
|
self.assertTrue(any(call[:2] == ["docker", "cp"] and call[2].endswith("-wal")
|
|
for call in installer.runner.calls),
|
|
"the SQLite write-ahead log sidecar must be copied with the database")
|
|
self.assertEqual((prefix / "data/ntfy/auth.db").read_bytes(), b"migrated")
|
|
|
|
def test_migration_keeps_existing_native_data_without_opt_in(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
prefix = Path(directory) / "prefix"
|
|
(prefix / "data").mkdir(parents=True)
|
|
(prefix / "data/pipelines.db").write_bytes(b"native")
|
|
recorder = Recorder(volumes=("pipelines_pipelines-data",))
|
|
installer = self.installer(directory, recorder)
|
|
self.assertTrue(installer.migrate_docker(prefix, replace=False))
|
|
self.assertEqual((prefix / "data/pipelines.db").read_bytes(), b"native")
|
|
self.assertFalse(any(call[1:2] == ["cp"] for call in recorder.calls))
|
|
|
|
def test_absent_docker_volumes_is_a_noop(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
recorder = Recorder(volumes=())
|
|
installer = self.installer(directory, recorder)
|
|
self.assertFalse(installer.migrate_docker(Path(directory) / "prefix", replace=False))
|
|
self.assertFalse(any(call[0] == "docker" and call[1] == "create" for call in recorder.calls))
|
|
|
|
def test_uninspectable_docker_volumes_abort_instead_of_skipping_migration(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
recorder = Recorder(fail_volume_ls=True)
|
|
installer = self.installer(directory, recorder)
|
|
with self.assertRaisesRegex(ValueError, "Could not inspect Docker volumes"):
|
|
installer.migrate_docker(Path(directory) / "prefix", replace=False)
|
|
|
|
def test_missing_docker_cli_is_a_noop_not_a_failure(self):
|
|
def no_docker(command, **kwargs):
|
|
raise FileNotFoundError(command[0])
|
|
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
installer = setup_native.Setup(directory, prefix=Path(directory) / "prefix",
|
|
runner=no_docker, resolver=resolution, environ={})
|
|
self.assertEqual(installer.docker_volumes(), ())
|
|
|
|
def test_requested_migration_failure_aborts(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
prefix = Path(directory) / "prefix"
|
|
(prefix / "data").mkdir(parents=True)
|
|
recorder = Recorder(volumes=("pipelines_pipelines-data",), fail_copy=True)
|
|
installer = self.installer(directory, recorder)
|
|
with self.assertRaisesRegex(ValueError, "install aborted"):
|
|
installer.migrate_docker(prefix, replace=False)
|
|
|
|
def test_full_run_migrates_docker_auth_store_before_native_accounts(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
recorder = Recorder(volumes=("pipelines_ntfy-data",))
|
|
installer = self.installer(directory, recorder)
|
|
prefix = Path(directory) / "prefix"
|
|
with patch.object(setup_native.shutil, "which", side_effect=self.which_all()), \
|
|
patch.object(setup_native.lan, "detect_lan", return_value="192.168.1.20"), \
|
|
patch.object(setup_native.configure, "host_timezone", return_value="Europe/London"):
|
|
installer.run(setup_native.arguments([]))
|
|
# The migrated store must survive bootstrap instead of being shadowed.
|
|
self.assertEqual((prefix / "data/ntfy/auth.db").read_bytes(), b"migrated")
|
|
copy_index = next(i for i, call in enumerate(recorder.calls)
|
|
if call[0] == "docker" and call[1:2] == ["cp"])
|
|
add_index = next(i for i, call in enumerate(recorder.calls)
|
|
if "pipeline-publisher" in call and "user" in call)
|
|
self.assertLess(copy_index, add_index, "auth store migrated after native accounts were created")
|
|
|
|
def test_missing_local_image_aborts_migration(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
prefix = Path(directory) / "prefix"
|
|
(prefix / "data").mkdir(parents=True)
|
|
recorder = Recorder(volumes=("pipelines_pipelines-data",), image=None)
|
|
installer = self.installer(directory, recorder)
|
|
with self.assertRaisesRegex(ValueError, "no local Pipelines image"):
|
|
installer.migrate_docker(prefix, replace=False)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|