Simplify Go installation and support standard OpenCode releases

This commit is contained in:
2026-10-08 23:17:38 +01:00
parent 0f30a001c2
commit accbe63e39
101 changed files with 1331 additions and 682 deletions
+4 -300
View File
@@ -1,306 +1,10 @@
#!/usr/bin/env python3
"""Install a clean native Pipelines service and authenticated Docker ntfy."""
import argparse
import getpass
import hashlib
import importlib.util
import json
import os
"""Compatibility entry point: install Go binaries only, with standard Go paths."""
from pathlib import Path
import platform
import plistlib
import re
import secrets
import shutil
import socket
import subprocess
import sys
from manage import DEFAULT_PREFIX, MARKER, compose, load, run, start, wait_health
SOURCE = Path(__file__).resolve().parents[1]
NAME = re.compile(r"^[A-Za-z][A-Za-z0-9_-]{0,63}$")
TOPIC = re.compile(r"^[-_A-Za-z0-9]{1,64}$")
def choose_port(preferred):
for port in range(preferred, min(preferred + 100, 65536)):
with socket.socket() as sock:
try:
sock.bind(("0.0.0.0", port))
return port
except OSError:
pass
raise ValueError("No available local service port")
def default_host():
tailscale = shutil.which("tailscale")
if tailscale:
try:
data = json.loads(run([tailscale, "status", "--json"], quiet=True, timeout=10).stdout)
if data.get("BackendState") == "Running":
return next(ip for ip in data["TailscaleIPs"] if ":" not in ip)
except (RuntimeError, ValueError, KeyError, StopIteration, subprocess.SubprocessError):
pass
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
try:
sock.connect(("192.0.2.1", 80))
return sock.getsockname()[0]
except OSError:
return "127.0.0.1"
def timezone():
spec = importlib.util.spec_from_file_location("configure", SOURCE / "scripts/configure-compose.py")
module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module)
return module.host_timezone()
def write_private(path, text):
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
with path.open("x") as file:
file.write(text)
path.chmod(0o600)
def ntfy_compose(config):
# JSON scalars are valid YAML scalars and prevent interpolation/injection.
q = lambda value: json.dumps(str(value))
return f'''services:
ntfy:
image: binwiederhier/ntfy:v2.29.0
command: serve
restart: unless-stopped
environment:
TZ: {q(config['timezone'])}
NTFY_BASE_URL: {q(config['public_url'])}
NTFY_LISTEN_HTTP: ":80"
NTFY_CACHE_FILE: /var/lib/ntfy/cache.db
NTFY_CACHE_DURATION: 168h
NTFY_AUTH_FILE: /var/lib/ntfy/auth.db
NTFY_AUTH_DEFAULT_ACCESS: deny-all
NTFY_ENABLE_LOGIN: "true"
NTFY_ATTACHMENT_CACHE_DIR: /var/lib/ntfy/attachments
NTFY_ATTACHMENT_EXPIRY_DURATION: 168h
NTFY_ATTACHMENT_TOTAL_SIZE_LIMIT: 1G
NTFY_ATTACHMENT_FILE_SIZE_LIMIT: 1M
ports:
- "0.0.0.0:{config['ntfy_port']}:80"
volumes:
- ntfy-data:/var/lib/ntfy
volumes:
ntfy-data:
'''
def make_registration(config):
root = Path(config["prefix"])
command = [config["python"], str(root / "lib/manage.py"), "supervise", "--prefix", str(root)]
if platform.system() == "Darwin":
path = Path.home() / "Library/LaunchAgents" / (config["label"] + ".plist")
path.parent.mkdir(parents=True, exist_ok=True)
if path.exists():
raise ValueError("Service registration already exists; refusing to overwrite")
with path.open("wb") as file:
plistlib.dump({"Label": config["label"], "ProgramArguments": command,
"RunAtLoad": True, "KeepAlive": {"SuccessfulExit": False},
"ExitTimeOut": 75, "ThrottleInterval": 10,
"StandardOutPath": str(root / "logs/supervisor.log"),
"StandardErrorPath": str(root / "logs/supervisor.log")}, file)
else:
path = Path.home() / ".config/systemd/user" / (config["label"] + ".service")
path.parent.mkdir(parents=True, exist_ok=True)
if path.exists():
raise ValueError("Service registration already exists; refusing to overwrite")
# systemd quotes its own command arguments, not shell strings.
quote = lambda value: '"' + value.replace("\\", "\\\\").replace('"', '\\"').replace("%", "%%") + '"'
path.write_text("[Unit]\nDescription=Pipelines server and ntfy\nAfter=default.target\n\n"
"[Service]\nType=simple\nExecStart=" + " ".join(map(quote, command)) +
"\nRestart=on-failure\nRestartSec=10\nTimeoutStopSec=90\n\n[Install]\nWantedBy=default.target\n")
return str(path)
def install(args):
if platform.system() not in ("Darwin", "Linux"):
raise ValueError("Supported: macOS and Linux (run as your normal user, not root)")
if os.getuid() == 0:
raise ValueError("Do not use sudo: this install reuses your user's OpenCode account")
root = args.prefix.expanduser().resolve()
if root in (Path("/"), Path.home().resolve(), SOURCE):
raise ValueError("Choose a dedicated installation directory")
docker = shutil.which("docker")
if not docker:
raise ValueError("Install/start Docker Desktop or Docker Engine first")
run([docker, "info", "--format", "{{.ServerVersion}}"], quiet=True, timeout=15)
run([docker, "compose", "version"], quiet=True, timeout=15)
if (root / MARKER).exists() and not load(root).get("uninstalled"):
config = load(root)
if not config.get("bootstrapped"):
raise ValueError("A partial installation exists; inspect it or run uninstall.sh --prefix ... --purge before retrying")
start(config)
print(f"Existing installation started; settings/history preserved: {root}")
return
retained = load(root) if (root / MARKER).exists() else None
if root.exists() and any(root.iterdir()) and retained is None:
raise ValueError("Installation directory is not empty; refusing to overwrite unrelated files")
if not args.no_autostart and platform.system() == "Linux":
run(["systemctl", "--user", "show-environment"], quiet=True)
opencode = args.opencode or shutil.which("opencode") or shutil.which("opencode2")
if not opencode:
npm=shutil.which("npm")
if not npm:
raise ValueError("Install OpenCode V2 first (npm install -g @opencode/cli), or install npm for automatic CLI setup")
if not args.yes and input("OpenCode V2 is missing. Install its pinned CLI privately using npm? [Y/n] ").strip().lower()=="n":
raise ValueError("OpenCode installation canceled")
root.mkdir(parents=True,exist_ok=True,mode=0o700)
run([npm,"install","--prefix",str(root / "tools"),"@opencode/cli@2.0.20"],quiet=True,timeout=240)
opencode=str(root / "tools/node_modules/.bin/opencode")
opencode = str(Path(opencode).resolve())
version = run([opencode, "--version"], quiet=True, timeout=15).stdout
if not re.search(r"(?:^|\s)v?2\.\d+", version):
raise ValueError("OpenCode V2 is required; your existing login/config will be reused, never exported")
auth = run([opencode, "auth", "list"], quiet=True, timeout=30).stdout
if "OpenCode Go" not in auth:
if args.yes:
raise ValueError("Connect OpenCode Go first: opencode auth login")
print("OpenCode Go is not connected. Complete your existing OpenCode login now.")
subprocess.run([opencode, "auth", "login"], check=True)
auth = run([opencode, "auth", "list"], quiet=True, timeout=30).stdout
if "OpenCode Go" not in auth:
raise ValueError("OpenCode Go login not confirmed; installation canceled")
# This exercises discovery/start as the same user, without generating model output.
run([opencode, "api", "get", "/api/info"], quiet=True, timeout=30)
root.mkdir(parents=True, exist_ok=True, mode=0o700)
root.chmod(0o700)
for directory in ("bin", "lib", "data", "logs", "secrets", "workspace"):
(root / directory).mkdir(exist_ok=True, mode=0o700)
packaged = SOURCE / "release-bin"
if packaged.is_dir():
metadata = json.loads((packaged / "platform.json").read_text())
if metadata != {"os": platform.system(), "arch": platform.machine()}:
raise ValueError("Release archive is for a different OS/CPU")
for name in ("pipelines-server", "pipelines-tui", "pipelines"):
shutil.copy2(packaged / name, root / "bin" / name)
else:
go = shutil.which("go")
if not go:
raise ValueError("Source install needs Go 1.27+; alternatively use a platform release archive")
run([go, "install", "-trimpath", "./cmd/server", "./cmd/tui", "./cmd/pipelines"],
env=dict(os.environ, GOBIN=str(root / "bin")), quiet=True, timeout=240)
for command in ("server", "tui"):
(root / "bin" / command).replace(root / "bin" / ("pipelines-" + command))
username = retained["username"] if retained else (args.username or ("phone-reader" if args.yes else input("Phone username [phone-reader]: ").strip() or "phone-reader"))
topic = retained["topic"] if retained else (args.topic or ("pipelines" if args.yes else input("Notification topic [pipelines]: ").strip() or "pipelines"))
if not NAME.fullmatch(username) or username == "pipeline-publisher" or not TOPIC.fullmatch(topic):
raise ValueError("Use safe alphanumeric usernames/topics; pipeline-publisher is reserved")
server_port = retained["server_port"] if retained else choose_port(args.server_port)
ntfy_port = retained["ntfy_port"] if retained else choose_port(args.ntfy_port)
if server_port == ntfy_port:
ntfy_port = choose_port(ntfy_port + 1)
public_url = retained["public_url"] if retained else args.public_url or f"http://{default_host()}:{ntfy_port}"
from urllib.parse import urlparse
url = urlparse(public_url)
if url.scheme not in ("http", "https") or not url.hostname or url.username or url.password or url.path not in ("", "/") or url.query or url.fragment or any(c in public_url for c in "$\n\r\x00"):
raise ValueError("Phone URL must be an HTTP(S) base URL without credentials, topic, query, or interpolation")
chosen_workspace=args.workspace or (Path(retained["workspace"]) if retained else root / "workspace")
workspace = str(chosen_workspace.expanduser().resolve())
if not Path(workspace).is_dir():
raise ValueError("Workspace directory must exist")
suffix = hashlib.sha256(str(root).encode()).hexdigest()[:12]
bindir = Path("/usr/local/bin") if args.system_command else args.bin_dir.expanduser().resolve()
launcher = bindir / "pipelines"
if launcher.exists() or launcher.is_symlink():
if not launcher.is_symlink() or launcher.resolve() != root / "bin/pipelines":
raise ValueError("A pipelines command already exists in the target bin directory; choose --bin-dir")
# Only explicit server-selection variables are copied; never API keys/tokens.
opencode_env = {key: value for key, value in os.environ.items() if key in ("XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_STATE_HOME", "OPENCODE_SERVER", "OPENCODE_DB")}
config = dict(format=1, prefix=str(root), docker=docker, python=sys.executable,
opencode=opencode, opencode_env=opencode_env, path=os.pathsep.join([str(root / "bin"), str(Path(opencode).parent), os.environ.get("PATH", "")]),
project="pipelines-" + suffix, label="io.pipelines." + suffix,
server_port=server_port, ntfy_port=ntfy_port, public_url=public_url.rstrip("/"),
topic=topic, username=username, timezone=timezone(), workspace=workspace,
workspace_roots=[str(path.expanduser().resolve()) for path in (args.workspace_root or ([Path(p) for p in retained["workspace_roots"]] if retained else [Path.home()]))],
launcher=str(launcher), registration=None, bootstrapped=bool(retained and retained.get("bootstrapped")))
manifest = root / MARKER
manifest.write_text(json.dumps(config, indent=2)); manifest.chmod(0o600)
(root / "compose.yaml").write_text(ntfy_compose(config))
shutil.copy2(SOURCE / "scripts/manage.py", root / "lib/manage.py")
alias = root / "bin/opencode"
if not alias.exists():
alias.symlink_to(opencode)
# The PATH command is the compiled Go binary, not a shell/Python launcher.
if not config["bootstrapped"]:
if args.password_file:
password = args.password_file.read_text().strip()
elif args.yes:
password = secrets.token_urlsafe(18)
else:
password = getpass.getpass("Phone password (blank = generate): ") or secrets.token_urlsafe(18)
if len(password) < 8 or any(c in password for c in "\r\n\x00"):
raise ValueError("Phone password must have at least 8 characters and no line breaks")
compose(config, "up", "-d", "ntfy", quiet=True)
wait_health(f"http://127.0.0.1:{ntfy_port}/v1/health")
for user, secret in (("pipeline-publisher", secrets.token_urlsafe(32)), (username, password)):
compose(config, "exec", "-T", "-e", "NTFY_PASSWORD", "ntfy", "ntfy", "user", "add", user,
env=dict(os.environ, NTFY_PASSWORD=secret), quiet=True)
compose(config, "exec", "-T", "ntfy", "ntfy", "access", "pipeline-publisher", topic, "write-only", quiet=True)
compose(config, "exec", "-T", "ntfy", "ntfy", "access", username, topic, "read-only", quiet=True)
output = compose(config, "exec", "-T", "ntfy", "ntfy", "token", "add", "pipeline-publisher", quiet=True).stdout
tokens = re.findall(r"\btk_[A-Za-z0-9]{29}\b", output)
if len(tokens) != 1:
raise RuntimeError("Could not extract the publishing token; credentials were not printed")
write_private(root / "secrets/publisher-token", tokens[0] + "\n")
write_private(root / "secrets/phone-login.txt", f"Server: {public_url}\nTopic: {topic}\nUsername: {username}\nPassword: {password}\n")
config["bootstrapped"] = True
if not args.no_autostart:
config["registration"] = make_registration(config)
manifest.write_text(json.dumps(config, indent=2))
if args.system_command and not bindir.exists():
subprocess.run(["sudo", "mkdir", "-p", str(bindir)], check=True)
else:
bindir.mkdir(parents=True, exist_ok=True)
if not launcher.is_symlink():
if args.system_command and not os.access(bindir, os.W_OK):
# Privilege is used ONLY for this command symlink, never for the server/auth.
subprocess.run(["sudo", "ln", "-s", str(root / "bin/pipelines"), str(launcher)], check=True)
else:
launcher.symlink_to(root / "bin/pipelines")
start(config)
print(f"Installed. Run: {launcher}\nManage both services: pipelines start|stop|restart|status|doctor|logs")
print(f"Phone server: {config['public_url']}\nTopic: {topic}\nUsername: {username}")
print(f"Password is private: {root / 'secrets/phone-login.txt'}")
print("Fresh installations have no pipelines. In the TUI press b for rendered examples/templates.")
print("ntfy is exposed on LAN/VPN with authentication. Use HTTPS or an encrypted VPN outside a trusted LAN.")
if str(bindir) not in os.environ.get("PATH", "").split(os.pathsep):
print(f"Add {bindir} to PATH, or use the full command path above.")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--prefix", type=Path, default=DEFAULT_PREFIX)
parser.add_argument("--bin-dir", type=Path, default=Path.home() / ".local/bin")
parser.add_argument("--system-command",action="store_true",help="Put the command in /usr/local/bin (sudo only for the symlink; services remain per-user)")
parser.add_argument("--username")
parser.add_argument("--topic")
parser.add_argument("--password-file", type=Path)
parser.add_argument("--public-url")
parser.add_argument("--workspace", type=Path)
parser.add_argument("--workspace-root",type=Path,action="append",help="Allowed repository root; repeatable (default: your home)")
parser.add_argument("--opencode", help="Path to the host OpenCode V2 CLI")
parser.add_argument("--server-port", type=int, default=8080)
parser.add_argument("--ntfy-port", type=int, default=8090)
parser.add_argument("--no-autostart", action="store_true", help="Use a managed background process rather than launchd/systemd")
parser.add_argument("--yes", action="store_true", help="Accept defaults and generate a private phone password")
args = parser.parse_args()
if not 1024 <= args.server_port <= 65535 or not 1024 <= args.ntfy_port <= 65535:
parser.error("Use unprivileged ports 1024–65535")
os.chdir(SOURCE)
install(args)
if __name__ == "__main__":
try:
main()
except (ValueError, RuntimeError, OSError, subprocess.SubprocessError) as error:
sys.exit(str(error))
if len(sys.argv) != 1:
sys.exit("Install binaries with go install ./cmd/...; set GOBIN to choose the destination. Configure and start services separately.")
sys.exit(subprocess.call(["go", "install", "-trimpath", "./cmd/..."], cwd=Path(__file__).resolve().parents[1]))
+1 -1
View File
@@ -36,7 +36,7 @@ def package(output, version, target):
if system not in ("darwin", "linux") or arch not in ("arm64", "amd64"):
raise ValueError("Supported: source, darwin-arm64/amd64, linux-arm64/amd64")
env = dict(os.environ, GOOS=system, GOARCH=arch, CGO_ENABLED="0")
for command, binary in (("server", "pipelines-server"), ("tui", "pipelines-tui"), ("pipelines", "pipelines")):
for command, binary in (("pipelines-server", "pipelines-server"), ("pipelines-tui", "pipelines-tui"), ("pipelines", "pipelines")):
subprocess.run(["go", "build", "-trimpath", "-o", str(binaries / binary), "./cmd/" + command],
cwd=ROOT, env=env, check=True)
(binaries / "platform.json").write_text(json.dumps({"os": "Darwin" if system == "darwin" else "Linux", "arch": "arm64" if arch == "arm64" and system == "darwin" else "aarch64" if arch == "arm64" else "x86_64"}))
+1 -1
View File
@@ -10,4 +10,4 @@ export PIPELINES_NTFY_URL="${PIPELINES_NTFY_URL:-http://127.0.0.1:8090/pipelines
if [ -z "${PIPELINES_NTFY_TOKEN:-}" ]; then
export PIPELINES_NTFY_TOKEN_FILE="${PIPELINES_NTFY_TOKEN_FILE:-$(pwd)/secrets/ntfy-token.txt}"
fi
exec go run ./cmd/server "$@"
exec go run ./cmd/pipelines-server "$@"
+22 -76
View File
@@ -1,90 +1,36 @@
import importlib.util
import json
import os
from pathlib import Path
import socket
import subprocess
import tempfile
import unittest
from unittest import mock
import install
import manage
class InstallationTests(unittest.TestCase):
def test_launcher_forwards_application_arguments_unchanged(self):
config = dict(prefix="/owned", server_port=12345, opencode="/host/opencode", path="/owned/bin:/usr/bin")
for command in (["edit", "a name", "change X", "--json"], ["--url", "ws://other/connect", "list", "--json"], ["--help"], ["run", "workflow", "task", "--yes", "--json"]):
with mock.patch.object(manage,"load",return_value=config), mock.patch.object(manage.sys,"argv",["manage.py","--prefix","/owned",*command]), mock.patch.object(manage.os,"execve") as execute:
manage.main()
binary,args,env=execute.call_args.args
self.assertEqual(args,["/owned/bin/pipelines",*command])
self.assertEqual(env["PIPELINES_URL"],"ws://127.0.0.1:12345/connect")
def test_compose_has_only_ntfy_and_no_credentials(self):
config = dict(timezone="Europe/London", public_url="http://100.1.2.3:8090", ntfy_port=8090)
text = install.ntfy_compose(config)
self.assertIn("binwiederhier/ntfy:v2.29.0", text)
self.assertIn("deny-all", text)
self.assertNotIn("opencode", text)
self.assertNotIn("password", text.lower())
self.assertNotIn("pipelines:", text)
def test_detect_free_port_avoids_listener(self):
with socket.socket() as listener:
listener.bind(("0.0.0.0", 0)); listener.listen()
occupied = listener.getsockname()[1]
self.assertNotEqual(install.choose_port(occupied), occupied)
def test_secret_files_private_and_not_overwritten(self):
def test_standard_installer_needs_only_go_and_honors_gobin(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory, "secrets/token")
install.write_private(path, "private")
self.assertEqual(path.stat().st_mode & 0o777, 0o600)
with self.assertRaises(FileExistsError):
install.write_private(path, "replacement")
self.assertEqual(path.read_text(), "private")
root=Path(directory)
capture=root/"args"
executable=root/"go"
executable.write_text('#!/bin/sh\nprintf "%s\\n" "$GOBIN" "$@" > "'+str(capture)+'"\n')
executable.chmod(0o700)
env=dict(os.environ,PATH=str(root)+os.pathsep+os.environ["PATH"],GOBIN=str(root/"bin"))
subprocess.run([str(Path(__file__).resolve().parents[1]/"install.sh")],env=env,check=True)
self.assertEqual(capture.read_text().splitlines(),[str(root/"bin"),"install","-trimpath","./cmd/..."])
self.assertFalse((root/"bin").exists())
def test_management_requires_owned_installation_marker(self):
with tempfile.TemporaryDirectory() as directory:
with self.assertRaises(ValueError):
manage.load(directory)
root = Path(directory).resolve()
(root / manage.MARKER).write_text(json.dumps(dict(format=1, prefix=str(root))))
self.assertEqual(manage.load(root)["prefix"], str(root))
(root / manage.MARKER).write_text(json.dumps(dict(format=1, prefix="/")))
with self.assertRaises(ValueError):
manage.load(root)
def test_standard_installer_rejects_legacy_provisioning_flags(self):
result=subprocess.run([str(Path(__file__).resolve().parents[1]/"install.sh"),"--system-command"],capture_output=True,text=True)
self.assertEqual(result.returncode,2)
self.assertIn("GOBIN",result.stderr)
def test_pid_reuse_cannot_signal_unrelated_process(self):
def test_legacy_management_requires_owned_marker(self):
with tempfile.TemporaryDirectory() as directory:
Path(directory, "supervisor.pid").write_text(str(os.getpid()))
with self.assertRaises(ValueError):manage.load(directory)
def test_legacy_pid_reuse_cannot_signal_unrelated_process(self):
with tempfile.TemporaryDirectory() as directory:
Path(directory,"supervisor.pid").write_text(str(os.getpid()))
self.assertIsNone(manage.managed_pid(dict(prefix=directory)))
def test_default_uninstall_preserves_data_and_open_code(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory, "app"); root.mkdir()
for name in ("bin", "lib", "data", "secrets"):
(root / name).mkdir()
(root / "data/pipelines.db").write_text("history")
(root / "secrets/publisher-token").write_text("credential")
config = dict(format=1, prefix=str(root), launcher=str(Path(directory, "pipelines")), registration=None)
Path(config["launcher"]).symlink_to(root / "bin/pipelines")
with mock.patch.object(manage, "stop"), mock.patch.object(manage, "compose") as compose:
manage.uninstall(config, yes=True)
compose.assert_called_once_with(config, "down", quiet=True)
self.assertEqual((root / "data/pipelines.db").read_text(), "history")
self.assertEqual((root / "secrets/publisher-token").read_text(), "credential")
self.assertFalse((root / "bin").exists())
self.assertFalse(Path(config["launcher"]).is_symlink())
def test_service_registration_quotes_paths(self):
with tempfile.TemporaryDirectory(prefix="path with spaces ") as directory:
root = Path(directory);(root / "logs").mkdir()
config = dict(prefix=str(root), python="/usr/bin/python3", label="io.pipelines.test")
with mock.patch.object(install.platform,"system",return_value="Linux"),mock.patch.object(install.Path,"home",return_value=root):
path = Path(install.make_registration(config))
self.assertIn('"'+str(root / 'lib/manage.py')+'"', path.read_text())
self.assertIn("Restart=on-failure", path.read_text())
if __name__ == "__main__":
unittest.main()
if __name__=="__main__":unittest.main()