Initial pipelines server, CLI, TUI, and Go installation

This commit is contained in:
2026-10-08 22:22:17 +01:00
commit 0f30a001c2
143 changed files with 17557 additions and 0 deletions
+72
View File
@@ -0,0 +1,72 @@
#!/usr/bin/env python3
"""Create local Compose settings without overwriting settings or credentials."""
import argparse
import os
from pathlib import Path
from urllib.parse import urlparse
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
def host_timezone():
candidates = [os.environ.get("TZ", "")]
link = str(Path("/etc/localtime").resolve())
if "/zoneinfo/" in link:
candidates.append(link.split("/zoneinfo/", 1)[1])
path = Path("/etc/timezone")
if path.exists():
candidates.append(path.read_text().strip())
for candidate in candidates:
if not candidate:
continue
try:
ZoneInfo(candidate)
return candidate
except (ZoneInfoNotFoundError, ValueError):
continue
raise ValueError("Cannot detect the host timezone; provide --timezone explicitly")
def configure(directory, ntfy_url, bind_address, timezone=None, port=8081):
directory = Path(directory)
url = urlparse(ntfy_url)
if (url.scheme not in ("http", "https") or not url.hostname or url.username
or url.password or url.query or url.fragment or url.path not in ("", "/")):
raise ValueError("--ntfy-url must be an HTTP(S) base URL without credentials or a topic")
timezone = timezone or host_timezone()
ZoneInfo(timezone)
for value in (ntfy_url, bind_address, timezone):
if any(char in value for char in "\r\n\x00$#'\" "):
raise ValueError("Unsafe configuration value")
if not 1 <= port <= 65535:
raise ValueError("Invalid pipeline port")
settings = directory / ".env"
if settings.exists():
raise ValueError(".env exists; edit it rather than overwrite existing settings")
secrets = directory / "secrets"
secrets.mkdir(mode=0o700, exist_ok=True)
token = secrets / "ntfy-token.txt"
# Empty until ntfy has issued a token. Never replace an existing credential.
try:
fd = os.open(token, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
os.close(fd)
except FileExistsError:
pass
with settings.open("x") as file:
file.write(f"TZ={timezone}\nNTFY_BASE_URL={ntfy_url.rstrip('/')}\n"
f"NTFY_BIND_ADDRESS={bind_address}\nPIPELINES_PORT={port}\n")
return timezone
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--ntfy-url", required=True)
parser.add_argument("--bind-address", default="127.0.0.1")
parser.add_argument("--timezone")
parser.add_argument("--port", type=int, default=8081)
args = parser.parse_args()
try:
zone = configure(Path(__file__).resolve().parents[1], args.ntfy_url,
args.bind_address, args.timezone, args.port)
except (ValueError, ZoneInfoNotFoundError) as error:
parser.error(str(error))
print(f"Created .env with timezone {zone}. Configure the ntfy token before starting pipelines.")
+306
View File
@@ -0,0 +1,306 @@
#!/usr/bin/env python3
"""Install a clean native Pipelines service and authenticated Docker ntfy."""
import argparse
import getpass
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import platform
import plistlib
import re
import secrets
import shutil
import socket
import subprocess
import sys
from manage import DEFAULT_PREFIX, MARKER, compose, load, run, start, wait_health
SOURCE = Path(__file__).resolve().parents[1]
NAME = re.compile(r"^[A-Za-z][A-Za-z0-9_-]{0,63}$")
TOPIC = re.compile(r"^[-_A-Za-z0-9]{1,64}$")
def choose_port(preferred):
for port in range(preferred, min(preferred + 100, 65536)):
with socket.socket() as sock:
try:
sock.bind(("0.0.0.0", port))
return port
except OSError:
pass
raise ValueError("No available local service port")
def default_host():
tailscale = shutil.which("tailscale")
if tailscale:
try:
data = json.loads(run([tailscale, "status", "--json"], quiet=True, timeout=10).stdout)
if data.get("BackendState") == "Running":
return next(ip for ip in data["TailscaleIPs"] if ":" not in ip)
except (RuntimeError, ValueError, KeyError, StopIteration, subprocess.SubprocessError):
pass
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
try:
sock.connect(("192.0.2.1", 80))
return sock.getsockname()[0]
except OSError:
return "127.0.0.1"
def timezone():
spec = importlib.util.spec_from_file_location("configure", SOURCE / "scripts/configure-compose.py")
module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module)
return module.host_timezone()
def write_private(path, text):
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
with path.open("x") as file:
file.write(text)
path.chmod(0o600)
def ntfy_compose(config):
# JSON scalars are valid YAML scalars and prevent interpolation/injection.
q = lambda value: json.dumps(str(value))
return f'''services:
ntfy:
image: binwiederhier/ntfy:v2.29.0
command: serve
restart: unless-stopped
environment:
TZ: {q(config['timezone'])}
NTFY_BASE_URL: {q(config['public_url'])}
NTFY_LISTEN_HTTP: ":80"
NTFY_CACHE_FILE: /var/lib/ntfy/cache.db
NTFY_CACHE_DURATION: 168h
NTFY_AUTH_FILE: /var/lib/ntfy/auth.db
NTFY_AUTH_DEFAULT_ACCESS: deny-all
NTFY_ENABLE_LOGIN: "true"
NTFY_ATTACHMENT_CACHE_DIR: /var/lib/ntfy/attachments
NTFY_ATTACHMENT_EXPIRY_DURATION: 168h
NTFY_ATTACHMENT_TOTAL_SIZE_LIMIT: 1G
NTFY_ATTACHMENT_FILE_SIZE_LIMIT: 1M
ports:
- "0.0.0.0:{config['ntfy_port']}:80"
volumes:
- ntfy-data:/var/lib/ntfy
volumes:
ntfy-data:
'''
def make_registration(config):
root = Path(config["prefix"])
command = [config["python"], str(root / "lib/manage.py"), "supervise", "--prefix", str(root)]
if platform.system() == "Darwin":
path = Path.home() / "Library/LaunchAgents" / (config["label"] + ".plist")
path.parent.mkdir(parents=True, exist_ok=True)
if path.exists():
raise ValueError("Service registration already exists; refusing to overwrite")
with path.open("wb") as file:
plistlib.dump({"Label": config["label"], "ProgramArguments": command,
"RunAtLoad": True, "KeepAlive": {"SuccessfulExit": False},
"ExitTimeOut": 75, "ThrottleInterval": 10,
"StandardOutPath": str(root / "logs/supervisor.log"),
"StandardErrorPath": str(root / "logs/supervisor.log")}, file)
else:
path = Path.home() / ".config/systemd/user" / (config["label"] + ".service")
path.parent.mkdir(parents=True, exist_ok=True)
if path.exists():
raise ValueError("Service registration already exists; refusing to overwrite")
# systemd quotes its own command arguments, not shell strings.
quote = lambda value: '"' + value.replace("\\", "\\\\").replace('"', '\\"').replace("%", "%%") + '"'
path.write_text("[Unit]\nDescription=Pipelines server and ntfy\nAfter=default.target\n\n"
"[Service]\nType=simple\nExecStart=" + " ".join(map(quote, command)) +
"\nRestart=on-failure\nRestartSec=10\nTimeoutStopSec=90\n\n[Install]\nWantedBy=default.target\n")
return str(path)
def install(args):
if platform.system() not in ("Darwin", "Linux"):
raise ValueError("Supported: macOS and Linux (run as your normal user, not root)")
if os.getuid() == 0:
raise ValueError("Do not use sudo: this install reuses your user's OpenCode account")
root = args.prefix.expanduser().resolve()
if root in (Path("/"), Path.home().resolve(), SOURCE):
raise ValueError("Choose a dedicated installation directory")
docker = shutil.which("docker")
if not docker:
raise ValueError("Install/start Docker Desktop or Docker Engine first")
run([docker, "info", "--format", "{{.ServerVersion}}"], quiet=True, timeout=15)
run([docker, "compose", "version"], quiet=True, timeout=15)
if (root / MARKER).exists() and not load(root).get("uninstalled"):
config = load(root)
if not config.get("bootstrapped"):
raise ValueError("A partial installation exists; inspect it or run uninstall.sh --prefix ... --purge before retrying")
start(config)
print(f"Existing installation started; settings/history preserved: {root}")
return
retained = load(root) if (root / MARKER).exists() else None
if root.exists() and any(root.iterdir()) and retained is None:
raise ValueError("Installation directory is not empty; refusing to overwrite unrelated files")
if not args.no_autostart and platform.system() == "Linux":
run(["systemctl", "--user", "show-environment"], quiet=True)
opencode = args.opencode or shutil.which("opencode") or shutil.which("opencode2")
if not opencode:
npm=shutil.which("npm")
if not npm:
raise ValueError("Install OpenCode V2 first (npm install -g @opencode/cli), or install npm for automatic CLI setup")
if not args.yes and input("OpenCode V2 is missing. Install its pinned CLI privately using npm? [Y/n] ").strip().lower()=="n":
raise ValueError("OpenCode installation canceled")
root.mkdir(parents=True,exist_ok=True,mode=0o700)
run([npm,"install","--prefix",str(root / "tools"),"@opencode/cli@2.0.20"],quiet=True,timeout=240)
opencode=str(root / "tools/node_modules/.bin/opencode")
opencode = str(Path(opencode).resolve())
version = run([opencode, "--version"], quiet=True, timeout=15).stdout
if not re.search(r"(?:^|\s)v?2\.\d+", version):
raise ValueError("OpenCode V2 is required; your existing login/config will be reused, never exported")
auth = run([opencode, "auth", "list"], quiet=True, timeout=30).stdout
if "OpenCode Go" not in auth:
if args.yes:
raise ValueError("Connect OpenCode Go first: opencode auth login")
print("OpenCode Go is not connected. Complete your existing OpenCode login now.")
subprocess.run([opencode, "auth", "login"], check=True)
auth = run([opencode, "auth", "list"], quiet=True, timeout=30).stdout
if "OpenCode Go" not in auth:
raise ValueError("OpenCode Go login not confirmed; installation canceled")
# This exercises discovery/start as the same user, without generating model output.
run([opencode, "api", "get", "/api/info"], quiet=True, timeout=30)
root.mkdir(parents=True, exist_ok=True, mode=0o700)
root.chmod(0o700)
for directory in ("bin", "lib", "data", "logs", "secrets", "workspace"):
(root / directory).mkdir(exist_ok=True, mode=0o700)
packaged = SOURCE / "release-bin"
if packaged.is_dir():
metadata = json.loads((packaged / "platform.json").read_text())
if metadata != {"os": platform.system(), "arch": platform.machine()}:
raise ValueError("Release archive is for a different OS/CPU")
for name in ("pipelines-server", "pipelines-tui", "pipelines"):
shutil.copy2(packaged / name, root / "bin" / name)
else:
go = shutil.which("go")
if not go:
raise ValueError("Source install needs Go 1.27+; alternatively use a platform release archive")
run([go, "install", "-trimpath", "./cmd/server", "./cmd/tui", "./cmd/pipelines"],
env=dict(os.environ, GOBIN=str(root / "bin")), quiet=True, timeout=240)
for command in ("server", "tui"):
(root / "bin" / command).replace(root / "bin" / ("pipelines-" + command))
username = retained["username"] if retained else (args.username or ("phone-reader" if args.yes else input("Phone username [phone-reader]: ").strip() or "phone-reader"))
topic = retained["topic"] if retained else (args.topic or ("pipelines" if args.yes else input("Notification topic [pipelines]: ").strip() or "pipelines"))
if not NAME.fullmatch(username) or username == "pipeline-publisher" or not TOPIC.fullmatch(topic):
raise ValueError("Use safe alphanumeric usernames/topics; pipeline-publisher is reserved")
server_port = retained["server_port"] if retained else choose_port(args.server_port)
ntfy_port = retained["ntfy_port"] if retained else choose_port(args.ntfy_port)
if server_port == ntfy_port:
ntfy_port = choose_port(ntfy_port + 1)
public_url = retained["public_url"] if retained else args.public_url or f"http://{default_host()}:{ntfy_port}"
from urllib.parse import urlparse
url = urlparse(public_url)
if url.scheme not in ("http", "https") or not url.hostname or url.username or url.password or url.path not in ("", "/") or url.query or url.fragment or any(c in public_url for c in "$\n\r\x00"):
raise ValueError("Phone URL must be an HTTP(S) base URL without credentials, topic, query, or interpolation")
chosen_workspace=args.workspace or (Path(retained["workspace"]) if retained else root / "workspace")
workspace = str(chosen_workspace.expanduser().resolve())
if not Path(workspace).is_dir():
raise ValueError("Workspace directory must exist")
suffix = hashlib.sha256(str(root).encode()).hexdigest()[:12]
bindir = Path("/usr/local/bin") if args.system_command else args.bin_dir.expanduser().resolve()
launcher = bindir / "pipelines"
if launcher.exists() or launcher.is_symlink():
if not launcher.is_symlink() or launcher.resolve() != root / "bin/pipelines":
raise ValueError("A pipelines command already exists in the target bin directory; choose --bin-dir")
# Only explicit server-selection variables are copied; never API keys/tokens.
opencode_env = {key: value for key, value in os.environ.items() if key in ("XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_STATE_HOME", "OPENCODE_SERVER", "OPENCODE_DB")}
config = dict(format=1, prefix=str(root), docker=docker, python=sys.executable,
opencode=opencode, opencode_env=opencode_env, path=os.pathsep.join([str(root / "bin"), str(Path(opencode).parent), os.environ.get("PATH", "")]),
project="pipelines-" + suffix, label="io.pipelines." + suffix,
server_port=server_port, ntfy_port=ntfy_port, public_url=public_url.rstrip("/"),
topic=topic, username=username, timezone=timezone(), workspace=workspace,
workspace_roots=[str(path.expanduser().resolve()) for path in (args.workspace_root or ([Path(p) for p in retained["workspace_roots"]] if retained else [Path.home()]))],
launcher=str(launcher), registration=None, bootstrapped=bool(retained and retained.get("bootstrapped")))
manifest = root / MARKER
manifest.write_text(json.dumps(config, indent=2)); manifest.chmod(0o600)
(root / "compose.yaml").write_text(ntfy_compose(config))
shutil.copy2(SOURCE / "scripts/manage.py", root / "lib/manage.py")
alias = root / "bin/opencode"
if not alias.exists():
alias.symlink_to(opencode)
# The PATH command is the compiled Go binary, not a shell/Python launcher.
if not config["bootstrapped"]:
if args.password_file:
password = args.password_file.read_text().strip()
elif args.yes:
password = secrets.token_urlsafe(18)
else:
password = getpass.getpass("Phone password (blank = generate): ") or secrets.token_urlsafe(18)
if len(password) < 8 or any(c in password for c in "\r\n\x00"):
raise ValueError("Phone password must have at least 8 characters and no line breaks")
compose(config, "up", "-d", "ntfy", quiet=True)
wait_health(f"http://127.0.0.1:{ntfy_port}/v1/health")
for user, secret in (("pipeline-publisher", secrets.token_urlsafe(32)), (username, password)):
compose(config, "exec", "-T", "-e", "NTFY_PASSWORD", "ntfy", "ntfy", "user", "add", user,
env=dict(os.environ, NTFY_PASSWORD=secret), quiet=True)
compose(config, "exec", "-T", "ntfy", "ntfy", "access", "pipeline-publisher", topic, "write-only", quiet=True)
compose(config, "exec", "-T", "ntfy", "ntfy", "access", username, topic, "read-only", quiet=True)
output = compose(config, "exec", "-T", "ntfy", "ntfy", "token", "add", "pipeline-publisher", quiet=True).stdout
tokens = re.findall(r"\btk_[A-Za-z0-9]{29}\b", output)
if len(tokens) != 1:
raise RuntimeError("Could not extract the publishing token; credentials were not printed")
write_private(root / "secrets/publisher-token", tokens[0] + "\n")
write_private(root / "secrets/phone-login.txt", f"Server: {public_url}\nTopic: {topic}\nUsername: {username}\nPassword: {password}\n")
config["bootstrapped"] = True
if not args.no_autostart:
config["registration"] = make_registration(config)
manifest.write_text(json.dumps(config, indent=2))
if args.system_command and not bindir.exists():
subprocess.run(["sudo", "mkdir", "-p", str(bindir)], check=True)
else:
bindir.mkdir(parents=True, exist_ok=True)
if not launcher.is_symlink():
if args.system_command and not os.access(bindir, os.W_OK):
# Privilege is used ONLY for this command symlink, never for the server/auth.
subprocess.run(["sudo", "ln", "-s", str(root / "bin/pipelines"), str(launcher)], check=True)
else:
launcher.symlink_to(root / "bin/pipelines")
start(config)
print(f"Installed. Run: {launcher}\nManage both services: pipelines start|stop|restart|status|doctor|logs")
print(f"Phone server: {config['public_url']}\nTopic: {topic}\nUsername: {username}")
print(f"Password is private: {root / 'secrets/phone-login.txt'}")
print("Fresh installations have no pipelines. In the TUI press b for rendered examples/templates.")
print("ntfy is exposed on LAN/VPN with authentication. Use HTTPS or an encrypted VPN outside a trusted LAN.")
if str(bindir) not in os.environ.get("PATH", "").split(os.pathsep):
print(f"Add {bindir} to PATH, or use the full command path above.")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--prefix", type=Path, default=DEFAULT_PREFIX)
parser.add_argument("--bin-dir", type=Path, default=Path.home() / ".local/bin")
parser.add_argument("--system-command",action="store_true",help="Put the command in /usr/local/bin (sudo only for the symlink; services remain per-user)")
parser.add_argument("--username")
parser.add_argument("--topic")
parser.add_argument("--password-file", type=Path)
parser.add_argument("--public-url")
parser.add_argument("--workspace", type=Path)
parser.add_argument("--workspace-root",type=Path,action="append",help="Allowed repository root; repeatable (default: your home)")
parser.add_argument("--opencode", help="Path to the host OpenCode V2 CLI")
parser.add_argument("--server-port", type=int, default=8080)
parser.add_argument("--ntfy-port", type=int, default=8090)
parser.add_argument("--no-autostart", action="store_true", help="Use a managed background process rather than launchd/systemd")
parser.add_argument("--yes", action="store_true", help="Accept defaults and generate a private phone password")
args = parser.parse_args()
if not 1024 <= args.server_port <= 65535 or not 1024 <= args.ntfy_port <= 65535:
parser.error("Use unprivileged ports 1024–65535")
os.chdir(SOURCE)
install(args)
if __name__ == "__main__":
try:
main()
except (ValueError, RuntimeError, OSError, subprocess.SubprocessError) as error:
sys.exit(str(error))
+252
View File
@@ -0,0 +1,252 @@
#!/usr/bin/env python3
"""One per-user lifecycle for the native server and its private Docker ntfy."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import platform
import shutil
import signal
import subprocess
import sys
import time
import urllib.request
DEFAULT_PREFIX = Path.home() / ".local/share/pipelines"
MARKER = "pipelines-install.json"
def load(prefix):
prefix = Path(prefix).expanduser().resolve()
path = prefix / MARKER
if not path.is_file() or path.is_symlink() or path.stat().st_uid != os.getuid():
raise ValueError("Not a managed Pipelines installation owned by this user")
config = json.loads(path.read_text())
if config.get("format") != 1 or config.get("prefix") != str(prefix):
raise ValueError("Invalid installation manifest; refusing to manage paths")
return config
def run(args, *, env=None, quiet=False, check=True, timeout=120):
result = subprocess.run(args, env=env, capture_output=True, text=True, timeout=timeout)
if check and result.returncode:
# No subprocess output here: ntfy token/password commands may contain secrets.
raise RuntimeError(f"Command failed ({result.returncode}): {args[0]} {args[1] if len(args)>1 else ''}")
if not quiet and result.stdout:
print(result.stdout.rstrip())
return result
def compose(config, *args, **kwargs):
return run([config["docker"], "compose", "--project-name", config["project"],
"--file", str(Path(config["prefix"]) / "compose.yaml"), *args], **kwargs)
def healthy(url):
try:
with urllib.request.urlopen(url, timeout=2) as response:
return response.status == 200 and json.load(response).get("healthy") is True
except (OSError, ValueError):
return False
def wait_health(url, timeout=40):
end = time.monotonic() + timeout
while time.monotonic() < end:
if healthy(url):
return
time.sleep(0.25)
raise RuntimeError(f"Service did not become healthy: {url}; see pipelines logs")
def managed_pid(config):
path = Path(config["prefix"]) / "supervisor.pid"
try:
pid = int(path.read_text())
if pid <= 1:
return None
command = subprocess.check_output(["ps", "-p", str(pid), "-o", "command="], text=True)
# Never signal an arbitrary/reused PID.
if str(Path(config["prefix"]) / "lib/manage.py") not in command or "supervise" not in command:
return None
return pid
except (OSError, ValueError, subprocess.CalledProcessError):
return None
def registration(config):
return config.get("registration")
def start(config):
root = Path(config["prefix"])
if registration(config):
if platform.system() == "Darwin":
target = f"gui/{os.getuid()}/{config['label']}"
if run(["launchctl", "print", target], quiet=True, check=False).returncode:
run(["launchctl", "bootstrap", f"gui/{os.getuid()}", registration(config)], quiet=True)
run(["launchctl", "kickstart", target], quiet=True)
else:
run(["systemctl", "--user", "daemon-reload"], quiet=True)
run(["systemctl", "--user", "enable", "--now", config["label"] + ".service"], quiet=True)
elif managed_pid(config) is None:
log = (root / "logs/supervisor.log").open("a")
subprocess.Popen([config["python"], str(root / "lib/manage.py"), "supervise", "--prefix", str(root)],
stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True)
log.close()
wait_health(f"http://127.0.0.1:{config['server_port']}/healthz", timeout=75)
def stop(config):
if registration(config):
if platform.system() == "Darwin":
run(["launchctl", "bootout", f"gui/{os.getuid()}", registration(config)], quiet=True, check=False)
else:
run(["systemctl", "--user", "stop", config["label"] + ".service"], quiet=True)
pid = managed_pid(config)
if pid is not None:
os.kill(pid, signal.SIGTERM)
deadline = time.monotonic() + 80
while managed_pid(config) is not None and time.monotonic() < deadline:
time.sleep(0.2)
if managed_pid(config) is not None:
raise RuntimeError("Supervisor did not stop; inspect logs before trying again")
compose(config, "stop", "ntfy", quiet=True)
def supervise(config):
root = Path(config["prefix"])
with (root / "supervisor.lock").open("a") as lock:
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
return
pidfile = root / "supervisor.pid"
pidfile.write_text(str(os.getpid()))
stopped = False
child = None
def stopping(*_):
nonlocal stopped
stopped = True
if child is not None and child.poll() is None:
child.terminate()
signal.signal(signal.SIGTERM, stopping)
signal.signal(signal.SIGINT, stopping)
try:
compose(config, "up", "-d", "ntfy", quiet=True)
wait_health(f"http://127.0.0.1:{config['ntfy_port']}/v1/health")
if stopped:
return
env = dict(os.environ)
env.update(config.get("opencode_env", {}))
env.update(HOME=str(Path.home()), PATH=config["path"], TZ=config["timezone"],
PIPELINES_NTFY_URL=f"http://127.0.0.1:{config['ntfy_port']}/{config['topic']}",
PIPELINES_NTFY_PUBLIC_URL=config["public_url"],
PIPELINES_NTFY_TOKEN_FILE=str(root / "secrets/publisher-token"))
env.pop("PIPELINES_NTFY_TOKEN", None)
with (root / "logs/server.log").open("a") as log:
child = subprocess.Popen([str(root / "bin/pipelines-server"), "-addr",
f"127.0.0.1:{config['server_port']}", "-db", str(root / "data/pipelines.db"),
*sum((["-workspace-root", path] for path in config["workspace_roots"]), [])],
cwd=config["workspace"], env=env, stdout=log, stderr=log)
while child.poll() is None and not stopped:
time.sleep(0.25)
if not stopped and child.returncode:
raise RuntimeError("Pipeline server exited; see logs/server.log")
finally:
if child is not None and child.poll() is None:
child.terminate()
try:
child.wait(timeout=70)
except subprocess.TimeoutExpired:
child.kill()
child.wait()
compose(config, "stop", "ntfy", quiet=True, check=False)
pidfile.unlink(missing_ok=True)
def uninstall(config, purge=False, yes=False):
root = Path(config["prefix"])
if not yes and input("Uninstall Pipelines? [y/N] ").strip().lower() != "y":
return
if purge and not yes and input("Also DELETE pipeline history, credentials, and this install's ntfy volume? Type DELETE: ") != "DELETE":
raise ValueError("Purge canceled; nothing deleted")
stop(config)
if registration(config):
if platform.system() != "Darwin":
run(["systemctl", "--user", "disable", config["label"] + ".service"], quiet=True, check=False)
Path(registration(config)).unlink(missing_ok=True)
compose(config, "down", *(["--volumes"] if purge else []), quiet=True)
link = Path(config["launcher"])
if link.is_symlink() and link.resolve() == (root / "bin/pipelines").resolve():
if str(link)=="/usr/local/bin/pipelines" and not os.access(link.parent,os.W_OK):
subprocess.run(["sudo","rm",str(link)],check=True)
else:
link.unlink()
if purge:
# Only the explicitly identified, owned installation directory is removed.
shutil.rmtree(root)
print("Removed installation and its pipeline/ntfy data. Host OpenCode was untouched.")
else:
for path in (root / "bin", root / "lib"):
if path.exists():
shutil.rmtree(path)
config["registration"] = None
config["uninstalled"] = True
(root / MARKER).write_text(json.dumps(config, indent=2))
print(f"Uninstalled. Data and ntfy volume retained at {root}; reinstall to reuse them.")
def main():
lifecycle = ("start", "stop", "restart", "status", "doctor", "logs", "supervise", "uninstall")
# Strip only the launcher's installation selector. All application arguments
# belong to the shared Go CLI (including global flags before the command).
selector = argparse.ArgumentParser(add_help=False)
selector.add_argument("--prefix", type=Path, default=DEFAULT_PREFIX)
selected, forwarded = selector.parse_known_args()
config = load(selected.prefix)
if not forwarded or forwarded[0] not in lifecycle:
binary = str(Path(config["prefix"]) / "bin/pipelines")
env = dict(os.environ)
env.update(config.get("opencode_env", {}))
env.update(PIPELINES_URL=f"ws://127.0.0.1:{config['server_port']}/connect",
PIPELINES_OPENCODE=config["opencode"], PATH=config["path"])
os.execve(binary, [binary, *(forwarded or ["tui"])], env)
return
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("action", nargs="?", default="tui")
parser.add_argument("--prefix", type=Path, default=DEFAULT_PREFIX)
parser.add_argument("--purge", action="store_true")
parser.add_argument("--yes", action="store_true")
args, rest = parser.parse_known_args()
if args.action in lifecycle and rest:
parser.error("unexpected arguments: " + " ".join(rest))
if args.action == "start":
start(config)
elif args.action == "stop":
stop(config)
elif args.action == "restart":
stop(config); start(config)
elif args.action in ("status", "doctor"):
print("Pipeline server:", "healthy" if healthy(f"http://127.0.0.1:{config['server_port']}/healthz") else "stopped/unhealthy")
print("ntfy:", "healthy" if healthy(f"http://127.0.0.1:{config['ntfy_port']}/v1/health") else "stopped/unhealthy")
print("Phone server:", config["public_url"], "· topic:", config["topic"], "· username:", config["username"])
if args.action == "doctor":
run([config["opencode"], "api", "get", "/api/info"], quiet=True)
print("Host OpenCode API: responding (model execution is a separate check)")
elif args.action == "logs":
os.execvp("tail", ["tail", "-n", "100", "-f", str(Path(config["prefix"]) / "logs/server.log")])
elif args.action == "supervise":
supervise(config)
elif args.action == "uninstall":
uninstall(config, args.purge, args.yes)
if __name__ == "__main__":
try:
main()
except (ValueError, RuntimeError, OSError, subprocess.SubprocessError) as error:
sys.exit(str(error))
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env python3
"""Build an allowlisted tester archive: never include runtime databases or secrets."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import subprocess
import tarfile
import tempfile
ROOT = Path(__file__).resolve().parents[1]
def source_files():
files = [ROOT / name for name in ("go.mod", "go.sum", "README.md", "install.sh", "uninstall.sh")]
for directory, patterns in {"cmd": ["*.go"], "internal": ["*.go", "*.md"], "examples": ["*.go", "*.md"],
"docs": ["*.md"], "scripts": ["*.py", "*.sh"]}.items():
for pattern in patterns:
files.extend((ROOT / directory).rglob(pattern))
return sorted({path for path in files if path.is_file() and not path.is_symlink()})
def package(output, version, target):
if not version or any(c not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-_" for c in version):
raise ValueError("Invalid release version")
output = Path(output).resolve(); output.mkdir(parents=True, exist_ok=True)
name = f"pipelines-{version}-{target}"
archive = output / (name + ".tar.gz")
if archive.exists():
raise ValueError("Archive exists; use a new version/output directory")
with tempfile.TemporaryDirectory(prefix="pipelines-package-", dir=output) as temporary:
binaries = Path(temporary)
if target != "source":
system, arch = target.split("-")
if system not in ("darwin", "linux") or arch not in ("arm64", "amd64"):
raise ValueError("Supported: source, darwin-arm64/amd64, linux-arm64/amd64")
env = dict(os.environ, GOOS=system, GOARCH=arch, CGO_ENABLED="0")
for command, binary in (("server", "pipelines-server"), ("tui", "pipelines-tui"), ("pipelines", "pipelines")):
subprocess.run(["go", "build", "-trimpath", "-o", str(binaries / binary), "./cmd/" + command],
cwd=ROOT, env=env, check=True)
(binaries / "platform.json").write_text(json.dumps({"os": "Darwin" if system == "darwin" else "Linux", "arch": "arm64" if arch == "arm64" and system == "darwin" else "aarch64" if arch == "arm64" else "x86_64"}))
with tarfile.open(archive, "w:gz") as tar:
for path in source_files():
tar.add(path, arcname=str(Path(name) / path.relative_to(ROOT)), recursive=False)
if target != "source":
for path in sorted(binaries.iterdir()):
tar.add(path, arcname=str(Path(name) / "release-bin" / path.name), recursive=False)
digest = hashlib.sha256(archive.read_bytes()).hexdigest()
archive.with_suffix(archive.suffix + ".sha256").write_text(digest + " " + archive.name + "\n")
return archive
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--platform", default="source")
parser.add_argument("--version", default="0.1.0-beta.1")
parser.add_argument("--output", type=Path, default=ROOT / "dist")
args = parser.parse_args()
print(package(args.output, args.version, args.platform))
+13
View File
@@ -0,0 +1,13 @@
#!/bin/sh
# Keep OpenCode local; only ntfy runs in Docker. Public URL contains no secret.
set -eu
cd "$(dirname "$0")/.."
if [ -z "${PIPELINES_NTFY_PUBLIC_URL:-}" ]; then
PIPELINES_NTFY_PUBLIC_URL=$(sed -n 's/^NTFY_BASE_URL=//p' .env)
fi
export PIPELINES_NTFY_PUBLIC_URL
export PIPELINES_NTFY_URL="${PIPELINES_NTFY_URL:-http://127.0.0.1:8090/pipelines}"
if [ -z "${PIPELINES_NTFY_TOKEN:-}" ]; then
export PIPELINES_NTFY_TOKEN_FILE="${PIPELINES_NTFY_TOKEN_FILE:-$(pwd)/secrets/ntfy-token.txt}"
fi
exec go run ./cmd/server "$@"
+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/env python3
"""Extract only the token from ntfy's CLI response; never print the credential."""
import os
from pathlib import Path
import re
import sys
def save_token(output, directory):
tokens = re.findall(r"\btk_[A-Za-z0-9]{29}\b", output)
if len(tokens) != 1:
raise ValueError("ntfy did not return exactly one token; existing credentials were not changed")
directory = Path(directory)
secrets = directory / "secrets"
secrets.mkdir(mode=0o700, exist_ok=True)
secrets.chmod(0o700)
path = secrets / "ntfy-token.txt"
if path.is_symlink() or (path.exists() and path.stat().st_size != 0):
raise ValueError("Token file already contains credentials; refusing to overwrite")
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(fd, "w") as file:
file.write(tokens[0] + "\n")
# Compose's file secret is a bind mount. Container uid 10001 needs read access;
# the private host parent directory restricts access by other host users.
path.chmod(0o444)
if __name__ == "__main__":
try:
save_token(sys.stdin.read(), Path(__file__).resolve().parents[1])
except ValueError as error:
sys.exit(str(error))
print("Saved ntfy publishing token; restart pipelines if it was already running.")
+42
View File
@@ -0,0 +1,42 @@
import importlib.util
from pathlib import Path
import tempfile
import unittest
spec = importlib.util.spec_from_file_location("configure", Path(__file__).with_name("configure-compose.py"))
configure = importlib.util.module_from_spec(spec)
spec.loader.exec_module(configure)
class ConfigureTests(unittest.TestCase):
def test_detect_timezone_and_preserve_existing_files(self):
with tempfile.TemporaryDirectory() as directory:
zone = configure.configure(directory, "http://192.168.1.20:8090", "0.0.0.0")
self.assertTrue(zone)
settings = Path(directory, ".env")
original = settings.read_bytes()
self.assertIn(f"TZ={zone}", settings.read_text())
self.assertEqual(Path(directory, "secrets/ntfy-token.txt").stat().st_mode & 0o777, 0o600)
with self.assertRaises(ValueError):
configure.configure(directory, "http://192.168.1.21:8090", "0.0.0.0")
self.assertEqual(settings.read_bytes(), original)
def test_existing_token_is_not_overwritten(self):
with tempfile.TemporaryDirectory() as directory:
secrets = Path(directory, "secrets")
secrets.mkdir()
token = secrets / "ntfy-token.txt"
token.write_text("existing-secret")
configure.configure(directory, "http://192.168.1.20:8090", "0.0.0.0", "Europe/London")
self.assertEqual(token.read_text(), "existing-secret")
def test_reject_bad_url_and_injected_values(self):
for url in ["file:///foo", "https://user:pass@example.com", "https://example.com/topic", "https://example.com?token=secret"]:
with tempfile.TemporaryDirectory() as directory, self.assertRaises(ValueError):
configure.configure(directory, url, "0.0.0.0", "Europe/London")
with tempfile.TemporaryDirectory() as directory, self.assertRaises(ValueError):
configure.configure(directory, "https://example.com", "0.0.0.0\nEVIL=yes", "Europe/London")
if __name__ == "__main__":
unittest.main()
+90
View File
@@ -0,0 +1,90 @@
import importlib.util
import json
import os
from pathlib import Path
import socket
import tempfile
import unittest
from unittest import mock
import install
import manage
class InstallationTests(unittest.TestCase):
def test_launcher_forwards_application_arguments_unchanged(self):
config = dict(prefix="/owned", server_port=12345, opencode="/host/opencode", path="/owned/bin:/usr/bin")
for command in (["edit", "a name", "change X", "--json"], ["--url", "ws://other/connect", "list", "--json"], ["--help"], ["run", "workflow", "task", "--yes", "--json"]):
with mock.patch.object(manage,"load",return_value=config), mock.patch.object(manage.sys,"argv",["manage.py","--prefix","/owned",*command]), mock.patch.object(manage.os,"execve") as execute:
manage.main()
binary,args,env=execute.call_args.args
self.assertEqual(args,["/owned/bin/pipelines",*command])
self.assertEqual(env["PIPELINES_URL"],"ws://127.0.0.1:12345/connect")
def test_compose_has_only_ntfy_and_no_credentials(self):
config = dict(timezone="Europe/London", public_url="http://100.1.2.3:8090", ntfy_port=8090)
text = install.ntfy_compose(config)
self.assertIn("binwiederhier/ntfy:v2.29.0", text)
self.assertIn("deny-all", text)
self.assertNotIn("opencode", text)
self.assertNotIn("password", text.lower())
self.assertNotIn("pipelines:", text)
def test_detect_free_port_avoids_listener(self):
with socket.socket() as listener:
listener.bind(("0.0.0.0", 0)); listener.listen()
occupied = listener.getsockname()[1]
self.assertNotEqual(install.choose_port(occupied), occupied)
def test_secret_files_private_and_not_overwritten(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory, "secrets/token")
install.write_private(path, "private")
self.assertEqual(path.stat().st_mode & 0o777, 0o600)
with self.assertRaises(FileExistsError):
install.write_private(path, "replacement")
self.assertEqual(path.read_text(), "private")
def test_management_requires_owned_installation_marker(self):
with tempfile.TemporaryDirectory() as directory:
with self.assertRaises(ValueError):
manage.load(directory)
root = Path(directory).resolve()
(root / manage.MARKER).write_text(json.dumps(dict(format=1, prefix=str(root))))
self.assertEqual(manage.load(root)["prefix"], str(root))
(root / manage.MARKER).write_text(json.dumps(dict(format=1, prefix="/")))
with self.assertRaises(ValueError):
manage.load(root)
def test_pid_reuse_cannot_signal_unrelated_process(self):
with tempfile.TemporaryDirectory() as directory:
Path(directory, "supervisor.pid").write_text(str(os.getpid()))
self.assertIsNone(manage.managed_pid(dict(prefix=directory)))
def test_default_uninstall_preserves_data_and_open_code(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory, "app"); root.mkdir()
for name in ("bin", "lib", "data", "secrets"):
(root / name).mkdir()
(root / "data/pipelines.db").write_text("history")
(root / "secrets/publisher-token").write_text("credential")
config = dict(format=1, prefix=str(root), launcher=str(Path(directory, "pipelines")), registration=None)
Path(config["launcher"]).symlink_to(root / "bin/pipelines")
with mock.patch.object(manage, "stop"), mock.patch.object(manage, "compose") as compose:
manage.uninstall(config, yes=True)
compose.assert_called_once_with(config, "down", quiet=True)
self.assertEqual((root / "data/pipelines.db").read_text(), "history")
self.assertEqual((root / "secrets/publisher-token").read_text(), "credential")
self.assertFalse((root / "bin").exists())
self.assertFalse(Path(config["launcher"]).is_symlink())
def test_service_registration_quotes_paths(self):
with tempfile.TemporaryDirectory(prefix="path with spaces ") as directory:
root = Path(directory);(root / "logs").mkdir()
config = dict(prefix=str(root), python="/usr/bin/python3", label="io.pipelines.test")
with mock.patch.object(install.platform,"system",return_value="Linux"),mock.patch.object(install.Path,"home",return_value=root):
path = Path(install.make_registration(config))
self.assertIn('"'+str(root / 'lib/manage.py')+'"', path.read_text())
self.assertIn("Restart=on-failure", path.read_text())
if __name__ == "__main__":
unittest.main()
+21
View File
@@ -0,0 +1,21 @@
from pathlib import Path
import tarfile
import tempfile
import unittest
import package
class PackageTests(unittest.TestCase):
def test_source_archive_excludes_runtime_and_credentials(self):
with tempfile.TemporaryDirectory() as directory:
archive = package.package(directory, "test", "source")
with tarfile.open(archive) as tar:
names = tar.getnames()
self.assertTrue(any(name.endswith("install.sh") for name in names))
self.assertTrue(any(name.endswith("examples/anime-today.md") for name in names))
self.assertFalse(any("/secrets/" in name or name.endswith(".db") or "/.env" in name or "/__pycache__/" in name for name in names))
self.assertTrue(archive.with_suffix(archive.suffix + ".sha256").exists())
if __name__ == "__main__":
unittest.main()
+36
View File
@@ -0,0 +1,36 @@
import importlib.util
from pathlib import Path
import tempfile
import unittest
spec = importlib.util.spec_from_file_location("save_token", Path(__file__).with_name("save-ntfy-token.py"))
save_token = importlib.util.module_from_spec(spec)
spec.loader.exec_module(save_token)
class TokenTests(unittest.TestCase):
def test_extract_prose_and_protect_source_directory(self):
token = "tk_" + "a" * 29
with tempfile.TemporaryDirectory() as directory:
save_token.save_token(f"token {token} created for user pipeline-publisher, never expires\n", directory)
path = Path(directory, "secrets/ntfy-token.txt")
self.assertEqual(path.read_text(), token + "\n")
self.assertEqual(path.stat().st_mode & 0o777, 0o444)
self.assertEqual(path.parent.stat().st_mode & 0o777, 0o700)
with self.assertRaises(ValueError):
save_token.save_token(f"token {token} created", directory)
self.assertEqual(path.read_text(), token + "\n")
def test_failures_do_not_replace_empty_placeholder(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory, "secrets/ntfy-token.txt")
path.parent.mkdir()
path.write_text("")
for output in ["", "connection failed", "tk_" + "a" * 29 + " tk_" + "b" * 29]:
with self.assertRaises(ValueError):
save_token.save_token(output, directory)
self.assertEqual(path.read_text(), "")
if __name__ == "__main__":
unittest.main()