merge: fresh installs work unattended; pipelines update

This commit is contained in:
2026-10-23 23:42:29 +01:00
17 changed files with 233 additions and 34 deletions
+2 -2
View File
@@ -32,7 +32,7 @@ SESSIONS = [
"title": "Answer an approval",
"agent": "build",
"model": "deepseek-v4.1",
"directory": "/Users/alan/repos/pipelines",
"directory": "/home/user/project",
"created_at": 1760620000000,
"updated_at": 1760620600000,
"busy": False,
@@ -42,7 +42,7 @@ SESSIONS = [
"title": "Already resolved",
"agent": "build",
"model": "deepseek-v4.1",
"directory": "/Users/alan/repos/pipelines",
"directory": "/home/user/project",
"created_at": 1760620000000,
"updated_at": 1760620600000,
"busy": False,
+1 -1
View File
@@ -58,7 +58,7 @@ class _FixtureGateway implements ApiGateway {
'chat_directories': [
'/workspace/demo',
'/workspace/archive',
'/Users/alan/repos/pipelines/tmp/flutter-chat-list',
'/home/user/project',
],
};
case 'chat_backends':
+1 -1
View File
@@ -7,7 +7,7 @@
# verdict comes from the suite's own final line, which is echoed with the
# totals. Logs go to build/check/ for the caller's evidence.
set -u
FLUTTER="${FLUTTER:-/Users/alan/development/flutter/bin/flutter}"
FLUTTER="${FLUTTER:-flutter}"
mkdir -p build/check
"$FLUTTER" analyze > build/check/analyze.log 2>&1
if ! grep -q "No issues found" build/check/analyze.log; then
+6 -6
View File
@@ -5,21 +5,21 @@
# The build number is minutes since the epoch, so every build is newer than
# the last. The APK and version.json go to build/updates/ (kept across preview
# publishes) and are copied into the live preview at build/web/updates/,
# served behind the web login at https://<host>:8096/updates/. Never restarts
# served behind the web login at /updates/. Never restarts
# any server.
set -eu
notes="${1:-}"
# The phone's default server: the web proxy over Tailscale (sign in once in
# Settings). Override with PIPELINES_SERVER_URL.
server="${PIPELINES_SERVER_URL:-wss://alans-macbook-pro.tail6bf4dc.ts.net:8443/connect}"
FLUTTER="${FLUTTER:-/Users/alan/development/flutter/bin/flutter}"
# The phone's default server, e.g. wss://host:8443/connect (required). An
# installation builds its APK with ./install --apk instead.
server="${PIPELINES_SERVER_URL:?set PIPELINES_SERVER_URL to the phone default server, e.g. wss://host/connect}"
FLUTTER="${FLUTTER:-flutter}"
export JAVA_HOME="${JAVA_HOME:-/Applications/Android Studio.app/Contents/jbr/Contents/Home}"
export ANDROID_HOME="${ANDROID_HOME:-$HOME/Library/Android/sdk}"
code=$(( $(date +%s) / 60 ))
name="0.1.$code"
"$FLUTTER" build apk --release --build-number="$code" --build-name="$name" \
--dart-define=PIPELINES_SERVER_URL="$server" \
--dart-define=PIPELINES_UPDATES_URL="${PIPELINES_UPDATES_URL:-https://alans-macbook-pro.tail6bf4dc.ts.net:8096/updates/}"
${PIPELINES_UPDATES_URL:+--dart-define=PIPELINES_UPDATES_URL="$PIPELINES_UPDATES_URL"}
mkdir -p build/updates
cp build/app/outputs/flutter-apk/app-release.apk build/updates/pipelines.apk
cat > build/updates/version.json <<JSON
+1 -1
View File
@@ -8,7 +8,7 @@
# directory and the new main.dart.js hash. Never restarts any server.
set -eu
name="${1:?usage: sh tool/publish_preview.sh <name>}"
FLUTTER="${FLUTTER:-/Users/alan/development/flutter/bin/flutter}"
FLUTTER="${FLUTTER:-flutter}"
"$FLUTTER" build web -o build/web-next
rollback="build/web-before-${name}-$(date +%s)"
if [ -d build/web ]; then mv build/web "$rollback"; fi
+1 -1
View File
@@ -128,7 +128,7 @@ const _fixtureHits = <JsonMap>[
'id': 457,
'pipeline_name': 'Plan and land',
'status': 'succeeded',
'workspace': '/Users/alan/repos/pipelines',
'workspace': '/home/user/project',
'created_at': '2026-10-20T08:12:00Z',
'finished_at': '2026-10-20T08:42:00Z',
},
-2
View File
@@ -77,8 +77,6 @@ func banner(w io.Writer, s startup) {
}
if s.ClaudeToken {
row("Claude", "service token present")
} else {
row("Claude", "no service token; as a background service run `claude setup-token` and save it to ~/.config/pipelines/claude-token (chmod 600)")
}
if s.HarnessErr != nil {
row("Harnesses", "unreadable: "+s.HarnessErr.Error())
+14
View File
@@ -14,8 +14,22 @@ cd pipelines
./install --docker # explicit Docker server + ntfy
./install --web # rebuild the web app (Docker) after app changes
./install --apk # optional: build the Android app (Docker, slower)
pipelines update # later: pull, reinstall the same way, restart
```
## Updating
`pipelines update` brings an installation to the newest source. It waits while
runs are executing, fast-forwards the checkout it was installed from
(`git pull --ff-only`), re-runs `./install` with the options recorded at install
time, restarts the server, web proxy and ntfy, and prints the startup report.
Data, the web password, ntfy accounts and the Android signing key are kept; the
web app and APK are rebuilt only when their sources changed. Phones pick up a
new APK (after `./install --apk`) from Settings → App updates.
`./install --fresh` starts with empty data even when volumes from an earlier
Docker installation exist (by default they are migrated).
## Native installation (default)
`./install`, `./install --native` and `./install native` run
+1
View File
@@ -72,5 +72,6 @@ var serviceHelp = map[string]string{
"restart": "Stop, then start both services, retaining definitions, run history, and credentials.",
"doctor": "Check both health endpoints and host OpenCode API discovery; does not execute a model.",
"logs": "Follow the native server log. Press Ctrl+C to stop following.",
"update": "Update this installation: wait for executing runs, pull the source checkout it was installed from, re-run its installer the same way (data and credentials kept), restart, and print the startup report.",
"uninstall": "Remove verified Go binaries from this command's bin directory; retain data and drafts. --yes skips confirmation. Existing managed installations can be selected explicitly with PIPELINES_PREFIX.",
}
+1
View File
@@ -53,6 +53,7 @@ SERVER / INTERFACE
service enable Configure user autostart
service disable | status Stop/remove autostart, or inspect OS service status
skill install Install/update the shared OpenCode and Claude Code workflow skill
update Update this installation from its source checkout
uninstall [--yes] Remove verified Go binaries; retain data and drafts
Legacy managed installs: select PIPELINES_PREFIX explicitly for start/stop,
+1 -1
View File
@@ -100,7 +100,7 @@ func (c Config) Environment() error {
func Service(action string) bool {
switch action {
case "start", "stop", "restart", "doctor", "logs", "uninstall":
case "start", "stop", "restart", "doctor", "logs", "uninstall", "update":
return true
}
return false
+49 -1
View File
@@ -309,6 +309,52 @@ def report(config):
print("==")
def active_runs(config):
"""Runs the server is still executing; an update waits for them."""
binary = str(Path(config["prefix"]) / "bin/pipelines")
result = run([binary, "--url", f"ws://127.0.0.1:{config['server_port']}/connect", "runs", "--json"],
quiet=True, check=False, timeout=60)
if result.returncode:
return None
try:
data = json.loads(result.stdout)
except ValueError:
return None
runs = data.get("runs", []) if isinstance(data, dict) else data
return [item for item in runs if isinstance(item, dict) and item.get("executing")]
def update(config):
"""Brings this installation to the newest source: pull, reinstall the same
way it was installed (data and credentials kept), restart, report."""
source = Path(config.get("source") or "")
if not (source / "install.sh").is_file():
raise ValueError("This installation does not record its source checkout; "
"run ./install from your Pipelines checkout to update it")
if healthy(f"http://127.0.0.1:{config['server_port']}/healthz"):
active = active_runs(config)
if active is None:
raise RuntimeError("Could not read the server's runs; update after checking `pipelines runs`")
if active:
raise RuntimeError(f"{len(active)} run(s) are still executing; update after they finish")
if (source / ".git").exists():
upstream = subprocess.run(["git", "-C", str(source), "rev-parse", "--abbrev-ref", "@{u}"],
capture_output=True, text=True)
if upstream.returncode:
print("The checkout tracks no upstream branch; installing it as it is.")
else:
pulled = subprocess.run(["git", "-C", str(source), "pull", "--ff-only"], text=True)
if pulled.returncode:
raise RuntimeError("git pull --ff-only failed; resolve the checkout, then rerun `pipelines update`")
installed = subprocess.run(["sh", str(source / "install.sh"), *config.get("install_args", [])])
if installed.returncode:
raise RuntimeError("The installer failed; the previous version keeps running")
config = load(config["prefix"])
stop(config)
start(config)
report(config)
def supervise(config):
root = Path(config["prefix"])
with (root / "supervisor.lock").open("a") as lock:
@@ -434,7 +480,7 @@ def uninstall(config, purge=False, yes=False):
def main():
lifecycle = ("start", "stop", "restart", "status", "doctor", "logs", "supervise", "uninstall")
lifecycle = ("start", "stop", "restart", "status", "doctor", "logs", "supervise", "uninstall", "update")
# Strip only the launcher's installation selector. All application arguments
# belong to the shared Go CLI (including global flags before the command).
selector = argparse.ArgumentParser(add_help=False)
@@ -478,6 +524,8 @@ def main():
print("Host OpenCode API: responding (model execution is a separate check)")
elif args.action == "logs":
os.execvp("tail", ["tail", "-n", "100", "-f", str(Path(config["prefix"]) / "logs/server.log")])
elif args.action == "update":
update(config)
elif args.action == "supervise":
supervise(config)
elif args.action == "uninstall":
+17 -8
View File
@@ -34,12 +34,15 @@ def save_token(output, path, mode=0o444):
return path
def bootstrap_accounts(run, token_path, mode=0o444, log=print):
def bootstrap_accounts(run, token_path, mode=0o444, log=print, publisher_password=None,
reader_password=None):
"""Create the pipeline-publisher and phone-reader accounts on an ntfy server.
``run(args, capture=False, interactive=False)`` executes the ntfy CLI (directly
or through Compose) and returns the completed process. Existing accounts and an
existing token file are preserved on reinstall.
``run(args, capture=False, interactive=False, password=None)`` executes the ntfy
CLI (directly or through Compose) and returns the completed process; a given
password is supplied without a prompt. Without one the CLI asks on the
terminal. Existing accounts and an existing token file are preserved on
reinstall.
"""
token_path = Path(token_path)
if token_path.is_symlink():
@@ -47,14 +50,20 @@ def bootstrap_accounts(run, token_path, mode=0o444, log=print):
if token_path.exists() and token_path.stat().st_size:
log("Keeping existing ntfy publishing token.")
else:
log("Set a password for ntfy's pipeline-publisher account (not a model-provider account).")
run(["user", "add", "--ignore-exists", "pipeline-publisher"], interactive=True)
if publisher_password is None:
log("Set a password for ntfy's pipeline-publisher account (not a model-provider account).")
run(["user", "add", "--ignore-exists", "pipeline-publisher"], interactive=True)
else:
run(["user", "add", "--ignore-exists", "pipeline-publisher"], password=publisher_password)
run(["access", "pipeline-publisher", "pipelines", "write-only"])
result = run(["token", "add", "pipeline-publisher"], capture=True)
save_token(result.stdout + result.stderr, token_path, mode)
# --ignore-exists preserves the phone account's password on reinstall.
log("Set the phone-reader password if this is a new ntfy account; existing accounts are preserved.")
run(["user", "add", "--ignore-exists", "phone-reader"], interactive=True)
if reader_password is None:
log("Set the phone-reader password if this is a new ntfy account; existing accounts are preserved.")
run(["user", "add", "--ignore-exists", "phone-reader"], interactive=True)
else:
run(["user", "add", "--ignore-exists", "phone-reader"], password=reader_password)
run(["access", "phone-reader", "pipelines", "read-only"])
+56 -9
View File
@@ -20,6 +20,7 @@ import platform
import getpass
import plistlib
import secrets
import time
import shlex
import shutil
import subprocess
@@ -61,10 +62,12 @@ DEFAULT_BIND_ADDRESS = "127.0.0.1"
class Setup:
def __init__(self, directory, prefix=None, runner=subprocess.run, resolver=None, environ=None):
def __init__(self, directory, prefix=None, runner=subprocess.run, resolver=None, environ=None,
popen=subprocess.Popen):
self.directory = Path(directory)
self.prefix_argument = prefix
self.runner = runner
self.popen = popen
self.resolver = resolver or release_lookup.resolve_ntfy
self.environ = environ if environ is not None else os.environ
@@ -543,22 +546,54 @@ class Setup:
# --- ntfy users and publishing token --------------------------------
def ntfy(self, prefix, config_path, args, capture=False):
def ntfy(self, prefix, config_path, args, capture=False, password=None):
# The account subcommands locate the server configuration through
# NTFY_CONFIG_FILE; --config is a `serve`-only flag.
env = dict(self.environ)
env["NTFY_CONFIG_FILE"] = str(config_path)
if password is not None:
# `ntfy user add` takes the password from here instead of a prompt.
env["NTFY_PASSWORD"] = password
result = self.runner([str(prefix / "bin/ntfy"), *args], capture_output=capture, text=True, env=env)
if result.returncode:
step = " ".join(args[:2])
raise ValueError(f"ntfy {step} failed; the native install is incomplete")
return result
def bootstrap_ntfy(self, prefix, config_path):
def run_ntfy(args, capture=False, interactive=False):
return self.ntfy(prefix, config_path, list(args), capture=capture)
def ensure_ntfy_auth_db(self, prefix, config_path):
"""ntfy creates its account database the first time its server runs;
the account commands refuse to run before that, so a fresh install
runs it once."""
auth = config_path.parent / "auth.db"
if auth.exists():
return
env = dict(self.environ)
process = self.popen([str(prefix / "bin/ntfy"), "serve", "--config", str(config_path)], env=env)
try:
deadline = time.monotonic() + 20
while not auth.exists():
if process.poll() is not None or time.monotonic() > deadline:
raise ValueError("ntfy did not create its account database; is its port free?")
time.sleep(0.2)
finally:
if process.poll() is None:
process.terminate()
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
process.kill()
process.wait()
tokens.bootstrap_accounts(run_ntfy, prefix / "secrets/publisher-token", 0o600)
def bootstrap_ntfy(self, prefix, config_path, reader_password=None):
def run_ntfy(args, capture=False, interactive=False, password=None):
return self.ntfy(prefix, config_path, list(args), capture=capture, password=password)
self.ensure_ntfy_auth_db(prefix, config_path)
# The publisher authenticates with its token; its password is never
# needed, so it is random. The phone signs in with the web password.
tokens.bootstrap_accounts(run_ntfy, prefix / "secrets/publisher-token", 0o600,
publisher_password=secrets.token_urlsafe(24),
reader_password=reader_password)
# --- Docker migration ------------------------------------------------
@@ -722,8 +757,12 @@ class Setup:
# Migrate existing Docker data before creating native ntfy accounts so a
# migrated auth store (the phone's credentials) is preserved rather than
# shadowed by a freshly bootstrapped one.
self.migrate_docker(prefix, args.replace_data)
self.bootstrap_ntfy(prefix, config_path)
if args.fresh:
print("Starting empty (--fresh); existing Docker volumes are left untouched.")
else:
self.migrate_docker(prefix, args.replace_data)
self.bootstrap_ntfy(prefix, config_path,
reader_password=json.loads(Path(web["auth_file"]).read_text())["password"])
timezone = args.timezone
if not timezone:
try:
@@ -764,6 +803,9 @@ class Setup:
"config": str(config_path), "base_url": base_url,
"version": resolution.get("tag") or resolution.get("version")},
"web": web,
# `pipelines update` re-runs this installer from here, the same way.
"source": str(self.directory),
"install_args": list(getattr(args, "argv", [])),
}
self.register_service(service_path, manifest)
self.write_manifest(prefix, manifest)
@@ -787,6 +829,7 @@ class Setup:
"Check your login service manager and that ports "
f"{args.port}, {args.ntfy_port} and {args.web_port} are free (a running Docker installation may use them).")
print(f"ntfy is configured for {base_url}; server, web proxy and ntfy have login autostart ({service_path}).")
print("Phone notifications: subscribe to the 'pipelines' topic as phone-reader with the web password.")
print(f"Web: http://{lan_ip if args.web_host == '0.0.0.0' else args.web_host}:{args.web_port}/ "
f"(sign-in password in {web['auth_file']}; existing credentials retained).")
print("Login autostart is already enabled. To enable/start it again, run:\n "
@@ -820,9 +863,13 @@ def arguments(argv=None):
help="Explicitly enable pipeline publication; fresh native installs default to enabled")
role.add_argument("--no-authoring", dest="allow_authoring", action="store_false",
help="Execution-only server; disable pipeline publication (reinstalls otherwise preserve the recorded role)")
parser.add_argument("--fresh", action="store_true",
help="Start with empty data even when an earlier Docker installation's volumes exist")
parser.add_argument("--replace-data", action="store_true",
help="Allow Docker migration to replace an existing native database/auth store")
return parser.parse_args(argv)
parsed = parser.parse_args(argv)
parsed.argv = list(sys.argv[1:] if argv is None else argv)
return parsed
if __name__ == "__main__":
+41
View File
@@ -177,6 +177,47 @@ class ManageNativeTests(unittest.TestCase):
self.assertEqual(calls[0], ["launchctl", "enable", f"gui/{os.getuid()}/net.a-wan.pipelines"])
self.assertIn(["launchctl", "kickstart", f"gui/{os.getuid()}/net.a-wan.pipelines"], calls)
def test_update_reinstalls_the_same_way_and_restarts(self):
with tempfile.TemporaryDirectory() as directory:
source = Path(directory) / "src"
source.mkdir()
(source / "install.sh").write_text("#!/bin/sh\n")
config = self.config(Path(directory))
config["source"] = str(source)
config["install_args"] = ["--port", "18180"]
calls = []
done = subprocess.CompletedProcess([], 0)
with patch.object(manage, "healthy", return_value=True), \
patch.object(manage, "active_runs", return_value=[]), \
patch.object(manage.subprocess, "run", side_effect=lambda cmd, **kw: calls.append(cmd) or done), \
patch.object(manage, "load", return_value=config), \
patch.object(manage, "stop") as stop, patch.object(manage, "start") as start, \
patch.object(manage, "report") as report:
manage.update(config)
self.assertEqual(calls[-1], ["sh", str(source / "install.sh"), "--port", "18180"])
stop.assert_called_once()
start.assert_called_once()
report.assert_called_once()
def test_update_waits_for_active_runs(self):
with tempfile.TemporaryDirectory() as directory:
source = Path(directory)
(source / "install.sh").write_text("#!/bin/sh\n")
config = self.config(Path(directory))
config["source"] = str(source)
with patch.object(manage, "healthy", return_value=True), \
patch.object(manage, "active_runs", return_value=[{"id": 1}]), \
patch.object(manage.subprocess, "run") as ran:
with self.assertRaisesRegex(RuntimeError, "still executing"):
manage.update(config)
ran.assert_not_called()
def test_update_needs_a_recorded_source(self):
with tempfile.TemporaryDirectory() as directory:
config = self.config(Path(directory))
with self.assertRaisesRegex(ValueError, "source checkout"):
manage.update(config)
def test_report_prints_the_server_block_and_access(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
+17
View File
@@ -1,3 +1,4 @@
import subprocess
import importlib.util
from pathlib import Path
import tempfile
@@ -68,5 +69,21 @@ class TokenTests(unittest.TestCase):
self.assertEqual(path.stat().st_mode & 0o777, 0o600)
class NonInteractiveAccounts(unittest.TestCase):
def test_given_passwords_are_used_without_prompts(self):
calls = []
def run(args, capture=False, interactive=False, password=None):
calls.append((tuple(args), interactive, password))
out = "token tk_abcdefghijklmnopqrstuvwxyz123 created" if args[0] == "token" else ""
return subprocess.CompletedProcess(args, 0, stdout=out, stderr="")
with tempfile.TemporaryDirectory() as directory:
save_token.bootstrap_accounts(run, Path(directory) / "token", 0o600, log=lambda _m: None,
publisher_password="p1", reader_password="p2")
adds = [(args[3], interactive, password) for args, interactive, password in calls if args[:2] == ("user", "add")]
self.assertEqual(adds, [("pipeline-publisher", False, "p1"), ("phone-reader", False, "p2")])
if __name__ == "__main__":
unittest.main()
+24 -1
View File
@@ -29,6 +29,24 @@ def resolution():
"version": NTFY_VERSION, "tag": "v" + NTFY_VERSION}
class _Served:
def poll(self):
return None
def terminate(self):
pass
def wait(self, timeout=None):
return 0
def fake_ntfy_serve(command, **kwargs):
"""`ntfy serve` creates its account database next to its configuration."""
config = Path(command[command.index("--config") + 1])
(config.parent / "auth.db").write_bytes(b"")
return _Served()
class Recorder:
def __init__(self, token=TOKEN, volumes=(), image="pipelines-pipelines:latest",
fail_create=False, fail_copy=False, fail_volume_ls=False, fail_build=False,
@@ -115,7 +133,7 @@ class SetupNativeTests(unittest.TestCase):
# A disposable XDG config home keeps the client default out of the real
# user configuration during tests.
return setup_native.Setup(directory, prefix=Path(directory) / "prefix", runner=recorder,
resolver=resolution,
resolver=resolution, popen=fake_ntfy_serve,
environ={"XDG_CONFIG_HOME": str(Path(directory) / "config"),
"PIPELINES_WEB_PASSWORD": "chosen-password"})
@@ -236,6 +254,11 @@ class SetupNativeTests(unittest.TestCase):
web = installer.configure_web(prefix, setup_native.arguments([]))
self.assertEqual(json.loads(Path(web["auth_file"]).read_text())["password"], "typed-password")
def test_fresh_install_skips_docker_migration(self):
args = setup_native.arguments(["--fresh"])
self.assertTrue(args.fresh)
self.assertFalse(setup_native.arguments([]).fresh)
def test_invalid_web_auth_is_retained(self):
with tempfile.TemporaryDirectory() as directory:
installer = self.installer(directory, Recorder())