diff --git a/.approval_check/chat_stub_approval.py b/.approval_check/chat_stub_approval.py index 6a8fb474..afbd53df 100644 --- a/.approval_check/chat_stub_approval.py +++ b/.approval_check/chat_stub_approval.py @@ -32,7 +32,7 @@ SESSIONS = [ "title": "Answer an approval", "agent": "build", "model": "deepseek-v4.1", - "directory": "/Users/alan/repos/pipelines", + "directory": "/home/user/project", "created_at": 1760620000000, "updated_at": 1760620600000, "busy": False, @@ -42,7 +42,7 @@ SESSIONS = [ "title": "Already resolved", "agent": "build", "model": "deepseek-v4.1", - "directory": "/Users/alan/repos/pipelines", + "directory": "/home/user/project", "created_at": 1760620000000, "updated_at": 1760620600000, "busy": False, diff --git a/app/tool/chat_sessions_probe_entry.dart b/app/tool/chat_sessions_probe_entry.dart index 6b9bcd85..3b3ef3e9 100644 --- a/app/tool/chat_sessions_probe_entry.dart +++ b/app/tool/chat_sessions_probe_entry.dart @@ -58,7 +58,7 @@ class _FixtureGateway implements ApiGateway { 'chat_directories': [ '/workspace/demo', '/workspace/archive', - '/Users/alan/repos/pipelines/tmp/flutter-chat-list', + '/home/user/project', ], }; case 'chat_backends': diff --git a/app/tool/check_full.sh b/app/tool/check_full.sh index 78536b92..c2c55bad 100644 --- a/app/tool/check_full.sh +++ b/app/tool/check_full.sh @@ -7,7 +7,7 @@ # verdict comes from the suite's own final line, which is echoed with the # totals. Logs go to build/check/ for the caller's evidence. set -u -FLUTTER="${FLUTTER:-/Users/alan/development/flutter/bin/flutter}" +FLUTTER="${FLUTTER:-flutter}" mkdir -p build/check "$FLUTTER" analyze > build/check/analyze.log 2>&1 if ! grep -q "No issues found" build/check/analyze.log; then diff --git a/app/tool/publish_apk.sh b/app/tool/publish_apk.sh index 23f95df1..f4479504 100644 --- a/app/tool/publish_apk.sh +++ b/app/tool/publish_apk.sh @@ -5,21 +5,21 @@ # The build number is minutes since the epoch, so every build is newer than # the last. The APK and version.json go to build/updates/ (kept across preview # publishes) and are copied into the live preview at build/web/updates/, -# served behind the web login at https://:8096/updates/. Never restarts +# served behind the web login at /updates/. Never restarts # any server. set -eu notes="${1:-}" -# The phone's default server: the web proxy over Tailscale (sign in once in -# Settings). Override with PIPELINES_SERVER_URL. -server="${PIPELINES_SERVER_URL:-wss://alans-macbook-pro.tail6bf4dc.ts.net:8443/connect}" -FLUTTER="${FLUTTER:-/Users/alan/development/flutter/bin/flutter}" +# The phone's default server, e.g. wss://host:8443/connect (required). An +# installation builds its APK with ./install --apk instead. +server="${PIPELINES_SERVER_URL:?set PIPELINES_SERVER_URL to the phone default server, e.g. wss://host/connect}" +FLUTTER="${FLUTTER:-flutter}" export JAVA_HOME="${JAVA_HOME:-/Applications/Android Studio.app/Contents/jbr/Contents/Home}" export ANDROID_HOME="${ANDROID_HOME:-$HOME/Library/Android/sdk}" code=$(( $(date +%s) / 60 )) name="0.1.$code" "$FLUTTER" build apk --release --build-number="$code" --build-name="$name" \ --dart-define=PIPELINES_SERVER_URL="$server" \ - --dart-define=PIPELINES_UPDATES_URL="${PIPELINES_UPDATES_URL:-https://alans-macbook-pro.tail6bf4dc.ts.net:8096/updates/}" + ${PIPELINES_UPDATES_URL:+--dart-define=PIPELINES_UPDATES_URL="$PIPELINES_UPDATES_URL"} mkdir -p build/updates cp build/app/outputs/flutter-apk/app-release.apk build/updates/pipelines.apk cat > build/updates/version.json <[ 'id': 457, 'pipeline_name': 'Plan and land', 'status': 'succeeded', - 'workspace': '/Users/alan/repos/pipelines', + 'workspace': '/home/user/project', 'created_at': '2026-10-20T08:12:00Z', 'finished_at': '2026-10-20T08:42:00Z', }, diff --git a/cmd/pipelines-server/banner.go b/cmd/pipelines-server/banner.go index 288e0903..f32da007 100644 --- a/cmd/pipelines-server/banner.go +++ b/cmd/pipelines-server/banner.go @@ -77,8 +77,6 @@ func banner(w io.Writer, s startup) { } if s.ClaudeToken { row("Claude", "service token present") - } else { - row("Claude", "no service token; as a background service run `claude setup-token` and save it to ~/.config/pipelines/claude-token (chmod 600)") } if s.HarnessErr != nil { row("Harnesses", "unreadable: "+s.HarnessErr.Error()) diff --git a/docs/install.md b/docs/install.md index 87431b8c..a057736f 100644 --- a/docs/install.md +++ b/docs/install.md @@ -14,8 +14,22 @@ cd pipelines ./install --docker # explicit Docker server + ntfy ./install --web # rebuild the web app (Docker) after app changes ./install --apk # optional: build the Android app (Docker, slower) +pipelines update # later: pull, reinstall the same way, restart ``` +## Updating + +`pipelines update` brings an installation to the newest source. It waits while +runs are executing, fast-forwards the checkout it was installed from +(`git pull --ff-only`), re-runs `./install` with the options recorded at install +time, restarts the server, web proxy and ntfy, and prints the startup report. +Data, the web password, ntfy accounts and the Android signing key are kept; the +web app and APK are rebuilt only when their sources changed. Phones pick up a +new APK (after `./install --apk`) from Settings → App updates. + +`./install --fresh` starts with empty data even when volumes from an earlier +Docker installation exist (by default they are migrated). + ## Native installation (default) `./install`, `./install --native` and `./install native` run diff --git a/internal/cli/commands.go b/internal/cli/commands.go index d94e9cb7..4daef5f2 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -72,5 +72,6 @@ var serviceHelp = map[string]string{ "restart": "Stop, then start both services, retaining definitions, run history, and credentials.", "doctor": "Check both health endpoints and host OpenCode API discovery; does not execute a model.", "logs": "Follow the native server log. Press Ctrl+C to stop following.", + "update": "Update this installation: wait for executing runs, pull the source checkout it was installed from, re-run its installer the same way (data and credentials kept), restart, and print the startup report.", "uninstall": "Remove verified Go binaries from this command's bin directory; retain data and drafts. --yes skips confirmation. Existing managed installations can be selected explicitly with PIPELINES_PREFIX.", } diff --git a/internal/cli/help.go b/internal/cli/help.go index 435b8238..03f491c9 100644 --- a/internal/cli/help.go +++ b/internal/cli/help.go @@ -53,6 +53,7 @@ SERVER / INTERFACE service enable Configure user autostart service disable | status Stop/remove autostart, or inspect OS service status skill install Install/update the shared OpenCode and Claude Code workflow skill + update Update this installation from its source checkout uninstall [--yes] Remove verified Go binaries; retain data and drafts Legacy managed installs: select PIPELINES_PREFIX explicitly for start/stop, diff --git a/internal/installation/installation.go b/internal/installation/installation.go index 32f0ded4..9859ce0d 100644 --- a/internal/installation/installation.go +++ b/internal/installation/installation.go @@ -100,7 +100,7 @@ func (c Config) Environment() error { func Service(action string) bool { switch action { - case "start", "stop", "restart", "doctor", "logs", "uninstall": + case "start", "stop", "restart", "doctor", "logs", "uninstall", "update": return true } return false diff --git a/scripts/manage.py b/scripts/manage.py index ae74db17..b3d56e61 100644 --- a/scripts/manage.py +++ b/scripts/manage.py @@ -309,6 +309,52 @@ def report(config): print("==") +def active_runs(config): + """Runs the server is still executing; an update waits for them.""" + binary = str(Path(config["prefix"]) / "bin/pipelines") + result = run([binary, "--url", f"ws://127.0.0.1:{config['server_port']}/connect", "runs", "--json"], + quiet=True, check=False, timeout=60) + if result.returncode: + return None + try: + data = json.loads(result.stdout) + except ValueError: + return None + runs = data.get("runs", []) if isinstance(data, dict) else data + return [item for item in runs if isinstance(item, dict) and item.get("executing")] + + +def update(config): + """Brings this installation to the newest source: pull, reinstall the same + way it was installed (data and credentials kept), restart, report.""" + source = Path(config.get("source") or "") + if not (source / "install.sh").is_file(): + raise ValueError("This installation does not record its source checkout; " + "run ./install from your Pipelines checkout to update it") + if healthy(f"http://127.0.0.1:{config['server_port']}/healthz"): + active = active_runs(config) + if active is None: + raise RuntimeError("Could not read the server's runs; update after checking `pipelines runs`") + if active: + raise RuntimeError(f"{len(active)} run(s) are still executing; update after they finish") + if (source / ".git").exists(): + upstream = subprocess.run(["git", "-C", str(source), "rev-parse", "--abbrev-ref", "@{u}"], + capture_output=True, text=True) + if upstream.returncode: + print("The checkout tracks no upstream branch; installing it as it is.") + else: + pulled = subprocess.run(["git", "-C", str(source), "pull", "--ff-only"], text=True) + if pulled.returncode: + raise RuntimeError("git pull --ff-only failed; resolve the checkout, then rerun `pipelines update`") + installed = subprocess.run(["sh", str(source / "install.sh"), *config.get("install_args", [])]) + if installed.returncode: + raise RuntimeError("The installer failed; the previous version keeps running") + config = load(config["prefix"]) + stop(config) + start(config) + report(config) + + def supervise(config): root = Path(config["prefix"]) with (root / "supervisor.lock").open("a") as lock: @@ -434,7 +480,7 @@ def uninstall(config, purge=False, yes=False): def main(): - lifecycle = ("start", "stop", "restart", "status", "doctor", "logs", "supervise", "uninstall") + lifecycle = ("start", "stop", "restart", "status", "doctor", "logs", "supervise", "uninstall", "update") # Strip only the launcher's installation selector. All application arguments # belong to the shared Go CLI (including global flags before the command). selector = argparse.ArgumentParser(add_help=False) @@ -478,6 +524,8 @@ def main(): print("Host OpenCode API: responding (model execution is a separate check)") elif args.action == "logs": os.execvp("tail", ["tail", "-n", "100", "-f", str(Path(config["prefix"]) / "logs/server.log")]) + elif args.action == "update": + update(config) elif args.action == "supervise": supervise(config) elif args.action == "uninstall": diff --git a/scripts/save-ntfy-token.py b/scripts/save-ntfy-token.py index 42ea320a..d4b781ac 100644 --- a/scripts/save-ntfy-token.py +++ b/scripts/save-ntfy-token.py @@ -34,12 +34,15 @@ def save_token(output, path, mode=0o444): return path -def bootstrap_accounts(run, token_path, mode=0o444, log=print): +def bootstrap_accounts(run, token_path, mode=0o444, log=print, publisher_password=None, + reader_password=None): """Create the pipeline-publisher and phone-reader accounts on an ntfy server. - ``run(args, capture=False, interactive=False)`` executes the ntfy CLI (directly - or through Compose) and returns the completed process. Existing accounts and an - existing token file are preserved on reinstall. + ``run(args, capture=False, interactive=False, password=None)`` executes the ntfy + CLI (directly or through Compose) and returns the completed process; a given + password is supplied without a prompt. Without one the CLI asks on the + terminal. Existing accounts and an existing token file are preserved on + reinstall. """ token_path = Path(token_path) if token_path.is_symlink(): @@ -47,14 +50,20 @@ def bootstrap_accounts(run, token_path, mode=0o444, log=print): if token_path.exists() and token_path.stat().st_size: log("Keeping existing ntfy publishing token.") else: - log("Set a password for ntfy's pipeline-publisher account (not a model-provider account).") - run(["user", "add", "--ignore-exists", "pipeline-publisher"], interactive=True) + if publisher_password is None: + log("Set a password for ntfy's pipeline-publisher account (not a model-provider account).") + run(["user", "add", "--ignore-exists", "pipeline-publisher"], interactive=True) + else: + run(["user", "add", "--ignore-exists", "pipeline-publisher"], password=publisher_password) run(["access", "pipeline-publisher", "pipelines", "write-only"]) result = run(["token", "add", "pipeline-publisher"], capture=True) save_token(result.stdout + result.stderr, token_path, mode) # --ignore-exists preserves the phone account's password on reinstall. - log("Set the phone-reader password if this is a new ntfy account; existing accounts are preserved.") - run(["user", "add", "--ignore-exists", "phone-reader"], interactive=True) + if reader_password is None: + log("Set the phone-reader password if this is a new ntfy account; existing accounts are preserved.") + run(["user", "add", "--ignore-exists", "phone-reader"], interactive=True) + else: + run(["user", "add", "--ignore-exists", "phone-reader"], password=reader_password) run(["access", "phone-reader", "pipelines", "read-only"]) diff --git a/scripts/setup-native.py b/scripts/setup-native.py index 598cc8d5..340b86d6 100644 --- a/scripts/setup-native.py +++ b/scripts/setup-native.py @@ -20,6 +20,7 @@ import platform import getpass import plistlib import secrets +import time import shlex import shutil import subprocess @@ -61,10 +62,12 @@ DEFAULT_BIND_ADDRESS = "127.0.0.1" class Setup: - def __init__(self, directory, prefix=None, runner=subprocess.run, resolver=None, environ=None): + def __init__(self, directory, prefix=None, runner=subprocess.run, resolver=None, environ=None, + popen=subprocess.Popen): self.directory = Path(directory) self.prefix_argument = prefix self.runner = runner + self.popen = popen self.resolver = resolver or release_lookup.resolve_ntfy self.environ = environ if environ is not None else os.environ @@ -543,22 +546,54 @@ class Setup: # --- ntfy users and publishing token -------------------------------- - def ntfy(self, prefix, config_path, args, capture=False): + def ntfy(self, prefix, config_path, args, capture=False, password=None): # The account subcommands locate the server configuration through # NTFY_CONFIG_FILE; --config is a `serve`-only flag. env = dict(self.environ) env["NTFY_CONFIG_FILE"] = str(config_path) + if password is not None: + # `ntfy user add` takes the password from here instead of a prompt. + env["NTFY_PASSWORD"] = password result = self.runner([str(prefix / "bin/ntfy"), *args], capture_output=capture, text=True, env=env) if result.returncode: step = " ".join(args[:2]) raise ValueError(f"ntfy {step} failed; the native install is incomplete") return result - def bootstrap_ntfy(self, prefix, config_path): - def run_ntfy(args, capture=False, interactive=False): - return self.ntfy(prefix, config_path, list(args), capture=capture) + def ensure_ntfy_auth_db(self, prefix, config_path): + """ntfy creates its account database the first time its server runs; + the account commands refuse to run before that, so a fresh install + runs it once.""" + auth = config_path.parent / "auth.db" + if auth.exists(): + return + env = dict(self.environ) + process = self.popen([str(prefix / "bin/ntfy"), "serve", "--config", str(config_path)], env=env) + try: + deadline = time.monotonic() + 20 + while not auth.exists(): + if process.poll() is not None or time.monotonic() > deadline: + raise ValueError("ntfy did not create its account database; is its port free?") + time.sleep(0.2) + finally: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait() - tokens.bootstrap_accounts(run_ntfy, prefix / "secrets/publisher-token", 0o600) + def bootstrap_ntfy(self, prefix, config_path, reader_password=None): + def run_ntfy(args, capture=False, interactive=False, password=None): + return self.ntfy(prefix, config_path, list(args), capture=capture, password=password) + + self.ensure_ntfy_auth_db(prefix, config_path) + # The publisher authenticates with its token; its password is never + # needed, so it is random. The phone signs in with the web password. + tokens.bootstrap_accounts(run_ntfy, prefix / "secrets/publisher-token", 0o600, + publisher_password=secrets.token_urlsafe(24), + reader_password=reader_password) # --- Docker migration ------------------------------------------------ @@ -722,8 +757,12 @@ class Setup: # Migrate existing Docker data before creating native ntfy accounts so a # migrated auth store (the phone's credentials) is preserved rather than # shadowed by a freshly bootstrapped one. - self.migrate_docker(prefix, args.replace_data) - self.bootstrap_ntfy(prefix, config_path) + if args.fresh: + print("Starting empty (--fresh); existing Docker volumes are left untouched.") + else: + self.migrate_docker(prefix, args.replace_data) + self.bootstrap_ntfy(prefix, config_path, + reader_password=json.loads(Path(web["auth_file"]).read_text())["password"]) timezone = args.timezone if not timezone: try: @@ -764,6 +803,9 @@ class Setup: "config": str(config_path), "base_url": base_url, "version": resolution.get("tag") or resolution.get("version")}, "web": web, + # `pipelines update` re-runs this installer from here, the same way. + "source": str(self.directory), + "install_args": list(getattr(args, "argv", [])), } self.register_service(service_path, manifest) self.write_manifest(prefix, manifest) @@ -787,6 +829,7 @@ class Setup: "Check your login service manager and that ports " f"{args.port}, {args.ntfy_port} and {args.web_port} are free (a running Docker installation may use them).") print(f"ntfy is configured for {base_url}; server, web proxy and ntfy have login autostart ({service_path}).") + print("Phone notifications: subscribe to the 'pipelines' topic as phone-reader with the web password.") print(f"Web: http://{lan_ip if args.web_host == '0.0.0.0' else args.web_host}:{args.web_port}/ " f"(sign-in password in {web['auth_file']}; existing credentials retained).") print("Login autostart is already enabled. To enable/start it again, run:\n " @@ -820,9 +863,13 @@ def arguments(argv=None): help="Explicitly enable pipeline publication; fresh native installs default to enabled") role.add_argument("--no-authoring", dest="allow_authoring", action="store_false", help="Execution-only server; disable pipeline publication (reinstalls otherwise preserve the recorded role)") + parser.add_argument("--fresh", action="store_true", + help="Start with empty data even when an earlier Docker installation's volumes exist") parser.add_argument("--replace-data", action="store_true", help="Allow Docker migration to replace an existing native database/auth store") - return parser.parse_args(argv) + parsed = parser.parse_args(argv) + parsed.argv = list(sys.argv[1:] if argv is None else argv) + return parsed if __name__ == "__main__": diff --git a/scripts/test_manage_native.py b/scripts/test_manage_native.py index 0f248de7..f2e262ff 100644 --- a/scripts/test_manage_native.py +++ b/scripts/test_manage_native.py @@ -177,6 +177,47 @@ class ManageNativeTests(unittest.TestCase): self.assertEqual(calls[0], ["launchctl", "enable", f"gui/{os.getuid()}/net.a-wan.pipelines"]) self.assertIn(["launchctl", "kickstart", f"gui/{os.getuid()}/net.a-wan.pipelines"], calls) + def test_update_reinstalls_the_same_way_and_restarts(self): + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "src" + source.mkdir() + (source / "install.sh").write_text("#!/bin/sh\n") + config = self.config(Path(directory)) + config["source"] = str(source) + config["install_args"] = ["--port", "18180"] + calls = [] + done = subprocess.CompletedProcess([], 0) + with patch.object(manage, "healthy", return_value=True), \ + patch.object(manage, "active_runs", return_value=[]), \ + patch.object(manage.subprocess, "run", side_effect=lambda cmd, **kw: calls.append(cmd) or done), \ + patch.object(manage, "load", return_value=config), \ + patch.object(manage, "stop") as stop, patch.object(manage, "start") as start, \ + patch.object(manage, "report") as report: + manage.update(config) + self.assertEqual(calls[-1], ["sh", str(source / "install.sh"), "--port", "18180"]) + stop.assert_called_once() + start.assert_called_once() + report.assert_called_once() + + def test_update_waits_for_active_runs(self): + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) + (source / "install.sh").write_text("#!/bin/sh\n") + config = self.config(Path(directory)) + config["source"] = str(source) + with patch.object(manage, "healthy", return_value=True), \ + patch.object(manage, "active_runs", return_value=[{"id": 1}]), \ + patch.object(manage.subprocess, "run") as ran: + with self.assertRaisesRegex(RuntimeError, "still executing"): + manage.update(config) + ran.assert_not_called() + + def test_update_needs_a_recorded_source(self): + with tempfile.TemporaryDirectory() as directory: + config = self.config(Path(directory)) + with self.assertRaisesRegex(ValueError, "source checkout"): + manage.update(config) + def test_report_prints_the_server_block_and_access(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/scripts/test_save_ntfy_token.py b/scripts/test_save_ntfy_token.py index ad0d2aac..fbe74784 100644 --- a/scripts/test_save_ntfy_token.py +++ b/scripts/test_save_ntfy_token.py @@ -1,3 +1,4 @@ +import subprocess import importlib.util from pathlib import Path import tempfile @@ -68,5 +69,21 @@ class TokenTests(unittest.TestCase): self.assertEqual(path.stat().st_mode & 0o777, 0o600) +class NonInteractiveAccounts(unittest.TestCase): + def test_given_passwords_are_used_without_prompts(self): + calls = [] + + def run(args, capture=False, interactive=False, password=None): + calls.append((tuple(args), interactive, password)) + out = "token tk_abcdefghijklmnopqrstuvwxyz123 created" if args[0] == "token" else "" + return subprocess.CompletedProcess(args, 0, stdout=out, stderr="") + + with tempfile.TemporaryDirectory() as directory: + save_token.bootstrap_accounts(run, Path(directory) / "token", 0o600, log=lambda _m: None, + publisher_password="p1", reader_password="p2") + adds = [(args[3], interactive, password) for args, interactive, password in calls if args[:2] == ("user", "add")] + self.assertEqual(adds, [("pipeline-publisher", False, "p1"), ("phone-reader", False, "p2")]) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/test_setup_native.py b/scripts/test_setup_native.py index 01f1e713..e5c41be9 100644 --- a/scripts/test_setup_native.py +++ b/scripts/test_setup_native.py @@ -29,6 +29,24 @@ def resolution(): "version": NTFY_VERSION, "tag": "v" + NTFY_VERSION} +class _Served: + def poll(self): + return None + + def terminate(self): + pass + + def wait(self, timeout=None): + return 0 + + +def fake_ntfy_serve(command, **kwargs): + """`ntfy serve` creates its account database next to its configuration.""" + config = Path(command[command.index("--config") + 1]) + (config.parent / "auth.db").write_bytes(b"") + return _Served() + + class Recorder: def __init__(self, token=TOKEN, volumes=(), image="pipelines-pipelines:latest", fail_create=False, fail_copy=False, fail_volume_ls=False, fail_build=False, @@ -115,7 +133,7 @@ class SetupNativeTests(unittest.TestCase): # A disposable XDG config home keeps the client default out of the real # user configuration during tests. return setup_native.Setup(directory, prefix=Path(directory) / "prefix", runner=recorder, - resolver=resolution, + resolver=resolution, popen=fake_ntfy_serve, environ={"XDG_CONFIG_HOME": str(Path(directory) / "config"), "PIPELINES_WEB_PASSWORD": "chosen-password"}) @@ -236,6 +254,11 @@ class SetupNativeTests(unittest.TestCase): web = installer.configure_web(prefix, setup_native.arguments([])) self.assertEqual(json.loads(Path(web["auth_file"]).read_text())["password"], "typed-password") + def test_fresh_install_skips_docker_migration(self): + args = setup_native.arguments(["--fresh"]) + self.assertTrue(args.fresh) + self.assertFalse(setup_native.arguments([]).fresh) + def test_invalid_web_auth_is_retained(self): with tempfile.TemporaryDirectory() as directory: installer = self.installer(directory, Recorder())