From 5ca6de310e338329d918d9d964e09abac696c983 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Thu, 12 Nov 2026 01:00:46 +0000 Subject: [PATCH] Quiet native installer output --- scripts/app_build.py | 5 ++ scripts/setup-native.py | 120 ++++++++++++++++++++-------------- scripts/test_manage_native.py | 21 ++++++ 3 files changed, 97 insertions(+), 49 deletions(-) diff --git a/scripts/app_build.py b/scripts/app_build.py index b0c0a05a..e51250a3 100644 --- a/scripts/app_build.py +++ b/scripts/app_build.py @@ -48,6 +48,7 @@ SKIPPED = {"build", ".dart_tool", ".gradle", ".cxx", "local.properties", ".DS_St # generated cannot be mistaken for complete web releases. WEB_CACHE_SCHEMA = "web-release-metadata-v1" WEB_RELEASE_SCHEMA = 1 +VERBOSE = False def flutter_version(app=APP): @@ -103,6 +104,7 @@ def web_cache_complete(entry): def docker(runner, *args, quiet=False): + quiet = quiet or not VERBOSE result = runner(["docker", *args], capture_output=quiet, text=True) if result.returncode: detail = (result.stderr or "").strip().splitlines()[-1:] if quiet else [] @@ -410,12 +412,15 @@ def main(argv=None): parser.add_argument("--publish", action="store_true", help="Publish desktop archive and update feed to this installation") parser.add_argument("--output", type=Path, help="Desktop archive output directory (default: build cache)") parser.add_argument("--flutter", help="Native Flutter executable for macOS and Android (or PIPELINES_FLUTTER/PATH)") + parser.add_argument("--verbose", action="store_true", help="Show full build and tool output") build_mode = parser.add_mutually_exclusive_group() build_mode.add_argument("--native", dest="native", action="store_true", default=None, help="APK: build on this machine with the host Flutter and Android SDK") build_mode.add_argument("--docker", dest="native", action="store_false", help="APK: build in Docker even when a host toolchain exists") args = parser.parse_args(argv) + global VERBOSE + VERBOSE = args.verbose if args.kind in ("linux", "macos"): spec = importlib.util.spec_from_file_location("pipelines_desktop_build", Path(__file__).with_name("app_desktop_build.py")) desktop = importlib.util.module_from_spec(spec) diff --git a/scripts/setup-native.py b/scripts/setup-native.py index 6a3a6b16..7c343d28 100644 --- a/scripts/setup-native.py +++ b/scripts/setup-native.py @@ -72,6 +72,29 @@ class Setup: self.popen = popen self.resolver = resolver or release_lookup.resolve_ntfy self.environ = environ if environ is not None else os.environ + self.verbose = False + self.install_log = None + + def command(self, arguments, **kwargs): + """Run an installer command, keeping tool output out of normal output.""" + if self.verbose: + return self.runner(arguments, **kwargs) + kwargs = dict(kwargs) + kwargs.update(capture_output=True, text=True) + result = self.runner(arguments, **kwargs) + if self.install_log: + self.install_log.parent.mkdir(parents=True, exist_ok=True) + with self.install_log.open("a") as log: + log.write(f"$ {shlex.join(map(str, arguments))}\n") + if result.stdout: + log.write(result.stdout) + if result.stderr: + log.write(result.stderr) + log.write(f"[exit {result.returncode}]\n") + return result + + def step(self, text): + print(f"• {text}") # --- detection ------------------------------------------------------- @@ -261,7 +284,7 @@ class Setup: for field, flag in (("model", "--model"), ("pipeline_model", "--pipeline-model")): if self._valid_model(current.get(field)): command.extend([flag, current[field]]) - result = self.runner(command, cwd=str(Path.home()), env=dict(self.environ)) + result = self.command(command, cwd=str(Path.home()), env=dict(self.environ)) if result.returncode: raise ValueError("Model selection did not complete; native setup is not ready. Rerun ./install or supply --model PROVIDER/MODEL.") return self.configure_harnesses(harness, model, allow_picker=False) @@ -340,7 +363,7 @@ class Setup: # Tool-manager shims dispatch by their invoked name (for example Volta). # Resolving the symlink would run volta-shim instead of node. node = str(Path(shutil.which("node")).absolute()) - version = self.runner([node, "--version"], capture_output=True, text=True) + version = self.command([node, "--version"], capture_output=True, text=True) try: major = int(version.stdout.strip().removeprefix("v").split(".")[0]) except (ValueError, AttributeError): @@ -425,7 +448,7 @@ class Setup: raise ValueError("Docker is not running, so the web app cannot be built. The server is never " "installed without it; start Docker and rerun ./install.") try: - built = app_build.build("web", runner=self.runner, + built = app_build.build("web", runner=self.command, arguments=app_build.update_signing.dart_defines(self.environ)) except RuntimeError as error: raise ValueError(f"The web app build failed ({error}); nothing was installed") from None @@ -453,7 +476,7 @@ class Setup: plist = Path.home() / f"Library/LaunchAgents/{self.LEGACY_WEB_LABEL}.plist" if not plist.is_file(): return - self.runner(["launchctl", "bootout", f"gui/{os.getuid()}/{self.LEGACY_WEB_LABEL}"], + self.command(["launchctl", "bootout", f"gui/{os.getuid()}/{self.LEGACY_WEB_LABEL}"], capture_output=True, text=True) plist.rename(plist.with_name(plist.name + ".disabled")) print(f"Retired the old standalone web agent {self.LEGACY_WEB_LABEL} (kept as {plist.name}.disabled).") @@ -468,7 +491,7 @@ class Setup: if not tailscale: raise ValueError(f"The public web URL {public_url} is a tailnet address but tailscale is not on PATH") port = parsed.port or 443 - result = self.runner([tailscale, "serve", "--bg", f"--https={port}", f"http://127.0.0.1:{web_port}"], + result = self.command([tailscale, "serve", "--bg", f"--https={port}", f"http://127.0.0.1:{web_port}"], capture_output=True, text=True) if result.returncode: raise ValueError(f"tailscale serve could not point {public_url} at the web app: " @@ -488,7 +511,7 @@ class Setup: shutil.rmtree(staging, ignore_errors=True) staging.mkdir(parents=True) for command in self.COMMANDS: - result = self.runner( + result = self.command( ["go", "build", "-trimpath", "-o", str(staging / command), "./cmd/" + command], cwd=str(self.directory)) if result.returncode: @@ -557,7 +580,7 @@ class Setup: return f"{version} (commit {commit})" if commit else str(version) def install_skill(self, prefix): - result = self.runner([str(prefix / "bin/pipelines"), "skill", "install"], cwd=str(self.directory)) + result = self.command([str(prefix / "bin/pipelines"), "skill", "install"], cwd=str(self.directory)) if result.returncode: raise ValueError("pipelines skill install failed; the native install is incomplete") @@ -588,10 +611,10 @@ class Setup: return path self.require_compiler() pinned = f"{module}@{version}" - result = self.runner(["go", "mod", "download", pinned], cwd=str(self.directory)) + result = self.command(["go", "mod", "download", pinned], cwd=str(self.directory)) if result.returncode: raise ValueError(f"go mod download {pinned} failed; the native install is incomplete") - cache = self.runner(["go", "env", "GOMODCACHE"], cwd=str(self.directory), + cache = self.command(["go", "env", "GOMODCACHE"], cwd=str(self.directory), capture_output=True, text=True) if cache.returncode or not (cache.stdout or "").strip(): raise ValueError("Could not locate the Go module cache to build ntfy from") @@ -606,7 +629,7 @@ class Setup: shutil.copytree(source, checkout, symlinks=True) # The module cache is read-only and carries no built web assets; the # upstream Makefile creates the same placeholders for local builds. - self.runner(["chmod", "-R", "u+w", str(checkout)]) + self.command(["chmod", "-R", "u+w", str(checkout)]) (checkout / "server/docs").mkdir(parents=True, exist_ok=True) (checkout / "server/site").mkdir(parents=True, exist_ok=True) (checkout / "server/docs/index.html").touch() @@ -614,7 +637,7 @@ class Setup: built = Path(staging) / "ntfy" env = dict(self.environ) env["CGO_ENABLED"] = "1" - build = self.runner( + build = self.command( ["go", "build", "-trimpath", "-tags", NTFY_BUILD_TAGS, "-ldflags", f"-s -w -X main.version={version}", "-o", str(built), "."], cwd=str(checkout), env=env) @@ -670,7 +693,7 @@ class Setup: if password is not None: # `ntfy user add` takes the password from here instead of a prompt. env["NTFY_PASSWORD"] = password - result = self.runner([str(prefix / "bin/ntfy"), *args], capture_output=capture, text=True, env=env) + result = self.command([str(prefix / "bin/ntfy"), *args], capture_output=capture, text=True, env=env) if result.returncode: step = " ".join(args[:2]) raise ValueError(f"ntfy {step} failed; the native install is incomplete") @@ -726,7 +749,7 @@ class Setup: def docker_volumes(self): try: - result = self.runner(["docker", "volume", "ls", "--format", "{{.Name}}"], + result = self.command(["docker", "volume", "ls", "--format", "{{.Name}}"], capture_output=True, text=True) except FileNotFoundError: # No Docker CLI at all: there is no Docker installation to migrate. @@ -743,7 +766,7 @@ class Setup: return tuple(name for name in ("pipelines_pipelines-data", "pipelines_ntfy-data") if name in names) def _migration_image(self): - result = self.runner(["docker", "image", "ls", "--format", "{{.Repository}}:{{.Tag}}"], + result = self.command(["docker", "image", "ls", "--format", "{{.Repository}}:{{.Tag}}"], capture_output=True, text=True) if result.returncode: return None @@ -759,12 +782,12 @@ class Setup: print(f"Keeping existing native {destination.name}; not overwriting it during migration.") return False name = f"pipelines-migrate-{os.getpid()}" - created = self.runner(["docker", "create", "--name", name, "--volume", f"{volume}:/mnt:ro", image], + created = self.command(["docker", "create", "--name", name, "--volume", f"{volume}:/mnt:ro", image], capture_output=True, text=True) if created.returncode: raise ValueError(f"Could not open Docker volume {volume} for migration; install aborted") try: - result = self.runner(["docker", "cp", f"{name}:/mnt/{inner}", str(destination)], + result = self.command(["docker", "cp", f"{name}:/mnt/{inner}", str(destination)], capture_output=True, text=True) if result.returncode: raise ValueError(f"Could not copy {inner} from Docker volume {volume}; install aborted") @@ -774,12 +797,12 @@ class Setup: for suffix in ("-wal",): sidecar = Path(str(destination) + suffix) sidecar.unlink(missing_ok=True) - copy = self.runner(["docker", "cp", f"{name}:/mnt/{inner}{suffix}", str(sidecar)], + copy = self.command(["docker", "cp", f"{name}:/mnt/{inner}{suffix}", str(sidecar)], capture_output=True, text=True) if copy.returncode or not sidecar.exists() or sidecar.stat().st_size == 0: sidecar.unlink(missing_ok=True) finally: - self.runner(["docker", "rm", "-f", name], capture_output=True, text=True, check=False) + self.command(["docker", "rm", "-f", name], capture_output=True, text=True, check=False) return True def migrate_docker(self, prefix, replace=False): @@ -871,9 +894,12 @@ class Setup: resolution = self.resolve_ntfy() # All or nothing: the web app and the server are both built before # either is installed, and a failed build leaves the old pair running. + self.step("Building the web app") web_build = self.build_app_web() self.prepare_layout(prefix) + self.step("Building Pipelines") self.build(prefix) + self.step("Configuring the installation") web = self.configure_web(prefix, args, web_build) self.install_binaries(prefix) self.check_path_shadow(prefix) @@ -909,7 +935,7 @@ class Setup: if self.environ.get(key): opencode_env[key] = self.environ[key] opencode_env["PIPELINES_DEFAULT_MODEL"] = args.model or pipeline_model - web_url = f"http://{web_access.default_url_host(web, lan_ip, web_access.tailnet_ipv4(self.runner))}:{args.web_port}" + web_url = f"http://{web_access.default_url_host(web, lan_ip, web_access.tailnet_ipv4(self.command))}:{args.web_port}" opencode_env["PIPELINES_WEB_PUBLIC_URL"] = web_public_url or web_url path_value = str(prefix / "bin") + os.pathsep + self.environ.get("PATH", "") manifest = { @@ -949,14 +975,9 @@ class Setup: # Docker address. endpoint = f"ws://127.0.0.1:{args.port}/connect" client_path = configure.configure_client(endpoint, self.environ) - print(f"Saved native client default: {endpoint} ({client_path}). Restart the TUI.") - override = self.environ.get("PIPELINES_URL", "") - if override and override != endpoint: - print("WARNING: PIPELINES_URL overrides the saved native address; unset it or pass --url explicitly.") - print(f"Native Pipelines installed at {prefix}." - f" ntfy {resolution.get('tag') or resolution.get('version')} was built from source.") + self.step("Starting the services") # Parity with the Docker installer: leave the server and ntfy running. - started = self.runner([python, str(prefix / "lib/manage.py"), "start", "--prefix", str(prefix)]) + started = self.command([python, str(prefix / "lib/manage.py"), "start", "--prefix", str(prefix)]) if started.returncode: raise ValueError( "The native services were installed but could not be started; run " @@ -965,29 +986,18 @@ class Setup: f"{args.port}, {args.ntfy_port} and {args.web_port} are free (a running Docker installation may use them).") self.retire_legacy_web() self.serve_on_tailnet(opencode_env["PIPELINES_WEB_PUBLIC_URL"], args.web_port) - print(f"ntfy is configured for {base_url}; server, web proxy and ntfy have login autostart ({service_path}).") - print("Phone notifications: subscribe to the 'pipelines' topic as phone-reader with the web password.") - print(f"Web: {web_url}/ (sign-in password in {web['auth_file']}; existing credentials retained).") - print("Web proxy access: " + ("loopback + tailnet" + (" + LAN" if args.web_lan else "") - if args.web_host == web_access.TAILNET else f"bound to {args.web_host}")) - for note in (*web_warnings, web_access.public_url_warning(opencode_env["PIPELINES_WEB_PUBLIC_URL"], web), - web_access.firewall_note(self.runner)): - if note: - print(note) - print("Login autostart is already enabled. To enable/start it again, run:\n " - + shlex.join([str(prefix / "bin/pipelines"), "service", "enable"])) - print("Android app (optional, slower): ./install --apk builds it in Docker and offers it at " - f"{opencode_env['PIPELINES_WEB_PUBLIC_URL']}/updates/.") - print("Configured server role: " + ("single-user authoring enabled (web pipeline creation/publication allowed)." - if allow_authoring else "execution-only (pipeline publication disabled).")) - if args.bind_address == "0.0.0.0": - print(f"Phone client server URL: ws://{lan_ip}:{args.port}/connect " - "(use a workspace path that exists on this server).") - else: - print(f"Unauthenticated API bound to {args.bind_address}:{args.port}; use the authenticated web URL.") + print("\nPipelines installed") + print(f"Install location: {prefix}") app_build.print_web_urls(manifest, lan_host=lan_ip) - print("OpenCode is required (default stage and chat harness) and is set up by ./install.") - print("Next: set up your agent harnesses: pipelines harness setup claude|codex|opencode|pi") + bin_dir = prefix / "bin" + path_entries = self.environ.get("PATH", "").split(os.pathsep) + if str(bin_dir) not in path_entries: + shell = Path(self.environ.get("SHELL", "")).name + rc = "~/.bashrc" if shell == "bash" else "~/.zshrc" if shell == "zsh" else "~/.profile" + displayed_bin = "$HOME/.local/share/pipelines/bin" if bin_dir == Path.home() / ".local/share/pipelines/bin" else str(bin_dir) + print(f"Add commands to PATH: echo 'export PATH=\"{displayed_bin}:$PATH\"' >> {rc}") + print(f"Update later: {prefix / 'bin/pipelines'} update") + print(f"Set up harnesses: {prefix / 'bin/pipelines'} harness setup claude|codex|opencode|pi") def arguments(argv=None): @@ -1020,17 +1030,29 @@ def arguments(argv=None): help="Start with empty data even when an earlier Docker installation's volumes exist") parser.add_argument("--replace-data", action="store_true", help="Allow Docker migration to replace an existing native database/auth store") + parser.add_argument("--verbose", action="store_true", help="Show full build and tool output") parsed = parser.parse_args(argv) parsed.argv = list(sys.argv[1:] if argv is None else argv) return parsed if __name__ == "__main__": + installer = Setup(Path(__file__).resolve().parents[1]) try: - Setup(Path(__file__).resolve().parents[1]).run(arguments()) + parsed = arguments() + installer.verbose = parsed.verbose + app_build.VERBOSE = parsed.verbose + prefix = parsed.prefix or Path(os.environ.get("PIPELINES_PREFIX") or DEFAULT_PREFIX).expanduser().resolve() + (prefix / "logs").mkdir(parents=True, exist_ok=True) + installer.install_log = prefix / "logs/install.log" + installer.run(parsed) except KeyboardInterrupt: print("\nNative setup canceled; existing configuration and data were retained.", file=sys.stderr) sys.exit(130) except (ValueError, OSError, json.JSONDecodeError, subprocess.SubprocessError, ZoneInfoNotFoundError) as error: + if installer.install_log and installer.install_log.is_file() and not installer.verbose: + tail = installer.install_log.read_text(errors="replace").splitlines()[-8:] + detail = "\n".join(tail) + sys.exit(f"Native setup incomplete: {error}\nBuild/tool log: {installer.install_log}\n{detail}") sys.exit(f"Native setup incomplete: {error}") diff --git a/scripts/test_manage_native.py b/scripts/test_manage_native.py index af699494..0463977f 100644 --- a/scripts/test_manage_native.py +++ b/scripts/test_manage_native.py @@ -26,6 +26,27 @@ class FakeProcess: class ManageNativeTests(unittest.TestCase): + def test_native_command_logs_tool_output_when_not_verbose(self): + with tempfile.TemporaryDirectory() as directory: + log = Path(directory) / "logs/install.log" + calls = [] + + def runner(command, **kwargs): + calls.append((command, kwargs)) + return subprocess.CompletedProcess(command, 1, stdout="build output\n", stderr="failure\n") + + installer = setup_native.Setup(directory, runner=runner) + installer.install_log = log + result = installer.command(["go", "build", "./cmd/pipelines"]) + self.assertEqual(result.returncode, 1) + self.assertIn("build output", log.read_text()) + self.assertIn("failure", log.read_text()) + self.assertTrue(calls[0][1]["capture_output"]) + + def test_native_arguments_accept_verbose(self): + parsed = setup_native.arguments(["--verbose"]) + self.assertTrue(parsed.verbose) + def service_config(self, home, system): config = self.config(home / "prefix") config["opencode_env"] = {"XDG_CONFIG_HOME": str(home / ".config")}