From 68042aad5dc75b8aeb33a6ccb96eb9b747f5286d Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sat, 10 Oct 2026 01:22:02 +0100 Subject: [PATCH] Mount host repositories for Docker execution with host user ownership --- compose.yaml | 25 ++++++++++++++++++-- docs/docker.md | 12 +++++++--- docs/install.md | 15 ++++++++++++ docs/ntfy.md | 31 ++++++++++++++++++++++--- internal/server/runner.go | 4 ++-- internal/server/workspaces_test.go | 5 ++++ internal/tui/run_detail.go | 2 +- scripts/configure-compose.py | 10 +++++++- scripts/setup-docker.py | 32 ++++++++++++++++++++++++++ scripts/test_setup_docker.py | 37 ++++++++++++++++++++++++++++++ 10 files changed, 161 insertions(+), 12 deletions(-) diff --git a/compose.yaml b/compose.yaml index bb632fe2..b4f0090e 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,6 +1,7 @@ services: pipelines: build: . + user: "${PIPELINES_UID:-10001}:${PIPELINES_GID:-10001}" init: true restart: unless-stopped stop_grace_period: 60s @@ -16,14 +17,34 @@ services: PIPELINES_HOST_IP: ${PIPELINES_HOST_IP:-} PIPELINES_PUBLIC_PORT: ${PIPELINES_PORT:-8080} PIPELINES_NTFY_READER: phone-reader + PIPELINES_UID: ${PIPELINES_UID:-10001} + PIPELINES_GID: ${PIPELINES_GID:-10001} secrets: - ntfy_token ports: - "${PIPELINES_BIND_ADDRESS:-0.0.0.0}:${PIPELINES_PORT:-8080}:8080" volumes: - - pipelines-data:/data - - opencode-home:/home/pipelines + - type: volume + source: pipelines-data + target: /data + volume: + nocopy: true + - type: volume + source: opencode-home + target: /home/pipelines + volume: + nocopy: true - pipelines-workspace:/workspace + - type: bind + source: ${PIPELINES_HOST_HOME:?Run ./install.sh to configure the host repository mount} + target: ${PIPELINES_HOST_HOME:?Run ./install.sh to configure the host repository mount} + bind: + create_host_path: false + - type: bind + source: ${PIPELINES_HOST_REPOSITORY:?Run ./install.sh to configure the host repository mount} + target: ${PIPELINES_HOST_REPOSITORY:?Run ./install.sh to configure the host repository mount} + bind: + create_host_path: false healthcheck: test: ["CMD", "pipelines", "--url", "ws://127.0.0.1:8080/connect", "status", "--json"] interval: 5s diff --git a/docs/docker.md b/docs/docker.md index 71ed245d..0220e0a4 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -102,6 +102,10 @@ both, and never puts the token in model prompts, outputs, or diagnostics. ```sh docker compose build pipelines +docker compose stop --timeout 60 pipelines +# Default managed volumes only; never substitute a host bind mount for these paths. +docker compose run --rm --no-deps --user 0:0 --entrypoint sh pipelines -c \ + 'chown -R "$PIPELINES_UID:$PIPELINES_GID" /data /home/pipelines /workspace' docker compose run --rm --no-deps --entrypoint opencode pipelines auth login docker compose up -d docker compose logs -f pipelines @@ -109,9 +113,11 @@ docker compose logs -f pipelines Select your preferred provider and follow its authentication flow. Credentials are stored in the dedicated `opencode-home` volume, separate from your host account. These -container commands are **not tested** here. The image includes Go/git for the -existing narrowly approved verification tools; the actual server runs as uid 10001. -The Docker workspace is persistent and separate from your host checkout. +interactive login is **not tested** in this regression. The Docker workspace/UID +checks were executed with the actual image and a mocked stage. The image includes +Go/git; installer-configured servers use the host user's UID/GID. The home and +installation checkout are bound at their original paths so runs edit the real +checkout, not the separate persistent image workspace at `/workspace`. Connect the native TUI: diff --git a/docs/install.md b/docs/install.md index 4b28d914..b14f63ed 100644 --- a/docs/install.md +++ b/docs/install.md @@ -45,6 +45,21 @@ checks ntfy health through Pipelines, and sends an authenticated test notificati It does not register an OS service, delete volumes, or seed pipeline definitions. Phone reception is not verified by installation. +Docker setup mounts the host user's home and the installation checkout at their +original absolute paths, and runs Pipelines/OpenCode using that host user's UID +and GID. This makes a client invocation directory such as +`/home/tank/repos/pipelines` refer to the actual host checkout rather than an +unavailable container path or the unrelated image checkout at `/workspace`. +Stage permissions/edit scopes still apply; this is a filesystem mount, not a +workspace allowlist or automatic file transfer from another client computer. +For repositories outside those mounts, add a Compose bind at the same absolute +path. Reinstallation stops the old Pipelines service before changing ownership +of its managed data/auth/workspace volumes. It never recursively chowns the host +home/repository, deletes credentials, or removes volumes. Existing data/auth +volumes use `nocopy` so Docker cannot reset a freshly prepared empty volume's +ownership to the image UID. Setup verifies that the server can enter the host +checkout before reporting completion. + For fresh configuration you can supply `--ntfy-url https://ntfy.example.com --bind-address 127.0.0.1 --timezone Europe/London --port 8081` directly to `./install.sh`. The defaults are ready for direct phone access on your trusted LAN. Only use diff --git a/docs/ntfy.md b/docs/ntfy.md index 40d737ca..79c6ab98 100644 --- a/docs/ntfy.md +++ b/docs/ntfy.md @@ -92,9 +92,9 @@ access. Prefer a dedicated publishing user with write-only access to this topic. The root `compose.yaml` runs **both the pipeline server and ntfy** with persistent volumes, internal service networking, and a private topic. See the [Docker + Android setup](docker.md) for bootstrap, credentials, and timezone -detection. The previous ntfy-only Compose file has been replaced. Docker's daemon -was unavailable; deployment is **not tested**. HTTP integration tests use an -ntfy-compatible test server, not a running ntfy instance. +detection. The previous ntfy-only Compose file has been replaced. Isolated real +Docker health and publishing checks have passed; phone reception and the user's +remote Tailscale connectivity remain **not tested** locally. Use a TLS reverse proxy or a private VPN (e.g. Tailscale). A phone cannot reach your computer's `127.0.0.1`; that address is only for local publishing/proxying. @@ -106,6 +106,31 @@ server is another container, localhost is that container, not the ntfy host. ## Phone +### Tailscale address times out while LAN works + +Use `http://:8090`, not port 80; enable Tailscale on both the +server and phone. The default Compose mapping binds ntfy to `0.0.0.0:8090`. +Binding to all interfaces does not bypass tailnet grants/ACLs, host firewalls, +or Docker forwarding rules. A timeout alone cannot identify which is blocking. +Do not disable firewalls or publicly forward this port as a workaround. + +On the server, these are read-only diagnostics (the remote checks are not tested +from this development machine): + +```sh +tailscale status +docker compose port ntfy 80 +curl --connect-timeout 5 http://127.0.0.1:8090/v1/health +curl --connect-timeout 5 "http://$(tailscale ip -4):8090/v1/health" +``` + +If the host health checks succeed but the phone times out, check that the phone +is in the intended tailnet and that tailnet access rules/firewalls allow this +phone to reach server TCP 8090. Once reachable, log in as `phone-reader` for the +Tailscale server entry, separately from the LAN entry. Set `NTFY_BASE_URL` to a +phone-reachable address if attachment URLs currently point to a LAN-only address; +that setting is not a substitute for fixing a TCP timeout. + Install the ntfy app, add your own server, authenticate with the reader account, and subscribe to `anime`. Ensure notifications and required background operation are allowed. Outside your home network, both topic subscriptions and attachment diff --git a/internal/server/runner.go b/internal/server/runner.go index 6ba58235..d45630c7 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -63,11 +63,11 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru } selected, err := filepath.EvalSymlinks(request.Workspace) if err != nil { - return nil, fmt.Errorf("run workspace is unavailable") + return nil, fmt.Errorf("run workspace is unavailable on the execution server: %q: %w; for Docker, mount this host directory at the same absolute path and use its owner's UID/GID", request.Workspace, err) } info, err := os.Stat(selected) if err != nil || !info.IsDir() { - return nil, fmt.Errorf("run workspace must be a directory") + return nil, fmt.Errorf("run workspace %q must be an accessible directory on the execution server", request.Workspace) } workspace = selected } diff --git a/internal/server/workspaces_test.go b/internal/server/workspaces_test.go index 7e928fef..80d4c0f0 100644 --- a/internal/server/workspaces_test.go +++ b/internal/server/workspaces_test.go @@ -7,6 +7,7 @@ import ( "git.a-wan.net/alan/pipelines/internal/stage" "os" "path/filepath" + "strings" "testing" ) @@ -60,4 +61,8 @@ func TestClientSelectedRepositoryWorkspaceAndAudit(t *testing.T) { t.Fatal("invalid run directory accepted", path) } } + missing := filepath.Join(root, "missing") + if _, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "missing-diagnostic", Input: map[string]any{"prompt": "task"}, Workspace: missing}); err == nil || !strings.Contains(err.Error(), missing) || !strings.Contains(err.Error(), "mount this host directory") { + t.Fatal("missing workspace error hid requested path or Docker remedy", err) + } } diff --git a/internal/tui/run_detail.go b/internal/tui/run_detail.go index 176e189e..df58c2d1 100644 --- a/internal/tui/run_detail.go +++ b/internal/tui/run_detail.go @@ -545,7 +545,7 @@ func (m Model) runsDetailsView(height int) string { } } if m.runError != "" { - rows = append(rows, m.runError) + rows = append(rows, strings.Split(ansi.Wrap(lipgloss.NewStyle().Foreground(themeError).Render(m.runError), max(1, m.width-4), ""), "\n")...) } footer := "/ fuzzy search · j/k select · Enter inspect · " + m.runActions() if m.runSearchOpen { diff --git a/scripts/configure-compose.py b/scripts/configure-compose.py index 757844f8..e740aa1c 100644 --- a/scripts/configure-compose.py +++ b/scripts/configure-compose.py @@ -25,6 +25,12 @@ def host_timezone(): continue raise ValueError("Cannot detect the host timezone; provide --timezone explicitly") +def host_workspaces(directory): + home, repository = str(Path.home().resolve()), str(Path(directory).resolve()) + if any(any(char in value for char in "\r\n\x00$#'\"\\") or value in ("/", "/data", "/workspace", "/home/pipelines") for value in [home, repository]) or home == repository: + raise ValueError("Host home/checkout cannot be mounted safely at its original path; configure a repository bind mount explicitly") + return {"PIPELINES_HOST_HOME": "'"+home+"'", "PIPELINES_HOST_REPOSITORY": "'"+repository+"'", "PIPELINES_UID": str(os.getuid()), "PIPELINES_GID": str(os.getgid())} + def configure(directory, ntfy_url, bind_address, timezone=None, port=8081): directory = Path(directory) @@ -42,6 +48,7 @@ def configure(directory, ntfy_url, bind_address, timezone=None, port=8081): settings = directory / ".env" if settings.exists(): raise ValueError(".env exists; edit it rather than overwrite existing settings") + workspace_values = host_workspaces(directory) secrets = directory / "secrets" secrets.mkdir(mode=0o700, exist_ok=True) token = secrets / "ntfy-token.txt" @@ -53,7 +60,8 @@ def configure(directory, ntfy_url, bind_address, timezone=None, port=8081): pass with settings.open("x") as file: file.write(f"TZ={timezone}\nNTFY_BASE_URL={ntfy_url.rstrip('/')}\n" - f"NTFY_BIND_ADDRESS={bind_address}\nPIPELINES_PORT={port}\n") + f"NTFY_BIND_ADDRESS={bind_address}\nPIPELINES_PORT={port}\n") + for key, value in workspace_values.items(): file.write(key+"="+value+"\n") return timezone diff --git a/scripts/setup-docker.py b/scripts/setup-docker.py index 73f206e6..7320aec9 100644 --- a/scripts/setup-docker.py +++ b/scripts/setup-docker.py @@ -197,6 +197,34 @@ exec pipelines provider-check --model "$1" --yes --json --timeout 90s updated = updated.rstrip("\n")+"\n"+key+address+"\n" path.write_text(updated) + def configure_workspaces(self): + home = str(Path.home().resolve()) + path = self.directory/".env" + values = configure.host_workspaces(self.directory) + lines = path.read_text().splitlines(keepends=True) + text = "".join(key+"="+values[key]+"\n" if (key:=line.split("=",1)[0]) in values else line for line in lines) + for key,value in values.items(): + if not any(line.startswith(key+"=") for line in lines): text=text.rstrip("\n")+"\n"+key+"="+value+"\n" + path.write_text(text) + print("Host repositories mounted at their original paths under " + home + "; server runs with host UID/GID. Stage tool/edit permissions still apply.") + + def prepare_volume_ownership(self): + # Existing volumes may belong to the old image UID 10001. Change only + # ownership on container-managed data/auth volumes, never the host home. + resolved = json.loads(self.compose("config", "--format", "json", capture=True).stdout) + mounts = resolved.get("services", {}).get("pipelines", {}).get("volumes", []) + for target in ("/data", "/home/pipelines", "/workspace"): + if not any(mount.get("target") == target and mount.get("type") == "volume" for mount in mounts): + raise ValueError("Refusing ownership migration: " + target + " is not a managed Docker volume; no host directory ownership was changed") + self.compose("run", "--rm", "--no-deps", "-T", "--user", "0:0", "--entrypoint", "sh", "pipelines", + "-c", 'set -eu; chown -R "$PIPELINES_UID:$PIPELINES_GID" /data /home/pipelines /workspace') + + def verify_workspace(self): + directory = str(self.directory.resolve()) + self.compose("exec", "-T", "pipelines", "sh", "-c", 'set -eu; test -d "$1"; test -r "$1"; test -x "$1"; cd "$1"; test "$(pwd -P)" = "$1"', + "pipelines-workspace-check", directory, capture=True) + print("Verified: Docker server can enter the host installation repository at " + directory) + def verify(self): # Read the token inside the container; never put it in argv or output. probe = '''set -eu @@ -275,17 +303,21 @@ printf 'header = "Authorization: Bearer %s"\\n' "$(cat "$PIPELINES_NTFY_TOKEN_FI self.compose("version", capture=True) self.settings(args) self.publish_address() + self.configure_workspaces() self.update_host_client() # Validate the actual resolved Compose configuration before starting anything. self.compose("config", "--quiet", capture=True) self.compose("up", "-d", "--wait", "--wait-timeout", "60", "ntfy") self.bootstrap_ntfy() self.compose("build", "pipelines") + self.compose("stop", "--timeout", "60", "pipelines") + self.prepare_volume_ownership() self.compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "pipelines", "pipelines", "skill", "install") self.authenticate(args.relogin) self.validate_provider(args) # Recreate to reload file secrets and environment, without deleting volumes. self.compose("up", "-d", "--force-recreate", "--wait", "--wait-timeout", "60", "pipelines") + self.verify_workspace() self.verify() print("Docker setup complete. Use docker compose ps / logs to manage it; no OS user service was registered.") diff --git a/scripts/test_setup_docker.py b/scripts/test_setup_docker.py index cd513c0f..70be7b29 100644 --- a/scripts/test_setup_docker.py +++ b/scripts/test_setup_docker.py @@ -51,6 +51,41 @@ class DockerSetupTests(unittest.TestCase): with patch.object(setup.shutil,"which",return_value="/fixture/go"), self.assertRaises(ValueError): setup.Setup(".",runner=runner).verify_host_endpoint("ws://127.0.0.1:8081/connect") + def test_workspace_mount_uses_host_home_uid_and_preserves_settings(self): + with tempfile.TemporaryDirectory() as directory: + path=Path(directory,".env");path.write_text("CUSTOM=keep\nPIPELINES_UID=10001\n") + setup.Setup(directory).configure_workspaces() + content=path.read_text() + self.assertIn("CUSTOM=keep\n",content) + self.assertIn("PIPELINES_HOST_HOME='"+str(Path.home().resolve())+"'",content) + self.assertIn("PIPELINES_UID="+str(os.getuid()),content) + self.assertIn("PIPELINES_GID="+str(os.getgid()),content) + setup.Setup(directory).configure_workspaces() + self.assertEqual(path.read_text(),content) + + def test_ownership_migration_and_workspace_verification_are_scoped(self): + calls=[] + def runner(command,**kwargs): + calls.append(command) + config={"services":{"pipelines":{"volumes":[{"type":"volume","target":target} for target in ["/data","/home/pipelines","/workspace"]]}}} + return subprocess.CompletedProcess(command,0,stdout=setup.json.dumps(config) if "--format" in command else "",stderr="") + installer=setup.Setup(".",runner=runner) + installer.prepare_volume_ownership() + installer.verify_workspace() + self.assertIn("0:0",calls[1]) + self.assertEqual(calls[1][-1], 'set -eu; chown -R "$PIPELINES_UID:$PIPELINES_GID" /data /home/pipelines /workspace') + self.assertEqual(calls[2][-1],str(Path(".").resolve())) + self.assertIn('cd "$1"',calls[2][-3]) + + def test_ownership_migration_never_chowns_host_bind_mounts(self): + calls=[] + def runner(command,**kwargs): + calls.append(command) + return subprocess.CompletedProcess(command,0,stdout='{"services":{"pipelines":{"volumes":[{"type":"bind","target":"/data"}]}}}',stderr="") + with self.assertRaises(ValueError): setup.Setup(".",runner=runner).prepare_volume_ownership() + self.assertEqual(len(calls),1) + self.assertFalse(any("chown" in part for part in calls[0])) + def test_validation_primes_its_own_service_and_passes_model_as_argument(self): with tempfile.TemporaryDirectory() as directory: root=Path(directory);(root/".env").write_text("CUSTOM=keep\n") @@ -264,6 +299,8 @@ fi out = '{"ntfy":{"health":"healthy","url":"http://192.168.1.20:8090"},"topic":"pipelines"}' if args[:1] == ["port"]: out = "127.0.0.1:8081\n" + if args == ["config", "--format", "json"]: + out = setup.json.dumps({"services":{"pipelines":{"volumes":[{"type":"volume","target":target} for target in ["/data","/home/pipelines","/workspace"]]}}}) if fail and (fail in args or (fail=="login" and "exec opencode auth login" in args[-1]) or (fail=="provider-check" and any("provider-check" in arg for arg in args))): code, out = 9, "private provider data/token" if fail == "host" and command[0] != "docker" and args[-2:] == ["status", "--json"]: