From f3ddd3e1ec5c153d4b2f3b78789e4bf0c7973731 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Mon, 9 Nov 2026 16:55:28 +0000 Subject: [PATCH] Attribute OpenCode code-mode tool calls, keep same-instant shown items in order, and record the chat tools notes --- app/notes/chat-pipelines-tools.md | 195 ++++++++++++++++++++++++ internal/pipeline/chat_tools.go | 4 +- internal/server/chat_tools_correlate.go | 106 ++++++++++--- internal/server/chat_tools_service.go | 2 +- internal/server/chat_tools_test.go | 36 +++++ 5 files changed, 319 insertions(+), 24 deletions(-) create mode 100644 app/notes/chat-pipelines-tools.md diff --git a/app/notes/chat-pipelines-tools.md b/app/notes/chat-pipelines-tools.md new file mode 100644 index 00000000..143182a7 --- /dev/null +++ b/app/notes/chat-pipelines-tools.md @@ -0,0 +1,195 @@ +# Chat: Pipelines tools (show file / image / HTML, notify phone) + +A chat's model can call four tools that act on Pipelines, not on the harness +or the workspace. The tools come from an MCP server that runs inside +`pipelines-server`. Each call adds an entry to the chat transcript, and the +entry is persisted. + +| Tool | Arguments | Effect | +| --- | --- | --- | +| `show_file` | `path`, `title?` | Shows an existing file inside the operator's home. The path can be absolute, `~/…`, or relative to the chat's directory. Images render inline; other files render as a preview card. | +| `show_image` | `url_or_path`, `title?`, `alt?` | Shows a home image file (it must be `image/*`) or an `https://` URL. | +| `show_html` | `html`, `title?`, `height_hint?` | Renders model-authored HTML or SVG inline in a sandbox, with an **Open** button for full screen. | +| `notify` | `title`, `message`, `priority?` | Sends a phone notification through the existing ntfy "system" destination. Tapping it opens `pipelines://chat/`. | + +Each tool returns one short text line to the model. A refusal comes back as +`isError` together with the reason. + +## Security model + +- **Endpoint.** The endpoint is `POST /mcp/chat/` on the server's own + listener. It speaks JSON-RPC 2.0 over MCP streamable HTTP: `initialize`, + `ping`, `tools/list`, `tools/call`, and notifications (answered `202`). + - `GET` and other methods get `405`. + - The protocol version is negotiated: 2025-11-25, 2025-06-18, 2025-03-26 or + 2024-11-05. + - Batches are refused, and request bodies are capped at 4 MiB. + - Requests that carry an `Origin` header (from a browser) get `403`, as do + requests from non-loopback peers. Unknown tokens get `404`. +- **Not exposed through the web.** `web/serve.mjs` and the peer relay forward + only an allowlist (`/connect`, `/files`, `/artifacts/`, `/chat-uploads`, + `/updates/`). Both new tests (`web/test/mcpRefused.test.mjs` and the + peerRelay test) confirm `/mcp…` never reaches the server, including for a + signed-in browser and `%6dcp`-encoded paths. +- **Tokens.** Each token is 256 random bits, stored in `chat_tool_tokens` with + the scope it serves: + - `session:`: one chat. Claude Code and Codex use this, and the token + stays stable across restarts so detached hosts keep working. + - `pending:`: a Codex thread before its id is known. Calls are refused until + the token is bound. + - `instance:opencode`: the shared OpenCode service. See "OpenCode attribution" + below. +- **Paths.** Paths use the same containment as `/files`: + `expandHomePath`, then `resolveInsideHome` (resolves every symlink), then + "regular file only", then the `specialFilesystem` check. The item records + the real path, and the client loads it from the chat's own machine's + `/files` (or `/peers//files` for a peer). +- **Image URLs.** Only `https` URLs without credentials, at most 2048 chars. + The client fetches them with a plain network image, without cookies. +- **HTML views.** These use the same inert view as artifact HTML previews: + - On web, an iframe with `sandbox="allow-popups allow-popups-to-escape-sandbox"` + and `srcdoc`. There are no scripts, no same-origin access, no forms and no + top navigation. + - Links are rewritten first: `#anchors` scroll inside the frame, + http(s)/mailto links open a new tab, and every other link is made inert. + - On Android/iOS, a WebView with JavaScript disabled that refuses navigation. + - Interactivity is therefore limited to CSS (`:hover`, `
`) and SVG. + The tool description tells the model this. + +## Limits + +- `show_html` documents are refused, not truncated, above **512 KiB**. The + model is told the size and the limit. +- At most **100 items are retained per chat**. HTML bodies are kept only for the + newest documents within **2 MiB per chat**; older documents keep their entry + but are marked `expired`, and the card says "This view is no longer retained". +- `notify` allows **5 per minute per chat** (a sliding window, held in memory). + Rate-limited calls record nothing. + - An undelivered notification is recorded with its reason, for example ntfy + not configured or switched off. + - `priority` is recorded, but ntfy always publishes at the default priority, + because `stage.PublishNtfy` has no priority parameter. + +## Protocol and persistence + +Items are stored in `chat_shown_items` (pipeline store). There are two ways +they reach clients: + +- The `chat_subscribe` reply carries `chat_shown: {session_id, items}` with + every retained item, oldest first. +- A pushed `chat_shown` frame carries one new item. + +The client interleaves items into the transcript by `created_at`. Items older +than the loaded history wait until that page loads. In a multi-machine chat, +the session id is rewritten by `ChatServerGateway`, and file URLs resolve +against the chat's own machine endpoint. + +The generic tool rows for `mcp__pipelines__*` (Claude), `pipelines_*` +(OpenCode tool parts) and `pipelines/*` (Codex) are hidden; failed calls stay +visible. OpenCode V2 runs MCP tools from its code-mode `execute` script, and +that row stays visible because it can contain other code. + +## Per-harness registration + +`server.ConfigureChatTools(store, addr)` runs at startup, before any harness +starts, and installs `chattools.Registry`. The registry's base URL is the +loopback address of `-addr`; a wildcard address maps to 127.0.0.1, and a +non-loopback-only bind disables the tools with a log line. + +- **Claude Code.** Started with `--mcp-config '{"mcpServers":{"pipelines":{"type":"http","url":…}}}'` + and `--allowedTools mcp__pipelines`. Both direct and hosted chats get the + session token. + - The config is added to the operator's own MCP servers; it does not replace + them. + - If a start fails with these flags, the chat logs the reason and retries + once without them (`session.toolsOff`). +- **Codex.** `thread/start` gets `config.mcp_servers.pipelines.url` with a + pending token, which is then bound to the returned thread id. + - A `thread/resume` (after "thread not found") registers the thread's + session token. + - If the App Server refuses the override, the thread starts or resumes + without it, and this is logged. + - Threads already loaded by the Codex desktop app before Pipelines touched + them have no tools. +- **OpenCode.** The spawned chat service gets + `mcp.pipelines = {type: "remote", url, enabled: true}` through + `OPENCODE_CONFIG_CONTENT`. This covers both `OpenChat` and the owned detached + `EnsureOwnedChatService`. + - The operator's chat overrides win. + - If the service does not start with the tools, it starts without them, and + this is logged. + - An operator-supplied endpoint (`-opencode-endpoint`) is not reconfigured, + so it has no tools. +- **Pi.** There is no Pi chat harness in this repository (chat backends are + opencode, claude and codex), so it is not supported. + +**OpenCode attribution.** One OpenCode service serves every chat, and its MCP +requests carry no session (verified: OpenCode 2.0.32 sends only +`User-Agent: opencode/latest/2.0.32/cli` and `Mcp-Protocol-Version`). + +A call on the instance token waits up to 3 s for exactly one chat whose +retained stream shows that call running. That means one of: + +- a `pipelines_` part with identical arguments, or +- a running code-mode `execute` script containing `pipelines.(`. If + several such scripts are running, they are told apart by the argument + values. + +When no chat matches, or several still do, the call is refused. It is never +guessed into a chat. Only chats Pipelines is observing can use the tools. + +## Verification + +- Go: + - `go test ./internal/server/ ./internal/chattools/ ./internal/pipeline/ ./internal/protocol/ ./internal/codex/ ./internal/codexhost/ ./internal/claudehost/`: all `ok` (server 72 s). + - `go test ./internal/claude/ ./internal/opencode/`: `ok` under `umask 022`. + Under this sandbox's `umask 077`, the existing tests + `TestServiceTokenIsPassedOnlyFromAPrivateFile` and + `TestStageDiscoveryRequiresPrivateRegistrationBeforeSessionWork` fail, + because both depend on 0644 file creation. They are unrelated to this + change. +- New Go tests: + - `chattools/mcp_test.go`: handshake, list, call, refusals, harness config. + - `server/chat_tools_test.go`: token isolation, pending binding, path + containment, image URLs, HTML limit and retention, notify rate limit, + instance and code-mode attribution. + - `server/chat_tools_http_test.go`: endpoint on the real listener, + `loopbackBase`. + - `claude|codex|opencode/pipelines_tools_test.go`: registration config and + the refuse-then-start-without fallback. +- `cd web && node --test test/`: `ℹ tests 18 … ℹ pass 18 ℹ fail 0`. +- Flutter: see the final report for the `tool/memwatch/full.sh test/features/chat` + verdict and `tool/analyze.sh`. +- Real end-to-end check, run on 2026-11-09: + - Setup: a scratch `pipelines-server` on `127.0.0.1:53834`, with + `HOME=/tmp/pl-e2e-tools/home`, a scratch database, + `PIPELINES_SHUTDOWN_SESSIONS=stop` and ntfy unset. + - OpenCode 2.0.32 (its default free model) was asked to `show_file + ~/scratch/checker.png` and to `show_html` an SVG circle. Its `execute` + script returned "Shown in the operator's Pipelines chat: checker.png (217 + bytes)" and "Shown … (137 bytes, sandboxed…)". + - `chat_subscribe` returned both items. The file item had + `path=/private/tmp/pl-e2e-tools/home/scratch/checker.png`, + `mime=image/png` and `size=217`. `/files` served that path with + `200 image/png 217`. + - `notify` was recorded as not delivered: "phone notifications (ntfy) are + not configured on this Pipelines server". + - A POST carrying an `Origin` header got `403`. + - The first OpenCode attempt was refused, and this is how code-mode + attribution was found to be necessary. +- Not run end to end: + - Claude Code under the isolated HOME answered "Not logged in", and + `~/.config/pipelines/claude-token` is empty. + - Codex has no app-server under the isolated HOME. + - Both registrations are covered only by unit tests against fake CLI + arguments and a fake App Server. +- Screenshots, real Chrome via `tool/capture` on the full-app fixture with + `?chats=1×tamps=1&shown=1&files=1`, all in + `notes/images/chat-pipelines-tools/`: + - `cards-light-880.png`: image, markdown card with first lines, HTML + diagram with Open, and notify row. + - `cards-dark-880.png` + - `cards-cupertino-560.png` + - `html-open-light-880.png`: full-screen HTML with zoom. + - `image-open-dark-880.png`: the existing viewer with download. +- Mobile (Android/iOS WebView, native thumbnails) is unverified: no device run. diff --git a/internal/pipeline/chat_tools.go b/internal/pipeline/chat_tools.go index 4538be8b..df56b2ce 100644 --- a/internal/pipeline/chat_tools.go +++ b/internal/pipeline/chat_tools.go @@ -99,7 +99,7 @@ func (s *Store) AppendChatShownItem(ctx context.Context, sessionID string, item return err } if _, err := tx.ExecContext(ctx, `DELETE FROM chat_shown_items WHERE session_id=? AND id NOT IN ( -SELECT id FROM chat_shown_items WHERE session_id=? ORDER BY created_at DESC, id DESC LIMIT ?)`, sessionID, sessionID, protocol.ChatShownLimit); err != nil { +SELECT id FROM chat_shown_items WHERE session_id=? ORDER BY created_at DESC, rowid DESC LIMIT ?)`, sessionID, sessionID, protocol.ChatShownLimit); err != nil { return err } items, err := chatShownItems(ctx, tx, sessionID) @@ -139,7 +139,7 @@ type chatShownQueryer interface { } func chatShownItems(ctx context.Context, db chatShownQueryer, sessionID string) ([]protocol.ChatShownItem, error) { - rows, err := db.QueryContext(ctx, `SELECT payload FROM chat_shown_items WHERE session_id=? ORDER BY created_at, id`, sessionID) + rows, err := db.QueryContext(ctx, `SELECT payload FROM chat_shown_items WHERE session_id=? ORDER BY created_at, rowid`, sessionID) if err != nil { return nil, err } diff --git a/internal/server/chat_tools_correlate.go b/internal/server/chat_tools_correlate.go index 4d735531..c2c4e4f3 100644 --- a/internal/server/chat_tools_correlate.go +++ b/internal/server/chat_tools_correlate.go @@ -5,26 +5,35 @@ import ( "encoding/json" "reflect" "slices" + "strings" "time" + "git.a-wan.net/alan/pipelines/internal/chattools" "git.a-wan.net/alan/pipelines/internal/protocol" ) +// openCodeCodeTool is OpenCode V2's code-mode tool: the model calls MCP tools +// from a script (`tools.pipelines.show_file({...})`) instead of as tool parts +// of their own, so the stream shows one running "execute" part. +const openCodeCodeTool = "execute" + // awaitToolCallSession attributes a call on a shared harness endpoint (one // OpenCode service serves every chat, and its MCP requests carry no session) -// to the chat whose retained stream shows a running tool part with one of -// names and exactly these arguments. The harness reports the part as running -// before or while it calls the tool, so this waits up to wait for it. It -// returns "" when no chat matches, and ambiguous when several distinct chats -// do: a call is never guessed into a chat. -func (h *chatHub) awaitToolCallSession(ctx context.Context, names []string, arguments json.RawMessage, wait time.Duration) (string, bool) { - var want any +// to the chat whose retained stream shows this call running: a tool part with +// the tool's harness name and exactly these arguments, or a running code-mode +// script that calls the tool. Several scripts are told apart by the argument +// values they contain. The harness reports the part as running before or while +// it calls the tool, so this waits up to wait for it. It returns "" when no +// chat matches, and ambiguous when several distinct chats still do: a call is +// never guessed into a chat. +func (h *chatHub) awaitToolCallSession(ctx context.Context, tool string, arguments json.RawMessage, wait time.Duration) (string, bool) { + var want map[string]any if json.Unmarshal(arguments, &want) != nil { return "", false } deadline := time.Now().Add(wait) for { - sessions := h.runningToolCallSessions(names, want) + sessions := h.runningToolCallSessions(tool, want) switch { case len(sessions) == 1: return sessions[0], false @@ -42,34 +51,89 @@ func (h *chatHub) awaitToolCallSession(ctx context.Context, names []string, argu } } -func (h *chatHub) runningToolCallSessions(names []string, want any) []string { +func (h *chatHub) runningToolCallSessions(tool string, want map[string]any) []string { + names := chattools.HarnessToolNames(tool) h.mu.Lock() defer h.mu.Unlock() - var sessions []string + var exact, scripts, narrowed []string for sessionID, turn := range h.turns { - if turnHasToolCall(turn, names, want) { - sessions = append(sessions, sessionID) + match := turnToolCallMatch(turn, names, tool, want) + if match == toolCallExact { + exact = append(exact, sessionID) + } + if match >= toolCallScript { + scripts = append(scripts, sessionID) + } + if match >= toolCallScriptWithArguments { + narrowed = append(narrowed, sessionID) } } - slices.Sort(sessions) - return sessions + for _, candidates := range [][]string{exact, scripts, narrowed} { + if len(candidates) == 1 { + return candidates + } + } + slices.Sort(scripts) + return scripts } -func turnHasToolCall(turn *chatTurn, names []string, want any) bool { +type toolCallMatch int + +const ( + toolCallNone toolCallMatch = iota + toolCallScript + toolCallScriptWithArguments + toolCallExact +) + +// turnToolCallMatch is the strongest evidence one chat's running tool parts +// give that it is making this call. +func turnToolCallMatch(turn *chatTurn, names []string, tool string, want map[string]any) toolCallMatch { + best := toolCallNone for _, message := range turn.messages { for _, part := range message.parts { p := part.part - if p.Type != "tool" || p.State == nil || !slices.Contains(names, p.Name) { + if p.Type != "tool" || p.State == nil || (p.State.Status != protocol.ChatToolRunning && p.State.Status != "pending") { continue } - if p.State.Status != protocol.ChatToolRunning && p.State.Status != "pending" { + if slices.Contains(names, p.Name) { + var got map[string]any + if json.Unmarshal(p.State.Input, &got) == nil && reflect.DeepEqual(got, want) { + return toolCallExact + } continue } - var got any - if json.Unmarshal(p.State.Input, &got) == nil && reflect.DeepEqual(got, want) { - return true + if p.Name != openCodeCodeTool { + continue } + var script struct { + Code string `json:"code"` + } + if json.Unmarshal(p.State.Input, &script) != nil || !strings.Contains(script.Code, chattools.ServerName+"."+tool+"(") { + continue + } + match := toolCallScript + if scriptHasArguments(script.Code, want) { + match = toolCallScriptWithArguments + } + best = max(best, match) } } - return false + return best +} + +// scriptHasArguments reports whether every string argument appears in code, +// literally or JSON-escaped. +func scriptHasArguments(code string, want map[string]any) bool { + for _, value := range want { + text, ok := value.(string) + if !ok || text == "" { + continue + } + escaped, _ := json.Marshal(text) + if !strings.Contains(code, text) && !strings.Contains(code, strings.Trim(string(escaped), `"`)) { + return false + } + } + return true } diff --git a/internal/server/chat_tools_service.go b/internal/server/chat_tools_service.go index 4c7705c9..5389d34f 100644 --- a/internal/server/chat_tools_service.go +++ b/internal/server/chat_tools_service.go @@ -98,7 +98,7 @@ func (s *chatToolService) session(ctx context.Context, token, tool string, argum return id, "" } if _, ok := strings.CutPrefix(scope, chatToolInstanceScope); ok && s.hub != nil { - id, ambiguous := s.hub.awaitToolCallSession(ctx, chattools.HarnessToolNames(tool), arguments, chatToolCorrelationWait) + id, ambiguous := s.hub.awaitToolCallSession(ctx, tool, arguments, chatToolCorrelationWait) if ambiguous { return "", "Pipelines could not tell which chat made this call (another chat made an identical one); retry with a different title." } diff --git a/internal/server/chat_tools_test.go b/internal/server/chat_tools_test.go index e124e903..16386883 100644 --- a/internal/server/chat_tools_test.go +++ b/internal/server/chat_tools_test.go @@ -292,3 +292,39 @@ func TestInstanceTokenCallsNeedOneMatchingRunningToolCall(t *testing.T) { t.Fatalf("two chats running the same call must be refused: %v", r) } } + +// OpenCode V2 calls MCP tools from its code-mode "execute" script, so the +// stream shows the script, not a tool part of the Pipelines tool's own name. +func TestInstanceTokenCallsMatchOpenCodeCodeModeScripts(t *testing.T) { + f := newChatToolFixture(t) + token := f.token(t, chatToolInstanceScope+"opencode") + saved := chatToolCorrelationWait + chatToolCorrelationWait = 100 * time.Millisecond + t.Cleanup(func() { chatToolCorrelationWait = saved }) + script := func(sessionID, part, code string) { + input, _ := json.Marshal(map[string]string{"code": code}) + f.hub.apply(protocol.ChatEventFrame{SessionID: sessionID, MessageID: "msg_" + sessionID, Kind: protocol.ChatEventTool, + Part: &protocol.ChatPart{Type: "tool", ID: part, Name: "execute", State: &protocol.ChatToolState{Status: protocol.ChatToolRunning, Input: input}}}) + } + if err := os.WriteFile(filepath.Join(f.home, "a.png"), []byte("\x89PNG\r\n\x1a\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(f.home, "b.png"), []byte("\x89PNG\r\n\x1a\n"), 0o600); err != nil { + t.Fatal(err) + } + script("ses_p", "prt_p", `return tools.pipelines.show_file({ path: "~/a.png", title: "Checker" })`) + script("ses_q", "prt_q", `return tools.other.show_file({ path: "~/a.png" })`) + if r := f.call(token, "show_file", `{"path":"~/a.png","title":"Checker"}`); r.IsError { + t.Fatal(r.Text) + } + if len(f.items(t, "ses_p")) != 1 || len(f.items(t, "ses_q")) != 0 { + t.Fatal("the script calling the Pipelines tool owns the call") + } + script("ses_q", "prt_q2", `await tools.pipelines.show_file({ path: "~/b.png" })`) + if r := f.call(token, "show_file", `{"path":"~/b.png"}`); r.IsError || len(f.items(t, "ses_q")) != 1 { + t.Fatalf("two scripts are told apart by their arguments: %v", r) + } + if r := f.call(token, "show_file", `{"path":"~/c.png"}`); !r.IsError { + t.Fatal("arguments neither script contains must not be guessed into a chat") + } +}