diff --git a/cmd/pipelines-server/main.go b/cmd/pipelines-server/main.go index 8de056d7..1300a2bf 100644 --- a/cmd/pipelines-server/main.go +++ b/cmd/pipelines-server/main.go @@ -70,6 +70,12 @@ func main() { } else if seeded > 0 { log.Printf("added %d example pipelines to the new database", seeded) } + // Every chat harness Pipelines launches registers the Pipelines tools (show + // a file, an image or an HTML view in the chat, notify the phone) as an MCP + // server on this listener's loopback address (internal/chattools). + if err := server.ConfigureChatTools(store, *addr); err != nil { + log.Printf("chat tools unavailable: %v", err) + } artifactsRoot := filepath.Join(filepath.Dir(*db), "artifacts") // Each new run gets a bounded one-line title from the configured default // harness; the summariser falls back to the prompt's first line, so title diff --git a/internal/chattools/harness_config.go b/internal/chattools/harness_config.go new file mode 100644 index 00000000..f1a46e1e --- /dev/null +++ b/internal/chattools/harness_config.go @@ -0,0 +1,49 @@ +package chattools + +import ( + "encoding/json" + "strings" +) + +// ClaudeMCPConfig is the inline JSON Claude Code takes as --mcp-config. +func ClaudeMCPConfig(url string) string { + data, _ := json.Marshal(map[string]any{"mcpServers": map[string]any{ + ServerName: map[string]any{"type": "http", "url": url}, + }}) + return string(data) +} + +// ClaudeAllowedTools pre-approves every Pipelines tool, so a call never waits +// on a permission card: the tools only present content to the operator. +func ClaudeAllowedTools() string { return "mcp__" + ServerName } + +// CodexConfig is the thread/start (and thread/resume) config override that +// registers the endpoint as a streamable HTTP MCP server for one thread. +func CodexConfig(url string) map[string]any { + return map[string]any{"mcp_servers": map[string]any{ + ServerName: map[string]any{"url": url}, + }} +} + +// OpenCodeConfig is the OpenCode config fragment registering the endpoint as a +// remote MCP server. +func OpenCodeConfig(url string) map[string]any { + return map[string]any{"mcp": map[string]any{ + ServerName: map[string]any{"type": "remote", "url": url, "enabled": true}, + }} +} + +// HarnessToolNames are the names one tool has in each harness's transcript. +func HarnessToolNames(tool string) []string { + return []string{"mcp__" + ServerName + "__" + tool, ServerName + "_" + tool, ServerName + "/" + tool} +} + +// ToolFromHarnessName returns the Pipelines tool a harness tool name denotes. +func ToolFromHarnessName(name string) (string, bool) { + for _, prefix := range []string{"mcp__" + ServerName + "__", ServerName + "_", ServerName + "/"} { + if tool, ok := strings.CutPrefix(name, prefix); ok && IsTool(tool) { + return tool, true + } + } + return "", false +} diff --git a/internal/chattools/mcp.go b/internal/chattools/mcp.go new file mode 100644 index 00000000..c318bae7 --- /dev/null +++ b/internal/chattools/mcp.go @@ -0,0 +1,172 @@ +package chattools + +import ( + "context" + "encoding/json" + "errors" + "io" + "net" + "net/http" + "slices" +) + +// MaxRequestBytes bounds one MCP request body: a full show_html document plus +// JSON escaping and the envelope. +const MaxRequestBytes = 4 << 20 + +// LatestProtocolVersion is the MCP revision this server implements. Older +// revisions a client asks for are accepted: the subset used here (initialize, +// tools/list, tools/call, ping over streamable HTTP with JSON responses) is the +// same in each. +const LatestProtocolVersion = "2025-06-18" + +var protocolVersions = []string{"2025-11-25", "2025-06-18", "2025-03-26", "2024-11-05"} + +// Result is one tool call's outcome: short text for the model. +type Result struct { + Text string + IsError bool +} + +// Service executes tool calls for the chat an endpoint token serves. +type Service interface { + // Known reports whether token is an issued endpoint token. + Known(ctx context.Context, token string) bool + // Call runs one tool. arguments is the raw JSON object the model sent. + Call(ctx context.Context, token, tool string, arguments json.RawMessage) Result +} + +// Handler serves the MCP streamable HTTP transport for one endpoint token. +type Handler struct { + Service Service + Version string +} + +type rpcRequest struct { + JSONRPC string `json:"jsonrpc"` + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method"` + Params json.RawMessage `json:"params,omitempty"` +} + +type rpcError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +type rpcResponse struct { + JSONRPC string `json:"jsonrpc"` + ID json.RawMessage `json:"id"` + Result any `json:"result,omitempty"` + Error *rpcError `json:"error,omitempty"` +} + +// Loopback reports whether a request came over the loopback interface. The +// endpoint is for harness processes on this machine only. +func Loopback(r *http.Request) bool { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return false + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +// ServeHTTP answers one POSTed JSON-RPC message. A browser request (any Origin +// header) and a non-loopback peer are refused, an unknown token is 404, and +// GET (the optional server-to-client stream) is not offered. +func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, token string) { + if r.Header.Get("Origin") != "" || !Loopback(r) { + http.Error(w, "forbidden", http.StatusForbidden) + return + } + if h.Service == nil || !h.Service.Known(r.Context(), token) { + http.NotFound(w, r) + return + } + if r.Method != http.MethodPost { + w.Header().Set("Allow", http.MethodPost) + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, MaxRequestBytes)) + if err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + writeRPC(w, rpcResponse{JSONRPC: "2.0", ID: json.RawMessage("null"), Error: &rpcError{Code: -32600, Message: "request exceeds 4 MiB"}}) + return + } + http.Error(w, "unreadable request", http.StatusBadRequest) + return + } + var request rpcRequest + if err := json.Unmarshal(body, &request); err != nil || request.JSONRPC != "2.0" || request.Method == "" { + writeRPC(w, rpcResponse{JSONRPC: "2.0", ID: json.RawMessage("null"), Error: &rpcError{Code: -32700, Message: "expected one JSON-RPC 2.0 request"}}) + return + } + if len(request.ID) == 0 || string(request.ID) == "null" { + // A notification (notifications/initialized, cancellations) has no reply. + w.WriteHeader(http.StatusAccepted) + return + } + response := rpcResponse{JSONRPC: "2.0", ID: request.ID} + response.Result, response.Error = h.dispatch(r.Context(), token, request) + writeRPC(w, response) +} + +func (h Handler) dispatch(ctx context.Context, token string, request rpcRequest) (any, *rpcError) { + switch request.Method { + case "initialize": + var params struct { + ProtocolVersion string `json:"protocolVersion"` + } + _ = json.Unmarshal(request.Params, ¶ms) + version := LatestProtocolVersion + if slices.Contains(protocolVersions, params.ProtocolVersion) { + version = params.ProtocolVersion + } + return map[string]any{ + "protocolVersion": version, + "capabilities": map[string]any{"tools": map[string]any{"listChanged": false}}, + "serverInfo": map[string]any{"name": ServerName, "title": "Pipelines", "version": h.version()}, + "instructions": Instructions, + }, nil + case "ping": + return map[string]any{}, nil + case "tools/list": + return map[string]any{"tools": definitions()}, nil + case "tools/call": + var params struct { + Name string `json:"name"` + Arguments json.RawMessage `json:"arguments"` + } + if err := json.Unmarshal(request.Params, ¶ms); err != nil || params.Name == "" { + return nil, &rpcError{Code: -32602, Message: "tools/call needs a tool name"} + } + if !IsTool(params.Name) { + return nil, &rpcError{Code: -32602, Message: "unknown tool: " + params.Name} + } + if len(params.Arguments) == 0 || string(params.Arguments) == "null" { + params.Arguments = json.RawMessage("{}") + } + result := h.Service.Call(ctx, token, params.Name, params.Arguments) + return map[string]any{ + "content": []map[string]any{{"type": "text", "text": result.Text}}, + "isError": result.IsError, + }, nil + } + return nil, &rpcError{Code: -32601, Message: "method not found: " + request.Method} +} + +func (h Handler) version() string { + if h.Version == "" { + return "dev" + } + return h.Version +} + +func writeRPC(w http.ResponseWriter, response rpcResponse) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + _ = json.NewEncoder(w).Encode(response) +} diff --git a/internal/chattools/mcp_test.go b/internal/chattools/mcp_test.go new file mode 100644 index 00000000..6cc9b7da --- /dev/null +++ b/internal/chattools/mcp_test.go @@ -0,0 +1,169 @@ +package chattools + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +type fakeService struct { + known map[string]bool + calls []string +} + +func (f *fakeService) Known(_ context.Context, token string) bool { return f.known[token] } + +func (f *fakeService) Call(_ context.Context, token, tool string, arguments json.RawMessage) Result { + f.calls = append(f.calls, token+" "+tool+" "+string(arguments)) + return Result{Text: "ok " + tool} +} + +func post(t *testing.T, h Handler, token, body string, mutate func(*http.Request)) *httptest.ResponseRecorder { + t.Helper() + request := httptest.NewRequest(http.MethodPost, "/mcp/chat/"+token, strings.NewReader(body)) + request.RemoteAddr = "127.0.0.1:50000" + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Accept", "application/json, text/event-stream") + if mutate != nil { + mutate(request) + } + recorder := httptest.NewRecorder() + h.ServeHTTP(recorder, request, token) + return recorder +} + +func decode(t *testing.T, recorder *httptest.ResponseRecorder) map[string]any { + t.Helper() + var out map[string]any + if err := json.Unmarshal(recorder.Body.Bytes(), &out); err != nil { + t.Fatalf("response %d %q: %v", recorder.Code, recorder.Body.String(), err) + } + return out +} + +func TestHandshakeListAndCall(t *testing.T) { + service := &fakeService{known: map[string]bool{"tok": true}} + h := Handler{Service: service, Version: "1.2.3"} + + init := decode(t, post(t, h, "tok", `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"x","version":"1"}}}`, nil)) + result := init["result"].(map[string]any) + if result["protocolVersion"] != "2025-03-26" { + t.Fatalf("a supported requested version must be echoed: %v", result) + } + if result["serverInfo"].(map[string]any)["name"] != ServerName || result["capabilities"].(map[string]any)["tools"] == nil { + t.Fatalf("initialize result = %v", result) + } + unknown := decode(t, post(t, h, "tok", `{"jsonrpc":"2.0","id":2,"method":"initialize","params":{"protocolVersion":"1999-01-01"}}`, nil)) + if unknown["result"].(map[string]any)["protocolVersion"] != LatestProtocolVersion { + t.Fatalf("an unknown version must get the latest: %v", unknown) + } + + if got := post(t, h, "tok", `{"jsonrpc":"2.0","method":"notifications/initialized"}`, nil); got.Code != http.StatusAccepted || got.Body.Len() != 0 { + t.Fatalf("a notification must be 202 with no body: %d %q", got.Code, got.Body.String()) + } + + list := decode(t, post(t, h, "tok", `{"jsonrpc":"2.0","id":"l","method":"tools/list"}`, nil)) + tools := list["result"].(map[string]any)["tools"].([]any) + var names []string + for _, tool := range tools { + entry := tool.(map[string]any) + names = append(names, entry["name"].(string)) + if entry["description"] == "" || entry["inputSchema"].(map[string]any)["type"] != "object" { + t.Fatalf("tool without description or schema: %v", entry) + } + } + if strings.Join(names, ",") != "show_file,show_image,show_html,notify" { + t.Fatalf("tools = %v", names) + } + if list["id"] != "l" { + t.Fatalf("a string id must be echoed: %v", list) + } + + call := decode(t, post(t, h, "tok", `{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"show_html","arguments":{"html":"x"}}}`, nil)) + content := call["result"].(map[string]any)["content"].([]any)[0].(map[string]any) + if content["type"] != "text" || content["text"] != "ok show_html" || call["result"].(map[string]any)["isError"] != false { + t.Fatalf("call result = %v", call) + } + if len(service.calls) != 1 || service.calls[0] != `tok show_html {"html":"x"}` { + t.Fatalf("service calls = %v", service.calls) + } + + bad := decode(t, post(t, h, "tok", `{"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"name":"rm_rf"}}`, nil)) + if bad["error"].(map[string]any)["code"].(float64) != -32602 { + t.Fatalf("an unknown tool must be invalid params: %v", bad) + } + missing := decode(t, post(t, h, "tok", `{"jsonrpc":"2.0","id":5,"method":"resources/list"}`, nil)) + if missing["error"].(map[string]any)["code"].(float64) != -32601 { + t.Fatalf("an unknown method must be method-not-found: %v", missing) + } + parse := decode(t, post(t, h, "tok", `[{"jsonrpc":"2.0","id":6,"method":"ping"}]`, nil)) + if parse["error"].(map[string]any)["code"].(float64) != -32700 { + t.Fatalf("a batch is refused: %v", parse) + } + if len(service.calls) != 1 { + t.Fatalf("refused requests must not call tools: %v", service.calls) + } +} + +func TestEndpointRefusesUnknownTokensBrowsersRemotePeersAndGET(t *testing.T) { + service := &fakeService{known: map[string]bool{"tok": true}} + h := Handler{Service: service} + ping := `{"jsonrpc":"2.0","id":1,"method":"ping"}` + if got := post(t, h, "other", ping, nil); got.Code != http.StatusNotFound { + t.Fatalf("unknown token = %d", got.Code) + } + if got := post(t, h, "tok", ping, func(r *http.Request) { r.Header.Set("Origin", "https://evil.example") }); got.Code != http.StatusForbidden { + t.Fatalf("browser origin = %d", got.Code) + } + if got := post(t, h, "tok", ping, func(r *http.Request) { r.RemoteAddr = "100.64.0.9:4000" }); got.Code != http.StatusForbidden { + t.Fatalf("remote peer = %d", got.Code) + } + if got := post(t, h, "tok", ping, func(r *http.Request) { r.RemoteAddr = "[::1]:4000" }); got.Code != http.StatusOK { + t.Fatalf("IPv6 loopback = %d", got.Code) + } + if got := post(t, h, "tok", "", func(r *http.Request) { r.Method = http.MethodGet }); got.Code != http.StatusMethodNotAllowed { + t.Fatalf("GET = %d", got.Code) + } + large := `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"show_html","arguments":{"html":"` + strings.Repeat("a", MaxRequestBytes) + `"}}}` + if got := decode(t, post(t, h, "tok", large, nil)); got["error"] == nil { + t.Fatalf("an oversized request must be refused: %v", got) + } + if len(service.calls) != 0 { + t.Fatalf("refused requests reached the service: %v", service.calls) + } +} + +func TestHarnessRegistrationConfig(t *testing.T) { + url := "http://127.0.0.1:8080/mcp/chat/abc" + var claude map[string]map[string]map[string]string + if err := json.Unmarshal([]byte(ClaudeMCPConfig(url)), &claude); err != nil { + t.Fatal(err) + } + if got := claude["mcpServers"]["pipelines"]; got["type"] != "http" || got["url"] != url { + t.Fatalf("claude config = %v", claude) + } + if ClaudeAllowedTools() != "mcp__pipelines" { + t.Fatalf("claude allowed tools = %q", ClaudeAllowedTools()) + } + codex := CodexConfig(url)["mcp_servers"].(map[string]any)["pipelines"].(map[string]any) + if codex["url"] != url { + t.Fatalf("codex config = %v", codex) + } + open := OpenCodeConfig(url)["mcp"].(map[string]any)["pipelines"].(map[string]any) + if open["type"] != "remote" || open["url"] != url || open["enabled"] != true { + t.Fatalf("opencode config = %v", open) + } + for _, name := range HarnessToolNames(ToolShowFile) { + if tool, ok := ToolFromHarnessName(name); !ok || tool != ToolShowFile { + t.Fatalf("%s not recognised", name) + } + } + for _, name := range []string{"pipelines_rm", "mcp__other__show_file", "show_file", "bash"} { + if _, ok := ToolFromHarnessName(name); ok { + t.Fatalf("%s must not be a Pipelines tool", name) + } + } +} diff --git a/internal/chattools/registry.go b/internal/chattools/registry.go new file mode 100644 index 00000000..a97fd6ab --- /dev/null +++ b/internal/chattools/registry.go @@ -0,0 +1,53 @@ +// Package chattools is the harness-neutral half of the Pipelines chat tools: +// the MCP server a chat's model calls to show the operator a file, an image or +// an HTML view, or to notify their phone. The tools act on Pipelines itself, +// never on the harness or the workspace. +// +// The server (internal/server) owns tokens, persistence and the tool effects. +// The harness adapters (claude, codex, opencode) only need the endpoint URL to +// register, which they obtain from the Registry configured here, so none of +// them imports the server. +package chattools + +import "sync" + +// ServerName is the MCP server name every harness registers. It fixes the tool +// names the harnesses report: Claude "mcp__pipelines__show_file", OpenCode +// "pipelines_show_file", Codex "pipelines/show_file". +const ServerName = "pipelines" + +// Registry issues the loopback MCP endpoint URLs harnesses register. Each URL +// embeds an unguessable token the server resolves to the chat it serves. +type Registry interface { + // SessionURL is the endpoint dedicated to one chat session, stable across + // server restarts so a detached harness process keeps working. + SessionURL(sessionID string) (string, error) + // PendingURL is an endpoint for a chat whose id the harness assigns when + // it creates it (a Codex thread). bind attaches it to that id; calls are + // refused until then. + PendingURL() (url string, bind func(sessionID string) error, err error) + // InstanceURL is the endpoint of one harness process that serves many + // chats (the OpenCode service). Its calls are attributed only to the one + // chat whose matching tool call is running at that moment. + InstanceURL(harness string) (string, error) +} + +var ( + mu sync.RWMutex + current Registry +) + +// Configure installs the registry the harness adapters use. A nil registry +// disables registration: chats start without the Pipelines tools. +func Configure(registry Registry) { + mu.Lock() + defer mu.Unlock() + current = registry +} + +// Current returns the configured registry, or nil when tools are disabled. +func Current() Registry { + mu.RLock() + defer mu.RUnlock() + return current +} diff --git a/internal/chattools/tools.go b/internal/chattools/tools.go new file mode 100644 index 00000000..db6e842b --- /dev/null +++ b/internal/chattools/tools.go @@ -0,0 +1,101 @@ +package chattools + +// The tool names, as the MCP server lists them. +const ( + ToolShowFile = "show_file" + ToolShowImage = "show_image" + ToolShowHTML = "show_html" + ToolNotify = "notify" +) + +// MaxHTMLBytes bounds one show_html document. A larger one is refused, never +// truncated: a cut document would render as something the model did not write. +const MaxHTMLBytes = 512 << 10 + +// IsTool reports whether name is one of the Pipelines tools. +func IsTool(name string) bool { + switch name { + case ToolShowFile, ToolShowImage, ToolShowHTML, ToolNotify: + return true + } + return false +} + +// Instructions is the MCP initialize instructions text. +const Instructions = "These tools act on Pipelines, the app the operator reads this chat in, not on your workspace. " + + "Use show_file / show_image instead of telling the operator a path to open, show_html for diagrams, charts or small visual views, " + + "and notify only when the operator should look at their phone now." + +type toolDefinition struct { + Name string `json:"name"` + Title string `json:"title,omitempty"` + Description string `json:"description"` + InputSchema map[string]any `json:"inputSchema"` + Annotations map[string]any `json:"annotations,omitempty"` +} + +func stringProperty(description string) map[string]any { + return map[string]any{"type": "string", "description": description} +} + +func objectSchema(required []string, properties map[string]any) map[string]any { + return map[string]any{"type": "object", "properties": properties, "required": required, "additionalProperties": false} +} + +// definitions is the tools/list result, in a fixed order. +func definitions() []toolDefinition { + return []toolDefinition{ + { + Name: ToolShowFile, + Title: "Show a file in the chat", + Description: "Show the operator an existing file from this machine directly in the Pipelines chat: images appear inline, " + + "documents (PDF, Markdown, text, CSV, HTML, and others) as a preview card they can tap to open or download. " + + "Use this instead of telling the operator a path to navigate to. The file must be inside the operator's home directory. " + + "A relative path is resolved against the chat's working directory.", + InputSchema: objectSchema([]string{"path"}, map[string]any{ + "path": stringProperty("Absolute path, ~/path, or a path relative to the chat's working directory."), + "title": stringProperty("Optional short caption shown above the file."), + }), + Annotations: map[string]any{"readOnlyHint": true, "openWorldHint": false}, + }, + { + Name: ToolShowImage, + Title: "Show an image in the chat", + Description: "Show the operator an image inline in the Pipelines chat, from a file on this machine (inside the operator's home) " + + "or from an https:// URL. The URL is loaded by the operator's device without cookies.", + InputSchema: objectSchema([]string{"url_or_path"}, map[string]any{ + "url_or_path": stringProperty("An https:// image URL, or a file path as for show_file."), + "title": stringProperty("Optional short caption."), + "alt": stringProperty("Optional description of the image for accessibility."), + }), + Annotations: map[string]any{"readOnlyHint": true}, + }, + { + Name: ToolShowHTML, + Title: "Show an HTML view in the chat", + Description: "Render a self-contained HTML or SVG document inline in the Pipelines chat: diagrams, charts, tables, small visual views. " + + "It is shown in a sandbox with no scripts, no network and no access to the app: use inline CSS and inline SVG, " + + "
for expandable parts and CSS :hover for interaction; JavaScript does not run and external resources do not load. " + + "The operator can open it full screen. At most 512 KiB.", + InputSchema: objectSchema([]string{"html"}, map[string]any{ + "html": stringProperty("The complete HTML (or a bare ) document."), + "title": stringProperty("Optional short caption."), + "height_hint": map[string]any{"type": "integer", "description": "Optional preferred inline height in logical pixels (120-600)."}, + }), + Annotations: map[string]any{"readOnlyHint": true, "openWorldHint": false}, + }, + { + Name: ToolNotify, + Title: "Notify the operator's phone", + Description: "Send a push notification to the operator's phone that opens this chat when tapped. " + + "Use it sparingly, for things they should know now (a long task finished, you are blocked on them). " + + "Limited to 5 per minute per chat. Returns whether delivery was accepted.", + InputSchema: objectSchema([]string{"title", "message"}, map[string]any{ + "title": stringProperty("Short notification title (one line)."), + "message": stringProperty("Notification text (one or two sentences)."), + "priority": map[string]any{"type": "string", "enum": []string{"low", "default", "high"}, "description": "Optional priority; default is default."}, + }), + Annotations: map[string]any{"readOnlyHint": false, "destructiveHint": false, "openWorldHint": true}, + }, + } +} diff --git a/internal/claude/claude.go b/internal/claude/claude.go index fd8c35b4..09d93cf0 100644 --- a/internal/claude/claude.go +++ b/internal/claude/claude.go @@ -320,6 +320,9 @@ type session struct { model string // CLI model selection (alias or full id); "" uses the CLI default mode string // permission mode id; "" uses the CLI's own default createdAt int64 + // toolsOff is set after a spawn carrying the Pipelines tools failed, so + // the chat keeps starting without them (chattools). + toolsOff bool proc *process starting chan struct{} diff --git a/internal/claude/pipelines_tools.go b/internal/claude/pipelines_tools.go new file mode 100644 index 00000000..577f87f3 --- /dev/null +++ b/internal/claude/pipelines_tools.go @@ -0,0 +1,43 @@ +package claude + +import ( + "log" + + "git.a-wan.net/alan/pipelines/internal/chattools" +) + +// pipelinesToolArgs registers the Pipelines MCP server for one chat: its own +// endpoint (a per-session token) as an inline --mcp-config, added to the +// operator's configured MCP servers rather than replacing them, with the tools +// pre-approved because they only present content in Pipelines. Without a +// configured registry, or when the endpoint cannot be issued, the chat starts +// without them. +func pipelinesToolArgs(sessionID string) []string { + registry := chattools.Current() + if registry == nil { + return nil + } + url, err := registry.SessionURL(sessionID) + if err != nil { + log.Printf("claude chat %s: Pipelines tools not registered: %v", sessionID, err) + return nil + } + return []string{"--mcp-config", chattools.ClaudeMCPConfig(url), "--allowedTools", chattools.ClaudeAllowedTools()} +} + +// disableTools reacts to a failed spawn: when the Pipelines tools were part of +// it, they are turned off for this chat (logged with the reason) and true asks +// the caller to retry without them. +func (s *session) disableTools(err error) bool { + if chattools.Current() == nil { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + if s.toolsOff { + return false + } + s.toolsOff = true + log.Printf("claude chat %s: start with the Pipelines tools failed (%v); retrying without them", s.id, err) + return true +} diff --git a/internal/claude/pipelines_tools_test.go b/internal/claude/pipelines_tools_test.go new file mode 100644 index 00000000..8943d9bb --- /dev/null +++ b/internal/claude/pipelines_tools_test.go @@ -0,0 +1,68 @@ +package claude + +import ( + "encoding/json" + "errors" + "slices" + "testing" + + "git.a-wan.net/alan/pipelines/internal/chattools" +) + +type toolRegistry struct{ fail bool } + +func (r toolRegistry) SessionURL(id string) (string, error) { + if r.fail { + return "", errors.New("no store") + } + return "http://127.0.0.1:9/mcp/chat/token-" + id, nil +} +func (toolRegistry) PendingURL() (string, func(string) error, error) { + return "", nil, errors.New("unused") +} +func (toolRegistry) InstanceURL(string) (string, error) { return "", errors.New("unused") } + +func TestCLIArgsRegisterThePipelinesToolsPerSession(t *testing.T) { + chat := &Chat{sessions: map[string]*session{}} + s := chat.sessionFor("id-1", "/fixture") + args, _ := s.cliArgs(false) + if slices.Contains(args, "--mcp-config") { + t.Fatalf("no registry must add no MCP config: %q", args) + } + + chattools.Configure(toolRegistry{}) + t.Cleanup(func() { chattools.Configure(nil) }) + args, _ = s.cliArgs(true) + var config struct { + MCPServers map[string]struct { + Type string `json:"type"` + URL string `json:"url"` + } `json:"mcpServers"` + } + if err := json.Unmarshal([]byte(argValue(args, "--mcp-config")), &config); err != nil { + t.Fatalf("--mcp-config is not JSON: %q", args) + } + if got := config.MCPServers["pipelines"]; got.Type != "http" || got.URL != "http://127.0.0.1:9/mcp/chat/token-id-1" { + t.Fatalf("mcp config = %+v", config) + } + if argValue(args, "--allowedTools") != "mcp__pipelines" || argValue(args, "--resume") != "id-1" { + t.Fatalf("args = %q", args) + } + + // A start that failed with the tools retries once without them. + if !s.disableTools(errors.New("unknown option --mcp-config")) { + t.Fatal("the first failure must ask for a retry without the tools") + } + if s.disableTools(errors.New("again")) { + t.Fatal("a chat already without the tools must not retry again") + } + if args, _ = s.cliArgs(true); slices.Contains(args, "--mcp-config") { + t.Fatalf("a chat whose start refused the tools must start without them: %q", args) + } + + chattools.Configure(toolRegistry{fail: true}) + other := chat.sessionFor("id-2", "/fixture") + if args, _ = other.cliArgs(false); slices.Contains(args, "--mcp-config") { + t.Fatalf("an unissued endpoint must start without the tools: %q", args) + } +} diff --git a/internal/claude/process.go b/internal/claude/process.go index e9bcc5cd..a82c1bec 100644 --- a/internal/claude/process.go +++ b/internal/claude/process.go @@ -95,8 +95,11 @@ func (s *session) cliArgs(resume bool) ([]string, string) { args = append(args, "--session-id", s.id) } s.mu.Lock() - model, mode, title := s.model, s.mode, s.title + model, mode, title, toolsOff := s.model, s.mode, s.title, s.toolsOff s.mu.Unlock() + if !toolsOff { + args = append(args, pipelinesToolArgs(s.id)...) + } if model != "" { args = append(args, "--model", model) } @@ -130,6 +133,18 @@ func (s *session) spawn(ctx context.Context) (*process, error) { if p := s.existingHost(); p != nil { return p, nil } + p, err := s.spawnOnce(ctx) + if err == nil || ctx.Err() != nil || !s.disableTools(err) { + return p, err + } + if p := s.existingHost(); p != nil { + return p, nil + } + return s.spawnOnce(ctx) +} + +// spawnOnce starts and initializes one CLI process with the current arguments. +func (s *session) spawnOnce(ctx context.Context) (*process, error) { args, mode := s.cliArgs(findTranscriptFile(s.id) != "") env, notices := s.chat.processEnv(ctx, s.directory) s.mu.Lock() diff --git a/internal/codex/chat.go b/internal/codex/chat.go index f8fd5baa..c0d64111 100644 --- a/internal/codex/chat.go +++ b/internal/codex/chat.go @@ -461,7 +461,7 @@ func (c *Chat) PromptWithMessageIDModelFiles(ctx context.Context, id, text, mess var refused *rpcCallError if errors.As(err, &refused) && refused.Message == "thread not found: "+id { var resumed json.RawMessage - resumed, err = c.call(ctx, "thread/resume", map[string]any{"threadId": id, "excludeTurns": true}) + resumed, err = c.resumeThread(ctx, map[string]any{"threadId": id, "excludeTurns": true}, id) if err == nil { var response struct { Thread struct { @@ -549,7 +549,7 @@ func (c *Chat) Create(ctx context.Context, title, directory string) (protocol.Ch if strings.TrimSpace(directory) == "" { return protocol.ChatSession{}, errors.New("a directory is required") } - raw, err := c.call(ctx, "thread/start", map[string]any{ + raw, err := c.startThread(ctx, map[string]any{ "cwd": directory, "sandbox": "workspace-write", }) diff --git a/internal/codex/pipelines_tools.go b/internal/codex/pipelines_tools.go new file mode 100644 index 00000000..c684743d --- /dev/null +++ b/internal/codex/pipelines_tools.go @@ -0,0 +1,76 @@ +package codex + +import ( + "context" + "encoding/json" + "errors" + "log" + "maps" + + "git.a-wan.net/alan/pipelines/internal/chattools" +) + +// withPipelinesTools returns params with the thread config override that +// registers the Pipelines MCP server at url, leaving params unchanged. +func withPipelinesTools(params map[string]any, url string) map[string]any { + next := maps.Clone(params) + next["config"] = chattools.CodexConfig(url) + return next +} + +// startThread calls thread/start registering the Pipelines tools for the new +// thread. Its id is assigned by the App Server, so the endpoint is issued +// pending and bound to the id the reply names. An App Server that refuses the +// override still gets the thread, started without the tools (logged). +func (c *Chat) startThread(ctx context.Context, params map[string]any) (json.RawMessage, error) { + registry := chattools.Current() + if registry == nil { + return c.call(ctx, "thread/start", params) + } + url, bind, err := registry.PendingURL() + if err != nil { + log.Printf("codex chat: Pipelines tools not registered: %v", err) + return c.call(ctx, "thread/start", params) + } + raw, err := c.call(ctx, "thread/start", withPipelinesTools(params, url)) + var refused *rpcCallError + if errors.As(err, &refused) { + log.Printf("codex chat: thread/start refused the Pipelines tools (%s); starting without them", refused.Message) + return c.call(ctx, "thread/start", params) + } + if err != nil { + return raw, err + } + var reply struct { + Thread struct { + ID string `json:"id"` + } `json:"thread"` + } + if json.Unmarshal(raw, &reply) == nil && reply.Thread.ID != "" { + if err := bind(reply.Thread.ID); err != nil { + log.Printf("codex chat %s: Pipelines tools endpoint not bound: %v", reply.Thread.ID, err) + } + } + return raw, nil +} + +// resumeThread calls thread/resume for id, registering the thread's own +// Pipelines tools endpoint; a refusal of the override resumes without it. +func (c *Chat) resumeThread(ctx context.Context, params map[string]any, id string) (json.RawMessage, error) { + registry := chattools.Current() + if registry == nil { + return c.call(ctx, "thread/resume", params) + } + url, err := registry.SessionURL(id) + if err != nil { + log.Printf("codex chat %s: Pipelines tools not registered: %v", id, err) + return c.call(ctx, "thread/resume", params) + } + raw, err := c.call(ctx, "thread/resume", withPipelinesTools(params, url)) + var refused *rpcCallError + if errors.As(err, &refused) && refused.Message != "thread not found: "+id { + log.Printf("codex chat %s: thread/resume refused the Pipelines tools (%s); resuming without them", id, refused.Message) + return c.call(ctx, "thread/resume", params) + } + return raw, err +} diff --git a/internal/codex/pipelines_tools_test.go b/internal/codex/pipelines_tools_test.go new file mode 100644 index 00000000..a5900651 --- /dev/null +++ b/internal/codex/pipelines_tools_test.go @@ -0,0 +1,100 @@ +package codex + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + + "git.a-wan.net/alan/pipelines/internal/chattools" +) + +type toolRegistry struct { + mu sync.Mutex + bound []string +} + +func (r *toolRegistry) SessionURL(id string) (string, error) { + return "http://127.0.0.1:9/mcp/chat/session-" + id, nil +} + +func (r *toolRegistry) PendingURL() (string, func(string) error, error) { + return "http://127.0.0.1:9/mcp/chat/pending", func(id string) error { + r.mu.Lock() + defer r.mu.Unlock() + r.bound = append(r.bound, id) + return nil + }, nil +} + +func (r *toolRegistry) InstanceURL(string) (string, error) { return "", errors.New("unused") } + +func mcpURL(t *testing.T, raw json.RawMessage) string { + t.Helper() + var params struct { + Config struct { + MCPServers map[string]struct { + URL string `json:"url"` + } `json:"mcp_servers"` + } `json:"config"` + } + if err := json.Unmarshal(raw, ¶ms); err != nil { + t.Fatal(err) + } + return params.Config.MCPServers["pipelines"].URL +} + +func TestThreadStartRegistersAndBindsThePipelinesTools(t *testing.T) { + registry := &toolRegistry{} + chattools.Configure(registry) + t.Cleanup(func() { chattools.Configure(nil) }) + var urls []string + chat := modelTestChat(t, func(method string, raw json.RawMessage) any { + switch method { + case "thread/start": + urls = append(urls, mcpURL(t, raw)) + return json.RawMessage(`{"thread":{"id":"thr-1","cwd":"/w"},"model":"gpt","modelProvider":"openai"}`) + case "thread/resume": + urls = append(urls, mcpURL(t, raw)) + return json.RawMessage(`{"thread":{"id":"thr-1"}}`) + } + return json.RawMessage(`{}`) + }) + if _, err := chat.Create(context.Background(), "", "/w"); err != nil { + t.Fatal(err) + } + if _, err := chat.resumeThread(context.Background(), map[string]any{"threadId": "thr-1"}, "thr-1"); err != nil { + t.Fatal(err) + } + if len(urls) != 2 || urls[0] != "http://127.0.0.1:9/mcp/chat/pending" || urls[1] != "http://127.0.0.1:9/mcp/chat/session-thr-1" { + t.Fatalf("registered urls = %v", urls) + } + if len(registry.bound) != 1 || registry.bound[0] != "thr-1" { + t.Fatalf("the pending endpoint must be bound to the new thread: %v", registry.bound) + } +} + +func TestThreadStartWithoutToolsWhenTheAppServerRefusesThem(t *testing.T) { + chattools.Configure(&toolRegistry{}) + t.Cleanup(func() { chattools.Configure(nil) }) + var attempts []string + chat := modelTestChat(t, func(method string, raw json.RawMessage) any { + if method != "thread/start" { + return json.RawMessage(`{}`) + } + url := mcpURL(t, raw) + attempts = append(attempts, url) + if url != "" { + return &rpcError{Code: -32602, Message: "invalid config: mcp_servers"} + } + return json.RawMessage(`{"thread":{"id":"thr-2","cwd":"/w"}}`) + }) + session, err := chat.Create(context.Background(), "", "/w") + if err != nil || session.ID != "thr-2" { + t.Fatalf("the chat must still start: %v %v", session, err) + } + if len(attempts) != 2 || attempts[0] == "" || attempts[1] != "" { + t.Fatalf("attempts = %q", attempts) + } +} diff --git a/internal/opencode/connection.go b/internal/opencode/connection.go index b6b71930..b067646a 100644 --- a/internal/opencode/connection.go +++ b/internal/opencode/connection.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "io" + "log" "net/http" "net/url" "os" @@ -307,13 +308,30 @@ func OpenChat(ctx context.Context, executable, directory string) (*Connection, e if err != nil { return nil, err } - if overrides != nil && backend == "v2" { - if c.configContent, err = configContent(os.Getenv("OPENCODE_CONFIG_CONTENT"), overrides); err != nil { + if backend != "v2" { + overrides = nil + } + // The Pipelines chat tools are registered on every spawned chat service. + withTools := withPipelinesTools(overrides) + if withTools != nil { + if c.configContent, err = configContent(os.Getenv("OPENCODE_CONFIG_CONTENT"), withTools); err != nil { return nil, err } } if err := c.startServe(ctx); err != nil { - return nil, err + if withTools == nil || ctx.Err() != nil { + return nil, err + } + log.Printf("OpenCode chat service did not start with the Pipelines tools (%v); starting without them", err) + c.process, c.configContent = nil, "" + if overrides != nil { + if c.configContent, err = configContent(os.Getenv("OPENCODE_CONFIG_CONTENT"), overrides); err != nil { + return nil, err + } + } + if err := c.startServe(ctx); err != nil { + return nil, err + } } return c, nil } diff --git a/internal/opencode/pipelines_tools.go b/internal/opencode/pipelines_tools.go new file mode 100644 index 00000000..31521cf2 --- /dev/null +++ b/internal/opencode/pipelines_tools.go @@ -0,0 +1,30 @@ +package opencode + +import ( + "log" + + "git.a-wan.net/alan/pipelines/internal/chattools" +) + +// withPipelinesTools adds the Pipelines MCP server to a chat service's config +// overrides. One OpenCode service serves every chat and its MCP requests name +// no session, so it registers the service's instance endpoint; the server +// attributes each call to the chat whose stream shows it running. The +// operator's own overrides win, including a server they named "pipelines". +// It returns overrides unchanged when the tools are not configured. +func withPipelinesTools(overrides map[string]any) map[string]any { + registry := chattools.Current() + if registry == nil { + return overrides + } + url, err := registry.InstanceURL("opencode") + if err != nil { + log.Printf("OpenCode chat: Pipelines tools not registered: %v", err) + return overrides + } + merged := chattools.OpenCodeConfig(url) + if overrides != nil { + mergeConfig(merged, overrides) + } + return merged +} diff --git a/internal/opencode/pipelines_tools_test.go b/internal/opencode/pipelines_tools_test.go new file mode 100644 index 00000000..2899162b --- /dev/null +++ b/internal/opencode/pipelines_tools_test.go @@ -0,0 +1,52 @@ +package opencode + +import ( + "encoding/json" + "errors" + "testing" + + "git.a-wan.net/alan/pipelines/internal/chattools" +) + +type toolRegistry struct{} + +func (toolRegistry) SessionURL(string) (string, error) { return "", errors.New("unused") } +func (toolRegistry) PendingURL() (string, func(string) error, error) { + return "", nil, errors.New("unused") +} +func (toolRegistry) InstanceURL(harness string) (string, error) { + return "http://127.0.0.1:9/mcp/chat/instance-" + harness, nil +} + +func TestChatServiceConfigRegistersThePipelinesTools(t *testing.T) { + if got := withPipelinesTools(nil); got != nil { + t.Fatalf("no registry must add nothing: %v", got) + } + chattools.Configure(toolRegistry{}) + t.Cleanup(func() { chattools.Configure(nil) }) + + content, err := configContent(`{"model":"x/y","mcp":{"mine":{"type":"local","command":["m"]}}}`, withPipelinesTools(nil)) + if err != nil { + t.Fatal(err) + } + var config struct { + Model string `json:"model"` + MCP map[string]map[string]any `json:"mcp"` + } + if err := json.Unmarshal([]byte(content), &config); err != nil { + t.Fatal(err) + } + if config.Model != "x/y" || config.MCP["mine"] == nil { + t.Fatalf("the operator's config must be kept: %s", content) + } + if got := config.MCP["pipelines"]; got["type"] != "remote" || got["url"] != "http://127.0.0.1:9/mcp/chat/instance-opencode" || got["enabled"] != true { + t.Fatalf("pipelines mcp = %v", got) + } + + // Chat overrides the operator set win, including their own "pipelines". + merged := withPipelinesTools(map[string]any{"mcp": map[string]any{"pipelines": map[string]any{"enabled": false}}, "limit": 1.0}) + server := merged["mcp"].(map[string]any)["pipelines"].(map[string]any) + if server["enabled"] != false || server["url"] == nil || merged["limit"] != 1.0 { + t.Fatalf("merged = %v", merged) + } +} diff --git a/internal/opencode/stage_owned.go b/internal/opencode/stage_owned.go index f325f694..871b9528 100644 --- a/internal/opencode/stage_owned.go +++ b/internal/opencode/stage_owned.go @@ -179,6 +179,19 @@ func EnsureOwnedChatService(ctx context.Context, executable, directory string) ( return nil, err } } + // The Pipelines chat tools are registered on the chat service; a service + // that does not start with them is started without (logged). + if withTools := withPipelinesTools(overrides); withTools != nil { + toolsContent, err := configContent(os.Getenv("OPENCODE_CONFIG_CONTENT"), withTools) + if err != nil { + return nil, err + } + c, err := ensureOwnedService(ctx, chatOwnedService, toolsContent, executable, directory, nil, "") + if err == nil || ctx.Err() != nil { + return c, err + } + log.Printf("OpenCode chat service did not start with the Pipelines tools (%v); starting without them", err) + } return ensureOwnedService(ctx, chatOwnedService, content, executable, directory, nil, "") } diff --git a/internal/pipeline/chat_tools.go b/internal/pipeline/chat_tools.go new file mode 100644 index 00000000..4538be8b --- /dev/null +++ b/internal/pipeline/chat_tools.go @@ -0,0 +1,159 @@ +package pipeline + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// The chat tool tables: the MCP endpoint tokens a harness registers (each +// names the scope it serves: "session:", "pending:" until a Codex thread +// id is bound, or "instance:") and the items a chat's model showed. +func (s *Store) initChatTools() error { + _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS chat_tool_tokens ( +token TEXT PRIMARY KEY, +scope TEXT NOT NULL, +created_at INTEGER NOT NULL); +CREATE INDEX IF NOT EXISTS chat_tool_tokens_scope ON chat_tool_tokens(scope); +CREATE TABLE IF NOT EXISTS chat_shown_items ( +id TEXT PRIMARY KEY, +session_id TEXT NOT NULL, +created_at INTEGER NOT NULL, +payload TEXT NOT NULL); +CREATE INDEX IF NOT EXISTS chat_shown_items_session ON chat_shown_items(session_id, created_at);`) + return err +} + +// ChatToolToken returns the token issued for scope, issuing one when none is. +func (s *Store) ChatToolToken(ctx context.Context, scope string) (string, error) { + var token string + err := s.db.QueryRowContext(ctx, `SELECT token FROM chat_tool_tokens WHERE scope=? ORDER BY created_at, token LIMIT 1`, scope).Scan(&token) + if err == nil { + return token, nil + } + if !errors.Is(err, sql.ErrNoRows) { + return "", err + } + return s.IssueChatToolToken(ctx, scope) +} + +// IssueChatToolToken records a new random token for scope. +func (s *Store) IssueChatToolToken(ctx context.Context, scope string) (string, error) { + var secret [32]byte + if _, err := rand.Read(secret[:]); err != nil { + return "", err + } + token := hex.EncodeToString(secret[:]) + if _, err := s.db.ExecContext(ctx, `INSERT INTO chat_tool_tokens(token,scope,created_at) VALUES(?,?,?)`, token, scope, time.Now().UnixMilli()); err != nil { + return "", err + } + return token, nil +} + +// ChatToolScope returns the scope a token serves (found false for an unknown +// token). +func (s *Store) ChatToolScope(ctx context.Context, token string) (string, bool, error) { + var scope string + err := s.db.QueryRowContext(ctx, `SELECT scope FROM chat_tool_tokens WHERE token=?`, token).Scan(&scope) + if errors.Is(err, sql.ErrNoRows) { + return "", false, nil + } + if err != nil { + return "", false, err + } + return scope, true, nil +} + +// BindChatToolToken moves a token to scope (a pending Codex token to its thread). +func (s *Store) BindChatToolToken(ctx context.Context, token, scope string) error { + result, err := s.db.ExecContext(ctx, `UPDATE chat_tool_tokens SET scope=? WHERE token=?`, scope, token) + if err != nil { + return err + } + if n, _ := result.RowsAffected(); n == 0 { + return errors.New("unknown chat tool token") + } + return nil +} + +// AppendChatShownItem records one shown item and applies the retention bounds: +// at most protocol.ChatShownLimit items per chat, and HTML bodies only for the +// newest documents within protocol.ChatShownHTMLBudget. +func (s *Store) AppendChatShownItem(ctx context.Context, sessionID string, item protocol.ChatShownItem) error { + payload, err := json.Marshal(item) + if err != nil { + return err + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.ExecContext(ctx, `INSERT INTO chat_shown_items(id,session_id,created_at,payload) VALUES(?,?,?,?)`, item.ID, sessionID, item.CreatedAt, string(payload)); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM chat_shown_items WHERE session_id=? AND id NOT IN ( +SELECT id FROM chat_shown_items WHERE session_id=? ORDER BY created_at DESC, id DESC LIMIT ?)`, sessionID, sessionID, protocol.ChatShownLimit); err != nil { + return err + } + items, err := chatShownItems(ctx, tx, sessionID) + if err != nil { + return err + } + budget := protocol.ChatShownHTMLBudget + for i := len(items) - 1; i >= 0; i-- { + current := items[i] + if current.Kind != protocol.ChatShownHTML || current.Expired { + continue + } + if len(current.HTML) <= budget { + budget -= len(current.HTML) + continue + } + budget = 0 + current.HTML, current.Expired = "", true + data, err := json.Marshal(current) + if err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `UPDATE chat_shown_items SET payload=? WHERE id=?`, string(data), current.ID); err != nil { + return err + } + } + return tx.Commit() +} + +// ChatShownItems returns a chat's retained items, oldest first. +func (s *Store) ChatShownItems(ctx context.Context, sessionID string) ([]protocol.ChatShownItem, error) { + return chatShownItems(ctx, s.db, sessionID) +} + +type chatShownQueryer interface { + QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) +} + +func chatShownItems(ctx context.Context, db chatShownQueryer, sessionID string) ([]protocol.ChatShownItem, error) { + rows, err := db.QueryContext(ctx, `SELECT payload FROM chat_shown_items WHERE session_id=? ORDER BY created_at, id`, sessionID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []protocol.ChatShownItem + for rows.Next() { + var payload string + if err := rows.Scan(&payload); err != nil { + return nil, err + } + var item protocol.ChatShownItem + if json.Unmarshal([]byte(payload), &item) == nil { + items = append(items, item) + } + } + return items, rows.Err() +} diff --git a/internal/pipeline/store.go b/internal/pipeline/store.go index 9a78b245..92642a95 100644 --- a/internal/pipeline/store.go +++ b/internal/pipeline/store.go @@ -105,6 +105,10 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU db.Close() return nil, err } + if err := s.initChatTools(); err != nil { + db.Close() + return nil, err + } if err := s.initRunSizes(); err != nil { db.Close() return nil, err diff --git a/internal/protocol/chat_shown.go b/internal/protocol/chat_shown.go new file mode 100644 index 00000000..932df197 --- /dev/null +++ b/internal/protocol/chat_shown.go @@ -0,0 +1,55 @@ +package protocol + +// ChatShown is the pushed frame type for an item a chat's model showed the +// operator through the Pipelines tools (show_file, show_image, show_html, +// notify). The chat_subscribe reply carries the retained list in the same +// ChatShown field. +const ChatShown = "chat_shown" + +// The ChatShownItem kinds. +const ( + ChatShownFile = "file" + ChatShownImage = "image" + ChatShownHTML = "html" + ChatShownNotify = "notify" +) + +// ChatShownLimit bounds the items retained per chat, and ChatShownHTMLBudget +// the HTML bytes kept across them: older documents beyond it keep their entry +// but drop the body (Expired). +const ( + ChatShownLimit = 100 + ChatShownHTMLBudget = 2 << 20 +) + +// ChatShownFrame is one chat's shown items, oldest first. +type ChatShownFrame struct { + SessionID string `json:"session_id"` + Items []ChatShownItem `json:"items"` +} + +// ChatShownItem is one entry. A file or image Path is an absolute path on the +// chat's own machine, served by that machine's /files endpoint; URL is an +// https image address the client loads without credentials. +type ChatShownItem struct { + ID string `json:"id"` + Kind string `json:"kind"` + CreatedAt int64 `json:"created_at"` + Title string `json:"title,omitempty"` + Path string `json:"path,omitempty"` + Name string `json:"name,omitempty"` + Size int64 `json:"size,omitempty"` + Mime string `json:"mime,omitempty"` + URL string `json:"url,omitempty"` + Alt string `json:"alt,omitempty"` + HTML string `json:"html,omitempty"` + HTMLBytes int `json:"html_bytes,omitempty"` + // HeightHint is the model's preferred inline height in logical pixels. + HeightHint int `json:"height_hint,omitempty"` + Expired bool `json:"expired,omitempty"` + Message string `json:"message,omitempty"` + Priority string `json:"priority,omitempty"` + Delivered bool `json:"delivered,omitempty"` + // Detail says why a notification was not delivered. + Detail string `json:"detail,omitempty"` +} diff --git a/internal/protocol/message.go b/internal/protocol/message.go index 3b4d0e37..305cb1d1 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -254,6 +254,10 @@ type Message struct { // chat_unqueue replies. ChatQueue *ChatQueueFrame `json:"chat_queue,omitempty"` ChatUnqueue *ChatUnqueueRequest `json:"chat_unqueue,omitempty"` + // ChatShown carries the items the chat's model showed through the Pipelines + // tools: the retained list on the chat_subscribe reply, one new item on a + // pushed chat_shown frame. + ChatShown *ChatShownFrame `json:"chat_shown,omitempty"` // ChatDirectories carries the operator's directory choices on a // chat_directories response (home first, then recent operator-session // directories, deduplicated and bounded). Additive/omitempty so older peers diff --git a/internal/server/chat_hub.go b/internal/server/chat_hub.go index 71686418..b43f86cf 100644 --- a/internal/server/chat_hub.go +++ b/internal/server/chat_hub.go @@ -74,6 +74,9 @@ type chatHub struct { // and its sequence source (chat_activity.go). activity map[string]protocol.ChatActivity activitySeq int64 + // shown holds the items chats showed through the Pipelines tools; the + // subscribe reply carries a chat's retained list. nil records none. + shown chatShownStore } // chatPart is the retained form of one part. Text and reasoning deltas append to diff --git a/internal/server/chat_stream.go b/internal/server/chat_stream.go index 223f752a..647c5a35 100644 --- a/internal/server/chat_stream.go +++ b/internal/server/chat_stream.go @@ -230,7 +230,7 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH // The reply carries the prompts the server still holds for the session, so // a reloaded client draws them as Queued without any device storage. queue := hub.queueFrame(query.SessionID) - _ = write(protocol.Message{Type: protocol.ChatSubscribe, ID: request.ID, ChatQueue: &queue}) + _ = write(protocol.Message{Type: protocol.ChatSubscribe, ID: request.ID, ChatQueue: &queue, ChatShown: hub.shownFrame(query.SessionID)}) if err := hub.ensureObserving(sub.room, broker); err != nil { sub.close() _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "could not observe session " + query.SessionID}) diff --git a/internal/server/chat_tools_correlate.go b/internal/server/chat_tools_correlate.go new file mode 100644 index 00000000..4d735531 --- /dev/null +++ b/internal/server/chat_tools_correlate.go @@ -0,0 +1,75 @@ +package server + +import ( + "context" + "encoding/json" + "reflect" + "slices" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// awaitToolCallSession attributes a call on a shared harness endpoint (one +// OpenCode service serves every chat, and its MCP requests carry no session) +// to the chat whose retained stream shows a running tool part with one of +// names and exactly these arguments. The harness reports the part as running +// before or while it calls the tool, so this waits up to wait for it. It +// returns "" when no chat matches, and ambiguous when several distinct chats +// do: a call is never guessed into a chat. +func (h *chatHub) awaitToolCallSession(ctx context.Context, names []string, arguments json.RawMessage, wait time.Duration) (string, bool) { + var want any + if json.Unmarshal(arguments, &want) != nil { + return "", false + } + deadline := time.Now().Add(wait) + for { + sessions := h.runningToolCallSessions(names, want) + switch { + case len(sessions) == 1: + return sessions[0], false + case len(sessions) > 1: + return "", true + } + if time.Now().After(deadline) { + return "", false + } + select { + case <-ctx.Done(): + return "", false + case <-time.After(50 * time.Millisecond): + } + } +} + +func (h *chatHub) runningToolCallSessions(names []string, want any) []string { + h.mu.Lock() + defer h.mu.Unlock() + var sessions []string + for sessionID, turn := range h.turns { + if turnHasToolCall(turn, names, want) { + sessions = append(sessions, sessionID) + } + } + slices.Sort(sessions) + return sessions +} + +func turnHasToolCall(turn *chatTurn, names []string, want any) bool { + for _, message := range turn.messages { + for _, part := range message.parts { + p := part.part + if p.Type != "tool" || p.State == nil || !slices.Contains(names, p.Name) { + continue + } + if p.State.Status != protocol.ChatToolRunning && p.State.Status != "pending" { + continue + } + var got any + if json.Unmarshal(p.State.Input, &got) == nil && reflect.DeepEqual(got, want) { + return true + } + } + } + return false +} diff --git a/internal/server/chat_tools_files.go b/internal/server/chat_tools_files.go new file mode 100644 index 00000000..2438f032 --- /dev/null +++ b/internal/server/chat_tools_files.go @@ -0,0 +1,143 @@ +package server + +import ( + "context" + "errors" + "io" + "log" + "mime" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// shownFile is a resolved file inside the operator's home. +type shownFile struct { + path string + name string + size int64 + mime string +} + +func (f shownFile) item(kind, title string) protocol.ChatShownItem { + return protocol.ChatShownItem{Kind: kind, Title: title, Path: f.path, Name: f.name, Size: f.size, Mime: f.mime} +} + +// resolveShownFile applies the /files endpoint's containment rules to a model +// supplied path: it must name an existing regular file whose real path (every +// symlink resolved) is inside the operator's home and not on a special +// filesystem. A relative path is taken from the chat's working directory. +func (s *chatToolService) resolveShownFile(ctx context.Context, sessionID, requested string) (shownFile, error) { + requested = strings.TrimSpace(requested) + if requested == "" { + return shownFile{}, errors.New("a path is required") + } + home, err := s.home() + if err != nil { + return shownFile{}, err + } + home = filepath.Clean(home) + realHome, err := filepath.EvalSymlinks(home) + if err != nil { + return shownFile{}, errors.New("the operator's home directory is not available") + } + if !filepath.IsAbs(requested) && !strings.HasPrefix(requested, "~") { + directory := s.chatSessionDirectory(ctx, sessionID) + if directory == "" { + return shownFile{}, errors.New("path " + requested + " is relative and this chat has no recorded working directory; pass an absolute path") + } + requested = filepath.Join(directory, requested) + } + target, err := expandHomePath(home, requested) + if err != nil { + return shownFile{}, err + } + _, realTarget, err := resolveInsideHome(realHome, target) + if err != nil { + if strings.Contains(err.Error(), "outside the home directory") { + return shownFile{}, errors.New("path " + requested + " is outside the operator's home directory; only files inside home can be shown") + } + return shownFile{}, err + } + info, err := os.Stat(realTarget) + if err != nil { + return shownFile{}, describePathError(requested, err) + } + if info.IsDir() { + return shownFile{}, errors.New(requested + " is a directory; show one file") + } + if !info.Mode().IsRegular() || specialFilesystem(realTarget) { + return shownFile{}, errors.New(requested + " is not a regular file") + } + return shownFile{path: realTarget, name: filepath.Base(realTarget), size: info.Size(), mime: shownFileMime(realTarget)}, nil +} + +// shownFileMime is the type the /files endpoint will serve: by extension, else +// sniffed from the first bytes. +func shownFileMime(path string) string { + if byExtension := mime.TypeByExtension(filepath.Ext(path)); byExtension != "" { + media, _, err := mime.ParseMediaType(byExtension) + if err == nil { + return media + } + } + file, err := os.Open(path) + if err != nil { + return "application/octet-stream" + } + defer file.Close() + var sample [512]byte + n, err := file.Read(sample[:]) + if err != nil && !errors.Is(err, io.EOF) { + return "application/octet-stream" + } + media, _, _ := mime.ParseMediaType(http.DetectContentType(sample[:n])) + if media == "" { + return "application/octet-stream" + } + return media +} + +// shownImageURL accepts only an absolute https URL without credentials. +func shownImageURL(raw string) (string, error) { + if len(raw) > 2048 { + return "", errors.New("the image URL is longer than 2048 characters") + } + parsed, err := url.Parse(raw) + if err != nil || parsed.Host == "" { + return "", errors.New("the image URL is not a valid absolute URL") + } + if parsed.Scheme != "https" { + return "", errors.New("only https:// image URLs can be shown") + } + if parsed.User != nil { + return "", errors.New("the image URL must not carry credentials") + } + return parsed.String(), nil +} + +// chatToolNotifier sends one model-requested phone notification for a chat and +// reports whether ntfy accepted it, or why not. +type chatToolNotifier func(ctx context.Context, sessionID, title, message string) (bool, string) + +// publishToolNotification publishes through the operator's existing "system" +// ntfy destination (the one chat attention uses), honoring their ntfy switch. +// Tapping it opens the chat (pipelines://chat/). +func (h *chatHub) publishToolNotification(ctx context.Context, sessionID, title, message string) (bool, string) { + if !chatAttentionConfigured() { + return false, "phone notifications (ntfy) are not configured on this Pipelines server" + } + if !chatNotificationSettings(ctx, h.settings).NtfyEnabled { + return false, "the operator has turned phone notifications off" + } + click, views := chatAttentionLinks(sessionID) + if _, err := chatAttentionPublish(ctx, "system", truncateUTF8(title, 256), truncateUTF8(message, 2000), "", nil, click, views...); err != nil { + log.Printf("chat tool notification session=%s: %v", sessionID, err) + return false, "the notification service refused it: " + err.Error() + } + return true, "" +} diff --git a/internal/server/chat_tools_http_test.go b/internal/server/chat_tools_http_test.go new file mode 100644 index 00000000..0d4ce7b0 --- /dev/null +++ b/internal/server/chat_tools_http_test.go @@ -0,0 +1,84 @@ +package server + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/chattools" + "git.a-wan.net/alan/pipelines/internal/pipeline" +) + +// The endpoint is served on the server's own listener: a harness registers +// the URL the registry issues and calls show_html over it. +func TestChatToolsEndpointOnTheServerListener(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "tools-http.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + handler := HandlerWithOptions(ctx, Options{}, store) + defer func() { cancel(); WaitRuns(handler) }() + srv := httptest.NewServer(handler) + defer srv.Close() + + if err := ConfigureChatTools(store, strings.TrimPrefix(srv.URL, "http://")); err != nil { + t.Fatal(err) + } + defer chattools.Configure(nil) + url, err := chattools.Current().SessionURL("ses_http") + if err != nil || !strings.HasPrefix(url, srv.URL+"/mcp/chat/") { + t.Fatalf("session url = %q %v", url, err) + } + rpc := func(target, body string) (int, map[string]any) { + request, _ := http.NewRequestWithContext(ctx, http.MethodPost, target, strings.NewReader(body)) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Accept", "application/json, text/event-stream") + response, err := http.DefaultClient.Do(request) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + var out map[string]any + _ = json.NewDecoder(response.Body).Decode(&out) + return response.StatusCode, out + } + if code, out := rpc(url, `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18"}}`); code != 200 || out["result"] == nil { + t.Fatalf("initialize = %d %v", code, out) + } + code, out := rpc(url, `{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"show_html","arguments":{"html":""}}}`) + if code != 200 || out["result"].(map[string]any)["isError"] != false { + t.Fatalf("tools/call = %d %v", code, out) + } + if items, _ := store.ChatShownItems(ctx, "ses_http"); len(items) != 1 { + t.Fatalf("items = %v", items) + } + if code, _ := rpc(srv.URL+"/mcp/chat/"+strings.Repeat("f", 64), `{"jsonrpc":"2.0","id":1,"method":"ping"}`); code != http.StatusNotFound { + t.Fatalf("unknown token = %d", code) + } +} + +func TestLoopbackBase(t *testing.T) { + for addr, want := range map[string]string{ + "127.0.0.1:8080": "http://127.0.0.1:8080", + ":8080": "http://127.0.0.1:8080", + "0.0.0.0:9": "http://127.0.0.1:9", + "[::]:9": "http://127.0.0.1:9", + "[::1]:9": "http://[::1]:9", + "localhost:7": "http://localhost:7", + } { + if got, err := loopbackBase(addr); err != nil || got != want { + t.Fatalf("%s = %q %v", addr, got, err) + } + } + if _, err := loopbackBase("100.64.0.2:8080"); err == nil { + t.Fatal("a non-loopback listener cannot serve the loopback-only endpoint") + } +} diff --git a/internal/server/chat_tools_run.go b/internal/server/chat_tools_run.go new file mode 100644 index 00000000..98119dab --- /dev/null +++ b/internal/server/chat_tools_run.go @@ -0,0 +1,129 @@ +package server + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "unicode/utf8" + + "git.a-wan.net/alan/pipelines/internal/chattools" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// run executes one tool for sessionID. A non-nil item is recorded and shown; +// the result is the short text the model reads. +func (s *chatToolService) run(ctx context.Context, sessionID, tool string, arguments json.RawMessage) (*protocol.ChatShownItem, chattools.Result) { + fail := func(format string, args ...any) (*protocol.ChatShownItem, chattools.Result) { + return nil, chattools.Result{Text: fmt.Sprintf(format, args...), IsError: true} + } + var in struct { + Path string `json:"path"` + URLOrPath string `json:"url_or_path"` + Title string `json:"title"` + Alt string `json:"alt"` + HTML string `json:"html"` + HeightHint int `json:"height_hint"` + Message string `json:"message"` + Priority string `json:"priority"` + } + if err := json.Unmarshal(arguments, &in); err != nil { + return fail("The arguments must be a JSON object matching the %s schema: %v", tool, err) + } + title := boundedLine(in.Title, 200) + switch tool { + case chattools.ToolShowFile: + file, err := s.resolveShownFile(ctx, sessionID, in.Path) + if err != nil { + return fail("Nothing was shown: %v", err) + } + item := file.item(protocol.ChatShownFile, title) + return &item, chattools.Result{Text: fmt.Sprintf("Shown in the operator's Pipelines chat: %s (%s).", file.name, humanBytes(file.size))} + case chattools.ToolShowImage: + source := strings.TrimSpace(in.URLOrPath) + alt := boundedLine(in.Alt, 500) + if strings.Contains(source, "://") { + address, err := shownImageURL(source) + if err != nil { + return fail("Nothing was shown: %v", err) + } + item := protocol.ChatShownItem{Kind: protocol.ChatShownImage, Title: title, URL: address, Alt: alt} + return &item, chattools.Result{Text: "Shown in the operator's Pipelines chat: the image at " + address + "."} + } + file, err := s.resolveShownFile(ctx, sessionID, source) + if err != nil { + return fail("Nothing was shown: %v", err) + } + if !strings.HasPrefix(file.mime, "image/") { + return fail("Nothing was shown: %s is %s, not an image; use show_file for documents.", file.name, file.mime) + } + item := file.item(protocol.ChatShownImage, title) + item.Alt = alt + return &item, chattools.Result{Text: fmt.Sprintf("Shown in the operator's Pipelines chat: the image %s (%s).", file.name, humanBytes(file.size))} + case chattools.ToolShowHTML: + if strings.TrimSpace(in.HTML) == "" { + return fail("Nothing was shown: html is empty.") + } + if len(in.HTML) > chattools.MaxHTMLBytes { + return fail("Nothing was shown: the document is %s; the limit is 512 KiB. Simplify it (inline SVG, fewer data points) and call show_html again.", humanBytes(int64(len(in.HTML)))) + } + if !utf8.ValidString(in.HTML) { + return fail("Nothing was shown: html is not valid UTF-8.") + } + item := protocol.ChatShownItem{Kind: protocol.ChatShownHTML, Title: title, HTML: in.HTML, HTMLBytes: len(in.HTML), HeightHint: clampHeightHint(in.HeightHint)} + return &item, chattools.Result{Text: fmt.Sprintf("Shown in the operator's Pipelines chat (%s, sandboxed: scripts and network are disabled).", humanBytes(int64(len(in.HTML))))} + case chattools.ToolNotify: + return s.runNotify(ctx, sessionID, title, in.Message, in.Priority) + } + return fail("Unknown tool %s.", tool) +} + +func (s *chatToolService) runNotify(ctx context.Context, sessionID, title, message, priority string) (*protocol.ChatShownItem, chattools.Result) { + message = boundedLine(message, 2000) + if title == "" || message == "" { + return nil, chattools.Result{Text: "Not sent: title and message are both required.", IsError: true} + } + switch priority { + case "": + priority = "default" + case "low", "default", "high": + default: + return nil, chattools.Result{Text: "Not sent: priority must be low, default or high.", IsError: true} + } + if !s.limiter.allow(sessionID, s.now()) { + return nil, chattools.Result{Text: "Not sent: this chat already sent 5 notifications in the last minute. Wait before notifying again.", IsError: true} + } + delivered, detail := s.notify(ctx, sessionID, title, message) + item := protocol.ChatShownItem{Kind: protocol.ChatShownNotify, Title: title, Message: message, Priority: priority, Delivered: delivered, Detail: detail} + if !delivered { + return &item, chattools.Result{Text: "Not delivered: " + detail + ".", IsError: true} + } + return &item, chattools.Result{Text: "Delivery accepted: the notification was handed to the operator's phone notification service (ntfy)."} +} + +// boundedLine folds text to one line of at most limit bytes. +func boundedLine(text string, limit int) string { + return truncateUTF8(singleLine(text), limit) +} + +func clampHeightHint(hint int) int { + switch { + case hint <= 0: + return 0 + case hint < 120: + return 120 + case hint > 600: + return 600 + } + return hint +} + +func humanBytes(n int64) string { + switch { + case n >= 1<<20: + return fmt.Sprintf("%.1f MiB", float64(n)/(1<<20)) + case n >= 1<<10: + return fmt.Sprintf("%.1f KiB", float64(n)/(1<<10)) + } + return fmt.Sprintf("%d bytes", n) +} diff --git a/internal/server/chat_tools_service.go b/internal/server/chat_tools_service.go new file mode 100644 index 00000000..4c7705c9 --- /dev/null +++ b/internal/server/chat_tools_service.go @@ -0,0 +1,190 @@ +package server + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "log" + "net/http" + "strings" + "sync" + "time" + + "git.a-wan.net/alan/pipelines/internal/chattools" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// chatShownStore persists the items a chat's model showed (pipeline.Store). +type chatShownStore interface { + AppendChatShownItem(ctx context.Context, sessionID string, item protocol.ChatShownItem) error + ChatShownItems(ctx context.Context, sessionID string) ([]protocol.ChatShownItem, error) +} + +// chatToolCorrelationWait is how long a call on a shared (instance) endpoint +// waits for the harness stream to report the matching running tool call. +var chatToolCorrelationWait = 3 * time.Second + +// chatToolService runs the Pipelines tools (chattools.Service) for the chat an +// endpoint token serves, records each shown item and pushes it to the chat's +// subscribers. +type chatToolService struct { + tokens chatToolTokenStore + shown chatShownStore + hub *chatHub + registry operatorRegistry + home func() (string, error) + notify chatToolNotifier + limiter *chatNotifyLimiter + now func() time.Time +} + +func newChatToolService(tokens chatToolTokenStore, shown chatShownStore, hub *chatHub, registry operatorRegistry) *chatToolService { + return &chatToolService{tokens: tokens, shown: shown, hub: hub, registry: registry, home: OperatorHomeDirectory, + notify: hub.publishToolNotification, limiter: newChatNotifyLimiter(5, time.Minute), now: time.Now} +} + +// registerChatTools serves the MCP endpoint on mux. The route exists only on +// the server's own listener; neither the web proxy nor the peer relay forwards +// it. +func registerChatTools(mux *http.ServeMux, service *chatToolService, version string) { + handler := chattools.Handler{Service: service, Version: version} + mux.HandleFunc(chatToolPath+"{token}", func(w http.ResponseWriter, r *http.Request) { + handler.ServeHTTP(w, r, r.PathValue("token")) + }) +} + +func (s *chatToolService) Known(ctx context.Context, token string) bool { + if s == nil || s.tokens == nil || len(token) != 64 { + return false + } + _, found, err := s.tokens.ChatToolScope(ctx, token) + return err == nil && found +} + +func (s *chatToolService) Call(ctx context.Context, token, tool string, arguments json.RawMessage) chattools.Result { + sessionID, refusal := s.session(ctx, token, tool, arguments) + if refusal != "" { + return chattools.Result{Text: refusal, IsError: true} + } + item, result := s.run(ctx, sessionID, tool, arguments) + if item == nil { + return result + } + item.ID = newChatShownID() + item.CreatedAt = s.now().UnixMilli() + if s.shown != nil { + if err := s.shown.AppendChatShownItem(ctx, sessionID, *item); err != nil { + log.Printf("chat tools session=%s: record %s: %v", sessionID, tool, err) + return chattools.Result{Text: "Pipelines could not record this item: " + err.Error(), IsError: true} + } + } + if s.hub != nil { + s.hub.fanout(sessionID, protocol.Message{Type: protocol.ChatShown, + ChatShown: &protocol.ChatShownFrame{SessionID: sessionID, Items: []protocol.ChatShownItem{*item}}}, "") + } + return result +} + +// session resolves the chat a call belongs to. A session token names it; a +// shared harness instance token is attributed to the one chat whose stream +// shows this exact call running, and refused when none or several do. +func (s *chatToolService) session(ctx context.Context, token, tool string, arguments json.RawMessage) (string, string) { + scope, found, err := s.tokens.ChatToolScope(ctx, token) + if err != nil || !found { + return "", "This Pipelines endpoint is not recognised." + } + if id, ok := strings.CutPrefix(scope, chatToolSessionScope); ok && id != "" { + return id, "" + } + if _, ok := strings.CutPrefix(scope, chatToolInstanceScope); ok && s.hub != nil { + id, ambiguous := s.hub.awaitToolCallSession(ctx, chattools.HarnessToolNames(tool), arguments, chatToolCorrelationWait) + if ambiguous { + return "", "Pipelines could not tell which chat made this call (another chat made an identical one); retry with a different title." + } + if id == "" { + return "", "Pipelines could not match this call to a chat it is showing; the tools work only in chats opened through Pipelines." + } + return id, "" + } + return "", "This chat is not ready for Pipelines tools yet." +} + +// chatSessionDirectory is the chat's recorded working directory, or "". +func (s *chatToolService) chatSessionDirectory(ctx context.Context, sessionID string) string { + if s.registry == nil { + return "" + } + session, found, err := s.registry.OperatorSession(ctx, sessionID) + if err != nil || !found { + return "" + } + return session.Directory +} + +func (s *chatToolService) chatTitle(ctx context.Context, sessionID string) string { + if s.registry == nil { + return "" + } + session, found, err := s.registry.OperatorSession(ctx, sessionID) + if err != nil || !found { + return "" + } + return session.Title +} + +func newChatShownID() string { + var b [12]byte + _, _ = rand.Read(b[:]) + return "shown_" + hex.EncodeToString(b[:]) +} + +// chatNotifyLimiter is a per-chat sliding-window rate limit. +type chatNotifyLimiter struct { + mu sync.Mutex + limit int + window time.Duration + sent map[string][]time.Time +} + +func newChatNotifyLimiter(limit int, window time.Duration) *chatNotifyLimiter { + return &chatNotifyLimiter{limit: limit, window: window, sent: map[string][]time.Time{}} +} + +// allow records one notification for sessionID at now unless the window is +// full. +func (l *chatNotifyLimiter) allow(sessionID string, now time.Time) bool { + l.mu.Lock() + defer l.mu.Unlock() + kept := l.sent[sessionID][:0] + for _, at := range l.sent[sessionID] { + if now.Sub(at) < l.window { + kept = append(kept, at) + } + } + if len(kept) >= l.limit { + l.sent[sessionID] = kept + return false + } + l.sent[sessionID] = append(kept, now) + return true +} + +// shownFrame is a chat's retained shown items for its subscribe reply, or nil +// when it has none (or none are recorded on this server). +func (h *chatHub) shownFrame(sessionID string) *protocol.ChatShownFrame { + if h == nil || h.shown == nil { + return nil + } + ctx, cancel := context.WithTimeout(h.baseCtx, 5*time.Second) + defer cancel() + items, err := h.shown.ChatShownItems(ctx, sessionID) + if err != nil { + log.Printf("chat shown items session=%s: %v", sessionID, err) + return nil + } + if len(items) == 0 { + return nil + } + return &protocol.ChatShownFrame{SessionID: sessionID, Items: items} +} diff --git a/internal/server/chat_tools_test.go b/internal/server/chat_tools_test.go new file mode 100644 index 00000000..e124e903 --- /dev/null +++ b/internal/server/chat_tools_test.go @@ -0,0 +1,294 @@ +package server + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/chattools" + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +type chatToolFixture struct { + store *pipeline.Store + service *chatToolService + hub *chatHub + home string + notified []string +} + +func newChatToolFixture(t *testing.T) *chatToolFixture { + t.Helper() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "tools.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { store.Close() }) + f := &chatToolFixture{store: store, hub: newChatHub(), home: t.TempDir()} + f.hub.shown = store + f.service = newChatToolService(store, store, f.hub, store) + f.service.home = func() (string, error) { return f.home, nil } + f.service.notify = func(_ context.Context, sessionID, title, message string) (bool, string) { + f.notified = append(f.notified, sessionID+": "+title) + return true, "" + } + return f +} + +func (f *chatToolFixture) token(t *testing.T, scope string) string { + t.Helper() + token, err := f.store.ChatToolToken(context.Background(), scope) + if err != nil { + t.Fatal(err) + } + return token +} + +func (f *chatToolFixture) call(token, tool, arguments string) chattools.Result { + return f.service.Call(context.Background(), token, tool, json.RawMessage(arguments)) +} + +func (f *chatToolFixture) items(t *testing.T, sessionID string) []protocol.ChatShownItem { + t.Helper() + items, err := f.store.ChatShownItems(context.Background(), sessionID) + if err != nil { + t.Fatal(err) + } + return items +} + +func TestChatToolTokensAttributeCallsToExactlyOneChat(t *testing.T) { + f := newChatToolFixture(t) + a, b := f.token(t, chatToolSessionScope+"ses_a"), f.token(t, chatToolSessionScope+"ses_b") + if a == b || len(a) != 64 || !f.service.Known(context.Background(), a) { + t.Fatalf("tokens must be distinct 256-bit values: %q %q", a, b) + } + if again := f.token(t, chatToolSessionScope+"ses_a"); again != a { + t.Fatal("a session keeps its token (a detached harness keeps its endpoint)") + } + if f.service.Known(context.Background(), strings.Repeat("0", 64)) { + t.Fatal("an unknown token must be refused") + } + if r := f.call(a, "show_html", `{"html":"

A

","title":"For A"}`); r.IsError { + t.Fatal(r.Text) + } + if got := f.items(t, "ses_a"); len(got) != 1 || got[0].Kind != protocol.ChatShownHTML || got[0].Title != "For A" || got[0].HTML != "

A

" || got[0].ID == "" || got[0].CreatedAt == 0 { + t.Fatalf("ses_a items = %+v", got) + } + if got := f.items(t, "ses_b"); len(got) != 0 { + t.Fatalf("another chat must not receive the item: %+v", got) + } + frame := f.hub.shownFrame("ses_a") + if frame == nil || frame.SessionID != "ses_a" || len(frame.Items) != 1 { + t.Fatalf("subscribe frame = %+v", frame) + } + if f.hub.shownFrame("ses_b") != nil { + t.Fatal("a chat with no items has no frame") + } + + registry := chatToolRegistry{store: f.store, base: "http://127.0.0.1:1"} + url, bind, err := registry.PendingURL() + if err != nil { + t.Fatal(err) + } + pending := strings.TrimPrefix(url, "http://127.0.0.1:1"+chatToolPath) + if r := f.call(pending, "show_html", `{"html":"x"}`); !r.IsError { + t.Fatal("an unbound pending token must not show anything") + } + if err := bind("019a6f7e-1c2d-7e3f-8a9b-0c1d2e3f4a5b"); err != nil { + t.Fatal(err) + } + if r := f.call(pending, "show_html", `{"html":"x"}`); r.IsError || len(f.items(t, "019a6f7e-1c2d-7e3f-8a9b-0c1d2e3f4a5b")) != 1 { + t.Fatalf("a bound token serves its thread: %v", r) + } +} + +func TestShowFileStaysInsideHome(t *testing.T) { + f := newChatToolFixture(t) + token := f.token(t, chatToolSessionScope+"ses_f") + inside := filepath.Join(f.home, "work", "report.md") + if err := os.MkdirAll(filepath.Dir(inside), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(inside, []byte("# Report\n"), 0o600); err != nil { + t.Fatal(err) + } + outside := filepath.Join(t.TempDir(), "secret.txt") + if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(f.home, "link.txt")); err != nil { + t.Fatal(err) + } + if err := f.store.RecordOperatorSession(context.Background(), "ses_f", filepath.Join(f.home, "work"), "Files", protocol.ChatSessionTagOperator); err != nil { + t.Fatal(err) + } + + r := f.call(token, "show_file", `{"path":"`+inside+`","title":"The report"}`) + if r.IsError || !strings.Contains(r.Text, "report.md") { + t.Fatalf("inside home: %v", r) + } + realHome, _ := filepath.EvalSymlinks(f.home) + got := f.items(t, "ses_f") + if len(got) != 1 || got[0].Path != filepath.Join(realHome, "work", "report.md") || got[0].Name != "report.md" || got[0].Size != 9 || !strings.HasPrefix(got[0].Mime, "text/") { + t.Fatalf("item = %+v", got) + } + if r := f.call(token, "show_file", `{"path":"report.md"}`); r.IsError { + t.Fatalf("a relative path resolves against the chat's directory: %v", r) + } + if r := f.call(token, "show_file", `{"path":"~/work/report.md"}`); r.IsError { + t.Fatalf("a ~/ path resolves in home: %v", r) + } + for _, refused := range []string{outside, filepath.Join(f.home, "link.txt"), filepath.Join(f.home, "work"), filepath.Join(f.home, "missing.png"), "../../../etc/passwd", ""} { + if r := f.call(token, "show_file", `{"path":"`+refused+`"}`); !r.IsError { + t.Fatalf("%q must be refused: %v", refused, r) + } + } + if len(f.items(t, "ses_f")) != 3 { + t.Fatal("refused paths must record nothing") + } +} + +func TestShowImageAcceptsHomeImagesAndHTTPSOnly(t *testing.T) { + f := newChatToolFixture(t) + token := f.token(t, chatToolSessionScope+"ses_i") + image := filepath.Join(f.home, "plot.png") + if err := os.WriteFile(image, []byte("\x89PNG\r\n\x1a\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(f.home, "notes.txt"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + if r := f.call(token, "show_image", `{"url_or_path":"`+image+`","alt":"A plot"}`); r.IsError { + t.Fatal(r.Text) + } + if r := f.call(token, "show_image", `{"url_or_path":"https://example.com/a.png","title":"Remote"}`); r.IsError { + t.Fatal(r.Text) + } + for _, refused := range []string{"http://example.com/a.png", "https://user:pw@example.com/a.png", "javascript:alert(1)", "file:///etc/passwd", filepath.Join(f.home, "notes.txt")} { + if r := f.call(token, "show_image", `{"url_or_path":"`+refused+`"}`); !r.IsError { + t.Fatalf("%q must be refused: %v", refused, r) + } + } + got := f.items(t, "ses_i") + if len(got) != 2 || got[0].Kind != protocol.ChatShownImage || got[0].Mime != "image/png" || got[0].Alt != "A plot" || got[1].URL != "https://example.com/a.png" || got[1].Path != "" { + t.Fatalf("items = %+v", got) + } +} + +func TestShowHTMLSizeLimitAndRetention(t *testing.T) { + f := newChatToolFixture(t) + token := f.token(t, chatToolSessionScope+"ses_h") + if r := f.call(token, "show_html", `{"html":"`+strings.Repeat("a", chattools.MaxHTMLBytes+1)+`"}`); !r.IsError || !strings.Contains(r.Text, "512 KiB") { + t.Fatalf("an oversized document must be refused with the limit: %v", r) + } + if r := f.call(token, "show_html", `{"html":" "}`); !r.IsError { + t.Fatal("an empty document must be refused") + } + if len(f.items(t, "ses_h")) != 0 { + t.Fatal("refused documents must record nothing") + } + full := strings.Repeat("b", chattools.MaxHTMLBytes) + for range 5 { + if r := f.call(token, "show_html", `{"html":"`+full+`","height_hint":5000}`); r.IsError { + t.Fatal(r.Text) + } + } + got := f.items(t, "ses_h") + if len(got) != 5 || got[0].HeightHint != 600 { + t.Fatalf("items = %d, hint %d", len(got), got[0].HeightHint) + } + // 2 MiB of HTML is retained: the newest four bodies, the oldest expired. + if !got[0].Expired || got[0].HTML != "" || got[0].HTMLBytes != chattools.MaxHTMLBytes { + t.Fatalf("the oldest body must be dropped under the budget: expired=%v len=%d", got[0].Expired, len(got[0].HTML)) + } + for _, item := range got[1:] { + if item.Expired || len(item.HTML) != chattools.MaxHTMLBytes { + t.Fatal("the newest bodies must be kept") + } + } + for range protocol.ChatShownLimit + 5 { + f.service.now = func() time.Time { return time.Now().Add(time.Hour) } + if r := f.call(token, "show_html", `{"html":"x"}`); r.IsError { + t.Fatal(r.Text) + } + } + if got := f.items(t, "ses_h"); len(got) != protocol.ChatShownLimit { + t.Fatalf("a chat retains at most %d items, got %d", protocol.ChatShownLimit, len(got)) + } +} + +func TestNotifyIsRateLimitedPerChat(t *testing.T) { + f := newChatToolFixture(t) + a, b := f.token(t, chatToolSessionScope+"ses_n1"), f.token(t, chatToolSessionScope+"ses_n2") + now := time.Unix(1_800_000_000, 0) + f.service.now = func() time.Time { return now } + for i := range 5 { + if r := f.call(a, "notify", `{"title":"Done","message":"Build finished","priority":"high"}`); r.IsError { + t.Fatalf("notification %d: %v", i, r) + } + } + if r := f.call(a, "notify", `{"title":"Done","message":"again"}`); !r.IsError || !strings.Contains(r.Text, "5 notifications") { + t.Fatalf("the sixth in a minute must be refused: %v", r) + } + if r := f.call(b, "notify", `{"title":"Other","message":"chat"}`); r.IsError { + t.Fatalf("another chat has its own limit: %v", r) + } + now = now.Add(61 * time.Second) + if r := f.call(a, "notify", `{"title":"Later","message":"ok"}`); r.IsError { + t.Fatalf("the window slides: %v", r) + } + if r := f.call(a, "notify", `{"title":"Bad","message":"x","priority":"urgent"}`); !r.IsError { + t.Fatal("an unknown priority must be refused") + } + if len(f.notified) != 7 || f.notified[0] != "ses_n1: Done" { + t.Fatalf("notified = %v", f.notified) + } + got := f.items(t, "ses_n1") + if len(got) != 6 || got[0].Kind != protocol.ChatShownNotify || !got[0].Delivered || got[0].Priority != "high" { + t.Fatalf("items = %+v", got) + } + + f.service.notify = func(context.Context, string, string, string) (bool, string) { return false, "ntfy is not configured" } + r := f.call(b, "notify", `{"title":"Fails","message":"x"}`) + if !r.IsError || !strings.Contains(r.Text, "ntfy is not configured") { + t.Fatalf("an undelivered notification is reported: %v", r) + } + if last := f.items(t, "ses_n2"); len(last) != 2 || last[1].Delivered || last[1].Detail != "ntfy is not configured" { + t.Fatalf("the undelivered entry is recorded: %+v", last) + } +} + +func TestInstanceTokenCallsNeedOneMatchingRunningToolCall(t *testing.T) { + f := newChatToolFixture(t) + token := f.token(t, chatToolInstanceScope+"opencode") + saved := chatToolCorrelationWait + chatToolCorrelationWait = 100 * time.Millisecond + t.Cleanup(func() { chatToolCorrelationWait = saved }) + arguments := `{"html":"

diagram

","title":"Plan"}` + if r := f.call(token, "show_html", arguments); !r.IsError { + t.Fatal("a shared-instance call with no running tool call must be refused") + } + running := func(sessionID, name, input string) { + f.hub.apply(protocol.ChatEventFrame{SessionID: sessionID, MessageID: "msg_" + sessionID, Kind: protocol.ChatEventTool, + Part: &protocol.ChatPart{Type: "tool", ID: "prt_" + sessionID, Name: name, State: &protocol.ChatToolState{Status: protocol.ChatToolRunning, Input: json.RawMessage(input)}}}) + } + running("ses_x", "pipelines_show_html", `{"title":"Plan","html":"

diagram

"}`) + running("ses_y", "pipelines_show_html", `{"html":"

other

"}`) + running("ses_z", "bash", arguments) + if r := f.call(token, "show_html", arguments); r.IsError { + t.Fatal(r.Text) + } + if len(f.items(t, "ses_x")) != 1 || len(f.items(t, "ses_y"))+len(f.items(t, "ses_z")) != 0 { + t.Fatal("the call belongs to the chat running exactly this call") + } + running("ses_y", "pipelines_show_html", arguments) + if r := f.call(token, "show_html", arguments); !r.IsError || !strings.Contains(r.Text, "identical") { + t.Fatalf("two chats running the same call must be refused: %v", r) + } +} diff --git a/internal/server/chat_tools_tokens.go b/internal/server/chat_tools_tokens.go new file mode 100644 index 00000000..4999e86f --- /dev/null +++ b/internal/server/chat_tools_tokens.go @@ -0,0 +1,110 @@ +package server + +import ( + "context" + "errors" + "net" + "time" + + "git.a-wan.net/alan/pipelines/internal/chattools" +) + +// chatToolTokenStore persists the MCP endpoint tokens (pipeline.Store). +type chatToolTokenStore interface { + ChatToolToken(ctx context.Context, scope string) (string, error) + IssueChatToolToken(ctx context.Context, scope string) (string, error) + ChatToolScope(ctx context.Context, token string) (string, bool, error) + BindChatToolToken(ctx context.Context, token, scope string) error +} + +// The token scopes. +const ( + chatToolSessionScope = "session:" + chatToolPendingScope = "pending:" + chatToolInstanceScope = "instance:" +) + +// chatToolPath is the loopback route prefix of the MCP endpoint. The web proxy +// and the peer relay forward an allowlist that does not include it. +const chatToolPath = "/mcp/chat/" + +// chatToolRegistry issues the endpoint URLs harnesses register +// (chattools.Registry) on the server's own loopback listener. +type chatToolRegistry struct { + store chatToolTokenStore + base string +} + +func (r chatToolRegistry) url(token string) string { return r.base + chatToolPath + token } + +func (r chatToolRegistry) SessionURL(sessionID string) (string, error) { + if !validChatSessionID(sessionID) { + return "", errors.New("invalid chat session id") + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + token, err := r.store.ChatToolToken(ctx, chatToolSessionScope+sessionID) + if err != nil { + return "", err + } + return r.url(token), nil +} + +func (r chatToolRegistry) PendingURL() (string, func(string) error, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + token, err := r.store.IssueChatToolToken(ctx, chatToolPendingScope) + if err != nil { + return "", nil, err + } + bind := func(sessionID string) error { + if !validChatSessionID(sessionID) { + return errors.New("invalid chat session id") + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return r.store.BindChatToolToken(ctx, token, chatToolSessionScope+sessionID) + } + return r.url(token), bind, nil +} + +func (r chatToolRegistry) InstanceURL(harness string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + token, err := r.store.ChatToolToken(ctx, chatToolInstanceScope+harness) + if err != nil { + return "", err + } + return r.url(token), nil +} + +// loopbackBase is the http origin harnesses on this machine use to reach a +// server listening on addr: a wildcard or blank host becomes 127.0.0.1. A +// server bound only to a non-loopback address cannot serve the tools: the +// endpoint answers loopback peers alone. +func loopbackBase(addr string) (string, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return "", err + } + ip := net.ParseIP(host) + switch { + case host == "localhost" || (ip != nil && ip.IsLoopback()): + return "http://" + net.JoinHostPort(host, port), nil + case host == "" || (ip != nil && ip.IsUnspecified()): + return "http://" + net.JoinHostPort("127.0.0.1", port), nil + } + return "", errors.New("the chat tools need a loopback or wildcard listen address, not " + addr) +} + +// ConfigureChatTools makes every chat harness Pipelines launches register the +// Pipelines MCP server on the listener at addr. It is called once at startup, +// before any harness starts. +func ConfigureChatTools(store chatToolTokenStore, addr string) error { + base, err := loopbackBase(addr) + if err != nil { + return err + } + chattools.Configure(chatToolRegistry{store: store, base: base}) + return nil +} diff --git a/internal/server/websocket.go b/internal/server/websocket.go index ce791257..c28b0639 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -153,6 +153,11 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin } runs.serveArtifact(w, r) }) + if len(stores) > 0 { + // The Pipelines chat tools' MCP endpoint, for harnesses on this machine. + hub.shown = stores[0] + registerChatTools(mux, newChatToolService(stores[0], stores[0], hub, registry), build.Version) + } mux.HandleFunc("GET /discovery", serveServerDiscovery) mux.HandleFunc("GET /files", serveHomeFile) mux.HandleFunc("POST /files", serveHomeFileUpload) diff --git a/web/test/mcpRefused.test.mjs b/web/test/mcpRefused.test.mjs new file mode 100644 index 00000000..0a064a53 --- /dev/null +++ b/web/test/mcpRefused.test.mjs @@ -0,0 +1,63 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import net from "node:net"; +import { createServer } from "node:http"; +import { once } from "node:events"; + +// The Pipelines chat tools' MCP endpoint (/mcp/chat/) is for harness +// processes on the server's own machine. The authenticated proxy must never +// forward it, even for a signed-in browser. +const here = fileURLToPath(new URL("..", import.meta.url)); + +async function freePort() { + const server = net.createServer().listen(0, "127.0.0.1"); + await once(server, "listening"); + const { port } = server.address(); + await new Promise((done) => server.close(done)); + return port; +} + +test("the proxy does not expose the loopback MCP chat-tools endpoint", async (t) => { + const requests = []; + const upstream = createServer((req, res) => { + requests.push(`${req.method} ${req.url}`); + res.writeHead(200, { "Content-Type": "application/json" }).end('{"jsonrpc":"2.0","id":1,"result":{}}'); + }).listen(0, "127.0.0.1"); + await once(upstream, "listening"); + t.after(() => new Promise((done) => upstream.close(done))); + const dir = await mkdtemp(join(tmpdir(), "serve-mcp-")); + const auth = join(dir, "auth.json"); + await writeFile(auth, JSON.stringify({ password: "password-1", secret: "s".repeat(40) })); + const port = await freePort(); + const child = spawn(process.execPath, ["serve.mjs", "--port", String(port), + "--pipelines", `127.0.0.1:${upstream.address().port}`], { + cwd: here, + env: { ...process.env, PIPELINES_WEB_AUTH: "", PIPELINES_WEB_AUTH_FILE: auth }, + stdio: ["ignore", "pipe", "inherit"], + }); + t.after(async () => { const exited = once(child, "exit"); child.kill(); await exited; }); + await once(child.stdout, "data"); + const base = `http://127.0.0.1:${port}`; + const login = await fetch(base + "/login", { + method: "POST", body: new URLSearchParams({ password: "password-1" }), redirect: "manual", + }); + const cookie = login.headers.get("set-cookie").split(";")[0]; + const token = "a".repeat(64); + for (const path of [`/mcp/chat/${token}`, "/mcp", "/mcp/", `/%6dcp/chat/${token}`]) { + for (const method of ["GET", "POST"]) { + const reply = await fetch(base + path, { + method, + headers: { Cookie: cookie, "Content-Type": "application/json", Accept: "application/json, text/event-stream" }, + body: method === "POST" ? '{"jsonrpc":"2.0","id":1,"method":"tools/list"}' : undefined, + }); + const text = await reply.text(); + assert.ok(!text.includes('"jsonrpc"'), `${method} ${path} reached the server: ${reply.status} ${text}`); + } + } + assert.deepEqual(requests, []); +}); diff --git a/web/test/peerRelay.test.mjs b/web/test/peerRelay.test.mjs index 6869a5fc..77aa60e7 100644 --- a/web/test/peerRelay.test.mjs +++ b/web/test/peerRelay.test.mjs @@ -585,3 +585,18 @@ test("raw peer HTML and legacy SVG files retain a local sandbox and no cookie au assert.match(await response.text(), /