From e9d902213ed310ec8f94b84bb74597e18dc0c594 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 18 Oct 2026 12:54:36 +0100 Subject: [PATCH 1/2] Preserve background shell visibility across readback races Reconcile authoritative shell snapshots with bounded lifecycle journals so concurrent exits and creates cannot be overwritten, coalesce observation setup, and retain verified ownership across reconnects. Add deterministic reconciliation and cancellation regressions plus stable limited-visibility handling.\n\nChecked packaged templates and authoring skill references: none need updating. The task-owned docs note records the stale shared chat documentation for docs-fold; shared documentation was not edited. --- docs/notes/background-shell-visibility.md | 37 +++ internal/opencode/chat.go | 209 +++++++++++++- .../opencode/chat_shell_reconcile_test.go | 116 ++++++++ internal/opencode/chat_shell_snapshot.go | 72 +++++ internal/opencode/chat_v2_test.go | 42 +++ internal/server/chat.go | 12 + internal/server/chat_hub.go | 12 +- internal/server/chat_multiclient_test.go | 2 + internal/server/chat_room.go | 186 ++++++++++++- internal/server/chat_shell_reconcile_test.go | 261 ++++++++++++++++++ internal/server/chat_stream.go | 13 +- internal/server/chat_test.go | 99 ++++++- 12 files changed, 1036 insertions(+), 25 deletions(-) create mode 100644 docs/notes/background-shell-visibility.md create mode 100644 internal/opencode/chat_shell_reconcile_test.go create mode 100644 internal/opencode/chat_shell_snapshot.go create mode 100644 internal/server/chat_shell_reconcile_test.go diff --git a/docs/notes/background-shell-visibility.md b/docs/notes/background-shell-visibility.md new file mode 100644 index 00000000..583e5f90 --- /dev/null +++ b/docs/notes/background-shell-visibility.md @@ -0,0 +1,37 @@ +# Background shell visibility + +The chat server now opens the broker stream for read-only subscribers and reconciles verified running shell operations from OpenCode's authoritative `/api/shell` readback. The packaged OpenCode skill references and chat documentation should describe that shell visibility survives reconnects only when OpenCode reports explicit `metadata.sessionID` ownership; shared working directories are not ownership evidence. + +## Current source behavior + +Observers share one initial readback per room/stream attachment. The adapter and hub +fold bounded lifecycle journals over each snapshot, so a stale read cannot resurrect +an exited shell or erase a concurrent create. Shell replay and live delivery enqueue +in fold order. Cancellation rejects old-room callbacks and releases local identity +state before reconnect. Journals retain at most 256 identities and 1 MiB of id/command +text; capacity/read failures surface stable limited-visibility text, never broker +response bodies. No shell polling was added. OpenCode's terminal `timeout` and +`killed` statuses are excluded from retained running operations as well as `exited`. + +## Verification evidence — 2026-10-18 + +`opencode api get /openapi.json` authenticated through supported service discovery. +The live schema advertises shell list/detail/output and generic `metadata`; it does +not type `metadata.sessionID`. A throwaway session +`ses_eb12ab7b3ffe0ZAle7PHpftHMG` and its own `sleep 1` shell +`sh_14ed57134001e6fuYqvn62pKit` verified actual create/detail readback preserving +explicit `metadata.sessionID`, status `running`, subsequent `exited`/exit 0, and +absence from the running list. This verifies explicit API metadata, not automatic +agent-tool ownership injection. The CLI event attempt yielded no captured events; +live-provider event ordering is **not tested**. Forced ordering evidence comes from +channel/barrier regressions, including a real WebSocket handler on a loopback scratch +server with fresh SQLite, not from the deployed service. + +## Documentation follow-up / deployment + +`docs/chat-verification.md` and the chat subscribe/replay section of `docs/cli.md` +are stale: docs-fold should describe authoritative ownership/readback, coalesced +observation, bounded reconciliation and limited-visibility errors, distinguishing +fixture ordering evidence from the live readback above. Shared docs and packaged +skill references were not edited here. No deployment, installation, service restart +or physical-phone/UI verification was performed; this changes no rendered layout. diff --git a/internal/opencode/chat.go b/internal/opencode/chat.go index 40adc005..8fae8cd9 100644 --- a/internal/opencode/chat.go +++ b/internal/opencode/chat.go @@ -28,7 +28,9 @@ var ErrPermissionUnsupported = errors.New("OpenCode permission approval is not i // semantics, the session-id shape and the 64 KiB / 1 MiB bounds of Connection so // no second transport or credential path is introduced. type Chat struct { - conn *Connection + conn *Connection + shellMu sync.Mutex + shells map[string]*ChatStreamState // mu serializes the adapter's HTTP calls. Connection records its last // operation and status on the shared connection, so two calls must not // overlap; the permission poller runs alongside the read loop's @@ -294,6 +296,114 @@ func decodeChatPermissions(sessionID string, data []byte) ([]protocol.ChatPermis return frames, nil } +// Shells reads the endpoint's authoritative shell list and returns only running +// operations whose metadata explicitly names sessionID. The shell API is global; +// filtering locally avoids guessing an unsupported query filter and never assigns +// ownership from a shared working directory. A successful read also seeds the +// stream state used to attribute later shell.exited events, which carry no session. +func (c *Chat) Shells(ctx context.Context, sessionID string) ([]protocol.ChatShell, error) { + if !ValidSessionID(sessionID) { + return nil, fmt.Errorf("invalid OpenCode session id") + } + state, err := c.streamStateContext(ctx, sessionID) + if err != nil { + return nil, err + } + // One read per state; waiting and I/O never hold the event-state mutex. + state.mu.Lock() + if state.readGate == nil { + state.readGate = make(chan struct{}, 1) + } + gate := state.readGate + state.mu.Unlock() + select { + case gate <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + defer func() { <-gate }() + state.mu.Lock() + state.readback = &ShellSnapshot{} + state.mu.Unlock() + defer func() { state.mu.Lock(); state.readback = nil; state.mu.Unlock() }() + data, err := c.call(ctx, "get", "/api/shell", nil) + if err != nil { + return nil, err + } + var response struct { + Data []chatShellInfo `json:"data"` + } + if err := json.Unmarshal(data, &response); err != nil || response.Data == nil { + if err := json.Unmarshal(data, &response.Data); err != nil { + return nil, fmt.Errorf("invalid OpenCode shell response") + } + } + out := make([]protocol.ChatShell, 0, len(response.Data)) + for _, info := range response.Data { + if info.ID == "" || info.shellSessionID() != sessionID || (info.Status != "" && info.Status != protocol.ChatShellRunning) { + continue + } + status := info.Status + if status == "" { + status = protocol.ChatShellRunning + } + out = append(out, protocol.ChatShell{ID: info.ID, Status: status, Command: info.Command}) + } + state.mu.Lock() + defer state.mu.Unlock() + out, err = state.readback.Merge(out) + if err != nil { + return nil, err + } + if err := ctx.Err(); err != nil { + return nil, err + } + ids := make([]string, 0, len(out)) + for _, shell := range out { + ids = append(ids, shell.ID) + } + state.replaceShellsLocked(ids) + return out, nil +} + +// streamState returns the session's retained, verified shell identities. It is +// deliberately kept across stream replacement so an exit after reconnect remains +// attributable; Shells reconciles its contents from authoritative readback on +// each attach rather than replacing the pointer a live scanner is using. +func (c *Chat) streamState(sessionID string) *ChatStreamState { + state, _ := c.streamStateContext(context.Background(), sessionID) + return state +} + +func (c *Chat) streamStateContext(ctx context.Context, sessionID string) (*ChatStreamState, error) { + c.shellMu.Lock() + defer c.shellMu.Unlock() + if err := ctx.Err(); err != nil { + return nil, err + } + if c.shells == nil { + c.shells = map[string]*ChatStreamState{} + } + state := c.shells[sessionID] + if state == nil { + state = &ChatStreamState{} + c.shells[sessionID] = state + } + return state, nil +} + +// ReleaseShells discards verified shell identities after the last room observer +// disconnects. A future attach calls Shells for a fresh authoritative snapshot, +// so this bounds state without attributing a foreign shell during the gap. +func (c *Chat) ReleaseShells(sessionID string) { + if sessionID == "" { + return + } + c.shellMu.Lock() + delete(c.shells, sessionID) + c.shellMu.Unlock() +} + // Stream subscribes to the OpenCode event firehose and invokes emit for every // frame belonging to sessionID. It reads one bounded SSE frame at a time and // never polls the per-session message log; the global firehose is the only @@ -306,6 +416,12 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr if !sessionID.MatchString(id) { return fmt.Errorf("invalid OpenCode session id") } + // Pin state before opening/announcing the stream so a canceled scanner + // cannot acquire a replacement room's identity map. + state, err := c.streamStateContext(ctx, id) + if err != nil { + return err + } body, err := c.conn.EventStream(ctx) if err != nil { return err @@ -314,7 +430,7 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr if ready != nil { ready() } - return scanChatEvents(ctx, body, id, emit) + return c.scanChatEventsWithState(ctx, body, id, state, emit) } // scanChatEvents turns an SSE body into protocol frames, one bounded frame at a @@ -322,10 +438,13 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr // malformed JSON fails the stream instead of producing a malformed frame. The // last assistant message id is retained so a later tool/step event that omits it // still attributes every frame to a stable message. -func scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit func(protocol.ChatEventFrame) error) error { +func (c *Chat) scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit func(protocol.ChatEventFrame) error) error { + return c.scanChatEventsWithState(ctx, body, sessionID, c.streamState(sessionID), emit) +} + +func (c *Chat) scanChatEventsWithState(ctx context.Context, body io.Reader, sessionID string, state *ChatStreamState, emit func(protocol.ChatEventFrame) error) error { scanner := bufio.NewScanner(body) scanner.Buffer(make([]byte, 0, 64<<10), MaxResponse) - state := &ChatStreamState{} for scanner.Scan() { if err := ctx.Err(); err != nil { return err @@ -346,7 +465,7 @@ func scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit continue } if frame.MessageID != "" { - state.LastMessageID = frame.MessageID + state.setLastMessageID(frame.MessageID) } if err := emit(frame); err != nil { return err @@ -449,8 +568,29 @@ func (i *chatShellInfo) shellSessionID() string { // what attributes the exit to its session: another session's shell is an // unknown id and is ignored. type ChatStreamState struct { + mu sync.Mutex LastMessageID string Shells map[string]struct{} + readGate chan struct{} + readback *ShellSnapshot +} + +func (s *ChatStreamState) lastMessageID() string { + if s == nil { + return "" + } + s.mu.Lock() + defer s.mu.Unlock() + return s.LastMessageID +} + +func (s *ChatStreamState) setLastMessageID(id string) { + if s == nil || id == "" { + return + } + s.mu.Lock() + s.LastMessageID = id + s.mu.Unlock() } // knowsShell reports whether id is a shell this stream has already surfaced for @@ -459,6 +599,8 @@ func (s *ChatStreamState) knowsShell(id string) bool { if s == nil || id == "" { return false } + s.mu.Lock() + defer s.mu.Unlock() _, ok := s.Shells[id] return ok } @@ -468,6 +610,8 @@ func (s *ChatStreamState) markShell(id string) { if s == nil || id == "" { return } + s.mu.Lock() + defer s.mu.Unlock() if s.Shells == nil { s.Shells = map[string]struct{}{} } @@ -476,12 +620,37 @@ func (s *ChatStreamState) markShell(id string) { // forgetShell drops id once its exit has been surfaced. func (s *ChatStreamState) forgetShell(id string) { - if s == nil || s.Shells == nil { + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if s.Shells == nil { return } delete(s.Shells, id) } +// replaceShells applies one authoritative running-shell snapshot without +// replacing the state object an already attached event scanner owns. +func (s *ChatStreamState) replaceShells(ids []string) { + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.replaceShellsLocked(ids) +} + +func (s *ChatStreamState) replaceShellsLocked(ids []string) { + s.Shells = make(map[string]struct{}, len(ids)) + for _, id := range ids { + if id != "" { + s.Shells[id] = struct{}{} + } + } +} + // NormalizeChatEvent maps one OpenCode firehose `data:` payload to a // protocol.ChatEventFrame, folding the small per-session context ([stream]) it // needs across frames. ok is false for an event type this adapter does not @@ -528,7 +697,7 @@ func NormalizeChatEvent(payload []byte, sessionID string, stream *ChatStreamStat messageID = data.MessageIDAlt } if messageID == "" && stream != nil { - messageID = stream.LastMessageID + messageID = stream.lastMessageID() } // Every pushed frame must carry a stable message id so the client can // attribute it; without one this is not a chat frame. @@ -590,20 +759,38 @@ func normalizeShellEvent(eventType string, data chatEvent, sessionID string, str if status == "" { status = protocol.ChatShellRunning } - stream.markShell(data.Info.ID) + shell := &protocol.ChatShell{ID: data.Info.ID, Status: status, Command: data.Info.Command} + if stream != nil { + stream.mu.Lock() + stream.readback.Record(shell) + if stream.Shells == nil { + stream.Shells = map[string]struct{}{} + } + if status == protocol.ChatShellRunning && len(stream.Shells) < chatShellIdentityLimit { + stream.Shells[shell.ID] = struct{}{} + } + stream.mu.Unlock() + } return protocol.ChatEventFrame{ SessionID: sessionID, Kind: protocol.ChatEventShell, - Shell: &protocol.ChatShell{ID: data.Info.ID, Status: status, Command: data.Info.Command}, + Shell: shell, }, true, nil } if data.ID == "" { return protocol.ChatEventFrame{}, false, fmt.Errorf("malformed OpenCode shell.exited event") } - if !stream.knowsShell(data.ID) { + if stream == nil { + return protocol.ChatEventFrame{}, false, nil + } + stream.mu.Lock() + stream.readback.Record(&protocol.ChatShell{ID: data.ID, Status: protocol.ChatShellExited}) + _, known := stream.Shells[data.ID] + delete(stream.Shells, data.ID) + stream.mu.Unlock() + if !known { return protocol.ChatEventFrame{}, false, nil } - stream.forgetShell(data.ID) status := data.Status if status == "" { status = protocol.ChatShellExited diff --git a/internal/opencode/chat_shell_reconcile_test.go b/internal/opencode/chat_shell_reconcile_test.go new file mode 100644 index 00000000..087080d0 --- /dev/null +++ b/internal/opencode/chat_shell_reconcile_test.go @@ -0,0 +1,116 @@ +package opencode + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +func TestChatShellReadbackBarrier(t *testing.T) { + for _, known := range []bool{false, true} { + t.Run(fmt.Sprint(known), func(t *testing.T) { + read, apply := make(chan struct{}), make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + close(read) + select { + case <-apply: + fmt.Fprint(w, `{"data":[{"id":"sh_old","status":"running","metadata":{"sessionID":"ses_1"}},{"id":"sh_foreign","status":"running","metadata":{"sessionID":"ses_other"}}]}`) + case <-r.Context().Done(): + } + })) + defer srv.Close() + c := NewChat(&Connection{Backend: "v2", URL: srv.URL}) + state := c.streamState("ses_1") + if known { + state.markShell("sh_old") + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + result := make(chan []protocol.ChatShell, 1) + errs := make(chan error, 1) + go func() { shells, err := c.Shells(ctx, "ses_1"); result <- shells; errs <- err }() + select { + case <-read: + case <-ctx.Done(): + t.Fatal("read did not reach barrier") + } + // Both a previously verified exit and an exit first learned while + // reading must suppress the stale running snapshot. + frame, ok, err := NormalizeChatEvent([]byte(`{"type":"shell.exited","data":{"id":"sh_old","status":"exited","exit":0}}`), "ses_1", state) + if err != nil || ok != known || (ok && frame.Shell.Status != protocol.ChatShellExited) { + t.Fatalf("exit = %+v %v %v", frame, ok, err) + } + frame, ok, err = NormalizeChatEvent([]byte(`{"type":"shell.created","data":{"info":{"id":"sh_new","status":"running","command":"sleep 10","metadata":{"sessionID":"ses_1"}}}}`), "ses_1", state) + if err != nil || !ok || frame.Shell.ID != "sh_new" { + t.Fatalf("create = %+v %v %v", frame, ok, err) + } + close(apply) + got := <-result + if err := <-errs; err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].ID != "sh_new" || got[0].Command != "sleep 10" { + t.Fatalf("readback lost events/leaked ownership: %+v", got) + } + if state.knowsShell("sh_old") || state.knowsShell("sh_foreign") || !state.knowsShell("sh_new") { + t.Fatal("adapter identity state regressed") + } + // User-visible subsequent exit remains attributable after reconciliation. + frame, ok, err = NormalizeChatEvent([]byte(`{"type":"shell.exited","data":{"id":"sh_new","status":"exited","exit":0}}`), "ses_1", state) + if err != nil || !ok || frame.Shell.ID != "sh_new" { + t.Fatalf("new shell exit lost: %+v %v %v", frame, ok, err) + } + }) + } +} + +func TestChatShellReadbackCancelWaiter(t *testing.T) { + read := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { close(read); <-r.Context().Done() })) + defer srv.Close() + c := NewChat(&Connection{Backend: "v2", URL: srv.URL}) + ctx, cancel := context.WithCancel(context.Background()) + result := make(chan error, 2) + go func() { _, err := c.Shells(ctx, "ses_1"); result <- err }() + select { + case <-read: + case <-time.After(time.Second): + t.Fatal("read stuck") + } + go func() { _, err := c.Shells(ctx, "ses_1"); result <- err }() + cancel() + for range 2 { + select { + case err := <-result: + if err == nil { + t.Fatal("canceled read succeeded") + } + case <-time.After(time.Second): + t.Fatal("read or waiter stuck") + } + } + old := c.streamState("ses_1") + c.ReleaseShells("ses_1") + if c.streamState("ses_1") == old { + t.Fatal("reconnect reused released state") + } +} + +func TestShellSnapshotBounded(t *testing.T) { + var snapshot ShellSnapshot + for i := 0; i <= chatShellIdentityLimit; i++ { + snapshot.Record(&protocol.ChatShell{ID: fmt.Sprintf("sh_%d", i), Status: protocol.ChatShellExited}) + } + if len(snapshot.changes) != chatShellIdentityLimit { + t.Fatalf("unbounded journal: %d", len(snapshot.changes)) + } + if _, err := snapshot.Merge([]protocol.ChatShell{{ID: "sh_0", Status: protocol.ChatShellRunning}}); err == nil { + t.Fatal("overflow accepted stale snapshot") + } +} diff --git a/internal/opencode/chat_shell_snapshot.go b/internal/opencode/chat_shell_snapshot.go new file mode 100644 index 00000000..8559a741 --- /dev/null +++ b/internal/opencode/chat_shell_snapshot.go @@ -0,0 +1,72 @@ +package opencode + +import ( + "fmt" + "sort" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// ShellSnapshot records changes while an authoritative read is in flight. Its +// owner serializes access. Unknown exits may be recorded, but can only remove an +// id: they never establish ownership. Overflow rejects the snapshot, not events. +type ShellSnapshot struct { + changes map[string]protocol.ChatShell + bytes int + overflow bool +} + +const chatShellIdentityLimit = 256 + +func (s *ShellSnapshot) Record(shell *protocol.ChatShell) { + if s == nil || shell == nil || shell.ID == "" || s.overflow { + return + } + if s.changes == nil { + s.changes = make(map[string]protocol.ChatShell) + } + old := s.changes[shell.ID] + n := s.bytes - len(old.ID) - len(old.Command) + len(shell.ID) + len(shell.Command) + if n > MaxResponse || (old.ID == "" && len(s.changes) >= chatShellIdentityLimit) { + s.overflow = true + return + } + // A terminal event wins even if a duplicate create is delivered later. + if old.ID != "" && old.Status != protocol.ChatShellRunning { + return + } + s.changes[shell.ID] = *shell + s.bytes = n +} + +func (s *ShellSnapshot) Merge(shells []protocol.ChatShell) ([]protocol.ChatShell, error) { + if s.overflow { + return nil, fmt.Errorf("shell reconciliation capacity exceeded") + } + byID := make(map[string]protocol.ChatShell) + for _, shell := range shells { + if shell.ID != "" && shell.Status == protocol.ChatShellRunning { + byID[shell.ID] = shell + } + } + for id, shell := range s.changes { + if shell.Status == protocol.ChatShellRunning { + byID[id] = shell + } else { + delete(byID, id) + } + } + if len(byID) > chatShellIdentityLimit { + return nil, fmt.Errorf("shell reconciliation capacity exceeded") + } + ids := make([]string, 0, len(byID)) + for id := range byID { + ids = append(ids, id) + } + sort.Strings(ids) + out := make([]protocol.ChatShell, 0, len(ids)) + for _, id := range ids { + out = append(out, byID[id]) + } + return out, nil +} diff --git a/internal/opencode/chat_v2_test.go b/internal/opencode/chat_v2_test.go index 00cb2242..25fc7024 100644 --- a/internal/opencode/chat_v2_test.go +++ b/internal/opencode/chat_v2_test.go @@ -395,3 +395,45 @@ func TestOpenChatRejectsUnrunnableCLI(t *testing.T) { t.Fatal("a missing CLI must fail detection, not open an endpoint") } } + +// TestChatShellReadbackSeedsReplacementStream proves an authoritative shell +// readback survives a later stream replacement: shell.exited has no session id, +// so it must still clear the verified shell after reconnect. +func TestChatShellReadbackSeedsReplacementStream(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/shell": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[{"id":"sh_1","status":"running","command":"sleep 30","metadata":{"sessionID":"ses_1"}},{"id":"sh_other","status":"running","command":"sleep 30","metadata":{"sessionID":"ses_other"}}]}`)) + case "/api/event": + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte("data: {\"type\":\"shell.exited\",\"data\":{\"id\":\"sh_1\",\"status\":\"exited\",\"exit\":0}}\n\n")) + } + })) + defer srv.Close() + chat := NewChat(&Connection{Backend: "v2", URL: srv.URL}) + shells, err := chat.Shells(context.Background(), "ses_1") + if err != nil || len(shells) != 1 || shells[0].ID != "sh_1" { + t.Fatalf("Shells = %+v, %v", shells, err) + } + var got []protocol.ChatEventFrame + if err := chat.Stream(context.Background(), "ses_1", nil, func(frame protocol.ChatEventFrame) error { got = append(got, frame); return nil }); err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Shell == nil || got[0].Shell.ID != "sh_1" || got[0].Shell.Status != protocol.ChatShellExited { + t.Fatalf("exit after readback = %+v", got) + } +} + +func TestChatReleaseShellsBoundsDisconnectedState(t *testing.T) { + chat := NewChat(&Connection{}) + state := chat.streamState("ses_1") + state.markShell("sh_1") + chat.ReleaseShells("ses_1") + chat.shellMu.Lock() + _, retained := chat.shells["ses_1"] + chat.shellMu.Unlock() + if retained { + t.Fatal("released session shell state was retained") + } +} diff --git a/internal/server/chat.go b/internal/server/chat.go index 1601fbfc..1a26dfa6 100644 --- a/internal/server/chat.go +++ b/internal/server/chat.go @@ -32,9 +32,21 @@ type chatBroker interface { // Permissions lists the session's pending OpenCode approval requests so the // handler can push chat_permission frames. Permissions(ctx context.Context, sessionID string) ([]protocol.ChatPermissionFrame, error) + // Shells returns only shell operations whose ownership is authoritatively + // verified for sessionID. It is the reconnect/readback source. + Shells(ctx context.Context, sessionID string) ([]protocol.ChatShell, error) Reply(ctx context.Context, request protocol.ChatReplyRequest) error } +// chatShellReleaser is implemented by brokers that retain verified shell ids for +// attributing exit events. The hub calls it only after the last observer leaves +// a room; a later attach obtains a fresh authoritative snapshot. +// This is a bounded local-cache operation: it must not perform I/O or call the +// hub, which serializes it with room replacement under its mutex. +type chatShellReleaser interface { + ReleaseShells(sessionID string) +} + // operatorRegistry is the server-side operator-session seam. *pipeline.Store // satisfies it. It is a table of its own, separate from the pipeline stage // ownership (SessionOwner/run_session_owners) that settles a run: chat_sessions diff --git a/internal/server/chat_hub.go b/internal/server/chat_hub.go index 134e3fd6..9bd7e32e 100644 --- a/internal/server/chat_hub.go +++ b/internal/server/chat_hub.go @@ -133,6 +133,13 @@ func (h *chatHub) apply(frame protocol.ChatEventFrame) { } h.mu.Lock() defer h.mu.Unlock() + h.applyLocked(frame) +} + +func (h *chatHub) applyLocked(frame protocol.ChatEventFrame) { + if frame.SessionID == "" { + return + } turn := h.turnLocked(frame.SessionID) switch frame.Kind { case protocol.ChatEventStep: @@ -144,6 +151,9 @@ func (h *chatHub) apply(frame protocol.ChatEventFrame) { case protocol.ChatEventShell: // A shell frame carries no message id (it outlives the tool call that // started it), so it is folded before the message-id gate below. + if room := h.rooms[frame.SessionID]; room != nil { + room.shellReadback.Record(frame.Shell) + } turn.applyShell(frame.Shell) return } @@ -240,7 +250,7 @@ func (t *chatTurn) applyShell(shell *protocol.ChatShell) { if shell == nil || shell.ID == "" { return } - if shell.Status == protocol.ChatShellExited { + if shell.Status != protocol.ChatShellRunning { old, ok := t.shells[shell.ID] if !ok { return diff --git a/internal/server/chat_multiclient_test.go b/internal/server/chat_multiclient_test.go index 43d373a5..d6b3dde6 100644 --- a/internal/server/chat_multiclient_test.go +++ b/internal/server/chat_multiclient_test.go @@ -82,6 +82,8 @@ func (c *channelChat) Session(context.Context, string) (protocol.ChatSession, bo return protocol.ChatSession{}, false, nil } +func (c *channelChat) Shells(context.Context, string) ([]protocol.ChatShell, error) { return nil, nil } + func (c *channelChat) Permissions(context.Context, string) ([]protocol.ChatPermissionFrame, error) { c.mu.Lock() defer c.mu.Unlock() diff --git a/internal/server/chat_room.go b/internal/server/chat_room.go index 9ae2450c..a8d0ecc5 100644 --- a/internal/server/chat_room.go +++ b/internal/server/chat_room.go @@ -106,6 +106,11 @@ type chatQueuedPrompt struct { acked bool } +type chatObservation struct { + done chan struct{} + err error // published by closing done; belongs to this attempt, not the next +} + // chatRoom is one session's shared live channel. Every field is guarded by the // owning chatHub's mutex. type chatRoom struct { @@ -117,6 +122,11 @@ type chatRoom struct { queue []*chatQueuedPrompt running bool streaming bool + // observing is non-nil while one caller is attaching the shared broker + // stream. It prevents concurrent subscribers from opening duplicate global + // event streams before streaming can be published. + observing *chatObservation + shellReadback *opencode.ShellSnapshot // buffering holds the turn's frames until the prompting client's ack has been // written, so the ack always precedes the frames it acknowledges. It is set // only while the very first turn subscribes to the firehose. @@ -160,6 +170,81 @@ func (h *chatHub) roomLocked(sessionID string) *chatRoom { return room } +// ensureObserving starts the firehose even for a read-only subscriber, then +// reconciles retained shells from the endpoint's authoritative list. A bounded +// journal folds concurrent lifecycle events over that snapshot atomically. All +// observers share this reconciliation, with no broker I/O under the hub mutex. +func (h *chatHub) ensureObserving(room *chatRoom, broker chatBroker) error { + if broker == nil { + return errors.New("chat is not configured on this server") + } + h.mu.Lock() + if err := room.ctx.Err(); err != nil { + h.mu.Unlock() + return err + } + if waiting := room.observing; waiting != nil { + h.mu.Unlock() + select { + case <-waiting.done: + return waiting.err + case <-room.ctx.Done(): + return room.ctx.Err() + } + } + if room.streaming { + h.mu.Unlock() + return nil + } + room.observing = &chatObservation{done: make(chan struct{})} + waiting := room.observing + room.shellReadback = &opencode.ShellSnapshot{} + h.mu.Unlock() + var observeErr error + defer func() { + h.mu.Lock() + room.shellReadback = nil + waiting.err = observeErr + room.observing = nil + close(waiting.done) + h.mu.Unlock() + }() + if err := h.startStream(room, broker); err != nil { + observeErr = err + return err + } + shells, err := broker.Shells(room.ctx, room.sessionID) + h.mu.Lock() + if room.ctx.Err() != nil || h.rooms[room.sessionID] != room { + h.mu.Unlock() + observeErr = context.Canceled + return observeErr + } + if err == nil { + shells, err = room.shellReadback.Merge(shells) + } + if err != nil { + // The firehose remains useful when its authoritative shell readback is + // temporarily unavailable. Do not turn that read failure into a chat + // failure (or claim the last retained snapshot is current): clear the + // replay cache and tell attached clients that shell visibility is limited. + // Preserve events already folded during this read, but clear stale replay. + live, mergeErr := room.shellReadback.Merge(nil) + if mergeErr == nil { + h.replaceShellsLocked(room.sessionID, live) + } + h.mu.Unlock() + // Never log broker bodies/URLs: they may contain credentials or private output. + log.Printf("chat shell readback session=%s failed (%T)", room.sessionID, err) + h.fanout(room.sessionID, protocol.Message{Type: protocol.Error, Error: "background shell visibility is limited"}, "") + return nil + } + h.replaceShellsLocked(room.sessionID, shells) + room.shellReadback = nil + h.mu.Unlock() + return nil +} + // attach registers write as one client's subscription to sessionID's stream and // returns it, or the existing subscription when the same connection has already // attached. It is idempotent so a client may prompt and subscribe without @@ -209,6 +294,11 @@ func (h *chatHub) detach(sessionID string, sub *chatSub) { if len(room.subs) == 0 { delete(h.rooms, sessionID) room.cancel() + // ReleaseShells is a local identity-cache operation (no broker I/O). + // Complete it before a reconnect can acquire this session's new room. + if releaser, ok := room.broker.(chatShellReleaser); ok { + releaser.ReleaseShells(sessionID) + } } h.mu.Unlock() } @@ -265,9 +355,17 @@ func (h *chatHub) broadcastUser(frame protocol.ChatEventFrame, exceptClientID st // once it is written. A step-ended frame finishes the turn, letting the queue // advance. func (h *chatHub) emit(frame protocol.ChatEventFrame) { - h.apply(frame) + h.emitRoom(nil, frame) +} + +func (h *chatHub) emitRoom(source *chatRoom, frame protocol.ChatEventFrame) { h.mu.Lock() room := h.rooms[frame.SessionID] + if source != nil && (source != room || source.ctx.Err() != nil) { + h.mu.Unlock() + return + } + h.applyLocked(frame) buffered := false var targets []*chatSub if room != nil { @@ -283,6 +381,22 @@ func (h *chatHub) emit(frame protocol.ChatEventFrame) { if room != nil && frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepEnded { h.finishTurnLocked(room) } + // Shell replay and live lifecycle delivery must enqueue in the same order + // as folding. Otherwise an exit can overtake an older running replay. + if frame.Kind == protocol.ChatEventShell && !buffered { + msg := protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame} + var slow []*chatSub + for _, sub := range targets { + if !enqueueChatLocked(sub, msg) { + slow = append(slow, sub) + } + } + h.mu.Unlock() + for _, sub := range slow { + sub.close() + } + return + } h.mu.Unlock() if buffered { return @@ -293,6 +407,62 @@ func (h *chatHub) emit(frame protocol.ChatEventFrame) { } } +// enqueueChatLocked never blocks or calls detach under the hub mutex. +func enqueueChatLocked(sub *chatSub, msg protocol.Message) bool { + select { + case <-sub.done: + return true + default: + } + select { + case sub.ch <- msg: + return true + default: + return false + } +} + +func (h *chatHub) replayShells(sub *chatSub) { + h.mu.Lock() + slow := false + if h.rooms[sub.room.sessionID] == sub.room && sub.room.subs[sub.clientID] == sub { + turn := h.turnLocked(sub.room.sessionID) + for _, id := range turn.shellOrder { + shell := turn.shells[id] + frame := protocol.ChatEventFrame{SessionID: sub.room.sessionID, Kind: protocol.ChatEventShell, Shell: &shell} + if !enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}) { + slow = true + break + } + } + } + h.mu.Unlock() + if slow { + sub.close() + } +} + +// replaceShells reconciles a successful authoritative readback. It drops stale +// retained entries (including exits missed while disconnected) and upserts the +// verified running set without fabricating a lifecycle event for foreign shells. +func (h *chatHub) replaceShells(sessionID string, shells []protocol.ChatShell) { + h.mu.Lock() + defer h.mu.Unlock() + h.replaceShellsLocked(sessionID, shells) +} + +func (h *chatHub) replaceShellsLocked(sessionID string, shells []protocol.ChatShell) { + turn := h.turnLocked(sessionID) + for _, shell := range turn.shells { + turn.shrink(len(shell.Command)) + } + turn.shells = nil + turn.shellOrder = nil + for i := range shells { + turn.applyShell(&shells[i]) + } +} + // finishTurnLocked marks the room's turn finished and releases the turn runner's // wait. Callers must hold h.mu. func (h *chatHub) finishTurnLocked(room *chatRoom) { @@ -331,14 +501,20 @@ func (h *chatHub) flushBuffered(sessionID string) { targets = append(targets, sub) } } - h.mu.Unlock() + var slow []*chatSub for i := range pending { frame := pending[i] msg := protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame} for _, sub := range targets { - sub.send(msg) + if !enqueueChatLocked(sub, msg) { + slow = append(slow, sub) + } } } + h.mu.Unlock() + for _, sub := range slow { + sub.close() + } } // submitPrompt accepts one prompt: it starts the turn runner immediately when no @@ -476,7 +652,7 @@ func (h *chatHub) runTurn(room *chatRoom, p *chatQueuedPrompt, done chan struct{ return } if first { - if err := h.startStream(room, broker); err != nil { + if err := h.ensureObserving(room, broker); err != nil { h.failPrompt(room, p, err) h.endRoomTurn(room) return @@ -550,7 +726,7 @@ func (h *chatHub) startStream(room *chatRoom, broker chatBroker) error { result <- broker.Stream(ctx, room.sessionID, func() { once.Do(func() { close(attached) }) }, func(frame protocol.ChatEventFrame) error { - h.emit(frame) + h.emitRoom(room, frame) return nil }) }() diff --git a/internal/server/chat_shell_reconcile_test.go b/internal/server/chat_shell_reconcile_test.go new file mode 100644 index 00000000..9ada615b --- /dev/null +++ b/internal/server/chat_shell_reconcile_test.go @@ -0,0 +1,261 @@ +package server + +import ( + "context" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" + "github.com/coder/websocket" + "github.com/coder/websocket/wsjson" +) + +type shellBarrierBroker struct { + *fakeChat + read chan struct{} + apply chan struct{} + events chan shellBarrierEvent + stopped chan struct{} + reads atomic.Int32 + snapshot []protocol.ChatShell +} + +// Exercise the real websocket handler on a loopback scratch server and a fresh +// SQLite database. A marker on the shared stream bounds the replay assertion +// without sleeps, polling or a negative-read timeout. +func TestChatShellReconciliationScratchServer(t *testing.T) { + b := newShellBarrierBroker() + b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: b}, newOperatorStore(t))) + defer srv.Close() + conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil) + if err != nil { + t.Fatal(err) + } + defer conn.CloseNow() + subscribe := func(id string) { + t.Helper() + if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: id, ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil { + t.Fatal(err) + } + } + read := func() protocol.Message { + t.Helper() + var m protocol.Message + if err := wsjson.Read(ctx, conn, &m); err != nil { + t.Fatal(err) + } + return m + } + if m := read(); m.Type != protocol.PipelineList { + t.Fatalf("server greeting = %s", m.Type) + } + subscribe("first") + if m := read(); m.ID != "first" || m.Type != protocol.ChatSubscribe { + t.Fatalf("ack = %+v", m) + } + shellBarrierWait(t, b.read) + done := make(chan struct{}) + b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_old", Status: protocol.ChatShellExited}, done} + shellBarrierWait(t, done) + if m := read(); m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellExited { + t.Fatalf("exit not visible: %+v", m) + } + close(b.apply) + subscribe("second") + for { + m := read() + if m.ChatEvent != nil && m.ChatEvent.Shell.Status == protocol.ChatShellRunning { + t.Fatalf("stale replay: %+v", m) + } + if m.ID == "second" { + break + } + } + done = make(chan struct{}) + b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_marker", Status: protocol.ChatShellExited}, done} + shellBarrierWait(t, done) + for { + m := read() + if m.ChatEvent == nil { + continue + } + if m.ChatEvent.Shell.Status == protocol.ChatShellRunning { + t.Fatalf("ghost visible after apply: %+v", m) + } + if m.ChatEvent.Shell.ID == "sh_marker" { + break + } + } + if b.reads.Load() != 1 { + t.Fatalf("shared observer read count = %d", b.reads.Load()) + } + conn.CloseNow() + shellBarrierWait(t, b.stopped) +} + +type shellBarrierEvent struct { + shell protocol.ChatShell + done chan struct{} +} + +func newShellBarrierBroker() *shellBarrierBroker { + return &shellBarrierBroker{fakeChat: scriptedChat(), read: make(chan struct{}, 8), apply: make(chan struct{}), events: make(chan shellBarrierEvent), stopped: make(chan struct{})} +} + +func (b *shellBarrierBroker) Shells(ctx context.Context, _ string) ([]protocol.ChatShell, error) { + b.reads.Add(1) + snapshot := append([]protocol.ChatShell(nil), b.snapshot...) + b.read <- struct{}{} + select { + case <-b.apply: + return snapshot, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (b *shellBarrierBroker) Stream(ctx context.Context, id string, ready func(), emit func(protocol.ChatEventFrame) error) error { + defer close(b.stopped) + ready() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case event := <-b.events: + if err := emit(protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventShell, Shell: &event.shell}); err != nil { + return err + } + close(event.done) + } + } +} + +func shellBarrierWait(t *testing.T, ch <-chan struct{}) { + t.Helper() + select { + case <-ch: + case <-time.After(3 * time.Second): + t.Fatal("barrier timed out") + } +} + +func TestChatShellReconciliationBarrier(t *testing.T) { + for _, status := range []string{protocol.ChatShellExited, protocol.ChatShellRunning, "timeout", "killed"} { + t.Run(status, func(t *testing.T) { + b := newShellBarrierBroker() + b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}} + h := newChatHub() + messages := make(chan protocol.Message, 32) + sub := h.attach("ses_1", "one", "", b, func(m protocol.Message) error { messages <- m; return nil }) + t.Cleanup(sub.close) + result := make(chan error, 2) + go func() { result <- h.ensureObserving(sub.room, b) }() + shellBarrierWait(t, b.read) + // A second observer must share the in-flight read, not apply an + // independently completing older/newer snapshot out of order. + go func() { result <- h.ensureObserving(sub.room, b) }() + shell := protocol.ChatShell{ID: "sh_old", Status: status} + if status == protocol.ChatShellRunning { + shell.ID = "sh_new" + shell.Command = "sleep 10" + } + done := make(chan struct{}) + select { + case b.events <- shellBarrierEvent{shell, done}: + case <-time.After(time.Second): + t.Fatal("event blocked behind readback") + } + shellBarrierWait(t, done) + select { + case m := <-messages: + if m.ChatEvent == nil || m.ChatEvent.Shell == nil || m.ChatEvent.Shell.ID != shell.ID || m.ChatEvent.Shell.Status != status { + t.Fatalf("live shell = %+v", m) + } + case <-time.After(time.Second): + t.Fatal("live event not visible during read") + } + close(b.apply) + for range 2 { + select { + case err := <-result: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("observer stuck") + } + } + if b.reads.Load() != 1 { + t.Fatalf("redundant/out-of-order reads: %d", b.reads.Load()) + } + snapshot := h.shellSnapshot("ses_1") + if status == protocol.ChatShellRunning { + if len(snapshot) != 2 { + t.Fatalf("new create lost: %+v", snapshot) + } + } else if len(snapshot) != 0 { + t.Fatalf("terminal shell resurrected: %+v", snapshot) + } + // Assert the same retained state is replayed to the client. + h.replayShells(sub) + for range len(snapshot) { + select { + case m := <-messages: + if m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellRunning { + t.Fatalf("replay = %+v", m) + } + case <-time.After(time.Second): + t.Fatal("replay missing") + } + } + sub.close() + shellBarrierWait(t, b.stopped) + }) + } +} + +func TestChatShellReconciliationCancelReconnectBarrier(t *testing.T) { + b := newShellBarrierBroker() + h := newChatHub() + sub := h.attach("ses_1", "one", "", b, func(protocol.Message) error { return nil }) + results := make(chan error, 2) + go func() { results <- h.ensureObserving(sub.room, b) }() + shellBarrierWait(t, b.read) + go func() { results <- h.ensureObserving(sub.room, b) }() + sub.close() + for range 2 { + select { + case err := <-results: + if err == nil { + t.Fatal("canceled observer succeeded") + } + case <-time.After(time.Second): + t.Fatal("canceled waiter stuck") + } + } + shellBarrierWait(t, b.stopped) + next := newShellBarrierBroker() + next.snapshot = []protocol.ChatShell{{ID: "sh_new", Status: protocol.ChatShellRunning}} + close(next.apply) + newSub := h.attach("ses_1", "two", "", next, func(protocol.Message) error { return nil }) + t.Cleanup(newSub.close) + if err := h.ensureObserving(newSub.room, next); err != nil { + t.Fatal(err) + } + // A late callback from the canceled stream cannot poison the new room, + // nor can a foreign-session callback from the current stream. + h.emitRoom(sub.room, protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_ghost", Status: protocol.ChatShellRunning}}) + h.emitRoom(newSub.room, protocol.ChatEventFrame{SessionID: "ses_foreign", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_foreign", Status: protocol.ChatShellRunning}}) + got := h.shellSnapshot("ses_1") + if len(got) != 1 || got[0].ID != "sh_new" || len(h.shellSnapshot("ses_foreign")) != 0 { + t.Fatalf("reconnect state = %+v", got) + } + newSub.close() + shellBarrierWait(t, next.stopped) +} diff --git a/internal/server/chat_stream.go b/internal/server/chat_stream.go index 76c175e6..89ba6404 100644 --- a/internal/server/chat_stream.go +++ b/internal/server/chat_stream.go @@ -111,7 +111,14 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "could not attach to session " + query.SessionID}) return } + // Acknowledge before the stream is allowed to fan out: otherwise an immediately + // available retained shell can overtake the subscribe reply on this connection. _ = write(protocol.Message{Type: protocol.ChatSubscribe, ID: request.ID}) + if err := hub.ensureObserving(sub.room, broker); err != nil { + sub.close() + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "could not observe session " + query.SessionID}) + return + } // Replay every still-unresolved approval request the room holds, not only the // most recent one, so a client that attaches after two requests were raised // still sees both cards. @@ -124,9 +131,5 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH // shows them rather than nothing until they exit. The message snapshot cannot // carry a shell — a shell outlives the turn and is not a turn — so this is the // shell half of the resume. - for _, shell := range hub.shellSnapshot(query.SessionID) { - running := shell - frame := protocol.ChatEventFrame{SessionID: query.SessionID, Kind: protocol.ChatEventShell, Shell: &running} - sub.send(protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}) - } + hub.replayShells(sub) } diff --git a/internal/server/chat_test.go b/internal/server/chat_test.go index 29dbc4bb..d4a15cd2 100644 --- a/internal/server/chat_test.go +++ b/internal/server/chat_test.go @@ -3,6 +3,7 @@ package server import ( "context" "encoding/json" + "errors" "fmt" "net/http" "net/http/httptest" @@ -52,6 +53,8 @@ type fakeChat struct { permissionSessionID string permissions []protocol.ChatPermissionFrame permissionErr error + shells []protocol.ChatShell + shellErr error sessionCalls int sessions []protocol.ChatSession sessionErr error @@ -62,6 +65,8 @@ type fakeChat struct { started chan struct{} done chan struct{} startOnce sync.Once + streamCalls int + readyGate <-chan struct{} block bool subscribed bool promptBeforeSubscribe bool @@ -91,13 +96,18 @@ func (f *fakeChat) Prompt(context.Context, string, string) error { func (f *fakeChat) Stream(ctx context.Context, _ string, ready func(), emit func(protocol.ChatEventFrame) error) error { f.mu.Lock() f.subscribed = true + f.streamCalls++ + readyGate := f.readyGate f.mu.Unlock() - if ready != nil { - ready() - } if f.started != nil { f.startOnce.Do(func() { close(f.started) }) } + if readyGate != nil { + <-readyGate + } + if ready != nil { + ready() + } if f.block { <-ctx.Done() if f.done != nil { @@ -116,6 +126,42 @@ func (f *fakeChat) Stream(ctx context.Context, _ string, ready func(), emit func return nil } +// TestChatHubConcurrentObserversShareStream pins the start-up handoff: two +// clients that subscribe before the broker says ready still share one firehose. +func TestChatHubConcurrentObserversShareStream(t *testing.T) { + broker := scriptedChat() + broker.block = true + broker.started = make(chan struct{}) + gate := make(chan struct{}) + broker.readyGate = gate + hub := newChatHub() + hub.mu.Lock() + room := hub.roomLocked("ses_1") + hub.mu.Unlock() + + errs := make(chan error, 2) + go func() { errs <- hub.ensureObserving(room, broker) }() + select { + case <-broker.started: + case <-time.After(time.Second): + t.Fatal("first observer did not start the broker stream") + } + go func() { errs <- hub.ensureObserving(room, broker) }() + close(gate) + for range 2 { + if err := <-errs; err != nil { + t.Fatalf("ensureObserving: %v", err) + } + } + broker.mu.Lock() + calls := broker.streamCalls + broker.mu.Unlock() + if calls != 1 { + t.Fatalf("broker Stream calls = %d, want 1", calls) + } + room.cancel() +} + func (f *fakeChat) Interrupt(context.Context, string) error { f.mu.Lock() defer f.mu.Unlock() @@ -151,6 +197,12 @@ func (f *fakeChat) Reply(_ context.Context, request protocol.ChatReplyRequest) e // Permissions returns the seeded pending requests, or blocks until the caller's // context is canceled when permBlock is set (used to prove the poller stops on // disconnect). Every call is counted so a test can assert polling happened. +func (f *fakeChat) Shells(context.Context, string) ([]protocol.ChatShell, error) { + f.mu.Lock() + defer f.mu.Unlock() + return append([]protocol.ChatShell(nil), f.shells...), f.shellErr +} + func (f *fakeChat) Permissions(ctx context.Context, sessionID string) ([]protocol.ChatPermissionFrame, error) { f.mu.Lock() f.permissionCalls++ @@ -682,6 +734,9 @@ func TestChatInterruptEndsRunningPrompt(t *testing.T) { case r.Method == http.MethodGet && r.URL.Path == "/api/permission/request": w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"location":{"directory":"/work"},"data":[]}`)) + case r.Method == http.MethodGet && r.URL.Path == "/api/shell": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/event": flusher, ok := w.(http.Flusher) if !ok { @@ -995,6 +1050,7 @@ func TestChatPromptStreamsShellFrame(t *testing.T) { // shows the background process instead of nothing until it exits. func TestChatSubscribeReplaysRunningShell(t *testing.T) { broker := scriptedChat() + broker.shells = []protocol.ChatShell{{ID: "sh_1", Status: protocol.ChatShellRunning, Command: "sleep 120"}} broker.script = append([]protocol.ChatEventFrame{ {SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_1", Status: protocol.ChatShellRunning, Command: "sleep 120"}}, }, broker.script...) @@ -1039,6 +1095,40 @@ func TestChatSubscribeReplaysRunningShell(t *testing.T) { } } +// TestChatSubscribeKeepsStreamWhenShellReadbackFails ensures a temporary +// authoritative readback failure does not make chat unavailable or replay a +// stale process as running. The subscriber receives an explicit limitation +// while its broker stream remains attached for future lifecycle events. +func TestChatSubscribeKeepsStreamWhenShellReadbackFails(t *testing.T) { + broker := scriptedChat() + broker.block = true + broker.shellErr = errors.New("shell endpoint unavailable: https://user:secret@example.invalid private response body") + ctx, _, conn := dialChat(t, broker) + if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub1", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil { + t.Fatal(err) + } + var ack protocol.Message + if err := wsjson.Read(ctx, conn, &ack); err != nil { + t.Fatal(err) + } + if ack.Type != protocol.ChatSubscribe || ack.ID != "sub1" { + t.Fatalf("subscribe ack = %+v", ack) + } + var limited protocol.Message + if err := wsjson.Read(ctx, conn, &limited); err != nil { + t.Fatal(err) + } + if limited.Type != protocol.Error || limited.Error != "background shell visibility is limited" { + t.Fatalf("limited visibility notice = %+v", limited) + } + broker.mu.Lock() + calls := broker.streamCalls + broker.mu.Unlock() + if calls != 1 { + t.Fatalf("broker Stream calls = %d, want 1", calls) + } +} + // TestChatPromptSubscribesBeforePrompt pins the subscribe-before-prompt order: // the firehose has no replay cursor, so the subscription must be attached before // the prompt is posted or deltas emitted in between are lost. @@ -1079,6 +1169,9 @@ func realAdapterChat(t *testing.T, frames []string) []protocol.ChatEventFrame { case r.Method == http.MethodGet && r.URL.Path == "/api/permission/request": w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"location":{"directory":"/work"},"data":[]}`)) + case r.Method == http.MethodGet && r.URL.Path == "/api/shell": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/event": streams.Add(1) w.Header().Set("Content-Type", "text/event-stream") From 2cdb9a9e9300a12d1ef65e8bc6b5f9cb5431536e Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 18 Oct 2026 14:50:18 +0100 Subject: [PATCH 2/2] Stabilize guarded model delivery Add validated OpenCode-only delivery capabilities, canonical linked-worktree access checks, isolated candidate verification, and guarded main settlement. Wire the bounded reconnect-foreground controlled HTTP fixture through ordinary server construction so target verification obtains server-controlled provenance and rejects unknown or incompatible fixture evidence. Checked packaged templates and packaged authoring skill references; no updates are needed because this draft-only infrastructure activates no definition. Recorded the stale reference and activation guidance in docs/notes/delivery-stabilization.md for the docs fold. --- cmd/pipelines-server/main.go | 2 +- docs/notes/delivery-stabilization.md | 47 +++ internal/server/delivery_gate.go | 185 +++++++++++ internal/server/landing_access.go | 130 ++++++++ internal/server/landing_access_test.go | 304 +++++++++++++++++++ internal/server/runner.go | 61 +++- internal/server/verification_fixture.go | 135 ++++++++ internal/server/verification_fixture_test.go | 148 +++++++++ internal/server/websocket.go | 8 + internal/stage/agent.go | 1 + internal/stage/model.go | 16 +- internal/stage/tools.go | 75 +++++ internal/stage/tools_test.go | 104 +++++++ internal/stage/verification.go | 97 ++++++ internal/stage/verification_test.go | 27 ++ 15 files changed, 1335 insertions(+), 5 deletions(-) create mode 100644 docs/notes/delivery-stabilization.md create mode 100644 internal/server/delivery_gate.go create mode 100644 internal/server/landing_access.go create mode 100644 internal/server/landing_access_test.go create mode 100644 internal/server/verification_fixture.go create mode 100644 internal/server/verification_fixture_test.go create mode 100644 internal/stage/verification.go create mode 100644 internal/stage/verification_test.go diff --git a/cmd/pipelines-server/main.go b/cmd/pipelines-server/main.go index ec789886..54415ddf 100644 --- a/cmd/pipelines-server/main.go +++ b/cmd/pipelines-server/main.go @@ -47,7 +47,7 @@ func main() { // Each new run gets a bounded one-line title from the configured default // harness; the summariser falls back to the prompt's first line, so title // generation never blocks or fails run creation. - options := server.Options{ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason} + options := server.Options{ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason, VerificationFixtures: server.VerificationFixturesFromEnvironment(os.Getenv)} recorded := "" if value, err := store.OpenCodeEndpoint(ctx); err != nil { log.Printf("OpenCode endpoint setting unreadable: %v", err) diff --git a/docs/notes/delivery-stabilization.md b/docs/notes/delivery-stabilization.md new file mode 100644 index 00000000..bba21fd9 --- /dev/null +++ b/docs/notes/delivery-stabilization.md @@ -0,0 +1,47 @@ +# Delivery stabilization + +Date: 2026-10-18. + +The server now has draft-only `delivery=true` support for OpenCode model stages. +At run time it derives the canonical `main` worktree and linked-worktree Git +metadata from Git rather than accepting paths from a stage prompt. It verifies +the committed item tree in a disposable candidate worktree before fast-forwarding +canonical `main`; declared checks are bounded configuration, not proof that any +deployed definition has used them. It is denied for Pi (until equivalent +capability enforcement exists), foreign repositories, missing main worktrees, +and malformed capabilities. This item does not publish or deploy a definition. + +The source also supports an optional controlled-fixture verification declaration +(`backend`, `revision`, `capability`, and `fixture`). Ordinary server setup wires +an explicit, bounded `reconnect-foreground` HTTP fixture registry when +`PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_URL` and its expected `BACKEND`, +`REVISION`, `CAPABILITY`, and `STATE` values are configured. There is no default +remote URL. The server, not the model, probes that registered endpoint and +collects its returned backend identity, build revision, capability, fixture ID, +and state before comparing provenance. Unknown fixtures, incomplete setup, +wrong backend/revision/capability/state, non-200 responses, cancellation, and +oversized or malformed responses fail closed. A model result's labels are not +evidence. This is controlled local/backend fixture infrastructure, not evidence +of a rendered layout or a live Android device. +Definitions must be scratch-validated against the deployed server only after +the server support is deployed; no future delivery or verification definition +is activated by this source change. + +`internal/author/skill/references/format.md` and +`internal/author/skill/references/queue.md` are stale: a later docs fold should +describe `delivery=true`, OpenCode-only availability, candidate verification, +and the required activation order (deploy server support before scratch-validating +and then publishing a definition). +OpenCode V2 resolves external resources at a containing directory, so delivery +uses the verified repository metadata directory rather than an impossible +single-lock-file scope. It does not grant a parent, home, foreign repository, +or global policy. Unrestricted shell permissions are not an OS sandbox. + +Activation remains draft-only: deploy compatible server support, configure the +controlled fixture endpoint and matching provenance, scratch-validate the +fixture, then publish a definition. No definition is activated by this source +change. + +Recovery checklist: retain `connection-reconnect` and unfinished feature +branches; merge only complete committed trees that pass mapped checks; do not +infer delivery from a dirty worktree or an old deployed-backend capture. diff --git a/internal/server/delivery_gate.go b/internal/server/delivery_gate.go new file mode 100644 index 00000000..df1dc7c5 --- /dev/null +++ b/internal/server/delivery_gate.go @@ -0,0 +1,185 @@ +package server + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" + + "git.a-wan.net/alan/pipelines/internal/stage" +) + +// verifyDelivery proves that the stage landed a complete, committed tree before +// its success can settle the run. It re-derives the capability after the stage: +// a stage result and its input cannot select either the landing path or checks. +const maxDeliveryCheckOutput = 64 << 10 +const deliveryCleanupTimeout = 10 * time.Second + +// verifyDelivery verifies the committed item tree in a disposable worktree +// before fast-forwarding canonical main. Settlement deliberately performs only +// non-mutating repository checks after that mutation: a declared check must +// never be the first signal that the landed tree is known-red. +func verifyDelivery(ctx context.Context, workspace string, access stage.LandingAccess, checks []string) error { + lock, err := acquireDeliveryLock(ctx, access.LockPath) + if err != nil { + return err + } + defer lock.Close() + current, err := landingAccess(workspace) + if err != nil { + return fmt.Errorf("delivery access changed: %w", err) + } + if current != access { + return fmt.Errorf("delivery access changed during stage") + } + for _, directory := range []string{workspace, access.MainWorktree} { + status, err := gitOutput(directory, "status", "--porcelain=v1", "--untracked-files=all") + if err != nil { + return fmt.Errorf("delivery checkout status: %w", err) + } + if status != "" { + return fmt.Errorf("delivery checkout is not clean: %s", strings.Split(status, "\n")[0]) + } + } + itemHead, err := gitOutput(workspace, "rev-parse", "HEAD") + if err != nil { + return fmt.Errorf("delivery item commit: %w", err) + } + if err := runDeliveryCandidate(ctx, workspace, itemHead, checks); err != nil { + return err + } + mainHead, err := gitOutput(access.MainWorktree, "rev-parse", "HEAD") + if err != nil { + return fmt.Errorf("delivery main commit: %w", err) + } + if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", mainHead, itemHead).Run(); err != nil { + return fmt.Errorf("delivery item commit cannot fast-forward canonical main") + } + if output, err := exec.CommandContext(ctx, "git", "-C", access.MainWorktree, "merge", "--ff-only", itemHead).CombinedOutput(); err != nil { + return fmt.Errorf("delivery fast-forward canonical main: %w: %s", err, strings.TrimSpace(string(output))) + } + for _, directory := range []string{workspace, access.MainWorktree} { + status, err := gitOutput(directory, "status", "--porcelain=v1", "--untracked-files=all") + if err != nil { + return fmt.Errorf("delivery settlement status: %w", err) + } + if status != "" { + return fmt.Errorf("delivery checkout is not clean after checks: %s", strings.Split(status, "\n")[0]) + } + } + // Re-derive at the settlement boundary. A success means exactly the checked + // item commit is now reachable from the same canonical main worktree. + current, err = landingAccess(workspace) + if err != nil || current != access { + return fmt.Errorf("delivery access changed during settlement") + } + mainHead, err = gitOutput(access.MainWorktree, "rev-parse", "HEAD") + if err != nil { + return fmt.Errorf("delivery main commit: %w", err) + } + if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", itemHead, mainHead).Run(); err != nil { + return fmt.Errorf("delivery item commit is not an ancestor of canonical main") + } + return nil +} + +func acquireDeliveryLock(ctx context.Context, path string) (*os.File, error) { + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return nil, fmt.Errorf("delivery lock: %w", err) + } + for { + err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) + if err == nil { + return file, nil + } + if !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EAGAIN) { + _ = file.Close() + return nil, fmt.Errorf("delivery lock: %w", err) + } + select { + case <-ctx.Done(): + _ = file.Close() + return nil, fmt.Errorf("delivery lock: %w", ctx.Err()) + case <-time.After(20 * time.Millisecond): + } + } +} + +func runDeliveryCandidate(ctx context.Context, workspace, itemHead string, checks []string) error { + candidate, err := os.MkdirTemp(filepath.Dir(workspace), ".pipelines-delivery-") + if err != nil { + return fmt.Errorf("create delivery candidate: %w", err) + } + if err := os.Remove(candidate); err != nil { + return fmt.Errorf("prepare delivery candidate: %w", err) + } + if output, err := boundedCommandOutput(ctx, workspace, "worktree", "add", "--detach", candidate, itemHead); err != nil { + cleanupDeliveryCandidate(workspace, candidate) + return fmt.Errorf("create delivery candidate: %w: %s", err, strings.TrimSpace(string(output))) + } + defer cleanupDeliveryCandidate(workspace, candidate) + for _, check := range checks { + if err := runDeliveryCheck(ctx, candidate, check); err != nil { + return err + } + } + return nil +} + +// cleanupDeliveryCandidate always uses its own bounded context: the stage context +// may already be canceled when a check fails or times out. +func cleanupDeliveryCandidate(workspace, candidate string) { + ctx, cancel := context.WithTimeout(context.Background(), deliveryCleanupTimeout) + defer cancel() + _ = exec.CommandContext(ctx, "git", "-C", workspace, "worktree", "remove", "--force", candidate).Run() + _ = os.RemoveAll(candidate) +} + +func boundedCommandOutput(ctx context.Context, directory string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, "git", append([]string{"-C", directory}, args...)...) + var output boundedDeliveryOutput + cmd.Stdout = &output + cmd.Stderr = &output + err := cmd.Run() + return []byte(output.String()), err +} + +func runDeliveryCheck(ctx context.Context, directory, check string) error { + cmd := exec.CommandContext(ctx, "sh", "-c", check) + cmd.Dir = directory + var output boundedDeliveryOutput + cmd.Stdout = &output + cmd.Stderr = &output + if err := cmd.Run(); err != nil { + return fmt.Errorf("delivery check %q failed: %w: %s", check, err, output.String()) + } + return nil +} + +type boundedDeliveryOutput struct{ bytes.Buffer } + +func (b *boundedDeliveryOutput) Write(p []byte) (int, error) { + remaining := maxDeliveryCheckOutput - b.Len() + if remaining > 0 { + _, _ = b.Buffer.Write(p[:min(remaining, len(p))]) + } + return len(p), nil +} + +func (b *boundedDeliveryOutput) String() string { + text := strings.TrimSpace(b.Buffer.String()) + if b.Len() == maxDeliveryCheckOutput { + return text + " (output truncated)" + } + return text +} + +var _ io.Writer = (*boundedDeliveryOutput)(nil) diff --git a/internal/server/landing_access.go b/internal/server/landing_access.go new file mode 100644 index 00000000..a373d625 --- /dev/null +++ b/internal/server/landing_access.go @@ -0,0 +1,130 @@ +package server + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "git.a-wan.net/alan/pipelines/internal/stage" +) + +// landingAccess derives delivery paths from Git's verified worktree metadata, +// never from stage input. A missing canonical main worktree fails closed. +func landingAccess(workspace string) (stage.LandingAccess, error) { + workspace, err := canonicalDirectory(workspace) + if err != nil { + return stage.LandingAccess{}, fmt.Errorf("delivery workspace: %w", err) + } + gitDir, err := gitOutput(workspace, "rev-parse", "--git-dir") + if err != nil { + return stage.LandingAccess{}, fmt.Errorf("delivery workspace is not a Git worktree: %w", err) + } + commonDir, err := gitOutput(workspace, "rev-parse", "--git-common-dir") + if err != nil { + return stage.LandingAccess{}, fmt.Errorf("delivery repository metadata: %w", err) + } + gitDir, err = canonicalDirectory(resolveGitPath(workspace, gitDir)) + if err != nil { + return stage.LandingAccess{}, fmt.Errorf("delivery Git metadata: %w", err) + } + commonDir, err = canonicalDirectory(resolveGitPath(workspace, commonDir)) + if err != nil { + return stage.LandingAccess{}, fmt.Errorf("delivery common Git metadata: %w", err) + } + main, err := mainWorktree(workspace, commonDir) + if err != nil { + return stage.LandingAccess{}, err + } + // Delivery originates in a linked item worktree. Its metadata must be under + // the repository's canonical common directory, never a standalone or foreign + // checkout. Git's common directory is the bounded capability needed for refs, + // worktree state, and the landing lock. + if !pathWithin(filepath.Join(commonDir, "worktrees"), gitDir) { + return stage.LandingAccess{}, fmt.Errorf("delivery workspace is not a linked Git worktree") + } + return stage.LandingAccess{MainWorktree: main, GitMetadata: commonDir, LockPath: filepath.Join(commonDir, "pipelines-landing.lock")}, nil +} + +func mainWorktree(workspace, commonDir string) (string, error) { + list, err := gitOutput(workspace, "worktree", "list", "--porcelain") + if err != nil { + return "", fmt.Errorf("list repository worktrees: %w", err) + } + for _, record := range strings.Split(strings.TrimSpace(list), "\n\n") { + lines := strings.Split(record, "\n") + if len(lines) == 0 || !strings.HasPrefix(lines[0], "worktree ") || !worktreeHasBranch(lines, "refs/heads/main") { + continue + } + candidate, err := canonicalDirectory(strings.TrimPrefix(lines[0], "worktree ")) + if err != nil || candidate == workspace { + continue + } + candidateGitDir, err := gitOutput(candidate, "rev-parse", "--git-dir") + if err != nil { + continue + } + candidateGitDir, err = canonicalDirectory(resolveGitPath(candidate, candidateGitDir)) + if err != nil || candidateGitDir != commonDir { + continue + } + candidateCommon, err := gitOutput(candidate, "rev-parse", "--git-common-dir") + if err != nil { + continue + } + candidateCommon, err = canonicalDirectory(resolveGitPath(candidate, candidateCommon)) + if err == nil && candidateCommon == commonDir { + return candidate, nil + } + } + return "", fmt.Errorf("no canonical main worktree in the delivery repository") +} + +func worktreeHasBranch(lines []string, branch string) bool { + for _, line := range lines[1:] { + if line == "branch "+branch { + return true + } + } + return false +} + +func canonicalDirectory(path string) (string, error) { + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + return "", err + } + info, err := os.Stat(resolved) + if err != nil { + return "", err + } + if !info.IsDir() { + return "", fmt.Errorf("not a directory") + } + return filepath.Clean(resolved), nil +} + +func resolveGitPath(workspace, value string) string { + if filepath.IsAbs(value) { + return value + } + return filepath.Join(workspace, value) +} + +func pathWithin(parent, child string) bool { + rel, err := filepath.Rel(parent, child) + return err == nil && rel != "." && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + +func gitOutput(directory string, args ...string) (string, error) { + cmd := exec.Command("git", append([]string{"-C", directory}, args...)...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + data, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String())) + } + return strings.TrimSpace(string(data)), nil +} diff --git a/internal/server/landing_access_test.go b/internal/server/landing_access_test.go new file mode 100644 index 00000000..9d2d7ae8 --- /dev/null +++ b/internal/server/landing_access_test.go @@ -0,0 +1,304 @@ +package server + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func TestLandingAccessAcceptsOnlyVerifiedSameRepositoryMainWorktree(t *testing.T) { + root := t.TempDir() + main := filepath.Join(root, "main") + runGit(t, root, "init", "-b", "main", main) + runGit(t, main, "config", "user.email", "test@example.invalid") + runGit(t, main, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, main, "add", "README") + runGit(t, main, "commit", "-m", "base") + item := filepath.Join(root, "item") + runGit(t, main, "worktree", "add", "-b", "item", item) + access, err := landingAccess(item) + if err != nil { + t.Fatal(err) + } + canonicalMain, err := canonicalDirectory(main) + if err != nil { + t.Fatal(err) + } + if access.MainWorktree != canonicalMain || access.GitMetadata != filepath.Join(canonicalMain, ".git") || access.LockPath != filepath.Join(canonicalMain, ".git", "pipelines-landing.lock") { + t.Fatalf("unexpected access: %+v", access) + } + foreign := filepath.Join(root, "foreign") + runGit(t, root, "init", "-b", "main", foreign) + if _, err := landingAccess(foreign); err == nil { + t.Fatal("foreign repository without an item worktree gained landing access") + } + if _, err := landingAccess(main); err == nil { + t.Fatal("main worktree gained linked-worktree delivery access") + } + if err := os.RemoveAll(item); err != nil { + t.Fatal(err) + } + runGit(t, main, "worktree", "prune") + if _, err := landingAccess(main); err == nil { + t.Fatal("repository without a linked delivery worktree gained landing access") + } + secondItem := filepath.Join(root, "second-item") + runGit(t, main, "worktree", "add", "-b", "second-item", secondItem) + runGit(t, main, "branch", "-m", "main", "trunk") + if _, err := landingAccess(secondItem); err == nil { + t.Fatal("repository without a canonical main worktree gained landing access") + } +} + +func TestVerifyDeliveryRequiresCleanLandedMainAndPassingChecks(t *testing.T) { + root := t.TempDir() + main := filepath.Join(root, "main") + runGit(t, root, "init", "-b", "main", main) + runGit(t, main, "config", "user.email", "test@example.invalid") + runGit(t, main, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, main, "add", "README") + runGit(t, main, "commit", "-m", "base") + item := filepath.Join(root, "item") + runGit(t, main, "worktree", "add", "-b", "item", item) + if err := os.WriteFile(filepath.Join(item, "delivered"), []byte("complete\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, item, "add", "delivered") + runGit(t, item, "commit", "-m", "delivery") + runGit(t, main, "merge", "--ff-only", "item") + access, err := landingAccess(item) + if err != nil { + t.Fatal(err) + } + if err := verifyDelivery(context.Background(), item, access, []string{"test -f delivered"}); err != nil { + t.Fatalf("complete landed delivery rejected: %v", err) + } + if err := os.WriteFile(filepath.Join(item, "missing-helper"), []byte("dirty\n"), 0600); err != nil { + t.Fatal(err) + } + if err := verifyDelivery(context.Background(), item, access, []string{"true"}); err == nil { + t.Fatal("dirty task-owned helper was accepted") + } + if err := os.Remove(filepath.Join(item, "missing-helper")); err != nil { + t.Fatal(err) + } + if err := verifyDelivery(context.Background(), item, access, []string{"false"}); err == nil { + t.Fatal("failed required check was accepted") + } + if err := os.WriteFile(filepath.Join(item, "unlanded"), []byte("no\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, item, "add", "unlanded") + runGit(t, item, "commit", "-m", "unlanded") + if err := verifyDelivery(context.Background(), item, access, []string{"test -f unlanded"}); err != nil { + t.Fatalf("complete unlanded item was not landed after candidate verification: %v", err) + } + mainHead := runGitOutput(t, main, "rev-parse", "HEAD") + itemHead := runGitOutput(t, item, "rev-parse", "HEAD") + if mainHead != itemHead { + t.Fatal("verified item was not fast-forwarded into canonical main") + } +} + +func TestVerifyDeliveryFailingCandidateLeavesMainUnchanged(t *testing.T) { + root := t.TempDir() + main := filepath.Join(root, "main") + runGit(t, root, "init", "-b", "main", main) + runGit(t, main, "config", "user.email", "test@example.invalid") + runGit(t, main, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, main, "add", "README") + runGit(t, main, "commit", "-m", "base") + item := filepath.Join(root, "item") + runGit(t, main, "worktree", "add", "-b", "item", item) + if err := os.WriteFile(filepath.Join(item, "candidate"), []byte("candidate\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, item, "add", "candidate") + runGit(t, item, "commit", "-m", "candidate") + access, err := landingAccess(item) + if err != nil { + t.Fatal(err) + } + before := runGitOutput(t, main, "rev-parse", "HEAD") + if err := verifyDelivery(context.Background(), item, access, []string{"test -f candidate", "false"}); err == nil { + t.Fatal("failing candidate check landed main") + } + if after := runGitOutput(t, main, "rev-parse", "HEAD"); after != before { + t.Fatal("failing candidate check changed canonical main") + } +} + +func TestRunnerSettlesSuccessfulDeliveryAfterCandidateLanding(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + main := filepath.Join(root, "main") + runGit(t, root, "init", "-b", "main", main) + runGit(t, main, "config", "user.email", "test@example.invalid") + runGit(t, main, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, main, "add", "README") + runGit(t, main, "commit", "-m", "base") + item := filepath.Join(root, "item") + runGit(t, main, "worktree", "add", "-b", "item", item) + if err := os.WriteFile(filepath.Join(item, "unlanded"), []byte("no\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, item, "add", "unlanded") + runGit(t, item, "commit", "-m", "unlanded") + + store, err := pipeline.Open(filepath.Join(root, "runner.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + source := pipeline.Template([]string{"committer"}, true) + source = strings.Replace(source, "```configs", "```checks\ntrue\n```\n\n```configs", 1) + source = strings.Replace(source, "timeout=1h", "timeout=1h\ndelivery=true", 1) + if _, err := store.Submit(ctx, "delivery", source); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + r, err := newRunner(ctx, store) + if err != nil { + t.Fatal(err) + } + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "unlanded", Workspace: item, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + r.wg.Wait() + detail, err := store.GetRun(ctx, run.ID) + if err != nil { + t.Fatal(err) + } + if detail.Status != "succeeded" || len(detail.Steps) != 1 { + t.Fatalf("complete delivery did not settle successfully: %+v", detail) + } + if runGitOutput(t, main, "rev-parse", "HEAD") != runGitOutput(t, item, "rev-parse", "HEAD") { + t.Fatal("runner did not land the verified item commit") + } +} + +func TestRunnerRejectsWrongBackendVerificationEvidence(t *testing.T) { + ctx := context.Background() + source := pipeline.Template([]string{"worker"}, true) + source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground\n```\n\n```configs", 1) + r, store, id := submitRunner(t, ctx, source) + defer store.Close() + r.verifyTarget = func(context.Context, stage.VerificationTarget) (stage.VerificationEvidence, error) { + return stage.VerificationEvidence{Backend: "old-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"}, nil + } + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + run := startRun(t, r, id, "wrong-backend-evidence") + detail, err := store.GetRun(ctx, run.ID) + if err != nil { + t.Fatal(err) + } + if detail.Status != "failed" || len(detail.Steps) != 1 || !strings.Contains(detail.Steps[0].Diagnostic, "backend") { + t.Fatalf("wrong-backend evidence settled as success: %+v", detail) + } +} + +func TestRunnerIgnoresModelSuppliedVerificationEvidence(t *testing.T) { + ctx := context.Background() + source := pipeline.Template([]string{"worker"}, true) + source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground\n```\n\n```configs", 1) + r, store, id := submitRunner(t, ctx, source) + defer store.Close() + r.verifyTarget = func(context.Context, stage.VerificationTarget) (stage.VerificationEvidence, error) { + return stage.VerificationEvidence{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"}, nil + } + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{Outcome: "success", Output: stage.Values{"verification": map[string]any{"backend": "old-api", "revision": "stale", "capability": "invented", "fixture": "other"}}}, nil + } + run := startRun(t, r, id, "model-supplied-evidence") + detail, err := store.GetRun(ctx, run.ID) + if err != nil { + t.Fatal(err) + } + if detail.Status != "succeeded" || len(detail.Steps) != 1 { + t.Fatalf("controlled verification did not settle: %+v", detail) + } +} + +func TestMainWorktreeRejectsNearMatchAndLinkedMain(t *testing.T) { + for _, test := range []struct { + name string + branch string + }{ + {name: "mainline is not main", branch: "mainline"}, + {name: "linked main is not canonical", branch: "main"}, + } { + t.Run(test.name, func(t *testing.T) { + root := t.TempDir() + primary := filepath.Join(root, "primary") + runGit(t, root, "init", "-b", "trunk", primary) + runGit(t, primary, "config", "user.email", "test@example.invalid") + runGit(t, primary, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(primary, "README"), []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, primary, "add", "README") + runGit(t, primary, "commit", "-m", "base") + item := filepath.Join(root, "item") + runGit(t, primary, "worktree", "add", "-b", "item", item) + candidate := filepath.Join(root, test.branch) + runGit(t, primary, "worktree", "add", "-b", test.branch, candidate) + + commonDir, err := gitOutput(item, "rev-parse", "--git-common-dir") + if err != nil { + t.Fatal(err) + } + commonDir, err = canonicalDirectory(resolveGitPath(item, commonDir)) + if err != nil { + t.Fatal(err) + } + if _, err := mainWorktree(item, commonDir); err == nil { + t.Fatalf("%s gained canonical main access", test.name) + } + }) + } +} + +func runGit(t *testing.T, directory string, args ...string) { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", directory}, args...)...) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v: %s", args, err, output) + } +} + +func runGitOutput(t *testing.T, directory string, args ...string) string { + t.Helper() + output, err := exec.Command("git", append([]string{"-C", directory}, args...)...).Output() + if err != nil { + t.Fatalf("git %v: %v", args, err) + } + return strings.TrimSpace(string(output)) +} diff --git a/internal/server/runner.go b/internal/server/runner.go index efaa8787..260397fe 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -58,8 +58,11 @@ type runner struct { // directory. It defaults to stage.ExportSession; a failure is best-effort and // never changes the stage's status, outcome, target or result validation. exportSession func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) - wg sync.WaitGroup - schedulerWG sync.WaitGroup + // verifyTarget is installed only by server wiring for a controlled fixture + // workflow. Leaving it nil fails closed for target-aware definitions. + verifyTarget stage.VerificationRunner + wg sync.WaitGroup + schedulerWG sync.WaitGroup // attentionMu guards attentionSent, the per-run set of run-attention events // already delivered — by a stage's own notify policy or by the run-level // default — so neither republishes the same genuine event. @@ -1079,7 +1082,22 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na } } operations := &stage.OperationLog{} + var landing stage.LandingAccess + delivery, deliveryErr := stage.DeliveryEnabled(def) + if deliveryErr != nil { + finish("failed", "error", deliveryErr.Error()) + return + } + if delivery { + access, accessErr := landingAccess(workspace) + if accessErr != nil { + finish("failed", "error", "delivery access denied: "+accessErr.Error()) + return + } + landing = access + } client := r.agentClient(id, sequence, state.artifactDirs, workspace, artifactsDir, operations, request.WritePaths, tools) + client.LandingAccess = landing client.OnSession = func(session string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -1222,6 +1240,45 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na step.Output["salvage"] = result.Output } } + if err == nil && result.Outcome == "success" { + target, targetErr := stage.VerificationTargetFor(def) + if targetErr != nil { + err = targetErr + } else if target != nil { + if r.verifyTarget == nil { + err = fmt.Errorf("verification target %q requires a server-controlled fixture runner", target.Fixture) + } else { + evidence, verificationErr := r.verifyTarget(stageCtx, *target) + if verificationErr != nil { + err = verificationErr + } else { + err = stage.VerifyTargetEvidence(*target, evidence) + } + } + } + if err != nil { + step.Status = "failed" + step.Diagnostic = err.Error() + step.Outcome = "error" + step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input} + } + } + if err == nil && delivery && result.Outcome == "success" { + checks, checksErr := stage.DeliveryChecks(def) + if checksErr != nil { + err = checksErr + } else { + gateCtx, gateCancel := context.WithTimeout(stageCtx, modelConfig.Timeout) + err = verifyDelivery(gateCtx, workspace, landing, checks) + gateCancel() + } + if err != nil { + step.Status = "failed" + step.Diagnostic = err.Error() + step.Outcome = "error" + step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input} + } + } // A failed or timed-out step carries the session pointer even when the // bounded transcript would otherwise be only its stage_limits envelope. if err != nil && stageSession != "" { diff --git a/internal/server/verification_fixture.go b/internal/server/verification_fixture.go new file mode 100644 index 00000000..767723ab --- /dev/null +++ b/internal/server/verification_fixture.go @@ -0,0 +1,135 @@ +package server + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" + + "git.a-wan.net/alan/pipelines/internal/stage" +) + +const ( + verificationFixtureID = "reconnect-foreground" + verificationFixtureResponseLimit int64 = 64 << 10 + verificationFixtureTimeout = 10 * time.Second +) + +// VerificationFixture is a server-owned, explicitly registered probe. Its +// endpoint and expected provenance come from server configuration, never a +// stage definition or result. +type VerificationFixture struct { + ID, Endpoint, Backend, Revision, Capability, State string +} + +// VerificationFixturesFromEnvironment returns the single supported fixture +// registration when its endpoint is explicitly configured. There is no default +// endpoint: an unconfigured server fails target-aware verification closed. +func VerificationFixturesFromEnvironment(getenv func(string) string) []VerificationFixture { + if getenv == nil { + getenv = os.Getenv + } + endpoint := strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_URL")) + if endpoint == "" { + return nil + } + return []VerificationFixture{{ + ID: verificationFixtureID, + Endpoint: endpoint, + Backend: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_BACKEND")), + Revision: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_REVISION")), + Capability: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_CAPABILITY")), + State: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_STATE")), + }} +} + +type verificationFixtureRegistry struct { + fixtures map[string]VerificationFixture + err error +} + +func newVerificationFixtureRegistry(fixtures []VerificationFixture) *verificationFixtureRegistry { + r := &verificationFixtureRegistry{fixtures: make(map[string]VerificationFixture, len(fixtures))} + for _, fixture := range fixtures { + if err := validVerificationFixture(fixture); err != nil { + r.err = err + return r + } + if _, exists := r.fixtures[fixture.ID]; exists { + r.err = fmt.Errorf("duplicate verification fixture %q", fixture.ID) + return r + } + r.fixtures[fixture.ID] = fixture + } + return r +} + +func validVerificationFixture(fixture VerificationFixture) error { + if fixture.ID != verificationFixtureID { + return fmt.Errorf("unsupported verification fixture %q", fixture.ID) + } + for _, value := range []string{fixture.Endpoint, fixture.Backend, fixture.Revision, fixture.Capability, fixture.State} { + if value == "" || len(value) > 256 { + return fmt.Errorf("verification fixture %q has incomplete configuration", fixture.ID) + } + } + u, err := url.Parse(fixture.Endpoint) + if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") { + return fmt.Errorf("verification fixture %q has invalid endpoint", fixture.ID) + } + return nil +} + +func (r *verificationFixtureRegistry) verify(ctx context.Context, target stage.VerificationTarget) (stage.VerificationEvidence, error) { + if r.err != nil { + return stage.VerificationEvidence{}, r.err + } + fixture, exists := r.fixtures[target.Fixture] + if !exists { + return stage.VerificationEvidence{}, fmt.Errorf("unknown verification fixture %q", target.Fixture) + } + if fixture.Backend != target.Backend || fixture.Revision != target.Revision || fixture.Capability != target.Capability { + return stage.VerificationEvidence{}, fmt.Errorf("verification target does not match registered fixture %q", target.Fixture) + } + probeCtx, cancel := context.WithTimeout(ctx, verificationFixtureTimeout) + defer cancel() + req, err := http.NewRequestWithContext(probeCtx, http.MethodGet, fixture.Endpoint, nil) + if err != nil { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q request: %w", fixture.ID, err) + } + client := &http.Client{Timeout: verificationFixtureTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + response, err := client.Do(req) + if err != nil { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q probe: %w", fixture.ID, err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q probe returned %s", fixture.ID, response.Status) + } + var observed struct { + Backend string `json:"backend"` + Revision string `json:"revision"` + Capability string `json:"capability"` + Fixture string `json:"fixture"` + State string `json:"state"` + } + body, err := io.ReadAll(io.LimitReader(response.Body, verificationFixtureResponseLimit+1)) + if err != nil { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response: %w", fixture.ID, err) + } + if int64(len(body)) > verificationFixtureResponseLimit { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response exceeds %d bytes", fixture.ID, verificationFixtureResponseLimit) + } + if err := json.Unmarshal(body, &observed); err != nil { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response: %w", fixture.ID, err) + } + if observed.Backend != fixture.Backend || observed.Revision != fixture.Revision || observed.Capability != fixture.Capability || observed.Fixture != fixture.ID || observed.State != fixture.State { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q returned incompatible backend, revision, capability, fixture, or state", fixture.ID) + } + return stage.VerificationEvidence{Backend: observed.Backend, Revision: observed.Revision, Capability: observed.Capability, Fixture: observed.Fixture}, nil +} diff --git a/internal/server/verification_fixture_test.go b/internal/server/verification_fixture_test.go new file mode 100644 index 00000000..3f71ca96 --- /dev/null +++ b/internal/server/verification_fixture_test.go @@ -0,0 +1,148 @@ +package server + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func TestHandlerRunsRegisteredVerificationFixture(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + var mu sync.Mutex + response := map[string]string{"backend": "branch-api", "revision": "abc123", "capability": "controlled-state", "fixture": verificationFixtureID, "state": "ready"} + calls := 0 + fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + calls++ + _ = json.NewEncoder(w).Encode(response) + })) + defer fixture.Close() + + store, err := pipeline.Open(filepath.Join(t.TempDir(), "fixture.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + source := pipeline.Template([]string{"worker"}, true) + source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture="+verificationFixtureID+"\n```\n\n```configs", 1) + if _, err := store.Submit(ctx, "controlled-fixture", source); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + handler := HandlerWithOptions(ctx, Options{VerificationFixtures: []VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", State: "ready"}}}, store).(*runHandler) + defer func() { + cancel() + WaitRuns(handler) + }() + r := handler.runner + if r.verifyTarget == nil { + t.Fatal("normal server construction did not wire controlled fixture verification") + } + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + // These forged model labels must not influence fixture provenance. + return stage.Result{Outcome: "success", Output: stage.Values{"verification": map[string]any{"backend": "old-api", "revision": "stale", "capability": "forged", "fixture": "other"}}}, nil + } + start := func(key string) *protocol.RunDetail { + run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: key, Workspace: t.TempDir(), Input: map[string]any{"prompt": "verify"}}) + if err != nil { + t.Fatal(err) + } + r.wg.Wait() + detail, err := store.GetRun(ctx, run.ID) + if err != nil { + t.Fatal(err) + } + return detail + } + if detail := start("compatible"); detail.Status != "succeeded" { + t.Fatalf("compatible controlled fixture did not settle: %+v", detail) + } + mu.Lock() + response["revision"] = "old-build" + mu.Unlock() + if detail := start("stale"); detail.Status != "failed" || !strings.Contains(detail.Steps[0].Diagnostic, "incompatible") { + t.Fatalf("stale fixture provenance settled as success: %+v", detail) + } + mu.Lock() + if calls != 2 { + t.Fatalf("fixture probes = %d, want one per target-aware stage", calls) + } + mu.Unlock() +} + +func TestVerificationFixtureCancellationClosesProbe(t *testing.T) { + started := make(chan struct{}) + canceled := make(chan struct{}) + fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(started) + <-r.Context().Done() + close(canceled) + })) + defer fixture.Close() + registry := newVerificationFixtureRegistry([]VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", State: "ready"}}) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + _, err := registry.verify(ctx, stage.VerificationTarget{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: verificationFixtureID}) + done <- err + }() + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("fixture probe did not start") + } + cancel() + select { + case err := <-done: + if err == nil { + t.Fatal("canceled fixture probe succeeded") + } + case <-time.After(time.Second): + t.Fatal("canceled fixture probe did not return") + } + select { + case <-canceled: + case <-time.After(time.Second): + t.Fatal("canceled fixture request was not cleaned up") + } +} + +func TestVerificationFixtureRejectsUnknownAndMismatchedProvenance(t *testing.T) { + base := map[string]string{"backend": "branch-api", "revision": "abc123", "capability": "controlled-state", "fixture": verificationFixtureID, "state": "ready"} + for field, wrong := range map[string]string{"backend": "old-api", "revision": "old-build", "capability": "other-state", "state": "stale"} { + t.Run(field, func(t *testing.T) { + response := make(map[string]string, len(base)) + for key, value := range base { + response[key] = value + } + response[field] = wrong + fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(response) + })) + defer fixture.Close() + registry := newVerificationFixtureRegistry([]VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: base["backend"], Revision: base["revision"], Capability: base["capability"], State: base["state"]}}) + _, err := registry.verify(context.Background(), stage.VerificationTarget{Backend: base["backend"], Revision: base["revision"], Capability: base["capability"], Fixture: verificationFixtureID}) + if err == nil || !strings.Contains(err.Error(), "incompatible") { + t.Fatalf("mismatched %s provenance was accepted: %v", field, err) + } + }) + } + registry := newVerificationFixtureRegistry(nil) + if _, err := registry.verify(context.Background(), stage.VerificationTarget{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "unknown"}); err == nil || !strings.Contains(err.Error(), "unknown") { + t.Fatalf("unknown fixture was accepted: %v", err) + } +} diff --git a/internal/server/websocket.go b/internal/server/websocket.go index adb3184b..5e598223 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -49,6 +49,10 @@ type Options struct { // read time with no model call, which keeps tests and unconfigured servers // deterministic. StepVerdicts func(context.Context, map[string]any) string + // VerificationFixtures are explicit server-owned HTTP fixtures for optional + // target-aware verification. Definitions can name only a registered ID; they + // cannot select an endpoint or supply provenance. + VerificationFixtures []VerificationFixture } // Handler serves the persistent connection with default options. Canceling ctx @@ -81,6 +85,10 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin if runErr == nil && options.StepVerdicts != nil { runs.verdict = options.StepVerdicts } + if runErr == nil { + fixtures := newVerificationFixtureRegistry(options.VerificationFixtures) + runs.verifyTarget = fixtures.verify + } if runErr == nil { runs.schedulerWG.Add(1) go runs.scheduleLoop() diff --git a/internal/stage/agent.go b/internal/stage/agent.go index 57b23b12..dd91d1f3 100644 --- a/internal/stage/agent.go +++ b/internal/stage/agent.go @@ -19,6 +19,7 @@ type AgentClient struct { // ArtifactAccess is supplied only by the runner. Write is this visit's // directory; Read contains this and completed earlier visits in the same run. ArtifactAccess ArtifactAccess + LandingAccess LandingAccess OnSession func(string, bool) error SessionID string Transcript string diff --git a/internal/stage/model.go b/internal/stage/model.go index 13e3d1ff..18465c67 100644 --- a/internal/stage/model.go +++ b/internal/stage/model.go @@ -59,7 +59,7 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) { } for block := range def.Blocks { switch block { - case "output", "prompt", "edges", "configs", "tools", "permissions": + case "output", "prompt", "edges", "configs", "tools", "permissions", "checks", "verification": default: return ModelSpec{}, fmt.Errorf("unexpected block %q for model stage", block) } @@ -70,6 +70,9 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) { if _, err := StagePermissions(def); err != nil { return ModelSpec{}, err } + if _, err := VerificationTargetFor(def); err != nil { + return ModelSpec{}, err + } for _, key := range []string{"provider", "model"} { if def.Configs[key] == "" { return ModelSpec{}, fmt.Errorf("missing config %s", key) @@ -77,7 +80,7 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) { } for key := range def.Configs { switch key { - case "provider", "model", "retries", "timeout", "budget_usd", "budget_tokens", "harness": + case "provider", "model", "retries", "timeout", "budget_usd", "budget_tokens", "harness", "delivery": default: return ModelSpec{}, fmt.Errorf("unknown config %q", key) } @@ -92,6 +95,15 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) { if backend != "opencode" && backend != "pi" { return ModelSpec{}, fmt.Errorf("harness must be opencode or pi") } + if delivery := def.Configs["delivery"]; delivery != "" && delivery != "true" { + return ModelSpec{}, fmt.Errorf("delivery must be true when set") + } + if def.Configs["delivery"] == "true" && backend != "opencode" { + return ModelSpec{}, fmt.Errorf("delivery is supported only by the opencode harness") + } + if _, present := def.Blocks["checks"]; present && def.Configs["delivery"] != "true" { + return ModelSpec{}, fmt.Errorf("checks block is supported only by delivery stages") + } if backend == "pi" { tools, _, _ := ToolNames(def) if _, err := piTools(tools); err != nil { diff --git a/internal/stage/tools.go b/internal/stage/tools.go index 77477987..2f567e3b 100644 --- a/internal/stage/tools.go +++ b/internal/stage/tools.go @@ -14,6 +14,77 @@ type ArtifactAccess struct { Read []string } +// LandingAccess is the server-derived external capability for a verified +// delivery operation. Empty or malformed values grant nothing. +type LandingAccess struct { + MainWorktree string + GitMetadata string + LockPath string +} + +func (a LandingAccess) valid() bool { + for _, path := range []string{a.MainWorktree, a.GitMetadata, a.LockPath} { + if path == "" || !filepath.IsAbs(path) || filepath.Clean(path) != path || strings.ContainsAny(path, "*?\\") { + return false + } + } + // The server grants the canonical repository metadata directory as OpenCode + // V2's smallest external-directory unit. Keep the lock inside that directory + // so a caller cannot turn a valid metadata capability into access elsewhere. + return a.GitMetadata == filepath.Join(a.MainWorktree, ".git") && + a.LockPath == filepath.Join(a.GitMetadata, "pipelines-landing.lock") +} + +// DeliveryEnabled is the narrow, declarative capability check used before the +// runner supplies any landing paths. Only a validated model definition can ask +// for it; runtime paths are still derived by the server. +func DeliveryEnabled(def Definition) (bool, error) { + delivery := def.Configs["delivery"] + if delivery == "" { + return false, nil + } + if delivery != "true" { + return false, fmt.Errorf("delivery must be true when set") + } + if EffectiveType(def.Type) != ModelType { + return false, fmt.Errorf("delivery is supported only by model stages") + } + if _, err := (ModelStage{}).Spec(def); err != nil { + return false, fmt.Errorf("invalid delivery model stage: %w", err) + } + if _, err := DeliveryChecks(def); err != nil { + return false, err + } + return true, nil +} + +// DeliveryChecks returns the bounded, definition-declared checks that the +// runner executes against a committed delivery candidate before canonical main +// is changed. They +// are intentionally a block rather than a generic runner setting: each +// workflow owns its repository-specific verification commands. +func DeliveryChecks(def Definition) ([]string, error) { + text := strings.TrimSpace(def.Blocks["checks"]) + if text == "" { + return nil, fmt.Errorf("delivery stages require a nonempty checks block") + } + var checks []string + for _, line := range strings.Split(text, "\n") { + check := strings.TrimSpace(line) + if check == "" { + continue + } + if len(check) > 4096 { + return nil, fmt.Errorf("delivery check exceeds 4096 bytes") + } + checks = append(checks, check) + } + if len(checks) == 0 || len(checks) > 20 { + return nil, fmt.Errorf("delivery stages require 1 to 20 checks") + } + return checks, nil +} + // ToolNames validates model-stage permissions, not general graph semantics. // A present empty block explicitly denies all tools. func ToolNames(def Definition) ([]string, bool, error) { @@ -155,6 +226,10 @@ func (c *AgentClient) permissionRules() []map[string]string { allow("external_directory", filepath.ToSlash(filepath.Join(c.ArtifactAccess.Write, "*"))) allow("edit", filepath.ToSlash(filepath.Join(c.ArtifactAccess.Write, "*"))) } + if c.LandingAccess.valid() { + allow("external_directory", filepath.ToSlash(filepath.Join(c.LandingAccess.MainWorktree, "*"))) + allow("external_directory", filepath.ToSlash(filepath.Join(c.LandingAccess.GitMetadata, "*"))) + } for _, pattern := range []string{"../*", "*/../*"} { rules = append(rules, map[string]string{"action": "external_directory", "resource": pattern, "effect": "deny"}) } diff --git a/internal/stage/tools_test.go b/internal/stage/tools_test.go index ed823bc1..e7a20c0f 100644 --- a/internal/stage/tools_test.go +++ b/internal/stage/tools_test.go @@ -174,3 +174,107 @@ func TestArtifactPermissionsAreExactAndToolScoped(t *testing.T) { t.Fatal("missing artifact scope enabled an external directory") } } + +func TestLandingPermissionsAreExactAndServerDerived(t *testing.T) { + access := LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/pipelines/.git", LockPath: "/repos/pipelines/.git/pipelines-landing.lock"} + client := &AgentClient{Tools: []string{"read", "shell"}, LandingAccess: access} + rules := client.permissionRules() + for _, path := range []string{"/repos/pipelines/go.mod", "/repos/pipelines/.git/worktrees/item/HEAD", "/repos/pipelines/.git/pipelines-landing.lock"} { + if permissionEffect(rules, "external_directory", path) != "allow" { + t.Fatalf("verified landing path denied: %q", path) + } + } + for _, path := range []string{"/repos/other/go.mod", "/repos/pipelines/../other/go.mod", "/Users/alan/.git/config", "/.git/config"} { + if permissionEffect(rules, "external_directory", path) != "deny" { + t.Fatalf("unverified landing path allowed: %q", path) + } + } + client.LandingAccess.MainWorktree = "/repos/pipelines/../other" + if permissionEffect(client.permissionRules(), "external_directory", "/repos/other/go.mod") != "deny" { + t.Fatal("non-canonical landing capability allowed") + } + client.LandingAccess = LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/pipelines/.git", LockPath: "/tmp/pipelines-landing.lock"} + if permissionEffect(client.permissionRules(), "external_directory", "/tmp/pipelines-landing.lock") != "deny" { + t.Fatal("landing lock outside verified metadata allowed") + } + client.LandingAccess = LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/other/.git", LockPath: "/repos/other/.git/pipelines-landing.lock"} + if permissionEffect(client.permissionRules(), "external_directory", "/repos/other/.git/config") != "deny" { + t.Fatal("unrelated Git metadata allowed") + } +} + +func TestDeliveryEnabledRequiresValidatedModelStage(t *testing.T) { + def := modelDefinition() + def.Configs["delivery"] = "true" + def.Blocks["checks"] = "go test ./..." + if enabled, err := DeliveryEnabled(def); err != nil || !enabled { + t.Fatalf("valid delivery stage = %v, %v", enabled, err) + } + for name, change := range map[string]func(*Definition){ + "malformed value": func(d *Definition) { d.Configs["delivery"] = "yes" }, + "non-model": func(d *Definition) { d.Type = HumanType }, + "invalid model": func(d *Definition) { d.Configs["unexpected"] = "value" }, + } { + t.Run(name, func(t *testing.T) { + copy := def + copy.Configs = map[string]string{} + for key, value := range def.Configs { + copy.Configs[key] = value + } + change(©) + if enabled, err := DeliveryEnabled(copy); err == nil || enabled { + t.Fatalf("invalid delivery stage = %v, %v", enabled, err) + } + }) + } +} + +func TestDeliveryRejectsPiBeforeAgentExecution(t *testing.T) { + def := modelDefinition() + def.Configs["delivery"] = "true" + def.Configs["harness"] = "pi" + def.Blocks["checks"] = "true" + if err := (ModelStage{}).Validate(def); err == nil || !strings.Contains(err.Error(), "opencode harness") { + t.Fatal("Pi delivery was accepted", err) + } + delete(def.Configs, "delivery") + delete(def.Blocks, "checks") + if err := (ModelStage{}).Validate(def); err != nil { + t.Fatal("Pi non-delivery was changed", err) + } +} + +func TestDeliveryChecksRequireBoundedDeliveryOnlyDeclaration(t *testing.T) { + def := modelDefinition() + def.Configs["delivery"] = "true" + for name, mutate := range map[string]func(*Definition){ + "missing": func(d *Definition) {}, + "empty": func(d *Definition) { d.Blocks["checks"] = "\n" }, + "non delivery": func(d *Definition) { d.Blocks["checks"] = "go test ./..." }, + "too many checks": func(d *Definition) { d.Blocks["checks"] = strings.Repeat("true\n", 21) }, + "oversized check": func(d *Definition) { d.Blocks["checks"] = strings.Repeat("x", 4097) }, + } { + t.Run(name, func(t *testing.T) { + copy := def + copy.Blocks = map[string]string{} + copy.Configs = map[string]string{} + for key, value := range def.Blocks { + copy.Blocks[key] = value + } + for key, value := range def.Configs { + copy.Configs[key] = value + } + mutate(©) + if name == "non delivery" { + delete(copy.Configs, "delivery") + if _, err := (ModelStage{}).Spec(copy); err == nil { + t.Fatal("non-delivery checks block was accepted") + } + return + } + if _, err := DeliveryEnabled(copy); err == nil { + t.Fatal("invalid delivery checks were accepted") + } + }) + } +} diff --git a/internal/stage/verification.go b/internal/stage/verification.go new file mode 100644 index 00000000..a186aa15 --- /dev/null +++ b/internal/stage/verification.go @@ -0,0 +1,97 @@ +package stage + +import ( + "context" + "fmt" + "strings" +) + +// VerificationTarget declares the controlled fixture that can substantiate a +// target-specific behavior claim. It is intentionally data-only: the runner +// does not execute fixture input supplied by a stage result. +type VerificationTarget struct { + Backend string + Revision string + Capability string + Fixture string +} + +// VerificationEvidence is returned by a server-controlled fixture workflow, +// never parsed from a model's result. +type VerificationEvidence struct { + Backend string + Revision string + Capability string + Fixture string +} + +// VerificationRunner is the narrow server seam for a controlled fixture or +// workflow. The runner selects it during server setup; model output cannot +// select a backend, revision, fixture, or executable. +type VerificationRunner func(context.Context, VerificationTarget) (VerificationEvidence, error) + +// VerificationTargetFor parses an optional, bounded verification block. A +// declared target requires all provenance fields so an old or other-backend +// fixture cannot be presented as evidence for the current target. +func VerificationTargetFor(def Definition) (*VerificationTarget, error) { + text, declared := def.Blocks["verification"] + if !declared { + return nil, nil + } + if len(text) > 4096 { + return nil, fmt.Errorf("verification block exceeds 4096 bytes") + } + values := map[string]string{} + for _, line := range strings.Split(text, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + key, value, ok := strings.Cut(line, "=") + key, value = strings.TrimSpace(key), strings.TrimSpace(value) + if !ok || value == "" || len(value) > 256 { + return nil, fmt.Errorf("invalid verification entry %q", line) + } + switch key { + case "backend", "revision", "capability", "fixture": + default: + return nil, fmt.Errorf("unknown verification entry %q", key) + } + if _, exists := values[key]; exists { + return nil, fmt.Errorf("duplicate verification entry %q", key) + } + values[key] = value + } + target := &VerificationTarget{Backend: values["backend"], Revision: values["revision"], Capability: values["capability"], Fixture: values["fixture"]} + if target.Backend == "" || target.Revision == "" || target.Capability == "" || target.Fixture == "" { + return nil, fmt.Errorf("verification requires backend, revision, capability, and fixture") + } + return target, nil +} + +// VerifyTargetEvidence compares server-collected controlled-fixture provenance +// with the workflow declaration. It deliberately accepts no model result: a +// stage cannot substantiate its own target-specific behavior claim. +func VerifyTargetEvidence(target VerificationTarget, evidence VerificationEvidence) error { + for key, want := range map[string]string{ + "backend": target.Backend, "revision": target.Revision, "capability": target.Capability, "fixture": target.Fixture, + } { + got := map[string]string{ + "backend": evidence.Backend, "revision": evidence.Revision, "capability": evidence.Capability, "fixture": evidence.Fixture, + }[key] + if got != want { + return fmt.Errorf("verification evidence %s does not match declared target", key) + } + } + return nil +} + +// VerifyTargetResult is retained for callers that only need declaration +// validation. Result-provided provenance is intentionally not evidence. +func VerifyTargetResult(def Definition, _ Values) error { + target, err := VerificationTargetFor(def) + if err != nil || target == nil { + return err + } + return fmt.Errorf("verification target %q requires a server-controlled fixture runner", target.Fixture) +} diff --git a/internal/stage/verification_test.go b/internal/stage/verification_test.go new file mode 100644 index 00000000..d52d5b5a --- /dev/null +++ b/internal/stage/verification_test.go @@ -0,0 +1,27 @@ +package stage + +import "testing" + +func TestTargetVerificationRejectsWrongBackendEvidence(t *testing.T) { + def := modelDefinition() + def.Blocks["verification"] = "backend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground" + if err := (ModelStage{}).Validate(def); err != nil { + t.Fatal(err) + } + target, err := VerificationTargetFor(def) + if err != nil { + t.Fatal(err) + } + good := VerificationEvidence{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"} + if err := VerifyTargetEvidence(*target, good); err != nil { + t.Fatalf("matching fixture evidence rejected: %v", err) + } + wrongBackend := VerificationEvidence{Backend: "old-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"} + if err := VerifyTargetEvidence(*target, wrongBackend); err == nil { + t.Fatal("wrong-backend fixture evidence accepted") + } + def.Blocks["verification"] = "backend=branch-api\nrevision=abc123" + if err := (ModelStage{}).Validate(def); err == nil { + t.Fatal("incomplete target verification declaration accepted") + } +}