diff --git a/cmd/pipelines-server/main.go b/cmd/pipelines-server/main.go index ec789886..54415ddf 100644 --- a/cmd/pipelines-server/main.go +++ b/cmd/pipelines-server/main.go @@ -47,7 +47,7 @@ func main() { // Each new run gets a bounded one-line title from the configured default // harness; the summariser falls back to the prompt's first line, so title // generation never blocks or fails run creation. - options := server.Options{ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason} + options := server.Options{ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason, VerificationFixtures: server.VerificationFixturesFromEnvironment(os.Getenv)} recorded := "" if value, err := store.OpenCodeEndpoint(ctx); err != nil { log.Printf("OpenCode endpoint setting unreadable: %v", err) diff --git a/docs/notes/background-shell-visibility.md b/docs/notes/background-shell-visibility.md new file mode 100644 index 00000000..583e5f90 --- /dev/null +++ b/docs/notes/background-shell-visibility.md @@ -0,0 +1,37 @@ +# Background shell visibility + +The chat server now opens the broker stream for read-only subscribers and reconciles verified running shell operations from OpenCode's authoritative `/api/shell` readback. The packaged OpenCode skill references and chat documentation should describe that shell visibility survives reconnects only when OpenCode reports explicit `metadata.sessionID` ownership; shared working directories are not ownership evidence. + +## Current source behavior + +Observers share one initial readback per room/stream attachment. The adapter and hub +fold bounded lifecycle journals over each snapshot, so a stale read cannot resurrect +an exited shell or erase a concurrent create. Shell replay and live delivery enqueue +in fold order. Cancellation rejects old-room callbacks and releases local identity +state before reconnect. Journals retain at most 256 identities and 1 MiB of id/command +text; capacity/read failures surface stable limited-visibility text, never broker +response bodies. No shell polling was added. OpenCode's terminal `timeout` and +`killed` statuses are excluded from retained running operations as well as `exited`. + +## Verification evidence — 2026-10-18 + +`opencode api get /openapi.json` authenticated through supported service discovery. +The live schema advertises shell list/detail/output and generic `metadata`; it does +not type `metadata.sessionID`. A throwaway session +`ses_eb12ab7b3ffe0ZAle7PHpftHMG` and its own `sleep 1` shell +`sh_14ed57134001e6fuYqvn62pKit` verified actual create/detail readback preserving +explicit `metadata.sessionID`, status `running`, subsequent `exited`/exit 0, and +absence from the running list. This verifies explicit API metadata, not automatic +agent-tool ownership injection. The CLI event attempt yielded no captured events; +live-provider event ordering is **not tested**. Forced ordering evidence comes from +channel/barrier regressions, including a real WebSocket handler on a loopback scratch +server with fresh SQLite, not from the deployed service. + +## Documentation follow-up / deployment + +`docs/chat-verification.md` and the chat subscribe/replay section of `docs/cli.md` +are stale: docs-fold should describe authoritative ownership/readback, coalesced +observation, bounded reconciliation and limited-visibility errors, distinguishing +fixture ordering evidence from the live readback above. Shared docs and packaged +skill references were not edited here. No deployment, installation, service restart +or physical-phone/UI verification was performed; this changes no rendered layout. diff --git a/docs/notes/delivery-stabilization.md b/docs/notes/delivery-stabilization.md new file mode 100644 index 00000000..bba21fd9 --- /dev/null +++ b/docs/notes/delivery-stabilization.md @@ -0,0 +1,47 @@ +# Delivery stabilization + +Date: 2026-10-18. + +The server now has draft-only `delivery=true` support for OpenCode model stages. +At run time it derives the canonical `main` worktree and linked-worktree Git +metadata from Git rather than accepting paths from a stage prompt. It verifies +the committed item tree in a disposable candidate worktree before fast-forwarding +canonical `main`; declared checks are bounded configuration, not proof that any +deployed definition has used them. It is denied for Pi (until equivalent +capability enforcement exists), foreign repositories, missing main worktrees, +and malformed capabilities. This item does not publish or deploy a definition. + +The source also supports an optional controlled-fixture verification declaration +(`backend`, `revision`, `capability`, and `fixture`). Ordinary server setup wires +an explicit, bounded `reconnect-foreground` HTTP fixture registry when +`PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_URL` and its expected `BACKEND`, +`REVISION`, `CAPABILITY`, and `STATE` values are configured. There is no default +remote URL. The server, not the model, probes that registered endpoint and +collects its returned backend identity, build revision, capability, fixture ID, +and state before comparing provenance. Unknown fixtures, incomplete setup, +wrong backend/revision/capability/state, non-200 responses, cancellation, and +oversized or malformed responses fail closed. A model result's labels are not +evidence. This is controlled local/backend fixture infrastructure, not evidence +of a rendered layout or a live Android device. +Definitions must be scratch-validated against the deployed server only after +the server support is deployed; no future delivery or verification definition +is activated by this source change. + +`internal/author/skill/references/format.md` and +`internal/author/skill/references/queue.md` are stale: a later docs fold should +describe `delivery=true`, OpenCode-only availability, candidate verification, +and the required activation order (deploy server support before scratch-validating +and then publishing a definition). +OpenCode V2 resolves external resources at a containing directory, so delivery +uses the verified repository metadata directory rather than an impossible +single-lock-file scope. It does not grant a parent, home, foreign repository, +or global policy. Unrestricted shell permissions are not an OS sandbox. + +Activation remains draft-only: deploy compatible server support, configure the +controlled fixture endpoint and matching provenance, scratch-validate the +fixture, then publish a definition. No definition is activated by this source +change. + +Recovery checklist: retain `connection-reconnect` and unfinished feature +branches; merge only complete committed trees that pass mapped checks; do not +infer delivery from a dirty worktree or an old deployed-backend capture. diff --git a/internal/opencode/chat.go b/internal/opencode/chat.go index 43ac1bf1..cec4484b 100644 --- a/internal/opencode/chat.go +++ b/internal/opencode/chat.go @@ -30,7 +30,9 @@ var ErrPermissionUnsupported = errors.New("OpenCode permission approval is not i // call guards. History alone streams/project tool details before decoding; it // does not introduce a total-history cap or a second credential path. type Chat struct { - conn *Connection + conn *Connection + shellMu sync.Mutex + shells map[string]*ChatStreamState // mu serializes the adapter's HTTP calls. Connection records its last // operation and status on the shared connection, so two calls must not // overlap; the permission poller runs alongside the read loop's @@ -332,6 +334,114 @@ func decodeChatPermissions(sessionID string, data []byte) ([]protocol.ChatPermis return frames, nil } +// Shells reads the endpoint's authoritative shell list and returns only running +// operations whose metadata explicitly names sessionID. The shell API is global; +// filtering locally avoids guessing an unsupported query filter and never assigns +// ownership from a shared working directory. A successful read also seeds the +// stream state used to attribute later shell.exited events, which carry no session. +func (c *Chat) Shells(ctx context.Context, sessionID string) ([]protocol.ChatShell, error) { + if !ValidSessionID(sessionID) { + return nil, fmt.Errorf("invalid OpenCode session id") + } + state, err := c.streamStateContext(ctx, sessionID) + if err != nil { + return nil, err + } + // One read per state; waiting and I/O never hold the event-state mutex. + state.mu.Lock() + if state.readGate == nil { + state.readGate = make(chan struct{}, 1) + } + gate := state.readGate + state.mu.Unlock() + select { + case gate <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + defer func() { <-gate }() + state.mu.Lock() + state.readback = &ShellSnapshot{} + state.mu.Unlock() + defer func() { state.mu.Lock(); state.readback = nil; state.mu.Unlock() }() + data, err := c.call(ctx, "get", "/api/shell", nil) + if err != nil { + return nil, err + } + var response struct { + Data []chatShellInfo `json:"data"` + } + if err := json.Unmarshal(data, &response); err != nil || response.Data == nil { + if err := json.Unmarshal(data, &response.Data); err != nil { + return nil, fmt.Errorf("invalid OpenCode shell response") + } + } + out := make([]protocol.ChatShell, 0, len(response.Data)) + for _, info := range response.Data { + if info.ID == "" || info.shellSessionID() != sessionID || (info.Status != "" && info.Status != protocol.ChatShellRunning) { + continue + } + status := info.Status + if status == "" { + status = protocol.ChatShellRunning + } + out = append(out, protocol.ChatShell{ID: info.ID, Status: status, Command: info.Command}) + } + state.mu.Lock() + defer state.mu.Unlock() + out, err = state.readback.Merge(out) + if err != nil { + return nil, err + } + if err := ctx.Err(); err != nil { + return nil, err + } + ids := make([]string, 0, len(out)) + for _, shell := range out { + ids = append(ids, shell.ID) + } + state.replaceShellsLocked(ids) + return out, nil +} + +// streamState returns the session's retained, verified shell identities. It is +// deliberately kept across stream replacement so an exit after reconnect remains +// attributable; Shells reconciles its contents from authoritative readback on +// each attach rather than replacing the pointer a live scanner is using. +func (c *Chat) streamState(sessionID string) *ChatStreamState { + state, _ := c.streamStateContext(context.Background(), sessionID) + return state +} + +func (c *Chat) streamStateContext(ctx context.Context, sessionID string) (*ChatStreamState, error) { + c.shellMu.Lock() + defer c.shellMu.Unlock() + if err := ctx.Err(); err != nil { + return nil, err + } + if c.shells == nil { + c.shells = map[string]*ChatStreamState{} + } + state := c.shells[sessionID] + if state == nil { + state = &ChatStreamState{} + c.shells[sessionID] = state + } + return state, nil +} + +// ReleaseShells discards verified shell identities after the last room observer +// disconnects. A future attach calls Shells for a fresh authoritative snapshot, +// so this bounds state without attributing a foreign shell during the gap. +func (c *Chat) ReleaseShells(sessionID string) { + if sessionID == "" { + return + } + c.shellMu.Lock() + delete(c.shells, sessionID) + c.shellMu.Unlock() +} + // Stream subscribes to the OpenCode event firehose and invokes emit for every // frame belonging to sessionID. It reads one bounded SSE frame at a time and // never polls the per-session message log; the global firehose is the only @@ -344,6 +454,12 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr if !sessionID.MatchString(id) { return fmt.Errorf("invalid OpenCode session id") } + // Pin state before opening/announcing the stream so a canceled scanner + // cannot acquire a replacement room's identity map. + state, err := c.streamStateContext(ctx, id) + if err != nil { + return err + } body, err := c.conn.EventStream(ctx) if err != nil { return err @@ -352,7 +468,7 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr if ready != nil { ready() } - return scanChatEvents(ctx, body, id, emit) + return c.scanChatEventsWithState(ctx, body, id, state, emit) } // scanChatEvents turns an SSE body into protocol frames, one bounded frame at a @@ -360,10 +476,13 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr // malformed JSON fails the stream instead of producing a malformed frame. The // last assistant message id is retained so a later tool/step event that omits it // still attributes every frame to a stable message. -func scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit func(protocol.ChatEventFrame) error) error { +func (c *Chat) scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit func(protocol.ChatEventFrame) error) error { + return c.scanChatEventsWithState(ctx, body, sessionID, c.streamState(sessionID), emit) +} + +func (c *Chat) scanChatEventsWithState(ctx context.Context, body io.Reader, sessionID string, state *ChatStreamState, emit func(protocol.ChatEventFrame) error) error { scanner := bufio.NewScanner(body) scanner.Buffer(make([]byte, 0, 64<<10), MaxResponse) - state := &ChatStreamState{} for scanner.Scan() { if err := ctx.Err(); err != nil { return err @@ -384,7 +503,7 @@ func scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit continue } if frame.MessageID != "" { - state.LastMessageID = frame.MessageID + state.setLastMessageID(frame.MessageID) } if err := emit(frame); err != nil { return err @@ -487,8 +606,29 @@ func (i *chatShellInfo) shellSessionID() string { // what attributes the exit to its session: another session's shell is an // unknown id and is ignored. type ChatStreamState struct { + mu sync.Mutex LastMessageID string Shells map[string]struct{} + readGate chan struct{} + readback *ShellSnapshot +} + +func (s *ChatStreamState) lastMessageID() string { + if s == nil { + return "" + } + s.mu.Lock() + defer s.mu.Unlock() + return s.LastMessageID +} + +func (s *ChatStreamState) setLastMessageID(id string) { + if s == nil || id == "" { + return + } + s.mu.Lock() + s.LastMessageID = id + s.mu.Unlock() } // knowsShell reports whether id is a shell this stream has already surfaced for @@ -497,6 +637,8 @@ func (s *ChatStreamState) knowsShell(id string) bool { if s == nil || id == "" { return false } + s.mu.Lock() + defer s.mu.Unlock() _, ok := s.Shells[id] return ok } @@ -506,6 +648,8 @@ func (s *ChatStreamState) markShell(id string) { if s == nil || id == "" { return } + s.mu.Lock() + defer s.mu.Unlock() if s.Shells == nil { s.Shells = map[string]struct{}{} } @@ -514,12 +658,37 @@ func (s *ChatStreamState) markShell(id string) { // forgetShell drops id once its exit has been surfaced. func (s *ChatStreamState) forgetShell(id string) { - if s == nil || s.Shells == nil { + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if s.Shells == nil { return } delete(s.Shells, id) } +// replaceShells applies one authoritative running-shell snapshot without +// replacing the state object an already attached event scanner owns. +func (s *ChatStreamState) replaceShells(ids []string) { + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.replaceShellsLocked(ids) +} + +func (s *ChatStreamState) replaceShellsLocked(ids []string) { + s.Shells = make(map[string]struct{}, len(ids)) + for _, id := range ids { + if id != "" { + s.Shells[id] = struct{}{} + } + } +} + // NormalizeChatEvent maps one OpenCode firehose `data:` payload to a // protocol.ChatEventFrame, folding the small per-session context ([stream]) it // needs across frames. ok is false for an event type this adapter does not @@ -566,7 +735,7 @@ func NormalizeChatEvent(payload []byte, sessionID string, stream *ChatStreamStat messageID = data.MessageIDAlt } if messageID == "" && stream != nil { - messageID = stream.LastMessageID + messageID = stream.lastMessageID() } // Every pushed frame must carry a stable message id so the client can // attribute it; without one this is not a chat frame. @@ -628,20 +797,38 @@ func normalizeShellEvent(eventType string, data chatEvent, sessionID string, str if status == "" { status = protocol.ChatShellRunning } - stream.markShell(data.Info.ID) + shell := &protocol.ChatShell{ID: data.Info.ID, Status: status, Command: data.Info.Command} + if stream != nil { + stream.mu.Lock() + stream.readback.Record(shell) + if stream.Shells == nil { + stream.Shells = map[string]struct{}{} + } + if status == protocol.ChatShellRunning && len(stream.Shells) < chatShellIdentityLimit { + stream.Shells[shell.ID] = struct{}{} + } + stream.mu.Unlock() + } return protocol.ChatEventFrame{ SessionID: sessionID, Kind: protocol.ChatEventShell, - Shell: &protocol.ChatShell{ID: data.Info.ID, Status: status, Command: data.Info.Command}, + Shell: shell, }, true, nil } if data.ID == "" { return protocol.ChatEventFrame{}, false, fmt.Errorf("malformed OpenCode shell.exited event") } - if !stream.knowsShell(data.ID) { + if stream == nil { + return protocol.ChatEventFrame{}, false, nil + } + stream.mu.Lock() + stream.readback.Record(&protocol.ChatShell{ID: data.ID, Status: protocol.ChatShellExited}) + _, known := stream.Shells[data.ID] + delete(stream.Shells, data.ID) + stream.mu.Unlock() + if !known { return protocol.ChatEventFrame{}, false, nil } - stream.forgetShell(data.ID) status := data.Status if status == "" { status = protocol.ChatShellExited diff --git a/internal/opencode/chat_shell_reconcile_test.go b/internal/opencode/chat_shell_reconcile_test.go new file mode 100644 index 00000000..087080d0 --- /dev/null +++ b/internal/opencode/chat_shell_reconcile_test.go @@ -0,0 +1,116 @@ +package opencode + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +func TestChatShellReadbackBarrier(t *testing.T) { + for _, known := range []bool{false, true} { + t.Run(fmt.Sprint(known), func(t *testing.T) { + read, apply := make(chan struct{}), make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + close(read) + select { + case <-apply: + fmt.Fprint(w, `{"data":[{"id":"sh_old","status":"running","metadata":{"sessionID":"ses_1"}},{"id":"sh_foreign","status":"running","metadata":{"sessionID":"ses_other"}}]}`) + case <-r.Context().Done(): + } + })) + defer srv.Close() + c := NewChat(&Connection{Backend: "v2", URL: srv.URL}) + state := c.streamState("ses_1") + if known { + state.markShell("sh_old") + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + result := make(chan []protocol.ChatShell, 1) + errs := make(chan error, 1) + go func() { shells, err := c.Shells(ctx, "ses_1"); result <- shells; errs <- err }() + select { + case <-read: + case <-ctx.Done(): + t.Fatal("read did not reach barrier") + } + // Both a previously verified exit and an exit first learned while + // reading must suppress the stale running snapshot. + frame, ok, err := NormalizeChatEvent([]byte(`{"type":"shell.exited","data":{"id":"sh_old","status":"exited","exit":0}}`), "ses_1", state) + if err != nil || ok != known || (ok && frame.Shell.Status != protocol.ChatShellExited) { + t.Fatalf("exit = %+v %v %v", frame, ok, err) + } + frame, ok, err = NormalizeChatEvent([]byte(`{"type":"shell.created","data":{"info":{"id":"sh_new","status":"running","command":"sleep 10","metadata":{"sessionID":"ses_1"}}}}`), "ses_1", state) + if err != nil || !ok || frame.Shell.ID != "sh_new" { + t.Fatalf("create = %+v %v %v", frame, ok, err) + } + close(apply) + got := <-result + if err := <-errs; err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].ID != "sh_new" || got[0].Command != "sleep 10" { + t.Fatalf("readback lost events/leaked ownership: %+v", got) + } + if state.knowsShell("sh_old") || state.knowsShell("sh_foreign") || !state.knowsShell("sh_new") { + t.Fatal("adapter identity state regressed") + } + // User-visible subsequent exit remains attributable after reconciliation. + frame, ok, err = NormalizeChatEvent([]byte(`{"type":"shell.exited","data":{"id":"sh_new","status":"exited","exit":0}}`), "ses_1", state) + if err != nil || !ok || frame.Shell.ID != "sh_new" { + t.Fatalf("new shell exit lost: %+v %v %v", frame, ok, err) + } + }) + } +} + +func TestChatShellReadbackCancelWaiter(t *testing.T) { + read := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { close(read); <-r.Context().Done() })) + defer srv.Close() + c := NewChat(&Connection{Backend: "v2", URL: srv.URL}) + ctx, cancel := context.WithCancel(context.Background()) + result := make(chan error, 2) + go func() { _, err := c.Shells(ctx, "ses_1"); result <- err }() + select { + case <-read: + case <-time.After(time.Second): + t.Fatal("read stuck") + } + go func() { _, err := c.Shells(ctx, "ses_1"); result <- err }() + cancel() + for range 2 { + select { + case err := <-result: + if err == nil { + t.Fatal("canceled read succeeded") + } + case <-time.After(time.Second): + t.Fatal("read or waiter stuck") + } + } + old := c.streamState("ses_1") + c.ReleaseShells("ses_1") + if c.streamState("ses_1") == old { + t.Fatal("reconnect reused released state") + } +} + +func TestShellSnapshotBounded(t *testing.T) { + var snapshot ShellSnapshot + for i := 0; i <= chatShellIdentityLimit; i++ { + snapshot.Record(&protocol.ChatShell{ID: fmt.Sprintf("sh_%d", i), Status: protocol.ChatShellExited}) + } + if len(snapshot.changes) != chatShellIdentityLimit { + t.Fatalf("unbounded journal: %d", len(snapshot.changes)) + } + if _, err := snapshot.Merge([]protocol.ChatShell{{ID: "sh_0", Status: protocol.ChatShellRunning}}); err == nil { + t.Fatal("overflow accepted stale snapshot") + } +} diff --git a/internal/opencode/chat_shell_snapshot.go b/internal/opencode/chat_shell_snapshot.go new file mode 100644 index 00000000..8559a741 --- /dev/null +++ b/internal/opencode/chat_shell_snapshot.go @@ -0,0 +1,72 @@ +package opencode + +import ( + "fmt" + "sort" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// ShellSnapshot records changes while an authoritative read is in flight. Its +// owner serializes access. Unknown exits may be recorded, but can only remove an +// id: they never establish ownership. Overflow rejects the snapshot, not events. +type ShellSnapshot struct { + changes map[string]protocol.ChatShell + bytes int + overflow bool +} + +const chatShellIdentityLimit = 256 + +func (s *ShellSnapshot) Record(shell *protocol.ChatShell) { + if s == nil || shell == nil || shell.ID == "" || s.overflow { + return + } + if s.changes == nil { + s.changes = make(map[string]protocol.ChatShell) + } + old := s.changes[shell.ID] + n := s.bytes - len(old.ID) - len(old.Command) + len(shell.ID) + len(shell.Command) + if n > MaxResponse || (old.ID == "" && len(s.changes) >= chatShellIdentityLimit) { + s.overflow = true + return + } + // A terminal event wins even if a duplicate create is delivered later. + if old.ID != "" && old.Status != protocol.ChatShellRunning { + return + } + s.changes[shell.ID] = *shell + s.bytes = n +} + +func (s *ShellSnapshot) Merge(shells []protocol.ChatShell) ([]protocol.ChatShell, error) { + if s.overflow { + return nil, fmt.Errorf("shell reconciliation capacity exceeded") + } + byID := make(map[string]protocol.ChatShell) + for _, shell := range shells { + if shell.ID != "" && shell.Status == protocol.ChatShellRunning { + byID[shell.ID] = shell + } + } + for id, shell := range s.changes { + if shell.Status == protocol.ChatShellRunning { + byID[id] = shell + } else { + delete(byID, id) + } + } + if len(byID) > chatShellIdentityLimit { + return nil, fmt.Errorf("shell reconciliation capacity exceeded") + } + ids := make([]string, 0, len(byID)) + for id := range byID { + ids = append(ids, id) + } + sort.Strings(ids) + out := make([]protocol.ChatShell, 0, len(ids)) + for _, id := range ids { + out = append(out, byID[id]) + } + return out, nil +} diff --git a/internal/opencode/chat_v2_test.go b/internal/opencode/chat_v2_test.go index 00cb2242..25fc7024 100644 --- a/internal/opencode/chat_v2_test.go +++ b/internal/opencode/chat_v2_test.go @@ -395,3 +395,45 @@ func TestOpenChatRejectsUnrunnableCLI(t *testing.T) { t.Fatal("a missing CLI must fail detection, not open an endpoint") } } + +// TestChatShellReadbackSeedsReplacementStream proves an authoritative shell +// readback survives a later stream replacement: shell.exited has no session id, +// so it must still clear the verified shell after reconnect. +func TestChatShellReadbackSeedsReplacementStream(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/shell": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[{"id":"sh_1","status":"running","command":"sleep 30","metadata":{"sessionID":"ses_1"}},{"id":"sh_other","status":"running","command":"sleep 30","metadata":{"sessionID":"ses_other"}}]}`)) + case "/api/event": + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte("data: {\"type\":\"shell.exited\",\"data\":{\"id\":\"sh_1\",\"status\":\"exited\",\"exit\":0}}\n\n")) + } + })) + defer srv.Close() + chat := NewChat(&Connection{Backend: "v2", URL: srv.URL}) + shells, err := chat.Shells(context.Background(), "ses_1") + if err != nil || len(shells) != 1 || shells[0].ID != "sh_1" { + t.Fatalf("Shells = %+v, %v", shells, err) + } + var got []protocol.ChatEventFrame + if err := chat.Stream(context.Background(), "ses_1", nil, func(frame protocol.ChatEventFrame) error { got = append(got, frame); return nil }); err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Shell == nil || got[0].Shell.ID != "sh_1" || got[0].Shell.Status != protocol.ChatShellExited { + t.Fatalf("exit after readback = %+v", got) + } +} + +func TestChatReleaseShellsBoundsDisconnectedState(t *testing.T) { + chat := NewChat(&Connection{}) + state := chat.streamState("ses_1") + state.markShell("sh_1") + chat.ReleaseShells("ses_1") + chat.shellMu.Lock() + _, retained := chat.shells["ses_1"] + chat.shellMu.Unlock() + if retained { + t.Fatal("released session shell state was retained") + } +} diff --git a/internal/server/chat.go b/internal/server/chat.go index 99eb212c..e318761f 100644 --- a/internal/server/chat.go +++ b/internal/server/chat.go @@ -32,9 +32,21 @@ type chatBroker interface { // Permissions lists the session's pending OpenCode approval requests so the // handler can push chat_permission frames. Permissions(ctx context.Context, sessionID string) ([]protocol.ChatPermissionFrame, error) + // Shells returns only shell operations whose ownership is authoritatively + // verified for sessionID. It is the reconnect/readback source. + Shells(ctx context.Context, sessionID string) ([]protocol.ChatShell, error) Reply(ctx context.Context, request protocol.ChatReplyRequest) error } +// chatShellReleaser is implemented by brokers that retain verified shell ids for +// attributing exit events. The hub calls it only after the last observer leaves +// a room; a later attach obtains a fresh authoritative snapshot. +// This is a bounded local-cache operation: it must not perform I/O or call the +// hub, which serializes it with room replacement under its mutex. +type chatShellReleaser interface { + ReleaseShells(sessionID string) +} + // operatorRegistry is the server-side operator-session seam. *pipeline.Store // satisfies it. It is a table of its own, separate from the pipeline stage // ownership (SessionOwner/run_session_owners) that settles a run: chat_sessions diff --git a/internal/server/chat_hub.go b/internal/server/chat_hub.go index 134e3fd6..9bd7e32e 100644 --- a/internal/server/chat_hub.go +++ b/internal/server/chat_hub.go @@ -133,6 +133,13 @@ func (h *chatHub) apply(frame protocol.ChatEventFrame) { } h.mu.Lock() defer h.mu.Unlock() + h.applyLocked(frame) +} + +func (h *chatHub) applyLocked(frame protocol.ChatEventFrame) { + if frame.SessionID == "" { + return + } turn := h.turnLocked(frame.SessionID) switch frame.Kind { case protocol.ChatEventStep: @@ -144,6 +151,9 @@ func (h *chatHub) apply(frame protocol.ChatEventFrame) { case protocol.ChatEventShell: // A shell frame carries no message id (it outlives the tool call that // started it), so it is folded before the message-id gate below. + if room := h.rooms[frame.SessionID]; room != nil { + room.shellReadback.Record(frame.Shell) + } turn.applyShell(frame.Shell) return } @@ -240,7 +250,7 @@ func (t *chatTurn) applyShell(shell *protocol.ChatShell) { if shell == nil || shell.ID == "" { return } - if shell.Status == protocol.ChatShellExited { + if shell.Status != protocol.ChatShellRunning { old, ok := t.shells[shell.ID] if !ok { return diff --git a/internal/server/chat_multiclient_test.go b/internal/server/chat_multiclient_test.go index 61f15e03..648fbf6c 100644 --- a/internal/server/chat_multiclient_test.go +++ b/internal/server/chat_multiclient_test.go @@ -82,6 +82,8 @@ func (c *channelChat) Session(context.Context, string) (protocol.ChatSession, bo return protocol.ChatSession{}, false, nil } +func (c *channelChat) Shells(context.Context, string) ([]protocol.ChatShell, error) { return nil, nil } + func (c *channelChat) Permissions(context.Context, string) ([]protocol.ChatPermissionFrame, error) { c.mu.Lock() defer c.mu.Unlock() diff --git a/internal/server/chat_room.go b/internal/server/chat_room.go index 9ae2450c..a8d0ecc5 100644 --- a/internal/server/chat_room.go +++ b/internal/server/chat_room.go @@ -106,6 +106,11 @@ type chatQueuedPrompt struct { acked bool } +type chatObservation struct { + done chan struct{} + err error // published by closing done; belongs to this attempt, not the next +} + // chatRoom is one session's shared live channel. Every field is guarded by the // owning chatHub's mutex. type chatRoom struct { @@ -117,6 +122,11 @@ type chatRoom struct { queue []*chatQueuedPrompt running bool streaming bool + // observing is non-nil while one caller is attaching the shared broker + // stream. It prevents concurrent subscribers from opening duplicate global + // event streams before streaming can be published. + observing *chatObservation + shellReadback *opencode.ShellSnapshot // buffering holds the turn's frames until the prompting client's ack has been // written, so the ack always precedes the frames it acknowledges. It is set // only while the very first turn subscribes to the firehose. @@ -160,6 +170,81 @@ func (h *chatHub) roomLocked(sessionID string) *chatRoom { return room } +// ensureObserving starts the firehose even for a read-only subscriber, then +// reconciles retained shells from the endpoint's authoritative list. A bounded +// journal folds concurrent lifecycle events over that snapshot atomically. All +// observers share this reconciliation, with no broker I/O under the hub mutex. +func (h *chatHub) ensureObserving(room *chatRoom, broker chatBroker) error { + if broker == nil { + return errors.New("chat is not configured on this server") + } + h.mu.Lock() + if err := room.ctx.Err(); err != nil { + h.mu.Unlock() + return err + } + if waiting := room.observing; waiting != nil { + h.mu.Unlock() + select { + case <-waiting.done: + return waiting.err + case <-room.ctx.Done(): + return room.ctx.Err() + } + } + if room.streaming { + h.mu.Unlock() + return nil + } + room.observing = &chatObservation{done: make(chan struct{})} + waiting := room.observing + room.shellReadback = &opencode.ShellSnapshot{} + h.mu.Unlock() + var observeErr error + defer func() { + h.mu.Lock() + room.shellReadback = nil + waiting.err = observeErr + room.observing = nil + close(waiting.done) + h.mu.Unlock() + }() + if err := h.startStream(room, broker); err != nil { + observeErr = err + return err + } + shells, err := broker.Shells(room.ctx, room.sessionID) + h.mu.Lock() + if room.ctx.Err() != nil || h.rooms[room.sessionID] != room { + h.mu.Unlock() + observeErr = context.Canceled + return observeErr + } + if err == nil { + shells, err = room.shellReadback.Merge(shells) + } + if err != nil { + // The firehose remains useful when its authoritative shell readback is + // temporarily unavailable. Do not turn that read failure into a chat + // failure (or claim the last retained snapshot is current): clear the + // replay cache and tell attached clients that shell visibility is limited. + // Preserve events already folded during this read, but clear stale replay. + live, mergeErr := room.shellReadback.Merge(nil) + if mergeErr == nil { + h.replaceShellsLocked(room.sessionID, live) + } + h.mu.Unlock() + // Never log broker bodies/URLs: they may contain credentials or private output. + log.Printf("chat shell readback session=%s failed (%T)", room.sessionID, err) + h.fanout(room.sessionID, protocol.Message{Type: protocol.Error, Error: "background shell visibility is limited"}, "") + return nil + } + h.replaceShellsLocked(room.sessionID, shells) + room.shellReadback = nil + h.mu.Unlock() + return nil +} + // attach registers write as one client's subscription to sessionID's stream and // returns it, or the existing subscription when the same connection has already // attached. It is idempotent so a client may prompt and subscribe without @@ -209,6 +294,11 @@ func (h *chatHub) detach(sessionID string, sub *chatSub) { if len(room.subs) == 0 { delete(h.rooms, sessionID) room.cancel() + // ReleaseShells is a local identity-cache operation (no broker I/O). + // Complete it before a reconnect can acquire this session's new room. + if releaser, ok := room.broker.(chatShellReleaser); ok { + releaser.ReleaseShells(sessionID) + } } h.mu.Unlock() } @@ -265,9 +355,17 @@ func (h *chatHub) broadcastUser(frame protocol.ChatEventFrame, exceptClientID st // once it is written. A step-ended frame finishes the turn, letting the queue // advance. func (h *chatHub) emit(frame protocol.ChatEventFrame) { - h.apply(frame) + h.emitRoom(nil, frame) +} + +func (h *chatHub) emitRoom(source *chatRoom, frame protocol.ChatEventFrame) { h.mu.Lock() room := h.rooms[frame.SessionID] + if source != nil && (source != room || source.ctx.Err() != nil) { + h.mu.Unlock() + return + } + h.applyLocked(frame) buffered := false var targets []*chatSub if room != nil { @@ -283,6 +381,22 @@ func (h *chatHub) emit(frame protocol.ChatEventFrame) { if room != nil && frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepEnded { h.finishTurnLocked(room) } + // Shell replay and live lifecycle delivery must enqueue in the same order + // as folding. Otherwise an exit can overtake an older running replay. + if frame.Kind == protocol.ChatEventShell && !buffered { + msg := protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame} + var slow []*chatSub + for _, sub := range targets { + if !enqueueChatLocked(sub, msg) { + slow = append(slow, sub) + } + } + h.mu.Unlock() + for _, sub := range slow { + sub.close() + } + return + } h.mu.Unlock() if buffered { return @@ -293,6 +407,62 @@ func (h *chatHub) emit(frame protocol.ChatEventFrame) { } } +// enqueueChatLocked never blocks or calls detach under the hub mutex. +func enqueueChatLocked(sub *chatSub, msg protocol.Message) bool { + select { + case <-sub.done: + return true + default: + } + select { + case sub.ch <- msg: + return true + default: + return false + } +} + +func (h *chatHub) replayShells(sub *chatSub) { + h.mu.Lock() + slow := false + if h.rooms[sub.room.sessionID] == sub.room && sub.room.subs[sub.clientID] == sub { + turn := h.turnLocked(sub.room.sessionID) + for _, id := range turn.shellOrder { + shell := turn.shells[id] + frame := protocol.ChatEventFrame{SessionID: sub.room.sessionID, Kind: protocol.ChatEventShell, Shell: &shell} + if !enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}) { + slow = true + break + } + } + } + h.mu.Unlock() + if slow { + sub.close() + } +} + +// replaceShells reconciles a successful authoritative readback. It drops stale +// retained entries (including exits missed while disconnected) and upserts the +// verified running set without fabricating a lifecycle event for foreign shells. +func (h *chatHub) replaceShells(sessionID string, shells []protocol.ChatShell) { + h.mu.Lock() + defer h.mu.Unlock() + h.replaceShellsLocked(sessionID, shells) +} + +func (h *chatHub) replaceShellsLocked(sessionID string, shells []protocol.ChatShell) { + turn := h.turnLocked(sessionID) + for _, shell := range turn.shells { + turn.shrink(len(shell.Command)) + } + turn.shells = nil + turn.shellOrder = nil + for i := range shells { + turn.applyShell(&shells[i]) + } +} + // finishTurnLocked marks the room's turn finished and releases the turn runner's // wait. Callers must hold h.mu. func (h *chatHub) finishTurnLocked(room *chatRoom) { @@ -331,14 +501,20 @@ func (h *chatHub) flushBuffered(sessionID string) { targets = append(targets, sub) } } - h.mu.Unlock() + var slow []*chatSub for i := range pending { frame := pending[i] msg := protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame} for _, sub := range targets { - sub.send(msg) + if !enqueueChatLocked(sub, msg) { + slow = append(slow, sub) + } } } + h.mu.Unlock() + for _, sub := range slow { + sub.close() + } } // submitPrompt accepts one prompt: it starts the turn runner immediately when no @@ -476,7 +652,7 @@ func (h *chatHub) runTurn(room *chatRoom, p *chatQueuedPrompt, done chan struct{ return } if first { - if err := h.startStream(room, broker); err != nil { + if err := h.ensureObserving(room, broker); err != nil { h.failPrompt(room, p, err) h.endRoomTurn(room) return @@ -550,7 +726,7 @@ func (h *chatHub) startStream(room *chatRoom, broker chatBroker) error { result <- broker.Stream(ctx, room.sessionID, func() { once.Do(func() { close(attached) }) }, func(frame protocol.ChatEventFrame) error { - h.emit(frame) + h.emitRoom(room, frame) return nil }) }() diff --git a/internal/server/chat_shell_reconcile_test.go b/internal/server/chat_shell_reconcile_test.go new file mode 100644 index 00000000..9ada615b --- /dev/null +++ b/internal/server/chat_shell_reconcile_test.go @@ -0,0 +1,261 @@ +package server + +import ( + "context" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" + "github.com/coder/websocket" + "github.com/coder/websocket/wsjson" +) + +type shellBarrierBroker struct { + *fakeChat + read chan struct{} + apply chan struct{} + events chan shellBarrierEvent + stopped chan struct{} + reads atomic.Int32 + snapshot []protocol.ChatShell +} + +// Exercise the real websocket handler on a loopback scratch server and a fresh +// SQLite database. A marker on the shared stream bounds the replay assertion +// without sleeps, polling or a negative-read timeout. +func TestChatShellReconciliationScratchServer(t *testing.T) { + b := newShellBarrierBroker() + b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: b}, newOperatorStore(t))) + defer srv.Close() + conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil) + if err != nil { + t.Fatal(err) + } + defer conn.CloseNow() + subscribe := func(id string) { + t.Helper() + if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: id, ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil { + t.Fatal(err) + } + } + read := func() protocol.Message { + t.Helper() + var m protocol.Message + if err := wsjson.Read(ctx, conn, &m); err != nil { + t.Fatal(err) + } + return m + } + if m := read(); m.Type != protocol.PipelineList { + t.Fatalf("server greeting = %s", m.Type) + } + subscribe("first") + if m := read(); m.ID != "first" || m.Type != protocol.ChatSubscribe { + t.Fatalf("ack = %+v", m) + } + shellBarrierWait(t, b.read) + done := make(chan struct{}) + b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_old", Status: protocol.ChatShellExited}, done} + shellBarrierWait(t, done) + if m := read(); m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellExited { + t.Fatalf("exit not visible: %+v", m) + } + close(b.apply) + subscribe("second") + for { + m := read() + if m.ChatEvent != nil && m.ChatEvent.Shell.Status == protocol.ChatShellRunning { + t.Fatalf("stale replay: %+v", m) + } + if m.ID == "second" { + break + } + } + done = make(chan struct{}) + b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_marker", Status: protocol.ChatShellExited}, done} + shellBarrierWait(t, done) + for { + m := read() + if m.ChatEvent == nil { + continue + } + if m.ChatEvent.Shell.Status == protocol.ChatShellRunning { + t.Fatalf("ghost visible after apply: %+v", m) + } + if m.ChatEvent.Shell.ID == "sh_marker" { + break + } + } + if b.reads.Load() != 1 { + t.Fatalf("shared observer read count = %d", b.reads.Load()) + } + conn.CloseNow() + shellBarrierWait(t, b.stopped) +} + +type shellBarrierEvent struct { + shell protocol.ChatShell + done chan struct{} +} + +func newShellBarrierBroker() *shellBarrierBroker { + return &shellBarrierBroker{fakeChat: scriptedChat(), read: make(chan struct{}, 8), apply: make(chan struct{}), events: make(chan shellBarrierEvent), stopped: make(chan struct{})} +} + +func (b *shellBarrierBroker) Shells(ctx context.Context, _ string) ([]protocol.ChatShell, error) { + b.reads.Add(1) + snapshot := append([]protocol.ChatShell(nil), b.snapshot...) + b.read <- struct{}{} + select { + case <-b.apply: + return snapshot, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (b *shellBarrierBroker) Stream(ctx context.Context, id string, ready func(), emit func(protocol.ChatEventFrame) error) error { + defer close(b.stopped) + ready() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case event := <-b.events: + if err := emit(protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventShell, Shell: &event.shell}); err != nil { + return err + } + close(event.done) + } + } +} + +func shellBarrierWait(t *testing.T, ch <-chan struct{}) { + t.Helper() + select { + case <-ch: + case <-time.After(3 * time.Second): + t.Fatal("barrier timed out") + } +} + +func TestChatShellReconciliationBarrier(t *testing.T) { + for _, status := range []string{protocol.ChatShellExited, protocol.ChatShellRunning, "timeout", "killed"} { + t.Run(status, func(t *testing.T) { + b := newShellBarrierBroker() + b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}} + h := newChatHub() + messages := make(chan protocol.Message, 32) + sub := h.attach("ses_1", "one", "", b, func(m protocol.Message) error { messages <- m; return nil }) + t.Cleanup(sub.close) + result := make(chan error, 2) + go func() { result <- h.ensureObserving(sub.room, b) }() + shellBarrierWait(t, b.read) + // A second observer must share the in-flight read, not apply an + // independently completing older/newer snapshot out of order. + go func() { result <- h.ensureObserving(sub.room, b) }() + shell := protocol.ChatShell{ID: "sh_old", Status: status} + if status == protocol.ChatShellRunning { + shell.ID = "sh_new" + shell.Command = "sleep 10" + } + done := make(chan struct{}) + select { + case b.events <- shellBarrierEvent{shell, done}: + case <-time.After(time.Second): + t.Fatal("event blocked behind readback") + } + shellBarrierWait(t, done) + select { + case m := <-messages: + if m.ChatEvent == nil || m.ChatEvent.Shell == nil || m.ChatEvent.Shell.ID != shell.ID || m.ChatEvent.Shell.Status != status { + t.Fatalf("live shell = %+v", m) + } + case <-time.After(time.Second): + t.Fatal("live event not visible during read") + } + close(b.apply) + for range 2 { + select { + case err := <-result: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("observer stuck") + } + } + if b.reads.Load() != 1 { + t.Fatalf("redundant/out-of-order reads: %d", b.reads.Load()) + } + snapshot := h.shellSnapshot("ses_1") + if status == protocol.ChatShellRunning { + if len(snapshot) != 2 { + t.Fatalf("new create lost: %+v", snapshot) + } + } else if len(snapshot) != 0 { + t.Fatalf("terminal shell resurrected: %+v", snapshot) + } + // Assert the same retained state is replayed to the client. + h.replayShells(sub) + for range len(snapshot) { + select { + case m := <-messages: + if m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellRunning { + t.Fatalf("replay = %+v", m) + } + case <-time.After(time.Second): + t.Fatal("replay missing") + } + } + sub.close() + shellBarrierWait(t, b.stopped) + }) + } +} + +func TestChatShellReconciliationCancelReconnectBarrier(t *testing.T) { + b := newShellBarrierBroker() + h := newChatHub() + sub := h.attach("ses_1", "one", "", b, func(protocol.Message) error { return nil }) + results := make(chan error, 2) + go func() { results <- h.ensureObserving(sub.room, b) }() + shellBarrierWait(t, b.read) + go func() { results <- h.ensureObserving(sub.room, b) }() + sub.close() + for range 2 { + select { + case err := <-results: + if err == nil { + t.Fatal("canceled observer succeeded") + } + case <-time.After(time.Second): + t.Fatal("canceled waiter stuck") + } + } + shellBarrierWait(t, b.stopped) + next := newShellBarrierBroker() + next.snapshot = []protocol.ChatShell{{ID: "sh_new", Status: protocol.ChatShellRunning}} + close(next.apply) + newSub := h.attach("ses_1", "two", "", next, func(protocol.Message) error { return nil }) + t.Cleanup(newSub.close) + if err := h.ensureObserving(newSub.room, next); err != nil { + t.Fatal(err) + } + // A late callback from the canceled stream cannot poison the new room, + // nor can a foreign-session callback from the current stream. + h.emitRoom(sub.room, protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_ghost", Status: protocol.ChatShellRunning}}) + h.emitRoom(newSub.room, protocol.ChatEventFrame{SessionID: "ses_foreign", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_foreign", Status: protocol.ChatShellRunning}}) + got := h.shellSnapshot("ses_1") + if len(got) != 1 || got[0].ID != "sh_new" || len(h.shellSnapshot("ses_foreign")) != 0 { + t.Fatalf("reconnect state = %+v", got) + } + newSub.close() + shellBarrierWait(t, next.stopped) +} diff --git a/internal/server/chat_stream.go b/internal/server/chat_stream.go index 76c175e6..89ba6404 100644 --- a/internal/server/chat_stream.go +++ b/internal/server/chat_stream.go @@ -111,7 +111,14 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "could not attach to session " + query.SessionID}) return } + // Acknowledge before the stream is allowed to fan out: otherwise an immediately + // available retained shell can overtake the subscribe reply on this connection. _ = write(protocol.Message{Type: protocol.ChatSubscribe, ID: request.ID}) + if err := hub.ensureObserving(sub.room, broker); err != nil { + sub.close() + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "could not observe session " + query.SessionID}) + return + } // Replay every still-unresolved approval request the room holds, not only the // most recent one, so a client that attaches after two requests were raised // still sees both cards. @@ -124,9 +131,5 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH // shows them rather than nothing until they exit. The message snapshot cannot // carry a shell — a shell outlives the turn and is not a turn — so this is the // shell half of the resume. - for _, shell := range hub.shellSnapshot(query.SessionID) { - running := shell - frame := protocol.ChatEventFrame{SessionID: query.SessionID, Kind: protocol.ChatEventShell, Shell: &running} - sub.send(protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}) - } + hub.replayShells(sub) } diff --git a/internal/server/chat_test.go b/internal/server/chat_test.go index 11dfde0f..93740260 100644 --- a/internal/server/chat_test.go +++ b/internal/server/chat_test.go @@ -3,6 +3,7 @@ package server import ( "context" "encoding/json" + "errors" "fmt" "net/http" "net/http/httptest" @@ -53,6 +54,8 @@ type fakeChat struct { permissionSessionID string permissions []protocol.ChatPermissionFrame permissionErr error + shells []protocol.ChatShell + shellErr error sessionCalls int sessions []protocol.ChatSession sessionErr error @@ -63,6 +66,8 @@ type fakeChat struct { started chan struct{} done chan struct{} startOnce sync.Once + streamCalls int + readyGate <-chan struct{} block bool subscribed bool promptBeforeSubscribe bool @@ -92,13 +97,18 @@ func (f *fakeChat) Prompt(context.Context, string, string) error { func (f *fakeChat) Stream(ctx context.Context, _ string, ready func(), emit func(protocol.ChatEventFrame) error) error { f.mu.Lock() f.subscribed = true + f.streamCalls++ + readyGate := f.readyGate f.mu.Unlock() - if ready != nil { - ready() - } if f.started != nil { f.startOnce.Do(func() { close(f.started) }) } + if readyGate != nil { + <-readyGate + } + if ready != nil { + ready() + } if f.block { <-ctx.Done() if f.done != nil { @@ -117,6 +127,42 @@ func (f *fakeChat) Stream(ctx context.Context, _ string, ready func(), emit func return nil } +// TestChatHubConcurrentObserversShareStream pins the start-up handoff: two +// clients that subscribe before the broker says ready still share one firehose. +func TestChatHubConcurrentObserversShareStream(t *testing.T) { + broker := scriptedChat() + broker.block = true + broker.started = make(chan struct{}) + gate := make(chan struct{}) + broker.readyGate = gate + hub := newChatHub() + hub.mu.Lock() + room := hub.roomLocked("ses_1") + hub.mu.Unlock() + + errs := make(chan error, 2) + go func() { errs <- hub.ensureObserving(room, broker) }() + select { + case <-broker.started: + case <-time.After(time.Second): + t.Fatal("first observer did not start the broker stream") + } + go func() { errs <- hub.ensureObserving(room, broker) }() + close(gate) + for range 2 { + if err := <-errs; err != nil { + t.Fatalf("ensureObserving: %v", err) + } + } + broker.mu.Lock() + calls := broker.streamCalls + broker.mu.Unlock() + if calls != 1 { + t.Fatalf("broker Stream calls = %d, want 1", calls) + } + room.cancel() +} + func (f *fakeChat) Interrupt(context.Context, string) error { f.mu.Lock() defer f.mu.Unlock() @@ -153,6 +199,12 @@ func (f *fakeChat) Reply(_ context.Context, request protocol.ChatReplyRequest) e // Permissions returns the seeded pending requests, or blocks until the caller's // context is canceled when permBlock is set (used to prove the poller stops on // disconnect). Every call is counted so a test can assert polling happened. +func (f *fakeChat) Shells(context.Context, string) ([]protocol.ChatShell, error) { + f.mu.Lock() + defer f.mu.Unlock() + return append([]protocol.ChatShell(nil), f.shells...), f.shellErr +} + func (f *fakeChat) Permissions(ctx context.Context, sessionID string) ([]protocol.ChatPermissionFrame, error) { f.mu.Lock() f.permissionCalls++ @@ -671,6 +723,9 @@ func TestChatInterruptEndsRunningPrompt(t *testing.T) { case r.Method == http.MethodGet && r.URL.Path == "/api/permission/request": w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"location":{"directory":"/work"},"data":[]}`)) + case r.Method == http.MethodGet && r.URL.Path == "/api/shell": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/event": flusher, ok := w.(http.Flusher) if !ok { @@ -984,6 +1039,7 @@ func TestChatPromptStreamsShellFrame(t *testing.T) { // shows the background process instead of nothing until it exits. func TestChatSubscribeReplaysRunningShell(t *testing.T) { broker := scriptedChat() + broker.shells = []protocol.ChatShell{{ID: "sh_1", Status: protocol.ChatShellRunning, Command: "sleep 120"}} broker.script = append([]protocol.ChatEventFrame{ {SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_1", Status: protocol.ChatShellRunning, Command: "sleep 120"}}, }, broker.script...) @@ -1028,6 +1084,40 @@ func TestChatSubscribeReplaysRunningShell(t *testing.T) { } } +// TestChatSubscribeKeepsStreamWhenShellReadbackFails ensures a temporary +// authoritative readback failure does not make chat unavailable or replay a +// stale process as running. The subscriber receives an explicit limitation +// while its broker stream remains attached for future lifecycle events. +func TestChatSubscribeKeepsStreamWhenShellReadbackFails(t *testing.T) { + broker := scriptedChat() + broker.block = true + broker.shellErr = errors.New("shell endpoint unavailable: https://user:secret@example.invalid private response body") + ctx, _, conn := dialChat(t, broker) + if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub1", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil { + t.Fatal(err) + } + var ack protocol.Message + if err := wsjson.Read(ctx, conn, &ack); err != nil { + t.Fatal(err) + } + if ack.Type != protocol.ChatSubscribe || ack.ID != "sub1" { + t.Fatalf("subscribe ack = %+v", ack) + } + var limited protocol.Message + if err := wsjson.Read(ctx, conn, &limited); err != nil { + t.Fatal(err) + } + if limited.Type != protocol.Error || limited.Error != "background shell visibility is limited" { + t.Fatalf("limited visibility notice = %+v", limited) + } + broker.mu.Lock() + calls := broker.streamCalls + broker.mu.Unlock() + if calls != 1 { + t.Fatalf("broker Stream calls = %d, want 1", calls) + } +} + // TestChatPromptSubscribesBeforePrompt pins the subscribe-before-prompt order: // the firehose has no replay cursor, so the subscription must be attached before // the prompt is posted or deltas emitted in between are lost. @@ -1068,6 +1158,9 @@ func realAdapterChat(t *testing.T, frames []string) []protocol.ChatEventFrame { case r.Method == http.MethodGet && r.URL.Path == "/api/permission/request": w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"location":{"directory":"/work"},"data":[]}`)) + case r.Method == http.MethodGet && r.URL.Path == "/api/shell": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/event": streams.Add(1) w.Header().Set("Content-Type", "text/event-stream") diff --git a/internal/server/delivery_gate.go b/internal/server/delivery_gate.go new file mode 100644 index 00000000..df1dc7c5 --- /dev/null +++ b/internal/server/delivery_gate.go @@ -0,0 +1,185 @@ +package server + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" + + "git.a-wan.net/alan/pipelines/internal/stage" +) + +// verifyDelivery proves that the stage landed a complete, committed tree before +// its success can settle the run. It re-derives the capability after the stage: +// a stage result and its input cannot select either the landing path or checks. +const maxDeliveryCheckOutput = 64 << 10 +const deliveryCleanupTimeout = 10 * time.Second + +// verifyDelivery verifies the committed item tree in a disposable worktree +// before fast-forwarding canonical main. Settlement deliberately performs only +// non-mutating repository checks after that mutation: a declared check must +// never be the first signal that the landed tree is known-red. +func verifyDelivery(ctx context.Context, workspace string, access stage.LandingAccess, checks []string) error { + lock, err := acquireDeliveryLock(ctx, access.LockPath) + if err != nil { + return err + } + defer lock.Close() + current, err := landingAccess(workspace) + if err != nil { + return fmt.Errorf("delivery access changed: %w", err) + } + if current != access { + return fmt.Errorf("delivery access changed during stage") + } + for _, directory := range []string{workspace, access.MainWorktree} { + status, err := gitOutput(directory, "status", "--porcelain=v1", "--untracked-files=all") + if err != nil { + return fmt.Errorf("delivery checkout status: %w", err) + } + if status != "" { + return fmt.Errorf("delivery checkout is not clean: %s", strings.Split(status, "\n")[0]) + } + } + itemHead, err := gitOutput(workspace, "rev-parse", "HEAD") + if err != nil { + return fmt.Errorf("delivery item commit: %w", err) + } + if err := runDeliveryCandidate(ctx, workspace, itemHead, checks); err != nil { + return err + } + mainHead, err := gitOutput(access.MainWorktree, "rev-parse", "HEAD") + if err != nil { + return fmt.Errorf("delivery main commit: %w", err) + } + if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", mainHead, itemHead).Run(); err != nil { + return fmt.Errorf("delivery item commit cannot fast-forward canonical main") + } + if output, err := exec.CommandContext(ctx, "git", "-C", access.MainWorktree, "merge", "--ff-only", itemHead).CombinedOutput(); err != nil { + return fmt.Errorf("delivery fast-forward canonical main: %w: %s", err, strings.TrimSpace(string(output))) + } + for _, directory := range []string{workspace, access.MainWorktree} { + status, err := gitOutput(directory, "status", "--porcelain=v1", "--untracked-files=all") + if err != nil { + return fmt.Errorf("delivery settlement status: %w", err) + } + if status != "" { + return fmt.Errorf("delivery checkout is not clean after checks: %s", strings.Split(status, "\n")[0]) + } + } + // Re-derive at the settlement boundary. A success means exactly the checked + // item commit is now reachable from the same canonical main worktree. + current, err = landingAccess(workspace) + if err != nil || current != access { + return fmt.Errorf("delivery access changed during settlement") + } + mainHead, err = gitOutput(access.MainWorktree, "rev-parse", "HEAD") + if err != nil { + return fmt.Errorf("delivery main commit: %w", err) + } + if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", itemHead, mainHead).Run(); err != nil { + return fmt.Errorf("delivery item commit is not an ancestor of canonical main") + } + return nil +} + +func acquireDeliveryLock(ctx context.Context, path string) (*os.File, error) { + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return nil, fmt.Errorf("delivery lock: %w", err) + } + for { + err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) + if err == nil { + return file, nil + } + if !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EAGAIN) { + _ = file.Close() + return nil, fmt.Errorf("delivery lock: %w", err) + } + select { + case <-ctx.Done(): + _ = file.Close() + return nil, fmt.Errorf("delivery lock: %w", ctx.Err()) + case <-time.After(20 * time.Millisecond): + } + } +} + +func runDeliveryCandidate(ctx context.Context, workspace, itemHead string, checks []string) error { + candidate, err := os.MkdirTemp(filepath.Dir(workspace), ".pipelines-delivery-") + if err != nil { + return fmt.Errorf("create delivery candidate: %w", err) + } + if err := os.Remove(candidate); err != nil { + return fmt.Errorf("prepare delivery candidate: %w", err) + } + if output, err := boundedCommandOutput(ctx, workspace, "worktree", "add", "--detach", candidate, itemHead); err != nil { + cleanupDeliveryCandidate(workspace, candidate) + return fmt.Errorf("create delivery candidate: %w: %s", err, strings.TrimSpace(string(output))) + } + defer cleanupDeliveryCandidate(workspace, candidate) + for _, check := range checks { + if err := runDeliveryCheck(ctx, candidate, check); err != nil { + return err + } + } + return nil +} + +// cleanupDeliveryCandidate always uses its own bounded context: the stage context +// may already be canceled when a check fails or times out. +func cleanupDeliveryCandidate(workspace, candidate string) { + ctx, cancel := context.WithTimeout(context.Background(), deliveryCleanupTimeout) + defer cancel() + _ = exec.CommandContext(ctx, "git", "-C", workspace, "worktree", "remove", "--force", candidate).Run() + _ = os.RemoveAll(candidate) +} + +func boundedCommandOutput(ctx context.Context, directory string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, "git", append([]string{"-C", directory}, args...)...) + var output boundedDeliveryOutput + cmd.Stdout = &output + cmd.Stderr = &output + err := cmd.Run() + return []byte(output.String()), err +} + +func runDeliveryCheck(ctx context.Context, directory, check string) error { + cmd := exec.CommandContext(ctx, "sh", "-c", check) + cmd.Dir = directory + var output boundedDeliveryOutput + cmd.Stdout = &output + cmd.Stderr = &output + if err := cmd.Run(); err != nil { + return fmt.Errorf("delivery check %q failed: %w: %s", check, err, output.String()) + } + return nil +} + +type boundedDeliveryOutput struct{ bytes.Buffer } + +func (b *boundedDeliveryOutput) Write(p []byte) (int, error) { + remaining := maxDeliveryCheckOutput - b.Len() + if remaining > 0 { + _, _ = b.Buffer.Write(p[:min(remaining, len(p))]) + } + return len(p), nil +} + +func (b *boundedDeliveryOutput) String() string { + text := strings.TrimSpace(b.Buffer.String()) + if b.Len() == maxDeliveryCheckOutput { + return text + " (output truncated)" + } + return text +} + +var _ io.Writer = (*boundedDeliveryOutput)(nil) diff --git a/internal/server/landing_access.go b/internal/server/landing_access.go new file mode 100644 index 00000000..a373d625 --- /dev/null +++ b/internal/server/landing_access.go @@ -0,0 +1,130 @@ +package server + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "git.a-wan.net/alan/pipelines/internal/stage" +) + +// landingAccess derives delivery paths from Git's verified worktree metadata, +// never from stage input. A missing canonical main worktree fails closed. +func landingAccess(workspace string) (stage.LandingAccess, error) { + workspace, err := canonicalDirectory(workspace) + if err != nil { + return stage.LandingAccess{}, fmt.Errorf("delivery workspace: %w", err) + } + gitDir, err := gitOutput(workspace, "rev-parse", "--git-dir") + if err != nil { + return stage.LandingAccess{}, fmt.Errorf("delivery workspace is not a Git worktree: %w", err) + } + commonDir, err := gitOutput(workspace, "rev-parse", "--git-common-dir") + if err != nil { + return stage.LandingAccess{}, fmt.Errorf("delivery repository metadata: %w", err) + } + gitDir, err = canonicalDirectory(resolveGitPath(workspace, gitDir)) + if err != nil { + return stage.LandingAccess{}, fmt.Errorf("delivery Git metadata: %w", err) + } + commonDir, err = canonicalDirectory(resolveGitPath(workspace, commonDir)) + if err != nil { + return stage.LandingAccess{}, fmt.Errorf("delivery common Git metadata: %w", err) + } + main, err := mainWorktree(workspace, commonDir) + if err != nil { + return stage.LandingAccess{}, err + } + // Delivery originates in a linked item worktree. Its metadata must be under + // the repository's canonical common directory, never a standalone or foreign + // checkout. Git's common directory is the bounded capability needed for refs, + // worktree state, and the landing lock. + if !pathWithin(filepath.Join(commonDir, "worktrees"), gitDir) { + return stage.LandingAccess{}, fmt.Errorf("delivery workspace is not a linked Git worktree") + } + return stage.LandingAccess{MainWorktree: main, GitMetadata: commonDir, LockPath: filepath.Join(commonDir, "pipelines-landing.lock")}, nil +} + +func mainWorktree(workspace, commonDir string) (string, error) { + list, err := gitOutput(workspace, "worktree", "list", "--porcelain") + if err != nil { + return "", fmt.Errorf("list repository worktrees: %w", err) + } + for _, record := range strings.Split(strings.TrimSpace(list), "\n\n") { + lines := strings.Split(record, "\n") + if len(lines) == 0 || !strings.HasPrefix(lines[0], "worktree ") || !worktreeHasBranch(lines, "refs/heads/main") { + continue + } + candidate, err := canonicalDirectory(strings.TrimPrefix(lines[0], "worktree ")) + if err != nil || candidate == workspace { + continue + } + candidateGitDir, err := gitOutput(candidate, "rev-parse", "--git-dir") + if err != nil { + continue + } + candidateGitDir, err = canonicalDirectory(resolveGitPath(candidate, candidateGitDir)) + if err != nil || candidateGitDir != commonDir { + continue + } + candidateCommon, err := gitOutput(candidate, "rev-parse", "--git-common-dir") + if err != nil { + continue + } + candidateCommon, err = canonicalDirectory(resolveGitPath(candidate, candidateCommon)) + if err == nil && candidateCommon == commonDir { + return candidate, nil + } + } + return "", fmt.Errorf("no canonical main worktree in the delivery repository") +} + +func worktreeHasBranch(lines []string, branch string) bool { + for _, line := range lines[1:] { + if line == "branch "+branch { + return true + } + } + return false +} + +func canonicalDirectory(path string) (string, error) { + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + return "", err + } + info, err := os.Stat(resolved) + if err != nil { + return "", err + } + if !info.IsDir() { + return "", fmt.Errorf("not a directory") + } + return filepath.Clean(resolved), nil +} + +func resolveGitPath(workspace, value string) string { + if filepath.IsAbs(value) { + return value + } + return filepath.Join(workspace, value) +} + +func pathWithin(parent, child string) bool { + rel, err := filepath.Rel(parent, child) + return err == nil && rel != "." && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + +func gitOutput(directory string, args ...string) (string, error) { + cmd := exec.Command("git", append([]string{"-C", directory}, args...)...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + data, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String())) + } + return strings.TrimSpace(string(data)), nil +} diff --git a/internal/server/landing_access_test.go b/internal/server/landing_access_test.go new file mode 100644 index 00000000..9d2d7ae8 --- /dev/null +++ b/internal/server/landing_access_test.go @@ -0,0 +1,304 @@ +package server + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func TestLandingAccessAcceptsOnlyVerifiedSameRepositoryMainWorktree(t *testing.T) { + root := t.TempDir() + main := filepath.Join(root, "main") + runGit(t, root, "init", "-b", "main", main) + runGit(t, main, "config", "user.email", "test@example.invalid") + runGit(t, main, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, main, "add", "README") + runGit(t, main, "commit", "-m", "base") + item := filepath.Join(root, "item") + runGit(t, main, "worktree", "add", "-b", "item", item) + access, err := landingAccess(item) + if err != nil { + t.Fatal(err) + } + canonicalMain, err := canonicalDirectory(main) + if err != nil { + t.Fatal(err) + } + if access.MainWorktree != canonicalMain || access.GitMetadata != filepath.Join(canonicalMain, ".git") || access.LockPath != filepath.Join(canonicalMain, ".git", "pipelines-landing.lock") { + t.Fatalf("unexpected access: %+v", access) + } + foreign := filepath.Join(root, "foreign") + runGit(t, root, "init", "-b", "main", foreign) + if _, err := landingAccess(foreign); err == nil { + t.Fatal("foreign repository without an item worktree gained landing access") + } + if _, err := landingAccess(main); err == nil { + t.Fatal("main worktree gained linked-worktree delivery access") + } + if err := os.RemoveAll(item); err != nil { + t.Fatal(err) + } + runGit(t, main, "worktree", "prune") + if _, err := landingAccess(main); err == nil { + t.Fatal("repository without a linked delivery worktree gained landing access") + } + secondItem := filepath.Join(root, "second-item") + runGit(t, main, "worktree", "add", "-b", "second-item", secondItem) + runGit(t, main, "branch", "-m", "main", "trunk") + if _, err := landingAccess(secondItem); err == nil { + t.Fatal("repository without a canonical main worktree gained landing access") + } +} + +func TestVerifyDeliveryRequiresCleanLandedMainAndPassingChecks(t *testing.T) { + root := t.TempDir() + main := filepath.Join(root, "main") + runGit(t, root, "init", "-b", "main", main) + runGit(t, main, "config", "user.email", "test@example.invalid") + runGit(t, main, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, main, "add", "README") + runGit(t, main, "commit", "-m", "base") + item := filepath.Join(root, "item") + runGit(t, main, "worktree", "add", "-b", "item", item) + if err := os.WriteFile(filepath.Join(item, "delivered"), []byte("complete\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, item, "add", "delivered") + runGit(t, item, "commit", "-m", "delivery") + runGit(t, main, "merge", "--ff-only", "item") + access, err := landingAccess(item) + if err != nil { + t.Fatal(err) + } + if err := verifyDelivery(context.Background(), item, access, []string{"test -f delivered"}); err != nil { + t.Fatalf("complete landed delivery rejected: %v", err) + } + if err := os.WriteFile(filepath.Join(item, "missing-helper"), []byte("dirty\n"), 0600); err != nil { + t.Fatal(err) + } + if err := verifyDelivery(context.Background(), item, access, []string{"true"}); err == nil { + t.Fatal("dirty task-owned helper was accepted") + } + if err := os.Remove(filepath.Join(item, "missing-helper")); err != nil { + t.Fatal(err) + } + if err := verifyDelivery(context.Background(), item, access, []string{"false"}); err == nil { + t.Fatal("failed required check was accepted") + } + if err := os.WriteFile(filepath.Join(item, "unlanded"), []byte("no\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, item, "add", "unlanded") + runGit(t, item, "commit", "-m", "unlanded") + if err := verifyDelivery(context.Background(), item, access, []string{"test -f unlanded"}); err != nil { + t.Fatalf("complete unlanded item was not landed after candidate verification: %v", err) + } + mainHead := runGitOutput(t, main, "rev-parse", "HEAD") + itemHead := runGitOutput(t, item, "rev-parse", "HEAD") + if mainHead != itemHead { + t.Fatal("verified item was not fast-forwarded into canonical main") + } +} + +func TestVerifyDeliveryFailingCandidateLeavesMainUnchanged(t *testing.T) { + root := t.TempDir() + main := filepath.Join(root, "main") + runGit(t, root, "init", "-b", "main", main) + runGit(t, main, "config", "user.email", "test@example.invalid") + runGit(t, main, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, main, "add", "README") + runGit(t, main, "commit", "-m", "base") + item := filepath.Join(root, "item") + runGit(t, main, "worktree", "add", "-b", "item", item) + if err := os.WriteFile(filepath.Join(item, "candidate"), []byte("candidate\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, item, "add", "candidate") + runGit(t, item, "commit", "-m", "candidate") + access, err := landingAccess(item) + if err != nil { + t.Fatal(err) + } + before := runGitOutput(t, main, "rev-parse", "HEAD") + if err := verifyDelivery(context.Background(), item, access, []string{"test -f candidate", "false"}); err == nil { + t.Fatal("failing candidate check landed main") + } + if after := runGitOutput(t, main, "rev-parse", "HEAD"); after != before { + t.Fatal("failing candidate check changed canonical main") + } +} + +func TestRunnerSettlesSuccessfulDeliveryAfterCandidateLanding(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + main := filepath.Join(root, "main") + runGit(t, root, "init", "-b", "main", main) + runGit(t, main, "config", "user.email", "test@example.invalid") + runGit(t, main, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, main, "add", "README") + runGit(t, main, "commit", "-m", "base") + item := filepath.Join(root, "item") + runGit(t, main, "worktree", "add", "-b", "item", item) + if err := os.WriteFile(filepath.Join(item, "unlanded"), []byte("no\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, item, "add", "unlanded") + runGit(t, item, "commit", "-m", "unlanded") + + store, err := pipeline.Open(filepath.Join(root, "runner.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + source := pipeline.Template([]string{"committer"}, true) + source = strings.Replace(source, "```configs", "```checks\ntrue\n```\n\n```configs", 1) + source = strings.Replace(source, "timeout=1h", "timeout=1h\ndelivery=true", 1) + if _, err := store.Submit(ctx, "delivery", source); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + r, err := newRunner(ctx, store) + if err != nil { + t.Fatal(err) + } + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "unlanded", Workspace: item, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + r.wg.Wait() + detail, err := store.GetRun(ctx, run.ID) + if err != nil { + t.Fatal(err) + } + if detail.Status != "succeeded" || len(detail.Steps) != 1 { + t.Fatalf("complete delivery did not settle successfully: %+v", detail) + } + if runGitOutput(t, main, "rev-parse", "HEAD") != runGitOutput(t, item, "rev-parse", "HEAD") { + t.Fatal("runner did not land the verified item commit") + } +} + +func TestRunnerRejectsWrongBackendVerificationEvidence(t *testing.T) { + ctx := context.Background() + source := pipeline.Template([]string{"worker"}, true) + source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground\n```\n\n```configs", 1) + r, store, id := submitRunner(t, ctx, source) + defer store.Close() + r.verifyTarget = func(context.Context, stage.VerificationTarget) (stage.VerificationEvidence, error) { + return stage.VerificationEvidence{Backend: "old-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"}, nil + } + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + run := startRun(t, r, id, "wrong-backend-evidence") + detail, err := store.GetRun(ctx, run.ID) + if err != nil { + t.Fatal(err) + } + if detail.Status != "failed" || len(detail.Steps) != 1 || !strings.Contains(detail.Steps[0].Diagnostic, "backend") { + t.Fatalf("wrong-backend evidence settled as success: %+v", detail) + } +} + +func TestRunnerIgnoresModelSuppliedVerificationEvidence(t *testing.T) { + ctx := context.Background() + source := pipeline.Template([]string{"worker"}, true) + source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground\n```\n\n```configs", 1) + r, store, id := submitRunner(t, ctx, source) + defer store.Close() + r.verifyTarget = func(context.Context, stage.VerificationTarget) (stage.VerificationEvidence, error) { + return stage.VerificationEvidence{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"}, nil + } + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{Outcome: "success", Output: stage.Values{"verification": map[string]any{"backend": "old-api", "revision": "stale", "capability": "invented", "fixture": "other"}}}, nil + } + run := startRun(t, r, id, "model-supplied-evidence") + detail, err := store.GetRun(ctx, run.ID) + if err != nil { + t.Fatal(err) + } + if detail.Status != "succeeded" || len(detail.Steps) != 1 { + t.Fatalf("controlled verification did not settle: %+v", detail) + } +} + +func TestMainWorktreeRejectsNearMatchAndLinkedMain(t *testing.T) { + for _, test := range []struct { + name string + branch string + }{ + {name: "mainline is not main", branch: "mainline"}, + {name: "linked main is not canonical", branch: "main"}, + } { + t.Run(test.name, func(t *testing.T) { + root := t.TempDir() + primary := filepath.Join(root, "primary") + runGit(t, root, "init", "-b", "trunk", primary) + runGit(t, primary, "config", "user.email", "test@example.invalid") + runGit(t, primary, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(primary, "README"), []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + runGit(t, primary, "add", "README") + runGit(t, primary, "commit", "-m", "base") + item := filepath.Join(root, "item") + runGit(t, primary, "worktree", "add", "-b", "item", item) + candidate := filepath.Join(root, test.branch) + runGit(t, primary, "worktree", "add", "-b", test.branch, candidate) + + commonDir, err := gitOutput(item, "rev-parse", "--git-common-dir") + if err != nil { + t.Fatal(err) + } + commonDir, err = canonicalDirectory(resolveGitPath(item, commonDir)) + if err != nil { + t.Fatal(err) + } + if _, err := mainWorktree(item, commonDir); err == nil { + t.Fatalf("%s gained canonical main access", test.name) + } + }) + } +} + +func runGit(t *testing.T, directory string, args ...string) { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", directory}, args...)...) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v: %s", args, err, output) + } +} + +func runGitOutput(t *testing.T, directory string, args ...string) string { + t.Helper() + output, err := exec.Command("git", append([]string{"-C", directory}, args...)...).Output() + if err != nil { + t.Fatalf("git %v: %v", args, err) + } + return strings.TrimSpace(string(output)) +} diff --git a/internal/server/runner.go b/internal/server/runner.go index efaa8787..260397fe 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -58,8 +58,11 @@ type runner struct { // directory. It defaults to stage.ExportSession; a failure is best-effort and // never changes the stage's status, outcome, target or result validation. exportSession func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) - wg sync.WaitGroup - schedulerWG sync.WaitGroup + // verifyTarget is installed only by server wiring for a controlled fixture + // workflow. Leaving it nil fails closed for target-aware definitions. + verifyTarget stage.VerificationRunner + wg sync.WaitGroup + schedulerWG sync.WaitGroup // attentionMu guards attentionSent, the per-run set of run-attention events // already delivered — by a stage's own notify policy or by the run-level // default — so neither republishes the same genuine event. @@ -1079,7 +1082,22 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na } } operations := &stage.OperationLog{} + var landing stage.LandingAccess + delivery, deliveryErr := stage.DeliveryEnabled(def) + if deliveryErr != nil { + finish("failed", "error", deliveryErr.Error()) + return + } + if delivery { + access, accessErr := landingAccess(workspace) + if accessErr != nil { + finish("failed", "error", "delivery access denied: "+accessErr.Error()) + return + } + landing = access + } client := r.agentClient(id, sequence, state.artifactDirs, workspace, artifactsDir, operations, request.WritePaths, tools) + client.LandingAccess = landing client.OnSession = func(session string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -1222,6 +1240,45 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na step.Output["salvage"] = result.Output } } + if err == nil && result.Outcome == "success" { + target, targetErr := stage.VerificationTargetFor(def) + if targetErr != nil { + err = targetErr + } else if target != nil { + if r.verifyTarget == nil { + err = fmt.Errorf("verification target %q requires a server-controlled fixture runner", target.Fixture) + } else { + evidence, verificationErr := r.verifyTarget(stageCtx, *target) + if verificationErr != nil { + err = verificationErr + } else { + err = stage.VerifyTargetEvidence(*target, evidence) + } + } + } + if err != nil { + step.Status = "failed" + step.Diagnostic = err.Error() + step.Outcome = "error" + step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input} + } + } + if err == nil && delivery && result.Outcome == "success" { + checks, checksErr := stage.DeliveryChecks(def) + if checksErr != nil { + err = checksErr + } else { + gateCtx, gateCancel := context.WithTimeout(stageCtx, modelConfig.Timeout) + err = verifyDelivery(gateCtx, workspace, landing, checks) + gateCancel() + } + if err != nil { + step.Status = "failed" + step.Diagnostic = err.Error() + step.Outcome = "error" + step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input} + } + } // A failed or timed-out step carries the session pointer even when the // bounded transcript would otherwise be only its stage_limits envelope. if err != nil && stageSession != "" { diff --git a/internal/server/verification_fixture.go b/internal/server/verification_fixture.go new file mode 100644 index 00000000..767723ab --- /dev/null +++ b/internal/server/verification_fixture.go @@ -0,0 +1,135 @@ +package server + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" + + "git.a-wan.net/alan/pipelines/internal/stage" +) + +const ( + verificationFixtureID = "reconnect-foreground" + verificationFixtureResponseLimit int64 = 64 << 10 + verificationFixtureTimeout = 10 * time.Second +) + +// VerificationFixture is a server-owned, explicitly registered probe. Its +// endpoint and expected provenance come from server configuration, never a +// stage definition or result. +type VerificationFixture struct { + ID, Endpoint, Backend, Revision, Capability, State string +} + +// VerificationFixturesFromEnvironment returns the single supported fixture +// registration when its endpoint is explicitly configured. There is no default +// endpoint: an unconfigured server fails target-aware verification closed. +func VerificationFixturesFromEnvironment(getenv func(string) string) []VerificationFixture { + if getenv == nil { + getenv = os.Getenv + } + endpoint := strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_URL")) + if endpoint == "" { + return nil + } + return []VerificationFixture{{ + ID: verificationFixtureID, + Endpoint: endpoint, + Backend: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_BACKEND")), + Revision: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_REVISION")), + Capability: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_CAPABILITY")), + State: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_STATE")), + }} +} + +type verificationFixtureRegistry struct { + fixtures map[string]VerificationFixture + err error +} + +func newVerificationFixtureRegistry(fixtures []VerificationFixture) *verificationFixtureRegistry { + r := &verificationFixtureRegistry{fixtures: make(map[string]VerificationFixture, len(fixtures))} + for _, fixture := range fixtures { + if err := validVerificationFixture(fixture); err != nil { + r.err = err + return r + } + if _, exists := r.fixtures[fixture.ID]; exists { + r.err = fmt.Errorf("duplicate verification fixture %q", fixture.ID) + return r + } + r.fixtures[fixture.ID] = fixture + } + return r +} + +func validVerificationFixture(fixture VerificationFixture) error { + if fixture.ID != verificationFixtureID { + return fmt.Errorf("unsupported verification fixture %q", fixture.ID) + } + for _, value := range []string{fixture.Endpoint, fixture.Backend, fixture.Revision, fixture.Capability, fixture.State} { + if value == "" || len(value) > 256 { + return fmt.Errorf("verification fixture %q has incomplete configuration", fixture.ID) + } + } + u, err := url.Parse(fixture.Endpoint) + if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") { + return fmt.Errorf("verification fixture %q has invalid endpoint", fixture.ID) + } + return nil +} + +func (r *verificationFixtureRegistry) verify(ctx context.Context, target stage.VerificationTarget) (stage.VerificationEvidence, error) { + if r.err != nil { + return stage.VerificationEvidence{}, r.err + } + fixture, exists := r.fixtures[target.Fixture] + if !exists { + return stage.VerificationEvidence{}, fmt.Errorf("unknown verification fixture %q", target.Fixture) + } + if fixture.Backend != target.Backend || fixture.Revision != target.Revision || fixture.Capability != target.Capability { + return stage.VerificationEvidence{}, fmt.Errorf("verification target does not match registered fixture %q", target.Fixture) + } + probeCtx, cancel := context.WithTimeout(ctx, verificationFixtureTimeout) + defer cancel() + req, err := http.NewRequestWithContext(probeCtx, http.MethodGet, fixture.Endpoint, nil) + if err != nil { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q request: %w", fixture.ID, err) + } + client := &http.Client{Timeout: verificationFixtureTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + response, err := client.Do(req) + if err != nil { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q probe: %w", fixture.ID, err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q probe returned %s", fixture.ID, response.Status) + } + var observed struct { + Backend string `json:"backend"` + Revision string `json:"revision"` + Capability string `json:"capability"` + Fixture string `json:"fixture"` + State string `json:"state"` + } + body, err := io.ReadAll(io.LimitReader(response.Body, verificationFixtureResponseLimit+1)) + if err != nil { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response: %w", fixture.ID, err) + } + if int64(len(body)) > verificationFixtureResponseLimit { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response exceeds %d bytes", fixture.ID, verificationFixtureResponseLimit) + } + if err := json.Unmarshal(body, &observed); err != nil { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response: %w", fixture.ID, err) + } + if observed.Backend != fixture.Backend || observed.Revision != fixture.Revision || observed.Capability != fixture.Capability || observed.Fixture != fixture.ID || observed.State != fixture.State { + return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q returned incompatible backend, revision, capability, fixture, or state", fixture.ID) + } + return stage.VerificationEvidence{Backend: observed.Backend, Revision: observed.Revision, Capability: observed.Capability, Fixture: observed.Fixture}, nil +} diff --git a/internal/server/verification_fixture_test.go b/internal/server/verification_fixture_test.go new file mode 100644 index 00000000..3f71ca96 --- /dev/null +++ b/internal/server/verification_fixture_test.go @@ -0,0 +1,148 @@ +package server + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func TestHandlerRunsRegisteredVerificationFixture(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + var mu sync.Mutex + response := map[string]string{"backend": "branch-api", "revision": "abc123", "capability": "controlled-state", "fixture": verificationFixtureID, "state": "ready"} + calls := 0 + fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + calls++ + _ = json.NewEncoder(w).Encode(response) + })) + defer fixture.Close() + + store, err := pipeline.Open(filepath.Join(t.TempDir(), "fixture.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + source := pipeline.Template([]string{"worker"}, true) + source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture="+verificationFixtureID+"\n```\n\n```configs", 1) + if _, err := store.Submit(ctx, "controlled-fixture", source); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + handler := HandlerWithOptions(ctx, Options{VerificationFixtures: []VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", State: "ready"}}}, store).(*runHandler) + defer func() { + cancel() + WaitRuns(handler) + }() + r := handler.runner + if r.verifyTarget == nil { + t.Fatal("normal server construction did not wire controlled fixture verification") + } + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + // These forged model labels must not influence fixture provenance. + return stage.Result{Outcome: "success", Output: stage.Values{"verification": map[string]any{"backend": "old-api", "revision": "stale", "capability": "forged", "fixture": "other"}}}, nil + } + start := func(key string) *protocol.RunDetail { + run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: key, Workspace: t.TempDir(), Input: map[string]any{"prompt": "verify"}}) + if err != nil { + t.Fatal(err) + } + r.wg.Wait() + detail, err := store.GetRun(ctx, run.ID) + if err != nil { + t.Fatal(err) + } + return detail + } + if detail := start("compatible"); detail.Status != "succeeded" { + t.Fatalf("compatible controlled fixture did not settle: %+v", detail) + } + mu.Lock() + response["revision"] = "old-build" + mu.Unlock() + if detail := start("stale"); detail.Status != "failed" || !strings.Contains(detail.Steps[0].Diagnostic, "incompatible") { + t.Fatalf("stale fixture provenance settled as success: %+v", detail) + } + mu.Lock() + if calls != 2 { + t.Fatalf("fixture probes = %d, want one per target-aware stage", calls) + } + mu.Unlock() +} + +func TestVerificationFixtureCancellationClosesProbe(t *testing.T) { + started := make(chan struct{}) + canceled := make(chan struct{}) + fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(started) + <-r.Context().Done() + close(canceled) + })) + defer fixture.Close() + registry := newVerificationFixtureRegistry([]VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", State: "ready"}}) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + _, err := registry.verify(ctx, stage.VerificationTarget{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: verificationFixtureID}) + done <- err + }() + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("fixture probe did not start") + } + cancel() + select { + case err := <-done: + if err == nil { + t.Fatal("canceled fixture probe succeeded") + } + case <-time.After(time.Second): + t.Fatal("canceled fixture probe did not return") + } + select { + case <-canceled: + case <-time.After(time.Second): + t.Fatal("canceled fixture request was not cleaned up") + } +} + +func TestVerificationFixtureRejectsUnknownAndMismatchedProvenance(t *testing.T) { + base := map[string]string{"backend": "branch-api", "revision": "abc123", "capability": "controlled-state", "fixture": verificationFixtureID, "state": "ready"} + for field, wrong := range map[string]string{"backend": "old-api", "revision": "old-build", "capability": "other-state", "state": "stale"} { + t.Run(field, func(t *testing.T) { + response := make(map[string]string, len(base)) + for key, value := range base { + response[key] = value + } + response[field] = wrong + fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(response) + })) + defer fixture.Close() + registry := newVerificationFixtureRegistry([]VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: base["backend"], Revision: base["revision"], Capability: base["capability"], State: base["state"]}}) + _, err := registry.verify(context.Background(), stage.VerificationTarget{Backend: base["backend"], Revision: base["revision"], Capability: base["capability"], Fixture: verificationFixtureID}) + if err == nil || !strings.Contains(err.Error(), "incompatible") { + t.Fatalf("mismatched %s provenance was accepted: %v", field, err) + } + }) + } + registry := newVerificationFixtureRegistry(nil) + if _, err := registry.verify(context.Background(), stage.VerificationTarget{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "unknown"}); err == nil || !strings.Contains(err.Error(), "unknown") { + t.Fatalf("unknown fixture was accepted: %v", err) + } +} diff --git a/internal/server/websocket.go b/internal/server/websocket.go index adb3184b..5e598223 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -49,6 +49,10 @@ type Options struct { // read time with no model call, which keeps tests and unconfigured servers // deterministic. StepVerdicts func(context.Context, map[string]any) string + // VerificationFixtures are explicit server-owned HTTP fixtures for optional + // target-aware verification. Definitions can name only a registered ID; they + // cannot select an endpoint or supply provenance. + VerificationFixtures []VerificationFixture } // Handler serves the persistent connection with default options. Canceling ctx @@ -81,6 +85,10 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin if runErr == nil && options.StepVerdicts != nil { runs.verdict = options.StepVerdicts } + if runErr == nil { + fixtures := newVerificationFixtureRegistry(options.VerificationFixtures) + runs.verifyTarget = fixtures.verify + } if runErr == nil { runs.schedulerWG.Add(1) go runs.scheduleLoop() diff --git a/internal/stage/agent.go b/internal/stage/agent.go index 57b23b12..dd91d1f3 100644 --- a/internal/stage/agent.go +++ b/internal/stage/agent.go @@ -19,6 +19,7 @@ type AgentClient struct { // ArtifactAccess is supplied only by the runner. Write is this visit's // directory; Read contains this and completed earlier visits in the same run. ArtifactAccess ArtifactAccess + LandingAccess LandingAccess OnSession func(string, bool) error SessionID string Transcript string diff --git a/internal/stage/model.go b/internal/stage/model.go index 13e3d1ff..18465c67 100644 --- a/internal/stage/model.go +++ b/internal/stage/model.go @@ -59,7 +59,7 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) { } for block := range def.Blocks { switch block { - case "output", "prompt", "edges", "configs", "tools", "permissions": + case "output", "prompt", "edges", "configs", "tools", "permissions", "checks", "verification": default: return ModelSpec{}, fmt.Errorf("unexpected block %q for model stage", block) } @@ -70,6 +70,9 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) { if _, err := StagePermissions(def); err != nil { return ModelSpec{}, err } + if _, err := VerificationTargetFor(def); err != nil { + return ModelSpec{}, err + } for _, key := range []string{"provider", "model"} { if def.Configs[key] == "" { return ModelSpec{}, fmt.Errorf("missing config %s", key) @@ -77,7 +80,7 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) { } for key := range def.Configs { switch key { - case "provider", "model", "retries", "timeout", "budget_usd", "budget_tokens", "harness": + case "provider", "model", "retries", "timeout", "budget_usd", "budget_tokens", "harness", "delivery": default: return ModelSpec{}, fmt.Errorf("unknown config %q", key) } @@ -92,6 +95,15 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) { if backend != "opencode" && backend != "pi" { return ModelSpec{}, fmt.Errorf("harness must be opencode or pi") } + if delivery := def.Configs["delivery"]; delivery != "" && delivery != "true" { + return ModelSpec{}, fmt.Errorf("delivery must be true when set") + } + if def.Configs["delivery"] == "true" && backend != "opencode" { + return ModelSpec{}, fmt.Errorf("delivery is supported only by the opencode harness") + } + if _, present := def.Blocks["checks"]; present && def.Configs["delivery"] != "true" { + return ModelSpec{}, fmt.Errorf("checks block is supported only by delivery stages") + } if backend == "pi" { tools, _, _ := ToolNames(def) if _, err := piTools(tools); err != nil { diff --git a/internal/stage/tools.go b/internal/stage/tools.go index 77477987..2f567e3b 100644 --- a/internal/stage/tools.go +++ b/internal/stage/tools.go @@ -14,6 +14,77 @@ type ArtifactAccess struct { Read []string } +// LandingAccess is the server-derived external capability for a verified +// delivery operation. Empty or malformed values grant nothing. +type LandingAccess struct { + MainWorktree string + GitMetadata string + LockPath string +} + +func (a LandingAccess) valid() bool { + for _, path := range []string{a.MainWorktree, a.GitMetadata, a.LockPath} { + if path == "" || !filepath.IsAbs(path) || filepath.Clean(path) != path || strings.ContainsAny(path, "*?\\") { + return false + } + } + // The server grants the canonical repository metadata directory as OpenCode + // V2's smallest external-directory unit. Keep the lock inside that directory + // so a caller cannot turn a valid metadata capability into access elsewhere. + return a.GitMetadata == filepath.Join(a.MainWorktree, ".git") && + a.LockPath == filepath.Join(a.GitMetadata, "pipelines-landing.lock") +} + +// DeliveryEnabled is the narrow, declarative capability check used before the +// runner supplies any landing paths. Only a validated model definition can ask +// for it; runtime paths are still derived by the server. +func DeliveryEnabled(def Definition) (bool, error) { + delivery := def.Configs["delivery"] + if delivery == "" { + return false, nil + } + if delivery != "true" { + return false, fmt.Errorf("delivery must be true when set") + } + if EffectiveType(def.Type) != ModelType { + return false, fmt.Errorf("delivery is supported only by model stages") + } + if _, err := (ModelStage{}).Spec(def); err != nil { + return false, fmt.Errorf("invalid delivery model stage: %w", err) + } + if _, err := DeliveryChecks(def); err != nil { + return false, err + } + return true, nil +} + +// DeliveryChecks returns the bounded, definition-declared checks that the +// runner executes against a committed delivery candidate before canonical main +// is changed. They +// are intentionally a block rather than a generic runner setting: each +// workflow owns its repository-specific verification commands. +func DeliveryChecks(def Definition) ([]string, error) { + text := strings.TrimSpace(def.Blocks["checks"]) + if text == "" { + return nil, fmt.Errorf("delivery stages require a nonempty checks block") + } + var checks []string + for _, line := range strings.Split(text, "\n") { + check := strings.TrimSpace(line) + if check == "" { + continue + } + if len(check) > 4096 { + return nil, fmt.Errorf("delivery check exceeds 4096 bytes") + } + checks = append(checks, check) + } + if len(checks) == 0 || len(checks) > 20 { + return nil, fmt.Errorf("delivery stages require 1 to 20 checks") + } + return checks, nil +} + // ToolNames validates model-stage permissions, not general graph semantics. // A present empty block explicitly denies all tools. func ToolNames(def Definition) ([]string, bool, error) { @@ -155,6 +226,10 @@ func (c *AgentClient) permissionRules() []map[string]string { allow("external_directory", filepath.ToSlash(filepath.Join(c.ArtifactAccess.Write, "*"))) allow("edit", filepath.ToSlash(filepath.Join(c.ArtifactAccess.Write, "*"))) } + if c.LandingAccess.valid() { + allow("external_directory", filepath.ToSlash(filepath.Join(c.LandingAccess.MainWorktree, "*"))) + allow("external_directory", filepath.ToSlash(filepath.Join(c.LandingAccess.GitMetadata, "*"))) + } for _, pattern := range []string{"../*", "*/../*"} { rules = append(rules, map[string]string{"action": "external_directory", "resource": pattern, "effect": "deny"}) } diff --git a/internal/stage/tools_test.go b/internal/stage/tools_test.go index ed823bc1..e7a20c0f 100644 --- a/internal/stage/tools_test.go +++ b/internal/stage/tools_test.go @@ -174,3 +174,107 @@ func TestArtifactPermissionsAreExactAndToolScoped(t *testing.T) { t.Fatal("missing artifact scope enabled an external directory") } } + +func TestLandingPermissionsAreExactAndServerDerived(t *testing.T) { + access := LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/pipelines/.git", LockPath: "/repos/pipelines/.git/pipelines-landing.lock"} + client := &AgentClient{Tools: []string{"read", "shell"}, LandingAccess: access} + rules := client.permissionRules() + for _, path := range []string{"/repos/pipelines/go.mod", "/repos/pipelines/.git/worktrees/item/HEAD", "/repos/pipelines/.git/pipelines-landing.lock"} { + if permissionEffect(rules, "external_directory", path) != "allow" { + t.Fatalf("verified landing path denied: %q", path) + } + } + for _, path := range []string{"/repos/other/go.mod", "/repos/pipelines/../other/go.mod", "/Users/alan/.git/config", "/.git/config"} { + if permissionEffect(rules, "external_directory", path) != "deny" { + t.Fatalf("unverified landing path allowed: %q", path) + } + } + client.LandingAccess.MainWorktree = "/repos/pipelines/../other" + if permissionEffect(client.permissionRules(), "external_directory", "/repos/other/go.mod") != "deny" { + t.Fatal("non-canonical landing capability allowed") + } + client.LandingAccess = LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/pipelines/.git", LockPath: "/tmp/pipelines-landing.lock"} + if permissionEffect(client.permissionRules(), "external_directory", "/tmp/pipelines-landing.lock") != "deny" { + t.Fatal("landing lock outside verified metadata allowed") + } + client.LandingAccess = LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/other/.git", LockPath: "/repos/other/.git/pipelines-landing.lock"} + if permissionEffect(client.permissionRules(), "external_directory", "/repos/other/.git/config") != "deny" { + t.Fatal("unrelated Git metadata allowed") + } +} + +func TestDeliveryEnabledRequiresValidatedModelStage(t *testing.T) { + def := modelDefinition() + def.Configs["delivery"] = "true" + def.Blocks["checks"] = "go test ./..." + if enabled, err := DeliveryEnabled(def); err != nil || !enabled { + t.Fatalf("valid delivery stage = %v, %v", enabled, err) + } + for name, change := range map[string]func(*Definition){ + "malformed value": func(d *Definition) { d.Configs["delivery"] = "yes" }, + "non-model": func(d *Definition) { d.Type = HumanType }, + "invalid model": func(d *Definition) { d.Configs["unexpected"] = "value" }, + } { + t.Run(name, func(t *testing.T) { + copy := def + copy.Configs = map[string]string{} + for key, value := range def.Configs { + copy.Configs[key] = value + } + change(©) + if enabled, err := DeliveryEnabled(copy); err == nil || enabled { + t.Fatalf("invalid delivery stage = %v, %v", enabled, err) + } + }) + } +} + +func TestDeliveryRejectsPiBeforeAgentExecution(t *testing.T) { + def := modelDefinition() + def.Configs["delivery"] = "true" + def.Configs["harness"] = "pi" + def.Blocks["checks"] = "true" + if err := (ModelStage{}).Validate(def); err == nil || !strings.Contains(err.Error(), "opencode harness") { + t.Fatal("Pi delivery was accepted", err) + } + delete(def.Configs, "delivery") + delete(def.Blocks, "checks") + if err := (ModelStage{}).Validate(def); err != nil { + t.Fatal("Pi non-delivery was changed", err) + } +} + +func TestDeliveryChecksRequireBoundedDeliveryOnlyDeclaration(t *testing.T) { + def := modelDefinition() + def.Configs["delivery"] = "true" + for name, mutate := range map[string]func(*Definition){ + "missing": func(d *Definition) {}, + "empty": func(d *Definition) { d.Blocks["checks"] = "\n" }, + "non delivery": func(d *Definition) { d.Blocks["checks"] = "go test ./..." }, + "too many checks": func(d *Definition) { d.Blocks["checks"] = strings.Repeat("true\n", 21) }, + "oversized check": func(d *Definition) { d.Blocks["checks"] = strings.Repeat("x", 4097) }, + } { + t.Run(name, func(t *testing.T) { + copy := def + copy.Blocks = map[string]string{} + copy.Configs = map[string]string{} + for key, value := range def.Blocks { + copy.Blocks[key] = value + } + for key, value := range def.Configs { + copy.Configs[key] = value + } + mutate(©) + if name == "non delivery" { + delete(copy.Configs, "delivery") + if _, err := (ModelStage{}).Spec(copy); err == nil { + t.Fatal("non-delivery checks block was accepted") + } + return + } + if _, err := DeliveryEnabled(copy); err == nil { + t.Fatal("invalid delivery checks were accepted") + } + }) + } +} diff --git a/internal/stage/verification.go b/internal/stage/verification.go new file mode 100644 index 00000000..a186aa15 --- /dev/null +++ b/internal/stage/verification.go @@ -0,0 +1,97 @@ +package stage + +import ( + "context" + "fmt" + "strings" +) + +// VerificationTarget declares the controlled fixture that can substantiate a +// target-specific behavior claim. It is intentionally data-only: the runner +// does not execute fixture input supplied by a stage result. +type VerificationTarget struct { + Backend string + Revision string + Capability string + Fixture string +} + +// VerificationEvidence is returned by a server-controlled fixture workflow, +// never parsed from a model's result. +type VerificationEvidence struct { + Backend string + Revision string + Capability string + Fixture string +} + +// VerificationRunner is the narrow server seam for a controlled fixture or +// workflow. The runner selects it during server setup; model output cannot +// select a backend, revision, fixture, or executable. +type VerificationRunner func(context.Context, VerificationTarget) (VerificationEvidence, error) + +// VerificationTargetFor parses an optional, bounded verification block. A +// declared target requires all provenance fields so an old or other-backend +// fixture cannot be presented as evidence for the current target. +func VerificationTargetFor(def Definition) (*VerificationTarget, error) { + text, declared := def.Blocks["verification"] + if !declared { + return nil, nil + } + if len(text) > 4096 { + return nil, fmt.Errorf("verification block exceeds 4096 bytes") + } + values := map[string]string{} + for _, line := range strings.Split(text, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + key, value, ok := strings.Cut(line, "=") + key, value = strings.TrimSpace(key), strings.TrimSpace(value) + if !ok || value == "" || len(value) > 256 { + return nil, fmt.Errorf("invalid verification entry %q", line) + } + switch key { + case "backend", "revision", "capability", "fixture": + default: + return nil, fmt.Errorf("unknown verification entry %q", key) + } + if _, exists := values[key]; exists { + return nil, fmt.Errorf("duplicate verification entry %q", key) + } + values[key] = value + } + target := &VerificationTarget{Backend: values["backend"], Revision: values["revision"], Capability: values["capability"], Fixture: values["fixture"]} + if target.Backend == "" || target.Revision == "" || target.Capability == "" || target.Fixture == "" { + return nil, fmt.Errorf("verification requires backend, revision, capability, and fixture") + } + return target, nil +} + +// VerifyTargetEvidence compares server-collected controlled-fixture provenance +// with the workflow declaration. It deliberately accepts no model result: a +// stage cannot substantiate its own target-specific behavior claim. +func VerifyTargetEvidence(target VerificationTarget, evidence VerificationEvidence) error { + for key, want := range map[string]string{ + "backend": target.Backend, "revision": target.Revision, "capability": target.Capability, "fixture": target.Fixture, + } { + got := map[string]string{ + "backend": evidence.Backend, "revision": evidence.Revision, "capability": evidence.Capability, "fixture": evidence.Fixture, + }[key] + if got != want { + return fmt.Errorf("verification evidence %s does not match declared target", key) + } + } + return nil +} + +// VerifyTargetResult is retained for callers that only need declaration +// validation. Result-provided provenance is intentionally not evidence. +func VerifyTargetResult(def Definition, _ Values) error { + target, err := VerificationTargetFor(def) + if err != nil || target == nil { + return err + } + return fmt.Errorf("verification target %q requires a server-controlled fixture runner", target.Fixture) +} diff --git a/internal/stage/verification_test.go b/internal/stage/verification_test.go new file mode 100644 index 00000000..d52d5b5a --- /dev/null +++ b/internal/stage/verification_test.go @@ -0,0 +1,27 @@ +package stage + +import "testing" + +func TestTargetVerificationRejectsWrongBackendEvidence(t *testing.T) { + def := modelDefinition() + def.Blocks["verification"] = "backend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground" + if err := (ModelStage{}).Validate(def); err != nil { + t.Fatal(err) + } + target, err := VerificationTargetFor(def) + if err != nil { + t.Fatal(err) + } + good := VerificationEvidence{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"} + if err := VerifyTargetEvidence(*target, good); err != nil { + t.Fatalf("matching fixture evidence rejected: %v", err) + } + wrongBackend := VerificationEvidence{Backend: "old-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"} + if err := VerifyTargetEvidence(*target, wrongBackend); err == nil { + t.Fatal("wrong-backend fixture evidence accepted") + } + def.Blocks["verification"] = "backend=branch-api\nrevision=abc123" + if err := (ModelStage{}).Validate(def); err == nil { + t.Fatal("incomplete target verification declaration accepted") + } +}