diff --git a/docs/notes/background-shell-visibility.md b/docs/notes/background-shell-visibility.md new file mode 100644 index 00000000..583e5f90 --- /dev/null +++ b/docs/notes/background-shell-visibility.md @@ -0,0 +1,37 @@ +# Background shell visibility + +The chat server now opens the broker stream for read-only subscribers and reconciles verified running shell operations from OpenCode's authoritative `/api/shell` readback. The packaged OpenCode skill references and chat documentation should describe that shell visibility survives reconnects only when OpenCode reports explicit `metadata.sessionID` ownership; shared working directories are not ownership evidence. + +## Current source behavior + +Observers share one initial readback per room/stream attachment. The adapter and hub +fold bounded lifecycle journals over each snapshot, so a stale read cannot resurrect +an exited shell or erase a concurrent create. Shell replay and live delivery enqueue +in fold order. Cancellation rejects old-room callbacks and releases local identity +state before reconnect. Journals retain at most 256 identities and 1 MiB of id/command +text; capacity/read failures surface stable limited-visibility text, never broker +response bodies. No shell polling was added. OpenCode's terminal `timeout` and +`killed` statuses are excluded from retained running operations as well as `exited`. + +## Verification evidence — 2026-10-18 + +`opencode api get /openapi.json` authenticated through supported service discovery. +The live schema advertises shell list/detail/output and generic `metadata`; it does +not type `metadata.sessionID`. A throwaway session +`ses_eb12ab7b3ffe0ZAle7PHpftHMG` and its own `sleep 1` shell +`sh_14ed57134001e6fuYqvn62pKit` verified actual create/detail readback preserving +explicit `metadata.sessionID`, status `running`, subsequent `exited`/exit 0, and +absence from the running list. This verifies explicit API metadata, not automatic +agent-tool ownership injection. The CLI event attempt yielded no captured events; +live-provider event ordering is **not tested**. Forced ordering evidence comes from +channel/barrier regressions, including a real WebSocket handler on a loopback scratch +server with fresh SQLite, not from the deployed service. + +## Documentation follow-up / deployment + +`docs/chat-verification.md` and the chat subscribe/replay section of `docs/cli.md` +are stale: docs-fold should describe authoritative ownership/readback, coalesced +observation, bounded reconciliation and limited-visibility errors, distinguishing +fixture ordering evidence from the live readback above. Shared docs and packaged +skill references were not edited here. No deployment, installation, service restart +or physical-phone/UI verification was performed; this changes no rendered layout. diff --git a/internal/opencode/chat.go b/internal/opencode/chat.go index 40adc005..8fae8cd9 100644 --- a/internal/opencode/chat.go +++ b/internal/opencode/chat.go @@ -28,7 +28,9 @@ var ErrPermissionUnsupported = errors.New("OpenCode permission approval is not i // semantics, the session-id shape and the 64 KiB / 1 MiB bounds of Connection so // no second transport or credential path is introduced. type Chat struct { - conn *Connection + conn *Connection + shellMu sync.Mutex + shells map[string]*ChatStreamState // mu serializes the adapter's HTTP calls. Connection records its last // operation and status on the shared connection, so two calls must not // overlap; the permission poller runs alongside the read loop's @@ -294,6 +296,114 @@ func decodeChatPermissions(sessionID string, data []byte) ([]protocol.ChatPermis return frames, nil } +// Shells reads the endpoint's authoritative shell list and returns only running +// operations whose metadata explicitly names sessionID. The shell API is global; +// filtering locally avoids guessing an unsupported query filter and never assigns +// ownership from a shared working directory. A successful read also seeds the +// stream state used to attribute later shell.exited events, which carry no session. +func (c *Chat) Shells(ctx context.Context, sessionID string) ([]protocol.ChatShell, error) { + if !ValidSessionID(sessionID) { + return nil, fmt.Errorf("invalid OpenCode session id") + } + state, err := c.streamStateContext(ctx, sessionID) + if err != nil { + return nil, err + } + // One read per state; waiting and I/O never hold the event-state mutex. + state.mu.Lock() + if state.readGate == nil { + state.readGate = make(chan struct{}, 1) + } + gate := state.readGate + state.mu.Unlock() + select { + case gate <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + defer func() { <-gate }() + state.mu.Lock() + state.readback = &ShellSnapshot{} + state.mu.Unlock() + defer func() { state.mu.Lock(); state.readback = nil; state.mu.Unlock() }() + data, err := c.call(ctx, "get", "/api/shell", nil) + if err != nil { + return nil, err + } + var response struct { + Data []chatShellInfo `json:"data"` + } + if err := json.Unmarshal(data, &response); err != nil || response.Data == nil { + if err := json.Unmarshal(data, &response.Data); err != nil { + return nil, fmt.Errorf("invalid OpenCode shell response") + } + } + out := make([]protocol.ChatShell, 0, len(response.Data)) + for _, info := range response.Data { + if info.ID == "" || info.shellSessionID() != sessionID || (info.Status != "" && info.Status != protocol.ChatShellRunning) { + continue + } + status := info.Status + if status == "" { + status = protocol.ChatShellRunning + } + out = append(out, protocol.ChatShell{ID: info.ID, Status: status, Command: info.Command}) + } + state.mu.Lock() + defer state.mu.Unlock() + out, err = state.readback.Merge(out) + if err != nil { + return nil, err + } + if err := ctx.Err(); err != nil { + return nil, err + } + ids := make([]string, 0, len(out)) + for _, shell := range out { + ids = append(ids, shell.ID) + } + state.replaceShellsLocked(ids) + return out, nil +} + +// streamState returns the session's retained, verified shell identities. It is +// deliberately kept across stream replacement so an exit after reconnect remains +// attributable; Shells reconciles its contents from authoritative readback on +// each attach rather than replacing the pointer a live scanner is using. +func (c *Chat) streamState(sessionID string) *ChatStreamState { + state, _ := c.streamStateContext(context.Background(), sessionID) + return state +} + +func (c *Chat) streamStateContext(ctx context.Context, sessionID string) (*ChatStreamState, error) { + c.shellMu.Lock() + defer c.shellMu.Unlock() + if err := ctx.Err(); err != nil { + return nil, err + } + if c.shells == nil { + c.shells = map[string]*ChatStreamState{} + } + state := c.shells[sessionID] + if state == nil { + state = &ChatStreamState{} + c.shells[sessionID] = state + } + return state, nil +} + +// ReleaseShells discards verified shell identities after the last room observer +// disconnects. A future attach calls Shells for a fresh authoritative snapshot, +// so this bounds state without attributing a foreign shell during the gap. +func (c *Chat) ReleaseShells(sessionID string) { + if sessionID == "" { + return + } + c.shellMu.Lock() + delete(c.shells, sessionID) + c.shellMu.Unlock() +} + // Stream subscribes to the OpenCode event firehose and invokes emit for every // frame belonging to sessionID. It reads one bounded SSE frame at a time and // never polls the per-session message log; the global firehose is the only @@ -306,6 +416,12 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr if !sessionID.MatchString(id) { return fmt.Errorf("invalid OpenCode session id") } + // Pin state before opening/announcing the stream so a canceled scanner + // cannot acquire a replacement room's identity map. + state, err := c.streamStateContext(ctx, id) + if err != nil { + return err + } body, err := c.conn.EventStream(ctx) if err != nil { return err @@ -314,7 +430,7 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr if ready != nil { ready() } - return scanChatEvents(ctx, body, id, emit) + return c.scanChatEventsWithState(ctx, body, id, state, emit) } // scanChatEvents turns an SSE body into protocol frames, one bounded frame at a @@ -322,10 +438,13 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr // malformed JSON fails the stream instead of producing a malformed frame. The // last assistant message id is retained so a later tool/step event that omits it // still attributes every frame to a stable message. -func scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit func(protocol.ChatEventFrame) error) error { +func (c *Chat) scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit func(protocol.ChatEventFrame) error) error { + return c.scanChatEventsWithState(ctx, body, sessionID, c.streamState(sessionID), emit) +} + +func (c *Chat) scanChatEventsWithState(ctx context.Context, body io.Reader, sessionID string, state *ChatStreamState, emit func(protocol.ChatEventFrame) error) error { scanner := bufio.NewScanner(body) scanner.Buffer(make([]byte, 0, 64<<10), MaxResponse) - state := &ChatStreamState{} for scanner.Scan() { if err := ctx.Err(); err != nil { return err @@ -346,7 +465,7 @@ func scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit continue } if frame.MessageID != "" { - state.LastMessageID = frame.MessageID + state.setLastMessageID(frame.MessageID) } if err := emit(frame); err != nil { return err @@ -449,8 +568,29 @@ func (i *chatShellInfo) shellSessionID() string { // what attributes the exit to its session: another session's shell is an // unknown id and is ignored. type ChatStreamState struct { + mu sync.Mutex LastMessageID string Shells map[string]struct{} + readGate chan struct{} + readback *ShellSnapshot +} + +func (s *ChatStreamState) lastMessageID() string { + if s == nil { + return "" + } + s.mu.Lock() + defer s.mu.Unlock() + return s.LastMessageID +} + +func (s *ChatStreamState) setLastMessageID(id string) { + if s == nil || id == "" { + return + } + s.mu.Lock() + s.LastMessageID = id + s.mu.Unlock() } // knowsShell reports whether id is a shell this stream has already surfaced for @@ -459,6 +599,8 @@ func (s *ChatStreamState) knowsShell(id string) bool { if s == nil || id == "" { return false } + s.mu.Lock() + defer s.mu.Unlock() _, ok := s.Shells[id] return ok } @@ -468,6 +610,8 @@ func (s *ChatStreamState) markShell(id string) { if s == nil || id == "" { return } + s.mu.Lock() + defer s.mu.Unlock() if s.Shells == nil { s.Shells = map[string]struct{}{} } @@ -476,12 +620,37 @@ func (s *ChatStreamState) markShell(id string) { // forgetShell drops id once its exit has been surfaced. func (s *ChatStreamState) forgetShell(id string) { - if s == nil || s.Shells == nil { + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if s.Shells == nil { return } delete(s.Shells, id) } +// replaceShells applies one authoritative running-shell snapshot without +// replacing the state object an already attached event scanner owns. +func (s *ChatStreamState) replaceShells(ids []string) { + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.replaceShellsLocked(ids) +} + +func (s *ChatStreamState) replaceShellsLocked(ids []string) { + s.Shells = make(map[string]struct{}, len(ids)) + for _, id := range ids { + if id != "" { + s.Shells[id] = struct{}{} + } + } +} + // NormalizeChatEvent maps one OpenCode firehose `data:` payload to a // protocol.ChatEventFrame, folding the small per-session context ([stream]) it // needs across frames. ok is false for an event type this adapter does not @@ -528,7 +697,7 @@ func NormalizeChatEvent(payload []byte, sessionID string, stream *ChatStreamStat messageID = data.MessageIDAlt } if messageID == "" && stream != nil { - messageID = stream.LastMessageID + messageID = stream.lastMessageID() } // Every pushed frame must carry a stable message id so the client can // attribute it; without one this is not a chat frame. @@ -590,20 +759,38 @@ func normalizeShellEvent(eventType string, data chatEvent, sessionID string, str if status == "" { status = protocol.ChatShellRunning } - stream.markShell(data.Info.ID) + shell := &protocol.ChatShell{ID: data.Info.ID, Status: status, Command: data.Info.Command} + if stream != nil { + stream.mu.Lock() + stream.readback.Record(shell) + if stream.Shells == nil { + stream.Shells = map[string]struct{}{} + } + if status == protocol.ChatShellRunning && len(stream.Shells) < chatShellIdentityLimit { + stream.Shells[shell.ID] = struct{}{} + } + stream.mu.Unlock() + } return protocol.ChatEventFrame{ SessionID: sessionID, Kind: protocol.ChatEventShell, - Shell: &protocol.ChatShell{ID: data.Info.ID, Status: status, Command: data.Info.Command}, + Shell: shell, }, true, nil } if data.ID == "" { return protocol.ChatEventFrame{}, false, fmt.Errorf("malformed OpenCode shell.exited event") } - if !stream.knowsShell(data.ID) { + if stream == nil { + return protocol.ChatEventFrame{}, false, nil + } + stream.mu.Lock() + stream.readback.Record(&protocol.ChatShell{ID: data.ID, Status: protocol.ChatShellExited}) + _, known := stream.Shells[data.ID] + delete(stream.Shells, data.ID) + stream.mu.Unlock() + if !known { return protocol.ChatEventFrame{}, false, nil } - stream.forgetShell(data.ID) status := data.Status if status == "" { status = protocol.ChatShellExited diff --git a/internal/opencode/chat_shell_reconcile_test.go b/internal/opencode/chat_shell_reconcile_test.go new file mode 100644 index 00000000..087080d0 --- /dev/null +++ b/internal/opencode/chat_shell_reconcile_test.go @@ -0,0 +1,116 @@ +package opencode + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +func TestChatShellReadbackBarrier(t *testing.T) { + for _, known := range []bool{false, true} { + t.Run(fmt.Sprint(known), func(t *testing.T) { + read, apply := make(chan struct{}), make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + close(read) + select { + case <-apply: + fmt.Fprint(w, `{"data":[{"id":"sh_old","status":"running","metadata":{"sessionID":"ses_1"}},{"id":"sh_foreign","status":"running","metadata":{"sessionID":"ses_other"}}]}`) + case <-r.Context().Done(): + } + })) + defer srv.Close() + c := NewChat(&Connection{Backend: "v2", URL: srv.URL}) + state := c.streamState("ses_1") + if known { + state.markShell("sh_old") + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + result := make(chan []protocol.ChatShell, 1) + errs := make(chan error, 1) + go func() { shells, err := c.Shells(ctx, "ses_1"); result <- shells; errs <- err }() + select { + case <-read: + case <-ctx.Done(): + t.Fatal("read did not reach barrier") + } + // Both a previously verified exit and an exit first learned while + // reading must suppress the stale running snapshot. + frame, ok, err := NormalizeChatEvent([]byte(`{"type":"shell.exited","data":{"id":"sh_old","status":"exited","exit":0}}`), "ses_1", state) + if err != nil || ok != known || (ok && frame.Shell.Status != protocol.ChatShellExited) { + t.Fatalf("exit = %+v %v %v", frame, ok, err) + } + frame, ok, err = NormalizeChatEvent([]byte(`{"type":"shell.created","data":{"info":{"id":"sh_new","status":"running","command":"sleep 10","metadata":{"sessionID":"ses_1"}}}}`), "ses_1", state) + if err != nil || !ok || frame.Shell.ID != "sh_new" { + t.Fatalf("create = %+v %v %v", frame, ok, err) + } + close(apply) + got := <-result + if err := <-errs; err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].ID != "sh_new" || got[0].Command != "sleep 10" { + t.Fatalf("readback lost events/leaked ownership: %+v", got) + } + if state.knowsShell("sh_old") || state.knowsShell("sh_foreign") || !state.knowsShell("sh_new") { + t.Fatal("adapter identity state regressed") + } + // User-visible subsequent exit remains attributable after reconciliation. + frame, ok, err = NormalizeChatEvent([]byte(`{"type":"shell.exited","data":{"id":"sh_new","status":"exited","exit":0}}`), "ses_1", state) + if err != nil || !ok || frame.Shell.ID != "sh_new" { + t.Fatalf("new shell exit lost: %+v %v %v", frame, ok, err) + } + }) + } +} + +func TestChatShellReadbackCancelWaiter(t *testing.T) { + read := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { close(read); <-r.Context().Done() })) + defer srv.Close() + c := NewChat(&Connection{Backend: "v2", URL: srv.URL}) + ctx, cancel := context.WithCancel(context.Background()) + result := make(chan error, 2) + go func() { _, err := c.Shells(ctx, "ses_1"); result <- err }() + select { + case <-read: + case <-time.After(time.Second): + t.Fatal("read stuck") + } + go func() { _, err := c.Shells(ctx, "ses_1"); result <- err }() + cancel() + for range 2 { + select { + case err := <-result: + if err == nil { + t.Fatal("canceled read succeeded") + } + case <-time.After(time.Second): + t.Fatal("read or waiter stuck") + } + } + old := c.streamState("ses_1") + c.ReleaseShells("ses_1") + if c.streamState("ses_1") == old { + t.Fatal("reconnect reused released state") + } +} + +func TestShellSnapshotBounded(t *testing.T) { + var snapshot ShellSnapshot + for i := 0; i <= chatShellIdentityLimit; i++ { + snapshot.Record(&protocol.ChatShell{ID: fmt.Sprintf("sh_%d", i), Status: protocol.ChatShellExited}) + } + if len(snapshot.changes) != chatShellIdentityLimit { + t.Fatalf("unbounded journal: %d", len(snapshot.changes)) + } + if _, err := snapshot.Merge([]protocol.ChatShell{{ID: "sh_0", Status: protocol.ChatShellRunning}}); err == nil { + t.Fatal("overflow accepted stale snapshot") + } +} diff --git a/internal/opencode/chat_shell_snapshot.go b/internal/opencode/chat_shell_snapshot.go new file mode 100644 index 00000000..8559a741 --- /dev/null +++ b/internal/opencode/chat_shell_snapshot.go @@ -0,0 +1,72 @@ +package opencode + +import ( + "fmt" + "sort" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// ShellSnapshot records changes while an authoritative read is in flight. Its +// owner serializes access. Unknown exits may be recorded, but can only remove an +// id: they never establish ownership. Overflow rejects the snapshot, not events. +type ShellSnapshot struct { + changes map[string]protocol.ChatShell + bytes int + overflow bool +} + +const chatShellIdentityLimit = 256 + +func (s *ShellSnapshot) Record(shell *protocol.ChatShell) { + if s == nil || shell == nil || shell.ID == "" || s.overflow { + return + } + if s.changes == nil { + s.changes = make(map[string]protocol.ChatShell) + } + old := s.changes[shell.ID] + n := s.bytes - len(old.ID) - len(old.Command) + len(shell.ID) + len(shell.Command) + if n > MaxResponse || (old.ID == "" && len(s.changes) >= chatShellIdentityLimit) { + s.overflow = true + return + } + // A terminal event wins even if a duplicate create is delivered later. + if old.ID != "" && old.Status != protocol.ChatShellRunning { + return + } + s.changes[shell.ID] = *shell + s.bytes = n +} + +func (s *ShellSnapshot) Merge(shells []protocol.ChatShell) ([]protocol.ChatShell, error) { + if s.overflow { + return nil, fmt.Errorf("shell reconciliation capacity exceeded") + } + byID := make(map[string]protocol.ChatShell) + for _, shell := range shells { + if shell.ID != "" && shell.Status == protocol.ChatShellRunning { + byID[shell.ID] = shell + } + } + for id, shell := range s.changes { + if shell.Status == protocol.ChatShellRunning { + byID[id] = shell + } else { + delete(byID, id) + } + } + if len(byID) > chatShellIdentityLimit { + return nil, fmt.Errorf("shell reconciliation capacity exceeded") + } + ids := make([]string, 0, len(byID)) + for id := range byID { + ids = append(ids, id) + } + sort.Strings(ids) + out := make([]protocol.ChatShell, 0, len(ids)) + for _, id := range ids { + out = append(out, byID[id]) + } + return out, nil +} diff --git a/internal/opencode/chat_v2_test.go b/internal/opencode/chat_v2_test.go index 00cb2242..25fc7024 100644 --- a/internal/opencode/chat_v2_test.go +++ b/internal/opencode/chat_v2_test.go @@ -395,3 +395,45 @@ func TestOpenChatRejectsUnrunnableCLI(t *testing.T) { t.Fatal("a missing CLI must fail detection, not open an endpoint") } } + +// TestChatShellReadbackSeedsReplacementStream proves an authoritative shell +// readback survives a later stream replacement: shell.exited has no session id, +// so it must still clear the verified shell after reconnect. +func TestChatShellReadbackSeedsReplacementStream(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/shell": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[{"id":"sh_1","status":"running","command":"sleep 30","metadata":{"sessionID":"ses_1"}},{"id":"sh_other","status":"running","command":"sleep 30","metadata":{"sessionID":"ses_other"}}]}`)) + case "/api/event": + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte("data: {\"type\":\"shell.exited\",\"data\":{\"id\":\"sh_1\",\"status\":\"exited\",\"exit\":0}}\n\n")) + } + })) + defer srv.Close() + chat := NewChat(&Connection{Backend: "v2", URL: srv.URL}) + shells, err := chat.Shells(context.Background(), "ses_1") + if err != nil || len(shells) != 1 || shells[0].ID != "sh_1" { + t.Fatalf("Shells = %+v, %v", shells, err) + } + var got []protocol.ChatEventFrame + if err := chat.Stream(context.Background(), "ses_1", nil, func(frame protocol.ChatEventFrame) error { got = append(got, frame); return nil }); err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Shell == nil || got[0].Shell.ID != "sh_1" || got[0].Shell.Status != protocol.ChatShellExited { + t.Fatalf("exit after readback = %+v", got) + } +} + +func TestChatReleaseShellsBoundsDisconnectedState(t *testing.T) { + chat := NewChat(&Connection{}) + state := chat.streamState("ses_1") + state.markShell("sh_1") + chat.ReleaseShells("ses_1") + chat.shellMu.Lock() + _, retained := chat.shells["ses_1"] + chat.shellMu.Unlock() + if retained { + t.Fatal("released session shell state was retained") + } +} diff --git a/internal/server/chat.go b/internal/server/chat.go index 1601fbfc..1a26dfa6 100644 --- a/internal/server/chat.go +++ b/internal/server/chat.go @@ -32,9 +32,21 @@ type chatBroker interface { // Permissions lists the session's pending OpenCode approval requests so the // handler can push chat_permission frames. Permissions(ctx context.Context, sessionID string) ([]protocol.ChatPermissionFrame, error) + // Shells returns only shell operations whose ownership is authoritatively + // verified for sessionID. It is the reconnect/readback source. + Shells(ctx context.Context, sessionID string) ([]protocol.ChatShell, error) Reply(ctx context.Context, request protocol.ChatReplyRequest) error } +// chatShellReleaser is implemented by brokers that retain verified shell ids for +// attributing exit events. The hub calls it only after the last observer leaves +// a room; a later attach obtains a fresh authoritative snapshot. +// This is a bounded local-cache operation: it must not perform I/O or call the +// hub, which serializes it with room replacement under its mutex. +type chatShellReleaser interface { + ReleaseShells(sessionID string) +} + // operatorRegistry is the server-side operator-session seam. *pipeline.Store // satisfies it. It is a table of its own, separate from the pipeline stage // ownership (SessionOwner/run_session_owners) that settles a run: chat_sessions diff --git a/internal/server/chat_hub.go b/internal/server/chat_hub.go index 134e3fd6..9bd7e32e 100644 --- a/internal/server/chat_hub.go +++ b/internal/server/chat_hub.go @@ -133,6 +133,13 @@ func (h *chatHub) apply(frame protocol.ChatEventFrame) { } h.mu.Lock() defer h.mu.Unlock() + h.applyLocked(frame) +} + +func (h *chatHub) applyLocked(frame protocol.ChatEventFrame) { + if frame.SessionID == "" { + return + } turn := h.turnLocked(frame.SessionID) switch frame.Kind { case protocol.ChatEventStep: @@ -144,6 +151,9 @@ func (h *chatHub) apply(frame protocol.ChatEventFrame) { case protocol.ChatEventShell: // A shell frame carries no message id (it outlives the tool call that // started it), so it is folded before the message-id gate below. + if room := h.rooms[frame.SessionID]; room != nil { + room.shellReadback.Record(frame.Shell) + } turn.applyShell(frame.Shell) return } @@ -240,7 +250,7 @@ func (t *chatTurn) applyShell(shell *protocol.ChatShell) { if shell == nil || shell.ID == "" { return } - if shell.Status == protocol.ChatShellExited { + if shell.Status != protocol.ChatShellRunning { old, ok := t.shells[shell.ID] if !ok { return diff --git a/internal/server/chat_multiclient_test.go b/internal/server/chat_multiclient_test.go index 43d373a5..d6b3dde6 100644 --- a/internal/server/chat_multiclient_test.go +++ b/internal/server/chat_multiclient_test.go @@ -82,6 +82,8 @@ func (c *channelChat) Session(context.Context, string) (protocol.ChatSession, bo return protocol.ChatSession{}, false, nil } +func (c *channelChat) Shells(context.Context, string) ([]protocol.ChatShell, error) { return nil, nil } + func (c *channelChat) Permissions(context.Context, string) ([]protocol.ChatPermissionFrame, error) { c.mu.Lock() defer c.mu.Unlock() diff --git a/internal/server/chat_room.go b/internal/server/chat_room.go index 9ae2450c..a8d0ecc5 100644 --- a/internal/server/chat_room.go +++ b/internal/server/chat_room.go @@ -106,6 +106,11 @@ type chatQueuedPrompt struct { acked bool } +type chatObservation struct { + done chan struct{} + err error // published by closing done; belongs to this attempt, not the next +} + // chatRoom is one session's shared live channel. Every field is guarded by the // owning chatHub's mutex. type chatRoom struct { @@ -117,6 +122,11 @@ type chatRoom struct { queue []*chatQueuedPrompt running bool streaming bool + // observing is non-nil while one caller is attaching the shared broker + // stream. It prevents concurrent subscribers from opening duplicate global + // event streams before streaming can be published. + observing *chatObservation + shellReadback *opencode.ShellSnapshot // buffering holds the turn's frames until the prompting client's ack has been // written, so the ack always precedes the frames it acknowledges. It is set // only while the very first turn subscribes to the firehose. @@ -160,6 +170,81 @@ func (h *chatHub) roomLocked(sessionID string) *chatRoom { return room } +// ensureObserving starts the firehose even for a read-only subscriber, then +// reconciles retained shells from the endpoint's authoritative list. A bounded +// journal folds concurrent lifecycle events over that snapshot atomically. All +// observers share this reconciliation, with no broker I/O under the hub mutex. +func (h *chatHub) ensureObserving(room *chatRoom, broker chatBroker) error { + if broker == nil { + return errors.New("chat is not configured on this server") + } + h.mu.Lock() + if err := room.ctx.Err(); err != nil { + h.mu.Unlock() + return err + } + if waiting := room.observing; waiting != nil { + h.mu.Unlock() + select { + case <-waiting.done: + return waiting.err + case <-room.ctx.Done(): + return room.ctx.Err() + } + } + if room.streaming { + h.mu.Unlock() + return nil + } + room.observing = &chatObservation{done: make(chan struct{})} + waiting := room.observing + room.shellReadback = &opencode.ShellSnapshot{} + h.mu.Unlock() + var observeErr error + defer func() { + h.mu.Lock() + room.shellReadback = nil + waiting.err = observeErr + room.observing = nil + close(waiting.done) + h.mu.Unlock() + }() + if err := h.startStream(room, broker); err != nil { + observeErr = err + return err + } + shells, err := broker.Shells(room.ctx, room.sessionID) + h.mu.Lock() + if room.ctx.Err() != nil || h.rooms[room.sessionID] != room { + h.mu.Unlock() + observeErr = context.Canceled + return observeErr + } + if err == nil { + shells, err = room.shellReadback.Merge(shells) + } + if err != nil { + // The firehose remains useful when its authoritative shell readback is + // temporarily unavailable. Do not turn that read failure into a chat + // failure (or claim the last retained snapshot is current): clear the + // replay cache and tell attached clients that shell visibility is limited. + // Preserve events already folded during this read, but clear stale replay. + live, mergeErr := room.shellReadback.Merge(nil) + if mergeErr == nil { + h.replaceShellsLocked(room.sessionID, live) + } + h.mu.Unlock() + // Never log broker bodies/URLs: they may contain credentials or private output. + log.Printf("chat shell readback session=%s failed (%T)", room.sessionID, err) + h.fanout(room.sessionID, protocol.Message{Type: protocol.Error, Error: "background shell visibility is limited"}, "") + return nil + } + h.replaceShellsLocked(room.sessionID, shells) + room.shellReadback = nil + h.mu.Unlock() + return nil +} + // attach registers write as one client's subscription to sessionID's stream and // returns it, or the existing subscription when the same connection has already // attached. It is idempotent so a client may prompt and subscribe without @@ -209,6 +294,11 @@ func (h *chatHub) detach(sessionID string, sub *chatSub) { if len(room.subs) == 0 { delete(h.rooms, sessionID) room.cancel() + // ReleaseShells is a local identity-cache operation (no broker I/O). + // Complete it before a reconnect can acquire this session's new room. + if releaser, ok := room.broker.(chatShellReleaser); ok { + releaser.ReleaseShells(sessionID) + } } h.mu.Unlock() } @@ -265,9 +355,17 @@ func (h *chatHub) broadcastUser(frame protocol.ChatEventFrame, exceptClientID st // once it is written. A step-ended frame finishes the turn, letting the queue // advance. func (h *chatHub) emit(frame protocol.ChatEventFrame) { - h.apply(frame) + h.emitRoom(nil, frame) +} + +func (h *chatHub) emitRoom(source *chatRoom, frame protocol.ChatEventFrame) { h.mu.Lock() room := h.rooms[frame.SessionID] + if source != nil && (source != room || source.ctx.Err() != nil) { + h.mu.Unlock() + return + } + h.applyLocked(frame) buffered := false var targets []*chatSub if room != nil { @@ -283,6 +381,22 @@ func (h *chatHub) emit(frame protocol.ChatEventFrame) { if room != nil && frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepEnded { h.finishTurnLocked(room) } + // Shell replay and live lifecycle delivery must enqueue in the same order + // as folding. Otherwise an exit can overtake an older running replay. + if frame.Kind == protocol.ChatEventShell && !buffered { + msg := protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame} + var slow []*chatSub + for _, sub := range targets { + if !enqueueChatLocked(sub, msg) { + slow = append(slow, sub) + } + } + h.mu.Unlock() + for _, sub := range slow { + sub.close() + } + return + } h.mu.Unlock() if buffered { return @@ -293,6 +407,62 @@ func (h *chatHub) emit(frame protocol.ChatEventFrame) { } } +// enqueueChatLocked never blocks or calls detach under the hub mutex. +func enqueueChatLocked(sub *chatSub, msg protocol.Message) bool { + select { + case <-sub.done: + return true + default: + } + select { + case sub.ch <- msg: + return true + default: + return false + } +} + +func (h *chatHub) replayShells(sub *chatSub) { + h.mu.Lock() + slow := false + if h.rooms[sub.room.sessionID] == sub.room && sub.room.subs[sub.clientID] == sub { + turn := h.turnLocked(sub.room.sessionID) + for _, id := range turn.shellOrder { + shell := turn.shells[id] + frame := protocol.ChatEventFrame{SessionID: sub.room.sessionID, Kind: protocol.ChatEventShell, Shell: &shell} + if !enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}) { + slow = true + break + } + } + } + h.mu.Unlock() + if slow { + sub.close() + } +} + +// replaceShells reconciles a successful authoritative readback. It drops stale +// retained entries (including exits missed while disconnected) and upserts the +// verified running set without fabricating a lifecycle event for foreign shells. +func (h *chatHub) replaceShells(sessionID string, shells []protocol.ChatShell) { + h.mu.Lock() + defer h.mu.Unlock() + h.replaceShellsLocked(sessionID, shells) +} + +func (h *chatHub) replaceShellsLocked(sessionID string, shells []protocol.ChatShell) { + turn := h.turnLocked(sessionID) + for _, shell := range turn.shells { + turn.shrink(len(shell.Command)) + } + turn.shells = nil + turn.shellOrder = nil + for i := range shells { + turn.applyShell(&shells[i]) + } +} + // finishTurnLocked marks the room's turn finished and releases the turn runner's // wait. Callers must hold h.mu. func (h *chatHub) finishTurnLocked(room *chatRoom) { @@ -331,14 +501,20 @@ func (h *chatHub) flushBuffered(sessionID string) { targets = append(targets, sub) } } - h.mu.Unlock() + var slow []*chatSub for i := range pending { frame := pending[i] msg := protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame} for _, sub := range targets { - sub.send(msg) + if !enqueueChatLocked(sub, msg) { + slow = append(slow, sub) + } } } + h.mu.Unlock() + for _, sub := range slow { + sub.close() + } } // submitPrompt accepts one prompt: it starts the turn runner immediately when no @@ -476,7 +652,7 @@ func (h *chatHub) runTurn(room *chatRoom, p *chatQueuedPrompt, done chan struct{ return } if first { - if err := h.startStream(room, broker); err != nil { + if err := h.ensureObserving(room, broker); err != nil { h.failPrompt(room, p, err) h.endRoomTurn(room) return @@ -550,7 +726,7 @@ func (h *chatHub) startStream(room *chatRoom, broker chatBroker) error { result <- broker.Stream(ctx, room.sessionID, func() { once.Do(func() { close(attached) }) }, func(frame protocol.ChatEventFrame) error { - h.emit(frame) + h.emitRoom(room, frame) return nil }) }() diff --git a/internal/server/chat_shell_reconcile_test.go b/internal/server/chat_shell_reconcile_test.go new file mode 100644 index 00000000..9ada615b --- /dev/null +++ b/internal/server/chat_shell_reconcile_test.go @@ -0,0 +1,261 @@ +package server + +import ( + "context" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" + "github.com/coder/websocket" + "github.com/coder/websocket/wsjson" +) + +type shellBarrierBroker struct { + *fakeChat + read chan struct{} + apply chan struct{} + events chan shellBarrierEvent + stopped chan struct{} + reads atomic.Int32 + snapshot []protocol.ChatShell +} + +// Exercise the real websocket handler on a loopback scratch server and a fresh +// SQLite database. A marker on the shared stream bounds the replay assertion +// without sleeps, polling or a negative-read timeout. +func TestChatShellReconciliationScratchServer(t *testing.T) { + b := newShellBarrierBroker() + b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: b}, newOperatorStore(t))) + defer srv.Close() + conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil) + if err != nil { + t.Fatal(err) + } + defer conn.CloseNow() + subscribe := func(id string) { + t.Helper() + if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: id, ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil { + t.Fatal(err) + } + } + read := func() protocol.Message { + t.Helper() + var m protocol.Message + if err := wsjson.Read(ctx, conn, &m); err != nil { + t.Fatal(err) + } + return m + } + if m := read(); m.Type != protocol.PipelineList { + t.Fatalf("server greeting = %s", m.Type) + } + subscribe("first") + if m := read(); m.ID != "first" || m.Type != protocol.ChatSubscribe { + t.Fatalf("ack = %+v", m) + } + shellBarrierWait(t, b.read) + done := make(chan struct{}) + b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_old", Status: protocol.ChatShellExited}, done} + shellBarrierWait(t, done) + if m := read(); m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellExited { + t.Fatalf("exit not visible: %+v", m) + } + close(b.apply) + subscribe("second") + for { + m := read() + if m.ChatEvent != nil && m.ChatEvent.Shell.Status == protocol.ChatShellRunning { + t.Fatalf("stale replay: %+v", m) + } + if m.ID == "second" { + break + } + } + done = make(chan struct{}) + b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_marker", Status: protocol.ChatShellExited}, done} + shellBarrierWait(t, done) + for { + m := read() + if m.ChatEvent == nil { + continue + } + if m.ChatEvent.Shell.Status == protocol.ChatShellRunning { + t.Fatalf("ghost visible after apply: %+v", m) + } + if m.ChatEvent.Shell.ID == "sh_marker" { + break + } + } + if b.reads.Load() != 1 { + t.Fatalf("shared observer read count = %d", b.reads.Load()) + } + conn.CloseNow() + shellBarrierWait(t, b.stopped) +} + +type shellBarrierEvent struct { + shell protocol.ChatShell + done chan struct{} +} + +func newShellBarrierBroker() *shellBarrierBroker { + return &shellBarrierBroker{fakeChat: scriptedChat(), read: make(chan struct{}, 8), apply: make(chan struct{}), events: make(chan shellBarrierEvent), stopped: make(chan struct{})} +} + +func (b *shellBarrierBroker) Shells(ctx context.Context, _ string) ([]protocol.ChatShell, error) { + b.reads.Add(1) + snapshot := append([]protocol.ChatShell(nil), b.snapshot...) + b.read <- struct{}{} + select { + case <-b.apply: + return snapshot, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (b *shellBarrierBroker) Stream(ctx context.Context, id string, ready func(), emit func(protocol.ChatEventFrame) error) error { + defer close(b.stopped) + ready() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case event := <-b.events: + if err := emit(protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventShell, Shell: &event.shell}); err != nil { + return err + } + close(event.done) + } + } +} + +func shellBarrierWait(t *testing.T, ch <-chan struct{}) { + t.Helper() + select { + case <-ch: + case <-time.After(3 * time.Second): + t.Fatal("barrier timed out") + } +} + +func TestChatShellReconciliationBarrier(t *testing.T) { + for _, status := range []string{protocol.ChatShellExited, protocol.ChatShellRunning, "timeout", "killed"} { + t.Run(status, func(t *testing.T) { + b := newShellBarrierBroker() + b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}} + h := newChatHub() + messages := make(chan protocol.Message, 32) + sub := h.attach("ses_1", "one", "", b, func(m protocol.Message) error { messages <- m; return nil }) + t.Cleanup(sub.close) + result := make(chan error, 2) + go func() { result <- h.ensureObserving(sub.room, b) }() + shellBarrierWait(t, b.read) + // A second observer must share the in-flight read, not apply an + // independently completing older/newer snapshot out of order. + go func() { result <- h.ensureObserving(sub.room, b) }() + shell := protocol.ChatShell{ID: "sh_old", Status: status} + if status == protocol.ChatShellRunning { + shell.ID = "sh_new" + shell.Command = "sleep 10" + } + done := make(chan struct{}) + select { + case b.events <- shellBarrierEvent{shell, done}: + case <-time.After(time.Second): + t.Fatal("event blocked behind readback") + } + shellBarrierWait(t, done) + select { + case m := <-messages: + if m.ChatEvent == nil || m.ChatEvent.Shell == nil || m.ChatEvent.Shell.ID != shell.ID || m.ChatEvent.Shell.Status != status { + t.Fatalf("live shell = %+v", m) + } + case <-time.After(time.Second): + t.Fatal("live event not visible during read") + } + close(b.apply) + for range 2 { + select { + case err := <-result: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("observer stuck") + } + } + if b.reads.Load() != 1 { + t.Fatalf("redundant/out-of-order reads: %d", b.reads.Load()) + } + snapshot := h.shellSnapshot("ses_1") + if status == protocol.ChatShellRunning { + if len(snapshot) != 2 { + t.Fatalf("new create lost: %+v", snapshot) + } + } else if len(snapshot) != 0 { + t.Fatalf("terminal shell resurrected: %+v", snapshot) + } + // Assert the same retained state is replayed to the client. + h.replayShells(sub) + for range len(snapshot) { + select { + case m := <-messages: + if m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellRunning { + t.Fatalf("replay = %+v", m) + } + case <-time.After(time.Second): + t.Fatal("replay missing") + } + } + sub.close() + shellBarrierWait(t, b.stopped) + }) + } +} + +func TestChatShellReconciliationCancelReconnectBarrier(t *testing.T) { + b := newShellBarrierBroker() + h := newChatHub() + sub := h.attach("ses_1", "one", "", b, func(protocol.Message) error { return nil }) + results := make(chan error, 2) + go func() { results <- h.ensureObserving(sub.room, b) }() + shellBarrierWait(t, b.read) + go func() { results <- h.ensureObserving(sub.room, b) }() + sub.close() + for range 2 { + select { + case err := <-results: + if err == nil { + t.Fatal("canceled observer succeeded") + } + case <-time.After(time.Second): + t.Fatal("canceled waiter stuck") + } + } + shellBarrierWait(t, b.stopped) + next := newShellBarrierBroker() + next.snapshot = []protocol.ChatShell{{ID: "sh_new", Status: protocol.ChatShellRunning}} + close(next.apply) + newSub := h.attach("ses_1", "two", "", next, func(protocol.Message) error { return nil }) + t.Cleanup(newSub.close) + if err := h.ensureObserving(newSub.room, next); err != nil { + t.Fatal(err) + } + // A late callback from the canceled stream cannot poison the new room, + // nor can a foreign-session callback from the current stream. + h.emitRoom(sub.room, protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_ghost", Status: protocol.ChatShellRunning}}) + h.emitRoom(newSub.room, protocol.ChatEventFrame{SessionID: "ses_foreign", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_foreign", Status: protocol.ChatShellRunning}}) + got := h.shellSnapshot("ses_1") + if len(got) != 1 || got[0].ID != "sh_new" || len(h.shellSnapshot("ses_foreign")) != 0 { + t.Fatalf("reconnect state = %+v", got) + } + newSub.close() + shellBarrierWait(t, next.stopped) +} diff --git a/internal/server/chat_stream.go b/internal/server/chat_stream.go index 76c175e6..89ba6404 100644 --- a/internal/server/chat_stream.go +++ b/internal/server/chat_stream.go @@ -111,7 +111,14 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "could not attach to session " + query.SessionID}) return } + // Acknowledge before the stream is allowed to fan out: otherwise an immediately + // available retained shell can overtake the subscribe reply on this connection. _ = write(protocol.Message{Type: protocol.ChatSubscribe, ID: request.ID}) + if err := hub.ensureObserving(sub.room, broker); err != nil { + sub.close() + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "could not observe session " + query.SessionID}) + return + } // Replay every still-unresolved approval request the room holds, not only the // most recent one, so a client that attaches after two requests were raised // still sees both cards. @@ -124,9 +131,5 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH // shows them rather than nothing until they exit. The message snapshot cannot // carry a shell — a shell outlives the turn and is not a turn — so this is the // shell half of the resume. - for _, shell := range hub.shellSnapshot(query.SessionID) { - running := shell - frame := protocol.ChatEventFrame{SessionID: query.SessionID, Kind: protocol.ChatEventShell, Shell: &running} - sub.send(protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}) - } + hub.replayShells(sub) } diff --git a/internal/server/chat_test.go b/internal/server/chat_test.go index 29dbc4bb..d4a15cd2 100644 --- a/internal/server/chat_test.go +++ b/internal/server/chat_test.go @@ -3,6 +3,7 @@ package server import ( "context" "encoding/json" + "errors" "fmt" "net/http" "net/http/httptest" @@ -52,6 +53,8 @@ type fakeChat struct { permissionSessionID string permissions []protocol.ChatPermissionFrame permissionErr error + shells []protocol.ChatShell + shellErr error sessionCalls int sessions []protocol.ChatSession sessionErr error @@ -62,6 +65,8 @@ type fakeChat struct { started chan struct{} done chan struct{} startOnce sync.Once + streamCalls int + readyGate <-chan struct{} block bool subscribed bool promptBeforeSubscribe bool @@ -91,13 +96,18 @@ func (f *fakeChat) Prompt(context.Context, string, string) error { func (f *fakeChat) Stream(ctx context.Context, _ string, ready func(), emit func(protocol.ChatEventFrame) error) error { f.mu.Lock() f.subscribed = true + f.streamCalls++ + readyGate := f.readyGate f.mu.Unlock() - if ready != nil { - ready() - } if f.started != nil { f.startOnce.Do(func() { close(f.started) }) } + if readyGate != nil { + <-readyGate + } + if ready != nil { + ready() + } if f.block { <-ctx.Done() if f.done != nil { @@ -116,6 +126,42 @@ func (f *fakeChat) Stream(ctx context.Context, _ string, ready func(), emit func return nil } +// TestChatHubConcurrentObserversShareStream pins the start-up handoff: two +// clients that subscribe before the broker says ready still share one firehose. +func TestChatHubConcurrentObserversShareStream(t *testing.T) { + broker := scriptedChat() + broker.block = true + broker.started = make(chan struct{}) + gate := make(chan struct{}) + broker.readyGate = gate + hub := newChatHub() + hub.mu.Lock() + room := hub.roomLocked("ses_1") + hub.mu.Unlock() + + errs := make(chan error, 2) + go func() { errs <- hub.ensureObserving(room, broker) }() + select { + case <-broker.started: + case <-time.After(time.Second): + t.Fatal("first observer did not start the broker stream") + } + go func() { errs <- hub.ensureObserving(room, broker) }() + close(gate) + for range 2 { + if err := <-errs; err != nil { + t.Fatalf("ensureObserving: %v", err) + } + } + broker.mu.Lock() + calls := broker.streamCalls + broker.mu.Unlock() + if calls != 1 { + t.Fatalf("broker Stream calls = %d, want 1", calls) + } + room.cancel() +} + func (f *fakeChat) Interrupt(context.Context, string) error { f.mu.Lock() defer f.mu.Unlock() @@ -151,6 +197,12 @@ func (f *fakeChat) Reply(_ context.Context, request protocol.ChatReplyRequest) e // Permissions returns the seeded pending requests, or blocks until the caller's // context is canceled when permBlock is set (used to prove the poller stops on // disconnect). Every call is counted so a test can assert polling happened. +func (f *fakeChat) Shells(context.Context, string) ([]protocol.ChatShell, error) { + f.mu.Lock() + defer f.mu.Unlock() + return append([]protocol.ChatShell(nil), f.shells...), f.shellErr +} + func (f *fakeChat) Permissions(ctx context.Context, sessionID string) ([]protocol.ChatPermissionFrame, error) { f.mu.Lock() f.permissionCalls++ @@ -682,6 +734,9 @@ func TestChatInterruptEndsRunningPrompt(t *testing.T) { case r.Method == http.MethodGet && r.URL.Path == "/api/permission/request": w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"location":{"directory":"/work"},"data":[]}`)) + case r.Method == http.MethodGet && r.URL.Path == "/api/shell": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/event": flusher, ok := w.(http.Flusher) if !ok { @@ -995,6 +1050,7 @@ func TestChatPromptStreamsShellFrame(t *testing.T) { // shows the background process instead of nothing until it exits. func TestChatSubscribeReplaysRunningShell(t *testing.T) { broker := scriptedChat() + broker.shells = []protocol.ChatShell{{ID: "sh_1", Status: protocol.ChatShellRunning, Command: "sleep 120"}} broker.script = append([]protocol.ChatEventFrame{ {SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_1", Status: protocol.ChatShellRunning, Command: "sleep 120"}}, }, broker.script...) @@ -1039,6 +1095,40 @@ func TestChatSubscribeReplaysRunningShell(t *testing.T) { } } +// TestChatSubscribeKeepsStreamWhenShellReadbackFails ensures a temporary +// authoritative readback failure does not make chat unavailable or replay a +// stale process as running. The subscriber receives an explicit limitation +// while its broker stream remains attached for future lifecycle events. +func TestChatSubscribeKeepsStreamWhenShellReadbackFails(t *testing.T) { + broker := scriptedChat() + broker.block = true + broker.shellErr = errors.New("shell endpoint unavailable: https://user:secret@example.invalid private response body") + ctx, _, conn := dialChat(t, broker) + if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub1", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil { + t.Fatal(err) + } + var ack protocol.Message + if err := wsjson.Read(ctx, conn, &ack); err != nil { + t.Fatal(err) + } + if ack.Type != protocol.ChatSubscribe || ack.ID != "sub1" { + t.Fatalf("subscribe ack = %+v", ack) + } + var limited protocol.Message + if err := wsjson.Read(ctx, conn, &limited); err != nil { + t.Fatal(err) + } + if limited.Type != protocol.Error || limited.Error != "background shell visibility is limited" { + t.Fatalf("limited visibility notice = %+v", limited) + } + broker.mu.Lock() + calls := broker.streamCalls + broker.mu.Unlock() + if calls != 1 { + t.Fatalf("broker Stream calls = %d, want 1", calls) + } +} + // TestChatPromptSubscribesBeforePrompt pins the subscribe-before-prompt order: // the firehose has no replay cursor, so the subscription must be attached before // the prompt is posted or deltas emitted in between are lost. @@ -1079,6 +1169,9 @@ func realAdapterChat(t *testing.T, frames []string) []protocol.ChatEventFrame { case r.Method == http.MethodGet && r.URL.Path == "/api/permission/request": w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"location":{"directory":"/work"},"data":[]}`)) + case r.Method == http.MethodGet && r.URL.Path == "/api/shell": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/event": streams.Add(1) w.Header().Set("Content-Type", "text/event-stream")