Merge background shell visibility reconciliation
This commit is contained in:
@@ -0,0 +1,37 @@
|
||||
# Background shell visibility
|
||||
|
||||
The chat server now opens the broker stream for read-only subscribers and reconciles verified running shell operations from OpenCode's authoritative `/api/shell` readback. The packaged OpenCode skill references and chat documentation should describe that shell visibility survives reconnects only when OpenCode reports explicit `metadata.sessionID` ownership; shared working directories are not ownership evidence.
|
||||
|
||||
## Current source behavior
|
||||
|
||||
Observers share one initial readback per room/stream attachment. The adapter and hub
|
||||
fold bounded lifecycle journals over each snapshot, so a stale read cannot resurrect
|
||||
an exited shell or erase a concurrent create. Shell replay and live delivery enqueue
|
||||
in fold order. Cancellation rejects old-room callbacks and releases local identity
|
||||
state before reconnect. Journals retain at most 256 identities and 1 MiB of id/command
|
||||
text; capacity/read failures surface stable limited-visibility text, never broker
|
||||
response bodies. No shell polling was added. OpenCode's terminal `timeout` and
|
||||
`killed` statuses are excluded from retained running operations as well as `exited`.
|
||||
|
||||
## Verification evidence — 2026-10-18
|
||||
|
||||
`opencode api get /openapi.json` authenticated through supported service discovery.
|
||||
The live schema advertises shell list/detail/output and generic `metadata`; it does
|
||||
not type `metadata.sessionID`. A throwaway session
|
||||
`ses_eb12ab7b3ffe0ZAle7PHpftHMG` and its own `sleep 1` shell
|
||||
`sh_14ed57134001e6fuYqvn62pKit` verified actual create/detail readback preserving
|
||||
explicit `metadata.sessionID`, status `running`, subsequent `exited`/exit 0, and
|
||||
absence from the running list. This verifies explicit API metadata, not automatic
|
||||
agent-tool ownership injection. The CLI event attempt yielded no captured events;
|
||||
live-provider event ordering is **not tested**. Forced ordering evidence comes from
|
||||
channel/barrier regressions, including a real WebSocket handler on a loopback scratch
|
||||
server with fresh SQLite, not from the deployed service.
|
||||
|
||||
## Documentation follow-up / deployment
|
||||
|
||||
`docs/chat-verification.md` and the chat subscribe/replay section of `docs/cli.md`
|
||||
are stale: docs-fold should describe authoritative ownership/readback, coalesced
|
||||
observation, bounded reconciliation and limited-visibility errors, distinguishing
|
||||
fixture ordering evidence from the live readback above. Shared docs and packaged
|
||||
skill references were not edited here. No deployment, installation, service restart
|
||||
or physical-phone/UI verification was performed; this changes no rendered layout.
|
||||
+198
-11
@@ -28,7 +28,9 @@ var ErrPermissionUnsupported = errors.New("OpenCode permission approval is not i
|
||||
// semantics, the session-id shape and the 64 KiB / 1 MiB bounds of Connection so
|
||||
// no second transport or credential path is introduced.
|
||||
type Chat struct {
|
||||
conn *Connection
|
||||
conn *Connection
|
||||
shellMu sync.Mutex
|
||||
shells map[string]*ChatStreamState
|
||||
// mu serializes the adapter's HTTP calls. Connection records its last
|
||||
// operation and status on the shared connection, so two calls must not
|
||||
// overlap; the permission poller runs alongside the read loop's
|
||||
@@ -294,6 +296,114 @@ func decodeChatPermissions(sessionID string, data []byte) ([]protocol.ChatPermis
|
||||
return frames, nil
|
||||
}
|
||||
|
||||
// Shells reads the endpoint's authoritative shell list and returns only running
|
||||
// operations whose metadata explicitly names sessionID. The shell API is global;
|
||||
// filtering locally avoids guessing an unsupported query filter and never assigns
|
||||
// ownership from a shared working directory. A successful read also seeds the
|
||||
// stream state used to attribute later shell.exited events, which carry no session.
|
||||
func (c *Chat) Shells(ctx context.Context, sessionID string) ([]protocol.ChatShell, error) {
|
||||
if !ValidSessionID(sessionID) {
|
||||
return nil, fmt.Errorf("invalid OpenCode session id")
|
||||
}
|
||||
state, err := c.streamStateContext(ctx, sessionID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// One read per state; waiting and I/O never hold the event-state mutex.
|
||||
state.mu.Lock()
|
||||
if state.readGate == nil {
|
||||
state.readGate = make(chan struct{}, 1)
|
||||
}
|
||||
gate := state.readGate
|
||||
state.mu.Unlock()
|
||||
select {
|
||||
case gate <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
defer func() { <-gate }()
|
||||
state.mu.Lock()
|
||||
state.readback = &ShellSnapshot{}
|
||||
state.mu.Unlock()
|
||||
defer func() { state.mu.Lock(); state.readback = nil; state.mu.Unlock() }()
|
||||
data, err := c.call(ctx, "get", "/api/shell", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var response struct {
|
||||
Data []chatShellInfo `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &response); err != nil || response.Data == nil {
|
||||
if err := json.Unmarshal(data, &response.Data); err != nil {
|
||||
return nil, fmt.Errorf("invalid OpenCode shell response")
|
||||
}
|
||||
}
|
||||
out := make([]protocol.ChatShell, 0, len(response.Data))
|
||||
for _, info := range response.Data {
|
||||
if info.ID == "" || info.shellSessionID() != sessionID || (info.Status != "" && info.Status != protocol.ChatShellRunning) {
|
||||
continue
|
||||
}
|
||||
status := info.Status
|
||||
if status == "" {
|
||||
status = protocol.ChatShellRunning
|
||||
}
|
||||
out = append(out, protocol.ChatShell{ID: info.ID, Status: status, Command: info.Command})
|
||||
}
|
||||
state.mu.Lock()
|
||||
defer state.mu.Unlock()
|
||||
out, err = state.readback.Merge(out)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids := make([]string, 0, len(out))
|
||||
for _, shell := range out {
|
||||
ids = append(ids, shell.ID)
|
||||
}
|
||||
state.replaceShellsLocked(ids)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// streamState returns the session's retained, verified shell identities. It is
|
||||
// deliberately kept across stream replacement so an exit after reconnect remains
|
||||
// attributable; Shells reconciles its contents from authoritative readback on
|
||||
// each attach rather than replacing the pointer a live scanner is using.
|
||||
func (c *Chat) streamState(sessionID string) *ChatStreamState {
|
||||
state, _ := c.streamStateContext(context.Background(), sessionID)
|
||||
return state
|
||||
}
|
||||
|
||||
func (c *Chat) streamStateContext(ctx context.Context, sessionID string) (*ChatStreamState, error) {
|
||||
c.shellMu.Lock()
|
||||
defer c.shellMu.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.shells == nil {
|
||||
c.shells = map[string]*ChatStreamState{}
|
||||
}
|
||||
state := c.shells[sessionID]
|
||||
if state == nil {
|
||||
state = &ChatStreamState{}
|
||||
c.shells[sessionID] = state
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
// ReleaseShells discards verified shell identities after the last room observer
|
||||
// disconnects. A future attach calls Shells for a fresh authoritative snapshot,
|
||||
// so this bounds state without attributing a foreign shell during the gap.
|
||||
func (c *Chat) ReleaseShells(sessionID string) {
|
||||
if sessionID == "" {
|
||||
return
|
||||
}
|
||||
c.shellMu.Lock()
|
||||
delete(c.shells, sessionID)
|
||||
c.shellMu.Unlock()
|
||||
}
|
||||
|
||||
// Stream subscribes to the OpenCode event firehose and invokes emit for every
|
||||
// frame belonging to sessionID. It reads one bounded SSE frame at a time and
|
||||
// never polls the per-session message log; the global firehose is the only
|
||||
@@ -306,6 +416,12 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr
|
||||
if !sessionID.MatchString(id) {
|
||||
return fmt.Errorf("invalid OpenCode session id")
|
||||
}
|
||||
// Pin state before opening/announcing the stream so a canceled scanner
|
||||
// cannot acquire a replacement room's identity map.
|
||||
state, err := c.streamStateContext(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := c.conn.EventStream(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -314,7 +430,7 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr
|
||||
if ready != nil {
|
||||
ready()
|
||||
}
|
||||
return scanChatEvents(ctx, body, id, emit)
|
||||
return c.scanChatEventsWithState(ctx, body, id, state, emit)
|
||||
}
|
||||
|
||||
// scanChatEvents turns an SSE body into protocol frames, one bounded frame at a
|
||||
@@ -322,10 +438,13 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr
|
||||
// malformed JSON fails the stream instead of producing a malformed frame. The
|
||||
// last assistant message id is retained so a later tool/step event that omits it
|
||||
// still attributes every frame to a stable message.
|
||||
func scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit func(protocol.ChatEventFrame) error) error {
|
||||
func (c *Chat) scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit func(protocol.ChatEventFrame) error) error {
|
||||
return c.scanChatEventsWithState(ctx, body, sessionID, c.streamState(sessionID), emit)
|
||||
}
|
||||
|
||||
func (c *Chat) scanChatEventsWithState(ctx context.Context, body io.Reader, sessionID string, state *ChatStreamState, emit func(protocol.ChatEventFrame) error) error {
|
||||
scanner := bufio.NewScanner(body)
|
||||
scanner.Buffer(make([]byte, 0, 64<<10), MaxResponse)
|
||||
state := &ChatStreamState{}
|
||||
for scanner.Scan() {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
@@ -346,7 +465,7 @@ func scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit
|
||||
continue
|
||||
}
|
||||
if frame.MessageID != "" {
|
||||
state.LastMessageID = frame.MessageID
|
||||
state.setLastMessageID(frame.MessageID)
|
||||
}
|
||||
if err := emit(frame); err != nil {
|
||||
return err
|
||||
@@ -449,8 +568,29 @@ func (i *chatShellInfo) shellSessionID() string {
|
||||
// what attributes the exit to its session: another session's shell is an
|
||||
// unknown id and is ignored.
|
||||
type ChatStreamState struct {
|
||||
mu sync.Mutex
|
||||
LastMessageID string
|
||||
Shells map[string]struct{}
|
||||
readGate chan struct{}
|
||||
readback *ShellSnapshot
|
||||
}
|
||||
|
||||
func (s *ChatStreamState) lastMessageID() string {
|
||||
if s == nil {
|
||||
return ""
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.LastMessageID
|
||||
}
|
||||
|
||||
func (s *ChatStreamState) setLastMessageID(id string) {
|
||||
if s == nil || id == "" {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.LastMessageID = id
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// knowsShell reports whether id is a shell this stream has already surfaced for
|
||||
@@ -459,6 +599,8 @@ func (s *ChatStreamState) knowsShell(id string) bool {
|
||||
if s == nil || id == "" {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
_, ok := s.Shells[id]
|
||||
return ok
|
||||
}
|
||||
@@ -468,6 +610,8 @@ func (s *ChatStreamState) markShell(id string) {
|
||||
if s == nil || id == "" {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.Shells == nil {
|
||||
s.Shells = map[string]struct{}{}
|
||||
}
|
||||
@@ -476,12 +620,37 @@ func (s *ChatStreamState) markShell(id string) {
|
||||
|
||||
// forgetShell drops id once its exit has been surfaced.
|
||||
func (s *ChatStreamState) forgetShell(id string) {
|
||||
if s == nil || s.Shells == nil {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.Shells == nil {
|
||||
return
|
||||
}
|
||||
delete(s.Shells, id)
|
||||
}
|
||||
|
||||
// replaceShells applies one authoritative running-shell snapshot without
|
||||
// replacing the state object an already attached event scanner owns.
|
||||
func (s *ChatStreamState) replaceShells(ids []string) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.replaceShellsLocked(ids)
|
||||
}
|
||||
|
||||
func (s *ChatStreamState) replaceShellsLocked(ids []string) {
|
||||
s.Shells = make(map[string]struct{}, len(ids))
|
||||
for _, id := range ids {
|
||||
if id != "" {
|
||||
s.Shells[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NormalizeChatEvent maps one OpenCode firehose `data:` payload to a
|
||||
// protocol.ChatEventFrame, folding the small per-session context ([stream]) it
|
||||
// needs across frames. ok is false for an event type this adapter does not
|
||||
@@ -528,7 +697,7 @@ func NormalizeChatEvent(payload []byte, sessionID string, stream *ChatStreamStat
|
||||
messageID = data.MessageIDAlt
|
||||
}
|
||||
if messageID == "" && stream != nil {
|
||||
messageID = stream.LastMessageID
|
||||
messageID = stream.lastMessageID()
|
||||
}
|
||||
// Every pushed frame must carry a stable message id so the client can
|
||||
// attribute it; without one this is not a chat frame.
|
||||
@@ -590,20 +759,38 @@ func normalizeShellEvent(eventType string, data chatEvent, sessionID string, str
|
||||
if status == "" {
|
||||
status = protocol.ChatShellRunning
|
||||
}
|
||||
stream.markShell(data.Info.ID)
|
||||
shell := &protocol.ChatShell{ID: data.Info.ID, Status: status, Command: data.Info.Command}
|
||||
if stream != nil {
|
||||
stream.mu.Lock()
|
||||
stream.readback.Record(shell)
|
||||
if stream.Shells == nil {
|
||||
stream.Shells = map[string]struct{}{}
|
||||
}
|
||||
if status == protocol.ChatShellRunning && len(stream.Shells) < chatShellIdentityLimit {
|
||||
stream.Shells[shell.ID] = struct{}{}
|
||||
}
|
||||
stream.mu.Unlock()
|
||||
}
|
||||
return protocol.ChatEventFrame{
|
||||
SessionID: sessionID,
|
||||
Kind: protocol.ChatEventShell,
|
||||
Shell: &protocol.ChatShell{ID: data.Info.ID, Status: status, Command: data.Info.Command},
|
||||
Shell: shell,
|
||||
}, true, nil
|
||||
}
|
||||
if data.ID == "" {
|
||||
return protocol.ChatEventFrame{}, false, fmt.Errorf("malformed OpenCode shell.exited event")
|
||||
}
|
||||
if !stream.knowsShell(data.ID) {
|
||||
if stream == nil {
|
||||
return protocol.ChatEventFrame{}, false, nil
|
||||
}
|
||||
stream.mu.Lock()
|
||||
stream.readback.Record(&protocol.ChatShell{ID: data.ID, Status: protocol.ChatShellExited})
|
||||
_, known := stream.Shells[data.ID]
|
||||
delete(stream.Shells, data.ID)
|
||||
stream.mu.Unlock()
|
||||
if !known {
|
||||
return protocol.ChatEventFrame{}, false, nil
|
||||
}
|
||||
stream.forgetShell(data.ID)
|
||||
status := data.Status
|
||||
if status == "" {
|
||||
status = protocol.ChatShellExited
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
package opencode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
func TestChatShellReadbackBarrier(t *testing.T) {
|
||||
for _, known := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint(known), func(t *testing.T) {
|
||||
read, apply := make(chan struct{}), make(chan struct{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
close(read)
|
||||
select {
|
||||
case <-apply:
|
||||
fmt.Fprint(w, `{"data":[{"id":"sh_old","status":"running","metadata":{"sessionID":"ses_1"}},{"id":"sh_foreign","status":"running","metadata":{"sessionID":"ses_other"}}]}`)
|
||||
case <-r.Context().Done():
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
c := NewChat(&Connection{Backend: "v2", URL: srv.URL})
|
||||
state := c.streamState("ses_1")
|
||||
if known {
|
||||
state.markShell("sh_old")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
result := make(chan []protocol.ChatShell, 1)
|
||||
errs := make(chan error, 1)
|
||||
go func() { shells, err := c.Shells(ctx, "ses_1"); result <- shells; errs <- err }()
|
||||
select {
|
||||
case <-read:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("read did not reach barrier")
|
||||
}
|
||||
// Both a previously verified exit and an exit first learned while
|
||||
// reading must suppress the stale running snapshot.
|
||||
frame, ok, err := NormalizeChatEvent([]byte(`{"type":"shell.exited","data":{"id":"sh_old","status":"exited","exit":0}}`), "ses_1", state)
|
||||
if err != nil || ok != known || (ok && frame.Shell.Status != protocol.ChatShellExited) {
|
||||
t.Fatalf("exit = %+v %v %v", frame, ok, err)
|
||||
}
|
||||
frame, ok, err = NormalizeChatEvent([]byte(`{"type":"shell.created","data":{"info":{"id":"sh_new","status":"running","command":"sleep 10","metadata":{"sessionID":"ses_1"}}}}`), "ses_1", state)
|
||||
if err != nil || !ok || frame.Shell.ID != "sh_new" {
|
||||
t.Fatalf("create = %+v %v %v", frame, ok, err)
|
||||
}
|
||||
close(apply)
|
||||
got := <-result
|
||||
if err := <-errs; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].ID != "sh_new" || got[0].Command != "sleep 10" {
|
||||
t.Fatalf("readback lost events/leaked ownership: %+v", got)
|
||||
}
|
||||
if state.knowsShell("sh_old") || state.knowsShell("sh_foreign") || !state.knowsShell("sh_new") {
|
||||
t.Fatal("adapter identity state regressed")
|
||||
}
|
||||
// User-visible subsequent exit remains attributable after reconciliation.
|
||||
frame, ok, err = NormalizeChatEvent([]byte(`{"type":"shell.exited","data":{"id":"sh_new","status":"exited","exit":0}}`), "ses_1", state)
|
||||
if err != nil || !ok || frame.Shell.ID != "sh_new" {
|
||||
t.Fatalf("new shell exit lost: %+v %v %v", frame, ok, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatShellReadbackCancelWaiter(t *testing.T) {
|
||||
read := make(chan struct{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { close(read); <-r.Context().Done() }))
|
||||
defer srv.Close()
|
||||
c := NewChat(&Connection{Backend: "v2", URL: srv.URL})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
result := make(chan error, 2)
|
||||
go func() { _, err := c.Shells(ctx, "ses_1"); result <- err }()
|
||||
select {
|
||||
case <-read:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("read stuck")
|
||||
}
|
||||
go func() { _, err := c.Shells(ctx, "ses_1"); result <- err }()
|
||||
cancel()
|
||||
for range 2 {
|
||||
select {
|
||||
case err := <-result:
|
||||
if err == nil {
|
||||
t.Fatal("canceled read succeeded")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("read or waiter stuck")
|
||||
}
|
||||
}
|
||||
old := c.streamState("ses_1")
|
||||
c.ReleaseShells("ses_1")
|
||||
if c.streamState("ses_1") == old {
|
||||
t.Fatal("reconnect reused released state")
|
||||
}
|
||||
}
|
||||
|
||||
func TestShellSnapshotBounded(t *testing.T) {
|
||||
var snapshot ShellSnapshot
|
||||
for i := 0; i <= chatShellIdentityLimit; i++ {
|
||||
snapshot.Record(&protocol.ChatShell{ID: fmt.Sprintf("sh_%d", i), Status: protocol.ChatShellExited})
|
||||
}
|
||||
if len(snapshot.changes) != chatShellIdentityLimit {
|
||||
t.Fatalf("unbounded journal: %d", len(snapshot.changes))
|
||||
}
|
||||
if _, err := snapshot.Merge([]protocol.ChatShell{{ID: "sh_0", Status: protocol.ChatShellRunning}}); err == nil {
|
||||
t.Fatal("overflow accepted stale snapshot")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package opencode
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// ShellSnapshot records changes while an authoritative read is in flight. Its
|
||||
// owner serializes access. Unknown exits may be recorded, but can only remove an
|
||||
// id: they never establish ownership. Overflow rejects the snapshot, not events.
|
||||
type ShellSnapshot struct {
|
||||
changes map[string]protocol.ChatShell
|
||||
bytes int
|
||||
overflow bool
|
||||
}
|
||||
|
||||
const chatShellIdentityLimit = 256
|
||||
|
||||
func (s *ShellSnapshot) Record(shell *protocol.ChatShell) {
|
||||
if s == nil || shell == nil || shell.ID == "" || s.overflow {
|
||||
return
|
||||
}
|
||||
if s.changes == nil {
|
||||
s.changes = make(map[string]protocol.ChatShell)
|
||||
}
|
||||
old := s.changes[shell.ID]
|
||||
n := s.bytes - len(old.ID) - len(old.Command) + len(shell.ID) + len(shell.Command)
|
||||
if n > MaxResponse || (old.ID == "" && len(s.changes) >= chatShellIdentityLimit) {
|
||||
s.overflow = true
|
||||
return
|
||||
}
|
||||
// A terminal event wins even if a duplicate create is delivered later.
|
||||
if old.ID != "" && old.Status != protocol.ChatShellRunning {
|
||||
return
|
||||
}
|
||||
s.changes[shell.ID] = *shell
|
||||
s.bytes = n
|
||||
}
|
||||
|
||||
func (s *ShellSnapshot) Merge(shells []protocol.ChatShell) ([]protocol.ChatShell, error) {
|
||||
if s.overflow {
|
||||
return nil, fmt.Errorf("shell reconciliation capacity exceeded")
|
||||
}
|
||||
byID := make(map[string]protocol.ChatShell)
|
||||
for _, shell := range shells {
|
||||
if shell.ID != "" && shell.Status == protocol.ChatShellRunning {
|
||||
byID[shell.ID] = shell
|
||||
}
|
||||
}
|
||||
for id, shell := range s.changes {
|
||||
if shell.Status == protocol.ChatShellRunning {
|
||||
byID[id] = shell
|
||||
} else {
|
||||
delete(byID, id)
|
||||
}
|
||||
}
|
||||
if len(byID) > chatShellIdentityLimit {
|
||||
return nil, fmt.Errorf("shell reconciliation capacity exceeded")
|
||||
}
|
||||
ids := make([]string, 0, len(byID))
|
||||
for id := range byID {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
out := make([]protocol.ChatShell, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
out = append(out, byID[id])
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -395,3 +395,45 @@ func TestOpenChatRejectsUnrunnableCLI(t *testing.T) {
|
||||
t.Fatal("a missing CLI must fail detection, not open an endpoint")
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatShellReadbackSeedsReplacementStream proves an authoritative shell
|
||||
// readback survives a later stream replacement: shell.exited has no session id,
|
||||
// so it must still clear the verified shell after reconnect.
|
||||
func TestChatShellReadbackSeedsReplacementStream(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api/shell":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data":[{"id":"sh_1","status":"running","command":"sleep 30","metadata":{"sessionID":"ses_1"}},{"id":"sh_other","status":"running","command":"sleep 30","metadata":{"sessionID":"ses_other"}}]}`))
|
||||
case "/api/event":
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
_, _ = w.Write([]byte("data: {\"type\":\"shell.exited\",\"data\":{\"id\":\"sh_1\",\"status\":\"exited\",\"exit\":0}}\n\n"))
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
chat := NewChat(&Connection{Backend: "v2", URL: srv.URL})
|
||||
shells, err := chat.Shells(context.Background(), "ses_1")
|
||||
if err != nil || len(shells) != 1 || shells[0].ID != "sh_1" {
|
||||
t.Fatalf("Shells = %+v, %v", shells, err)
|
||||
}
|
||||
var got []protocol.ChatEventFrame
|
||||
if err := chat.Stream(context.Background(), "ses_1", nil, func(frame protocol.ChatEventFrame) error { got = append(got, frame); return nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Shell == nil || got[0].Shell.ID != "sh_1" || got[0].Shell.Status != protocol.ChatShellExited {
|
||||
t.Fatalf("exit after readback = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatReleaseShellsBoundsDisconnectedState(t *testing.T) {
|
||||
chat := NewChat(&Connection{})
|
||||
state := chat.streamState("ses_1")
|
||||
state.markShell("sh_1")
|
||||
chat.ReleaseShells("ses_1")
|
||||
chat.shellMu.Lock()
|
||||
_, retained := chat.shells["ses_1"]
|
||||
chat.shellMu.Unlock()
|
||||
if retained {
|
||||
t.Fatal("released session shell state was retained")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,9 +32,21 @@ type chatBroker interface {
|
||||
// Permissions lists the session's pending OpenCode approval requests so the
|
||||
// handler can push chat_permission frames.
|
||||
Permissions(ctx context.Context, sessionID string) ([]protocol.ChatPermissionFrame, error)
|
||||
// Shells returns only shell operations whose ownership is authoritatively
|
||||
// verified for sessionID. It is the reconnect/readback source.
|
||||
Shells(ctx context.Context, sessionID string) ([]protocol.ChatShell, error)
|
||||
Reply(ctx context.Context, request protocol.ChatReplyRequest) error
|
||||
}
|
||||
|
||||
// chatShellReleaser is implemented by brokers that retain verified shell ids for
|
||||
// attributing exit events. The hub calls it only after the last observer leaves
|
||||
// a room; a later attach obtains a fresh authoritative snapshot.
|
||||
// This is a bounded local-cache operation: it must not perform I/O or call the
|
||||
// hub, which serializes it with room replacement under its mutex.
|
||||
type chatShellReleaser interface {
|
||||
ReleaseShells(sessionID string)
|
||||
}
|
||||
|
||||
// operatorRegistry is the server-side operator-session seam. *pipeline.Store
|
||||
// satisfies it. It is a table of its own, separate from the pipeline stage
|
||||
// ownership (SessionOwner/run_session_owners) that settles a run: chat_sessions
|
||||
|
||||
@@ -133,6 +133,13 @@ func (h *chatHub) apply(frame protocol.ChatEventFrame) {
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.applyLocked(frame)
|
||||
}
|
||||
|
||||
func (h *chatHub) applyLocked(frame protocol.ChatEventFrame) {
|
||||
if frame.SessionID == "" {
|
||||
return
|
||||
}
|
||||
turn := h.turnLocked(frame.SessionID)
|
||||
switch frame.Kind {
|
||||
case protocol.ChatEventStep:
|
||||
@@ -144,6 +151,9 @@ func (h *chatHub) apply(frame protocol.ChatEventFrame) {
|
||||
case protocol.ChatEventShell:
|
||||
// A shell frame carries no message id (it outlives the tool call that
|
||||
// started it), so it is folded before the message-id gate below.
|
||||
if room := h.rooms[frame.SessionID]; room != nil {
|
||||
room.shellReadback.Record(frame.Shell)
|
||||
}
|
||||
turn.applyShell(frame.Shell)
|
||||
return
|
||||
}
|
||||
@@ -240,7 +250,7 @@ func (t *chatTurn) applyShell(shell *protocol.ChatShell) {
|
||||
if shell == nil || shell.ID == "" {
|
||||
return
|
||||
}
|
||||
if shell.Status == protocol.ChatShellExited {
|
||||
if shell.Status != protocol.ChatShellRunning {
|
||||
old, ok := t.shells[shell.ID]
|
||||
if !ok {
|
||||
return
|
||||
|
||||
@@ -82,6 +82,8 @@ func (c *channelChat) Session(context.Context, string) (protocol.ChatSession, bo
|
||||
return protocol.ChatSession{}, false, nil
|
||||
}
|
||||
|
||||
func (c *channelChat) Shells(context.Context, string) ([]protocol.ChatShell, error) { return nil, nil }
|
||||
|
||||
func (c *channelChat) Permissions(context.Context, string) ([]protocol.ChatPermissionFrame, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
@@ -106,6 +106,11 @@ type chatQueuedPrompt struct {
|
||||
acked bool
|
||||
}
|
||||
|
||||
type chatObservation struct {
|
||||
done chan struct{}
|
||||
err error // published by closing done; belongs to this attempt, not the next
|
||||
}
|
||||
|
||||
// chatRoom is one session's shared live channel. Every field is guarded by the
|
||||
// owning chatHub's mutex.
|
||||
type chatRoom struct {
|
||||
@@ -117,6 +122,11 @@ type chatRoom struct {
|
||||
queue []*chatQueuedPrompt
|
||||
running bool
|
||||
streaming bool
|
||||
// observing is non-nil while one caller is attaching the shared broker
|
||||
// stream. It prevents concurrent subscribers from opening duplicate global
|
||||
// event streams before streaming can be published.
|
||||
observing *chatObservation
|
||||
shellReadback *opencode.ShellSnapshot
|
||||
// buffering holds the turn's frames until the prompting client's ack has been
|
||||
// written, so the ack always precedes the frames it acknowledges. It is set
|
||||
// only while the very first turn subscribes to the firehose.
|
||||
@@ -160,6 +170,81 @@ func (h *chatHub) roomLocked(sessionID string) *chatRoom {
|
||||
return room
|
||||
}
|
||||
|
||||
// ensureObserving starts the firehose even for a read-only subscriber, then
|
||||
// reconciles retained shells from the endpoint's authoritative list. A bounded
|
||||
// journal folds concurrent lifecycle events over that snapshot atomically. All
|
||||
// observers share this reconciliation, with no broker I/O under the hub mutex.
|
||||
func (h *chatHub) ensureObserving(room *chatRoom, broker chatBroker) error {
|
||||
if broker == nil {
|
||||
return errors.New("chat is not configured on this server")
|
||||
}
|
||||
h.mu.Lock()
|
||||
if err := room.ctx.Err(); err != nil {
|
||||
h.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
if waiting := room.observing; waiting != nil {
|
||||
h.mu.Unlock()
|
||||
select {
|
||||
case <-waiting.done:
|
||||
return waiting.err
|
||||
case <-room.ctx.Done():
|
||||
return room.ctx.Err()
|
||||
}
|
||||
}
|
||||
if room.streaming {
|
||||
h.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
room.observing = &chatObservation{done: make(chan struct{})}
|
||||
waiting := room.observing
|
||||
room.shellReadback = &opencode.ShellSnapshot{}
|
||||
h.mu.Unlock()
|
||||
var observeErr error
|
||||
defer func() {
|
||||
h.mu.Lock()
|
||||
room.shellReadback = nil
|
||||
waiting.err = observeErr
|
||||
room.observing = nil
|
||||
close(waiting.done)
|
||||
h.mu.Unlock()
|
||||
}()
|
||||
if err := h.startStream(room, broker); err != nil {
|
||||
observeErr = err
|
||||
return err
|
||||
}
|
||||
shells, err := broker.Shells(room.ctx, room.sessionID)
|
||||
h.mu.Lock()
|
||||
if room.ctx.Err() != nil || h.rooms[room.sessionID] != room {
|
||||
h.mu.Unlock()
|
||||
observeErr = context.Canceled
|
||||
return observeErr
|
||||
}
|
||||
if err == nil {
|
||||
shells, err = room.shellReadback.Merge(shells)
|
||||
}
|
||||
if err != nil {
|
||||
// The firehose remains useful when its authoritative shell readback is
|
||||
// temporarily unavailable. Do not turn that read failure into a chat
|
||||
// failure (or claim the last retained snapshot is current): clear the
|
||||
// replay cache and tell attached clients that shell visibility is limited.
|
||||
// Preserve events already folded during this read, but clear stale replay.
|
||||
live, mergeErr := room.shellReadback.Merge(nil)
|
||||
if mergeErr == nil {
|
||||
h.replaceShellsLocked(room.sessionID, live)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
// Never log broker bodies/URLs: they may contain credentials or private output.
|
||||
log.Printf("chat shell readback session=%s failed (%T)", room.sessionID, err)
|
||||
h.fanout(room.sessionID, protocol.Message{Type: protocol.Error, Error: "background shell visibility is limited"}, "")
|
||||
return nil
|
||||
}
|
||||
h.replaceShellsLocked(room.sessionID, shells)
|
||||
room.shellReadback = nil
|
||||
h.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// attach registers write as one client's subscription to sessionID's stream and
|
||||
// returns it, or the existing subscription when the same connection has already
|
||||
// attached. It is idempotent so a client may prompt and subscribe without
|
||||
@@ -209,6 +294,11 @@ func (h *chatHub) detach(sessionID string, sub *chatSub) {
|
||||
if len(room.subs) == 0 {
|
||||
delete(h.rooms, sessionID)
|
||||
room.cancel()
|
||||
// ReleaseShells is a local identity-cache operation (no broker I/O).
|
||||
// Complete it before a reconnect can acquire this session's new room.
|
||||
if releaser, ok := room.broker.(chatShellReleaser); ok {
|
||||
releaser.ReleaseShells(sessionID)
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
@@ -265,9 +355,17 @@ func (h *chatHub) broadcastUser(frame protocol.ChatEventFrame, exceptClientID st
|
||||
// once it is written. A step-ended frame finishes the turn, letting the queue
|
||||
// advance.
|
||||
func (h *chatHub) emit(frame protocol.ChatEventFrame) {
|
||||
h.apply(frame)
|
||||
h.emitRoom(nil, frame)
|
||||
}
|
||||
|
||||
func (h *chatHub) emitRoom(source *chatRoom, frame protocol.ChatEventFrame) {
|
||||
h.mu.Lock()
|
||||
room := h.rooms[frame.SessionID]
|
||||
if source != nil && (source != room || source.ctx.Err() != nil) {
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
h.applyLocked(frame)
|
||||
buffered := false
|
||||
var targets []*chatSub
|
||||
if room != nil {
|
||||
@@ -283,6 +381,22 @@ func (h *chatHub) emit(frame protocol.ChatEventFrame) {
|
||||
if room != nil && frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepEnded {
|
||||
h.finishTurnLocked(room)
|
||||
}
|
||||
// Shell replay and live lifecycle delivery must enqueue in the same order
|
||||
// as folding. Otherwise an exit can overtake an older running replay.
|
||||
if frame.Kind == protocol.ChatEventShell && !buffered {
|
||||
msg := protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}
|
||||
var slow []*chatSub
|
||||
for _, sub := range targets {
|
||||
if !enqueueChatLocked(sub, msg) {
|
||||
slow = append(slow, sub)
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
for _, sub := range slow {
|
||||
sub.close()
|
||||
}
|
||||
return
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if buffered {
|
||||
return
|
||||
@@ -293,6 +407,62 @@ func (h *chatHub) emit(frame protocol.ChatEventFrame) {
|
||||
}
|
||||
}
|
||||
|
||||
// enqueueChatLocked never blocks or calls detach under the hub mutex.
|
||||
func enqueueChatLocked(sub *chatSub, msg protocol.Message) bool {
|
||||
select {
|
||||
case <-sub.done:
|
||||
return true
|
||||
default:
|
||||
}
|
||||
select {
|
||||
case sub.ch <- msg:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (h *chatHub) replayShells(sub *chatSub) {
|
||||
h.mu.Lock()
|
||||
slow := false
|
||||
if h.rooms[sub.room.sessionID] == sub.room && sub.room.subs[sub.clientID] == sub {
|
||||
turn := h.turnLocked(sub.room.sessionID)
|
||||
for _, id := range turn.shellOrder {
|
||||
shell := turn.shells[id]
|
||||
frame := protocol.ChatEventFrame{SessionID: sub.room.sessionID, Kind: protocol.ChatEventShell, Shell: &shell}
|
||||
if !enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}) {
|
||||
slow = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if slow {
|
||||
sub.close()
|
||||
}
|
||||
}
|
||||
|
||||
// replaceShells reconciles a successful authoritative readback. It drops stale
|
||||
// retained entries (including exits missed while disconnected) and upserts the
|
||||
// verified running set without fabricating a lifecycle event for foreign shells.
|
||||
func (h *chatHub) replaceShells(sessionID string, shells []protocol.ChatShell) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.replaceShellsLocked(sessionID, shells)
|
||||
}
|
||||
|
||||
func (h *chatHub) replaceShellsLocked(sessionID string, shells []protocol.ChatShell) {
|
||||
turn := h.turnLocked(sessionID)
|
||||
for _, shell := range turn.shells {
|
||||
turn.shrink(len(shell.Command))
|
||||
}
|
||||
turn.shells = nil
|
||||
turn.shellOrder = nil
|
||||
for i := range shells {
|
||||
turn.applyShell(&shells[i])
|
||||
}
|
||||
}
|
||||
|
||||
// finishTurnLocked marks the room's turn finished and releases the turn runner's
|
||||
// wait. Callers must hold h.mu.
|
||||
func (h *chatHub) finishTurnLocked(room *chatRoom) {
|
||||
@@ -331,14 +501,20 @@ func (h *chatHub) flushBuffered(sessionID string) {
|
||||
targets = append(targets, sub)
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
var slow []*chatSub
|
||||
for i := range pending {
|
||||
frame := pending[i]
|
||||
msg := protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}
|
||||
for _, sub := range targets {
|
||||
sub.send(msg)
|
||||
if !enqueueChatLocked(sub, msg) {
|
||||
slow = append(slow, sub)
|
||||
}
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
for _, sub := range slow {
|
||||
sub.close()
|
||||
}
|
||||
}
|
||||
|
||||
// submitPrompt accepts one prompt: it starts the turn runner immediately when no
|
||||
@@ -476,7 +652,7 @@ func (h *chatHub) runTurn(room *chatRoom, p *chatQueuedPrompt, done chan struct{
|
||||
return
|
||||
}
|
||||
if first {
|
||||
if err := h.startStream(room, broker); err != nil {
|
||||
if err := h.ensureObserving(room, broker); err != nil {
|
||||
h.failPrompt(room, p, err)
|
||||
h.endRoomTurn(room)
|
||||
return
|
||||
@@ -550,7 +726,7 @@ func (h *chatHub) startStream(room *chatRoom, broker chatBroker) error {
|
||||
result <- broker.Stream(ctx, room.sessionID, func() {
|
||||
once.Do(func() { close(attached) })
|
||||
}, func(frame protocol.ChatEventFrame) error {
|
||||
h.emit(frame)
|
||||
h.emitRoom(room, frame)
|
||||
return nil
|
||||
})
|
||||
}()
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"github.com/coder/websocket"
|
||||
"github.com/coder/websocket/wsjson"
|
||||
)
|
||||
|
||||
type shellBarrierBroker struct {
|
||||
*fakeChat
|
||||
read chan struct{}
|
||||
apply chan struct{}
|
||||
events chan shellBarrierEvent
|
||||
stopped chan struct{}
|
||||
reads atomic.Int32
|
||||
snapshot []protocol.ChatShell
|
||||
}
|
||||
|
||||
// Exercise the real websocket handler on a loopback scratch server and a fresh
|
||||
// SQLite database. A marker on the shared stream bounds the replay assertion
|
||||
// without sleeps, polling or a negative-read timeout.
|
||||
func TestChatShellReconciliationScratchServer(t *testing.T) {
|
||||
b := newShellBarrierBroker()
|
||||
b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: b}, newOperatorStore(t)))
|
||||
defer srv.Close()
|
||||
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.CloseNow()
|
||||
subscribe := func(id string) {
|
||||
t.Helper()
|
||||
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: id, ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
read := func() protocol.Message {
|
||||
t.Helper()
|
||||
var m protocol.Message
|
||||
if err := wsjson.Read(ctx, conn, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
if m := read(); m.Type != protocol.PipelineList {
|
||||
t.Fatalf("server greeting = %s", m.Type)
|
||||
}
|
||||
subscribe("first")
|
||||
if m := read(); m.ID != "first" || m.Type != protocol.ChatSubscribe {
|
||||
t.Fatalf("ack = %+v", m)
|
||||
}
|
||||
shellBarrierWait(t, b.read)
|
||||
done := make(chan struct{})
|
||||
b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_old", Status: protocol.ChatShellExited}, done}
|
||||
shellBarrierWait(t, done)
|
||||
if m := read(); m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellExited {
|
||||
t.Fatalf("exit not visible: %+v", m)
|
||||
}
|
||||
close(b.apply)
|
||||
subscribe("second")
|
||||
for {
|
||||
m := read()
|
||||
if m.ChatEvent != nil && m.ChatEvent.Shell.Status == protocol.ChatShellRunning {
|
||||
t.Fatalf("stale replay: %+v", m)
|
||||
}
|
||||
if m.ID == "second" {
|
||||
break
|
||||
}
|
||||
}
|
||||
done = make(chan struct{})
|
||||
b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_marker", Status: protocol.ChatShellExited}, done}
|
||||
shellBarrierWait(t, done)
|
||||
for {
|
||||
m := read()
|
||||
if m.ChatEvent == nil {
|
||||
continue
|
||||
}
|
||||
if m.ChatEvent.Shell.Status == protocol.ChatShellRunning {
|
||||
t.Fatalf("ghost visible after apply: %+v", m)
|
||||
}
|
||||
if m.ChatEvent.Shell.ID == "sh_marker" {
|
||||
break
|
||||
}
|
||||
}
|
||||
if b.reads.Load() != 1 {
|
||||
t.Fatalf("shared observer read count = %d", b.reads.Load())
|
||||
}
|
||||
conn.CloseNow()
|
||||
shellBarrierWait(t, b.stopped)
|
||||
}
|
||||
|
||||
type shellBarrierEvent struct {
|
||||
shell protocol.ChatShell
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func newShellBarrierBroker() *shellBarrierBroker {
|
||||
return &shellBarrierBroker{fakeChat: scriptedChat(), read: make(chan struct{}, 8), apply: make(chan struct{}), events: make(chan shellBarrierEvent), stopped: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (b *shellBarrierBroker) Shells(ctx context.Context, _ string) ([]protocol.ChatShell, error) {
|
||||
b.reads.Add(1)
|
||||
snapshot := append([]protocol.ChatShell(nil), b.snapshot...)
|
||||
b.read <- struct{}{}
|
||||
select {
|
||||
case <-b.apply:
|
||||
return snapshot, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (b *shellBarrierBroker) Stream(ctx context.Context, id string, ready func(), emit func(protocol.ChatEventFrame) error) error {
|
||||
defer close(b.stopped)
|
||||
ready()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case event := <-b.events:
|
||||
if err := emit(protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventShell, Shell: &event.shell}); err != nil {
|
||||
return err
|
||||
}
|
||||
close(event.done)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func shellBarrierWait(t *testing.T, ch <-chan struct{}) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-ch:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("barrier timed out")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatShellReconciliationBarrier(t *testing.T) {
|
||||
for _, status := range []string{protocol.ChatShellExited, protocol.ChatShellRunning, "timeout", "killed"} {
|
||||
t.Run(status, func(t *testing.T) {
|
||||
b := newShellBarrierBroker()
|
||||
b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}}
|
||||
h := newChatHub()
|
||||
messages := make(chan protocol.Message, 32)
|
||||
sub := h.attach("ses_1", "one", "", b, func(m protocol.Message) error { messages <- m; return nil })
|
||||
t.Cleanup(sub.close)
|
||||
result := make(chan error, 2)
|
||||
go func() { result <- h.ensureObserving(sub.room, b) }()
|
||||
shellBarrierWait(t, b.read)
|
||||
// A second observer must share the in-flight read, not apply an
|
||||
// independently completing older/newer snapshot out of order.
|
||||
go func() { result <- h.ensureObserving(sub.room, b) }()
|
||||
shell := protocol.ChatShell{ID: "sh_old", Status: status}
|
||||
if status == protocol.ChatShellRunning {
|
||||
shell.ID = "sh_new"
|
||||
shell.Command = "sleep 10"
|
||||
}
|
||||
done := make(chan struct{})
|
||||
select {
|
||||
case b.events <- shellBarrierEvent{shell, done}:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("event blocked behind readback")
|
||||
}
|
||||
shellBarrierWait(t, done)
|
||||
select {
|
||||
case m := <-messages:
|
||||
if m.ChatEvent == nil || m.ChatEvent.Shell == nil || m.ChatEvent.Shell.ID != shell.ID || m.ChatEvent.Shell.Status != status {
|
||||
t.Fatalf("live shell = %+v", m)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("live event not visible during read")
|
||||
}
|
||||
close(b.apply)
|
||||
for range 2 {
|
||||
select {
|
||||
case err := <-result:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("observer stuck")
|
||||
}
|
||||
}
|
||||
if b.reads.Load() != 1 {
|
||||
t.Fatalf("redundant/out-of-order reads: %d", b.reads.Load())
|
||||
}
|
||||
snapshot := h.shellSnapshot("ses_1")
|
||||
if status == protocol.ChatShellRunning {
|
||||
if len(snapshot) != 2 {
|
||||
t.Fatalf("new create lost: %+v", snapshot)
|
||||
}
|
||||
} else if len(snapshot) != 0 {
|
||||
t.Fatalf("terminal shell resurrected: %+v", snapshot)
|
||||
}
|
||||
// Assert the same retained state is replayed to the client.
|
||||
h.replayShells(sub)
|
||||
for range len(snapshot) {
|
||||
select {
|
||||
case m := <-messages:
|
||||
if m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellRunning {
|
||||
t.Fatalf("replay = %+v", m)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("replay missing")
|
||||
}
|
||||
}
|
||||
sub.close()
|
||||
shellBarrierWait(t, b.stopped)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatShellReconciliationCancelReconnectBarrier(t *testing.T) {
|
||||
b := newShellBarrierBroker()
|
||||
h := newChatHub()
|
||||
sub := h.attach("ses_1", "one", "", b, func(protocol.Message) error { return nil })
|
||||
results := make(chan error, 2)
|
||||
go func() { results <- h.ensureObserving(sub.room, b) }()
|
||||
shellBarrierWait(t, b.read)
|
||||
go func() { results <- h.ensureObserving(sub.room, b) }()
|
||||
sub.close()
|
||||
for range 2 {
|
||||
select {
|
||||
case err := <-results:
|
||||
if err == nil {
|
||||
t.Fatal("canceled observer succeeded")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("canceled waiter stuck")
|
||||
}
|
||||
}
|
||||
shellBarrierWait(t, b.stopped)
|
||||
next := newShellBarrierBroker()
|
||||
next.snapshot = []protocol.ChatShell{{ID: "sh_new", Status: protocol.ChatShellRunning}}
|
||||
close(next.apply)
|
||||
newSub := h.attach("ses_1", "two", "", next, func(protocol.Message) error { return nil })
|
||||
t.Cleanup(newSub.close)
|
||||
if err := h.ensureObserving(newSub.room, next); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A late callback from the canceled stream cannot poison the new room,
|
||||
// nor can a foreign-session callback from the current stream.
|
||||
h.emitRoom(sub.room, protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_ghost", Status: protocol.ChatShellRunning}})
|
||||
h.emitRoom(newSub.room, protocol.ChatEventFrame{SessionID: "ses_foreign", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_foreign", Status: protocol.ChatShellRunning}})
|
||||
got := h.shellSnapshot("ses_1")
|
||||
if len(got) != 1 || got[0].ID != "sh_new" || len(h.shellSnapshot("ses_foreign")) != 0 {
|
||||
t.Fatalf("reconnect state = %+v", got)
|
||||
}
|
||||
newSub.close()
|
||||
shellBarrierWait(t, next.stopped)
|
||||
}
|
||||
@@ -111,7 +111,14 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "could not attach to session " + query.SessionID})
|
||||
return
|
||||
}
|
||||
// Acknowledge before the stream is allowed to fan out: otherwise an immediately
|
||||
// available retained shell can overtake the subscribe reply on this connection.
|
||||
_ = write(protocol.Message{Type: protocol.ChatSubscribe, ID: request.ID})
|
||||
if err := hub.ensureObserving(sub.room, broker); err != nil {
|
||||
sub.close()
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "could not observe session " + query.SessionID})
|
||||
return
|
||||
}
|
||||
// Replay every still-unresolved approval request the room holds, not only the
|
||||
// most recent one, so a client that attaches after two requests were raised
|
||||
// still sees both cards.
|
||||
@@ -124,9 +131,5 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH
|
||||
// shows them rather than nothing until they exit. The message snapshot cannot
|
||||
// carry a shell — a shell outlives the turn and is not a turn — so this is the
|
||||
// shell half of the resume.
|
||||
for _, shell := range hub.shellSnapshot(query.SessionID) {
|
||||
running := shell
|
||||
frame := protocol.ChatEventFrame{SessionID: query.SessionID, Kind: protocol.ChatEventShell, Shell: &running}
|
||||
sub.send(protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame})
|
||||
}
|
||||
hub.replayShells(sub)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package server
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -52,6 +53,8 @@ type fakeChat struct {
|
||||
permissionSessionID string
|
||||
permissions []protocol.ChatPermissionFrame
|
||||
permissionErr error
|
||||
shells []protocol.ChatShell
|
||||
shellErr error
|
||||
sessionCalls int
|
||||
sessions []protocol.ChatSession
|
||||
sessionErr error
|
||||
@@ -62,6 +65,8 @@ type fakeChat struct {
|
||||
started chan struct{}
|
||||
done chan struct{}
|
||||
startOnce sync.Once
|
||||
streamCalls int
|
||||
readyGate <-chan struct{}
|
||||
block bool
|
||||
subscribed bool
|
||||
promptBeforeSubscribe bool
|
||||
@@ -91,13 +96,18 @@ func (f *fakeChat) Prompt(context.Context, string, string) error {
|
||||
func (f *fakeChat) Stream(ctx context.Context, _ string, ready func(), emit func(protocol.ChatEventFrame) error) error {
|
||||
f.mu.Lock()
|
||||
f.subscribed = true
|
||||
f.streamCalls++
|
||||
readyGate := f.readyGate
|
||||
f.mu.Unlock()
|
||||
if ready != nil {
|
||||
ready()
|
||||
}
|
||||
if f.started != nil {
|
||||
f.startOnce.Do(func() { close(f.started) })
|
||||
}
|
||||
if readyGate != nil {
|
||||
<-readyGate
|
||||
}
|
||||
if ready != nil {
|
||||
ready()
|
||||
}
|
||||
if f.block {
|
||||
<-ctx.Done()
|
||||
if f.done != nil {
|
||||
@@ -116,6 +126,42 @@ func (f *fakeChat) Stream(ctx context.Context, _ string, ready func(), emit func
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestChatHubConcurrentObserversShareStream pins the start-up handoff: two
|
||||
// clients that subscribe before the broker says ready still share one firehose.
|
||||
func TestChatHubConcurrentObserversShareStream(t *testing.T) {
|
||||
broker := scriptedChat()
|
||||
broker.block = true
|
||||
broker.started = make(chan struct{})
|
||||
gate := make(chan struct{})
|
||||
broker.readyGate = gate
|
||||
hub := newChatHub()
|
||||
hub.mu.Lock()
|
||||
room := hub.roomLocked("ses_1")
|
||||
hub.mu.Unlock()
|
||||
|
||||
errs := make(chan error, 2)
|
||||
go func() { errs <- hub.ensureObserving(room, broker) }()
|
||||
select {
|
||||
case <-broker.started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("first observer did not start the broker stream")
|
||||
}
|
||||
go func() { errs <- hub.ensureObserving(room, broker) }()
|
||||
close(gate)
|
||||
for range 2 {
|
||||
if err := <-errs; err != nil {
|
||||
t.Fatalf("ensureObserving: %v", err)
|
||||
}
|
||||
}
|
||||
broker.mu.Lock()
|
||||
calls := broker.streamCalls
|
||||
broker.mu.Unlock()
|
||||
if calls != 1 {
|
||||
t.Fatalf("broker Stream calls = %d, want 1", calls)
|
||||
}
|
||||
room.cancel()
|
||||
}
|
||||
|
||||
func (f *fakeChat) Interrupt(context.Context, string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
@@ -151,6 +197,12 @@ func (f *fakeChat) Reply(_ context.Context, request protocol.ChatReplyRequest) e
|
||||
// Permissions returns the seeded pending requests, or blocks until the caller's
|
||||
// context is canceled when permBlock is set (used to prove the poller stops on
|
||||
// disconnect). Every call is counted so a test can assert polling happened.
|
||||
func (f *fakeChat) Shells(context.Context, string) ([]protocol.ChatShell, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]protocol.ChatShell(nil), f.shells...), f.shellErr
|
||||
}
|
||||
|
||||
func (f *fakeChat) Permissions(ctx context.Context, sessionID string) ([]protocol.ChatPermissionFrame, error) {
|
||||
f.mu.Lock()
|
||||
f.permissionCalls++
|
||||
@@ -682,6 +734,9 @@ func TestChatInterruptEndsRunningPrompt(t *testing.T) {
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/permission/request":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"location":{"directory":"/work"},"data":[]}`))
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/shell":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data":[]}`))
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/event":
|
||||
flusher, ok := w.(http.Flusher)
|
||||
if !ok {
|
||||
@@ -995,6 +1050,7 @@ func TestChatPromptStreamsShellFrame(t *testing.T) {
|
||||
// shows the background process instead of nothing until it exits.
|
||||
func TestChatSubscribeReplaysRunningShell(t *testing.T) {
|
||||
broker := scriptedChat()
|
||||
broker.shells = []protocol.ChatShell{{ID: "sh_1", Status: protocol.ChatShellRunning, Command: "sleep 120"}}
|
||||
broker.script = append([]protocol.ChatEventFrame{
|
||||
{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_1", Status: protocol.ChatShellRunning, Command: "sleep 120"}},
|
||||
}, broker.script...)
|
||||
@@ -1039,6 +1095,40 @@ func TestChatSubscribeReplaysRunningShell(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatSubscribeKeepsStreamWhenShellReadbackFails ensures a temporary
|
||||
// authoritative readback failure does not make chat unavailable or replay a
|
||||
// stale process as running. The subscriber receives an explicit limitation
|
||||
// while its broker stream remains attached for future lifecycle events.
|
||||
func TestChatSubscribeKeepsStreamWhenShellReadbackFails(t *testing.T) {
|
||||
broker := scriptedChat()
|
||||
broker.block = true
|
||||
broker.shellErr = errors.New("shell endpoint unavailable: https://user:secret@example.invalid private response body")
|
||||
ctx, _, conn := dialChat(t, broker)
|
||||
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub1", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ack protocol.Message
|
||||
if err := wsjson.Read(ctx, conn, &ack); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ack.Type != protocol.ChatSubscribe || ack.ID != "sub1" {
|
||||
t.Fatalf("subscribe ack = %+v", ack)
|
||||
}
|
||||
var limited protocol.Message
|
||||
if err := wsjson.Read(ctx, conn, &limited); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if limited.Type != protocol.Error || limited.Error != "background shell visibility is limited" {
|
||||
t.Fatalf("limited visibility notice = %+v", limited)
|
||||
}
|
||||
broker.mu.Lock()
|
||||
calls := broker.streamCalls
|
||||
broker.mu.Unlock()
|
||||
if calls != 1 {
|
||||
t.Fatalf("broker Stream calls = %d, want 1", calls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatPromptSubscribesBeforePrompt pins the subscribe-before-prompt order:
|
||||
// the firehose has no replay cursor, so the subscription must be attached before
|
||||
// the prompt is posted or deltas emitted in between are lost.
|
||||
@@ -1079,6 +1169,9 @@ func realAdapterChat(t *testing.T, frames []string) []protocol.ChatEventFrame {
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/permission/request":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"location":{"directory":"/work"},"data":[]}`))
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/shell":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data":[]}`))
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/event":
|
||||
streams.Add(1)
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
|
||||
Reference in New Issue
Block a user