package server import ( "context" "net/http/httptest" "strings" "sync/atomic" "testing" "time" "git.a-wan.net/alan/pipelines/internal/protocol" "github.com/coder/websocket" "github.com/coder/websocket/wsjson" ) type shellBarrierBroker struct { *fakeChat read chan struct{} apply chan struct{} events chan shellBarrierEvent stopped chan struct{} reads atomic.Int32 snapshot []protocol.ChatShell } // Exercise the real websocket handler on a loopback scratch server and a fresh // SQLite database. A marker on the shared stream bounds the replay assertion // without sleeps, polling or a negative-read timeout. func TestChatShellReconciliationScratchServer(t *testing.T) { b := newShellBarrierBroker() b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}} ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: b}, newOperatorStore(t))) defer srv.Close() conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil) if err != nil { t.Fatal(err) } defer conn.CloseNow() subscribe := func(id string) { t.Helper() if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: id, ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil { t.Fatal(err) } } read := func() protocol.Message { t.Helper() var m protocol.Message if err := wsjson.Read(ctx, conn, &m); err != nil { t.Fatal(err) } return m } if m := read(); m.Type != protocol.PipelineList { t.Fatalf("server greeting = %s", m.Type) } subscribe("first") if m := read(); m.ID != "first" || m.Type != protocol.ChatSubscribe { t.Fatalf("ack = %+v", m) } shellBarrierWait(t, b.read) done := make(chan struct{}) b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_old", Status: protocol.ChatShellExited}, done} shellBarrierWait(t, done) if m := read(); m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellExited { t.Fatalf("exit not visible: %+v", m) } close(b.apply) subscribe("second") for { m := read() if m.ChatEvent != nil && m.ChatEvent.Shell.Status == protocol.ChatShellRunning { t.Fatalf("stale replay: %+v", m) } if m.ID == "second" { break } } done = make(chan struct{}) b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_marker", Status: protocol.ChatShellExited}, done} shellBarrierWait(t, done) for { m := read() if m.ChatEvent == nil { continue } if m.ChatEvent.Shell.Status == protocol.ChatShellRunning { t.Fatalf("ghost visible after apply: %+v", m) } if m.ChatEvent.Shell.ID == "sh_marker" { break } } if b.reads.Load() != 1 { t.Fatalf("shared observer read count = %d", b.reads.Load()) } conn.CloseNow() shellBarrierWait(t, b.stopped) } type shellBarrierEvent struct { shell protocol.ChatShell done chan struct{} } func newShellBarrierBroker() *shellBarrierBroker { return &shellBarrierBroker{fakeChat: scriptedChat(), read: make(chan struct{}, 8), apply: make(chan struct{}), events: make(chan shellBarrierEvent), stopped: make(chan struct{})} } func (b *shellBarrierBroker) Shells(ctx context.Context, _ string) ([]protocol.ChatShell, error) { b.reads.Add(1) snapshot := append([]protocol.ChatShell(nil), b.snapshot...) b.read <- struct{}{} select { case <-b.apply: return snapshot, nil case <-ctx.Done(): return nil, ctx.Err() } } func (b *shellBarrierBroker) Stream(ctx context.Context, id string, ready func(), emit func(protocol.ChatEventFrame) error) error { defer close(b.stopped) ready() for { select { case <-ctx.Done(): return ctx.Err() case event := <-b.events: if err := emit(protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventShell, Shell: &event.shell}); err != nil { return err } close(event.done) } } } func shellBarrierWait(t *testing.T, ch <-chan struct{}) { t.Helper() select { case <-ch: case <-time.After(3 * time.Second): t.Fatal("barrier timed out") } } func TestChatShellReconciliationBarrier(t *testing.T) { for _, status := range []string{protocol.ChatShellExited, protocol.ChatShellRunning, "timeout", "killed"} { t.Run(status, func(t *testing.T) { b := newShellBarrierBroker() b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}} h := newChatHub() messages := make(chan protocol.Message, 32) sub := h.attach("ses_1", "one", "", b, func(m protocol.Message) error { messages <- m; return nil }) t.Cleanup(sub.close) result := make(chan error, 2) go func() { result <- h.ensureObserving(sub.room, b) }() shellBarrierWait(t, b.read) // A second observer must share the in-flight read, not apply an // independently completing older/newer snapshot out of order. go func() { result <- h.ensureObserving(sub.room, b) }() shell := protocol.ChatShell{ID: "sh_old", Status: status} if status == protocol.ChatShellRunning { shell.ID = "sh_new" shell.Command = "sleep 10" } done := make(chan struct{}) select { case b.events <- shellBarrierEvent{shell, done}: case <-time.After(time.Second): t.Fatal("event blocked behind readback") } shellBarrierWait(t, done) select { case m := <-messages: if m.ChatEvent == nil || m.ChatEvent.Shell == nil || m.ChatEvent.Shell.ID != shell.ID || m.ChatEvent.Shell.Status != status { t.Fatalf("live shell = %+v", m) } case <-time.After(time.Second): t.Fatal("live event not visible during read") } close(b.apply) for range 2 { select { case err := <-result: if err != nil { t.Fatal(err) } case <-time.After(time.Second): t.Fatal("observer stuck") } } if b.reads.Load() != 1 { t.Fatalf("redundant/out-of-order reads: %d", b.reads.Load()) } snapshot := h.shellSnapshot("ses_1") if status == protocol.ChatShellRunning { if len(snapshot) != 2 { t.Fatalf("new create lost: %+v", snapshot) } } else if len(snapshot) != 0 { t.Fatalf("terminal shell resurrected: %+v", snapshot) } // Assert the same retained state is replayed to the client. h.replayShells(sub) for range len(snapshot) { select { case m := <-messages: if m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellRunning { t.Fatalf("replay = %+v", m) } case <-time.After(time.Second): t.Fatal("replay missing") } } sub.close() shellBarrierWait(t, b.stopped) }) } } func TestChatShellReconciliationCancelReconnectBarrier(t *testing.T) { b := newShellBarrierBroker() h := newChatHub() sub := h.attach("ses_1", "one", "", b, func(protocol.Message) error { return nil }) results := make(chan error, 2) go func() { results <- h.ensureObserving(sub.room, b) }() shellBarrierWait(t, b.read) go func() { results <- h.ensureObserving(sub.room, b) }() sub.close() for range 2 { select { case err := <-results: if err == nil { t.Fatal("canceled observer succeeded") } case <-time.After(time.Second): t.Fatal("canceled waiter stuck") } } shellBarrierWait(t, b.stopped) next := newShellBarrierBroker() next.snapshot = []protocol.ChatShell{{ID: "sh_new", Status: protocol.ChatShellRunning}} close(next.apply) newSub := h.attach("ses_1", "two", "", next, func(protocol.Message) error { return nil }) t.Cleanup(newSub.close) if err := h.ensureObserving(newSub.room, next); err != nil { t.Fatal(err) } // A late callback from the canceled stream cannot poison the new room, // nor can a foreign-session callback from the current stream. h.emitRoom(sub.room, protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_ghost", Status: protocol.ChatShellRunning}}) h.emitRoom(newSub.room, protocol.ChatEventFrame{SessionID: "ses_foreign", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_foreign", Status: protocol.ChatShellRunning}}) got := h.shellSnapshot("ses_1") if len(got) != 1 || got[0].ID != "sh_new" || len(h.shellSnapshot("ses_foreign")) != 0 { t.Fatalf("reconnect state = %+v", got) } newSub.close() shellBarrierWait(t, next.stopped) }