e9d902213e
Reconcile authoritative shell snapshots with bounded lifecycle journals so concurrent exits and creates cannot be overwritten, coalesce observation setup, and retain verified ownership across reconnects. Add deterministic reconciliation and cancellation regressions plus stable limited-visibility handling.\n\nChecked packaged templates and authoring skill references: none need updating. The task-owned docs note records the stale shared chat documentation for docs-fold; shared documentation was not edited.
262 lines
8.1 KiB
Go
262 lines
8.1 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"github.com/coder/websocket"
|
|
"github.com/coder/websocket/wsjson"
|
|
)
|
|
|
|
type shellBarrierBroker struct {
|
|
*fakeChat
|
|
read chan struct{}
|
|
apply chan struct{}
|
|
events chan shellBarrierEvent
|
|
stopped chan struct{}
|
|
reads atomic.Int32
|
|
snapshot []protocol.ChatShell
|
|
}
|
|
|
|
// Exercise the real websocket handler on a loopback scratch server and a fresh
|
|
// SQLite database. A marker on the shared stream bounds the replay assertion
|
|
// without sleeps, polling or a negative-read timeout.
|
|
func TestChatShellReconciliationScratchServer(t *testing.T) {
|
|
b := newShellBarrierBroker()
|
|
b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: b}, newOperatorStore(t)))
|
|
defer srv.Close()
|
|
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer conn.CloseNow()
|
|
subscribe := func(id string) {
|
|
t.Helper()
|
|
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: id, ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
read := func() protocol.Message {
|
|
t.Helper()
|
|
var m protocol.Message
|
|
if err := wsjson.Read(ctx, conn, &m); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return m
|
|
}
|
|
if m := read(); m.Type != protocol.PipelineList {
|
|
t.Fatalf("server greeting = %s", m.Type)
|
|
}
|
|
subscribe("first")
|
|
if m := read(); m.ID != "first" || m.Type != protocol.ChatSubscribe {
|
|
t.Fatalf("ack = %+v", m)
|
|
}
|
|
shellBarrierWait(t, b.read)
|
|
done := make(chan struct{})
|
|
b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_old", Status: protocol.ChatShellExited}, done}
|
|
shellBarrierWait(t, done)
|
|
if m := read(); m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellExited {
|
|
t.Fatalf("exit not visible: %+v", m)
|
|
}
|
|
close(b.apply)
|
|
subscribe("second")
|
|
for {
|
|
m := read()
|
|
if m.ChatEvent != nil && m.ChatEvent.Shell.Status == protocol.ChatShellRunning {
|
|
t.Fatalf("stale replay: %+v", m)
|
|
}
|
|
if m.ID == "second" {
|
|
break
|
|
}
|
|
}
|
|
done = make(chan struct{})
|
|
b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_marker", Status: protocol.ChatShellExited}, done}
|
|
shellBarrierWait(t, done)
|
|
for {
|
|
m := read()
|
|
if m.ChatEvent == nil {
|
|
continue
|
|
}
|
|
if m.ChatEvent.Shell.Status == protocol.ChatShellRunning {
|
|
t.Fatalf("ghost visible after apply: %+v", m)
|
|
}
|
|
if m.ChatEvent.Shell.ID == "sh_marker" {
|
|
break
|
|
}
|
|
}
|
|
if b.reads.Load() != 1 {
|
|
t.Fatalf("shared observer read count = %d", b.reads.Load())
|
|
}
|
|
conn.CloseNow()
|
|
shellBarrierWait(t, b.stopped)
|
|
}
|
|
|
|
type shellBarrierEvent struct {
|
|
shell protocol.ChatShell
|
|
done chan struct{}
|
|
}
|
|
|
|
func newShellBarrierBroker() *shellBarrierBroker {
|
|
return &shellBarrierBroker{fakeChat: scriptedChat(), read: make(chan struct{}, 8), apply: make(chan struct{}), events: make(chan shellBarrierEvent), stopped: make(chan struct{})}
|
|
}
|
|
|
|
func (b *shellBarrierBroker) Shells(ctx context.Context, _ string) ([]protocol.ChatShell, error) {
|
|
b.reads.Add(1)
|
|
snapshot := append([]protocol.ChatShell(nil), b.snapshot...)
|
|
b.read <- struct{}{}
|
|
select {
|
|
case <-b.apply:
|
|
return snapshot, nil
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
}
|
|
|
|
func (b *shellBarrierBroker) Stream(ctx context.Context, id string, ready func(), emit func(protocol.ChatEventFrame) error) error {
|
|
defer close(b.stopped)
|
|
ready()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case event := <-b.events:
|
|
if err := emit(protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventShell, Shell: &event.shell}); err != nil {
|
|
return err
|
|
}
|
|
close(event.done)
|
|
}
|
|
}
|
|
}
|
|
|
|
func shellBarrierWait(t *testing.T, ch <-chan struct{}) {
|
|
t.Helper()
|
|
select {
|
|
case <-ch:
|
|
case <-time.After(3 * time.Second):
|
|
t.Fatal("barrier timed out")
|
|
}
|
|
}
|
|
|
|
func TestChatShellReconciliationBarrier(t *testing.T) {
|
|
for _, status := range []string{protocol.ChatShellExited, protocol.ChatShellRunning, "timeout", "killed"} {
|
|
t.Run(status, func(t *testing.T) {
|
|
b := newShellBarrierBroker()
|
|
b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}}
|
|
h := newChatHub()
|
|
messages := make(chan protocol.Message, 32)
|
|
sub := h.attach("ses_1", "one", "", b, func(m protocol.Message) error { messages <- m; return nil })
|
|
t.Cleanup(sub.close)
|
|
result := make(chan error, 2)
|
|
go func() { result <- h.ensureObserving(sub.room, b) }()
|
|
shellBarrierWait(t, b.read)
|
|
// A second observer must share the in-flight read, not apply an
|
|
// independently completing older/newer snapshot out of order.
|
|
go func() { result <- h.ensureObserving(sub.room, b) }()
|
|
shell := protocol.ChatShell{ID: "sh_old", Status: status}
|
|
if status == protocol.ChatShellRunning {
|
|
shell.ID = "sh_new"
|
|
shell.Command = "sleep 10"
|
|
}
|
|
done := make(chan struct{})
|
|
select {
|
|
case b.events <- shellBarrierEvent{shell, done}:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("event blocked behind readback")
|
|
}
|
|
shellBarrierWait(t, done)
|
|
select {
|
|
case m := <-messages:
|
|
if m.ChatEvent == nil || m.ChatEvent.Shell == nil || m.ChatEvent.Shell.ID != shell.ID || m.ChatEvent.Shell.Status != status {
|
|
t.Fatalf("live shell = %+v", m)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("live event not visible during read")
|
|
}
|
|
close(b.apply)
|
|
for range 2 {
|
|
select {
|
|
case err := <-result:
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("observer stuck")
|
|
}
|
|
}
|
|
if b.reads.Load() != 1 {
|
|
t.Fatalf("redundant/out-of-order reads: %d", b.reads.Load())
|
|
}
|
|
snapshot := h.shellSnapshot("ses_1")
|
|
if status == protocol.ChatShellRunning {
|
|
if len(snapshot) != 2 {
|
|
t.Fatalf("new create lost: %+v", snapshot)
|
|
}
|
|
} else if len(snapshot) != 0 {
|
|
t.Fatalf("terminal shell resurrected: %+v", snapshot)
|
|
}
|
|
// Assert the same retained state is replayed to the client.
|
|
h.replayShells(sub)
|
|
for range len(snapshot) {
|
|
select {
|
|
case m := <-messages:
|
|
if m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellRunning {
|
|
t.Fatalf("replay = %+v", m)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("replay missing")
|
|
}
|
|
}
|
|
sub.close()
|
|
shellBarrierWait(t, b.stopped)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestChatShellReconciliationCancelReconnectBarrier(t *testing.T) {
|
|
b := newShellBarrierBroker()
|
|
h := newChatHub()
|
|
sub := h.attach("ses_1", "one", "", b, func(protocol.Message) error { return nil })
|
|
results := make(chan error, 2)
|
|
go func() { results <- h.ensureObserving(sub.room, b) }()
|
|
shellBarrierWait(t, b.read)
|
|
go func() { results <- h.ensureObserving(sub.room, b) }()
|
|
sub.close()
|
|
for range 2 {
|
|
select {
|
|
case err := <-results:
|
|
if err == nil {
|
|
t.Fatal("canceled observer succeeded")
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("canceled waiter stuck")
|
|
}
|
|
}
|
|
shellBarrierWait(t, b.stopped)
|
|
next := newShellBarrierBroker()
|
|
next.snapshot = []protocol.ChatShell{{ID: "sh_new", Status: protocol.ChatShellRunning}}
|
|
close(next.apply)
|
|
newSub := h.attach("ses_1", "two", "", next, func(protocol.Message) error { return nil })
|
|
t.Cleanup(newSub.close)
|
|
if err := h.ensureObserving(newSub.room, next); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// A late callback from the canceled stream cannot poison the new room,
|
|
// nor can a foreign-session callback from the current stream.
|
|
h.emitRoom(sub.room, protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_ghost", Status: protocol.ChatShellRunning}})
|
|
h.emitRoom(newSub.room, protocol.ChatEventFrame{SessionID: "ses_foreign", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_foreign", Status: protocol.ChatShellRunning}})
|
|
got := h.shellSnapshot("ses_1")
|
|
if len(got) != 1 || got[0].ID != "sh_new" || len(h.shellSnapshot("ses_foreign")) != 0 {
|
|
t.Fatalf("reconnect state = %+v", got)
|
|
}
|
|
newSub.close()
|
|
shellBarrierWait(t, next.stopped)
|
|
}
|